From 441d065418ad87d55862768876089969dcb57d94 Mon Sep 17 00:00:00 2001 From: Eugenio Paolantonio Date: Tue, 12 Nov 2024 15:38:25 +0100 Subject: [PATCH] gocd: slfo-stagings: slfo-stagings: fail on broken/failed products (SLFO:Main) Rather than using verify-repo-built-successful.py, check with the API directly whether the product is blocked, scheduled or building. Since it might happen for a product being broken/failed (even after a successful pkglistgen run), just fail in that case. This is #3191, but for SUSE:SLFO:Main rather than 1.1. Signed-off-by: Eugenio Paolantonio --- gocd/slfo-stagings.gocd.yaml | 198 +++++++++++++++++++------------ gocd/slfo-stagings.gocd.yaml.erb | 18 +-- 2 files changed, 132 insertions(+), 84 deletions(-) diff --git a/gocd/slfo-stagings.gocd.yaml b/gocd/slfo-stagings.gocd.yaml index 5d15bbae..f13d017b 100644 --- a/gocd/slfo-stagings.gocd.yaml +++ b/gocd/slfo-stagings.gocd.yaml @@ -122,20 +122,24 @@ pipelines: - staging-bot tasks: - script: |- - minutes=0 + minutes=1 osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build" export PYTHONPATH=$PWD/scripts - while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do - if [ ${ret} -eq 2 ]; then - echo "product repository not found. Project configuration issue?" >&2 - exit 1 - elif [ ${minutes} -gt 180 ]; then - echo "Product is still failing after timeout, exiting..." >&2 + ## Let the scheduler warm up first + sleep 60 + while osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -E 'statuscount code="(blocked|scheduled|building)"'; do + if [ ${minutes} -gt 180 ]; then + echo "Product is still building after timeout, exiting..." >&2 exit 1 fi sleep 60 minutes=$(expr $minutes + 1) done + ## Always fail on broken/failed products + if osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -qE 'statuscount code="(broken|failed)"'; then + echo "Some products failed to build, exiting..." >&2 + exit 1 + fi - Enable.images.repo: resources: @@ -211,20 +215,24 @@ pipelines: - staging-bot tasks: - script: |- - minutes=0 + minutes=1 osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build" export PYTHONPATH=$PWD/scripts - while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do - if [ ${ret} -eq 2 ]; then - echo "product repository not found. Project configuration issue?" >&2 - exit 1 - elif [ ${minutes} -gt 180 ]; then - echo "Product is still failing after timeout, exiting..." >&2 + ## Let the scheduler warm up first + sleep 60 + while osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -E 'statuscount code="(blocked|scheduled|building)"'; do + if [ ${minutes} -gt 180 ]; then + echo "Product is still building after timeout, exiting..." >&2 exit 1 fi sleep 60 minutes=$(expr $minutes + 1) done + ## Always fail on broken/failed products + if osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -qE 'statuscount code="(broken|failed)"'; then + echo "Some products failed to build, exiting..." >&2 + exit 1 + fi - Enable.images.repo: resources: @@ -300,20 +308,24 @@ pipelines: - staging-bot tasks: - script: |- - minutes=0 + minutes=1 osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build" export PYTHONPATH=$PWD/scripts - while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do - if [ ${ret} -eq 2 ]; then - echo "product repository not found. Project configuration issue?" >&2 - exit 1 - elif [ ${minutes} -gt 180 ]; then - echo "Product is still failing after timeout, exiting..." >&2 + ## Let the scheduler warm up first + sleep 60 + while osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -E 'statuscount code="(blocked|scheduled|building)"'; do + if [ ${minutes} -gt 180 ]; then + echo "Product is still building after timeout, exiting..." >&2 exit 1 fi sleep 60 minutes=$(expr $minutes + 1) done + ## Always fail on broken/failed products + if osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -qE 'statuscount code="(broken|failed)"'; then + echo "Some products failed to build, exiting..." >&2 + exit 1 + fi - Enable.images.repo: resources: @@ -389,20 +401,24 @@ pipelines: - staging-bot tasks: - script: |- - minutes=0 + minutes=1 osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build" export PYTHONPATH=$PWD/scripts - while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do - if [ ${ret} -eq 2 ]; then - echo "product repository not found. Project configuration issue?" >&2 - exit 1 - elif [ ${minutes} -gt 180 ]; then - echo "Product is still failing after timeout, exiting..." >&2 + ## Let the scheduler warm up first + sleep 60 + while osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -E 'statuscount code="(blocked|scheduled|building)"'; do + if [ ${minutes} -gt 180 ]; then + echo "Product is still building after timeout, exiting..." >&2 exit 1 fi sleep 60 minutes=$(expr $minutes + 1) done + ## Always fail on broken/failed products + if osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -qE 'statuscount code="(broken|failed)"'; then + echo "Some products failed to build, exiting..." >&2 + exit 1 + fi - Enable.images.repo: resources: @@ -478,20 +494,24 @@ pipelines: - staging-bot tasks: - script: |- - minutes=0 + minutes=1 osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build" export PYTHONPATH=$PWD/scripts - while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do - if [ ${ret} -eq 2 ]; then - echo "product repository not found. Project configuration issue?" >&2 - exit 1 - elif [ ${minutes} -gt 180 ]; then - echo "Product is still failing after timeout, exiting..." >&2 + ## Let the scheduler warm up first + sleep 60 + while osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -E 'statuscount code="(blocked|scheduled|building)"'; do + if [ ${minutes} -gt 180 ]; then + echo "Product is still building after timeout, exiting..." >&2 exit 1 fi sleep 60 minutes=$(expr $minutes + 1) done + ## Always fail on broken/failed products + if osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -qE 'statuscount code="(broken|failed)"'; then + echo "Some products failed to build, exiting..." >&2 + exit 1 + fi - Enable.images.repo: resources: @@ -567,20 +587,24 @@ pipelines: - staging-bot tasks: - script: |- - minutes=0 + minutes=1 osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build" export PYTHONPATH=$PWD/scripts - while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do - if [ ${ret} -eq 2 ]; then - echo "product repository not found. Project configuration issue?" >&2 - exit 1 - elif [ ${minutes} -gt 180 ]; then - echo "Product is still failing after timeout, exiting..." >&2 + ## Let the scheduler warm up first + sleep 60 + while osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -E 'statuscount code="(blocked|scheduled|building)"'; do + if [ ${minutes} -gt 180 ]; then + echo "Product is still building after timeout, exiting..." >&2 exit 1 fi sleep 60 minutes=$(expr $minutes + 1) done + ## Always fail on broken/failed products + if osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -qE 'statuscount code="(broken|failed)"'; then + echo "Some products failed to build, exiting..." >&2 + exit 1 + fi - Enable.images.repo: resources: @@ -656,20 +680,24 @@ pipelines: - staging-bot tasks: - script: |- - minutes=0 + minutes=1 osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build" export PYTHONPATH=$PWD/scripts - while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do - if [ ${ret} -eq 2 ]; then - echo "product repository not found. Project configuration issue?" >&2 - exit 1 - elif [ ${minutes} -gt 180 ]; then - echo "Product is still failing after timeout, exiting..." >&2 + ## Let the scheduler warm up first + sleep 60 + while osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -E 'statuscount code="(blocked|scheduled|building)"'; do + if [ ${minutes} -gt 180 ]; then + echo "Product is still building after timeout, exiting..." >&2 exit 1 fi sleep 60 minutes=$(expr $minutes + 1) done + ## Always fail on broken/failed products + if osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -qE 'statuscount code="(broken|failed)"'; then + echo "Some products failed to build, exiting..." >&2 + exit 1 + fi - Enable.images.repo: resources: @@ -745,20 +773,24 @@ pipelines: - staging-bot tasks: - script: |- - minutes=0 + minutes=1 osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build" export PYTHONPATH=$PWD/scripts - while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do - if [ ${ret} -eq 2 ]; then - echo "product repository not found. Project configuration issue?" >&2 - exit 1 - elif [ ${minutes} -gt 180 ]; then - echo "Product is still failing after timeout, exiting..." >&2 + ## Let the scheduler warm up first + sleep 60 + while osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -E 'statuscount code="(blocked|scheduled|building)"'; do + if [ ${minutes} -gt 180 ]; then + echo "Product is still building after timeout, exiting..." >&2 exit 1 fi sleep 60 minutes=$(expr $minutes + 1) done + ## Always fail on broken/failed products + if osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -qE 'statuscount code="(broken|failed)"'; then + echo "Some products failed to build, exiting..." >&2 + exit 1 + fi - Enable.images.repo: resources: @@ -834,20 +866,24 @@ pipelines: - staging-bot tasks: - script: |- - minutes=0 + minutes=1 osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build" export PYTHONPATH=$PWD/scripts - while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do - if [ ${ret} -eq 2 ]; then - echo "product repository not found. Project configuration issue?" >&2 - exit 1 - elif [ ${minutes} -gt 180 ]; then - echo "Product is still failing after timeout, exiting..." >&2 + ## Let the scheduler warm up first + sleep 60 + while osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -E 'statuscount code="(blocked|scheduled|building)"'; do + if [ ${minutes} -gt 180 ]; then + echo "Product is still building after timeout, exiting..." >&2 exit 1 fi sleep 60 minutes=$(expr $minutes + 1) done + ## Always fail on broken/failed products + if osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -qE 'statuscount code="(broken|failed)"'; then + echo "Some products failed to build, exiting..." >&2 + exit 1 + fi - Enable.images.repo: resources: @@ -923,20 +959,24 @@ pipelines: - staging-bot tasks: - script: |- - minutes=0 + minutes=1 osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build" export PYTHONPATH=$PWD/scripts - while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do - if [ ${ret} -eq 2 ]; then - echo "product repository not found. Project configuration issue?" >&2 - exit 1 - elif [ ${minutes} -gt 180 ]; then - echo "Product is still failing after timeout, exiting..." >&2 + ## Let the scheduler warm up first + sleep 60 + while osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -E 'statuscount code="(blocked|scheduled|building)"'; do + if [ ${minutes} -gt 180 ]; then + echo "Product is still building after timeout, exiting..." >&2 exit 1 fi sleep 60 minutes=$(expr $minutes + 1) done + ## Always fail on broken/failed products + if osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -qE 'statuscount code="(broken|failed)"'; then + echo "Some products failed to build, exiting..." >&2 + exit 1 + fi - Enable.images.repo: resources: @@ -1012,20 +1052,24 @@ pipelines: - staging-bot tasks: - script: |- - minutes=0 + minutes=1 osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build" export PYTHONPATH=$PWD/scripts - while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do - if [ ${ret} -eq 2 ]; then - echo "product repository not found. Project configuration issue?" >&2 - exit 1 - elif [ ${minutes} -gt 180 ]; then - echo "Product is still failing after timeout, exiting..." >&2 + ## Let the scheduler warm up first + sleep 60 + while osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -E 'statuscount code="(blocked|scheduled|building)"'; do + if [ ${minutes} -gt 180 ]; then + echo "Product is still building after timeout, exiting..." >&2 exit 1 fi sleep 60 minutes=$(expr $minutes + 1) done + ## Always fail on broken/failed products + if osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -qE 'statuscount code="(broken|failed)"'; then + echo "Some products failed to build, exiting..." >&2 + exit 1 + fi - Enable.images.repo: resources: diff --git a/gocd/slfo-stagings.gocd.yaml.erb b/gocd/slfo-stagings.gocd.yaml.erb index 5684b966..7df80517 100644 --- a/gocd/slfo-stagings.gocd.yaml.erb +++ b/gocd/slfo-stagings.gocd.yaml.erb @@ -123,20 +123,24 @@ pipelines: - staging-bot tasks: - script: |- - minutes=0 + minutes=1 osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=product&flag=build" export PYTHONPATH=$PWD/scripts - while ./scripts/gocd/verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r product; ret=$?; [ ${ret} -ne 0 ]; do - if [ ${ret} -eq 2 ]; then - echo "product repository not found. Project configuration issue?" >&2 - exit 1 - elif [ ${minutes} -gt 180 ]; then - echo "Product is still failing after timeout, exiting..." >&2 + ## Let the scheduler warm up first + sleep 60 + while osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -E 'statuscount code="(blocked|scheduled|building)"'; do + if [ ${minutes} -gt 180 ]; then + echo "Product is still building after timeout, exiting..." >&2 exit 1 fi sleep 60 minutes=$(expr $minutes + 1) done + ## Always fail on broken/failed products + if osc -A $STAGING_API api -X GET "/build/$STAGING_PROJECT/_result?view=summary&repository=product&arch=local" | grep -qE 'statuscount code="(broken|failed)"'; then + echo "Some products failed to build, exiting..." >&2 + exit 1 + fi - Enable.images.repo: resources: