From 2ba9745c2eb53184fc23745a30e006d003aaaf31 Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Fri, 21 Sep 2018 12:42:16 -0400 Subject: [PATCH] Add time limit for successfully deploying - Add config var for max wait time for a deployment - Move exit function and trap to the beginning of the script - Execute the rollout status command using timeout -- Use signal 2 (SIGINT; same pressing CTRL+C) -- Abort the command if it is still running when max time is reached -- If the command was aborted, rollback this deployment --- deploy/kubernetes/atst-update-deploy.sh | 28 ++++++++++++++++--------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/deploy/kubernetes/atst-update-deploy.sh b/deploy/kubernetes/atst-update-deploy.sh index c7b92364..a95fb262 100755 --- a/deploy/kubernetes/atst-update-deploy.sh +++ b/deploy/kubernetes/atst-update-deploy.sh @@ -8,6 +8,17 @@ set -o errexit set -o nounset # set -o xtrace +# Config +MAX_DEPLOY_WAIT='5m' + +# Remove the K8S CA file when the script exits +function cleanup { + printf "Cleaning up...\n" + rm -vf "${HOME}/k8s_ca.crt" + printf "Cleaning done." +} +trap cleanup EXIT + # Decode and save the K8S CA cert echo "${K8S_CA_CRT}" | base64 -d - > "${HOME}/k8s_ca.crt" @@ -31,13 +42,10 @@ kubectl config current-context kubectl -n atat set image deployment.apps/atst atst="${ATAT_DOCKER_REGISTRY_URL}/${PROD_IMAGE_NAME}:${GIT_SHA}" # Wait for deployment to finish -kubectl -n atat rollout status deployment/atst - -# Remove the K8S CA file when the script exits -function cleanup { - printf "Cleaning up...\n" - rm -vf "${HOME}/k8s_ca.crt" - printf "Cleaning done." -} - -trap cleanup EXIT +if ! timeout -s 2 "${MAX_DEPLOY_WAIT}" kubectl -n atat rollout status deployment/atst +then + # Deploy did not finish before max wait time; abort and rollback the deploy + kubectl -n atat rollout undo deployment/atst + # Exit with a non-zero return code + exit 2 +fi