From 387f957aa4963fd7d1754f554d0413f081302e11 Mon Sep 17 00:00:00 2001 From: dandds Date: Tue, 12 Nov 2019 16:46:56 -0500 Subject: [PATCH] Add CircleCI config for staging deployment. This generalizes the deploy step into a configurable CircleCI command. The available parameters are: - `namespace`: the K8s namespace to alter - `tag`: the docker tag to apply to the image The script for applying migrations to the K8s environment and the corresponding K8s Job config have been generalized so that they can be configured to run in the specified namespace. The main workflow has been updated so that the appropriate deployment will happen, depending on whether we are merging to staging or master. In the future, we could look to add an additional workflow based around Git tags for production. Note that this also removes the creation of the `latest` tag from CD. That tag is no longer hard-coded into our K8s config and so there's no longer a need to update it in our container registry. --- .circleci/config.yml | 157 +++++++++++++++++++++-------------- deploy/shared/migration.yaml | 2 +- script/cluster_migration | 14 ++-- 3 files changed, 104 insertions(+), 69 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a0ec6dcc..10ed47a8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -54,6 +54,75 @@ commands: name: Apply the default permission sets command: docker run --network atat -e PGDATABASE=<< parameters.pgdatabase >> << parameters.container_env >> atat:builder .venv/bin/python script/seed_roles.py + deploy: + parameters: + namespace: + type: string + default: atat + tag: + type: string + default: ${AZURE_SERVER_NAME}/atat:latest + steps: + - checkout + - setup_remote_docker: + docker_layer_caching: true + version: 18.06.0-ce + - restore_docker_image + - run: + name: Install Azure CLI + command: | + apk update + apk add bash py-pip + apk add --virtual=build \ + linux-headers gcc libffi-dev musl-dev openssl-dev python-dev make + pip --no-cache-dir install -U pip + pip --no-cache-dir install azure-cli + apk del --purge build + - run: + name: Login to Azure CLI + command: | + az login \ + --service-principal \ + --tenant $AZURE_SP_TENANT \ + --password $AZURE_SP_PASSWORD \ + --username $AZURE_SP + echo "Successfully logged in to Azure CLI." + az acr login --name $AZURE_REGISTRY + - run: + name: Install kubectl + command: | + apk add curl + export KUBECTL_VERSION=$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt) + curl -LO https://storage.googleapis.com/kubernetes-release/release/$KUBECTL_VERSION/bin/linux/amd64/kubectl + chmod +x ./kubectl + mv ./kubectl /usr/local/bin + - run: + name: Configure kubectl + command: | + apk add libssl1.0 + az aks get-credentials --name ${CLUSTER_NAME} --resource-group ${RESOURCE_GROUP} + - run: + name: Tag images + command: | + docker tag atat:latest << parameters.tag >> + - run: + name: Push image + command: | + docker push << parameters.tag >> + - run: + name: Add gettext package + command: apk add gettext + - run: + command: K8S_NAMESPACE=<< parameters.namespace >> CONTAINER_IMAGE=<< parameters.tag >> /bin/sh ./script/cluster_migration + name: Apply Migrations and Seed Roles + - run: + name: Update Kubernetes cluster + command: | + kubectl set image deployment.apps/atst atst=<< parameters.tag >> --namespace=<< parameters.namespace >> + kubectl set image deployment.apps/atst-worker atst-worker=<< parameters.tag >> --namespace=<< parameters.namespace >> + kubectl set image deployment.apps/atst-beat atst-beat=<< parameters.tag >> --namespace=<< parameters.namespace >> + kubectl set image cronjobs.batch/crls crls=<< parameters.tag >> --namespace=<< parameters.namespace >> + jobs: docker-build: docker: @@ -160,7 +229,7 @@ jobs: atat:builder \ /bin/sh -c "pipenv install --dev && /bin/sh script/sync-crls && pipenv run pytest --no-cov tests/check_crl_parse.py" - deploy: + deploy-staging: docker: - image: docker:18.06.0-ce-git environment: @@ -168,67 +237,21 @@ jobs: RESOURCE_GROUP: atat CLUSTER_NAME: atat-cluster steps: - - checkout - - setup_remote_docker: - docker_layer_caching: true - version: 18.06.0-ce - - restore_docker_image - - run: - name: Install Azure CLI - command: | - apk update - apk add bash py-pip - apk add --virtual=build \ - linux-headers gcc libffi-dev musl-dev openssl-dev python-dev make - pip --no-cache-dir install -U pip - pip --no-cache-dir install azure-cli - apk del --purge build - - run: - name: Login to Azure CLI - command: | - az login \ - --service-principal \ - --tenant $AZURE_SP_TENANT \ - --password $AZURE_SP_PASSWORD \ - --username $AZURE_SP - echo "Successfully logged in to Azure CLI." - az acr login --name $AZURE_REGISTRY - - run: - name: Install kubectl - command: | - apk add curl - export KUBECTL_VERSION=$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt) - curl -LO https://storage.googleapis.com/kubernetes-release/release/$KUBECTL_VERSION/bin/linux/amd64/kubectl - chmod +x ./kubectl - mv ./kubectl /usr/local/bin - - run: - name: Configure kubectl - command: | - apk add libssl1.0 - az aks get-credentials --name ${CLUSTER_NAME} --resource-group ${RESOURCE_GROUP} - - run: - name: Tag images - command: | - docker tag atat:latest ${AZURE_SERVER_NAME}/atat:atat-${CIRCLE_SHA1} - docker tag atat:latest ${AZURE_SERVER_NAME}/atat:latest - - run: - name: Push image - command: | - docker push ${AZURE_SERVER_NAME}/atat:atat-${CIRCLE_SHA1} - docker push ${AZURE_SERVER_NAME}/atat:latest - - run: - name: Add gettext package - command: apk add gettext - - run: - command: CONTAINER_IMAGE=${AZURE_SERVER_NAME}/atat:atat-${CIRCLE_SHA1} /bin/sh ./script/cluster_migration - name: Apply Migrations and Seed Roles - - run: - name: Update Kubernetes cluster - command: | - kubectl set image deployment.apps/atst atst=${AZURE_SERVER_NAME}/atat:atat-${CIRCLE_SHA1} --namespace=atat - kubectl set image deployment.apps/atst-worker atst-worker=${AZURE_SERVER_NAME}/atat:atat-${CIRCLE_SHA1} --namespace=atat - kubectl set image deployment.apps/atst-beat atst-beat=${AZURE_SERVER_NAME}/atat:atat-${CIRCLE_SHA1} --namespace=atat - kubectl set image cronjobs.batch/crls crls=${AZURE_SERVER_NAME}/atat:atat-${CIRCLE_SHA1} --namespace=atat + - deploy: + namespace: staging + tag: ${AZURE_SERVER_NAME}/atat:staging-${CIRCLE_SHA1} + + deploy-master: + docker: + - image: docker:18.06.0-ce-git + environment: + AZURE_REGISTRY: pwatat + RESOURCE_GROUP: atat + CLUSTER_NAME: atat-cluster + steps: + - deploy: + namespace: atat + tag: ${AZURE_SERVER_NAME}/atat:master-${CIRCLE_SHA1} workflows: version: 2 @@ -241,7 +264,15 @@ workflows: - integration-tests: requires: - docker-build - - deploy: + - deploy-staging: + requires: + - test + - integration-tests + filters: + branches: + only: + - staging + - deploy-master: requires: - test - integration-tests diff --git a/deploy/shared/migration.yaml b/deploy/shared/migration.yaml index c26ca543..d571c84d 100644 --- a/deploy/shared/migration.yaml +++ b/deploy/shared/migration.yaml @@ -2,7 +2,7 @@ apiVersion: batch/v1 kind: Job metadata: name: migration - namespace: atat + namespace: $K8S_NAMESPACE spec: ttlSecondsAfterFinished: 100 backoffLimit: 2 diff --git a/script/cluster_migration b/script/cluster_migration index 03a85a29..ad08e9ab 100755 --- a/script/cluster_migration +++ b/script/cluster_migration @@ -10,15 +10,19 @@ if [ -z "${MIGRATION_TIMEOUT+is_set}" ]; then MIGRATION_TIMEOUT=120s fi +if [ -z "${K8S_NAMESPACE+is_set}" ]; then + K8S_NAMESPACE=atat +fi + echo "Creating job..." -envsubst < deploy/shared/migration.yaml | $K8S_CMD -n atat apply -f - +envsubst < deploy/shared/migration.yaml | $K8S_CMD -n ${K8S_NAMESPACE} apply -f - echo "Wait for job to finish or timeout..." -JOB_SUCCESS=$(${K8S_CMD} -n atat wait --for=condition=complete --timeout=${MIGRATION_TIMEOUT} job/migration) +JOB_SUCCESS=$(${K8S_CMD} -n ${K8S_NAMESPACE} wait --for=condition=complete --timeout=${MIGRATION_TIMEOUT} job/migration) delete_job () { echo "Deleting job..." - $K8S_CMD -n atat delete job migration + $K8S_CMD -n ${K8S_NAMESPACE} delete job migration } if echo "$JOB_SUCCESS" | grep -q "condition met"; then @@ -26,9 +30,9 @@ if echo "$JOB_SUCCESS" | grep -q "condition met"; then delete_job exit 0 else - POD_NAME=$(${K8S_CMD} -n atat get pods -l job-name=migration -o=jsonpath='{.items[0].metadata.name}') + POD_NAME=$(${K8S_CMD} -n ${K8S_NAMESPACE} get pods -l job-name=migration -o=jsonpath='{.items[0].metadata.name}') echo "Job failed:" - $K8S_CMD -n atat logs $POD_NAME + $K8S_CMD -n ${K8S_NAMESPACE} logs $POD_NAME delete_job exit 1 fi