From f3e032fc03b2167898526048544d7debd918e6ad Mon Sep 17 00:00:00 2001 From: dandds Date: Tue, 30 Jul 2019 15:14:09 -0400 Subject: [PATCH] Finalize CD config and add k8s job for migrations. Add CircleCI config for both CSPs to: - build the Docker image and push it to the registry - run a short-lived k8s job to apply migrations and see data - update the images for the Flask pods and rq worker pods --- .circleci/config.yml | 89 ++++++++++++++++++++++++++++++++++++++- Dockerfile | 1 + k8s/shared/migration.yaml | 42 ++++++++++++++++++ script/cluster_migration | 34 +++++++++++++++ 4 files changed, 164 insertions(+), 2 deletions(-) create mode 100644 k8s/shared/migration.yaml create mode 100755 script/cluster_migration diff --git a/.circleci/config.yml b/.circleci/config.yml index 73d4da23..04e9f70c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,6 +5,7 @@ orbs: aws-eks: circleci/aws-eks@0.1.0 azure-acr: circleci/azure-acr@0.1.1 azure-aks: circleci/azure-aks@0.2.0 + kubernetes: circleci/kubernetes@0.3.0 defaults: appEnvironment: &appEnvironment @@ -16,6 +17,27 @@ defaults: PIP_VERSION: 18.* CRL_STORAGE_PROVIDER: CLOUDFILES +commands: + migration_setup: + parameters: + container_image: + type: string + steps: + - attach_workspace: + at: . + - run: + name: Setup Environment Variables + command: | + echo 'export CONTAINER_IMAGE="<< parameters.container_image >>"' >> $BASH_ENV + - run: sudo apt-get update + - run: sudo apt-get install gettext + - kubernetes/install + migration_apply: + steps: + - run: + command: ./script/cluster_migration + name: Apply Migrations and Seed Roles + jobs: app_setup: docker: @@ -76,6 +98,27 @@ jobs: name: "Run Tests" command: ./script/cibuild + aws-migration: + executor: aws-eks/python3 + steps: + - migration_setup: + container_image: "$AWS_ECR_ACCOUNT_URL/atat:$CIRCLE_SHA1" + - aws-eks/update-kubeconfig-with-authenticator: + cluster-name: atat + aws-region: "${AWS_REGION}" + - migration_apply + + azure-migration: + executor: azure-aks/default + steps: + - migration_setup: + container_image: "$AZURE_SERVER_NAME/atat:$CIRCLE_SHA1" + - azure-aks/update-kubeconfig-with-credentials: + cluster-name: atat-cluster + install-kubectl: true + perform-login: true + resource-group: atat + - migration_apply workflows: version: 2 @@ -96,6 +139,27 @@ workflows: branches: only: - master + - azure-migration: + requires: + - azure-acr/build_and_push_image + filters: + branches: + only: + - master + - azure-aks/update-container-image: + cluster-name: atat-cluster + container-image-updates: "atst=${AZURE_SERVER_NAME}/atat:${CIRCLE_SHA1}" + namespace: atat + resource-name: deployment.apps/atst + resource-group: atat + # uncomment below for debugging + # show-kubectl-command: true + requires: + - azure-migration + filters: + branches: + only: + - master - azure-aks/update-container-image: cluster-name: atat-cluster container-image-updates: "atst-worker=${AZURE_SERVER_NAME}/atat:${CIRCLE_SHA1}" @@ -105,7 +169,7 @@ workflows: # uncomment below for debugging # show-kubectl-command: true requires: - - azure-acr/build_and_push_image + - azure-migration filters: branches: only: @@ -119,6 +183,13 @@ workflows: branches: only: - master + - aws-migration: + requires: + - aws-ecr/build_and_push_image + filters: + branches: + only: + - master - aws-eks/update-container-image: cluster-name: atat container-image-updates: "atst=${AWS_ECR_ACCOUNT_URL}/atat:${CIRCLE_SHA1}" @@ -128,7 +199,21 @@ workflows: # uncomment below for debugging # show-kubectl-command: true requires: - - aws-ecr/build_and_push_image + - aws-migration + filters: + branches: + only: + - master + - aws-eks/update-container-image: + cluster-name: atat + container-image-updates: "atst-worker=${AWS_ECR_ACCOUNT_URL}/atat:${CIRCLE_SHA1}" + namespace: atat + resource-name: deployment.apps/atst-worker + aws-region: "${AWS_REGION}" + # uncomment below for debugging + # show-kubectl-command: true + requires: + - aws-migration filters: branches: only: diff --git a/Dockerfile b/Dockerfile index 681515b4..2785b815 100644 --- a/Dockerfile +++ b/Dockerfile @@ -78,6 +78,7 @@ COPY --from=builder /install/atst/ ./atst/ COPY --from=builder /install/config/ ./config/ COPY --from=builder /install/templates/ ./templates/ COPY --from=builder /install/translations.yaml . +COPY --from=builder /install/script/seed_roles.py ./script/seed_roles.py COPY --from=builder /install/static/ ./static/ COPY --from=builder /install/uwsgi.ini . COPY --from=builder /usr/local/bin/uwsgi /usr/local/bin/uwsgi diff --git a/k8s/shared/migration.yaml b/k8s/shared/migration.yaml new file mode 100644 index 00000000..cbf7da6b --- /dev/null +++ b/k8s/shared/migration.yaml @@ -0,0 +1,42 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: migration + namespace: atat +spec: + ttlSecondsAfterFinished: 100 + backoffLimit: 2 + template: + spec: + containers: + - name: migration + image: $CONTAINER_IMAGE + command: [ + "/bin/sh", "-c" + ] + args: + - | + /opt/atat/atst/.venv/bin/python \ + /opt/atat/atst/.venv/bin/alembic \ + upgrade head \ + && \ + /opt/atat/atst/.venv/bin/python \ + /opt/atat/atst/script/seed_roles.py + envFrom: + - configMapRef: + name: atst-envvars + - configMapRef: + name: atst-worker-envvars + volumeMounts: + - name: atst-config + mountPath: "/opt/atat/atst/atst-overrides.ini" + subPath: atst-overrides.ini + volumes: + - name: atst-config + secret: + secretName: atst-config-ini + items: + - key: override.ini + path: atst-overrides.ini + mode: 0644 + restartPolicy: Never diff --git a/script/cluster_migration b/script/cluster_migration new file mode 100755 index 00000000..2c011365 --- /dev/null +++ b/script/cluster_migration @@ -0,0 +1,34 @@ +#!/bin/sh + +if [ -z "${K8S_CONTEXT+is_set}" ]; then + K8S_CMD="kubectl" +else + K8S_CMD="kubectl --context=$K8S_CONTEXT" +fi + +if [ -z "${MIGRATION_TIMEOUT+is_set}" ]; then + MIGRATION_TIMEOUT=120s +fi + +echo "Creating job..." +envsubst < k8s/shared/migration.yaml | $K8S_CMD -n atat apply -f - + +echo "Wait for job to finish or timeout..." +JOB_SUCCESS=$(${K8S_CMD} -n atat wait --for=condition=complete --timeout=${MIGRATION_TIMEOUT} job/migration) + +delete_job () { + echo "Deleting job..." + $K8S_CMD -n atat delete job migration +} + +if echo "$JOB_SUCCESS" | grep -q "condition met"; then + echo "Job ran successfully." + delete_job + exit 0 +else + POD_NAME=$(${K8S_CMD} -n atat get pods -l job-name=migration -o=jsonpath='{.items[0].metadata.name}') + echo "Job failed:" + $K8S_CMD -n atat log $POD_NAME + delete_job + exit 1 +fi