Finalize CD config and add k8s job for migrations.

Add CircleCI config for both CSPs to:
- build the Docker image and push it to the registry
- run a short-lived k8s job to apply migrations and see data
- update the images for the Flask pods and rq worker pods
This commit is contained in:
dandds 2019-07-30 15:14:09 -04:00
parent 4c70d59c5d
commit f3e032fc03
4 changed files with 164 additions and 2 deletions

View File

@ -5,6 +5,7 @@ orbs:
aws-eks: circleci/aws-eks@0.1.0
azure-acr: circleci/azure-acr@0.1.1
azure-aks: circleci/azure-aks@0.2.0
kubernetes: circleci/kubernetes@0.3.0
defaults:
appEnvironment: &appEnvironment
@ -16,6 +17,27 @@ defaults:
PIP_VERSION: 18.*
CRL_STORAGE_PROVIDER: CLOUDFILES
commands:
migration_setup:
parameters:
container_image:
type: string
steps:
- attach_workspace:
at: .
- run:
name: Setup Environment Variables
command: |
echo 'export CONTAINER_IMAGE="<< parameters.container_image >>"' >> $BASH_ENV
- run: sudo apt-get update
- run: sudo apt-get install gettext
- kubernetes/install
migration_apply:
steps:
- run:
command: ./script/cluster_migration
name: Apply Migrations and Seed Roles
jobs:
app_setup:
docker:
@ -76,6 +98,27 @@ jobs:
name: "Run Tests"
command: ./script/cibuild
aws-migration:
executor: aws-eks/python3
steps:
- migration_setup:
container_image: "$AWS_ECR_ACCOUNT_URL/atat:$CIRCLE_SHA1"
- aws-eks/update-kubeconfig-with-authenticator:
cluster-name: atat
aws-region: "${AWS_REGION}"
- migration_apply
azure-migration:
executor: azure-aks/default
steps:
- migration_setup:
container_image: "$AZURE_SERVER_NAME/atat:$CIRCLE_SHA1"
- azure-aks/update-kubeconfig-with-credentials:
cluster-name: atat-cluster
install-kubectl: true
perform-login: true
resource-group: atat
- migration_apply
workflows:
version: 2
@ -96,6 +139,27 @@ workflows:
branches:
only:
- master
- azure-migration:
requires:
- azure-acr/build_and_push_image
filters:
branches:
only:
- master
- azure-aks/update-container-image:
cluster-name: atat-cluster
container-image-updates: "atst=${AZURE_SERVER_NAME}/atat:${CIRCLE_SHA1}"
namespace: atat
resource-name: deployment.apps/atst
resource-group: atat
# uncomment below for debugging
# show-kubectl-command: true
requires:
- azure-migration
filters:
branches:
only:
- master
- azure-aks/update-container-image:
cluster-name: atat-cluster
container-image-updates: "atst-worker=${AZURE_SERVER_NAME}/atat:${CIRCLE_SHA1}"
@ -105,7 +169,7 @@ workflows:
# uncomment below for debugging
# show-kubectl-command: true
requires:
- azure-acr/build_and_push_image
- azure-migration
filters:
branches:
only:
@ -119,6 +183,13 @@ workflows:
branches:
only:
- master
- aws-migration:
requires:
- aws-ecr/build_and_push_image
filters:
branches:
only:
- master
- aws-eks/update-container-image:
cluster-name: atat
container-image-updates: "atst=${AWS_ECR_ACCOUNT_URL}/atat:${CIRCLE_SHA1}"
@ -128,7 +199,21 @@ workflows:
# uncomment below for debugging
# show-kubectl-command: true
requires:
- aws-ecr/build_and_push_image
- aws-migration
filters:
branches:
only:
- master
- aws-eks/update-container-image:
cluster-name: atat
container-image-updates: "atst-worker=${AWS_ECR_ACCOUNT_URL}/atat:${CIRCLE_SHA1}"
namespace: atat
resource-name: deployment.apps/atst-worker
aws-region: "${AWS_REGION}"
# uncomment below for debugging
# show-kubectl-command: true
requires:
- aws-migration
filters:
branches:
only:

View File

@ -78,6 +78,7 @@ COPY --from=builder /install/atst/ ./atst/
COPY --from=builder /install/config/ ./config/
COPY --from=builder /install/templates/ ./templates/
COPY --from=builder /install/translations.yaml .
COPY --from=builder /install/script/seed_roles.py ./script/seed_roles.py
COPY --from=builder /install/static/ ./static/
COPY --from=builder /install/uwsgi.ini .
COPY --from=builder /usr/local/bin/uwsgi /usr/local/bin/uwsgi

42
k8s/shared/migration.yaml Normal file
View File

@ -0,0 +1,42 @@
apiVersion: batch/v1
kind: Job
metadata:
name: migration
namespace: atat
spec:
ttlSecondsAfterFinished: 100
backoffLimit: 2
template:
spec:
containers:
- name: migration
image: $CONTAINER_IMAGE
command: [
"/bin/sh", "-c"
]
args:
- |
/opt/atat/atst/.venv/bin/python \
/opt/atat/atst/.venv/bin/alembic \
upgrade head \
&& \
/opt/atat/atst/.venv/bin/python \
/opt/atat/atst/script/seed_roles.py
envFrom:
- configMapRef:
name: atst-envvars
- configMapRef:
name: atst-worker-envvars
volumeMounts:
- name: atst-config
mountPath: "/opt/atat/atst/atst-overrides.ini"
subPath: atst-overrides.ini
volumes:
- name: atst-config
secret:
secretName: atst-config-ini
items:
- key: override.ini
path: atst-overrides.ini
mode: 0644
restartPolicy: Never

34
script/cluster_migration Executable file
View File

@ -0,0 +1,34 @@
#!/bin/sh
if [ -z "${K8S_CONTEXT+is_set}" ]; then
K8S_CMD="kubectl"
else
K8S_CMD="kubectl --context=$K8S_CONTEXT"
fi
if [ -z "${MIGRATION_TIMEOUT+is_set}" ]; then
MIGRATION_TIMEOUT=120s
fi
echo "Creating job..."
envsubst < k8s/shared/migration.yaml | $K8S_CMD -n atat apply -f -
echo "Wait for job to finish or timeout..."
JOB_SUCCESS=$(${K8S_CMD} -n atat wait --for=condition=complete --timeout=${MIGRATION_TIMEOUT} job/migration)
delete_job () {
echo "Deleting job..."
$K8S_CMD -n atat delete job migration
}
if echo "$JOB_SUCCESS" | grep -q "condition met"; then
echo "Job ran successfully."
delete_job
exit 0
else
POD_NAME=$(${K8S_CMD} -n atat get pods -l job-name=migration -o=jsonpath='{.items[0].metadata.name}')
echo "Job failed:"
$K8S_CMD -n atat log $POD_NAME
delete_job
exit 1
fi