diff --git a/.circleci/config.yml b/.circleci/config.yml index 45770805..c22dbf93 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,25 +1,44 @@ version: 2.0 + +defaults: + working_directory: &workingDirectory /opt/atat/atst + sourceImage: &sourceImage registry.atat.codes:443/atat-app-builder:circleci-cd + sourceAuth: &sourceAuth + username: $REGISTRY_USERNAME + password: $REGISTRY_PASSWORD + appEnvironment: &appEnvironment + KEEP_EXISTING_VENV: true + PGHOST: localhost + PGUSER: root + PGDATABASE: circle_test + REDIS_URI: redis://localhost:6379 + dockerCmdEnvironment: &dockerCmdEnvironment + APP_USER: atst + APP_GROUP: atat + APP_DIR: /opt/atat/atst + ATAT_DOCKER_REGISTRY_URL: registry.atat.codes:443 + CONTAINER_NAME: atst-container + USR_BIN_DIR: /usr/bin + PYTHON_SITE_PACKAGES_DIR: /usr/lib/python3.6/site-packages + PROD_IMAGE_NAME: atst-prod + jobs: - build: + app_setup: docker: - - image: registry.atat.codes:443/atat-app-builder:circleci - auth: - username: $REGISTRY_USERNAME - password: $REGISTRY_PASSWORD - environment: - KEEP_EXISTING_VENV: true - PGHOST: localhost - PGUSER: root - PGDATABASE: circle_test - REDIS_URI: redis://localhost:6379 + - image: *sourceImage + auth: *sourceAuth + environment: *appEnvironment - image: circleci/postgres:9.6.5-alpine-ram - image: circleci/redis:4-alpine3.8 + working_directory: *workingDirectory steps: - checkout - run: name: "Clone Submodules" command: | git submodule update --init --recursive + - attach_workspace: + at: . - restore_cache: name: "Load Cache: Pipenv References" keys: @@ -38,6 +57,10 @@ jobs: - yarn-v1-{{ .Branch }}-{{ checksum "yarn.lock" }} - yarn-v1-{{ .Branch }}- - yarn-v1- + - restore_cache: + name: "Load Cache: Node Modules" + keys: + - node-v1-{{ .Branch }}-{{ checksum "yarn.lock" }} - run: ./script/setup - save_cache: name: "Save Cache: Pipenv Refrences" @@ -54,12 +77,141 @@ jobs: paths: - ~/.cache/yarn key: yarn-v1-{{ .Branch }}-{{ checksum "yarn.lock" }} - - restore_cache: - keys: - - disa-crls - - run: ./script/sync-crls - save_cache: + name: "Save Cache: Node Modules" + paths: + - ./node_modules + key: node-v1-{{ .Branch }}-{{ checksum "yarn.lock" }} + - restore_cache: + name: "Load Cache: CRLs" + keys: + - disa-crls-v2 + - run: + name: "Update CRLs" + command: ./script/sync-crls + - save_cache: + name: "Save Cache: CRLs" paths: - ./crl - key: disa-crls - - run: ./script/cibuild + key: disa-crls-v2-{{ .Branch }}-{{ epoch}} + - run: + name: "Generate build info" + command: ./script/generate_build_info.sh + - persist_to_workspace: + root: . + paths: + - . + + test: + docker: + - image: *sourceImage + auth: *sourceAuth + environment: *appEnvironment + - image: circleci/postgres:9.6.5-alpine-ram + - image: circleci/redis:4-alpine3.8 + working_directory: *workingDirectory + steps: + - attach_workspace: + at: . + - run: + name: "Run Tests" + command: ./script/cibuild + + build_and_push_image: + docker: + - image: *sourceImage + auth: *sourceAuth + environment: *dockerCmdEnvironment + working_directory: *workingDirectory + steps: + - attach_workspace: + at: . + - setup_remote_docker: + version: 18.05.0-ce + - run: + name: "Export GIT_SHA" + command: echo "export GIT_SHA=$(git rev-parse --short HEAD)" >> $BASH_ENV + - run: + name: "Generate the Target Image Name" + command: echo "export IMAGE_NAME=\"${ATAT_DOCKER_REGISTRY_URL}/${PROD_IMAGE_NAME}:${GIT_SHA}-circleci\"" >> $BASH_ENV + - run: + name: "Start a Fresh Container" + command: docker run -d --entrypoint='/bin/sh' -ti --name ${CONTAINER_NAME} alpine:3.8 + - run: + name: "Create the App Directory" + command: docker exec -t ${CONTAINER_NAME} mkdir -p ${APP_DIR} + - run: + name: "Copy Workspace Contents into the Container" + command: docker cp . ${CONTAINER_NAME}:${APP_DIR} + - run: + name: "Run Alpine Setup" + command: docker exec -t --workdir ${APP_DIR} ${CONTAINER_NAME} ./script/alpine_setup + - run: + name: "Copy System Site Packages into the Container" + command: docker cp -a ${PYTHON_SITE_PACKAGES_DIR}/. ${CONTAINER_NAME}:${PYTHON_SITE_PACKAGES_DIR} + - run: + name: "Copy USR_BIN Contents into the Container" + command: docker cp -a ${USR_BIN_DIR}/. ${CONTAINER_NAME}:${USR_BIN_DIR} + - run: + name: "Run Fix Permissions" + command: docker exec -t --workdir ${APP_DIR} ${CONTAINER_NAME} ./script/fix_permissions ${APP_USER} ${APP_GROUP} + - run: + name: "Commit Container Changes to New Image" + command: | + docker commit \ + --change="ENV APP_USER \"${APP_USER}\"" \ + --change="ENV APP_GROUP \"${APP_GROUP}\"" \ + --change="ENV APP_DIR \"${APP_DIR}\"" \ + --change='ENTRYPOINT ["/usr/bin/dumb-init", "--"]' \ + --change="CMD [\"bash\", \"-c\", \"${APP_DIR}/script/uwsgi_server\"]" \ + --change="WORKDIR ${APP_DIR}" \ + --change="USER \"${APP_USER}\"" \ + ${CONTAINER_NAME} \ + ${IMAGE_NAME} + - run: + name: "Publish ATST Image" + command: | + docker image ls + docker login -u ${REGISTRY_USERNAME} -p ${REGISTRY_PASSWORD} ${ATAT_DOCKER_REGISTRY_URL} + docker push ${IMAGE_NAME} + docker logout + + deploy: + docker: + - image: *sourceImage + auth: *sourceAuth + environment: *dockerCmdEnvironment + working_directory: *workingDirectory + steps: + - attach_workspace: + at: . + - run: + name: "Export GIT_SHA" + command: echo "export GIT_SHA=$(git rev-parse --short HEAD)" >> $BASH_ENV + - run: + name: "Generate the Target Image Name" + command: echo "export IMAGE_NAME=\"${ATAT_DOCKER_REGISTRY_URL}/${PROD_IMAGE_NAME}:${GIT_SHA}-circleci\"" >> $BASH_ENV + - run: + name: "Update Kubernetes Deployment" + command: ./deploy/kubernetes/atst-update-deploy.sh + +workflows: + version: 2 + run-tests: + jobs: + - app_setup + - test: + requires: + - app_setup + - build_and_push_image: + requires: + - test + filters: + branches: + only: master + - deploy: + requires: + - build_and_push_image + filters: + branches: + only: master diff --git a/.travis.yml b/.travis.yml index 5ea87ca8..d5167828 100644 --- a/.travis.yml +++ b/.travis.yml @@ -45,16 +45,9 @@ script: - docker container stop current-atst-tester - docker run --add-host "postgreshost:${postgres_ip}" --add-host "redishost:${redis_ip}" "${TESTER_IMAGE2_NAME}" -before_deploy: - - docker build --tag "${PROD_IMAGE_NAME}" . -f deploy/docker/prod/Dockerfile - - remote_image_name="${ATAT_DOCKER_REGISTRY_URL}/${PROD_IMAGE_NAME}:${GIT_SHA}" - - docker tag "${PROD_IMAGE_NAME}" "${remote_image_name}" - - docker images - - docker push "${remote_image_name}" - deploy: provider: script - script: deploy/kubernetes/atst-update-deploy.sh + script: echo "Deployment now handles by CircleCI" on: branch: master diff --git a/deploy/kubernetes/atst-check-deploy.sh b/deploy/kubernetes/atst-check-deploy.sh new file mode 100755 index 00000000..3b2f588c --- /dev/null +++ b/deploy/kubernetes/atst-check-deploy.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# + +set -o pipefail +set -o errexit +set -o nounset + +# Decode and save the K8S CA cert +echo "${K8S_CA_CRT}" | base64 -d - > "${HOME}/k8s_ca.crt" + +# Setup the local kubectl client +kubectl config set-context travis \ + --cluster=atat-cluster \ + --user=atat-deployer \ + --namespace=atat + +kubectl config set-cluster atat-cluster \ + --embed-certs=true \ + --server="${K8S_ENDPOINT}" \ + --certificate-authority="${HOME}/k8s_ca.crt" + +kubectl config set-credentials atat-deployer --token="$(echo ${K8S_USER_TOKEN} | base64 -d -)" + +kubectl config use-context travis +kubectl config current-context + +echo +echo "Current ATST Deployment Details:" +kubectl -n atat get deployment.apps/atst -o yaml + +# Remove the K8S CA file when the script exits +function cleanup { + printf "Cleaning up...\n" + rm -vf "${HOME}/k8s_ca.crt" + printf "Cleaning done." +} + +trap cleanup EXIT diff --git a/deploy/kubernetes/atst-debugger.yml b/deploy/kubernetes/atst-debugger.yml index d86abd0f..6708a02a 100644 --- a/deploy/kubernetes/atst-debugger.yml +++ b/deploy/kubernetes/atst-debugger.yml @@ -1,14 +1,16 @@ apiVersion: v1 kind: Pod metadata: - name: atst-debugger + labels: + app: atst-debugger + name: atst-debugger-v1 namespace: atat spec: securityContext: fsGroup: 101 containers: - - name: atst-debugger - image: registry.atat.codes:443/atst-prod:a1916b1 + - name: atst + image: registry.atat.codes:443/atst-prod:6329f8e args: ["/bin/bash", "-c", "while true; do date; sleep 45; done"] envFrom: - configMapRef: @@ -17,19 +19,85 @@ spec: - name: atst-config mountPath: "/opt/atat/atst/atst-overrides.ini" subPath: atst-overrides.ini + - name: nginx-client-ca-bundle + mountPath: "/opt/atat/atst/ssl/server-certs/ca-chain.pem" + subPath: client-ca-bundle.pem - name: uwsgi-config mountPath: "/opt/atat/atst/uwsgi-config.ini" subPath: uwsgi-config.ini - name: uwsgi-socket-dir mountPath: "/var/run/uwsgi" + - name: atst-nginx + image: nginx:alpine + ports: + - containerPort: 8442 + name: http + - containerPort: 8443 + name: https + volumeMounts: + - name: nginx-auth-tls + mountPath: "/etc/ssl/private" + - name: nginx-client-ca-bundle + mountPath: "/etc/ssl/client-ca-bundle.pem" + subPath: client-ca-bundle.pem + - name: nginx-config + mountPath: "/etc/nginx/conf.d/atst.conf" + subPath: atst.conf + - name: nginx-dhparam + mountPath: "/etc/ssl/dhparam.pem" + subPath: dhparam.pem + - name: nginx-htpasswd + mountPath: "/etc/nginx/.htpasswd" + subPath: .htpasswd + - name: uwsgi-socket-dir + mountPath: "/var/run/uwsgi" + imagePullSecrets: + - name: regcred volumes: - name: atst-config secret: secretName: atst-config-ini items: - - key: atst-overrides.ini + - key: override.ini path: atst-overrides.ini mode: 0644 + - name: nginx-auth-tls + secret: + secretName: auth-atst-ingress-tls + items: + - key: tls.crt + path: auth.atat.crt + mode: 0644 + - key: tls.key + path: auth.atat.key + mode: 0640 + - name: nginx-client-ca-bundle + secret: + secretName: nginx-client-ca-bundle + items: + - key: client-ca-bundle.pem + path: client-ca-bundle.pem + mode: 0666 + - name: nginx-config + configMap: + name: atst-nginx + items: + - key: nginx-config + path: atst.conf + - name: nginx-dhparam + secret: + secretName: dhparam-4096 + items: + - key: dhparam.pem + path: dhparam.pem + mode: 0640 + - name: nginx-htpasswd + secret: + secretName: atst-nginx-htpasswd + items: + - key: htpasswd + path: .htpasswd + mode: 0640 - name: uwsgi-config configMap: name: atst-config @@ -41,3 +109,18 @@ spec: emptyDir: medium: Memory restartPolicy: Never +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: atst-debugger + name: atst-debugger + namespace: atat +spec: + ports: + - name: http + port: 80 + targetPort: 8442 + selector: + app: atst-debugger diff --git a/deploy/kubernetes/atst-update-deploy.sh b/deploy/kubernetes/atst-update-deploy.sh index f83bbe01..75b3f6d6 100755 --- a/deploy/kubernetes/atst-update-deploy.sh +++ b/deploy/kubernetes/atst-update-deploy.sh @@ -8,8 +8,24 @@ set -o errexit set -o nounset # set -o xtrace +# Config +MAX_DEPLOY_WAIT='300' + +if [ "${IMAGE_NAME}x" = "x" ] +then + IMAGE_NAME="${ATAT_DOCKER_REGISTRY_URL}/${PROD_IMAGE_NAME}:${GIT_SHA}" +fi + +# Remove the K8S CA file when the script exits +function cleanup { + printf "Cleaning up...\n" + rm -vf "${HOME}/k8s_ca.crt" + printf "Cleaning done." +} +trap cleanup EXIT + # Decode and save the K8S CA cert -echo "${K8S_CA_CRT}" | base64 --decode -i > "${HOME}/k8s_ca.crt" +echo "${K8S_CA_CRT}" | base64 -d - > "${HOME}/k8s_ca.crt" # Setup the local kubectl client kubectl config set-context travis \ @@ -22,22 +38,19 @@ kubectl config set-cluster atat-cluster \ --server="${K8S_ENDPOINT}" \ --certificate-authority="${HOME}/k8s_ca.crt" -kubectl config set-credentials atat-deployer --token=`echo ${K8S_USER_TOKEN} | base64 --decode` +kubectl config set-credentials atat-deployer --token="$(echo ${K8S_USER_TOKEN} | base64 -d -)" kubectl config use-context travis kubectl config current-context # Update the ATST deployment -kubectl -n atat set image deployment.apps/atst atst="${ATAT_DOCKER_REGISTRY_URL}/${PROD_IMAGE_NAME}:${GIT_SHA}" +kubectl -n atat set image deployment.apps/atst atst="${IMAGE_NAME}" # Wait for deployment to finish -kubectl -n atat rollout status deployment/atst - -# Remove the K8S CA file when the script exits -function cleanup { - printf "Cleaning up...\n" - rm -vf "${HOME}/k8s_ca.crt" - printf "Cleaning done." -} - -trap cleanup EXIT +if ! timeout -t "${MAX_DEPLOY_WAIT}" -s INT kubectl -n atat rollout status deployment/atst +then + # Deploy did not finish before max wait time; abort and rollback the deploy + kubectl -n atat rollout undo deployment/atst + # Exit with a non-zero return code + exit 2 +fi diff --git a/script/cibuild b/script/cibuild index 1493008c..39afa18d 100755 --- a/script/cibuild +++ b/script/cibuild @@ -13,6 +13,9 @@ PYTHON_FILES="./app.py ./atst/** ./config" # Enable Python testing RUN_PYTHON_TESTS="true" +# Reset the DB, since the one script/setup created might not be persisted +RESET_DB="true" + # Check python formatting source ./script/format check diff --git a/script/fix_permissions b/script/fix_permissions new file mode 100755 index 00000000..1ec5acee --- /dev/null +++ b/script/fix_permissions @@ -0,0 +1,24 @@ +#!/bin/bash + +# script/fix_permissions: Updates the app directory with the correct user +# permissions (skipping node_modules since it is not +# required and very large) + +source "$(dirname "${0}")"/../script/include/global_header.inc.sh + +APP_USER="${1}" +APP_GROUP="${2}" + +if [ "${APP_USER}x" = "x" ] || [ "${APP_GROUP}x" = "x" ]; then + echo "ERROR: Missing username or groupname argument!" + echo "Received: *${APP_USER}:${APP_GROUP}*" + echo + exit 1 +fi + +chown "${APP_USER}:${APP_GROUP}" . +chown "${APP_USER}:${APP_GROUP}" ./* +for subdir in $(find . -type d -maxdepth 1 | grep -Ee '.[^/]' | grep -Fve 'node_modules') +do + chown "${APP_USER}:${APP_GROUP}" -R "${subdir}" +done diff --git a/script/generate_build_info.sh b/script/generate_build_info.sh index c40ef269..e660b933 100755 --- a/script/generate_build_info.sh +++ b/script/generate_build_info.sh @@ -111,7 +111,7 @@ cat > ${STATIC_DIR}/buildinfo.html < - +
BuildInfo (${BUILT_BY}BuildInfo (${BUILT_BY})
Container Image Creation Time: