Merge pull request #276 from dod-ccpo/circleci-cd

CircleCI CD

.circleci/config.yml

@@ -1,25 +1,44 @@
 version: 2.0
+
+defaults:
+  working_directory: &workingDirectory /opt/atat/atst
+  sourceImage: &sourceImage registry.atat.codes:443/atat-app-builder:circleci-cd
+  sourceAuth: &sourceAuth
+    username: $REGISTRY_USERNAME
+    password: $REGISTRY_PASSWORD
+  appEnvironment: &appEnvironment
+    KEEP_EXISTING_VENV: true
+    PGHOST: localhost
+    PGUSER: root
+    PGDATABASE: circle_test
+    REDIS_URI: redis://localhost:6379
+  dockerCmdEnvironment: &dockerCmdEnvironment
+    APP_USER: atst
+    APP_GROUP: atat
+    APP_DIR: /opt/atat/atst
+    ATAT_DOCKER_REGISTRY_URL: registry.atat.codes:443
+    CONTAINER_NAME: atst-container
+    USR_BIN_DIR: /usr/bin
+    PYTHON_SITE_PACKAGES_DIR: /usr/lib/python3.6/site-packages
+    PROD_IMAGE_NAME: atst-prod
+
 jobs:
-  build:
+  app_setup:
     docker:
-      - image: registry.atat.codes:443/atat-app-builder:circleci
+      - image: *sourceImage
-        auth:
+        auth: *sourceAuth
-          username: $REGISTRY_USERNAME
+        environment: *appEnvironment
-          password: $REGISTRY_PASSWORD
-        environment:
-          KEEP_EXISTING_VENV: true
-          PGHOST: localhost
-          PGUSER: root
-          PGDATABASE: circle_test
-          REDIS_URI: redis://localhost:6379
       - image: circleci/postgres:9.6.5-alpine-ram
       - image: circleci/redis:4-alpine3.8
+    working_directory: *workingDirectory
     steps:
       - checkout
       - run:
           name: "Clone Submodules"
           command: |
             git submodule update --init --recursive
+      - attach_workspace:
+          at: .
       - restore_cache:
           name: "Load Cache: Pipenv References"
           keys:
@@ -38,6 +57,10 @@ jobs:
             - yarn-v1-{{ .Branch }}-{{ checksum "yarn.lock" }}
             - yarn-v1-{{ .Branch }}-
             - yarn-v1-
+      - restore_cache:
+          name: "Load Cache: Node Modules"
+          keys:
+            - node-v1-{{ .Branch }}-{{ checksum "yarn.lock" }}
       - run: ./script/setup
       - save_cache:
           name: "Save Cache: Pipenv Refrences"
@@ -54,12 +77,141 @@ jobs:
           paths:
             - ~/.cache/yarn
           key: yarn-v1-{{ .Branch }}-{{ checksum "yarn.lock" }}
-      - restore_cache:
-          keys:
-            - disa-crls
-      - run: ./script/sync-crls
       - save_cache:
+          name: "Save Cache: Node Modules"
+          paths:
+            - ./node_modules
+          key: node-v1-{{ .Branch }}-{{ checksum "yarn.lock" }}
+      - restore_cache:
+          name: "Load Cache: CRLs"
+          keys:
+            - disa-crls-v2
+      - run: 
+          name: "Update CRLs"
+          command: ./script/sync-crls
+      - save_cache:
+          name: "Save Cache: CRLs"
           paths:
             - ./crl
-          key: disa-crls
+          key: disa-crls-v2-{{ .Branch }}-{{ epoch}}
-      - run: ./script/cibuild
+      - run:
+          name: "Generate build info"
+          command: ./script/generate_build_info.sh
+      - persist_to_workspace:
+          root: .
+          paths:
+            - .
+
+  test:
+    docker:
+      - image: *sourceImage
+        auth: *sourceAuth
+        environment: *appEnvironment
+      - image: circleci/postgres:9.6.5-alpine-ram
+      - image: circleci/redis:4-alpine3.8
+    working_directory: *workingDirectory
+    steps:
+      - attach_workspace:
+          at: .
+      - run:
+          name: "Run Tests"
+          command: ./script/cibuild
+
+  build_and_push_image:
+    docker:
+      - image: *sourceImage
+        auth: *sourceAuth
+        environment: *dockerCmdEnvironment
+    working_directory: *workingDirectory
+    steps:
+      - attach_workspace:
+          at: .
+      - setup_remote_docker:
+          version: 18.05.0-ce
+      - run:
+          name: "Export GIT_SHA"
+          command: echo "export GIT_SHA=$(git rev-parse --short HEAD)" >> $BASH_ENV
+      - run:
+          name: "Generate the Target Image Name"
+          command: echo "export IMAGE_NAME=\"${ATAT_DOCKER_REGISTRY_URL}/${PROD_IMAGE_NAME}:${GIT_SHA}-circleci\"" >> $BASH_ENV
+      - run:
+          name: "Start a Fresh Container"
+          command: docker run -d --entrypoint='/bin/sh' -ti --name ${CONTAINER_NAME} alpine:3.8
+      - run:
+          name: "Create the App Directory"
+          command: docker exec -t ${CONTAINER_NAME} mkdir -p ${APP_DIR}
+      - run:
+          name: "Copy Workspace Contents into the Container"
+          command: docker cp . ${CONTAINER_NAME}:${APP_DIR}
+      - run:
+          name: "Run Alpine Setup"
+          command: docker exec -t --workdir ${APP_DIR} ${CONTAINER_NAME} ./script/alpine_setup
+      - run:
+          name: "Copy System Site Packages into the Container"
+          command: docker cp -a ${PYTHON_SITE_PACKAGES_DIR}/. ${CONTAINER_NAME}:${PYTHON_SITE_PACKAGES_DIR}
+      - run:
+          name: "Copy USR_BIN Contents into the Container"
+          command: docker cp -a ${USR_BIN_DIR}/. ${CONTAINER_NAME}:${USR_BIN_DIR}
+      - run:
+          name: "Run Fix Permissions"
+          command: docker exec -t --workdir ${APP_DIR} ${CONTAINER_NAME} ./script/fix_permissions ${APP_USER} ${APP_GROUP}
+      - run:
+          name: "Commit Container Changes to New Image"
+          command: |
+            docker commit \
+              --change="ENV APP_USER \"${APP_USER}\"" \
+              --change="ENV APP_GROUP \"${APP_GROUP}\"" \
+              --change="ENV APP_DIR \"${APP_DIR}\"" \
+              --change='ENTRYPOINT ["/usr/bin/dumb-init", "--"]' \
+              --change="CMD [\"bash\", \"-c\", \"${APP_DIR}/script/uwsgi_server\"]" \
+              --change="WORKDIR ${APP_DIR}" \
+              --change="USER \"${APP_USER}\"" \
+              ${CONTAINER_NAME} \
+              ${IMAGE_NAME}
+      - run:
+          name: "Publish ATST Image"
+          command: |
+            docker image ls
+            docker login -u ${REGISTRY_USERNAME} -p ${REGISTRY_PASSWORD} ${ATAT_DOCKER_REGISTRY_URL}
+            docker push ${IMAGE_NAME}
+            docker logout
+
+  deploy:
+    docker:
+      - image: *sourceImage
+        auth: *sourceAuth
+        environment: *dockerCmdEnvironment
+    working_directory: *workingDirectory
+    steps:
+      - attach_workspace:
+          at: .
+      - run:
+          name: "Export GIT_SHA"
+          command: echo "export GIT_SHA=$(git rev-parse --short HEAD)" >> $BASH_ENV
+      - run:
+          name: "Generate the Target Image Name"
+          command: echo "export IMAGE_NAME=\"${ATAT_DOCKER_REGISTRY_URL}/${PROD_IMAGE_NAME}:${GIT_SHA}-circleci\"" >> $BASH_ENV
+      - run:
+          name: "Update Kubernetes Deployment"
+          command: ./deploy/kubernetes/atst-update-deploy.sh
+
+workflows:
+  version: 2
+  run-tests:
+    jobs:
+      - app_setup
+      - test:
+          requires:
+            - app_setup
+      - build_and_push_image:
+          requires:
+            - test
+          filters:
+            branches:
+              only: master
+      - deploy:
+          requires:
+            - build_and_push_image
+          filters:
+            branches:
+              only: master

.travis.yml

@@ -45,16 +45,9 @@ script:
   - docker container stop current-atst-tester
   - docker run --add-host "postgreshost:${postgres_ip}" --add-host "redishost:${redis_ip}" "${TESTER_IMAGE2_NAME}"
 
-before_deploy:
-  - docker build --tag "${PROD_IMAGE_NAME}" . -f deploy/docker/prod/Dockerfile
-  - remote_image_name="${ATAT_DOCKER_REGISTRY_URL}/${PROD_IMAGE_NAME}:${GIT_SHA}"
-  - docker tag "${PROD_IMAGE_NAME}" "${remote_image_name}"
-  - docker images
-  - docker push "${remote_image_name}"
-
 deploy:
   provider: script
-  script: deploy/kubernetes/atst-update-deploy.sh
+  script: echo "Deployment now handles by CircleCI"
   on:
     branch: master
 

deploy/kubernetes/atst-check-deploy.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+#
+
+set -o pipefail
+set -o errexit
+set -o nounset
+
+# Decode and save the K8S CA cert
+echo "${K8S_CA_CRT}" | base64 -d - > "${HOME}/k8s_ca.crt"
+
+# Setup the local kubectl client
+kubectl config set-context travis \
+    --cluster=atat-cluster \
+    --user=atat-deployer \
+    --namespace=atat
+
+kubectl config set-cluster atat-cluster \
+    --embed-certs=true \
+    --server="${K8S_ENDPOINT}"  \
+    --certificate-authority="${HOME}/k8s_ca.crt"
+
+kubectl config set-credentials atat-deployer --token="$(echo ${K8S_USER_TOKEN} | base64 -d -)"
+
+kubectl config use-context travis
+kubectl config current-context
+
+echo
+echo "Current ATST Deployment Details:"
+kubectl -n atat get deployment.apps/atst -o yaml
+
+# Remove the K8S CA file when the script exits
+function cleanup {
+    printf "Cleaning up...\n"
+    rm -vf "${HOME}/k8s_ca.crt"
+    printf "Cleaning done."
+}
+
+trap cleanup EXIT

deploy/kubernetes/atst-debugger.yml

@@ -1,14 +1,16 @@
 apiVersion: v1
 kind: Pod
 metadata:
-  name: atst-debugger
+  labels:
+    app: atst-debugger
+  name: atst-debugger-v1
   namespace: atat
 spec:
   securityContext:
     fsGroup: 101
   containers:
-    - name: atst-debugger
+    - name: atst
-      image: registry.atat.codes:443/atst-prod:a1916b1
+      image: registry.atat.codes:443/atst-prod:6329f8e
       args: ["/bin/bash", "-c", "while true; do date; sleep 45; done"]
       envFrom:
       - configMapRef:
@@ -17,19 +19,85 @@ spec:
         - name: atst-config
           mountPath: "/opt/atat/atst/atst-overrides.ini"
           subPath: atst-overrides.ini
+        - name: nginx-client-ca-bundle
+          mountPath: "/opt/atat/atst/ssl/server-certs/ca-chain.pem"
+          subPath: client-ca-bundle.pem
         - name: uwsgi-config
           mountPath: "/opt/atat/atst/uwsgi-config.ini"
           subPath: uwsgi-config.ini
         - name: uwsgi-socket-dir
           mountPath: "/var/run/uwsgi"
+    - name: atst-nginx
+      image: nginx:alpine
+      ports:
+        - containerPort: 8442
+          name: http
+        - containerPort: 8443
+          name: https
+      volumeMounts:
+        - name: nginx-auth-tls
+          mountPath: "/etc/ssl/private"
+        - name: nginx-client-ca-bundle
+          mountPath: "/etc/ssl/client-ca-bundle.pem"
+          subPath: client-ca-bundle.pem
+        - name: nginx-config
+          mountPath: "/etc/nginx/conf.d/atst.conf"
+          subPath: atst.conf
+        - name: nginx-dhparam
+          mountPath: "/etc/ssl/dhparam.pem"
+          subPath: dhparam.pem
+        - name: nginx-htpasswd
+          mountPath: "/etc/nginx/.htpasswd"
+          subPath: .htpasswd
+        - name: uwsgi-socket-dir
+          mountPath: "/var/run/uwsgi"
+  imagePullSecrets:
+    - name: regcred
   volumes:
     - name: atst-config
       secret:
         secretName: atst-config-ini
         items:
-        - key: atst-overrides.ini
+        - key: override.ini
           path: atst-overrides.ini
           mode: 0644
+    - name: nginx-auth-tls
+      secret:
+        secretName: auth-atst-ingress-tls
+        items:
+        - key: tls.crt
+          path: auth.atat.crt
+          mode: 0644
+        - key: tls.key
+          path: auth.atat.key
+          mode: 0640
+    - name: nginx-client-ca-bundle
+      secret:
+        secretName: nginx-client-ca-bundle
+        items:
+        - key: client-ca-bundle.pem
+          path: client-ca-bundle.pem
+          mode: 0666
+    - name: nginx-config
+      configMap:
+        name: atst-nginx
+        items:
+        - key: nginx-config
+          path: atst.conf
+    - name: nginx-dhparam
+      secret:
+        secretName: dhparam-4096
+        items:
+        - key: dhparam.pem
+          path: dhparam.pem
+          mode: 0640
+    - name: nginx-htpasswd
+      secret:
+        secretName: atst-nginx-htpasswd
+        items:
+        - key: htpasswd
+          path: .htpasswd
+          mode: 0640
     - name: uwsgi-config
       configMap:
         name: atst-config
@@ -41,3 +109,18 @@ spec:
       emptyDir:
         medium: Memory
   restartPolicy: Never
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: atst-debugger
+  name: atst-debugger
+  namespace: atat
+spec:
+  ports:
+  - name: http
+    port: 80
+    targetPort: 8442
+  selector:
+    app: atst-debugger

deploy/kubernetes/atst-update-deploy.sh

@@ -8,8 +8,24 @@ set -o errexit
 set -o nounset
 # set -o xtrace
 
+# Config
+MAX_DEPLOY_WAIT='300'
+
+if [ "${IMAGE_NAME}x" = "x" ]
+then
+    IMAGE_NAME="${ATAT_DOCKER_REGISTRY_URL}/${PROD_IMAGE_NAME}:${GIT_SHA}"
+fi
+
+# Remove the K8S CA file when the script exits
+function cleanup {
+    printf "Cleaning up...\n"
+    rm -vf "${HOME}/k8s_ca.crt"
+    printf "Cleaning done."
+}
+trap cleanup EXIT
+
 # Decode and save the K8S CA cert
-echo "${K8S_CA_CRT}" | base64 --decode -i > "${HOME}/k8s_ca.crt"
+echo "${K8S_CA_CRT}" | base64 -d - > "${HOME}/k8s_ca.crt"
 
 # Setup the local kubectl client
 kubectl config set-context travis \
@@ -22,22 +38,19 @@ kubectl config set-cluster atat-cluster \
     --server="${K8S_ENDPOINT}"  \
     --certificate-authority="${HOME}/k8s_ca.crt"
 
-kubectl config set-credentials atat-deployer --token=`echo ${K8S_USER_TOKEN} | base64 --decode`
+kubectl config set-credentials atat-deployer --token="$(echo ${K8S_USER_TOKEN} | base64 -d -)"
 
 kubectl config use-context travis
 kubectl config current-context
 
 # Update the ATST deployment
-kubectl -n atat set image deployment.apps/atst atst="${ATAT_DOCKER_REGISTRY_URL}/${PROD_IMAGE_NAME}:${GIT_SHA}"
+kubectl -n atat set image deployment.apps/atst atst="${IMAGE_NAME}"
 
 # Wait for deployment to finish
-kubectl -n atat rollout status deployment/atst
+if ! timeout -t "${MAX_DEPLOY_WAIT}" -s INT kubectl -n atat rollout status deployment/atst
-
+then
-# Remove the K8S CA file when the script exits
+    # Deploy did not finish before max wait time; abort and rollback the deploy
-function cleanup {
+    kubectl -n atat rollout undo deployment/atst
-    printf "Cleaning up...\n"
+    # Exit with a non-zero return code
-    rm -vf "${HOME}/k8s_ca.crt"
+    exit 2
-    printf "Cleaning done."
+fi
-}
-
-trap cleanup EXIT

script/cibuild

@@ -13,6 +13,9 @@ PYTHON_FILES="./app.py ./atst/** ./config"
 # Enable Python testing
 RUN_PYTHON_TESTS="true"
 
+# Reset the DB, since the one script/setup created might not be persisted
+RESET_DB="true"
+
 # Check python formatting
 source ./script/format check
 

script/fix_permissions

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# script/fix_permissions: Updates the app directory with the correct user 
+#                         permissions (skipping node_modules since it is not
+#                         required and very large)
+
+source "$(dirname "${0}")"/../script/include/global_header.inc.sh
+
+APP_USER="${1}"
+APP_GROUP="${2}"
+
+if [ "${APP_USER}x" = "x" ] || [ "${APP_GROUP}x" = "x" ]; then
+    echo "ERROR: Missing username or groupname argument!"
+    echo "Received: *${APP_USER}:${APP_GROUP}*"
+    echo
+    exit 1
+fi
+
+chown "${APP_USER}:${APP_GROUP}" .
+chown "${APP_USER}:${APP_GROUP}" ./*
+for subdir in $(find . -type d -maxdepth 1 | grep -Ee '.[^/]' | grep -Fve 'node_modules')
+do
+    chown "${APP_USER}:${APP_GROUP}" -R "${subdir}"
+done

script/generate_build_info.sh

@@ -111,7 +111,7 @@ cat > ${STATIC_DIR}/buildinfo.html <<ENDHTML
 <BODY>
 <TABLE border="1">
 <TR>
-        <TH colspan="2">BuildInfo (${BUILT_BY}</TH>
+        <TH colspan="2">BuildInfo (${BUILT_BY})</TH>
 </TR>
 <TR>
         <TD class="label">Container Image Creation Time:</TD>