Merge pull request #276 from dod-ccpo/circleci-cd

CircleCI CD
2018-10-16 11:37:09 -04:00 · 2018-10-16 11:37:09 -04:00 · 38610d0e0a
commit 38610d0e0a
parent dee0479403 714569eda4
8 changed files with 349 additions and 43 deletions
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@ -1,25 +1,44 @@
 version: 2.0
+
+defaults:
+  working_directory: &workingDirectory /opt/atat/atst
+  sourceImage: &sourceImage registry.atat.codes:443/atat-app-builder:circleci-cd
+  sourceAuth: &sourceAuth
+    username: $REGISTRY_USERNAME
+    password: $REGISTRY_PASSWORD
+  appEnvironment: &appEnvironment
+    KEEP_EXISTING_VENV: true
+    PGHOST: localhost
+    PGUSER: root
+    PGDATABASE: circle_test
+    REDIS_URI: redis://localhost:6379
+  dockerCmdEnvironment: &dockerCmdEnvironment
+    APP_USER: atst
+    APP_GROUP: atat
+    APP_DIR: /opt/atat/atst
+    ATAT_DOCKER_REGISTRY_URL: registry.atat.codes:443
+    CONTAINER_NAME: atst-container
+    USR_BIN_DIR: /usr/bin
+    PYTHON_SITE_PACKAGES_DIR: /usr/lib/python3.6/site-packages
+    PROD_IMAGE_NAME: atst-prod
+
 jobs:
-  build:
+  app_setup:
    docker:
-      - image: registry.atat.codes:443/atat-app-builder:circleci
-        auth:
-          username: $REGISTRY_USERNAME
-          password: $REGISTRY_PASSWORD
-        environment:
-          KEEP_EXISTING_VENV: true
-          PGHOST: localhost
-          PGUSER: root
-          PGDATABASE: circle_test
-          REDIS_URI: redis://localhost:6379
+      - image: *sourceImage
+        auth: *sourceAuth
+        environment: *appEnvironment
      - image: circleci/postgres:9.6.5-alpine-ram
      - image: circleci/redis:4-alpine3.8
+    working_directory: *workingDirectory
    steps:
      - checkout
      - run:
          name: "Clone Submodules"
          command: |
            git submodule update --init --recursive
+      - attach_workspace:
+          at: .
      - restore_cache:
          name: "Load Cache: Pipenv References"
          keys:
@ -38,6 +57,10 @@ jobs:
            - yarn-v1-{{ .Branch }}-{{ checksum "yarn.lock" }}
            - yarn-v1-{{ .Branch }}-
            - yarn-v1-
+      - restore_cache:
+          name: "Load Cache: Node Modules"
+          keys:
+            - node-v1-{{ .Branch }}-{{ checksum "yarn.lock" }}
      - run: ./script/setup
      - save_cache:
          name: "Save Cache: Pipenv Refrences"
@ -54,12 +77,141 @@ jobs:
          paths:
            - ~/.cache/yarn
          key: yarn-v1-{{ .Branch }}-{{ checksum "yarn.lock" }}
-      - restore_cache:
-          keys:
-            - disa-crls
-      - run: ./script/sync-crls
      - save_cache:
+          name: "Save Cache: Node Modules"
+          paths:
+            - ./node_modules
+          key: node-v1-{{ .Branch }}-{{ checksum "yarn.lock" }}
+      - restore_cache:
+          name: "Load Cache: CRLs"
+          keys:
+            - disa-crls-v2
+      - run: 
+          name: "Update CRLs"
+          command: ./script/sync-crls
+      - save_cache:
+          name: "Save Cache: CRLs"
          paths:
            - ./crl
-          key: disa-crls
-      - run: ./script/cibuild
+          key: disa-crls-v2-{{ .Branch }}-{{ epoch}}
+      - run:
+          name: "Generate build info"
+          command: ./script/generate_build_info.sh
+      - persist_to_workspace:
+          root: .
+          paths:
+            - .
+
+  test:
+    docker:
+      - image: *sourceImage
+        auth: *sourceAuth
+        environment: *appEnvironment
+      - image: circleci/postgres:9.6.5-alpine-ram
+      - image: circleci/redis:4-alpine3.8
+    working_directory: *workingDirectory
+    steps:
+      - attach_workspace:
+          at: .
+      - run:
+          name: "Run Tests"
+          command: ./script/cibuild
+
+  build_and_push_image:
+    docker:
+      - image: *sourceImage
+        auth: *sourceAuth
+        environment: *dockerCmdEnvironment
+    working_directory: *workingDirectory
+    steps:
+      - attach_workspace:
+          at: .
+      - setup_remote_docker:
+          version: 18.05.0-ce
+      - run:
+          name: "Export GIT_SHA"
+          command: echo "export GIT_SHA=$(git rev-parse --short HEAD)" >> $BASH_ENV
+      - run:
+          name: "Generate the Target Image Name"
+          command: echo "export IMAGE_NAME=\"${ATAT_DOCKER_REGISTRY_URL}/${PROD_IMAGE_NAME}:${GIT_SHA}-circleci\"" >> $BASH_ENV
+      - run:
+          name: "Start a Fresh Container"
+          command: docker run -d --entrypoint='/bin/sh' -ti --name ${CONTAINER_NAME} alpine:3.8
+      - run:
+          name: "Create the App Directory"
+          command: docker exec -t ${CONTAINER_NAME} mkdir -p ${APP_DIR}
+      - run:
+          name: "Copy Workspace Contents into the Container"
+          command: docker cp . ${CONTAINER_NAME}:${APP_DIR}
+      - run:
+          name: "Run Alpine Setup"
+          command: docker exec -t --workdir ${APP_DIR} ${CONTAINER_NAME} ./script/alpine_setup
+      - run:
+          name: "Copy System Site Packages into the Container"
+          command: docker cp -a ${PYTHON_SITE_PACKAGES_DIR}/. ${CONTAINER_NAME}:${PYTHON_SITE_PACKAGES_DIR}
+      - run:
+          name: "Copy USR_BIN Contents into the Container"
+          command: docker cp -a ${USR_BIN_DIR}/. ${CONTAINER_NAME}:${USR_BIN_DIR}
+      - run:
+          name: "Run Fix Permissions"
+          command: docker exec -t --workdir ${APP_DIR} ${CONTAINER_NAME} ./script/fix_permissions ${APP_USER} ${APP_GROUP}
+      - run:
+          name: "Commit Container Changes to New Image"
+          command: |
+            docker commit \
+              --change="ENV APP_USER \"${APP_USER}\"" \
+              --change="ENV APP_GROUP \"${APP_GROUP}\"" \
+              --change="ENV APP_DIR \"${APP_DIR}\"" \
+              --change='ENTRYPOINT ["/usr/bin/dumb-init", "--"]' \
+              --change="CMD [\"bash\", \"-c\", \"${APP_DIR}/script/uwsgi_server\"]" \
+              --change="WORKDIR ${APP_DIR}" \
+              --change="USER \"${APP_USER}\"" \
+              ${CONTAINER_NAME} \
+              ${IMAGE_NAME}
+      - run:
+          name: "Publish ATST Image"
+          command: |
+            docker image ls
+            docker login -u ${REGISTRY_USERNAME} -p ${REGISTRY_PASSWORD} ${ATAT_DOCKER_REGISTRY_URL}
+            docker push ${IMAGE_NAME}
+            docker logout
+
+  deploy:
+    docker:
+      - image: *sourceImage
+        auth: *sourceAuth
+        environment: *dockerCmdEnvironment
+    working_directory: *workingDirectory
+    steps:
+      - attach_workspace:
+          at: .
+      - run:
+          name: "Export GIT_SHA"
+          command: echo "export GIT_SHA=$(git rev-parse --short HEAD)" >> $BASH_ENV
+      - run:
+          name: "Generate the Target Image Name"
+          command: echo "export IMAGE_NAME=\"${ATAT_DOCKER_REGISTRY_URL}/${PROD_IMAGE_NAME}:${GIT_SHA}-circleci\"" >> $BASH_ENV
+      - run:
+          name: "Update Kubernetes Deployment"
+          command: ./deploy/kubernetes/atst-update-deploy.sh
+
+workflows:
+  version: 2
+  run-tests:
+    jobs:
+      - app_setup
+      - test:
+          requires:
+            - app_setup
+      - build_and_push_image:
+          requires:
+            - test
+          filters:
+            branches:
+              only: master
+      - deploy:
+          requires:
+            - build_and_push_image
+          filters:
+            branches:
+              only: master
--- a/.travis.yml
+++ b/.travis.yml
@ -45,16 +45,9 @@ script:
  - docker container stop current-atst-tester
  - docker run --add-host "postgreshost:${postgres_ip}" --add-host "redishost:${redis_ip}" "${TESTER_IMAGE2_NAME}"

-before_deploy:
-  - docker build --tag "${PROD_IMAGE_NAME}" . -f deploy/docker/prod/Dockerfile
-  - remote_image_name="${ATAT_DOCKER_REGISTRY_URL}/${PROD_IMAGE_NAME}:${GIT_SHA}"
-  - docker tag "${PROD_IMAGE_NAME}" "${remote_image_name}"
-  - docker images
-  - docker push "${remote_image_name}"
-
 deploy:
  provider: script
-  script: deploy/kubernetes/atst-update-deploy.sh
+  script: echo "Deployment now handles by CircleCI"
  on:
    branch: master

--- a/deploy/kubernetes/atst-check-deploy.sh
+++ b/deploy/kubernetes/atst-check-deploy.sh
@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+#
+
+set -o pipefail
+set -o errexit
+set -o nounset
+
+# Decode and save the K8S CA cert
+echo "${K8S_CA_CRT}" | base64 -d - > "${HOME}/k8s_ca.crt"
+
+# Setup the local kubectl client
+kubectl config set-context travis \
+    --cluster=atat-cluster \
+    --user=atat-deployer \
+    --namespace=atat
+
+kubectl config set-cluster atat-cluster \
+    --embed-certs=true \
+    --server="${K8S_ENDPOINT}"  \
+    --certificate-authority="${HOME}/k8s_ca.crt"
+
+kubectl config set-credentials atat-deployer --token="$(echo ${K8S_USER_TOKEN} | base64 -d -)"
+
+kubectl config use-context travis
+kubectl config current-context
+
+echo
+echo "Current ATST Deployment Details:"
+kubectl -n atat get deployment.apps/atst -o yaml
+
+# Remove the K8S CA file when the script exits
+function cleanup {
+    printf "Cleaning up...\n"
+    rm -vf "${HOME}/k8s_ca.crt"
+    printf "Cleaning done."
+}
+
+trap cleanup EXIT
--- a/deploy/kubernetes/atst-debugger.yml
+++ b/deploy/kubernetes/atst-debugger.yml
@ -1,14 +1,16 @@
 apiVersion: v1
 kind: Pod
 metadata:
-  name: atst-debugger
+  labels:
+    app: atst-debugger
+  name: atst-debugger-v1
  namespace: atat
 spec:
  securityContext:
    fsGroup: 101
  containers:
-    - name: atst-debugger
-      image: registry.atat.codes:443/atst-prod:a1916b1
+    - name: atst
+      image: registry.atat.codes:443/atst-prod:6329f8e
      args: ["/bin/bash", "-c", "while true; do date; sleep 45; done"]
      envFrom:
      - configMapRef:
@ -17,19 +19,85 @@ spec:
        - name: atst-config
          mountPath: "/opt/atat/atst/atst-overrides.ini"
          subPath: atst-overrides.ini
+        - name: nginx-client-ca-bundle
+          mountPath: "/opt/atat/atst/ssl/server-certs/ca-chain.pem"
+          subPath: client-ca-bundle.pem
        - name: uwsgi-config
          mountPath: "/opt/atat/atst/uwsgi-config.ini"
          subPath: uwsgi-config.ini
        - name: uwsgi-socket-dir
          mountPath: "/var/run/uwsgi"
+    - name: atst-nginx
+      image: nginx:alpine
+      ports:
+        - containerPort: 8442
+          name: http
+        - containerPort: 8443
+          name: https
+      volumeMounts:
+        - name: nginx-auth-tls
+          mountPath: "/etc/ssl/private"
+        - name: nginx-client-ca-bundle
+          mountPath: "/etc/ssl/client-ca-bundle.pem"
+          subPath: client-ca-bundle.pem
+        - name: nginx-config
+          mountPath: "/etc/nginx/conf.d/atst.conf"
+          subPath: atst.conf
+        - name: nginx-dhparam
+          mountPath: "/etc/ssl/dhparam.pem"
+          subPath: dhparam.pem
+        - name: nginx-htpasswd
+          mountPath: "/etc/nginx/.htpasswd"
+          subPath: .htpasswd
+        - name: uwsgi-socket-dir
+          mountPath: "/var/run/uwsgi"
+  imagePullSecrets:
+    - name: regcred
  volumes:
    - name: atst-config
      secret:
        secretName: atst-config-ini
        items:
-        - key: atst-overrides.ini
+        - key: override.ini
          path: atst-overrides.ini
          mode: 0644
+    - name: nginx-auth-tls
+      secret:
+        secretName: auth-atst-ingress-tls
+        items:
+        - key: tls.crt
+          path: auth.atat.crt
+          mode: 0644
+        - key: tls.key
+          path: auth.atat.key
+          mode: 0640
+    - name: nginx-client-ca-bundle
+      secret:
+        secretName: nginx-client-ca-bundle
+        items:
+        - key: client-ca-bundle.pem
+          path: client-ca-bundle.pem
+          mode: 0666
+    - name: nginx-config
+      configMap:
+        name: atst-nginx
+        items:
+        - key: nginx-config
+          path: atst.conf
+    - name: nginx-dhparam
+      secret:
+        secretName: dhparam-4096
+        items:
+        - key: dhparam.pem
+          path: dhparam.pem
+          mode: 0640
+    - name: nginx-htpasswd
+      secret:
+        secretName: atst-nginx-htpasswd
+        items:
+        - key: htpasswd
+          path: .htpasswd
+          mode: 0640
    - name: uwsgi-config
      configMap:
        name: atst-config
@ -41,3 +109,18 @@ spec:
      emptyDir:
        medium: Memory
  restartPolicy: Never
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: atst-debugger
+  name: atst-debugger
+  namespace: atat
+spec:
+  ports:
+  - name: http
+    port: 80
+    targetPort: 8442
+  selector:
+    app: atst-debugger
--- a/deploy/kubernetes/atst-update-deploy.sh
+++ b/deploy/kubernetes/atst-update-deploy.sh
@ -8,8 +8,24 @@ set -o errexit
 set -o nounset
 # set -o xtrace

+# Config
+MAX_DEPLOY_WAIT='300'
+
+if [ "${IMAGE_NAME}x" = "x" ]
+then
+    IMAGE_NAME="${ATAT_DOCKER_REGISTRY_URL}/${PROD_IMAGE_NAME}:${GIT_SHA}"
+fi
+
+# Remove the K8S CA file when the script exits
+function cleanup {
+    printf "Cleaning up...\n"
+    rm -vf "${HOME}/k8s_ca.crt"
+    printf "Cleaning done."
+}
+trap cleanup EXIT
+
 # Decode and save the K8S CA cert
-echo "${K8S_CA_CRT}" | base64 --decode -i > "${HOME}/k8s_ca.crt"
+echo "${K8S_CA_CRT}" | base64 -d - > "${HOME}/k8s_ca.crt"

 # Setup the local kubectl client
 kubectl config set-context travis \
@ -22,22 +38,19 @@ kubectl config set-cluster atat-cluster \
    --server="${K8S_ENDPOINT}"  \
    --certificate-authority="${HOME}/k8s_ca.crt"

-kubectl config set-credentials atat-deployer --token=`echo ${K8S_USER_TOKEN} | base64 --decode`
+kubectl config set-credentials atat-deployer --token="$(echo ${K8S_USER_TOKEN} | base64 -d -)"

 kubectl config use-context travis
 kubectl config current-context

 # Update the ATST deployment
-kubectl -n atat set image deployment.apps/atst atst="${ATAT_DOCKER_REGISTRY_URL}/${PROD_IMAGE_NAME}:${GIT_SHA}"
+kubectl -n atat set image deployment.apps/atst atst="${IMAGE_NAME}"

 # Wait for deployment to finish
-kubectl -n atat rollout status deployment/atst
-
-# Remove the K8S CA file when the script exits
-function cleanup {
-    printf "Cleaning up...\n"
-    rm -vf "${HOME}/k8s_ca.crt"
-    printf "Cleaning done."
-}
-
-trap cleanup EXIT
+if ! timeout -t "${MAX_DEPLOY_WAIT}" -s INT kubectl -n atat rollout status deployment/atst
+then
+    # Deploy did not finish before max wait time; abort and rollback the deploy
+    kubectl -n atat rollout undo deployment/atst
+    # Exit with a non-zero return code
+    exit 2
+fi
--- a/script/cibuild
+++ b/script/cibuild
@ -13,6 +13,9 @@ PYTHON_FILES="./app.py ./atst/** ./config"
 # Enable Python testing
 RUN_PYTHON_TESTS="true"

+# Reset the DB, since the one script/setup created might not be persisted
+RESET_DB="true"
+
 # Check python formatting
 source ./script/format check

--- a/script/fix_permissions
+++ b/script/fix_permissions
@ -0,0 +1,24 @@
+#!/bin/bash
+
+# script/fix_permissions: Updates the app directory with the correct user 
+#                         permissions (skipping node_modules since it is not
+#                         required and very large)
+
+source "$(dirname "${0}")"/../script/include/global_header.inc.sh
+
+APP_USER="${1}"
+APP_GROUP="${2}"
+
+if [ "${APP_USER}x" = "x" ] || [ "${APP_GROUP}x" = "x" ]; then
+    echo "ERROR: Missing username or groupname argument!"
+    echo "Received: *${APP_USER}:${APP_GROUP}*"
+    echo
+    exit 1
+fi
+
+chown "${APP_USER}:${APP_GROUP}" .
+chown "${APP_USER}:${APP_GROUP}" ./*
+for subdir in $(find . -type d -maxdepth 1 | grep -Ee '.[^/]' | grep -Fve 'node_modules')
+do
+    chown "${APP_USER}:${APP_GROUP}" -R "${subdir}"
+done
--- a/script/generate_build_info.sh
+++ b/script/generate_build_info.sh
@ -111,7 +111,7 @@ cat > ${STATIC_DIR}/buildinfo.html <<ENDHTML
 <BODY>
 <TABLE border="1">
 <TR>
-        <TH colspan="2">BuildInfo (${BUILT_BY}</TH>
+        <TH colspan="2">BuildInfo (${BUILT_BY})</TH>
 </TR>
 <TR>
        <TD class="label">Container Image Creation Time:</TD>