Merge pull request #1141 from dod-ccpo/consolidate-ci

Consolidate CI
This commit is contained in:
dandds 2019-10-30 16:06:46 -04:00 committed by GitHub
commit b5a2e6e929
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 167 additions and 179 deletions

View File

@ -1,81 +1,96 @@
version: 2.1 version: 2.1
orbs:
azure-acr: circleci/azure-acr@0.1.2
azure-aks: circleci/azure-aks@0.2.0
kubernetes: circleci/kubernetes@0.3.0
defaults:
appEnvironment: &appEnvironment
KEEP_EXISTING_VENV: true
PGHOST: localhost
PGUSER: postgres
PGDATABASE: circle_test
REDIS_URI: redis://localhost:6379
PIP_VERSION: 18.*
commands: commands:
migration_setup: cache_docker_image:
parameters:
container_image:
type: string
steps: steps:
- attach_workspace:
at: .
- run: - run:
name: Setup Environment Variables name: Save the docker images to a cache
command: | command: |
echo 'export CONTAINER_IMAGE="<< parameters.container_image >>"' >> $BASH_ENV mkdir -p docker-cache
- run: sudo apt-get update docker save -o docker-cache/atat.tar atat:latest
- run: sudo apt-get install gettext docker save -o docker-cache/builder.tar atat:builder
- kubernetes/install - save_cache:
migration_apply: key: docker-cache-{{ .Branch }}-{{ .Revision }}
paths:
- docker-cache
restore_docker_image:
steps:
- restore_cache:
keys:
- docker-cache-{{ .Branch }}-{{ .Revision }}
- run:
name: Restore Docker image from cache
command: |
docker load < docker-cache/atat.tar
docker load < docker-cache/builder.tar
setup_datastores:
parameters:
pgdatabase:
type: string
default: atat_test
container_env:
type: string
default: -e PGHOST=postgres -e REDIS_URI=redis://redis:6379
steps: steps:
- run: - run:
command: ./script/cluster_migration name: Set up temporary docker network
name: Apply Migrations and Seed Roles command: docker network create atat
- run:
name: Start redis
command: docker run -d --network atat --link redis:redis -p 6379:6379 --name redis circleci/redis:4-alpine3.8
- run:
name: Start postgres
command: docker run -d --network atat --link postgres:postgres -p 5432:5432 --name postgres circleci/postgres:10-alpine-ram
- run:
name: Wait for containers
command: sleep 3
- run:
name: Create database
command: "docker exec postgres createdb -U postgres << parameters.pgdatabase >>"
- run:
name: Apply migrations
command: docker run --network atat -e PGDATABASE=<< parameters.pgdatabase >> << parameters.container_env >> atat:builder .venv/bin/python .venv/bin/alembic upgrade head
- run:
name: Apply the default permission sets
command: docker run --network atat -e PGDATABASE=<< parameters.pgdatabase >> << parameters.container_env >> atat:builder .venv/bin/python script/seed_roles.py
jobs: jobs:
docker-build:
docker:
- image: docker:18.06.0-ce-git
steps:
- checkout
- setup_remote_docker:
docker_layer_caching: true
version: 18.06.0-ce
- run:
name: Build image
command: |
docker build . --target builder --build-arg CSP=azure -f ./Dockerfile -t atat:builder
docker build . --build-arg CSP=azure -f ./Dockerfile -t atat:latest
- cache_docker_image
test: test:
docker: docker:
- image: circleci/python:3.7.3-stretch-node - image: docker:18.06.0-ce-git
environment: *appEnvironment
- image: circleci/postgres:10-alpine-ram - image: circleci/postgres:10-alpine-ram
- image: circleci/redis:4-alpine3.8 - image: circleci/redis:4-alpine3.8
steps: steps:
- checkout - setup_remote_docker:
- run: sudo apt-get update docker_layer_caching: true
- run: sudo apt-get install postgresql-client version: 18.06.0-ce
- attach_workspace: - restore_docker_image
at: . - setup_datastores:
- run: ./script/setup pgdatabase: atat_test
- save_cache:
name: "Save Cache: Pipenv References"
paths:
- ~/.local/share
key: pipenv-v1-{{ .Branch }}-{{ checksum "Pipfile.lock" }}
- save_cache:
name: "Save Cache: Python Venv"
paths:
- ./.venv
key: venv-v1-{{ .Branch }}-{{ checksum "Pipfile.lock" }}
- save_cache:
name: "Save Cache: Yarn"
paths:
- ~/.cache/yarn
key: yarn-v1-{{ .Branch }}-{{ checksum "yarn.lock" }}
- save_cache:
name: "Save Cache: Node Modules"
paths:
- ./node_modules
key: node-v1-{{ .Branch }}-{{ checksum "yarn.lock" }}
- run: - run:
name: "Run Tests" name: Run CI tests
command: ./script/cibuild command: |
- persist_to_workspace: docker run \
root: . -e PGHOST=postgres \
paths: -e REDIS_URI=redis://redis:6379 \
- . --network atat \
atat:builder \
/bin/sh -c "pipenv install --dev && /bin/sh script/cibuild"
integration-tests: integration-tests:
docker: docker:
@ -83,25 +98,12 @@ jobs:
- image: circleci/postgres:10-alpine-ram - image: circleci/postgres:10-alpine-ram
- image: circleci/redis:4-alpine3.8 - image: circleci/redis:4-alpine3.8
steps: steps:
- attach_workspace:
at: .
- setup_remote_docker: - setup_remote_docker:
docker_layer_caching: true
version: 18.06.0-ce version: 18.06.0-ce
- run: - restore_docker_image
name: Remove existing font symlink - setup_datastores:
command: rm static/fonts pgdatabase: atat
- run:
name: Set up temporary docker network
command: docker network create atat
- run:
name: Build image
command: docker build . -t atat:latest
- run:
name: Start redis
command: docker run -d --network atat --link redis:redis -p 6379:6379 --name redis circleci/redis:4-alpine3.8
- run:
name: Start postgres
command: docker run -d --network atat --link postgres:postgres -p 5432:5432 --name postgres circleci/postgres:10-alpine-ram
- run: - run:
name: Start application container name: Start application container
command: | command: |
@ -112,25 +114,17 @@ jobs:
-p 8000:8000 \ -p 8000:8000 \
--network atat \ --network atat \
--name test-atat \ --name test-atat \
atat:latest \ atat:builder \
/bin/sh -c "
echo CLOUD_PROVIDER=mock > .env &&\
yarn build &&\
uwsgi \ uwsgi \
--callable app \ --callable app \
--module app \ --module app \
--plugin python3 \ --plugin python3 \
--virtualenv /opt/atat/atst/.venv \ --virtualenv /install/.venv \
--http-socket :8000 --http-socket :8000
- run: "
name: Wait for containers
command: sleep 3
- run:
name: Create database
command: docker exec postgres createdb -U postgres atat
- run:
name: Apply migrations
command: docker exec test-atat .venv/bin/python .venv/bin/alembic upgrade head
- run:
name: Apply the default permission sets
command: docker exec test-atat .venv/bin/python script/seed_roles.py
- run: - run:
name: Execute Ghost Inspector test suite name: Execute Ghost Inspector test suite
command: | command: |
@ -144,94 +138,91 @@ jobs:
--network atat \ --network atat \
ghostinspector/test-runner-standalone:latest ghostinspector/test-runner-standalone:latest
azure-migration: deploy:
executor: azure-aks/default docker:
- image: docker:18.06.0-ce-git
environment:
AZURE_REGISTRY: pwatat
RESOURCE_GROUP: atat
CLUSTER_NAME: atat-cluster
steps: steps:
- migration_setup: - checkout
container_image: "$AZURE_SERVER_NAME/atat:atat-$CIRCLE_SHA1" - setup_remote_docker:
- azure-aks/update-kubeconfig-with-credentials: docker_layer_caching: true
cluster-name: atat-cluster version: 18.06.0-ce
install-kubectl: true - restore_docker_image
perform-login: true - run:
resource-group: atat name: Install Azure CLI
- migration_apply command: |
apk update
# the azure-acr orb doesn't allow for multiple tags in the apk add bash py-pip
# build-and-push-image step, so instead we wrap our own job around it and run apk add --virtual=build \
# some additional Docker commands linux-headers gcc libffi-dev musl-dev openssl-dev python-dev make
azure-build-and-push-image: pip --no-cache-dir install -U pip
executor: azure-acr/default pip --no-cache-dir install azure-cli
steps: apk del --purge build
- azure-acr/build-and-push-image: - run:
extra-build-args: "--build-arg CSP=azure" name: Login to Azure CLI
login-server-name: "${AZURE_SERVER_NAME}" command: |
registry-name: pwatat az login \
repo: atat --service-principal \
tag: "atat-${CIRCLE_SHA1}" --tenant $AZURE_SP_TENANT \
- run: "docker tag ${AZURE_SERVER_NAME}/atat:atat-${CIRCLE_SHA1} ${AZURE_SERVER_NAME}/atat:latest" --password $AZURE_SP_PASSWORD \
- run: "docker push ${AZURE_SERVER_NAME}/atat:latest" --username $AZURE_SP
echo "Successfully logged in to Azure CLI."
az acr login --name $AZURE_REGISTRY
- run:
name: Install kubectl
command: |
apk add curl
export KUBECTL_VERSION=$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)
curl -LO https://storage.googleapis.com/kubernetes-release/release/$KUBECTL_VERSION/bin/linux/amd64/kubectl
chmod +x ./kubectl
mv ./kubectl /usr/local/bin
- run:
name: Configure kubectl
command: |
apk add libssl1.0
az aks get-credentials --name ${CLUSTER_NAME} --resource-group ${RESOURCE_GROUP}
- run:
name: Tag images
command: |
docker tag atat:latest ${AZURE_SERVER_NAME}/atat:atat-${CIRCLE_SHA1}
docker tag atat:latest ${AZURE_SERVER_NAME}/atat:latest
- run:
name: Push image
command: |
docker push ${AZURE_SERVER_NAME}/atat:atat-${CIRCLE_SHA1}
docker push ${AZURE_SERVER_NAME}/atat:latest
- run:
name: Add gettext package
command: apk add gettext
- run:
command: CONTAINER_IMAGE=${AZURE_SERVER_NAME}/atat:atat-${CIRCLE_SHA1} /bin/sh ./script/cluster_migration
name: Apply Migrations and Seed Roles
- run:
name: Update Kubernetes cluster
command: |
kubectl set image deployment.apps/atst atst=${AZURE_SERVER_NAME}/atat:atat-${CIRCLE_SHA1} --namespace=atat
kubectl set image deployment.apps/atst-worker atst-worker=${AZURE_SERVER_NAME}/atat:atat-${CIRCLE_SHA1} --namespace=atat
kubectl set image deployment.apps/atst-beat atst-beat=${AZURE_SERVER_NAME}/atat:atat-${CIRCLE_SHA1} --namespace=atat
workflows: workflows:
version: 2 version: 2
run-tests: run-tests:
jobs: jobs:
- test - docker-build
- test:
requires:
- docker-build
- integration-tests: - integration-tests:
requires: requires:
- test - docker-build
- azure-build-and-push-image: - deploy:
requires: requires:
- test
- integration-tests - integration-tests
filters: filters:
branches: branches:
only: only:
- master - master
- azure-migration:
requires:
- azure-build-and-push-image
filters:
branches:
only:
- master
- azure-aks/update-container-image:
cluster-name: atat-cluster
container-image-updates: "atst=${AZURE_SERVER_NAME}/atat:atat-${CIRCLE_SHA1}"
namespace: atat
resource-name: deployment.apps/atst
resource-group: atat
# uncomment below for debugging
# show-kubectl-command: true
requires:
- azure-migration
filters:
branches:
only:
- master
- azure-aks/update-container-image:
cluster-name: atat-cluster
container-image-updates: "atst-worker=${AZURE_SERVER_NAME}/atat:atat-${CIRCLE_SHA1}"
namespace: atat
resource-name: deployment.apps/atst-worker
resource-group: atat
# uncomment below for debugging
# show-kubectl-command: true
requires:
- azure-migration
filters:
branches:
only:
- master
- azure-aks/update-container-image:
cluster-name: atat-cluster
container-image-updates: "atst-beat=${AZURE_SERVER_NAME}/atat:atat-${CIRCLE_SHA1}"
namespace: atat
resource-name: deployment.apps/atst-beat
resource-group: atat
# uncomment below for debugging
# show-kubectl-command: true
requires:
- azure-migration
filters:
branches:
only:
- master

View File

@ -1,6 +1,7 @@
FROM python:3.7.3-alpine3.9 AS builder FROM python:3.7.3-alpine3.9 AS builder
ARG CSP ARG CSP
ENV TZ UTC
RUN mkdir -p /install/.venv RUN mkdir -p /install/.venv
WORKDIR /install WORKDIR /install
@ -51,7 +52,6 @@ ARG APP_DIR=/opt/atat/atst
# Environment variables # Environment variables
ENV APP_DIR "${APP_DIR}" ENV APP_DIR "${APP_DIR}"
ENV TZ UTC
# Create application directory # Create application directory
RUN set -x ; \ RUN set -x ; \

View File

@ -16,9 +16,6 @@ RUN_PYTHON_TESTS="true"
# Enable Javascript testing # Enable Javascript testing
RUN_JS_TESTS="true" RUN_JS_TESTS="true"
# Reset the DB, since the one script/setup created might not be persisted
RESET_DB="true"
# Check python formatting # Check python formatting
source ./script/format check source ./script/format check