From 6a776b4087d25ad316611eb8aca598a93aaa87d1 Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Sun, 12 Aug 2018 15:08:02 -0400 Subject: [PATCH 01/19] Add atat-deployer user for travis to use --- deploy/kubernetes/atst-travis-deployer.yml | 38 ++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 deploy/kubernetes/atst-travis-deployer.yml diff --git a/deploy/kubernetes/atst-travis-deployer.yml b/deploy/kubernetes/atst-travis-deployer.yml new file mode 100644 index 00000000..4b36ef10 --- /dev/null +++ b/deploy/kubernetes/atst-travis-deployer.yml @@ -0,0 +1,38 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: atat-deployer + namespace: atat +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + namespace: atat + name: travis-deploy-role +rules: +- apiGroups: ["extensions", "apps"] + resources: ["deployments"] + verbs: ["get", "create", "update", "patch"] +- apiGroups: [""] + resources: ["services"] + verbs: ["get", "create", "update", "patch"] +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "create", "update", "patch"] +- apiGroups: ["extensions", "apps"] + resources: ["ingresses"] + verbs: ["get", "create", "update", "patch"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: travis-role-binding + namespace: atat +subjects: +- kind: ServiceAccount + name: atat-deployer + namespace: atat +roleRef: + kind: Role + name: travis-deploy-role + apiGroup: rbac.authorization.k8s.io From 94d88d795e88607d8f3997f55a0086f0f684e05a Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Sun, 12 Aug 2018 15:31:00 -0400 Subject: [PATCH 02/19] Add script for updating source image of ATST deploy --- deploy/kubernetes/atst-update-deploy.sh | 40 +++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 deploy/kubernetes/atst-update-deploy.sh diff --git a/deploy/kubernetes/atst-update-deploy.sh b/deploy/kubernetes/atst-update-deploy.sh new file mode 100644 index 00000000..1ad9e9ee --- /dev/null +++ b/deploy/kubernetes/atst-update-deploy.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# +# deploy/kubernetes/atst-update-deploy.sh: Updates the existing ATST deployment +# with a new source image + +set -o pipefail +set -o errexit +set -o nounset +# set -o xtrace + +# Decode and save the K8S CA cert +echo "${K8S_CA_CRT}" | base64 --decode -i > "${HOME}/k8s_ca.crt" + +# Setup the local kubectl client +kubectl config set-cluster atat-cluster \ + --embed-certs=true \ + --server="${K8S_CLUSTER_ENDPOINT}" \ + --certificate-authority="${HOME}/k8s_ca.crt" + +kubectl config set-credentials atat-deployer --token="${K8S_USER_TOKEN}" + +kubectl config set-context travis \ + --cluster=atat-cluster \ + --user=atat-deployer \ + --namespace=atat + +kubectl config use-context travis +kubectl config current-context + +# Update the ATST deployment +kubectl set image deployment.apps/atst atst="${PROD_IMAGE_NAME}" + +# Remove the K8S CA file when the script exits +function cleanup { + printf "Cleaning up...\n" + rm -vf "${HOME}/k8s_ca.crt" + printf "Cleaning done." +} + +trap cleanup EXIT From 24625cf08fc399b59b5ac4faec3c466d867119d2 Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Sun, 12 Aug 2018 15:31:28 -0400 Subject: [PATCH 03/19] On successful master build, deploy new code version --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 5a0734c2..63000b89 100644 --- a/.travis.yml +++ b/.travis.yml @@ -47,6 +47,6 @@ before_deploy: deploy: provider: script - script: echo "** Image push only for now... stay tuned! **" + script: deploy/kubernetes/atst-update-deploy.sh on: branch: master From 531cef233da3522dcb36bd79e11a2e11d3729799 Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Tue, 14 Aug 2018 12:10:16 -0400 Subject: [PATCH 04/19] Add namespace to kubectl command --- deploy/kubernetes/atst-update-deploy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/kubernetes/atst-update-deploy.sh b/deploy/kubernetes/atst-update-deploy.sh index 1ad9e9ee..f7eba653 100644 --- a/deploy/kubernetes/atst-update-deploy.sh +++ b/deploy/kubernetes/atst-update-deploy.sh @@ -28,7 +28,7 @@ kubectl config use-context travis kubectl config current-context # Update the ATST deployment -kubectl set image deployment.apps/atst atst="${PROD_IMAGE_NAME}" +kubectl -n atat set image deployment.apps/atst atst="${PROD_IMAGE_NAME}" # Remove the K8S CA file when the script exits function cleanup { From 7337eee9e480353367da211c9c18ad0153126009 Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Tue, 14 Aug 2018 13:40:03 -0400 Subject: [PATCH 05/19] Add kubectl install --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 63000b89..c7141e38 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,6 @@ sudo: required language: minimal +dist: trusty services: - docker git: @@ -18,6 +19,8 @@ before_install: - sed -i 's/git@github.com:/https:\/\/github.com\//' .gitmodules # Manually initialize submodules - git submodule update --init --recursive +install: + - curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/v1.10.4/bin/linux/amd64/kubectl && chmod +x kubectl && sudo mv kubectl /usr/local/bin/ before_script: - rm -rf ./crl/* From 8b59e98a09962722d5434103ddf6fa6db49e84e5 Mon Sep 17 00:00:00 2001 From: Patrick Smith Date: Tue, 14 Aug 2018 14:16:59 -0400 Subject: [PATCH 06/19] Temporarily continuously deploy this branch --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c7141e38..a32ef87c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -52,4 +52,4 @@ deploy: provider: script script: deploy/kubernetes/atst-update-deploy.sh on: - branch: master + branch: continuous-deployment From 47baa270cef7cf110c1289d929801b1afeac4b64 Mon Sep 17 00:00:00 2001 From: Patrick Smith Date: Tue, 14 Aug 2018 14:35:09 -0400 Subject: [PATCH 07/19] Make deploy script executable --- deploy/kubernetes/atst-update-deploy.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 deploy/kubernetes/atst-update-deploy.sh diff --git a/deploy/kubernetes/atst-update-deploy.sh b/deploy/kubernetes/atst-update-deploy.sh old mode 100644 new mode 100755 From 717f67cee52ec7c7d5e6fc40c5f697485e28b345 Mon Sep 17 00:00:00 2001 From: Patrick Smith Date: Tue, 14 Aug 2018 15:16:35 -0400 Subject: [PATCH 08/19] Update kubernetes endpoint var name --- deploy/kubernetes/atst-update-deploy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/kubernetes/atst-update-deploy.sh b/deploy/kubernetes/atst-update-deploy.sh index f7eba653..caec55ee 100755 --- a/deploy/kubernetes/atst-update-deploy.sh +++ b/deploy/kubernetes/atst-update-deploy.sh @@ -14,7 +14,7 @@ echo "${K8S_CA_CRT}" | base64 --decode -i > "${HOME}/k8s_ca.crt" # Setup the local kubectl client kubectl config set-cluster atat-cluster \ --embed-certs=true \ - --server="${K8S_CLUSTER_ENDPOINT}" \ + --server="${K8S_ENDPOINT}" \ --certificate-authority="${HOME}/k8s_ca.crt" kubectl config set-credentials atat-deployer --token="${K8S_USER_TOKEN}" From 21b442ce3e2c1d0ec0afa13224dae682b3491266 Mon Sep 17 00:00:00 2001 From: Patrick Smith Date: Tue, 14 Aug 2018 15:49:42 -0400 Subject: [PATCH 09/19] Shuffle around kubectl setup commands --- deploy/kubernetes/atst-update-deploy.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/deploy/kubernetes/atst-update-deploy.sh b/deploy/kubernetes/atst-update-deploy.sh index caec55ee..719616ec 100755 --- a/deploy/kubernetes/atst-update-deploy.sh +++ b/deploy/kubernetes/atst-update-deploy.sh @@ -12,6 +12,11 @@ set -o nounset echo "${K8S_CA_CRT}" | base64 --decode -i > "${HOME}/k8s_ca.crt" # Setup the local kubectl client +kubectl config set-context travis \ + --cluster=atat-cluster \ + --user=atat-deployer \ + --namespace=atat + kubectl config set-cluster atat-cluster \ --embed-certs=true \ --server="${K8S_ENDPOINT}" \ @@ -19,11 +24,6 @@ kubectl config set-cluster atat-cluster \ kubectl config set-credentials atat-deployer --token="${K8S_USER_TOKEN}" -kubectl config set-context travis \ - --cluster=atat-cluster \ - --user=atat-deployer \ - --namespace=atat - kubectl config use-context travis kubectl config current-context From cba435c2332fc055bd7410c443c59c3c99a76ac7 Mon Sep 17 00:00:00 2001 From: Patrick Smith Date: Thu, 16 Aug 2018 10:22:07 -0400 Subject: [PATCH 10/19] Bump up memory requirement --- deploy/kubernetes/atst.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/kubernetes/atst.yml b/deploy/kubernetes/atst.yml index c62d7a6a..ab21c110 100644 --- a/deploy/kubernetes/atst.yml +++ b/deploy/kubernetes/atst.yml @@ -27,7 +27,7 @@ spec: image: registry.atat.codes:443/atst-prod:a1916b1 resources: requests: - memory: "2500Mi" + memory: "4500Mi" envFrom: - configMapRef: name: atst-envvars From 71a8c06c226c497e6cff7ae66225d2e2de506ba3 Mon Sep 17 00:00:00 2001 From: Patrick Smith Date: Thu, 16 Aug 2018 10:36:27 -0400 Subject: [PATCH 11/19] Even more memory please --- deploy/kubernetes/atst.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/kubernetes/atst.yml b/deploy/kubernetes/atst.yml index ab21c110..036cc10d 100644 --- a/deploy/kubernetes/atst.yml +++ b/deploy/kubernetes/atst.yml @@ -27,7 +27,7 @@ spec: image: registry.atat.codes:443/atst-prod:a1916b1 resources: requests: - memory: "4500Mi" + memory: "6000Mi" envFrom: - configMapRef: name: atst-envvars From 9deb602d22136bd644dde5ce01a67af5cd37bc7f Mon Sep 17 00:00:00 2001 From: Patrick Smith Date: Thu, 16 Aug 2018 11:59:29 -0400 Subject: [PATCH 12/19] Update image sha in deploy yml --- deploy/kubernetes/atst.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/kubernetes/atst.yml b/deploy/kubernetes/atst.yml index 036cc10d..d06b7a1f 100644 --- a/deploy/kubernetes/atst.yml +++ b/deploy/kubernetes/atst.yml @@ -24,7 +24,7 @@ spec: fsGroup: 101 containers: - name: atst - image: registry.atat.codes:443/atst-prod:a1916b1 + image: registry.atat.codes:443/atst-prod:23e5c04 resources: requests: memory: "6000Mi" From b82bbae0c5dbf6ac41a6f9719660e8abab41d78f Mon Sep 17 00:00:00 2001 From: Patrick Smith Date: Thu, 16 Aug 2018 13:40:54 -0400 Subject: [PATCH 13/19] base64 decode user token --- deploy/kubernetes/atst-update-deploy.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/kubernetes/atst-update-deploy.sh b/deploy/kubernetes/atst-update-deploy.sh index 719616ec..464669fd 100755 --- a/deploy/kubernetes/atst-update-deploy.sh +++ b/deploy/kubernetes/atst-update-deploy.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# deploy/kubernetes/atst-update-deploy.sh: Updates the existing ATST deployment +# deploy/kubernetes/atst-update-deploy.sh: Updates the existing ATST deployment # with a new source image set -o pipefail @@ -22,7 +22,7 @@ kubectl config set-cluster atat-cluster \ --server="${K8S_ENDPOINT}" \ --certificate-authority="${HOME}/k8s_ca.crt" -kubectl config set-credentials atat-deployer --token="${K8S_USER_TOKEN}" +kubectl config set-credentials atat-deployer --token=`echo ${K8S_USER_TOKEN} | base64 --decode` kubectl config use-context travis kubectl config current-context From 1acd5fb73a8a6998a492ecf732a41aad5fdf894d Mon Sep 17 00:00:00 2001 From: Patrick Smith Date: Thu, 16 Aug 2018 13:59:30 -0400 Subject: [PATCH 14/19] Use full remote image name when setting deployed image --- deploy/kubernetes/atst-update-deploy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/kubernetes/atst-update-deploy.sh b/deploy/kubernetes/atst-update-deploy.sh index 464669fd..96d3314c 100755 --- a/deploy/kubernetes/atst-update-deploy.sh +++ b/deploy/kubernetes/atst-update-deploy.sh @@ -28,7 +28,7 @@ kubectl config use-context travis kubectl config current-context # Update the ATST deployment -kubectl -n atat set image deployment.apps/atst atst="${PROD_IMAGE_NAME}" +kubectl -n atat set image deployment.apps/atst atst="${remote_image_name}" # Remove the K8S CA file when the script exits function cleanup { From 265b128255ad2d95a97e35086afdda037d1bbfee Mon Sep 17 00:00:00 2001 From: Patrick Smith Date: Thu, 16 Aug 2018 14:17:54 -0400 Subject: [PATCH 15/19] Build remote image name in update-deploy script --- .travis.yml | 4 ++-- deploy/kubernetes/atst-update-deploy.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index a32ef87c..b73cd0b4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,7 @@ env: - PROD_IMAGE_NAME=atst-prod - TESTER_IMAGE1_NAME=atst-tester-nocrls - TESTER_IMAGE2_NAME=atst-tester + - GIT_SHA="$(git rev-parse --short HEAD)" cache: directories: - crl @@ -42,8 +43,7 @@ script: before_deploy: - docker build --tag "${PROD_IMAGE_NAME}" . -f deploy/docker/prod/Dockerfile - - git_sha="$(git rev-parse --short HEAD)" - - remote_image_name="${ATAT_DOCKER_REGISTRY_URL}/${PROD_IMAGE_NAME}:${git_sha}" + - remote_image_name="${ATAT_DOCKER_REGISTRY_URL}/${PROD_IMAGE_NAME}:${GIT_SHA}" - docker tag "${PROD_IMAGE_NAME}" "${remote_image_name}" - docker images - docker push "${remote_image_name}" diff --git a/deploy/kubernetes/atst-update-deploy.sh b/deploy/kubernetes/atst-update-deploy.sh index 96d3314c..51219992 100755 --- a/deploy/kubernetes/atst-update-deploy.sh +++ b/deploy/kubernetes/atst-update-deploy.sh @@ -28,7 +28,7 @@ kubectl config use-context travis kubectl config current-context # Update the ATST deployment -kubectl -n atat set image deployment.apps/atst atst="${remote_image_name}" +kubectl -n atat set image deployment.apps/atst atst="${ATAT_DOCKER_REGISTRY_URL}/${PROD_IMAGE_NAME}:${GIT_SHA}" # Remove the K8S CA file when the script exits function cleanup { From 1cb68ecdaf0d8f6800e644e0f13408549e424a55 Mon Sep 17 00:00:00 2001 From: Patrick Smith Date: Thu, 16 Aug 2018 14:18:55 -0400 Subject: [PATCH 16/19] Wait for deployment to roll out before finishing --- deploy/kubernetes/atst-update-deploy.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/deploy/kubernetes/atst-update-deploy.sh b/deploy/kubernetes/atst-update-deploy.sh index 51219992..f83bbe01 100755 --- a/deploy/kubernetes/atst-update-deploy.sh +++ b/deploy/kubernetes/atst-update-deploy.sh @@ -30,6 +30,9 @@ kubectl config current-context # Update the ATST deployment kubectl -n atat set image deployment.apps/atst atst="${ATAT_DOCKER_REGISTRY_URL}/${PROD_IMAGE_NAME}:${GIT_SHA}" +# Wait for deployment to finish +kubectl -n atat rollout status deployment/atst + # Remove the K8S CA file when the script exits function cleanup { printf "Cleaning up...\n" From 2f3a0a4db1216a436daec62bd08a696529a7a793 Mon Sep 17 00:00:00 2001 From: Patrick Smith Date: Thu, 16 Aug 2018 16:08:47 -0400 Subject: [PATCH 17/19] Attempt to fix permissions on rollout status --- deploy/kubernetes/atst-travis-deployer.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/kubernetes/atst-travis-deployer.yml b/deploy/kubernetes/atst-travis-deployer.yml index 4b36ef10..4dd063b2 100644 --- a/deploy/kubernetes/atst-travis-deployer.yml +++ b/deploy/kubernetes/atst-travis-deployer.yml @@ -12,7 +12,7 @@ metadata: rules: - apiGroups: ["extensions", "apps"] resources: ["deployments"] - verbs: ["get", "create", "update", "patch"] + verbs: ["get", "watch", "create", "update", "patch"] - apiGroups: [""] resources: ["services"] verbs: ["get", "create", "update", "patch"] From f6c1b316397c3562c6dc334cf976e42ded16bd64 Mon Sep 17 00:00:00 2001 From: Patrick Smith Date: Thu, 16 Aug 2018 16:46:35 -0400 Subject: [PATCH 18/19] Continously deploy just the master branch --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b73cd0b4..83af6a1a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -52,4 +52,4 @@ deploy: provider: script script: deploy/kubernetes/atst-update-deploy.sh on: - branch: continuous-deployment + branch: master From 4578c2f1c80a0a283728a916c80283ddfcc1835b Mon Sep 17 00:00:00 2001 From: Patrick Smith Date: Fri, 17 Aug 2018 09:39:24 -0400 Subject: [PATCH 19/19] Don't remove cached CRLs everytime --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 83af6a1a..17db1075 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,7 +24,6 @@ install: - curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/v1.10.4/bin/linux/amd64/kubectl && chmod +x kubectl && sudo mv kubectl /usr/local/bin/ before_script: - - rm -rf ./crl/* - docker run -d --name postgres96 postgres:9.6-alpine - docker run -d --name redis redis:4.0.10-alpine - docker run --link postgres96:postgres96 --link redis:redis waisbrot/wait