From 52829a12365355663972ccb80bf31eb659df941c Mon Sep 17 00:00:00 2001 From: dandds Date: Mon, 15 Jul 2019 06:10:34 -0400 Subject: [PATCH 1/4] Add additional k8s config for AWS and Azure. - remove unused k8s environment config - experimental config for azure and aws - use nginx for proxy and add rq worker pod --- deploy/kubernetes/test/atat-deploy-role.yml | 26 ---- .../kubernetes/test/atst-nginx-configmap.yml | 99 -------------- .../kubernetes/uat/atst-nginx-configmap.yml | 99 -------------- .../uat => k8s/aws}/atst-configmap.yml | 2 +- .../aws}/atst-envvars-configmap.yml | 6 +- k8s/aws/atst-nginx-configmap.yml | 37 +++++ .../aws}/atst-worker-envvars-configmap.yml | 2 +- .../test/test.yml => k8s/aws/aws.yml | 126 +++--------------- .../test => k8s/azure}/atst-configmap.yml | 2 +- .../azure}/atst-envvars-configmap.yml | 7 +- k8s/azure/atst-nginx-configmap.yml | 37 +++++ .../azure}/atst-worker-envvars-configmap.yml | 2 +- .../uat/uat.yml => k8s/azure/azure.yml | 123 +++-------------- 13 files changed, 117 insertions(+), 451 deletions(-) delete mode 100644 deploy/kubernetes/test/atat-deploy-role.yml delete mode 100644 deploy/kubernetes/test/atst-nginx-configmap.yml delete mode 100644 deploy/kubernetes/uat/atst-nginx-configmap.yml rename {deploy/kubernetes/uat => k8s/aws}/atst-configmap.yml (98%) rename {deploy/kubernetes/uat => k8s/aws}/atst-envvars-configmap.yml (62%) create mode 100644 k8s/aws/atst-nginx-configmap.yml rename {deploy/kubernetes/uat => k8s/aws}/atst-worker-envvars-configmap.yml (87%) rename deploy/kubernetes/test/test.yml => k8s/aws/aws.yml (51%) rename {deploy/kubernetes/test => k8s/azure}/atst-configmap.yml (98%) rename {deploy/kubernetes/test => k8s/azure}/atst-envvars-configmap.yml (58%) create mode 100644 k8s/azure/atst-nginx-configmap.yml rename {deploy/kubernetes/test => k8s/azure}/atst-worker-envvars-configmap.yml (86%) rename deploy/kubernetes/uat/uat.yml => k8s/azure/azure.yml (51%) diff --git a/deploy/kubernetes/test/atat-deploy-role.yml b/deploy/kubernetes/test/atat-deploy-role.yml deleted file mode 100644 index 8a7b6f85..00000000 --- a/deploy/kubernetes/test/atat-deploy-role.yml +++ /dev/null @@ -1,26 +0,0 @@ -kind: Role -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - namespace: atat-test - name: atat-sample-update -rules: -- apiGroups: [""] - resources: ["pods"] - verbs: ["get", "list"] -- apiGroups: [""] - resources: ["pods/exec"] - verbs: ["create"] ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: atst-sample-role-binding - namespace: atat-test -subjects: -- kind: ServiceAccount - name: atat-deployer - namespace: atat -roleRef: - kind: Role - name: atat-sample-update - apiGroup: rbac.authorization.k8s.io diff --git a/deploy/kubernetes/test/atst-nginx-configmap.yml b/deploy/kubernetes/test/atst-nginx-configmap.yml deleted file mode 100644 index f0605c5b..00000000 --- a/deploy/kubernetes/test/atst-nginx-configmap.yml +++ /dev/null @@ -1,99 +0,0 @@ ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: atst-nginx - namespace: atat-test -data: - nginx-config: |- - server { - access_log /var/log/nginx/access.log json; - server_name test.atat.code.mil; - listen 8442; - listen [::]:8442 ipv6only=on; - if ($http_x_forwarded_proto != 'https') { - return 301 https://$host$request_uri; - } - location /login-redirect { - return 301 https://auth-test.atat.code.mil$request_uri; - } - location /login-dev { - try_files $uri @appbasicauth; - } - location / { - try_files $uri @app; - } - location @app { - include uwsgi_params; - uwsgi_pass unix:///var/run/uwsgi/uwsgi.socket; - uwsgi_param HTTP_X_REQUEST_ID $request_id; - } - location @appbasicauth { - include uwsgi_params; - uwsgi_pass unix:///var/run/uwsgi/uwsgi.socket; - auth_basic "Developer Access"; - auth_basic_user_file /etc/nginx/.htpasswd; - uwsgi_param HTTP_X_REQUEST_ID $request_id; - } - } - server { - access_log /var/log/nginx/access.log json; - server_name auth-test.atat.code.mil; - listen 8443 ssl; - listen [::]:8443 ssl ipv6only=on; - # SSL server certificate and private key - ssl_certificate /etc/ssl/private/auth.atat.crt; - ssl_certificate_key /etc/ssl/private/auth.atat.key; - # Set SSL protocols, ciphers, and related options - ssl_protocols TLSv1.3 TLSv1.2; - ssl_ciphers 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256'; - ssl_prefer_server_ciphers on; - ssl_ecdh_curve secp384r1; - ssl_dhparam /etc/ssl/dhparam.pem; - # SSL session options - ssl_session_timeout 4h; - ssl_session_cache shared:SSL:10m; # 1mb = ~4000 sessions - ssl_session_tickets off; - # OCSP Stapling - ssl_stapling on; - ssl_stapling_verify on; - resolver 8.8.8.8 8.8.4.4; - # Request and validate client certificate - ssl_verify_client on; - ssl_verify_depth 10; - ssl_client_certificate /etc/ssl/client-ca-bundle.pem; - # Guard against HTTPS -> HTTP downgrade - add_header Strict-Transport-Security "max-age=31536000; includeSubDomains; always"; - location / { - return 301 https://test.atat.code.mil$request_uri; - } - location /login-redirect { - try_files $uri @app; - } - location @app { - include uwsgi_params; - uwsgi_pass unix:///var/run/uwsgi/uwsgi.socket; - uwsgi_param HTTP_X_SSL_CLIENT_VERIFY $ssl_client_verify; - uwsgi_param HTTP_X_SSL_CLIENT_CERT $ssl_client_raw_cert; - uwsgi_param HTTP_X_SSL_CLIENT_S_DN $ssl_client_s_dn; - uwsgi_param HTTP_X_SSL_CLIENT_S_DN_LEGACY $ssl_client_s_dn_legacy; - uwsgi_param HTTP_X_SSL_CLIENT_I_DN $ssl_client_i_dn; - uwsgi_param HTTP_X_SSL_CLIENT_I_DN_LEGACY $ssl_client_i_dn_legacy; - uwsgi_param HTTP_X_REQUEST_ID $request_id; - } - } - nginx-json-log-config: |- - log_format json escape=json - '{' - '"timestamp":"$time_iso8601",' - '"msec":"$msec",' - '"request_id":"$request_id",' - '"remote_addr":"$remote_addr",' - '"remote_user":"$remote_user",' - '"request":"$request",' - '"status":$status,' - '"body_bytes_sent":$body_bytes_sent,' - '"referer":"$http_referer",' - '"user_agent":"$http_user_agent",' - '"http_x_forwarded_for":"$http_x_forwarded_for"' - '}'; diff --git a/deploy/kubernetes/uat/atst-nginx-configmap.yml b/deploy/kubernetes/uat/atst-nginx-configmap.yml deleted file mode 100644 index e8e932b1..00000000 --- a/deploy/kubernetes/uat/atst-nginx-configmap.yml +++ /dev/null @@ -1,99 +0,0 @@ ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: atst-nginx - namespace: atat-uat -data: - nginx-config: |- - server { - access_log /var/log/nginx/access.log json; - server_name uat.atat.code.mil; - listen 8442; - listen [::]:8442 ipv6only=on; - if ($http_x_forwarded_proto != 'https') { - return 301 https://$host$request_uri; - } - location /login-redirect { - return 301 https://auth-uat.atat.code.mil$request_uri; - } - location /login-dev { - try_files $uri @appbasicauth; - } - location / { - try_files $uri @app; - } - location @app { - include uwsgi_params; - uwsgi_pass unix:///var/run/uwsgi/uwsgi.socket; - uwsgi_param HTTP_X_REQUEST_ID $request_id; - } - location @appbasicauth { - include uwsgi_params; - uwsgi_pass unix:///var/run/uwsgi/uwsgi.socket; - auth_basic "Developer Access"; - auth_basic_user_file /etc/nginx/.htpasswd; - uwsgi_param HTTP_X_REQUEST_ID $request_id; - } - } - server { - access_log /var/log/nginx/access.log json; - server_name auth-uat.atat.code.mil; - listen 8443 ssl; - listen [::]:8443 ssl ipv6only=on; - # SSL server certificate and private key - ssl_certificate /etc/ssl/private/auth.atat.crt; - ssl_certificate_key /etc/ssl/private/auth.atat.key; - # Set SSL protocols, ciphers, and related options - ssl_protocols TLSv1.3 TLSv1.2; - ssl_ciphers 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256'; - ssl_prefer_server_ciphers on; - ssl_ecdh_curve secp384r1; - ssl_dhparam /etc/ssl/dhparam.pem; - # SSL session options - ssl_session_timeout 4h; - ssl_session_cache shared:SSL:10m; # 1mb = ~4000 sessions - ssl_session_tickets off; - # OCSP Stapling - ssl_stapling on; - ssl_stapling_verify on; - resolver 8.8.8.8 8.8.4.4; - # Request and validate client certificate - ssl_verify_client on; - ssl_verify_depth 10; - ssl_client_certificate /etc/ssl/client-ca-bundle.pem; - # Guard against HTTPS -> HTTP downgrade - add_header Strict-Transport-Security "max-age=31536000; includeSubDomains; always"; - location / { - return 301 https://uat.atat.code.mil$request_uri; - } - location /login-redirect { - try_files $uri @app; - } - location @app { - include uwsgi_params; - uwsgi_pass unix:///var/run/uwsgi/uwsgi.socket; - uwsgi_param HTTP_X_SSL_CLIENT_VERIFY $ssl_client_verify; - uwsgi_param HTTP_X_SSL_CLIENT_CERT $ssl_client_raw_cert; - uwsgi_param HTTP_X_SSL_CLIENT_S_DN $ssl_client_s_dn; - uwsgi_param HTTP_X_SSL_CLIENT_S_DN_LEGACY $ssl_client_s_dn_legacy; - uwsgi_param HTTP_X_SSL_CLIENT_I_DN $ssl_client_i_dn; - uwsgi_param HTTP_X_SSL_CLIENT_I_DN_LEGACY $ssl_client_i_dn_legacy; - uwsgi_param HTTP_X_REQUEST_ID $request_id; - } - } - nginx-json-log-config: |- - log_format json escape=json - '{' - '"timestamp":"$time_iso8601",' - '"msec":"$msec",' - '"request_id":"$request_id",' - '"remote_addr":"$remote_addr",' - '"remote_user":"$remote_user",' - '"request":"$request",' - '"status":$status,' - '"body_bytes_sent":$body_bytes_sent,' - '"referer":"$http_referer",' - '"user_agent":"$http_user_agent",' - '"http_x_forwarded_for":"$http_x_forwarded_for"' - '}'; diff --git a/deploy/kubernetes/uat/atst-configmap.yml b/k8s/aws/atst-configmap.yml similarity index 98% rename from deploy/kubernetes/uat/atst-configmap.yml rename to k8s/aws/atst-configmap.yml index 0482ea10..79f9a61b 100644 --- a/deploy/kubernetes/uat/atst-configmap.yml +++ b/k8s/aws/atst-configmap.yml @@ -3,7 +3,7 @@ apiVersion: v1 kind: ConfigMap metadata: name: atst-config - namespace: atat-uat + namespace: atat data: uwsgi-config: |- [uwsgi] diff --git a/deploy/kubernetes/uat/atst-envvars-configmap.yml b/k8s/aws/atst-envvars-configmap.yml similarity index 62% rename from deploy/kubernetes/uat/atst-envvars-configmap.yml rename to k8s/aws/atst-envvars-configmap.yml index 7821b5c0..fc65df51 100644 --- a/deploy/kubernetes/uat/atst-envvars-configmap.yml +++ b/k8s/aws/atst-envvars-configmap.yml @@ -3,12 +3,10 @@ apiVersion: v1 kind: ConfigMap metadata: name: atst-envvars - namespace: atat-uat + namespace: atat data: TZ: UTC FLASK_ENV: dev OVERRIDE_CONFIG_FULLPATH: /opt/atat/atst/atst-overrides.ini - UWSGI_CONFIG_FULLPATH: /opt/atat/atst/uwsgi-config.ini - RQ_QUEUES: atat-uat + UWSGI_CONFIG_FULLPATH: /opt/atat/atst/uwsgi.ini CRL_STORAGE_PROVIDER: CLOUDFILES - LOG_JSON: "true" diff --git a/k8s/aws/atst-nginx-configmap.yml b/k8s/aws/atst-nginx-configmap.yml new file mode 100644 index 00000000..3c38614b --- /dev/null +++ b/k8s/aws/atst-nginx-configmap.yml @@ -0,0 +1,37 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: atst-nginx + namespace: atat +data: + nginx-config: |- + server { + listen 8442; + server_name localhost; + + location / { + try_files $uri @app; + } + + location @app { + uwsgi_pass unix:///var/run/uwsgi/uwsgi.socket; + uwsgi_param QUERY_STRING $query_string; + uwsgi_param REQUEST_METHOD $request_method; + uwsgi_param CONTENT_TYPE $content_type; + uwsgi_param CONTENT_LENGTH $content_length; + + uwsgi_param REQUEST_URI $request_uri; + uwsgi_param PATH_INFO $document_uri; + uwsgi_param DOCUMENT_ROOT $document_root; + uwsgi_param SERVER_PROTOCOL $server_protocol; + uwsgi_param REQUEST_SCHEME $scheme; + uwsgi_param HTTPS $https if_not_empty; + + uwsgi_param REMOTE_ADDR $remote_addr; + uwsgi_param REMOTE_PORT $remote_port; + uwsgi_param SERVER_PORT $server_port; + uwsgi_param SERVER_NAME $server_name; + uwsgi_param HTTP_X_REQUEST_ID $request_id; + } + } diff --git a/deploy/kubernetes/uat/atst-worker-envvars-configmap.yml b/k8s/aws/atst-worker-envvars-configmap.yml similarity index 87% rename from deploy/kubernetes/uat/atst-worker-envvars-configmap.yml rename to k8s/aws/atst-worker-envvars-configmap.yml index 266c3c68..c1f8edde 100644 --- a/deploy/kubernetes/uat/atst-worker-envvars-configmap.yml +++ b/k8s/aws/atst-worker-envvars-configmap.yml @@ -3,7 +3,7 @@ apiVersion: v1 kind: ConfigMap metadata: name: atst-worker-envvars - namespace: atat-uat + namespace: atat data: TZ: UTC DISABLE_CRL_CHECK: "True" diff --git a/deploy/kubernetes/test/test.yml b/k8s/aws/aws.yml similarity index 51% rename from deploy/kubernetes/test/test.yml rename to k8s/aws/aws.yml index 631c847d..336d9434 100644 --- a/deploy/kubernetes/test/test.yml +++ b/k8s/aws/aws.yml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Namespace metadata: - name: atat-test + name: atat --- apiVersion: extensions/v1beta1 kind: Deployment @@ -10,7 +10,7 @@ metadata: labels: app: atst name: atst - namespace: atat-test + namespace: atat spec: selector: matchLabels: @@ -28,10 +28,10 @@ spec: fsGroup: 101 containers: - name: atst - image: registry.atat.code.mil:443/atst-prod:575cfce5 + image: 904153757533.dkr.ecr.us-east-2.amazonaws.com/atat:884d95ada21a5097f5c07f305d8e4e24d0f2a03f resources: requests: - memory: "2500Mi" + memory: "500Mi" envFrom: - configMapRef: name: atst-envvars @@ -39,40 +39,22 @@ spec: - name: atst-config mountPath: "/opt/atat/atst/atst-overrides.ini" subPath: atst-overrides.ini - - name: uwsgi-config - mountPath: "/opt/atat/atst/uwsgi-config.ini" - subPath: uwsgi-config.ini + - name: nginx-client-ca-bundle + mountPath: "/opt/atat/atst/ssl/server-certs/ca-chain.pem" + subPath: client-ca-bundle.pem - name: uwsgi-socket-dir mountPath: "/var/run/uwsgi" - - name: atst-nginx + - name: nginx image: nginx:alpine ports: - containerPort: 8442 name: http - - containerPort: 8443 - name: https volumeMounts: - - name: nginx-auth-tls - mountPath: "/etc/ssl/private" - - name: nginx-client-ca-bundle - mountPath: "/etc/ssl/client-ca-bundle.pem" - subPath: client-ca-bundle.pem - name: nginx-config mountPath: "/etc/nginx/conf.d/atst.conf" subPath: atst.conf - - name: nginx-config - mountPath: "/etc/nginx/conf.d/00json_log.conf" - subPath: 00json_log.conf - - name: nginx-dhparam - mountPath: "/etc/ssl/dhparam.pem" - subPath: dhparam.pem - - name: nginx-htpasswd - mountPath: "/etc/nginx/.htpasswd" - subPath: .htpasswd - name: uwsgi-socket-dir mountPath: "/var/run/uwsgi" - imagePullSecrets: - - name: regcred volumes: - name: atst-config secret: @@ -81,16 +63,6 @@ spec: - key: override.ini path: atst-overrides.ini mode: 0644 - - name: nginx-auth-tls - secret: - secretName: atst-auth-test-ingress-tls - items: - - key: tls.crt - path: auth.atat.crt - mode: 0644 - - key: tls.key - path: auth.atat.key - mode: 0640 - name: nginx-client-ca-bundle secret: secretName: nginx-client-ca-bundle @@ -104,29 +76,6 @@ spec: items: - key: nginx-config path: atst.conf - - key: nginx-json-log-config - path: 00json_log.conf - - name: nginx-dhparam - secret: - secretName: dhparam-4096 - items: - - key: dhparam.pem - path: dhparam.pem - mode: 0640 - - name: nginx-htpasswd - secret: - secretName: atst-nginx-htpasswd - items: - - key: htpasswd - path: .htpasswd - mode: 0640 - - name: uwsgi-config - configMap: - name: atst-config - items: - - key: uwsgi-config - path: uwsgi-config.ini - mode: 0644 - name: uwsgi-socket-dir emptyDir: medium: Memory @@ -137,7 +86,7 @@ metadata: labels: app: atst name: atst-worker - namespace: atat-test + namespace: atat spec: selector: matchLabels: @@ -155,8 +104,13 @@ spec: fsGroup: 101 containers: - name: atst-worker - image: registry.atat.code.mil:443/atst-prod:575cfce5 - args: ["/bin/bash", "-c", "/opt/atat/atst/script/rq_worker"] + image: 904153757533.dkr.ecr.us-east-2.amazonaws.com/atat:884d95ada21a5097f5c07f305d8e4e24d0f2a03f + args: [ + "/opt/atat/atst/.venv/bin/python", + "/opt/atat/atst/.venv/bin/flask", + "rq", + "worker" + ] resources: requests: memory: "500Mi" @@ -169,8 +123,6 @@ spec: - name: atst-config mountPath: "/opt/atat/atst/atst-overrides.ini" subPath: atst-overrides.ini - imagePullSecrets: - - name: regcred volumes: - name: atst-config secret: @@ -186,51 +138,11 @@ metadata: labels: app: atst name: atst - namespace: atat-test + namespace: atat spec: ports: - - name: http - port: 80 + - port: 80 targetPort: 8442 selector: role: web ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app: atst - name: atst-auth - namespace: atat-test -spec: - type: NodePort - ports: - - name: https - protocol: TCP - nodePort: 32711 - port: 8443 - selector: - role: web ---- -apiVersion: extensions/v1beta1 -kind: Ingress -metadata: - name: atst - namespace: atat-test - annotations: - kubernetes.io/tls-acme: "true" - kubernetes.io/ingress.class: "nginx" - nginx.ingress.kubernetes.io/proxy-body-size: 10m -spec: - tls: - - secretName: atst-test-ingress-tls - hosts: - - test.atat.code.mil - rules: - - host: test.atat.code.mil - http: - paths: - - path: / - backend: - serviceName: atst - servicePort: 80 + type: LoadBalancer diff --git a/deploy/kubernetes/test/atst-configmap.yml b/k8s/azure/atst-configmap.yml similarity index 98% rename from deploy/kubernetes/test/atst-configmap.yml rename to k8s/azure/atst-configmap.yml index 30c64f29..79f9a61b 100644 --- a/deploy/kubernetes/test/atst-configmap.yml +++ b/k8s/azure/atst-configmap.yml @@ -3,7 +3,7 @@ apiVersion: v1 kind: ConfigMap metadata: name: atst-config - namespace: atat-test + namespace: atat data: uwsgi-config: |- [uwsgi] diff --git a/deploy/kubernetes/test/atst-envvars-configmap.yml b/k8s/azure/atst-envvars-configmap.yml similarity index 58% rename from deploy/kubernetes/test/atst-envvars-configmap.yml rename to k8s/azure/atst-envvars-configmap.yml index 1ac4cb03..fc65df51 100644 --- a/deploy/kubernetes/test/atst-envvars-configmap.yml +++ b/k8s/azure/atst-envvars-configmap.yml @@ -3,13 +3,10 @@ apiVersion: v1 kind: ConfigMap metadata: name: atst-envvars - namespace: atat-test + namespace: atat data: TZ: UTC FLASK_ENV: dev OVERRIDE_CONFIG_FULLPATH: /opt/atat/atst/atst-overrides.ini - UWSGI_CONFIG_FULLPATH: /opt/atat/atst/uwsgi-config.ini - RQ_QUEUES: atat-test + UWSGI_CONFIG_FULLPATH: /opt/atat/atst/uwsgi.ini CRL_STORAGE_PROVIDER: CLOUDFILES - LOG_JSON: "true" - DEBUG: "false" diff --git a/k8s/azure/atst-nginx-configmap.yml b/k8s/azure/atst-nginx-configmap.yml new file mode 100644 index 00000000..3c38614b --- /dev/null +++ b/k8s/azure/atst-nginx-configmap.yml @@ -0,0 +1,37 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: atst-nginx + namespace: atat +data: + nginx-config: |- + server { + listen 8442; + server_name localhost; + + location / { + try_files $uri @app; + } + + location @app { + uwsgi_pass unix:///var/run/uwsgi/uwsgi.socket; + uwsgi_param QUERY_STRING $query_string; + uwsgi_param REQUEST_METHOD $request_method; + uwsgi_param CONTENT_TYPE $content_type; + uwsgi_param CONTENT_LENGTH $content_length; + + uwsgi_param REQUEST_URI $request_uri; + uwsgi_param PATH_INFO $document_uri; + uwsgi_param DOCUMENT_ROOT $document_root; + uwsgi_param SERVER_PROTOCOL $server_protocol; + uwsgi_param REQUEST_SCHEME $scheme; + uwsgi_param HTTPS $https if_not_empty; + + uwsgi_param REMOTE_ADDR $remote_addr; + uwsgi_param REMOTE_PORT $remote_port; + uwsgi_param SERVER_PORT $server_port; + uwsgi_param SERVER_NAME $server_name; + uwsgi_param HTTP_X_REQUEST_ID $request_id; + } + } diff --git a/deploy/kubernetes/test/atst-worker-envvars-configmap.yml b/k8s/azure/atst-worker-envvars-configmap.yml similarity index 86% rename from deploy/kubernetes/test/atst-worker-envvars-configmap.yml rename to k8s/azure/atst-worker-envvars-configmap.yml index e18f8619..c1f8edde 100644 --- a/deploy/kubernetes/test/atst-worker-envvars-configmap.yml +++ b/k8s/azure/atst-worker-envvars-configmap.yml @@ -3,7 +3,7 @@ apiVersion: v1 kind: ConfigMap metadata: name: atst-worker-envvars - namespace: atat-test + namespace: atat data: TZ: UTC DISABLE_CRL_CHECK: "True" diff --git a/deploy/kubernetes/uat/uat.yml b/k8s/azure/azure.yml similarity index 51% rename from deploy/kubernetes/uat/uat.yml rename to k8s/azure/azure.yml index cb4c67a1..c407adee 100644 --- a/deploy/kubernetes/uat/uat.yml +++ b/k8s/azure/azure.yml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Namespace metadata: - name: atat-uat + name: atat --- apiVersion: extensions/v1beta1 kind: Deployment @@ -10,7 +10,7 @@ metadata: labels: app: atst name: atst - namespace: atat-uat + namespace: atat spec: selector: matchLabels: @@ -28,10 +28,10 @@ spec: fsGroup: 101 containers: - name: atst - image: registry.atat.code.mil:443/atst-prod:03ee3438 + image: pwatat.azurecr.io/atat:884d95ada21a5097f5c07f305d8e4e24d0f2a03f resources: requests: - memory: "2500Mi" + memory: "500Mi" envFrom: - configMapRef: name: atst-envvars @@ -42,40 +42,19 @@ spec: - name: nginx-client-ca-bundle mountPath: "/opt/atat/atst/ssl/server-certs/ca-chain.pem" subPath: client-ca-bundle.pem - - name: uwsgi-config - mountPath: "/opt/atat/atst/uwsgi-config.ini" - subPath: uwsgi-config.ini - name: uwsgi-socket-dir mountPath: "/var/run/uwsgi" - - name: atst-nginx + - name: nginx image: nginx:alpine ports: - containerPort: 8442 name: http - - containerPort: 8443 - name: https volumeMounts: - - name: nginx-auth-tls - mountPath: "/etc/ssl/private" - - name: nginx-client-ca-bundle - mountPath: "/etc/ssl/client-ca-bundle.pem" - subPath: client-ca-bundle.pem - name: nginx-config mountPath: "/etc/nginx/conf.d/atst.conf" subPath: atst.conf - - name: nginx-config - mountPath: "/etc/nginx/conf.d/00json_log.conf" - subPath: 00json_log.conf - - name: nginx-dhparam - mountPath: "/etc/ssl/dhparam.pem" - subPath: dhparam.pem - - name: nginx-htpasswd - mountPath: "/etc/nginx/.htpasswd" - subPath: .htpasswd - name: uwsgi-socket-dir mountPath: "/var/run/uwsgi" - imagePullSecrets: - - name: regcred volumes: - name: atst-config secret: @@ -84,16 +63,6 @@ spec: - key: override.ini path: atst-overrides.ini mode: 0644 - - name: nginx-auth-tls - secret: - secretName: atst-auth-uat-ingress-tls - items: - - key: tls.crt - path: auth.atat.crt - mode: 0644 - - key: tls.key - path: auth.atat.key - mode: 0640 - name: nginx-client-ca-bundle secret: secretName: nginx-client-ca-bundle @@ -107,29 +76,6 @@ spec: items: - key: nginx-config path: atst.conf - - key: nginx-json-log-config - path: 00json_log.conf - - name: nginx-dhparam - secret: - secretName: dhparam-4096 - items: - - key: dhparam.pem - path: dhparam.pem - mode: 0640 - - name: nginx-htpasswd - secret: - secretName: atst-nginx-htpasswd - items: - - key: htpasswd - path: .htpasswd - mode: 0640 - - name: uwsgi-config - configMap: - name: atst-config - items: - - key: uwsgi-config - path: uwsgi-config.ini - mode: 0644 - name: uwsgi-socket-dir emptyDir: medium: Memory @@ -140,7 +86,7 @@ metadata: labels: app: atst name: atst-worker - namespace: atat-uat + namespace: atat spec: selector: matchLabels: @@ -158,8 +104,13 @@ spec: fsGroup: 101 containers: - name: atst-worker - image: registry.atat.code.mil:443/atst-prod:03ee3438 - args: ["/bin/bash", "-c", "/opt/atat/atst/script/rq_worker"] + image: pwatat.azurecr.io/atat:884d95ada21a5097f5c07f305d8e4e24d0f2a03f + args: [ + "/opt/atat/atst/.venv/bin/python", + "/opt/atat/atst/.venv/bin/flask", + "rq", + "worker" + ] resources: requests: memory: "500Mi" @@ -172,8 +123,6 @@ spec: - name: atst-config mountPath: "/opt/atat/atst/atst-overrides.ini" subPath: atst-overrides.ini - imagePullSecrets: - - name: regcred volumes: - name: atst-config secret: @@ -189,51 +138,11 @@ metadata: labels: app: atst name: atst - namespace: atat-uat + namespace: atat spec: ports: - - name: http - port: 80 + - port: 80 targetPort: 8442 selector: role: web ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app: atst - name: atst-auth - namespace: atat-uat -spec: - type: NodePort - ports: - - name: https - protocol: TCP - nodePort: 32701 - port: 8443 - selector: - role: web ---- -apiVersion: extensions/v1beta1 -kind: Ingress -metadata: - name: atst - namespace: atat-uat - annotations: - kubernetes.io/tls-acme: "true" - kubernetes.io/ingress.class: "nginx" - nginx.ingress.kubernetes.io/proxy-body-size: 10m -spec: - tls: - - secretName: atst-uat-ingress-tls - hosts: - - uat.atat.code.mil - rules: - - host: uat.atat.code.mil - http: - paths: - - path: / - backend: - serviceName: atst - servicePort: 80 + type: LoadBalancer From d056191b01bb5acc666fb67cf6eaabad1bfef845 Mon Sep 17 00:00:00 2001 From: dandds Date: Wed, 17 Jul 2019 15:24:04 -0400 Subject: [PATCH 2/4] Working config for CD. This adds the AWS and Azure CircleCI orbs for updating container images in a cluster. It installs the clients for both CSPs, configures kubectl with a programmatic user's auth information, and executes a `kubectl set image` command to reset the cluster image to the one that was just pushed to the container registry. --- .circleci/config.yml | 76 ++++++++++++++++++++++++++++++++++++++------ 1 file changed, 67 insertions(+), 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b9d73612..13a081fe 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,7 +2,9 @@ version: 2.1 orbs: aws-ecr: circleci/aws-ecr@4.0.1 + aws-eks: circleci/aws-eks@0.1.0 azure-acr: circleci/azure-acr@0.1.1 + azure-aks: circleci/azure-aks@0.2.0 defaults: appEnvironment: &appEnvironment @@ -83,15 +85,6 @@ workflows: - test: requires: - app_setup - - aws-ecr/build_and_push_image: - repo: atat - tag: "${CIRCLE_SHA1}" - requires: - - test - filters: - branches: - only: - - master - azure-acr/build_and_push_image: login-server-name: "${AZURE_SERVER_NAME}" registry-name: pwatat @@ -103,3 +96,68 @@ workflows: branches: only: - master + - azure-aks/update-container-image: + cluster-name: atat-cluster + container-image-updates: "atst=${AZURE_SERVER_NAME}/atat:${CIRCLE_SHA1}" + namespace: atat + resource-name: deployment.apps/atst + resource-group: atat + # uncomment below for debugging + # show-kubectl-command: true + requires: + - azure-acr/build_and_push_image + filters: + branches: + only: + - master + - azure-aks/update-container-image: + cluster-name: atat-cluster + container-image-updates: "atst-worker=${AZURE_SERVER_NAME}/atat:${CIRCLE_SHA1}" + namespace: atat + resource-name: deployment.apps/atst-worker + resource-group: atat + # uncomment below for debugging + # show-kubectl-command: true + requires: + - azure-acr/build_and_push_image + filters: + branches: + only: + - master + - aws-ecr/build_and_push_image: + repo: atat + tag: "${CIRCLE_SHA1}" + requires: + - test + filters: + branches: + only: + - master + - aws-eks/update-container-image: + cluster-name: atat + container-image-updates: "atst=${AWS_ECR_ACCOUNT_URL}/atat:${CIRCLE_SHA1}" + namespace: atat + resource-name: deployment.apps/atst + aws-region: "${AWS_REGION}" + # uncomment below for debugging + # show-kubectl-command: true + requires: + - aws-ecr/build_and_push_image + filters: + branches: + only: + - master + - aws-eks/update-container-image: + cluster-name: atat + container-image-updates: "atst-worker=${AWS_ECR_ACCOUNT_URL}/atat:${CIRCLE_SHA1}" + namespace: atat + resource-name: deployment.apps/atst-worker + aws-region: "${AWS_REGION}" + # uncomment below for debugging + # show-kubectl-command: true + requires: + - aws-ecr/build_and_push_image + filters: + branches: + only: + - master From 4c70d59c5d845a3b051043082440abd03c5241d3 Mon Sep 17 00:00:00 2001 From: dandds Date: Tue, 30 Jul 2019 10:59:01 -0400 Subject: [PATCH 3/4] Add k8s networking. - bind static azure IPs - Add load balancers for both CSPs --- .circleci/config.yml | 28 --------- k8s/aws/atst-nginx-configmap.yml | 91 ++++++++++++++++++++++-------- k8s/aws/aws.yml | 25 +++++++- k8s/azure/atst-nginx-configmap.yml | 91 ++++++++++++++++++++++-------- k8s/azure/azure.yml | 23 +++++++- 5 files changed, 182 insertions(+), 76 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 13a081fe..73d4da23 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -96,20 +96,6 @@ workflows: branches: only: - master - - azure-aks/update-container-image: - cluster-name: atat-cluster - container-image-updates: "atst=${AZURE_SERVER_NAME}/atat:${CIRCLE_SHA1}" - namespace: atat - resource-name: deployment.apps/atst - resource-group: atat - # uncomment below for debugging - # show-kubectl-command: true - requires: - - azure-acr/build_and_push_image - filters: - branches: - only: - - master - azure-aks/update-container-image: cluster-name: atat-cluster container-image-updates: "atst-worker=${AZURE_SERVER_NAME}/atat:${CIRCLE_SHA1}" @@ -147,17 +133,3 @@ workflows: branches: only: - master - - aws-eks/update-container-image: - cluster-name: atat - container-image-updates: "atst-worker=${AWS_ECR_ACCOUNT_URL}/atat:${CIRCLE_SHA1}" - namespace: atat - resource-name: deployment.apps/atst-worker - aws-region: "${AWS_REGION}" - # uncomment below for debugging - # show-kubectl-command: true - requires: - - aws-ecr/build_and_push_image - filters: - branches: - only: - - master diff --git a/k8s/aws/atst-nginx-configmap.yml b/k8s/aws/atst-nginx-configmap.yml index 3c38614b..cd0d051e 100644 --- a/k8s/aws/atst-nginx-configmap.yml +++ b/k8s/aws/atst-nginx-configmap.yml @@ -7,31 +7,78 @@ metadata: data: nginx-config: |- server { - listen 8442; - server_name localhost; - + server_name aws.atat.code.mil; + # access_log /var/log/nginx/access.log json; + listen 8442; + listen [::]:8442 ipv6only=on; + # if ($http_x_forwarded_proto != 'https') { + # return 301 https://$host$request_uri; + # } + location /login-redirect { + return 301 https://auth-aws.atat.code.mil$request_uri; + } + location /login-dev { + try_files $uri @appbasicauth; + } location / { try_files $uri @app; } - location @app { - uwsgi_pass unix:///var/run/uwsgi/uwsgi.socket; - uwsgi_param QUERY_STRING $query_string; - uwsgi_param REQUEST_METHOD $request_method; - uwsgi_param CONTENT_TYPE $content_type; - uwsgi_param CONTENT_LENGTH $content_length; - - uwsgi_param REQUEST_URI $request_uri; - uwsgi_param PATH_INFO $document_uri; - uwsgi_param DOCUMENT_ROOT $document_root; - uwsgi_param SERVER_PROTOCOL $server_protocol; - uwsgi_param REQUEST_SCHEME $scheme; - uwsgi_param HTTPS $https if_not_empty; - - uwsgi_param REMOTE_ADDR $remote_addr; - uwsgi_param REMOTE_PORT $remote_port; - uwsgi_param SERVER_PORT $server_port; - uwsgi_param SERVER_NAME $server_name; - uwsgi_param HTTP_X_REQUEST_ID $request_id; + include uwsgi_params; + uwsgi_pass unix:///var/run/uwsgi/uwsgi.socket; + uwsgi_param HTTP_X_REQUEST_ID $request_id; + } + location @appbasicauth { + include uwsgi_params; + uwsgi_pass unix:///var/run/uwsgi/uwsgi.socket; + auth_basic "Developer Access"; + auth_basic_user_file /etc/nginx/.htpasswd; + uwsgi_param HTTP_X_REQUEST_ID $request_id; + } + } + server { + # access_log /var/log/nginx/access.log json; + server_name auth-aws.atat.code.mil; + listen 8443; + listen [::]:8443 ipv6only=on; + # SSL server certificate and private key + # ssl_certificate /etc/ssl/private/auth.atat.crt; + # ssl_certificate_key /etc/ssl/private/auth.atat.key; + # Set SSL protocols, ciphers, and related options + # ssl_protocols TLSv1.3 TLSv1.2; + # ssl_ciphers 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256'; + # ssl_prefer_server_ciphers on; + # ssl_ecdh_curve secp384r1; + # ssl_dhparam /etc/ssl/dhparam.pem; + # SSL session options + # ssl_session_timeout 4h; + # ssl_session_cache shared:SSL:10m; # 1mb = ~4000 sessions + # ssl_session_tickets off; + # OCSP Stapling + # ssl_stapling on; + # ssl_stapling_verify on; + # resolver 8.8.8.8 8.8.4.4; + # Request and validate client certificate + # ssl_verify_client on; + # ssl_verify_depth 10; + # ssl_client_certificate /etc/ssl/client-ca-bundle.pem; + # Guard against HTTPS -> HTTP downgrade + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains; always"; + location / { + return 301 https://aws.atat.code.mil$request_uri; + } + location /login-redirect { + try_files $uri @app; + } + location @app { + include uwsgi_params; + uwsgi_pass unix:///var/run/uwsgi/uwsgi.socket; + # uwsgi_param HTTP_X_SSL_CLIENT_VERIFY $ssl_client_verify; + # uwsgi_param HTTP_X_SSL_CLIENT_CERT $ssl_client_raw_cert; + # uwsgi_param HTTP_X_SSL_CLIENT_S_DN $ssl_client_s_dn; + # uwsgi_param HTTP_X_SSL_CLIENT_S_DN_LEGACY $ssl_client_s_dn_legacy; + # uwsgi_param HTTP_X_SSL_CLIENT_I_DN $ssl_client_i_dn; + # uwsgi_param HTTP_X_SSL_CLIENT_I_DN_LEGACY $ssl_client_i_dn_legacy; + uwsgi_param HTTP_X_REQUEST_ID $request_id; } } diff --git a/k8s/aws/aws.yml b/k8s/aws/aws.yml index 336d9434..dfb434b0 100644 --- a/k8s/aws/aws.yml +++ b/k8s/aws/aws.yml @@ -48,7 +48,9 @@ spec: image: nginx:alpine ports: - containerPort: 8442 - name: http + name: main + - containerPort: 8443 + name: auth volumeMounts: - name: nginx-config mountPath: "/etc/nginx/conf.d/atst.conf" @@ -137,8 +139,10 @@ kind: Service metadata: labels: app: atst - name: atst + name: atst-main namespace: atat + annotations: + service.beta.kubernetes.io/aws-load-balancer-type: "nlb" spec: ports: - port: 80 @@ -146,3 +150,20 @@ spec: selector: role: web type: LoadBalancer +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: atst + name: atst-auth + namespace: atat + annotations: + service.beta.kubernetes.io/aws-load-balancer-type: "nlb" +spec: + ports: + - port: 80 + targetPort: 8443 + selector: + role: web + type: LoadBalancer diff --git a/k8s/azure/atst-nginx-configmap.yml b/k8s/azure/atst-nginx-configmap.yml index 3c38614b..77f69e5f 100644 --- a/k8s/azure/atst-nginx-configmap.yml +++ b/k8s/azure/atst-nginx-configmap.yml @@ -7,31 +7,78 @@ metadata: data: nginx-config: |- server { - listen 8442; - server_name localhost; - + server_name azure.atat.code.mil; + # access_log /var/log/nginx/access.log json; + listen 8442; + listen [::]:8442 ipv6only=on; + # if ($http_x_forwarded_proto != 'https') { + # return 301 https://$host$request_uri; + # } + location /login-redirect { + return 301 https://auth-azure.atat.code.mil$request_uri; + } + location /login-dev { + try_files $uri @appbasicauth; + } location / { try_files $uri @app; } - location @app { - uwsgi_pass unix:///var/run/uwsgi/uwsgi.socket; - uwsgi_param QUERY_STRING $query_string; - uwsgi_param REQUEST_METHOD $request_method; - uwsgi_param CONTENT_TYPE $content_type; - uwsgi_param CONTENT_LENGTH $content_length; - - uwsgi_param REQUEST_URI $request_uri; - uwsgi_param PATH_INFO $document_uri; - uwsgi_param DOCUMENT_ROOT $document_root; - uwsgi_param SERVER_PROTOCOL $server_protocol; - uwsgi_param REQUEST_SCHEME $scheme; - uwsgi_param HTTPS $https if_not_empty; - - uwsgi_param REMOTE_ADDR $remote_addr; - uwsgi_param REMOTE_PORT $remote_port; - uwsgi_param SERVER_PORT $server_port; - uwsgi_param SERVER_NAME $server_name; - uwsgi_param HTTP_X_REQUEST_ID $request_id; + include uwsgi_params; + uwsgi_pass unix:///var/run/uwsgi/uwsgi.socket; + uwsgi_param HTTP_X_REQUEST_ID $request_id; + } + location @appbasicauth { + include uwsgi_params; + uwsgi_pass unix:///var/run/uwsgi/uwsgi.socket; + auth_basic "Developer Access"; + auth_basic_user_file /etc/nginx/.htpasswd; + uwsgi_param HTTP_X_REQUEST_ID $request_id; + } + } + server { + # access_log /var/log/nginx/access.log json; + server_name auth-azure.atat.code.mil; + listen 8443; + listen [::]:8443 ipv6only=on; + # SSL server certificate and private key + # ssl_certificate /etc/ssl/private/auth.atat.crt; + # ssl_certificate_key /etc/ssl/private/auth.atat.key; + # Set SSL protocols, ciphers, and related options + # ssl_protocols TLSv1.3 TLSv1.2; + # ssl_ciphers 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256'; + # ssl_prefer_server_ciphers on; + # ssl_ecdh_curve secp384r1; + # ssl_dhparam /etc/ssl/dhparam.pem; + # SSL session options + # ssl_session_timeout 4h; + # ssl_session_cache shared:SSL:10m; # 1mb = ~4000 sessions + # ssl_session_tickets off; + # OCSP Stapling + # ssl_stapling on; + # ssl_stapling_verify on; + # resolver 8.8.8.8 8.8.4.4; + # Request and validate client certificate + # ssl_verify_client on; + # ssl_verify_depth 10; + # ssl_client_certificate /etc/ssl/client-ca-bundle.pem; + # Guard against HTTPS -> HTTP downgrade + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains; always"; + location / { + return 301 https://azure.atat.code.mil$request_uri; + } + location /login-redirect { + try_files $uri @app; + } + location @app { + include uwsgi_params; + uwsgi_pass unix:///var/run/uwsgi/uwsgi.socket; + # uwsgi_param HTTP_X_SSL_CLIENT_VERIFY $ssl_client_verify; + # uwsgi_param HTTP_X_SSL_CLIENT_CERT $ssl_client_raw_cert; + # uwsgi_param HTTP_X_SSL_CLIENT_S_DN $ssl_client_s_dn; + # uwsgi_param HTTP_X_SSL_CLIENT_S_DN_LEGACY $ssl_client_s_dn_legacy; + # uwsgi_param HTTP_X_SSL_CLIENT_I_DN $ssl_client_i_dn; + # uwsgi_param HTTP_X_SSL_CLIENT_I_DN_LEGACY $ssl_client_i_dn_legacy; + uwsgi_param HTTP_X_REQUEST_ID $request_id; } } diff --git a/k8s/azure/azure.yml b/k8s/azure/azure.yml index c407adee..2040e24e 100644 --- a/k8s/azure/azure.yml +++ b/k8s/azure/azure.yml @@ -48,7 +48,9 @@ spec: image: nginx:alpine ports: - containerPort: 8442 - name: http + name: main + - containerPort: 8443 + name: auth volumeMounts: - name: nginx-config mountPath: "/etc/nginx/conf.d/atst.conf" @@ -137,12 +139,29 @@ kind: Service metadata: labels: app: atst - name: atst + name: atst-main namespace: atat spec: + loadBalancerIP: 13.92.235.6 ports: - port: 80 targetPort: 8442 selector: role: web type: LoadBalancer +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: atst + name: atst-auth + namespace: atat +spec: + loadBalancerIP: 23.100.24.41 + ports: + - port: 80 + targetPort: 8443 + selector: + role: web + type: LoadBalancer From f3e032fc03b2167898526048544d7debd918e6ad Mon Sep 17 00:00:00 2001 From: dandds Date: Tue, 30 Jul 2019 15:14:09 -0400 Subject: [PATCH 4/4] Finalize CD config and add k8s job for migrations. Add CircleCI config for both CSPs to: - build the Docker image and push it to the registry - run a short-lived k8s job to apply migrations and see data - update the images for the Flask pods and rq worker pods --- .circleci/config.yml | 89 ++++++++++++++++++++++++++++++++++++++- Dockerfile | 1 + k8s/shared/migration.yaml | 42 ++++++++++++++++++ script/cluster_migration | 34 +++++++++++++++ 4 files changed, 164 insertions(+), 2 deletions(-) create mode 100644 k8s/shared/migration.yaml create mode 100755 script/cluster_migration diff --git a/.circleci/config.yml b/.circleci/config.yml index 73d4da23..04e9f70c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,6 +5,7 @@ orbs: aws-eks: circleci/aws-eks@0.1.0 azure-acr: circleci/azure-acr@0.1.1 azure-aks: circleci/azure-aks@0.2.0 + kubernetes: circleci/kubernetes@0.3.0 defaults: appEnvironment: &appEnvironment @@ -16,6 +17,27 @@ defaults: PIP_VERSION: 18.* CRL_STORAGE_PROVIDER: CLOUDFILES +commands: + migration_setup: + parameters: + container_image: + type: string + steps: + - attach_workspace: + at: . + - run: + name: Setup Environment Variables + command: | + echo 'export CONTAINER_IMAGE="<< parameters.container_image >>"' >> $BASH_ENV + - run: sudo apt-get update + - run: sudo apt-get install gettext + - kubernetes/install + migration_apply: + steps: + - run: + command: ./script/cluster_migration + name: Apply Migrations and Seed Roles + jobs: app_setup: docker: @@ -76,6 +98,27 @@ jobs: name: "Run Tests" command: ./script/cibuild + aws-migration: + executor: aws-eks/python3 + steps: + - migration_setup: + container_image: "$AWS_ECR_ACCOUNT_URL/atat:$CIRCLE_SHA1" + - aws-eks/update-kubeconfig-with-authenticator: + cluster-name: atat + aws-region: "${AWS_REGION}" + - migration_apply + + azure-migration: + executor: azure-aks/default + steps: + - migration_setup: + container_image: "$AZURE_SERVER_NAME/atat:$CIRCLE_SHA1" + - azure-aks/update-kubeconfig-with-credentials: + cluster-name: atat-cluster + install-kubectl: true + perform-login: true + resource-group: atat + - migration_apply workflows: version: 2 @@ -96,6 +139,27 @@ workflows: branches: only: - master + - azure-migration: + requires: + - azure-acr/build_and_push_image + filters: + branches: + only: + - master + - azure-aks/update-container-image: + cluster-name: atat-cluster + container-image-updates: "atst=${AZURE_SERVER_NAME}/atat:${CIRCLE_SHA1}" + namespace: atat + resource-name: deployment.apps/atst + resource-group: atat + # uncomment below for debugging + # show-kubectl-command: true + requires: + - azure-migration + filters: + branches: + only: + - master - azure-aks/update-container-image: cluster-name: atat-cluster container-image-updates: "atst-worker=${AZURE_SERVER_NAME}/atat:${CIRCLE_SHA1}" @@ -105,7 +169,7 @@ workflows: # uncomment below for debugging # show-kubectl-command: true requires: - - azure-acr/build_and_push_image + - azure-migration filters: branches: only: @@ -119,6 +183,13 @@ workflows: branches: only: - master + - aws-migration: + requires: + - aws-ecr/build_and_push_image + filters: + branches: + only: + - master - aws-eks/update-container-image: cluster-name: atat container-image-updates: "atst=${AWS_ECR_ACCOUNT_URL}/atat:${CIRCLE_SHA1}" @@ -128,7 +199,21 @@ workflows: # uncomment below for debugging # show-kubectl-command: true requires: - - aws-ecr/build_and_push_image + - aws-migration + filters: + branches: + only: + - master + - aws-eks/update-container-image: + cluster-name: atat + container-image-updates: "atst-worker=${AWS_ECR_ACCOUNT_URL}/atat:${CIRCLE_SHA1}" + namespace: atat + resource-name: deployment.apps/atst-worker + aws-region: "${AWS_REGION}" + # uncomment below for debugging + # show-kubectl-command: true + requires: + - aws-migration filters: branches: only: diff --git a/Dockerfile b/Dockerfile index 681515b4..2785b815 100644 --- a/Dockerfile +++ b/Dockerfile @@ -78,6 +78,7 @@ COPY --from=builder /install/atst/ ./atst/ COPY --from=builder /install/config/ ./config/ COPY --from=builder /install/templates/ ./templates/ COPY --from=builder /install/translations.yaml . +COPY --from=builder /install/script/seed_roles.py ./script/seed_roles.py COPY --from=builder /install/static/ ./static/ COPY --from=builder /install/uwsgi.ini . COPY --from=builder /usr/local/bin/uwsgi /usr/local/bin/uwsgi diff --git a/k8s/shared/migration.yaml b/k8s/shared/migration.yaml new file mode 100644 index 00000000..cbf7da6b --- /dev/null +++ b/k8s/shared/migration.yaml @@ -0,0 +1,42 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: migration + namespace: atat +spec: + ttlSecondsAfterFinished: 100 + backoffLimit: 2 + template: + spec: + containers: + - name: migration + image: $CONTAINER_IMAGE + command: [ + "/bin/sh", "-c" + ] + args: + - | + /opt/atat/atst/.venv/bin/python \ + /opt/atat/atst/.venv/bin/alembic \ + upgrade head \ + && \ + /opt/atat/atst/.venv/bin/python \ + /opt/atat/atst/script/seed_roles.py + envFrom: + - configMapRef: + name: atst-envvars + - configMapRef: + name: atst-worker-envvars + volumeMounts: + - name: atst-config + mountPath: "/opt/atat/atst/atst-overrides.ini" + subPath: atst-overrides.ini + volumes: + - name: atst-config + secret: + secretName: atst-config-ini + items: + - key: override.ini + path: atst-overrides.ini + mode: 0644 + restartPolicy: Never diff --git a/script/cluster_migration b/script/cluster_migration new file mode 100755 index 00000000..2c011365 --- /dev/null +++ b/script/cluster_migration @@ -0,0 +1,34 @@ +#!/bin/sh + +if [ -z "${K8S_CONTEXT+is_set}" ]; then + K8S_CMD="kubectl" +else + K8S_CMD="kubectl --context=$K8S_CONTEXT" +fi + +if [ -z "${MIGRATION_TIMEOUT+is_set}" ]; then + MIGRATION_TIMEOUT=120s +fi + +echo "Creating job..." +envsubst < k8s/shared/migration.yaml | $K8S_CMD -n atat apply -f - + +echo "Wait for job to finish or timeout..." +JOB_SUCCESS=$(${K8S_CMD} -n atat wait --for=condition=complete --timeout=${MIGRATION_TIMEOUT} job/migration) + +delete_job () { + echo "Deleting job..." + $K8S_CMD -n atat delete job migration +} + +if echo "$JOB_SUCCESS" | grep -q "condition met"; then + echo "Job ran successfully." + delete_job + exit 0 +else + POD_NAME=$(${K8S_CMD} -n atat get pods -l job-name=migration -o=jsonpath='{.items[0].metadata.name}') + echo "Job failed:" + $K8S_CMD -n atat log $POD_NAME + delete_job + exit 1 +fi