From 26c5b5ea7f3972b51e061466aba0066eefa2cd79 Mon Sep 17 00:00:00 2001 From: dandds Date: Thu, 14 Nov 2019 13:25:30 -0500 Subject: [PATCH 01/16] Add JSON logging back for NGINX container. This configures the NGINX container to log in JSON. It also updates the K8s config so that we mount all of the key/value pairs available in the atst-nginx ConfigMap as files in "/etc/nginx/conf.d" inside the container. This simplifies the config a little. --- deploy/azure/atst-nginx-configmap.yml | 26 ++++++++++++++++++++++++-- deploy/azure/azure.yml | 6 +----- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/deploy/azure/atst-nginx-configmap.yml b/deploy/azure/atst-nginx-configmap.yml index e19c6d54..22d6d97e 100644 --- a/deploy/azure/atst-nginx-configmap.yml +++ b/deploy/azure/atst-nginx-configmap.yml @@ -5,8 +5,10 @@ metadata: name: atst-nginx namespace: atat data: - nginx-config: |- + atst.conf: |- server { + access_log /var/log/nginx/access.log json; + listen ${PORT_PREFIX}342; server_name ${MAIN_DOMAIN}; root /usr/share/nginx/html; @@ -18,6 +20,8 @@ data: } } server { + access_log /var/log/nginx/access.log json; + listen ${PORT_PREFIX}343; server_name ${AUTH_DOMAIN}; root /usr/share/nginx/html; @@ -29,6 +33,8 @@ data: } } server { + access_log /var/log/nginx/access.log json; + server_name ${MAIN_DOMAIN}; # access_log /var/log/nginx/access.log json; listen ${PORT_PREFIX}442 ssl; @@ -58,7 +64,8 @@ data: } } server { - # access_log /var/log/nginx/access.log json; + access_log /var/log/nginx/access.log json; + server_name ${AUTH_DOMAIN}; listen ${PORT_PREFIX}443 ssl; listen [::]:${PORT_PREFIX}443 ssl ipv6only=on; @@ -88,3 +95,18 @@ data: uwsgi_param HTTP_X_REQUEST_ID $request_id; } } + 00json_log.conf: |- + log_format json escape=json + '{' + '"timestamp":"$time_iso8601",' + '"msec":"$msec",' + '"request_id":"$request_id",' + '"remote_addr":"$remote_addr",' + '"remote_user":"$remote_user",' + '"request":"$request",' + '"status":$status,' + '"body_bytes_sent":$body_bytes_sent,' + '"referer":"$http_referer",' + '"user_agent":"$http_user_agent",' + '"http_x_forwarded_for":"$http_x_forwarded_for"' + '}'; diff --git a/deploy/azure/azure.yml b/deploy/azure/azure.yml index 8d46fa4b..3ed3ea61 100644 --- a/deploy/azure/azure.yml +++ b/deploy/azure/azure.yml @@ -62,8 +62,7 @@ spec: name: auth volumeMounts: - name: nginx-config - mountPath: "/etc/nginx/conf.d/atst.conf" - subPath: atst.conf + mountPath: "/etc/nginx/conf.d/" - name: uwsgi-socket-dir mountPath: "/var/run/uwsgi" - name: nginx-htpasswd @@ -90,9 +89,6 @@ spec: - name: nginx-config configMap: name: atst-nginx - items: - - key: nginx-config - path: atst.conf - name: uwsgi-socket-dir emptyDir: medium: Memory From a3aa3e69352eadec8d2db976858461843495a4f5 Mon Sep 17 00:00:00 2001 From: dandds Date: Thu, 14 Nov 2019 14:20:17 -0500 Subject: [PATCH 02/16] Config for NGINX SSL/TLS. This adds additional SSL/TLS config to specify the acceptable TLS version, cipher suites, session cache, etc. Values are currently based on the Mozilla Foundation's recommendations for intermediate compatibility: https://wiki.mozilla.org/Security/Server_Side_TLS We will manage NGINX configuration snippets as a K8s ConfigMap so that they can be included in server blocks as-needed. --- deploy/azure/atst-nginx-configmap.yml | 8 +- deploy/azure/azure.yml | 143 ++++++++++++++------------ deploy/azure/kustomization.yaml | 1 + deploy/azure/nginx-snippets.yml | 24 +++++ 4 files changed, 106 insertions(+), 70 deletions(-) create mode 100644 deploy/azure/nginx-snippets.yml diff --git a/deploy/azure/atst-nginx-configmap.yml b/deploy/azure/atst-nginx-configmap.yml index 22d6d97e..b702924c 100644 --- a/deploy/azure/atst-nginx-configmap.yml +++ b/deploy/azure/atst-nginx-configmap.yml @@ -41,6 +41,9 @@ data: listen [::]:${PORT_PREFIX}442 ssl ipv6only=on; ssl_certificate /etc/ssl/private/atat.crt; ssl_certificate_key /etc/ssl/private/atat.key; + # additional SSL/TLS settings + include /etc/nginx/snippets/ssl.conf + location /login-redirect { return 301 https://auth-azure.atat.code.mil$request_uri; } @@ -75,8 +78,9 @@ data: ssl_verify_client on; ssl_verify_depth 10; ssl_client_certificate /etc/ssl/client-ca-bundle.pem; - # Guard against HTTPS -> HTTP downgrade - add_header Strict-Transport-Security "max-age=31536000; includeSubDomains; always"; + # additional SSL/TLS settings + include /etc/nginx/snippets/ssl.conf + location / { return 301 https://azure.atat.code.mil$request_uri; } diff --git a/deploy/azure/azure.yml b/deploy/azure/azure.yml index 3ed3ea61..4ed180fb 100644 --- a/deploy/azure/azure.yml +++ b/deploy/azure/azure.yml @@ -30,8 +30,8 @@ spec: - name: atst image: $CONTAINER_IMAGE envFrom: - - configMapRef: - name: atst-envvars + - configMapRef: + name: atst-envvars volumeMounts: - name: atst-config mountPath: "/opt/atat/atst/atst-overrides.ini" @@ -74,14 +74,16 @@ spec: mountPath: "/etc/ssl/" - name: acme mountPath: "/usr/share/nginx/html/.well-known/acme-challenge/" + - name: snippets + mountPath: "/etc/nginx/snippets/" volumes: - name: atst-config secret: secretName: atst-config-ini items: - - key: override.ini - path: atst-overrides.ini - mode: 0644 + - key: override.ini + path: atst-overrides.ini + mode: 0644 - name: nginx-client-ca-bundle configMap: name: nginx-client-ca-bundle @@ -96,19 +98,19 @@ spec: secret: secretName: atst-nginx-htpasswd items: - - key: htpasswd - path: .htpasswd - mode: 0640 + - key: htpasswd + path: .htpasswd + mode: 0640 - name: tls secret: secretName: azure-atat-code-mil-tls items: - - key: tls.crt - path: atat.crt - mode: 0644 - - key: tls.key - path: atat.key - mode: 0640 + - key: tls.crt + path: atat.crt + mode: 0644 + - key: tls.key + path: atat.key + mode: 0640 - name: crls-vol persistentVolumeClaim: claimName: crls-vol-claim @@ -116,9 +118,9 @@ spec: configMap: name: pgsslrootcert items: - - key: cert - path: pgsslrootcert.crt - mode: 0666 + - key: cert + path: pgsslrootcert.crt + mode: 0666 - name: acme configMap: name: acme-challenges @@ -128,9 +130,12 @@ spec: name: uwsgi-config defaultMode: 0666 items: - - key: uwsgi.ini - path: uwsgi.ini - mode: 0644 + - key: uwsgi.ini + path: uwsgi.ini + mode: 0644 + - name: snippets + configMap: + name: nginx-snippets --- apiVersion: extensions/v1beta1 kind: Deployment @@ -157,19 +162,20 @@ spec: containers: - name: atst-worker image: $CONTAINER_IMAGE - args: [ - "/opt/atat/atst/.venv/bin/python", - "/opt/atat/atst/.venv/bin/celery", - "-A", - "celery_worker.celery", - "worker", - "--loglevel=info" - ] + args: + [ + "/opt/atat/atst/.venv/bin/python", + "/opt/atat/atst/.venv/bin/celery", + "-A", + "celery_worker.celery", + "worker", + "--loglevel=info", + ] envFrom: - - configMapRef: - name: atst-envvars - - configMapRef: - name: atst-worker-envvars + - configMapRef: + name: atst-envvars + - configMapRef: + name: atst-worker-envvars volumeMounts: - name: atst-config mountPath: "/opt/atat/atst/atst-overrides.ini" @@ -182,16 +188,16 @@ spec: secret: secretName: atst-config-ini items: - - key: override.ini - path: atst-overrides.ini - mode: 0644 + - key: override.ini + path: atst-overrides.ini + mode: 0644 - name: pgsslrootcert configMap: name: pgsslrootcert items: - - key: cert - path: pgsslrootcert.crt - mode: 0666 + - key: cert + path: pgsslrootcert.crt + mode: 0666 --- apiVersion: extensions/v1beta1 kind: Deployment @@ -218,19 +224,20 @@ spec: containers: - name: atst-beat image: $CONTAINER_IMAGE - args: [ - "/opt/atat/atst/.venv/bin/python", - "/opt/atat/atst/.venv/bin/celery", - "-A", - "celery_worker.celery", - "beat", - "--loglevel=info" - ] + args: + [ + "/opt/atat/atst/.venv/bin/python", + "/opt/atat/atst/.venv/bin/celery", + "-A", + "celery_worker.celery", + "beat", + "--loglevel=info", + ] envFrom: - - configMapRef: - name: atst-envvars - - configMapRef: - name: atst-worker-envvars + - configMapRef: + name: atst-envvars + - configMapRef: + name: atst-worker-envvars volumeMounts: - name: atst-config mountPath: "/opt/atat/atst/atst-overrides.ini" @@ -243,16 +250,16 @@ spec: secret: secretName: atst-config-ini items: - - key: override.ini - path: atst-overrides.ini - mode: 0644 + - key: override.ini + path: atst-overrides.ini + mode: 0644 - name: pgsslrootcert configMap: name: pgsslrootcert items: - - key: cert - path: pgsslrootcert.crt - mode: 0666 + - key: cert + path: pgsslrootcert.crt + mode: 0666 --- apiVersion: v1 kind: Service @@ -264,12 +271,12 @@ metadata: spec: loadBalancerIP: 13.92.235.6 ports: - - port: 80 - targetPort: 8342 - name: http - - port: 443 - targetPort: 8442 - name: https + - port: 80 + targetPort: 8342 + name: http + - port: 443 + targetPort: 8442 + name: https selector: role: web type: LoadBalancer @@ -284,12 +291,12 @@ metadata: spec: loadBalancerIP: 23.100.24.41 ports: - - port: 80 - targetPort: 8343 - name: http - - port: 443 - targetPort: 8443 - name: https + - port: 80 + targetPort: 8343 + name: http + - port: 443 + targetPort: 8443 + name: https selector: role: web type: LoadBalancer diff --git a/deploy/azure/kustomization.yaml b/deploy/azure/kustomization.yaml index 43e6f813..9dee809c 100644 --- a/deploy/azure/kustomization.yaml +++ b/deploy/azure/kustomization.yaml @@ -11,3 +11,4 @@ resources: - nginx-client-ca-bundle.yml - acme-challenges.yml - aadpodidentity.yml + - nginx-snippets.yml diff --git a/deploy/azure/nginx-snippets.yml b/deploy/azure/nginx-snippets.yml new file mode 100644 index 00000000..fc38751f --- /dev/null +++ b/deploy/azure/nginx-snippets.yml @@ -0,0 +1,24 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: nginx-snippets + namespace: atat +data: + ssl.conf: |- + # Guard against HTTPS -> HTTP downgrade + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains; always"; + # Set SSL protocols, ciphers, and related options + ssl_protocols TLSv1.3 TLSv1.2; + ssl_ciphers 'TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384' + ssl_prefer_server_ciphers on; + ssl_ecdh_curve X25519:prime256v1:secp384r1; + ssl_dhparam /etc/ssl/dhparam.pem; + # SSL session options + ssl_session_timeout 4h; + ssl_session_cache shared:SSL:10m; # 1mb = ~4000 sessions + ssl_session_tickets off; + # OCSP Stapling + ssl_stapling on; + ssl_stapling_verify on; + resolver 8.8.8.8 8.8.4.4; From 6acc085a771803353af9f75882afd983cd6f178b Mon Sep 17 00:00:00 2001 From: tomdds Date: Mon, 25 Nov 2019 14:03:41 -0500 Subject: [PATCH 03/16] Use dhparam.pem from AZ Key Vault --- deploy/azure/azure.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/deploy/azure/azure.yml b/deploy/azure/azure.yml index 4ed180fb..58491c9c 100644 --- a/deploy/azure/azure.yml +++ b/deploy/azure/azure.yml @@ -23,6 +23,7 @@ spec: labels: app: atst role: web + aadpodidbinding: atat-kv-id-binding spec: securityContext: fsGroup: 101 @@ -76,6 +77,9 @@ spec: mountPath: "/usr/share/nginx/html/.well-known/acme-challenge/" - name: snippets mountPath: "/etc/nginx/snippets/" + - name: nginx-dhparam-secret + mountPath: "/etc/ssl/" + readOnly: true volumes: - name: atst-config secret: @@ -136,6 +140,16 @@ spec: - name: snippets configMap: name: nginx-snippets + - name: nginx-dhparam-secret + flexVolume: + driver: "azure/kv" + options: + usepodidentity: "true" + keyvaultname: "atat-vault-test" + keyvaultobjectnames: "dhparam4096" + keyvaultobjectaliases: "dhparam.pem" + keyvaultobjecttypes: secret + tenantid: "b5ab0e1e-09f8-4258-afb7-fb17654bc5b3" --- apiVersion: extensions/v1beta1 kind: Deployment From 949ffa294dd8f0bba586967548fe03aa1d4279d5 Mon Sep 17 00:00:00 2001 From: tomdds Date: Mon, 25 Nov 2019 14:38:56 -0500 Subject: [PATCH 04/16] Use a single FlexVolume for nginx secrets Just a name update for now, but we'll use the one flex volume to mount all the nginx related secrets going forward. --- deploy/azure/azure.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/azure/azure.yml b/deploy/azure/azure.yml index 58491c9c..0f0033c5 100644 --- a/deploy/azure/azure.yml +++ b/deploy/azure/azure.yml @@ -77,7 +77,7 @@ spec: mountPath: "/usr/share/nginx/html/.well-known/acme-challenge/" - name: snippets mountPath: "/etc/nginx/snippets/" - - name: nginx-dhparam-secret + - name: nginx-secret mountPath: "/etc/ssl/" readOnly: true volumes: @@ -140,7 +140,7 @@ spec: - name: snippets configMap: name: nginx-snippets - - name: nginx-dhparam-secret + - name: nginx-secret flexVolume: driver: "azure/kv" options: From 9469d1ff1b99ba64b89a94d373aa3ad508ec607f Mon Sep 17 00:00:00 2001 From: tomdds Date: Mon, 25 Nov 2019 14:40:10 -0500 Subject: [PATCH 05/16] Introduce TEMPLATE_ID variable for FlexVolume FlexVolume requires you specify the tenant id of the key vault instance, so this will need to be templated in for future enviroments --- deploy/README.md | 1 + deploy/azure/azure.yml | 2 +- script/k8s_config | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/deploy/README.md b/deploy/README.md index be66290d..731d07d7 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -14,6 +14,7 @@ The production configuration (azure.atat.code.mil, currently) is reflected in th - AUTH_DOMAIN: The host domain for the authentication endpoint for the environment. - KV_MI_ID: the fully qualified id (path) of the managed identity for the key vault (instructions on retrieving this are down in section on [Setting up FlexVol](#configuring-the-identity)). Example: /subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/RESOURCE_GROUP_NAME/providers/Microsoft.ManagedIdentity/userAssignedIdentities/MANAGED_IDENTITY_NAME - KV_MI_CLIENT_ID: The client id of the managed identity for the key vault. This is a GUID. +- TENANT_ID: The id of the active directory tenant in which the cluster and it's associated users exist. This is a GUID. We use envsubst to substitute values for these variables. There is a wrapper script (script/k8s_config) that will output the compiled configuration, using a combination of kustomize and envsubst. diff --git a/deploy/azure/azure.yml b/deploy/azure/azure.yml index 0f0033c5..ddbbfe18 100644 --- a/deploy/azure/azure.yml +++ b/deploy/azure/azure.yml @@ -149,7 +149,7 @@ spec: keyvaultobjectnames: "dhparam4096" keyvaultobjectaliases: "dhparam.pem" keyvaultobjecttypes: secret - tenantid: "b5ab0e1e-09f8-4258-afb7-fb17654bc5b3" + tenantid: $TENANT_ID --- apiVersion: extensions/v1beta1 kind: Deployment diff --git a/script/k8s_config b/script/k8s_config index ee3c9878..b489c942 100755 --- a/script/k8s_config +++ b/script/k8s_config @@ -13,6 +13,7 @@ SETTINGS=( AUTH_DOMAIN KV_MI_ID KV_MI_CLIENT_ID + TENANT_ID ) # Loop all expected settings. Track ones that are missing and build From 1c4e00e9142468d18391ba11cb4abe3ab7ff8c88 Mon Sep 17 00:00:00 2001 From: tomdds Date: Mon, 25 Nov 2019 15:01:13 -0500 Subject: [PATCH 06/16] Update Deploy Readme for FlexVol consumption Explain via example how you can use FlexVol to mount secrets in our containers. --- deploy/README.md | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/deploy/README.md b/deploy/README.md index 731d07d7..6f8c4cf1 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -218,3 +218,45 @@ Example values: 5. The file `deploy/azure/aadpodidentity.yml` is templated via Kustomize, so you'll need to include clientId (as `KV_MI_CLIENT_ID`) and id (as `KV_MI_ID`) of the managed identity as part of the call to Kustomize. +## Using the FlexVol + +There are 3 steps to using the FlexVol to access secrets from KeyVault + +1. For the resource in which you would like to mount a FlexVol, add a metadata label with the selector from `aadpodidentity.yml` + ``` + metadata: + labels: + app: atst + role: web + aadpodidbinding: atat-kv-id-binding + ``` + +2. Register the FlexVol as a mount and specifiy which secrets you want to mount, along with the file name they should have. The `keyvaultobjectnames`, `keyvaultobjectaliases`, and `keyvaultobjecttypes` correspond to one another, positionally. They are passed as semicolon delimited strings, examples below. + + ``` + - name: volume-of-secrets + flexVolume: + driver: "azure/kv" + options: + usepodidentity: "true" + keyvaultname: "" + keyvaultobjectnames: "mysecret;mykey;mycert" + keyvaultobjectaliases: "mysecret.pem;mykey.txt;mycert.crt" + keyvaultobjecttypes: "secret;key;cert" + tenantid: $TENANT_ID + ``` + +3. Tell the resource where to mount your new volume, using the same name that you specified for the volume above. + ``` + - name: nginx-secret + mountPath: "/usr/secrets/" + readOnly: true + ``` + +4. Once applied, the directory specified in the `mountPath` argument will contain the files you specified in the flexVolume. In our case, you would be able to do this: + ``` + $ kubectl exec -it CONTAINER_NAME -c atst ls /usr/secrets + mycert.crt + mykey.txt + mysecret.pem + ``` From 9b8d5e36626f989c11b468eb38b7f321be1a7923 Mon Sep 17 00:00:00 2001 From: tomdds Date: Tue, 26 Nov 2019 11:03:53 -0500 Subject: [PATCH 07/16] Document generation and updating of dhparams. --- deploy/README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/deploy/README.md b/deploy/README.md index 6f8c4cf1..25380293 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -170,6 +170,12 @@ Then: kubectl -n atat create secret tls azure-atat-code-mil-tls --key="[path to the private key]" --cert="[path to the full chain]" ``` +### Create the Diffie-Hellman parameters + +Diffie-Hellman parameters allow per-session encryption of SSL traffic to help improve security. We currently store our parameters in KeyVault, the value can be updated using the following command. Note: Generating the new paramter can take over 10 minutes and there won't be any output while it's running. +``` +az keyvault secret set --vault-name --name --value "$(openssl genpkey -genparam -algorithm DH -outform pem -pkeyopt dh_paramgen_prime_len:4096 2> /dev/null)" +``` --- # Setting Up FlexVol for Secrets From 26bb2f46141a5c25a83b0fe92561bc891f4bb4db Mon Sep 17 00:00:00 2001 From: tomdds Date: Tue, 26 Nov 2019 17:01:16 -0500 Subject: [PATCH 08/16] Use mounted all-in-one cert for nginx ssl Mount the combined key and cert for nginx ssl using flexvol and point the necessary nginx config at it. --- deploy/azure/atst-nginx-configmap.yml | 8 ++++---- deploy/azure/azure.yml | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/deploy/azure/atst-nginx-configmap.yml b/deploy/azure/atst-nginx-configmap.yml index b702924c..77de51f9 100644 --- a/deploy/azure/atst-nginx-configmap.yml +++ b/deploy/azure/atst-nginx-configmap.yml @@ -39,8 +39,8 @@ data: # access_log /var/log/nginx/access.log json; listen ${PORT_PREFIX}442 ssl; listen [::]:${PORT_PREFIX}442 ssl ipv6only=on; - ssl_certificate /etc/ssl/private/atat.crt; - ssl_certificate_key /etc/ssl/private/atat.key; + ssl_certificate /etc/ssl/atat.crt; + ssl_certificate_key /etc/ssl/atat.crt; # additional SSL/TLS settings include /etc/nginx/snippets/ssl.conf @@ -72,8 +72,8 @@ data: server_name ${AUTH_DOMAIN}; listen ${PORT_PREFIX}443 ssl; listen [::]:${PORT_PREFIX}443 ssl ipv6only=on; - ssl_certificate /etc/ssl/private/atat.crt; - ssl_certificate_key /etc/ssl/private/atat.key; + ssl_certificate /etc/ssl/atat.crt; + ssl_certificate_key /etc/ssl/atat.crt; # Request and validate client certificate ssl_verify_client on; ssl_verify_depth 10; diff --git a/deploy/azure/azure.yml b/deploy/azure/azure.yml index ddbbfe18..e391fc54 100644 --- a/deploy/azure/azure.yml +++ b/deploy/azure/azure.yml @@ -146,8 +146,8 @@ spec: options: usepodidentity: "true" keyvaultname: "atat-vault-test" - keyvaultobjectnames: "dhparam4096" - keyvaultobjectaliases: "dhparam.pem" + keyvaultobjectnames: "dhparam4096;staging-cert" + keyvaultobjectaliases: "dhparam.pem;atat.crt" keyvaultobjecttypes: secret tenantid: $TENANT_ID --- From 221e9ab26b77c39cb4e03fbf2a9371ca879ad1c1 Mon Sep 17 00:00:00 2001 From: tomdds Date: Tue, 26 Nov 2019 17:02:04 -0500 Subject: [PATCH 09/16] Add a staging overlay for the key vault name Currently we're just using the test vault, but in the future we want to be able to prescribe vault names for different environments via overlay. --- deploy/overlays/staging/kustomization.yaml | 1 + deploy/overlays/staging/vault_name.yml | 12 ++++++++++++ 2 files changed, 13 insertions(+) create mode 100644 deploy/overlays/staging/vault_name.yml diff --git a/deploy/overlays/staging/kustomization.yaml b/deploy/overlays/staging/kustomization.yaml index 83450cf5..51d34b15 100644 --- a/deploy/overlays/staging/kustomization.yaml +++ b/deploy/overlays/staging/kustomization.yaml @@ -7,6 +7,7 @@ patchesStrategicMerge: - replica_count.yml - ports.yml - envvars.yml + - vault_name.yml patchesJson6902: - target: group: extensions diff --git a/deploy/overlays/staging/vault_name.yml b/deploy/overlays/staging/vault_name.yml new file mode 100644 index 00000000..fa364270 --- /dev/null +++ b/deploy/overlays/staging/vault_name.yml @@ -0,0 +1,12 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: atst +spec: + template: + spec: + volumes: + - name: nginx-secret + flexVolume: + options: + keyvaultname: "atat-vault-test" From 36406372e3d464ccb7acbbd7a2268203e3d701e4 Mon Sep 17 00:00:00 2001 From: tomdds Date: Mon, 2 Dec 2019 11:41:56 -0500 Subject: [PATCH 10/16] Remove unused secret volume for tls key and cert --- deploy/azure/azure.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/deploy/azure/azure.yml b/deploy/azure/azure.yml index e391fc54..99eb847d 100644 --- a/deploy/azure/azure.yml +++ b/deploy/azure/azure.yml @@ -105,16 +105,6 @@ spec: - key: htpasswd path: .htpasswd mode: 0640 - - name: tls - secret: - secretName: azure-atat-code-mil-tls - items: - - key: tls.crt - path: atat.crt - mode: 0644 - - key: tls.key - path: atat.key - mode: 0640 - name: crls-vol persistentVolumeClaim: claimName: crls-vol-claim From 253ddaa49edcebe5ec2d3225ad67643e395418e8 Mon Sep 17 00:00:00 2001 From: tomdds Date: Mon, 2 Dec 2019 11:42:36 -0500 Subject: [PATCH 11/16] Properly register key vault object types --- deploy/azure/azure.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/azure/azure.yml b/deploy/azure/azure.yml index 99eb847d..5859d94a 100644 --- a/deploy/azure/azure.yml +++ b/deploy/azure/azure.yml @@ -138,7 +138,7 @@ spec: keyvaultname: "atat-vault-test" keyvaultobjectnames: "dhparam4096;staging-cert" keyvaultobjectaliases: "dhparam.pem;atat.crt" - keyvaultobjecttypes: secret + keyvaultobjecttypes: "secret;cert" tenantid: $TENANT_ID --- apiVersion: extensions/v1beta1 From 33ce02d0458b090c9cd6f0da7ea8cbec127a47fd Mon Sep 17 00:00:00 2001 From: tomdds Date: Mon, 2 Dec 2019 11:43:26 -0500 Subject: [PATCH 12/16] Better differentiate between master and staging vault config via overlay --- deploy/azure/azure.yml | 2 +- deploy/overlays/staging/{vault_name.yml => flex_vol.yml} | 1 + deploy/overlays/staging/kustomization.yaml | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) rename deploy/overlays/staging/{vault_name.yml => flex_vol.yml} (78%) diff --git a/deploy/azure/azure.yml b/deploy/azure/azure.yml index 5859d94a..c8c498c4 100644 --- a/deploy/azure/azure.yml +++ b/deploy/azure/azure.yml @@ -136,7 +136,7 @@ spec: options: usepodidentity: "true" keyvaultname: "atat-vault-test" - keyvaultobjectnames: "dhparam4096;staging-cert" + keyvaultobjectnames: "dhparam4096;master-cert" keyvaultobjectaliases: "dhparam.pem;atat.crt" keyvaultobjecttypes: "secret;cert" tenantid: $TENANT_ID diff --git a/deploy/overlays/staging/vault_name.yml b/deploy/overlays/staging/flex_vol.yml similarity index 78% rename from deploy/overlays/staging/vault_name.yml rename to deploy/overlays/staging/flex_vol.yml index fa364270..ef8ca168 100644 --- a/deploy/overlays/staging/vault_name.yml +++ b/deploy/overlays/staging/flex_vol.yml @@ -10,3 +10,4 @@ spec: flexVolume: options: keyvaultname: "atat-vault-test" + keyvaultobjectnames: "dhparam4096;staging-cert" diff --git a/deploy/overlays/staging/kustomization.yaml b/deploy/overlays/staging/kustomization.yaml index 51d34b15..38251002 100644 --- a/deploy/overlays/staging/kustomization.yaml +++ b/deploy/overlays/staging/kustomization.yaml @@ -7,7 +7,7 @@ patchesStrategicMerge: - replica_count.yml - ports.yml - envvars.yml - - vault_name.yml + - flex_vol.yml patchesJson6902: - target: group: extensions From 5006945cfe73d8880a874b3650dfff018a7d8678 Mon Sep 17 00:00:00 2001 From: tomdds Date: Mon, 2 Dec 2019 14:01:35 -0500 Subject: [PATCH 13/16] Remove tls volumeMount --- deploy/azure/azure.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/deploy/azure/azure.yml b/deploy/azure/azure.yml index c8c498c4..e8a9eacb 100644 --- a/deploy/azure/azure.yml +++ b/deploy/azure/azure.yml @@ -69,8 +69,6 @@ spec: - name: nginx-htpasswd mountPath: "/etc/nginx/.htpasswd" subPath: .htpasswd - - name: tls - mountPath: "/etc/ssl/private" - name: nginx-client-ca-bundle mountPath: "/etc/ssl/" - name: acme From 9ac52493080d84ac7b18261a1305907d08bd95bd Mon Sep 17 00:00:00 2001 From: tomdds Date: Mon, 2 Dec 2019 15:38:15 -0500 Subject: [PATCH 14/16] Add .env files to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 19f4acc5..d8e2290d 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,7 @@ static/buildinfo.* log/* config/dev.ini +.env* # CRLs /crl From df6ab4a01626af341bf2e7df3ade1d0b39a8528b Mon Sep 17 00:00:00 2001 From: tomdds Date: Mon, 2 Dec 2019 15:40:25 -0500 Subject: [PATCH 15/16] Fix some formatting problems in nginx configs --- deploy/azure/atst-nginx-configmap.yml | 4 ++-- deploy/azure/nginx-snippets.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy/azure/atst-nginx-configmap.yml b/deploy/azure/atst-nginx-configmap.yml index 77de51f9..b7959845 100644 --- a/deploy/azure/atst-nginx-configmap.yml +++ b/deploy/azure/atst-nginx-configmap.yml @@ -42,7 +42,7 @@ data: ssl_certificate /etc/ssl/atat.crt; ssl_certificate_key /etc/ssl/atat.crt; # additional SSL/TLS settings - include /etc/nginx/snippets/ssl.conf + include /etc/nginx/snippets/ssl.conf; location /login-redirect { return 301 https://auth-azure.atat.code.mil$request_uri; @@ -79,7 +79,7 @@ data: ssl_verify_depth 10; ssl_client_certificate /etc/ssl/client-ca-bundle.pem; # additional SSL/TLS settings - include /etc/nginx/snippets/ssl.conf + include /etc/nginx/snippets/ssl.conf; location / { return 301 https://azure.atat.code.mil$request_uri; diff --git a/deploy/azure/nginx-snippets.yml b/deploy/azure/nginx-snippets.yml index fc38751f..916d9524 100644 --- a/deploy/azure/nginx-snippets.yml +++ b/deploy/azure/nginx-snippets.yml @@ -10,7 +10,7 @@ data: add_header Strict-Transport-Security "max-age=31536000; includeSubDomains; always"; # Set SSL protocols, ciphers, and related options ssl_protocols TLSv1.3 TLSv1.2; - ssl_ciphers 'TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384' + ssl_ciphers TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384; ssl_prefer_server_ciphers on; ssl_ecdh_curve X25519:prime256v1:secp384r1; ssl_dhparam /etc/ssl/dhparam.pem; From 728bb5713f2d1d814f4f1dc5d2ab61b375f51ebb Mon Sep 17 00:00:00 2001 From: tomdds Date: Mon, 2 Dec 2019 15:41:46 -0500 Subject: [PATCH 16/16] Fix flexVol serving of nginx certificates FlexVol requires that you specify certificates as secrets in order to get both the certificate and private key in the appropriate format for nginx to consume. Additionally, flexvol shouldn't interfer with other secrets mounted in it's host directory. --- deploy/azure/atst-nginx-configmap.yml | 4 ++-- deploy/azure/azure.yml | 15 +++++++++------ deploy/overlays/staging/flex_vol.yml | 2 +- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/deploy/azure/atst-nginx-configmap.yml b/deploy/azure/atst-nginx-configmap.yml index b7959845..5f51c7d6 100644 --- a/deploy/azure/atst-nginx-configmap.yml +++ b/deploy/azure/atst-nginx-configmap.yml @@ -40,7 +40,7 @@ data: listen ${PORT_PREFIX}442 ssl; listen [::]:${PORT_PREFIX}442 ssl ipv6only=on; ssl_certificate /etc/ssl/atat.crt; - ssl_certificate_key /etc/ssl/atat.crt; + ssl_certificate_key /etc/ssl/atat.key; # additional SSL/TLS settings include /etc/nginx/snippets/ssl.conf; @@ -73,7 +73,7 @@ data: listen ${PORT_PREFIX}443 ssl; listen [::]:${PORT_PREFIX}443 ssl ipv6only=on; ssl_certificate /etc/ssl/atat.crt; - ssl_certificate_key /etc/ssl/atat.crt; + ssl_certificate_key /etc/ssl/atat.key; # Request and validate client certificate ssl_verify_client on; ssl_verify_depth 10; diff --git a/deploy/azure/azure.yml b/deploy/azure/azure.yml index e8a9eacb..02952029 100644 --- a/deploy/azure/azure.yml +++ b/deploy/azure/azure.yml @@ -70,14 +70,14 @@ spec: mountPath: "/etc/nginx/.htpasswd" subPath: .htpasswd - name: nginx-client-ca-bundle - mountPath: "/etc/ssl/" + mountPath: "/etc/ssl/client-ca-bundle.pem" + subPath: "client-ca-bundle.pem" - name: acme mountPath: "/usr/share/nginx/html/.well-known/acme-challenge/" - name: snippets mountPath: "/etc/nginx/snippets/" - name: nginx-secret mountPath: "/etc/ssl/" - readOnly: true volumes: - name: atst-config secret: @@ -89,7 +89,10 @@ spec: - name: nginx-client-ca-bundle configMap: name: nginx-client-ca-bundle - defaultMode: 0666 + defaultMode: 0444 + items: + - key: "client-ca-bundle.pem" + path: "client-ca-bundle.pem" - name: nginx-config configMap: name: atst-nginx @@ -134,9 +137,9 @@ spec: options: usepodidentity: "true" keyvaultname: "atat-vault-test" - keyvaultobjectnames: "dhparam4096;master-cert" - keyvaultobjectaliases: "dhparam.pem;atat.crt" - keyvaultobjecttypes: "secret;cert" + keyvaultobjectnames: "dhparam4096;master-cert;master-cert" + keyvaultobjectaliases: "dhparam.pem;atat.key;atat.crt" + keyvaultobjecttypes: "secret;secret;secret" tenantid: $TENANT_ID --- apiVersion: extensions/v1beta1 diff --git a/deploy/overlays/staging/flex_vol.yml b/deploy/overlays/staging/flex_vol.yml index ef8ca168..0ebeea84 100644 --- a/deploy/overlays/staging/flex_vol.yml +++ b/deploy/overlays/staging/flex_vol.yml @@ -10,4 +10,4 @@ spec: flexVolume: options: keyvaultname: "atat-vault-test" - keyvaultobjectnames: "dhparam4096;staging-cert" + keyvaultobjectnames: "dhparam4096;staging-cert;staging-cert"