diff --git a/.gitignore b/.gitignore index 19f4acc5..d8e2290d 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,7 @@ static/buildinfo.* log/* config/dev.ini +.env* # CRLs /crl diff --git a/deploy/README.md b/deploy/README.md index be66290d..25380293 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -14,6 +14,7 @@ The production configuration (azure.atat.code.mil, currently) is reflected in th - AUTH_DOMAIN: The host domain for the authentication endpoint for the environment. - KV_MI_ID: the fully qualified id (path) of the managed identity for the key vault (instructions on retrieving this are down in section on [Setting up FlexVol](#configuring-the-identity)). Example: /subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/RESOURCE_GROUP_NAME/providers/Microsoft.ManagedIdentity/userAssignedIdentities/MANAGED_IDENTITY_NAME - KV_MI_CLIENT_ID: The client id of the managed identity for the key vault. This is a GUID. +- TENANT_ID: The id of the active directory tenant in which the cluster and it's associated users exist. This is a GUID. We use envsubst to substitute values for these variables. There is a wrapper script (script/k8s_config) that will output the compiled configuration, using a combination of kustomize and envsubst. @@ -169,6 +170,12 @@ Then: kubectl -n atat create secret tls azure-atat-code-mil-tls --key="[path to the private key]" --cert="[path to the full chain]" ``` +### Create the Diffie-Hellman parameters + +Diffie-Hellman parameters allow per-session encryption of SSL traffic to help improve security. We currently store our parameters in KeyVault, the value can be updated using the following command. Note: Generating the new paramter can take over 10 minutes and there won't be any output while it's running. +``` +az keyvault secret set --vault-name --name --value "$(openssl genpkey -genparam -algorithm DH -outform pem -pkeyopt dh_paramgen_prime_len:4096 2> /dev/null)" +``` --- # Setting Up FlexVol for Secrets @@ -217,3 +224,45 @@ Example values: 5. The file `deploy/azure/aadpodidentity.yml` is templated via Kustomize, so you'll need to include clientId (as `KV_MI_CLIENT_ID`) and id (as `KV_MI_ID`) of the managed identity as part of the call to Kustomize. +## Using the FlexVol + +There are 3 steps to using the FlexVol to access secrets from KeyVault + +1. For the resource in which you would like to mount a FlexVol, add a metadata label with the selector from `aadpodidentity.yml` + ``` + metadata: + labels: + app: atst + role: web + aadpodidbinding: atat-kv-id-binding + ``` + +2. Register the FlexVol as a mount and specifiy which secrets you want to mount, along with the file name they should have. The `keyvaultobjectnames`, `keyvaultobjectaliases`, and `keyvaultobjecttypes` correspond to one another, positionally. They are passed as semicolon delimited strings, examples below. + + ``` + - name: volume-of-secrets + flexVolume: + driver: "azure/kv" + options: + usepodidentity: "true" + keyvaultname: "" + keyvaultobjectnames: "mysecret;mykey;mycert" + keyvaultobjectaliases: "mysecret.pem;mykey.txt;mycert.crt" + keyvaultobjecttypes: "secret;key;cert" + tenantid: $TENANT_ID + ``` + +3. Tell the resource where to mount your new volume, using the same name that you specified for the volume above. + ``` + - name: nginx-secret + mountPath: "/usr/secrets/" + readOnly: true + ``` + +4. Once applied, the directory specified in the `mountPath` argument will contain the files you specified in the flexVolume. In our case, you would be able to do this: + ``` + $ kubectl exec -it CONTAINER_NAME -c atst ls /usr/secrets + mycert.crt + mykey.txt + mysecret.pem + ``` diff --git a/deploy/azure/atst-nginx-configmap.yml b/deploy/azure/atst-nginx-configmap.yml index e19c6d54..5f51c7d6 100644 --- a/deploy/azure/atst-nginx-configmap.yml +++ b/deploy/azure/atst-nginx-configmap.yml @@ -5,8 +5,10 @@ metadata: name: atst-nginx namespace: atat data: - nginx-config: |- + atst.conf: |- server { + access_log /var/log/nginx/access.log json; + listen ${PORT_PREFIX}342; server_name ${MAIN_DOMAIN}; root /usr/share/nginx/html; @@ -18,6 +20,8 @@ data: } } server { + access_log /var/log/nginx/access.log json; + listen ${PORT_PREFIX}343; server_name ${AUTH_DOMAIN}; root /usr/share/nginx/html; @@ -29,12 +33,17 @@ data: } } server { + access_log /var/log/nginx/access.log json; + server_name ${MAIN_DOMAIN}; # access_log /var/log/nginx/access.log json; listen ${PORT_PREFIX}442 ssl; listen [::]:${PORT_PREFIX}442 ssl ipv6only=on; - ssl_certificate /etc/ssl/private/atat.crt; - ssl_certificate_key /etc/ssl/private/atat.key; + ssl_certificate /etc/ssl/atat.crt; + ssl_certificate_key /etc/ssl/atat.key; + # additional SSL/TLS settings + include /etc/nginx/snippets/ssl.conf; + location /login-redirect { return 301 https://auth-azure.atat.code.mil$request_uri; } @@ -58,18 +67,20 @@ data: } } server { - # access_log /var/log/nginx/access.log json; + access_log /var/log/nginx/access.log json; + server_name ${AUTH_DOMAIN}; listen ${PORT_PREFIX}443 ssl; listen [::]:${PORT_PREFIX}443 ssl ipv6only=on; - ssl_certificate /etc/ssl/private/atat.crt; - ssl_certificate_key /etc/ssl/private/atat.key; + ssl_certificate /etc/ssl/atat.crt; + ssl_certificate_key /etc/ssl/atat.key; # Request and validate client certificate ssl_verify_client on; ssl_verify_depth 10; ssl_client_certificate /etc/ssl/client-ca-bundle.pem; - # Guard against HTTPS -> HTTP downgrade - add_header Strict-Transport-Security "max-age=31536000; includeSubDomains; always"; + # additional SSL/TLS settings + include /etc/nginx/snippets/ssl.conf; + location / { return 301 https://azure.atat.code.mil$request_uri; } @@ -88,3 +99,18 @@ data: uwsgi_param HTTP_X_REQUEST_ID $request_id; } } + 00json_log.conf: |- + log_format json escape=json + '{' + '"timestamp":"$time_iso8601",' + '"msec":"$msec",' + '"request_id":"$request_id",' + '"remote_addr":"$remote_addr",' + '"remote_user":"$remote_user",' + '"request":"$request",' + '"status":$status,' + '"body_bytes_sent":$body_bytes_sent,' + '"referer":"$http_referer",' + '"user_agent":"$http_user_agent",' + '"http_x_forwarded_for":"$http_x_forwarded_for"' + '}'; diff --git a/deploy/azure/azure.yml b/deploy/azure/azure.yml index 8d46fa4b..02952029 100644 --- a/deploy/azure/azure.yml +++ b/deploy/azure/azure.yml @@ -23,6 +23,7 @@ spec: labels: app: atst role: web + aadpodidbinding: atat-kv-id-binding spec: securityContext: fsGroup: 101 @@ -30,8 +31,8 @@ spec: - name: atst image: $CONTAINER_IMAGE envFrom: - - configMapRef: - name: atst-envvars + - configMapRef: + name: atst-envvars volumeMounts: - name: atst-config mountPath: "/opt/atat/atst/atst-overrides.ini" @@ -62,37 +63,39 @@ spec: name: auth volumeMounts: - name: nginx-config - mountPath: "/etc/nginx/conf.d/atst.conf" - subPath: atst.conf + mountPath: "/etc/nginx/conf.d/" - name: uwsgi-socket-dir mountPath: "/var/run/uwsgi" - name: nginx-htpasswd mountPath: "/etc/nginx/.htpasswd" subPath: .htpasswd - - name: tls - mountPath: "/etc/ssl/private" - name: nginx-client-ca-bundle - mountPath: "/etc/ssl/" + mountPath: "/etc/ssl/client-ca-bundle.pem" + subPath: "client-ca-bundle.pem" - name: acme mountPath: "/usr/share/nginx/html/.well-known/acme-challenge/" + - name: snippets + mountPath: "/etc/nginx/snippets/" + - name: nginx-secret + mountPath: "/etc/ssl/" volumes: - name: atst-config secret: secretName: atst-config-ini items: - - key: override.ini - path: atst-overrides.ini - mode: 0644 + - key: override.ini + path: atst-overrides.ini + mode: 0644 - name: nginx-client-ca-bundle configMap: name: nginx-client-ca-bundle - defaultMode: 0666 + defaultMode: 0444 + items: + - key: "client-ca-bundle.pem" + path: "client-ca-bundle.pem" - name: nginx-config configMap: name: atst-nginx - items: - - key: nginx-config - path: atst.conf - name: uwsgi-socket-dir emptyDir: medium: Memory @@ -100,19 +103,9 @@ spec: secret: secretName: atst-nginx-htpasswd items: - - key: htpasswd - path: .htpasswd - mode: 0640 - - name: tls - secret: - secretName: azure-atat-code-mil-tls - items: - - key: tls.crt - path: atat.crt - mode: 0644 - - key: tls.key - path: atat.key - mode: 0640 + - key: htpasswd + path: .htpasswd + mode: 0640 - name: crls-vol persistentVolumeClaim: claimName: crls-vol-claim @@ -120,9 +113,9 @@ spec: configMap: name: pgsslrootcert items: - - key: cert - path: pgsslrootcert.crt - mode: 0666 + - key: cert + path: pgsslrootcert.crt + mode: 0666 - name: acme configMap: name: acme-challenges @@ -132,9 +125,22 @@ spec: name: uwsgi-config defaultMode: 0666 items: - - key: uwsgi.ini - path: uwsgi.ini - mode: 0644 + - key: uwsgi.ini + path: uwsgi.ini + mode: 0644 + - name: snippets + configMap: + name: nginx-snippets + - name: nginx-secret + flexVolume: + driver: "azure/kv" + options: + usepodidentity: "true" + keyvaultname: "atat-vault-test" + keyvaultobjectnames: "dhparam4096;master-cert;master-cert" + keyvaultobjectaliases: "dhparam.pem;atat.key;atat.crt" + keyvaultobjecttypes: "secret;secret;secret" + tenantid: $TENANT_ID --- apiVersion: extensions/v1beta1 kind: Deployment @@ -161,19 +167,20 @@ spec: containers: - name: atst-worker image: $CONTAINER_IMAGE - args: [ - "/opt/atat/atst/.venv/bin/python", - "/opt/atat/atst/.venv/bin/celery", - "-A", - "celery_worker.celery", - "worker", - "--loglevel=info" - ] + args: + [ + "/opt/atat/atst/.venv/bin/python", + "/opt/atat/atst/.venv/bin/celery", + "-A", + "celery_worker.celery", + "worker", + "--loglevel=info", + ] envFrom: - - configMapRef: - name: atst-envvars - - configMapRef: - name: atst-worker-envvars + - configMapRef: + name: atst-envvars + - configMapRef: + name: atst-worker-envvars volumeMounts: - name: atst-config mountPath: "/opt/atat/atst/atst-overrides.ini" @@ -186,16 +193,16 @@ spec: secret: secretName: atst-config-ini items: - - key: override.ini - path: atst-overrides.ini - mode: 0644 + - key: override.ini + path: atst-overrides.ini + mode: 0644 - name: pgsslrootcert configMap: name: pgsslrootcert items: - - key: cert - path: pgsslrootcert.crt - mode: 0666 + - key: cert + path: pgsslrootcert.crt + mode: 0666 --- apiVersion: extensions/v1beta1 kind: Deployment @@ -222,19 +229,20 @@ spec: containers: - name: atst-beat image: $CONTAINER_IMAGE - args: [ - "/opt/atat/atst/.venv/bin/python", - "/opt/atat/atst/.venv/bin/celery", - "-A", - "celery_worker.celery", - "beat", - "--loglevel=info" - ] + args: + [ + "/opt/atat/atst/.venv/bin/python", + "/opt/atat/atst/.venv/bin/celery", + "-A", + "celery_worker.celery", + "beat", + "--loglevel=info", + ] envFrom: - - configMapRef: - name: atst-envvars - - configMapRef: - name: atst-worker-envvars + - configMapRef: + name: atst-envvars + - configMapRef: + name: atst-worker-envvars volumeMounts: - name: atst-config mountPath: "/opt/atat/atst/atst-overrides.ini" @@ -247,16 +255,16 @@ spec: secret: secretName: atst-config-ini items: - - key: override.ini - path: atst-overrides.ini - mode: 0644 + - key: override.ini + path: atst-overrides.ini + mode: 0644 - name: pgsslrootcert configMap: name: pgsslrootcert items: - - key: cert - path: pgsslrootcert.crt - mode: 0666 + - key: cert + path: pgsslrootcert.crt + mode: 0666 --- apiVersion: v1 kind: Service @@ -268,12 +276,12 @@ metadata: spec: loadBalancerIP: 13.92.235.6 ports: - - port: 80 - targetPort: 8342 - name: http - - port: 443 - targetPort: 8442 - name: https + - port: 80 + targetPort: 8342 + name: http + - port: 443 + targetPort: 8442 + name: https selector: role: web type: LoadBalancer @@ -288,12 +296,12 @@ metadata: spec: loadBalancerIP: 23.100.24.41 ports: - - port: 80 - targetPort: 8343 - name: http - - port: 443 - targetPort: 8443 - name: https + - port: 80 + targetPort: 8343 + name: http + - port: 443 + targetPort: 8443 + name: https selector: role: web type: LoadBalancer diff --git a/deploy/azure/kustomization.yaml b/deploy/azure/kustomization.yaml index 43e6f813..9dee809c 100644 --- a/deploy/azure/kustomization.yaml +++ b/deploy/azure/kustomization.yaml @@ -11,3 +11,4 @@ resources: - nginx-client-ca-bundle.yml - acme-challenges.yml - aadpodidentity.yml + - nginx-snippets.yml diff --git a/deploy/azure/nginx-snippets.yml b/deploy/azure/nginx-snippets.yml new file mode 100644 index 00000000..916d9524 --- /dev/null +++ b/deploy/azure/nginx-snippets.yml @@ -0,0 +1,24 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: nginx-snippets + namespace: atat +data: + ssl.conf: |- + # Guard against HTTPS -> HTTP downgrade + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains; always"; + # Set SSL protocols, ciphers, and related options + ssl_protocols TLSv1.3 TLSv1.2; + ssl_ciphers TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384; + ssl_prefer_server_ciphers on; + ssl_ecdh_curve X25519:prime256v1:secp384r1; + ssl_dhparam /etc/ssl/dhparam.pem; + # SSL session options + ssl_session_timeout 4h; + ssl_session_cache shared:SSL:10m; # 1mb = ~4000 sessions + ssl_session_tickets off; + # OCSP Stapling + ssl_stapling on; + ssl_stapling_verify on; + resolver 8.8.8.8 8.8.4.4; diff --git a/deploy/overlays/staging/flex_vol.yml b/deploy/overlays/staging/flex_vol.yml new file mode 100644 index 00000000..0ebeea84 --- /dev/null +++ b/deploy/overlays/staging/flex_vol.yml @@ -0,0 +1,13 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: atst +spec: + template: + spec: + volumes: + - name: nginx-secret + flexVolume: + options: + keyvaultname: "atat-vault-test" + keyvaultobjectnames: "dhparam4096;staging-cert;staging-cert" diff --git a/deploy/overlays/staging/kustomization.yaml b/deploy/overlays/staging/kustomization.yaml index 83450cf5..38251002 100644 --- a/deploy/overlays/staging/kustomization.yaml +++ b/deploy/overlays/staging/kustomization.yaml @@ -7,6 +7,7 @@ patchesStrategicMerge: - replica_count.yml - ports.yml - envvars.yml + - flex_vol.yml patchesJson6902: - target: group: extensions diff --git a/script/k8s_config b/script/k8s_config index ee3c9878..b489c942 100755 --- a/script/k8s_config +++ b/script/k8s_config @@ -13,6 +13,7 @@ SETTINGS=( AUTH_DOMAIN KV_MI_ID KV_MI_CLIENT_ID + TENANT_ID ) # Loop all expected settings. Track ones that are missing and build