From b1cf89051a2d74bb1567c12906b626b0ff4ff267 Mon Sep 17 00:00:00 2001 From: dandds Date: Mon, 5 Aug 2019 09:39:47 -0400 Subject: [PATCH] Add kubernetes config and scripts for syncing CRLs. This adds a previous version of the CRL sync functionality back to the repo, with some small adjustments. We now grab the CRLs directly from their DISA URLs. The CRL sync is handled by a kubernetes cronjob that sync the files to a persistent volume that is mounted into each Flask app container. --- atst/domain/authnid/crl/util.py | 143 ++++++++++++++++++++++++++++++++ k8s/azure/azure.yml | 6 ++ k8s/azure/storage-class.yml | 46 ++++++++++ k8s/shared/crls-sync.yaml | 42 ++++++++++ script/sync-crls | 20 ++--- 5 files changed, 245 insertions(+), 12 deletions(-) create mode 100644 atst/domain/authnid/crl/util.py create mode 100644 k8s/azure/storage-class.yml create mode 100644 k8s/shared/crls-sync.yaml diff --git a/atst/domain/authnid/crl/util.py b/atst/domain/authnid/crl/util.py new file mode 100644 index 00000000..7b32a86e --- /dev/null +++ b/atst/domain/authnid/crl/util.py @@ -0,0 +1,143 @@ +import requests +import re +import os +import pendulum +from html.parser import HTMLParser + +MODIFIED_TIME_BUFFER = 15 * 60 + + +CRL_LIST = [ + "http://crl.disa.mil/crl/DODROOTCA2.crl", + "http://crl.disa.mil/crl/DODROOTCA3.crl", + "http://crl.disa.mil/crl/DODROOTCA4.crl", + "http://crl.disa.mil/crl/DODROOTCA5.crl", + "http://crl.disa.mil/crl/DODIDCA_33.crl", + "http://crl.disa.mil/crl/DODIDCA_34.crl", + "http://crl.disa.mil/crl/DODIDSWCA_35.crl", + "http://crl.disa.mil/crl/DODIDSWCA_36.crl", + "http://crl.disa.mil/crl/DODIDSWCA_37.crl", + "http://crl.disa.mil/crl/DODIDSWCA_38.crl", + "http://crl.disa.mil/crl/DODIDCA_39.crl", + "http://crl.disa.mil/crl/DODIDCA_40.crl", + "http://crl.disa.mil/crl/DODIDCA_41.crl", + "http://crl.disa.mil/crl/DODIDCA_42.crl", + "http://crl.disa.mil/crl/DODIDCA_43.crl", + "http://crl.disa.mil/crl/DODIDCA_44.crl", + "http://crl.disa.mil/crl/DODIDSWCA_45.crl", + "http://crl.disa.mil/crl/DODIDSWCA_46.crl", + "http://crl.disa.mil/crl/DODIDSWCA_47.crl", + "http://crl.disa.mil/crl/DODIDSWCA_48.crl", + "http://crl.disa.mil/crl/DODIDCA_49.crl", + "http://crl.disa.mil/crl/DODIDCA_50.crl", + "http://crl.disa.mil/crl/DODIDCA_51.crl", + "http://crl.disa.mil/crl/DODIDCA_52.crl", + "http://crl.disa.mil/crl/DODIDCA_59.crl", + "http://crl.disa.mil/crl/DODSWCA_53.crl", + "http://crl.disa.mil/crl/DODSWCA_54.crl", + "http://crl.disa.mil/crl/DODSWCA_55.crl", + "http://crl.disa.mil/crl/DODSWCA_56.crl", + "http://crl.disa.mil/crl/DODSWCA_57.crl", + "http://crl.disa.mil/crl/DODSWCA_58.crl", + "http://crl.disa.mil/crl/DODSWCA_60.crl", + "http://crl.disa.mil/crl/DODSWCA_61.crl", + "http://crl.disa.mil/crl/DODEMAILCA_33.crl", + "http://crl.disa.mil/crl/DODEMAILCA_34.crl", + "http://crl.disa.mil/crl/DODEMAILCA_39.crl", + "http://crl.disa.mil/crl/DODEMAILCA_40.crl", + "http://crl.disa.mil/crl/DODEMAILCA_41.crl", + "http://crl.disa.mil/crl/DODEMAILCA_42.crl", + "http://crl.disa.mil/crl/DODEMAILCA_43.crl", + "http://crl.disa.mil/crl/DODEMAILCA_44.crl", + "http://crl.disa.mil/crl/DODEMAILCA_49.crl", + "http://crl.disa.mil/crl/DODEMAILCA_50.crl", + "http://crl.disa.mil/crl/DODEMAILCA_51.crl", + "http://crl.disa.mil/crl/DODEMAILCA_52.crl", + "http://crl.disa.mil/crl/DODEMAILCA_59.crl", + "http://crl.disa.mil/crl/DODINTEROPERABILITYROOTCA1.crl ", + "http://crl.disa.mil/crl/DODINTEROPERABILITYROOTCA2.crl ", + "http://crl.disa.mil/crl/USDODCCEBINTEROPERABILITYROOTCA1.crl ", + "http://crl.disa.mil/crl/USDODCCEBINTEROPERABILITYROOTCA2.crl", + "http://crl.disa.mil/crl/DODNIPRINTERNALNPEROOTCA1.crl", + "http://crl.disa.mil/crl/DODNPEROOTCA1.crl", + "http://crl.disa.mil/crl/DMDNSIGNINGCA_1.crl", + "http://crl.disa.mil/crl/DODWCFROOTCA1.crl", +] + + +def crl_local_path(out_dir, crl_location): + name = re.split("/", crl_location)[-1] + crl = os.path.join(out_dir, name) + return crl + + +def existing_crl_modification_time(crl): + if os.path.exists(crl): + prev_time = os.path.getmtime(crl) + buffered = prev_time + MODIFIED_TIME_BUFFER + mod_time = prev_time if pendulum.now().timestamp() < buffered else buffered + dt = pendulum.from_timestamp(mod_time, tz="GMT") + return dt.format("ddd, DD MMM YYYY HH:mm:ss zz") + + else: + return False + + +def write_crl(out_dir, target_dir, crl_location): + crl = crl_local_path(out_dir, crl_location) + existing = crl_local_path(target_dir, crl_location) + options = {"stream": True} + mod_time = existing_crl_modification_time(existing) + if mod_time: + options["headers"] = {"If-Modified-Since": mod_time} + + with requests.get(crl_location, **options) as response: + if response.status_code == 304: + return False + + with open(crl, "wb") as crl_file: + for chunk in response.iter_content(chunk_size=1024): + if chunk: + crl_file.write(chunk) + + return True + + +def remove_bad_crl(out_dir, crl_location): + crl = crl_local_path(out_dir, crl_location) + os.remove(crl) + + +def refresh_crls(out_dir, target_dir, logger): + for crl_location in CRL_LIST: + logger.info("updating CRL from {}".format(crl_location)) + try: + if write_crl(out_dir, target_dir, crl_location): + logger.info("successfully synced CRL from {}".format(crl_location)) + else: + logger.info("no updates for CRL from {}".format(crl_location)) + except requests.exceptions.ChunkedEncodingError: + if logger: + logger.error( + "Error downloading {}, removing file and continuing anyway".format( + crl_location + ) + ) + remove_bad_crl(out_dir, crl_location) + + +if __name__ == "__main__": + import sys + import logging + + logging.basicConfig( + level=logging.INFO, format="[%(asctime)s]:%(levelname)s: %(message)s" + ) + logger = logging.getLogger() + logger.info("Updating CRLs") + try: + refresh_crls(sys.argv[1], sys.argv[2], logger) + except Exception as err: + logger.exception("Fatal error encountered, stopping") + sys.exit(1) + logger.info("Finished updating CRLs") diff --git a/k8s/azure/azure.yml b/k8s/azure/azure.yml index 097b1e15..ea645df5 100644 --- a/k8s/azure/azure.yml +++ b/k8s/azure/azure.yml @@ -44,6 +44,8 @@ spec: subPath: client-ca-bundle.pem - name: uwsgi-socket-dir mountPath: "/var/run/uwsgi" + - name: crls-vol + mountPath: "/opt/atat/atst/crls" - name: nginx image: nginx:alpine ports: @@ -109,6 +111,10 @@ spec: - key: tls.key path: atat.key mode: 0640 + - name: crls-vol + persistentVolumeClaim: + claimName: crls-vol-claim + --- apiVersion: extensions/v1beta1 kind: Deployment diff --git a/k8s/azure/storage-class.yml b/k8s/azure/storage-class.yml new file mode 100644 index 00000000..fba2908a --- /dev/null +++ b/k8s/azure/storage-class.yml @@ -0,0 +1,46 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: azurefile +provisioner: kubernetes.io/azure-file +mountOptions: + - dir_mode=0777 + - file_mode=0777 + - uid=1000 + - gid=1000 +parameters: + skuName: Standard_LRS +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:azure-cloud-provider +rules: +- apiGroups: [''] + resources: ['secrets'] + verbs: ['get','create'] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:azure-cloud-provider +roleRef: + kind: ClusterRole + apiGroup: rbac.authorization.k8s.io + name: system:azure-cloud-provider +subjects: +- kind: ServiceAccount + name: persistent-volume-binder + namespace: kube-system +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: crls-vol-claim +spec: + accessModes: + - ReadWriteMany + storageClassName: azurefile + resources: + requests: + storage: 1Gi diff --git a/k8s/shared/crls-sync.yaml b/k8s/shared/crls-sync.yaml new file mode 100644 index 00000000..541c026b --- /dev/null +++ b/k8s/shared/crls-sync.yaml @@ -0,0 +1,42 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: crls + namespace: atat +spec: + schedule: "0 * * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: crls + image: $CONTAINER_IMAGE + command: [ + "/bin/sh", "-c" + ] + args: [ + "/opt/atat/atst/script/sync-crls" + ] + envFrom: + - configMapRef: + name: atst-envvars + - configMapRef: + name: atst-worker-envvars + volumeMounts: + - name: atst-config + mountPath: "/opt/atat/atst/atst-overrides.ini" + subPath: atst-overrides.ini + - name: crls-vol + mountPath: "/opt/atat/atst/crls" + volumes: + - name: atst-config + secret: + secretName: atst-config-ini + items: + - key: override.ini + path: atst-overrides.ini + mode: 0644 + - name: crls-vol + persistentVolumeClaim: + claimName: crls-vol-claim diff --git a/script/sync-crls b/script/sync-crls index 82b6631b..7708e386 100755 --- a/script/sync-crls +++ b/script/sync-crls @@ -1,14 +1,10 @@ -#! .venv/bin/python -# Add root application dir to the python path -import os -import sys +#!/bin/bash -parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) -sys.path.append(parent_dir) +# script/sync-crls: update the DOD CRLs and place them where authnid expects them +set -e +cd "$(dirname "$0")/.." -from atst.app import make_config, make_app - -if __name__ == "__main__": - config = make_config({"DISABLE_CRL_CHECK": True}) - app = make_app(config) - app.csp.crls.sync_crls() +mkdir -p crl-tmp crls +pipenv run python ./atst/domain/authnid/crl/util.py crl-tmp crls +cp -r crl-tmp/* crls/ +rm -rf crl-tmp