diff --git a/atst/domain/authnid/crl/util.py b/atst/domain/authnid/crl/util.py new file mode 100644 index 00000000..7b32a86e --- /dev/null +++ b/atst/domain/authnid/crl/util.py @@ -0,0 +1,143 @@ +import requests +import re +import os +import pendulum +from html.parser import HTMLParser + +MODIFIED_TIME_BUFFER = 15 * 60 + + +CRL_LIST = [ + "http://crl.disa.mil/crl/DODROOTCA2.crl", + "http://crl.disa.mil/crl/DODROOTCA3.crl", + "http://crl.disa.mil/crl/DODROOTCA4.crl", + "http://crl.disa.mil/crl/DODROOTCA5.crl", + "http://crl.disa.mil/crl/DODIDCA_33.crl", + "http://crl.disa.mil/crl/DODIDCA_34.crl", + "http://crl.disa.mil/crl/DODIDSWCA_35.crl", + "http://crl.disa.mil/crl/DODIDSWCA_36.crl", + "http://crl.disa.mil/crl/DODIDSWCA_37.crl", + "http://crl.disa.mil/crl/DODIDSWCA_38.crl", + "http://crl.disa.mil/crl/DODIDCA_39.crl", + "http://crl.disa.mil/crl/DODIDCA_40.crl", + "http://crl.disa.mil/crl/DODIDCA_41.crl", + "http://crl.disa.mil/crl/DODIDCA_42.crl", + "http://crl.disa.mil/crl/DODIDCA_43.crl", + "http://crl.disa.mil/crl/DODIDCA_44.crl", + "http://crl.disa.mil/crl/DODIDSWCA_45.crl", + "http://crl.disa.mil/crl/DODIDSWCA_46.crl", + "http://crl.disa.mil/crl/DODIDSWCA_47.crl", + "http://crl.disa.mil/crl/DODIDSWCA_48.crl", + "http://crl.disa.mil/crl/DODIDCA_49.crl", + "http://crl.disa.mil/crl/DODIDCA_50.crl", + "http://crl.disa.mil/crl/DODIDCA_51.crl", + "http://crl.disa.mil/crl/DODIDCA_52.crl", + "http://crl.disa.mil/crl/DODIDCA_59.crl", + "http://crl.disa.mil/crl/DODSWCA_53.crl", + "http://crl.disa.mil/crl/DODSWCA_54.crl", + "http://crl.disa.mil/crl/DODSWCA_55.crl", + "http://crl.disa.mil/crl/DODSWCA_56.crl", + "http://crl.disa.mil/crl/DODSWCA_57.crl", + "http://crl.disa.mil/crl/DODSWCA_58.crl", + "http://crl.disa.mil/crl/DODSWCA_60.crl", + "http://crl.disa.mil/crl/DODSWCA_61.crl", + "http://crl.disa.mil/crl/DODEMAILCA_33.crl", + "http://crl.disa.mil/crl/DODEMAILCA_34.crl", + "http://crl.disa.mil/crl/DODEMAILCA_39.crl", + "http://crl.disa.mil/crl/DODEMAILCA_40.crl", + "http://crl.disa.mil/crl/DODEMAILCA_41.crl", + "http://crl.disa.mil/crl/DODEMAILCA_42.crl", + "http://crl.disa.mil/crl/DODEMAILCA_43.crl", + "http://crl.disa.mil/crl/DODEMAILCA_44.crl", + "http://crl.disa.mil/crl/DODEMAILCA_49.crl", + "http://crl.disa.mil/crl/DODEMAILCA_50.crl", + "http://crl.disa.mil/crl/DODEMAILCA_51.crl", + "http://crl.disa.mil/crl/DODEMAILCA_52.crl", + "http://crl.disa.mil/crl/DODEMAILCA_59.crl", + "http://crl.disa.mil/crl/DODINTEROPERABILITYROOTCA1.crl ", + "http://crl.disa.mil/crl/DODINTEROPERABILITYROOTCA2.crl ", + "http://crl.disa.mil/crl/USDODCCEBINTEROPERABILITYROOTCA1.crl ", + "http://crl.disa.mil/crl/USDODCCEBINTEROPERABILITYROOTCA2.crl", + "http://crl.disa.mil/crl/DODNIPRINTERNALNPEROOTCA1.crl", + "http://crl.disa.mil/crl/DODNPEROOTCA1.crl", + "http://crl.disa.mil/crl/DMDNSIGNINGCA_1.crl", + "http://crl.disa.mil/crl/DODWCFROOTCA1.crl", +] + + +def crl_local_path(out_dir, crl_location): + name = re.split("/", crl_location)[-1] + crl = os.path.join(out_dir, name) + return crl + + +def existing_crl_modification_time(crl): + if os.path.exists(crl): + prev_time = os.path.getmtime(crl) + buffered = prev_time + MODIFIED_TIME_BUFFER + mod_time = prev_time if pendulum.now().timestamp() < buffered else buffered + dt = pendulum.from_timestamp(mod_time, tz="GMT") + return dt.format("ddd, DD MMM YYYY HH:mm:ss zz") + + else: + return False + + +def write_crl(out_dir, target_dir, crl_location): + crl = crl_local_path(out_dir, crl_location) + existing = crl_local_path(target_dir, crl_location) + options = {"stream": True} + mod_time = existing_crl_modification_time(existing) + if mod_time: + options["headers"] = {"If-Modified-Since": mod_time} + + with requests.get(crl_location, **options) as response: + if response.status_code == 304: + return False + + with open(crl, "wb") as crl_file: + for chunk in response.iter_content(chunk_size=1024): + if chunk: + crl_file.write(chunk) + + return True + + +def remove_bad_crl(out_dir, crl_location): + crl = crl_local_path(out_dir, crl_location) + os.remove(crl) + + +def refresh_crls(out_dir, target_dir, logger): + for crl_location in CRL_LIST: + logger.info("updating CRL from {}".format(crl_location)) + try: + if write_crl(out_dir, target_dir, crl_location): + logger.info("successfully synced CRL from {}".format(crl_location)) + else: + logger.info("no updates for CRL from {}".format(crl_location)) + except requests.exceptions.ChunkedEncodingError: + if logger: + logger.error( + "Error downloading {}, removing file and continuing anyway".format( + crl_location + ) + ) + remove_bad_crl(out_dir, crl_location) + + +if __name__ == "__main__": + import sys + import logging + + logging.basicConfig( + level=logging.INFO, format="[%(asctime)s]:%(levelname)s: %(message)s" + ) + logger = logging.getLogger() + logger.info("Updating CRLs") + try: + refresh_crls(sys.argv[1], sys.argv[2], logger) + except Exception as err: + logger.exception("Fatal error encountered, stopping") + sys.exit(1) + logger.info("Finished updating CRLs") diff --git a/k8s/azure/azure.yml b/k8s/azure/azure.yml index 097b1e15..ea645df5 100644 --- a/k8s/azure/azure.yml +++ b/k8s/azure/azure.yml @@ -44,6 +44,8 @@ spec: subPath: client-ca-bundle.pem - name: uwsgi-socket-dir mountPath: "/var/run/uwsgi" + - name: crls-vol + mountPath: "/opt/atat/atst/crls" - name: nginx image: nginx:alpine ports: @@ -109,6 +111,10 @@ spec: - key: tls.key path: atat.key mode: 0640 + - name: crls-vol + persistentVolumeClaim: + claimName: crls-vol-claim + --- apiVersion: extensions/v1beta1 kind: Deployment diff --git a/k8s/azure/storage-class.yml b/k8s/azure/storage-class.yml new file mode 100644 index 00000000..fba2908a --- /dev/null +++ b/k8s/azure/storage-class.yml @@ -0,0 +1,46 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: azurefile +provisioner: kubernetes.io/azure-file +mountOptions: + - dir_mode=0777 + - file_mode=0777 + - uid=1000 + - gid=1000 +parameters: + skuName: Standard_LRS +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:azure-cloud-provider +rules: +- apiGroups: [''] + resources: ['secrets'] + verbs: ['get','create'] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:azure-cloud-provider +roleRef: + kind: ClusterRole + apiGroup: rbac.authorization.k8s.io + name: system:azure-cloud-provider +subjects: +- kind: ServiceAccount + name: persistent-volume-binder + namespace: kube-system +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: crls-vol-claim +spec: + accessModes: + - ReadWriteMany + storageClassName: azurefile + resources: + requests: + storage: 1Gi diff --git a/k8s/shared/crls-sync.yaml b/k8s/shared/crls-sync.yaml new file mode 100644 index 00000000..541c026b --- /dev/null +++ b/k8s/shared/crls-sync.yaml @@ -0,0 +1,42 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: crls + namespace: atat +spec: + schedule: "0 * * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: crls + image: $CONTAINER_IMAGE + command: [ + "/bin/sh", "-c" + ] + args: [ + "/opt/atat/atst/script/sync-crls" + ] + envFrom: + - configMapRef: + name: atst-envvars + - configMapRef: + name: atst-worker-envvars + volumeMounts: + - name: atst-config + mountPath: "/opt/atat/atst/atst-overrides.ini" + subPath: atst-overrides.ini + - name: crls-vol + mountPath: "/opt/atat/atst/crls" + volumes: + - name: atst-config + secret: + secretName: atst-config-ini + items: + - key: override.ini + path: atst-overrides.ini + mode: 0644 + - name: crls-vol + persistentVolumeClaim: + claimName: crls-vol-claim diff --git a/script/sync-crls b/script/sync-crls index 82b6631b..7708e386 100755 --- a/script/sync-crls +++ b/script/sync-crls @@ -1,14 +1,10 @@ -#! .venv/bin/python -# Add root application dir to the python path -import os -import sys +#!/bin/bash -parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) -sys.path.append(parent_dir) +# script/sync-crls: update the DOD CRLs and place them where authnid expects them +set -e +cd "$(dirname "$0")/.." -from atst.app import make_config, make_app - -if __name__ == "__main__": - config = make_config({"DISABLE_CRL_CHECK": True}) - app = make_app(config) - app.csp.crls.sync_crls() +mkdir -p crl-tmp crls +pipenv run python ./atst/domain/authnid/crl/util.py crl-tmp crls +cp -r crl-tmp/* crls/ +rm -rf crl-tmp