Add kubernetes config and scripts for syncing CRLs.

This adds a previous version of the CRL sync functionality back to the
repo, with some small adjustments. We now grab the CRLs directly from
their DISA URLs.

The CRL sync is handled by a kubernetes cronjob that sync the files to a
persistent volume that is mounted into each Flask app container.

atst/domain/authnid/crl/util.py

@@ -0,0 +1,143 @@
+import requests
+import re
+import os
+import pendulum
+from html.parser import HTMLParser
+
+MODIFIED_TIME_BUFFER = 15 * 60
+
+
+CRL_LIST = [
+    "http://crl.disa.mil/crl/DODROOTCA2.crl",
+    "http://crl.disa.mil/crl/DODROOTCA3.crl",
+    "http://crl.disa.mil/crl/DODROOTCA4.crl",
+    "http://crl.disa.mil/crl/DODROOTCA5.crl",
+    "http://crl.disa.mil/crl/DODIDCA_33.crl",
+    "http://crl.disa.mil/crl/DODIDCA_34.crl",
+    "http://crl.disa.mil/crl/DODIDSWCA_35.crl",
+    "http://crl.disa.mil/crl/DODIDSWCA_36.crl",
+    "http://crl.disa.mil/crl/DODIDSWCA_37.crl",
+    "http://crl.disa.mil/crl/DODIDSWCA_38.crl",
+    "http://crl.disa.mil/crl/DODIDCA_39.crl",
+    "http://crl.disa.mil/crl/DODIDCA_40.crl",
+    "http://crl.disa.mil/crl/DODIDCA_41.crl",
+    "http://crl.disa.mil/crl/DODIDCA_42.crl",
+    "http://crl.disa.mil/crl/DODIDCA_43.crl",
+    "http://crl.disa.mil/crl/DODIDCA_44.crl",
+    "http://crl.disa.mil/crl/DODIDSWCA_45.crl",
+    "http://crl.disa.mil/crl/DODIDSWCA_46.crl",
+    "http://crl.disa.mil/crl/DODIDSWCA_47.crl",
+    "http://crl.disa.mil/crl/DODIDSWCA_48.crl",
+    "http://crl.disa.mil/crl/DODIDCA_49.crl",
+    "http://crl.disa.mil/crl/DODIDCA_50.crl",
+    "http://crl.disa.mil/crl/DODIDCA_51.crl",
+    "http://crl.disa.mil/crl/DODIDCA_52.crl",
+    "http://crl.disa.mil/crl/DODIDCA_59.crl",
+    "http://crl.disa.mil/crl/DODSWCA_53.crl",
+    "http://crl.disa.mil/crl/DODSWCA_54.crl",
+    "http://crl.disa.mil/crl/DODSWCA_55.crl",
+    "http://crl.disa.mil/crl/DODSWCA_56.crl",
+    "http://crl.disa.mil/crl/DODSWCA_57.crl",
+    "http://crl.disa.mil/crl/DODSWCA_58.crl",
+    "http://crl.disa.mil/crl/DODSWCA_60.crl",
+    "http://crl.disa.mil/crl/DODSWCA_61.crl",
+    "http://crl.disa.mil/crl/DODEMAILCA_33.crl",
+    "http://crl.disa.mil/crl/DODEMAILCA_34.crl",
+    "http://crl.disa.mil/crl/DODEMAILCA_39.crl",
+    "http://crl.disa.mil/crl/DODEMAILCA_40.crl",
+    "http://crl.disa.mil/crl/DODEMAILCA_41.crl",
+    "http://crl.disa.mil/crl/DODEMAILCA_42.crl",
+    "http://crl.disa.mil/crl/DODEMAILCA_43.crl",
+    "http://crl.disa.mil/crl/DODEMAILCA_44.crl",
+    "http://crl.disa.mil/crl/DODEMAILCA_49.crl",
+    "http://crl.disa.mil/crl/DODEMAILCA_50.crl",
+    "http://crl.disa.mil/crl/DODEMAILCA_51.crl",
+    "http://crl.disa.mil/crl/DODEMAILCA_52.crl",
+    "http://crl.disa.mil/crl/DODEMAILCA_59.crl",
+    "http://crl.disa.mil/crl/DODINTEROPERABILITYROOTCA1.crl ",
+    "http://crl.disa.mil/crl/DODINTEROPERABILITYROOTCA2.crl ",
+    "http://crl.disa.mil/crl/USDODCCEBINTEROPERABILITYROOTCA1.crl ",
+    "http://crl.disa.mil/crl/USDODCCEBINTEROPERABILITYROOTCA2.crl",
+    "http://crl.disa.mil/crl/DODNIPRINTERNALNPEROOTCA1.crl",
+    "http://crl.disa.mil/crl/DODNPEROOTCA1.crl",
+    "http://crl.disa.mil/crl/DMDNSIGNINGCA_1.crl",
+    "http://crl.disa.mil/crl/DODWCFROOTCA1.crl",
+]
+
+
+def crl_local_path(out_dir, crl_location):
+    name = re.split("/", crl_location)[-1]
+    crl = os.path.join(out_dir, name)
+    return crl
+
+
+def existing_crl_modification_time(crl):
+    if os.path.exists(crl):
+        prev_time = os.path.getmtime(crl)
+        buffered = prev_time + MODIFIED_TIME_BUFFER
+        mod_time = prev_time if pendulum.now().timestamp() < buffered else buffered
+        dt = pendulum.from_timestamp(mod_time, tz="GMT")
+        return dt.format("ddd, DD MMM YYYY HH:mm:ss zz")
+
+    else:
+        return False
+
+
+def write_crl(out_dir, target_dir, crl_location):
+    crl = crl_local_path(out_dir, crl_location)
+    existing = crl_local_path(target_dir, crl_location)
+    options = {"stream": True}
+    mod_time = existing_crl_modification_time(existing)
+    if mod_time:
+        options["headers"] = {"If-Modified-Since": mod_time}
+
+    with requests.get(crl_location, **options) as response:
+        if response.status_code == 304:
+            return False
+
+        with open(crl, "wb") as crl_file:
+            for chunk in response.iter_content(chunk_size=1024):
+                if chunk:
+                    crl_file.write(chunk)
+
+    return True
+
+
+def remove_bad_crl(out_dir, crl_location):
+    crl = crl_local_path(out_dir, crl_location)
+    os.remove(crl)
+
+
+def refresh_crls(out_dir, target_dir, logger):
+    for crl_location in CRL_LIST:
+        logger.info("updating CRL from {}".format(crl_location))
+        try:
+            if write_crl(out_dir, target_dir, crl_location):
+                logger.info("successfully synced CRL from {}".format(crl_location))
+            else:
+                logger.info("no updates for CRL from {}".format(crl_location))
+        except requests.exceptions.ChunkedEncodingError:
+            if logger:
+                logger.error(
+                    "Error downloading {}, removing file and continuing anyway".format(
+                        crl_location
+                    )
+                )
+            remove_bad_crl(out_dir, crl_location)
+
+
+if __name__ == "__main__":
+    import sys
+    import logging
+
+    logging.basicConfig(
+        level=logging.INFO, format="[%(asctime)s]:%(levelname)s: %(message)s"
+    )
+    logger = logging.getLogger()
+    logger.info("Updating CRLs")
+    try:
+        refresh_crls(sys.argv[1], sys.argv[2], logger)
+    except Exception as err:
+        logger.exception("Fatal error encountered, stopping")
+        sys.exit(1)
+    logger.info("Finished updating CRLs")

k8s/azure/azure.yml

@@ -44,6 +44,8 @@ spec:
               subPath: client-ca-bundle.pem
             - name: uwsgi-socket-dir
               mountPath: "/var/run/uwsgi"
+            - name: crls-vol
+              mountPath: "/opt/atat/atst/crls"
         - name: nginx
           image: nginx:alpine
           ports:
@@ -109,6 +111,10 @@ spec:
             - key: tls.key
               path: atat.key
               mode: 0640
+        - name: crls-vol
+          persistentVolumeClaim:
+            claimName: crls-vol-claim
+
 ---
 apiVersion: extensions/v1beta1
 kind: Deployment

k8s/azure/storage-class.yml

@@ -0,0 +1,46 @@
+kind: StorageClass
+apiVersion: storage.k8s.io/v1
+metadata:
+  name: azurefile
+provisioner: kubernetes.io/azure-file
+mountOptions:
+  - dir_mode=0777
+  - file_mode=0777
+  - uid=1000
+  - gid=1000
+parameters:
+  skuName: Standard_LRS
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: system:azure-cloud-provider
+rules:
+- apiGroups: ['']
+  resources: ['secrets']
+  verbs:     ['get','create']
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: system:azure-cloud-provider
+roleRef:
+  kind: ClusterRole
+  apiGroup: rbac.authorization.k8s.io
+  name: system:azure-cloud-provider
+subjects:
+- kind: ServiceAccount
+  name: persistent-volume-binder
+  namespace: kube-system
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: crls-vol-claim
+spec:
+  accessModes:
+    - ReadWriteMany
+  storageClassName: azurefile
+  resources:
+    requests:
+      storage: 1Gi

k8s/shared/crls-sync.yaml

@@ -0,0 +1,42 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: crls
+  namespace: atat
+spec:
+  schedule: "0 * * * *"
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+          - name: crls
+            image: $CONTAINER_IMAGE
+            command: [
+              "/bin/sh", "-c"
+            ]
+            args: [
+              "/opt/atat/atst/script/sync-crls"
+            ]
+            envFrom:
+            - configMapRef:
+                name: atst-envvars
+            - configMapRef:
+                name: atst-worker-envvars
+            volumeMounts:
+              - name: atst-config
+                mountPath: "/opt/atat/atst/atst-overrides.ini"
+                subPath: atst-overrides.ini
+              - name: crls-vol
+                mountPath: "/opt/atat/atst/crls"
+          volumes:
+            - name: atst-config
+              secret:
+                secretName: atst-config-ini
+                items:
+                - key: override.ini
+                  path: atst-overrides.ini
+                  mode: 0644
+            - name: crls-vol
+              persistentVolumeClaim:
+                claimName: crls-vol-claim

script/sync-crls

@@ -1,14 +1,10 @@
-#! .venv/bin/python
+#!/bin/bash
-# Add root application dir to the python path
-import os
-import sys
 
-parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+# script/sync-crls: update the DOD CRLs and place them where authnid expects them
-sys.path.append(parent_dir)
+set -e
+cd "$(dirname "$0")/.."
 
-from atst.app import make_config, make_app
+mkdir -p crl-tmp crls
-
+pipenv run python ./atst/domain/authnid/crl/util.py crl-tmp crls
-if __name__ == "__main__":
+cp -r crl-tmp/* crls/
-    config = make_config({"DISABLE_CRL_CHECK": True})
+rm -rf crl-tmp
-    app = make_app(config)
-    app.csp.crls.sync_crls()