From b1cf89051a2d74bb1567c12906b626b0ff4ff267 Mon Sep 17 00:00:00 2001 From: dandds Date: Mon, 5 Aug 2019 09:39:47 -0400 Subject: [PATCH 1/4] Add kubernetes config and scripts for syncing CRLs. This adds a previous version of the CRL sync functionality back to the repo, with some small adjustments. We now grab the CRLs directly from their DISA URLs. The CRL sync is handled by a kubernetes cronjob that sync the files to a persistent volume that is mounted into each Flask app container. --- atst/domain/authnid/crl/util.py | 143 ++++++++++++++++++++++++++++++++ k8s/azure/azure.yml | 6 ++ k8s/azure/storage-class.yml | 46 ++++++++++ k8s/shared/crls-sync.yaml | 42 ++++++++++ script/sync-crls | 20 ++--- 5 files changed, 245 insertions(+), 12 deletions(-) create mode 100644 atst/domain/authnid/crl/util.py create mode 100644 k8s/azure/storage-class.yml create mode 100644 k8s/shared/crls-sync.yaml diff --git a/atst/domain/authnid/crl/util.py b/atst/domain/authnid/crl/util.py new file mode 100644 index 00000000..7b32a86e --- /dev/null +++ b/atst/domain/authnid/crl/util.py @@ -0,0 +1,143 @@ +import requests +import re +import os +import pendulum +from html.parser import HTMLParser + +MODIFIED_TIME_BUFFER = 15 * 60 + + +CRL_LIST = [ + "http://crl.disa.mil/crl/DODROOTCA2.crl", + "http://crl.disa.mil/crl/DODROOTCA3.crl", + "http://crl.disa.mil/crl/DODROOTCA4.crl", + "http://crl.disa.mil/crl/DODROOTCA5.crl", + "http://crl.disa.mil/crl/DODIDCA_33.crl", + "http://crl.disa.mil/crl/DODIDCA_34.crl", + "http://crl.disa.mil/crl/DODIDSWCA_35.crl", + "http://crl.disa.mil/crl/DODIDSWCA_36.crl", + "http://crl.disa.mil/crl/DODIDSWCA_37.crl", + "http://crl.disa.mil/crl/DODIDSWCA_38.crl", + "http://crl.disa.mil/crl/DODIDCA_39.crl", + "http://crl.disa.mil/crl/DODIDCA_40.crl", + "http://crl.disa.mil/crl/DODIDCA_41.crl", + "http://crl.disa.mil/crl/DODIDCA_42.crl", + "http://crl.disa.mil/crl/DODIDCA_43.crl", + "http://crl.disa.mil/crl/DODIDCA_44.crl", + "http://crl.disa.mil/crl/DODIDSWCA_45.crl", + "http://crl.disa.mil/crl/DODIDSWCA_46.crl", + "http://crl.disa.mil/crl/DODIDSWCA_47.crl", + "http://crl.disa.mil/crl/DODIDSWCA_48.crl", + "http://crl.disa.mil/crl/DODIDCA_49.crl", + "http://crl.disa.mil/crl/DODIDCA_50.crl", + "http://crl.disa.mil/crl/DODIDCA_51.crl", + "http://crl.disa.mil/crl/DODIDCA_52.crl", + "http://crl.disa.mil/crl/DODIDCA_59.crl", + "http://crl.disa.mil/crl/DODSWCA_53.crl", + "http://crl.disa.mil/crl/DODSWCA_54.crl", + "http://crl.disa.mil/crl/DODSWCA_55.crl", + "http://crl.disa.mil/crl/DODSWCA_56.crl", + "http://crl.disa.mil/crl/DODSWCA_57.crl", + "http://crl.disa.mil/crl/DODSWCA_58.crl", + "http://crl.disa.mil/crl/DODSWCA_60.crl", + "http://crl.disa.mil/crl/DODSWCA_61.crl", + "http://crl.disa.mil/crl/DODEMAILCA_33.crl", + "http://crl.disa.mil/crl/DODEMAILCA_34.crl", + "http://crl.disa.mil/crl/DODEMAILCA_39.crl", + "http://crl.disa.mil/crl/DODEMAILCA_40.crl", + "http://crl.disa.mil/crl/DODEMAILCA_41.crl", + "http://crl.disa.mil/crl/DODEMAILCA_42.crl", + "http://crl.disa.mil/crl/DODEMAILCA_43.crl", + "http://crl.disa.mil/crl/DODEMAILCA_44.crl", + "http://crl.disa.mil/crl/DODEMAILCA_49.crl", + "http://crl.disa.mil/crl/DODEMAILCA_50.crl", + "http://crl.disa.mil/crl/DODEMAILCA_51.crl", + "http://crl.disa.mil/crl/DODEMAILCA_52.crl", + "http://crl.disa.mil/crl/DODEMAILCA_59.crl", + "http://crl.disa.mil/crl/DODINTEROPERABILITYROOTCA1.crl ", + "http://crl.disa.mil/crl/DODINTEROPERABILITYROOTCA2.crl ", + "http://crl.disa.mil/crl/USDODCCEBINTEROPERABILITYROOTCA1.crl ", + "http://crl.disa.mil/crl/USDODCCEBINTEROPERABILITYROOTCA2.crl", + "http://crl.disa.mil/crl/DODNIPRINTERNALNPEROOTCA1.crl", + "http://crl.disa.mil/crl/DODNPEROOTCA1.crl", + "http://crl.disa.mil/crl/DMDNSIGNINGCA_1.crl", + "http://crl.disa.mil/crl/DODWCFROOTCA1.crl", +] + + +def crl_local_path(out_dir, crl_location): + name = re.split("/", crl_location)[-1] + crl = os.path.join(out_dir, name) + return crl + + +def existing_crl_modification_time(crl): + if os.path.exists(crl): + prev_time = os.path.getmtime(crl) + buffered = prev_time + MODIFIED_TIME_BUFFER + mod_time = prev_time if pendulum.now().timestamp() < buffered else buffered + dt = pendulum.from_timestamp(mod_time, tz="GMT") + return dt.format("ddd, DD MMM YYYY HH:mm:ss zz") + + else: + return False + + +def write_crl(out_dir, target_dir, crl_location): + crl = crl_local_path(out_dir, crl_location) + existing = crl_local_path(target_dir, crl_location) + options = {"stream": True} + mod_time = existing_crl_modification_time(existing) + if mod_time: + options["headers"] = {"If-Modified-Since": mod_time} + + with requests.get(crl_location, **options) as response: + if response.status_code == 304: + return False + + with open(crl, "wb") as crl_file: + for chunk in response.iter_content(chunk_size=1024): + if chunk: + crl_file.write(chunk) + + return True + + +def remove_bad_crl(out_dir, crl_location): + crl = crl_local_path(out_dir, crl_location) + os.remove(crl) + + +def refresh_crls(out_dir, target_dir, logger): + for crl_location in CRL_LIST: + logger.info("updating CRL from {}".format(crl_location)) + try: + if write_crl(out_dir, target_dir, crl_location): + logger.info("successfully synced CRL from {}".format(crl_location)) + else: + logger.info("no updates for CRL from {}".format(crl_location)) + except requests.exceptions.ChunkedEncodingError: + if logger: + logger.error( + "Error downloading {}, removing file and continuing anyway".format( + crl_location + ) + ) + remove_bad_crl(out_dir, crl_location) + + +if __name__ == "__main__": + import sys + import logging + + logging.basicConfig( + level=logging.INFO, format="[%(asctime)s]:%(levelname)s: %(message)s" + ) + logger = logging.getLogger() + logger.info("Updating CRLs") + try: + refresh_crls(sys.argv[1], sys.argv[2], logger) + except Exception as err: + logger.exception("Fatal error encountered, stopping") + sys.exit(1) + logger.info("Finished updating CRLs") diff --git a/k8s/azure/azure.yml b/k8s/azure/azure.yml index 097b1e15..ea645df5 100644 --- a/k8s/azure/azure.yml +++ b/k8s/azure/azure.yml @@ -44,6 +44,8 @@ spec: subPath: client-ca-bundle.pem - name: uwsgi-socket-dir mountPath: "/var/run/uwsgi" + - name: crls-vol + mountPath: "/opt/atat/atst/crls" - name: nginx image: nginx:alpine ports: @@ -109,6 +111,10 @@ spec: - key: tls.key path: atat.key mode: 0640 + - name: crls-vol + persistentVolumeClaim: + claimName: crls-vol-claim + --- apiVersion: extensions/v1beta1 kind: Deployment diff --git a/k8s/azure/storage-class.yml b/k8s/azure/storage-class.yml new file mode 100644 index 00000000..fba2908a --- /dev/null +++ b/k8s/azure/storage-class.yml @@ -0,0 +1,46 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: azurefile +provisioner: kubernetes.io/azure-file +mountOptions: + - dir_mode=0777 + - file_mode=0777 + - uid=1000 + - gid=1000 +parameters: + skuName: Standard_LRS +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:azure-cloud-provider +rules: +- apiGroups: [''] + resources: ['secrets'] + verbs: ['get','create'] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:azure-cloud-provider +roleRef: + kind: ClusterRole + apiGroup: rbac.authorization.k8s.io + name: system:azure-cloud-provider +subjects: +- kind: ServiceAccount + name: persistent-volume-binder + namespace: kube-system +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: crls-vol-claim +spec: + accessModes: + - ReadWriteMany + storageClassName: azurefile + resources: + requests: + storage: 1Gi diff --git a/k8s/shared/crls-sync.yaml b/k8s/shared/crls-sync.yaml new file mode 100644 index 00000000..541c026b --- /dev/null +++ b/k8s/shared/crls-sync.yaml @@ -0,0 +1,42 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: crls + namespace: atat +spec: + schedule: "0 * * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: crls + image: $CONTAINER_IMAGE + command: [ + "/bin/sh", "-c" + ] + args: [ + "/opt/atat/atst/script/sync-crls" + ] + envFrom: + - configMapRef: + name: atst-envvars + - configMapRef: + name: atst-worker-envvars + volumeMounts: + - name: atst-config + mountPath: "/opt/atat/atst/atst-overrides.ini" + subPath: atst-overrides.ini + - name: crls-vol + mountPath: "/opt/atat/atst/crls" + volumes: + - name: atst-config + secret: + secretName: atst-config-ini + items: + - key: override.ini + path: atst-overrides.ini + mode: 0644 + - name: crls-vol + persistentVolumeClaim: + claimName: crls-vol-claim diff --git a/script/sync-crls b/script/sync-crls index 82b6631b..7708e386 100755 --- a/script/sync-crls +++ b/script/sync-crls @@ -1,14 +1,10 @@ -#! .venv/bin/python -# Add root application dir to the python path -import os -import sys +#!/bin/bash -parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) -sys.path.append(parent_dir) +# script/sync-crls: update the DOD CRLs and place them where authnid expects them +set -e +cd "$(dirname "$0")/.." -from atst.app import make_config, make_app - -if __name__ == "__main__": - config = make_config({"DISABLE_CRL_CHECK": True}) - app = make_app(config) - app.csp.crls.sync_crls() +mkdir -p crl-tmp crls +pipenv run python ./atst/domain/authnid/crl/util.py crl-tmp crls +cp -r crl-tmp/* crls/ +rm -rf crl-tmp From 44141c002d4c11159ece915083470f0c065ac14b Mon Sep 17 00:00:00 2001 From: dandds Date: Mon, 5 Aug 2019 15:32:09 -0400 Subject: [PATCH 2/4] Fix up sync-crls script and add to Dockerfile. - Fix some python formatting and import issues - Fix dockerfile to include sync-crls script - Adjust sync-crls script to use paths and CLI tools available in the Docker container --- Dockerfile | 1 + atst/domain/authnid/crl/util.py | 4 ++-- k8s/{shared => azure}/crls-sync.yaml | 7 ++++--- script/sync-crls | 5 +++-- 4 files changed, 10 insertions(+), 7 deletions(-) rename k8s/{shared => azure}/crls-sync.yaml (84%) diff --git a/Dockerfile b/Dockerfile index 2785b815..ceb862db 100644 --- a/Dockerfile +++ b/Dockerfile @@ -79,6 +79,7 @@ COPY --from=builder /install/config/ ./config/ COPY --from=builder /install/templates/ ./templates/ COPY --from=builder /install/translations.yaml . COPY --from=builder /install/script/seed_roles.py ./script/seed_roles.py +COPY --from=builder /install/script/sync-crls ./script/sync-crls COPY --from=builder /install/static/ ./static/ COPY --from=builder /install/uwsgi.ini . COPY --from=builder /usr/local/bin/uwsgi /usr/local/bin/uwsgi diff --git a/atst/domain/authnid/crl/util.py b/atst/domain/authnid/crl/util.py index 7b32a86e..408a0d63 100644 --- a/atst/domain/authnid/crl/util.py +++ b/atst/domain/authnid/crl/util.py @@ -1,8 +1,8 @@ -import requests import re import os + import pendulum -from html.parser import HTMLParser +import requests MODIFIED_TIME_BUFFER = 15 * 60 diff --git a/k8s/shared/crls-sync.yaml b/k8s/azure/crls-sync.yaml similarity index 84% rename from k8s/shared/crls-sync.yaml rename to k8s/azure/crls-sync.yaml index 541c026b..a86272d4 100644 --- a/k8s/shared/crls-sync.yaml +++ b/k8s/azure/crls-sync.yaml @@ -1,4 +1,4 @@ -apiVersion: batch/v1 +apiVersion: batch/v1beta1 kind: CronJob metadata: name: crls @@ -9,14 +9,15 @@ spec: spec: template: spec: + restartPolicy: OnFailure containers: - name: crls - image: $CONTAINER_IMAGE + image: pwatat.azurecr.io/atat:8f1c8b5633ca70168837c885010e7d66d93562dc command: [ "/bin/sh", "-c" ] args: [ - "/opt/atat/atst/script/sync-crls" + "/opt/atat/atst/script/sync-crls", ] envFrom: - configMapRef: diff --git a/script/sync-crls b/script/sync-crls index 7708e386..6c33172e 100755 --- a/script/sync-crls +++ b/script/sync-crls @@ -1,10 +1,11 @@ -#!/bin/bash +#!/bin/sh # script/sync-crls: update the DOD CRLs and place them where authnid expects them set -e cd "$(dirname "$0")/.." mkdir -p crl-tmp crls -pipenv run python ./atst/domain/authnid/crl/util.py crl-tmp crls +# need to adjust this command +./.venv/bin/python ./atst/domain/authnid/crl/util.py crl-tmp crls cp -r crl-tmp/* crls/ rm -rf crl-tmp From bd3662e8ce8833b03be51f3e9c9c19b17e70b225 Mon Sep 17 00:00:00 2001 From: dandds Date: Tue, 6 Aug 2019 09:48:39 -0400 Subject: [PATCH 3/4] Add config for EFS persistent volume in AWS k8s cluster. Chose Elastic File Storage over EBS (Elastic Block Storage) because the latter can only be mounted into a single node. This adds the RBAC config and deployment for managing EFS mounts within the cluster. Largely depends on this efs-provisioner config: https://github.com/kubernetes-incubator/external-storage/tree/master/aws/efs The config has been hard-copied into the repo and updated for future reference. Note that the config requires an environment variable substitution and cannot be applied directly to the cluster. --- k8s/aws/aws.yml | 5 +++ k8s/aws/crls-sync.yaml | 43 +++++++++++++++++++++ k8s/aws/efs-rbac.yml | 66 ++++++++++++++++++++++++++++++++ k8s/aws/storage-class.yml | 80 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 194 insertions(+) create mode 100644 k8s/aws/crls-sync.yaml create mode 100644 k8s/aws/efs-rbac.yml create mode 100644 k8s/aws/storage-class.yml diff --git a/k8s/aws/aws.yml b/k8s/aws/aws.yml index 4fe84e70..8f8e46d0 100644 --- a/k8s/aws/aws.yml +++ b/k8s/aws/aws.yml @@ -44,6 +44,8 @@ spec: subPath: client-ca-bundle.pem - name: uwsgi-socket-dir mountPath: "/var/run/uwsgi" + - name: crls-vol + mountPath: "/opt/atat/atst/crls" - name: nginx image: nginx:alpine ports: @@ -109,6 +111,9 @@ spec: - key: tls.key path: atat.key mode: 0640 + - name: crls-vol + persistentVolumeClaim: + claimName: efs --- apiVersion: extensions/v1beta1 kind: Deployment diff --git a/k8s/aws/crls-sync.yaml b/k8s/aws/crls-sync.yaml new file mode 100644 index 00000000..2d7ee55e --- /dev/null +++ b/k8s/aws/crls-sync.yaml @@ -0,0 +1,43 @@ +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: crls + namespace: atat +spec: + schedule: "0 * * * *" + jobTemplate: + spec: + template: + spec: + restartPolicy: OnFailure + containers: + - name: crls + image: 904153757533.dkr.ecr.us-east-2.amazonaws.com/atat:8f1c8b5633ca70168837c885010e7d66d93562dc + command: [ + "/bin/sh", "-c" + ] + args: [ + "/opt/atat/atst/script/sync-crls", + ] + envFrom: + - configMapRef: + name: atst-envvars + - configMapRef: + name: atst-worker-envvars + volumeMounts: + - name: atst-config + mountPath: "/opt/atat/atst/atst-overrides.ini" + subPath: atst-overrides.ini + - name: crls-vol + mountPath: "/opt/atat/atst/crls" + volumes: + - name: atst-config + secret: + secretName: atst-config-ini + items: + - key: override.ini + path: atst-overrides.ini + mode: 0644 + - name: crls-vol + persistentVolumeClaim: + claimName: efs diff --git a/k8s/aws/efs-rbac.yml b/k8s/aws/efs-rbac.yml new file mode 100644 index 00000000..12496f02 --- /dev/null +++ b/k8s/aws/efs-rbac.yml @@ -0,0 +1,66 @@ +# This can't be run without substituting the EFSID environment variable. +# from https://github.com/kubernetes-incubator/external-storage/blob/master/aws/efs/deploy/rbac.yaml +--- +kind: ServiceAccount +apiVersion: v1 +metadata: + name: efs-provisioner +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: efs-provisioner-runner +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete", "describe"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "update", "patch"] + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "list", "watch", "create", "update", "patch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: run-efs-provisioner +subjects: + - kind: ServiceAccount + name: efs-provisioner + # replace with namespace where provisioner is deployed + namespace: atat +roleRef: + kind: ClusterRole + name: efs-provisioner-runner + apiGroup: rbac.authorization.k8s.io +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: leader-locking-efs-provisioner +rules: + - apiGroups: [""] + resources: ["endpoints"] + + verbs: ["get", "list", "watch", "create", "update", "patch"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: leader-locking-efs-provisioner +subjects: + - kind: ServiceAccount + name: efs-provisioner + # replace with namespace where provisioner is deployed + namespace: atat +roleRef: + kind: Role + name: leader-locking-efs-provisioner + apiGroup: rbac.authorization.k8s.io diff --git a/k8s/aws/storage-class.yml b/k8s/aws/storage-class.yml new file mode 100644 index 00000000..1df8d78c --- /dev/null +++ b/k8s/aws/storage-class.yml @@ -0,0 +1,80 @@ +# from https://github.com/kubernetes-incubator/external-storage/blob/master/aws/efs/deploy/manifest.yaml +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: efs-provisioner +data: + file.system.id: $EFSID + aws.region: us-east-2 + provisioner.name: example.com/aws-efs + dns.name: $EFSID.efs.us-east-2.amazonaws.com +--- +kind: Deployment +apiVersion: extensions/v1beta1 +metadata: + name: efs-provisioner +spec: + replicas: 1 + strategy: + type: Recreate + template: + metadata: + labels: + app: efs-provisioner + spec: + serviceAccountName: efs-provisioner + containers: + - name: efs-provisioner + image: quay.io/external_storage/efs-provisioner:latest + env: + - name: FILE_SYSTEM_ID + valueFrom: + configMapKeyRef: + name: efs-provisioner + key: file.system.id + - name: AWS_REGION + valueFrom: + configMapKeyRef: + name: efs-provisioner + key: aws.region + - name: DNS_NAME + valueFrom: + configMapKeyRef: + name: efs-provisioner + key: dns.name + optional: true + - name: PROVISIONER_NAME + valueFrom: + configMapKeyRef: + name: efs-provisioner + key: provisioner.name + volumeMounts: + - name: pv-volume + mountPath: /persistentvolumes + volumes: + - name: pv-volume + nfs: + server: $EFSID.efs.us-east-2.amazonaws.com + path: / +--- +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: aws-efs +provisioner: example.com/aws-efs +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: efs + annotations: + volume.beta.kubernetes.io/storage-class: "aws-efs" +spec: + accessModes: + - ReadWriteMany + storageClassName: aws-efs + resources: + requests: + storage: 1Mi +--- From 0468d5353a13514df2e84fe37e3be108d95b5425 Mon Sep 17 00:00:00 2001 From: dandds Date: Tue, 6 Aug 2019 12:59:02 -0400 Subject: [PATCH 4/4] Fix a few bugs in CRL handling. - Don't write a CRL to the cache if the response code is above 399. (We were getting HTML files as CRLs, d'oh). - Fix a kwarg in the CRL logger (extras -> extra). - Set Kubernetes clusters to log output as JSON. --- atst/domain/authnid/crl/__init__.py | 2 +- atst/domain/authnid/crl/util.py | 32 ++++++++++++++++++++-------- k8s/aws/atst-envvars-configmap.yml | 1 + k8s/azure/atst-envvars-configmap.yml | 1 + 4 files changed, 26 insertions(+), 10 deletions(-) diff --git a/atst/domain/authnid/crl/__init__.py b/atst/domain/authnid/crl/__init__.py index 6a5addd2..7865ef14 100644 --- a/atst/domain/authnid/crl/__init__.py +++ b/atst/domain/authnid/crl/__init__.py @@ -33,7 +33,7 @@ class CRLInterface: def _log(self, message, level=logging.INFO): if self.logger: - self.logger.log(level, message, extras={"tags": ["authorization", "crl"]}) + self.logger.log(level, message, extra={"tags": ["authorization", "crl"]}) def crl_check(self, cert): raise NotImplementedError() diff --git a/atst/domain/authnid/crl/util.py b/atst/domain/authnid/crl/util.py index 408a0d63..e9bc74fb 100644 --- a/atst/domain/authnid/crl/util.py +++ b/atst/domain/authnid/crl/util.py @@ -4,6 +4,11 @@ import os import pendulum import requests + +class CRLNotFoundError(Exception): + pass + + MODIFIED_TIME_BUFFER = 15 * 60 @@ -54,9 +59,9 @@ CRL_LIST = [ "http://crl.disa.mil/crl/DODEMAILCA_51.crl", "http://crl.disa.mil/crl/DODEMAILCA_52.crl", "http://crl.disa.mil/crl/DODEMAILCA_59.crl", - "http://crl.disa.mil/crl/DODINTEROPERABILITYROOTCA1.crl ", - "http://crl.disa.mil/crl/DODINTEROPERABILITYROOTCA2.crl ", - "http://crl.disa.mil/crl/USDODCCEBINTEROPERABILITYROOTCA1.crl ", + "http://crl.disa.mil/crl/DODINTEROPERABILITYROOTCA1.crl", + "http://crl.disa.mil/crl/DODINTEROPERABILITYROOTCA2.crl", + "http://crl.disa.mil/crl/USDODCCEBINTEROPERABILITYROOTCA1.crl", "http://crl.disa.mil/crl/USDODCCEBINTEROPERABILITYROOTCA2.crl", "http://crl.disa.mil/crl/DODNIPRINTERNALNPEROOTCA1.crl", "http://crl.disa.mil/crl/DODNPEROOTCA1.crl", @@ -92,6 +97,9 @@ def write_crl(out_dir, target_dir, crl_location): options["headers"] = {"If-Modified-Since": mod_time} with requests.get(crl_location, **options) as response: + if response.status_code > 399: + raise CRLNotFoundError() + if response.status_code == 304: return False @@ -108,6 +116,15 @@ def remove_bad_crl(out_dir, crl_location): os.remove(crl) +def log_error(logger, crl_location): + if logger: + logger.error( + "Error downloading {}, removing file and continuing anyway".format( + crl_location + ) + ) + + def refresh_crls(out_dir, target_dir, logger): for crl_location in CRL_LIST: logger.info("updating CRL from {}".format(crl_location)) @@ -117,13 +134,10 @@ def refresh_crls(out_dir, target_dir, logger): else: logger.info("no updates for CRL from {}".format(crl_location)) except requests.exceptions.ChunkedEncodingError: - if logger: - logger.error( - "Error downloading {}, removing file and continuing anyway".format( - crl_location - ) - ) + log_error(logger, crl_location) remove_bad_crl(out_dir, crl_location) + except CRLNotFoundError: + log_error(logger, crl_location) if __name__ == "__main__": diff --git a/k8s/aws/atst-envvars-configmap.yml b/k8s/aws/atst-envvars-configmap.yml index fc65df51..ed2c5c1c 100644 --- a/k8s/aws/atst-envvars-configmap.yml +++ b/k8s/aws/atst-envvars-configmap.yml @@ -10,3 +10,4 @@ data: OVERRIDE_CONFIG_FULLPATH: /opt/atat/atst/atst-overrides.ini UWSGI_CONFIG_FULLPATH: /opt/atat/atst/uwsgi.ini CRL_STORAGE_PROVIDER: CLOUDFILES + LOG_JSON: "true" diff --git a/k8s/azure/atst-envvars-configmap.yml b/k8s/azure/atst-envvars-configmap.yml index fc65df51..ed2c5c1c 100644 --- a/k8s/azure/atst-envvars-configmap.yml +++ b/k8s/azure/atst-envvars-configmap.yml @@ -10,3 +10,4 @@ data: OVERRIDE_CONFIG_FULLPATH: /opt/atat/atst/atst-overrides.ini UWSGI_CONFIG_FULLPATH: /opt/atat/atst/uwsgi.ini CRL_STORAGE_PROVIDER: CLOUDFILES + LOG_JSON: "true"