diff --git a/atst/domain/authnid/crl/__init__.py b/atst/domain/authnid/crl/__init__.py index 6ac18aba..037a8bb4 100644 --- a/atst/domain/authnid/crl/__init__.py +++ b/atst/domain/authnid/crl/__init__.py @@ -7,7 +7,7 @@ from OpenSSL import crypto, SSL from datetime import datetime from flask import current_app as app -from .util import load_crl_locations_cache, serialize_crl_locations_cache +from .util import load_crl_locations_cache, serialize_crl_locations_cache, CRL_LIST # error codes from OpenSSL: https://github.com/openssl/openssl/blob/2c75f03b39de2fa7d006bc0f0d7c58235a54d9bb/include/openssl/x509_vfy.h#L111 CRL_EXPIRED_ERROR_CODE = 12 @@ -75,13 +75,13 @@ class CRLCache(CRLInterface): crl_dir, store_class=crypto.X509Store, logger=None, - crl_update_func=None, + crl_list=CRL_LIST, ): self._crl_dir = crl_dir self.logger = logger self.store_class = store_class self.certificate_authorities = {} - self._crl_update_func = crl_update_func + self.crl_list = crl_list self._load_roots(root_location) self._build_crl_cache() @@ -101,7 +101,9 @@ class CRLCache(CRLInterface): try: self.crl_cache = load_crl_locations_cache(self._crl_dir) except FileNotFoundError: - self.crl_cache = serialize_crl_locations_cache(self._crl_dir) + self.crl_cache = serialize_crl_locations_cache( + self._crl_dir, crl_list=self.crl_list + ) def _load_crl(self, crl_location): with open(crl_location, "rb") as crl_file: @@ -117,19 +119,17 @@ class CRLCache(CRLInterface): store = self.store_class() self._log("STORE ID: {}. Building store.".format(id(store))) store.set_flags(crypto.X509StoreFlags.CRL_CHECK) - crl_info = self.crl_cache.get(issuer.der(), {}) + crl_location = self.crl_cache.get(issuer.der()) issuer_name = get_common_name(issuer) - if not crl_info: + if not crl_location: raise CRLInvalidException( "Could not find matching CRL for issuer with Common Name {}".format( issuer_name ) ) - self._manage_expiring(crl_info["expires"]) - - crl = self._load_crl(crl_info["location"]) + crl = self._load_crl(crl_location) store.add_crl(crl) self._log( @@ -141,17 +141,6 @@ class CRLCache(CRLInterface): store = self._add_certificate_chain_to_store(store, crl.get_issuer()) return store - def _manage_expiring(self, crl_expiry): - """ - If a CRL is expired and CRLCache has been given a function for updating - the physical CRL locations, run the update function and then rebuild - the CRL cache. - """ - current = datetime.now(crl_expiry.tzinfo) - if self._crl_update_func and current > crl_expiry: - self._crl_update_func() - self._build_crl_cache() - # this _should_ happen just twice for the DoD PKI (intermediary, root) but # theoretically it can build a longer certificate chain diff --git a/atst/domain/authnid/crl/util.py b/atst/domain/authnid/crl/util.py index 18e070c1..7668a71f 100644 --- a/atst/domain/authnid/crl/util.py +++ b/atst/domain/authnid/crl/util.py @@ -1,4 +1,3 @@ -from datetime import datetime import json import os import re @@ -19,161 +18,230 @@ MODIFIED_TIME_BUFFER = 15 * 60 CRL_LIST = [ - "http://crl.disa.mil/crl/DODROOTCA2.crl", - "http://crl.disa.mil/crl/DODROOTCA3.crl", - "http://crl.disa.mil/crl/DODROOTCA4.crl", - "http://crl.disa.mil/crl/DODROOTCA5.crl", - "http://crl.disa.mil/crl/DODIDCA_33.crl", - "http://crl.disa.mil/crl/DODIDCA_34.crl", - "http://crl.disa.mil/crl/DODIDSWCA_35.crl", - "http://crl.disa.mil/crl/DODIDSWCA_36.crl", - "http://crl.disa.mil/crl/DODIDSWCA_37.crl", - "http://crl.disa.mil/crl/DODIDSWCA_38.crl", - "http://crl.disa.mil/crl/DODIDCA_39.crl", - "http://crl.disa.mil/crl/DODIDCA_40.crl", - "http://crl.disa.mil/crl/DODIDCA_41.crl", - "http://crl.disa.mil/crl/DODIDCA_42.crl", - "http://crl.disa.mil/crl/DODIDCA_43.crl", - "http://crl.disa.mil/crl/DODIDCA_44.crl", - "http://crl.disa.mil/crl/DODIDSWCA_45.crl", - "http://crl.disa.mil/crl/DODIDSWCA_46.crl", - "http://crl.disa.mil/crl/DODIDSWCA_47.crl", - "http://crl.disa.mil/crl/DODIDSWCA_48.crl", - "http://crl.disa.mil/crl/DODIDCA_49.crl", - "http://crl.disa.mil/crl/DODIDCA_50.crl", - "http://crl.disa.mil/crl/DODIDCA_51.crl", - "http://crl.disa.mil/crl/DODIDCA_52.crl", - "http://crl.disa.mil/crl/DODIDCA_59.crl", - "http://crl.disa.mil/crl/DODSWCA_53.crl", - "http://crl.disa.mil/crl/DODSWCA_54.crl", - "http://crl.disa.mil/crl/DODSWCA_55.crl", - "http://crl.disa.mil/crl/DODSWCA_56.crl", - "http://crl.disa.mil/crl/DODSWCA_57.crl", - "http://crl.disa.mil/crl/DODSWCA_58.crl", - "http://crl.disa.mil/crl/DODSWCA_60.crl", - "http://crl.disa.mil/crl/DODSWCA_61.crl", - "http://crl.disa.mil/crl/DODEMAILCA_33.crl", - "http://crl.disa.mil/crl/DODEMAILCA_34.crl", - "http://crl.disa.mil/crl/DODEMAILCA_39.crl", - "http://crl.disa.mil/crl/DODEMAILCA_40.crl", - "http://crl.disa.mil/crl/DODEMAILCA_41.crl", - "http://crl.disa.mil/crl/DODEMAILCA_42.crl", - "http://crl.disa.mil/crl/DODEMAILCA_43.crl", - "http://crl.disa.mil/crl/DODEMAILCA_44.crl", - "http://crl.disa.mil/crl/DODEMAILCA_49.crl", - "http://crl.disa.mil/crl/DODEMAILCA_50.crl", - "http://crl.disa.mil/crl/DODEMAILCA_51.crl", - "http://crl.disa.mil/crl/DODEMAILCA_52.crl", - "http://crl.disa.mil/crl/DODEMAILCA_59.crl", - "http://crl.disa.mil/crl/DODINTEROPERABILITYROOTCA1.crl", - "http://crl.disa.mil/crl/DODINTEROPERABILITYROOTCA2.crl", - "http://crl.disa.mil/crl/USDODCCEBINTEROPERABILITYROOTCA1.crl", - "http://crl.disa.mil/crl/USDODCCEBINTEROPERABILITYROOTCA2.crl", - "http://crl.disa.mil/crl/DODNIPRINTERNALNPEROOTCA1.crl", - "http://crl.disa.mil/crl/DODNPEROOTCA1.crl", - "http://crl.disa.mil/crl/DMDNSIGNINGCA_1.crl", - "http://crl.disa.mil/crl/DODWCFROOTCA1.crl", + ( + "https://crl.gds.disa.mil/crl/DODROOTCA2.crl", + "305b310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311630140603550403130d446f4420526f6f742043412032", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/DODROOTCA3.crl", + "305b310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311630140603550403130d446f4420526f6f742043412033", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/DODROOTCA4.crl", + "305b310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311630140603550403130d446f4420526f6f742043412034", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/DODROOTCA5.crl", + "305b310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311630140603550403130d446f4420526f6f742043412035", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/DODIDCA_33.crl", + "305a310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311530130603550403130c444f442049442043412d3333", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/DODIDCA_34.crl", + "305a310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311530130603550403130c444f442049442043412d3334", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/DODIDSWCA_35.crl", + "305d310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311830160603550403130f444f442049442053572043412d3335", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/DODIDSWCA_36.crl", + "305d310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311830160603550403130f444f442049442053572043412d3336", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/DODIDSWCA_37.crl", + "305d310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311830160603550403130f444f442049442053572043412d3337", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/DODIDSWCA_38.crl", + "305d310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311830160603550403130f444f442049442053572043412d3338", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/DODIDCA_39.crl", + "305a310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311530130603550403130c444f442049442043412d3339", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/DODIDCA_40.crl", + "305a310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311530130603550403130c444f442049442043412d3430", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/DODIDCA_41.crl", + "305a310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311530130603550403130c444f442049442043412d3431", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/DODIDCA_42.crl", + "305a310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311530130603550403130c444f442049442043412d3432", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/DODIDCA_43.crl", + "305a310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311530130603550403130c444f442049442043412d3433", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/DODIDCA_44.crl", + "305a310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311530130603550403130c444f442049442043412d3434", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/DODIDSWCA_45.crl", + "305d310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311830160603550403130f444f442049442053572043412d3435", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/DODIDSWCA_46.crl", + "305d310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311830160603550403130f444f442049442053572043412d3436", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/DODIDSWCA_47.crl", + "305d310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311830160603550403130f444f442049442053572043412d3437", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/DODIDSWCA_48.crl", + "305d310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311830160603550403130f444f442049442053572043412d3438", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/DODIDCA_49.crl", + "305a310b300906035504061302555331183016060355040a0c0f552e532e20476f7665726e6d656e74310c300a060355040b0c03446f44310c300a060355040b0c03504b493115301306035504030c0c444f442049442043412d3439", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/DODIDCA_50.crl", + "305a310b300906035504061302555331183016060355040a0c0f552e532e20476f7665726e6d656e74310c300a060355040b0c03446f44310c300a060355040b0c03504b493115301306035504030c0c444f442049442043412d3530", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/DODIDCA_51.crl", + "305a310b300906035504061302555331183016060355040a0c0f552e532e20476f7665726e6d656e74310c300a060355040b0c03446f44310c300a060355040b0c03504b493115301306035504030c0c444f442049442043412d3531", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/DODIDCA_52.crl", + "305a310b300906035504061302555331183016060355040a0c0f552e532e20476f7665726e6d656e74310c300a060355040b0c03446f44310c300a060355040b0c03504b493115301306035504030c0c444f442049442043412d3532", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/DODIDCA_59.crl", + "305a310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311530130603550403130c444f442049442043412d3539", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/DODSWCA_53.crl", + "305a310b300906035504061302555331183016060355040a0c0f552e532e20476f7665726e6d656e74310c300a060355040b0c03446f44310c300a060355040b0c03504b493115301306035504030c0c444f442053572043412d3533", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/DODSWCA_54.crl", + "305a310b300906035504061302555331183016060355040a0c0f552e532e20476f7665726e6d656e74310c300a060355040b0c03446f44310c300a060355040b0c03504b493115301306035504030c0c444f442053572043412d3534", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/DODSWCA_55.crl", + "305a310b300906035504061302555331183016060355040a0c0f552e532e20476f7665726e6d656e74310c300a060355040b0c03446f44310c300a060355040b0c03504b493115301306035504030c0c444f442053572043412d3535", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/DODSWCA_56.crl", + "305a310b300906035504061302555331183016060355040a0c0f552e532e20476f7665726e6d656e74310c300a060355040b0c03446f44310c300a060355040b0c03504b493115301306035504030c0c444f442053572043412d3536", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/DODSWCA_57.crl", + "305a310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311530130603550403130c444f442053572043412d3537", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/DODSWCA_58.crl", + "305a310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311530130603550403130c444f442053572043412d3538", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/DODSWCA_60.crl", + "305a310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311530130603550403130c444f442053572043412d3630", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/DODSWCA_61.crl", + "305a310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311530130603550403130c444f442053572043412d3631", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/DODEMAILCA_33.crl", + "305d310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311830160603550403130f444f4420454d41494c2043412d3333", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/DODEMAILCA_34.crl", + "305d310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311830160603550403130f444f4420454d41494c2043412d3334", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/DODEMAILCA_39.crl", + "305d310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311830160603550403130f444f4420454d41494c2043412d3339", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/DODEMAILCA_40.crl", + "305d310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311830160603550403130f444f4420454d41494c2043412d3430", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/DODEMAILCA_41.crl", + "305d310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311830160603550403130f444f4420454d41494c2043412d3431", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/DODEMAILCA_42.crl", + "305d310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311830160603550403130f444f4420454d41494c2043412d3432", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/DODEMAILCA_43.crl", + "305d310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311830160603550403130f444f4420454d41494c2043412d3433", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/DODEMAILCA_44.crl", + "305d310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311830160603550403130f444f4420454d41494c2043412d3434", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/DODEMAILCA_49.crl", + "305d310b300906035504061302555331183016060355040a0c0f552e532e20476f7665726e6d656e74310c300a060355040b0c03446f44310c300a060355040b0c03504b493118301606035504030c0f444f4420454d41494c2043412d3439", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/DODEMAILCA_50.crl", + "305d310b300906035504061302555331183016060355040a0c0f552e532e20476f7665726e6d656e74310c300a060355040b0c03446f44310c300a060355040b0c03504b493118301606035504030c0f444f4420454d41494c2043412d3530", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/DODEMAILCA_51.crl", + "305d310b300906035504061302555331183016060355040a0c0f552e532e20476f7665726e6d656e74310c300a060355040b0c03446f44310c300a060355040b0c03504b493118301606035504030c0f444f4420454d41494c2043412d3531", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/DODEMAILCA_52.crl", + "305d310b300906035504061302555331183016060355040a0c0f552e532e20476f7665726e6d656e74310c300a060355040b0c03446f44310c300a060355040b0c03504b493118301606035504030c0f444f4420454d41494c2043412d3532", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/DODEMAILCA_59.crl", + "305d310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311830160603550403130f444f4420454d41494c2043412d3539", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/DODINTEROPERABILITYROOTCA1.crl", + "306c310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49312730250603550403131e446f4420496e7465726f7065726162696c69747920526f6f742043412031", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/DODINTEROPERABILITYROOTCA2.crl", + "306c310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49312730250603550403131e446f4420496e7465726f7065726162696c69747920526f6f742043412032", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/USDODCCEBINTEROPERABILITYROOTCA1.crl", + "3074310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49312f302d06035504031326555320446f44204343454220496e7465726f7065726162696c69747920526f6f742043412031", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/USDODCCEBINTEROPERABILITYROOTCA2.crl", + "3074310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49312f302d06035504031326555320446f44204343454220496e7465726f7065726162696c69747920526f6f742043412032", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/DODNIPRINTERNALNPEROOTCA1.crl", + "3075310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f4431143012060355040b130b496e7465726e616c4e5045312830260603550403131f446f44204e49505220496e7465726e616c204e504520526f6f742043412031", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/DODNPEROOTCA1.crl", + "305f310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311a301806035504031311446f44204e504520526f6f742043412031", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/DMDNSIGNINGCA_1.crl", + "305f310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311a301806035504031311444d444e205369676e696e672043412d31", # pragma: allowlist secret + ), + ( + "https://crl.gds.disa.mil/crl/DODWCFROOTCA1.crl", + "3063310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f443110300e060355040b130757434620504b49311a301806035504031311446f442057434620526f6f742043412031", # pragma: allowlist secret + ), ] -def scan_for_issuer_and_next_update(crl): - """ - Scans a CRL file byte-by-byte to find the issuer and nextUpdate fields. - - Per RFC 5280, the issuer is the fourth ASN1 sequence element to occur in a - DER-encoded CRL file (https://tools.ietf.org/html/rfc5280#section-5.1). - This function takes a brute-force approach and scans the file until if find - the fourth sequence, then begins collecting that and the following bytes to - construct the issuer. It stop doing this when it finds \x17, which begins - the thisUpdate field. It then scans for the next UTCTime element (the next - occurrence of \x17) and grabs the 13 following bytes. It parses the ASN1 - UTCTime byte string to derive a datetime object. - - :param crl: - The path to a CRL file on-disk. - - :return: - A two-element tuple. The first element is the raw DER bytes of the - issuer, the second is the parsed Python datetime object for nextUpdate. - """ - with open(crl, "rb") as f: - byte = f.read(1) - sequences = 0 - issuer_finished = False - issuer = b"" - while byte: - if not issuer_finished: - if byte == b"0" and sequences < 4: - sequences += 1 - - if byte == b"\x17" and sequences == 4: - issuer_finished = True - - if sequences == 4 and not issuer_finished: - issuer += byte - else: - if byte == b"\x17": - byte_str = f.read(13) - next_update = datetime.strptime( - byte_str[1:].decode(), "%y%m%d%H%M%S" - ) - return (issuer, next_update) - - byte = f.read(1) - - raise CRLParseError("CRL could not be scanned.") - - -def build_crl_locations_cache(crl_locations, logger=None): - crl_cache = {} - for crl_location in crl_locations: - try: - issuer_der, next_update = scan_for_issuer_and_next_update(crl_location) - crl_cache[issuer_der] = {"location": crl_location, "expires": next_update} - except CRLParseError: - if logger: - logger.warning( - "CRL could not be scanned for caching: {}".format(crl_location) - ) - continue - - return crl_cache - - JSON_CACHE = "crl_locations.json" -def _serialize_cache_items(cache): - return { - der.hex(): { - k: v.timestamp() if hasattr(v, "timestamp") else v - for (k, v) in data.items() - } - for (der, data) in cache.items() - } - - def _deserialize_cache_items(cache): - return { - bytes.fromhex(der): { - k: datetime.fromtimestamp(v) if isinstance(v, float) else v - for (k, v) in data.items() - } - for (der, data) in cache.items() - } - - -def serialize_crl_locations_cache(crl_dir, logger=None): - crl_locations = [ - "{}/{}".format(crl_dir, crl_path) for crl_path in os.listdir(crl_dir) - ] - location_cache = build_crl_locations_cache(crl_locations, logger=logger) - json_location = "{}/{}".format(crl_dir, JSON_CACHE) - with open(json_location, "w") as json_file: - json_ready = _serialize_cache_items(location_cache) - json.dump(json_ready, json_file) - - return location_cache + return {bytes.fromhex(der): data for (der, data) in cache.items()} def load_crl_locations_cache(crl_dir): @@ -183,6 +251,20 @@ def load_crl_locations_cache(crl_dir): return _deserialize_cache_items(cache) +def serialize_crl_locations_cache(crl_dir, crl_list=CRL_LIST): + crl_cache = {} + for crl_uri, crl_issuer in crl_list: + crl_path = crl_local_path(crl_dir, crl_uri) + if os.path.isfile(crl_path): + crl_cache[crl_issuer] = crl_path + + json_location = "{}/{}".format(crl_dir, JSON_CACHE) + with open(json_location, "w") as json_file: + json.dump(crl_cache, json_file) + + return {bytes.fromhex(k): v for k, v in crl_cache.items()} + + def crl_local_path(out_dir, crl_location): name = re.split("/", crl_location)[-1] crl = os.path.join(out_dir, name) @@ -214,14 +296,14 @@ def write_crl(out_dir, target_dir, crl_location): raise CRLNotFoundError() if response.status_code == 304: - return False + return (False, existing) with open(crl, "wb") as crl_file: for chunk in response.iter_content(chunk_size=1024): if chunk: crl_file.write(chunk) - return True + return (True, existing) def remove_bad_crl(out_dir, crl_location): @@ -238,19 +320,32 @@ def log_error(logger, crl_location): ) -def refresh_crls(out_dir, target_dir, logger): - for crl_location in CRL_LIST: - logger.info("updating CRL from {}".format(crl_location)) - try: - if write_crl(out_dir, target_dir, crl_location): - logger.info("successfully synced CRL from {}".format(crl_location)) - else: - logger.info("no updates for CRL from {}".format(crl_location)) - except requests.exceptions.ChunkedEncodingError: - log_error(logger, crl_location) - remove_bad_crl(out_dir, crl_location) - except CRLNotFoundError: - log_error(logger, crl_location) +def refresh_crl(out_dir, target_dir, crl_uri, logger): + logger.info("updating CRL from {}".format(crl_uri)) + try: + was_updated, crl_path = write_crl(out_dir, target_dir, crl_uri) + if was_updated: + logger.info("successfully synced CRL from {}".format(crl_uri)) + else: + logger.info("no updates for CRL from {}".format(crl_uri)) + + return crl_path + except requests.exceptions.ChunkedEncodingError: + log_error(logger, crl_uri) + remove_bad_crl(out_dir, crl_uri) + except CRLNotFoundError: + log_error(logger, crl_uri) + + +def sync_crls(tmp_location, final_location): + crl_cache = {} + for crl_uri, crl_issuer in CRL_LIST: + crl_path = refresh_crl(tmp_location, final_location, crl_uri, logger) + crl_cache[crl_issuer] = crl_path + + json_location = "{}/{}".format(final_location, JSON_CACHE) + with open(json_location, "w") as json_file: + json.dump(crl_cache, json_file) if __name__ == "__main__": @@ -265,8 +360,7 @@ if __name__ == "__main__": try: tmp_location = sys.argv[1] final_location = sys.argv[2] - refresh_crls(tmp_location, final_location, logger) - serialize_crl_locations_cache(tmp_location, logger=logger) + sync_crls(tmp_location, final_location) except Exception as err: logger.exception("Fatal error encountered, stopping") sys.exit(1) diff --git a/tests/check_crl_parse.py b/tests/check_crl_parse.py index a6f0d94c..efc38084 100644 --- a/tests/check_crl_parse.py +++ b/tests/check_crl_parse.py @@ -1,19 +1,16 @@ import os import pytest -from atst.domain.authnid.crl.util import scan_for_issuer_and_next_update +from atst.domain.authnid.crl.util import crl_local_path, CRL_LIST from tests.utils import parse_for_issuer_and_next_update CRL_DIR = "crls" -_CRLS = [ - "{}/{}".format(CRL_DIR, file_) for file_ in os.listdir(CRL_DIR) if ".crl" in file_ -] -@pytest.mark.parametrize("crl_path", _CRLS) -def test_crl_scan_against_parse(crl_path): +@pytest.mark.parametrize("crl_uri, issuer", CRL_LIST) +def test_crl_scan_against_parse(crl_uri, issuer): + crl_path = crl_local_path(CRL_DIR, crl_uri) parsed_der = parse_for_issuer_and_next_update(crl_path) - scanned_der = scan_for_issuer_and_next_update(crl_path) - assert parsed_der == scanned_der + assert issuer == parsed_der.hex() diff --git a/tests/domain/authnid/test_crl.py b/tests/domain/authnid/test_crl.py index 0460fc30..37686150 100644 --- a/tests/domain/authnid/test_crl.py +++ b/tests/domain/authnid/test_crl.py @@ -14,16 +14,14 @@ from atst.domain.authnid.crl import ( NoOpCRLCache, ) from atst.domain.authnid.crl.util import ( - scan_for_issuer_and_next_update, - build_crl_locations_cache, - serialize_crl_locations_cache, load_crl_locations_cache, + serialize_crl_locations_cache, CRLParseError, JSON_CACHE, ) from tests.mocks import FIXTURE_EMAIL_ADDRESS, DOD_CN -from tests.utils import FakeLogger, parse_for_issuer_and_next_update +from tests.utils import FakeLogger, parse_for_issuer_and_next_update, make_crl_list class MockX509Store: @@ -43,14 +41,13 @@ class MockX509Store: def test_can_build_crl_list(crl_file, ca_key, ca_file, make_crl, tmpdir): crl = make_crl(ca_key) - dir_ = os.path.dirname(crl_file) - serialize_crl_locations_cache(dir_) - cache = CRLCache(ca_file, dir_, store_class=MockX509Store) issuer_der = crl.issuer.public_bytes(default_backend()) + dir_ = os.path.dirname(crl_file) + serialize_crl_locations_cache(dir_, crl_list=[(str(crl_file), issuer_der.hex())]) + cache = CRLCache(ca_file, dir_, store_class=MockX509Store) assert len(cache.crl_cache.keys()) == 1 assert issuer_der in cache.crl_cache - assert cache.crl_cache[issuer_der]["location"] == crl_file - assert cache.crl_cache[issuer_der]["expires"] == crl.next_update + assert cache.crl_cache[issuer_der] == crl_file def test_can_build_trusted_root_list(app): @@ -81,7 +78,8 @@ def test_crl_validation_on_login( serialize_pki_object_to_disk(crl, crl_file, encoding=Encoding.DER) crl_dir = os.path.dirname(crl_file) - cache = CRLCache(ca_file, crl_dir) + crl_list = make_crl_list(good_cert, crl_file) + cache = CRLCache(ca_file, crl_dir, crl_list=crl_list) assert cache.crl_check(good_cert.public_bytes(Encoding.PEM).decode()) with pytest.raises(CRLRevocationException): cache.crl_check(bad_cert.public_bytes(Encoding.PEM).decode()) @@ -97,9 +95,10 @@ def test_can_dynamically_update_crls( serialize_pki_object_to_disk, ): crl_dir = os.path.dirname(crl_file) - cache = CRLCache(ca_file, crl_dir) client_cert = make_x509(rsa_key(), signer_key=ca_key, cn="chewbacca") client_pem = client_cert.public_bytes(Encoding.PEM) + crl_list = make_crl_list(client_cert, crl_file) + cache = CRLCache(ca_file, crl_dir, crl_list=crl_list) assert cache.crl_check(client_pem) revoked_crl = make_crl(ca_key, expired_serials=[client_cert.serial_number]) @@ -147,7 +146,8 @@ def test_expired_crl_raises_CRLInvalidException_with_failover_config_false( client_cert = make_x509(rsa_key(), signer_key=ca_key, cn="chewbacca") client_pem = client_cert.public_bytes(Encoding.PEM) crl_dir = os.path.dirname(expired_crl_file) - crl_cache = CRLCache(ca_file, crl_dir) + crl_list = make_crl_list(client_cert, expired_crl_file) + crl_cache = CRLCache(ca_file, crl_dir, crl_list=crl_list) with pytest.raises(CRLInvalidException): crl_cache.crl_check(client_pem) @@ -158,72 +158,13 @@ def test_expired_crl_passes_with_failover_config_true( client_cert = make_x509(rsa_key(), signer_key=ca_key, cn="chewbacca") client_pem = client_cert.public_bytes(Encoding.PEM) crl_dir = os.path.dirname(expired_crl_file) - crl_cache = CRLCache(ca_file, crl_dir) + crl_list = make_crl_list(client_cert, expired_crl_file) + crl_cache = CRLCache(ca_file, crl_dir, crl_list=crl_list) assert crl_cache.crl_check(client_pem) -def test_updates_expired_certs( - rsa_key, ca_file, expired_crl_file, crl_file, ca_key, make_x509 -): - """ - Given a CRLCache object with an expired CRL and a function for updating the - CRLs, the CRLCache should run the update function before checking a - certificate that requires the expired CRL. - """ - client_cert = make_x509(rsa_key(), signer_key=ca_key, cn="chewbacca") - client_pem = client_cert.public_bytes(Encoding.PEM) - - def _crl_update_func(): - shutil.copyfile(crl_file, expired_crl_file) - - crl_dir = os.path.dirname(expired_crl_file) - crl_cache = CRLCache(ca_file, crl_dir, crl_update_func=_crl_update_func) - crl_cache.crl_check(client_pem) - - -def test_scan_for_issuer_and_next_update(crl_file): - parsed = parse_for_issuer_and_next_update(crl_file) - scanned = scan_for_issuer_and_next_update(crl_file) - assert parsed == scanned - - -@pytest.fixture -def bad_crl(tmpdir): - bad_file = tmpdir.join("bad.crl") - with open(bad_file, "wb") as bad: - bad.write(b"definitely not a crl") - - return bad_file - - -def test_scan_for_issuer_and_next_update_with_bad_data(bad_crl): - with pytest.raises(CRLParseError): - scan_for_issuer_and_next_update(bad_crl) - - -def test_build_crl_locations_cache(crl_file): - issuer_der, next_update = parse_for_issuer_and_next_update(crl_file) - cache = build_crl_locations_cache([crl_file]) - assert cache == {issuer_der: {"location": crl_file, "expires": next_update}} - - -def test_build_crl_locations_cache_with_bad_data(crl_file, bad_crl): - logger = FakeLogger() - issuer_der, next_update = parse_for_issuer_and_next_update(crl_file) - cache = build_crl_locations_cache([crl_file, bad_crl], logger=logger) - assert cache == {issuer_der: {"location": crl_file, "expires": next_update}} - assert logger.messages - assert str(bad_crl) in logger.messages[0] - - -def test_serialize_crl_locations_cache(crl_file, bad_crl): - dir_ = os.path.dirname(crl_file) - serialize_crl_locations_cache(dir_) - assert os.path.isfile("{}/{}".format(dir_, JSON_CACHE)) - - -def test_load_crl_locations_cache(crl_file, bad_crl): +def test_load_crl_locations_cache(crl_file): dir_ = os.path.dirname(crl_file) serialize_crl_locations_cache(dir_) cache = load_crl_locations_cache(dir_) diff --git a/tests/test_auth.py b/tests/test_auth.py index de1c03f5..d6160491 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -6,14 +6,16 @@ from datetime import datetime from flask import session, url_for from cryptography.hazmat.primitives.serialization import Encoding -from .mocks import DOD_SDN_INFO, DOD_SDN, FIXTURE_EMAIL_ADDRESS from atst.domain.users import Users from atst.domain.permission_sets import PermissionSets from atst.domain.exceptions import NotFoundError from atst.domain.authnid.crl import CRLInvalidException from atst.domain.auth import UNPROTECTED_ROUTES from atst.domain.authnid.crl import CRLCache + from .factories import UserFactory +from .mocks import DOD_SDN_INFO, DOD_SDN, FIXTURE_EMAIL_ADDRESS +from .utils import make_crl_list MOCK_USER = {"id": "438567dd-25fa-4d83-a8cc-8aa8366cb24a"} @@ -149,7 +151,8 @@ def swap_crl_cache( crl = make_crl(ca_key) serialize_pki_object_to_disk(crl, crl_file, encoding=Encoding.DER) crl_dir = os.path.dirname(crl_file) - app.crl_cache = CRLCache(ca_file, crl_dir) + crl_list = make_crl_list(crl, crl_file) + app.crl_cache = CRLCache(ca_file, crl_dir, crl_list=crl_list) yield _swap_crl_cache @@ -175,7 +178,8 @@ def test_crl_validation_on_login( serialize_pki_object_to_disk(crl, crl_file, encoding=Encoding.DER) crl_dir = os.path.dirname(crl_file) - cache = CRLCache(ca_file, crl_dir) + crl_list = make_crl_list(good_cert, crl_file) + cache = CRLCache(ca_file, crl_dir, crl_list=crl_list) swap_crl_cache(cache) # bad cert is on the test CRL diff --git a/tests/utils.py b/tests/utils.py index 1a36b819..152c347a 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,7 +1,10 @@ -from flask import template_rendered from contextlib import contextmanager +import os from unittest.mock import Mock + from OpenSSL import crypto +from cryptography.hazmat.backends import default_backend +from flask import template_rendered from atst.utils.notification_sender import NotificationSender @@ -49,5 +52,10 @@ FakeNotificationSender = lambda: Mock(spec=NotificationSender) def parse_for_issuer_and_next_update(crl): with open(crl, "rb") as crl_file: parsed = crypto.load_crl(crypto.FILETYPE_ASN1, crl_file.read()) - next_update = parsed.to_cryptography().next_update - return (parsed.get_issuer().der(), next_update) + return parsed.get_issuer().der() + + +def make_crl_list(x509_obj, x509_path): + issuer = x509_obj.issuer.public_bytes(default_backend()) + filename = os.path.basename(x509_path) + return [(filename, issuer.hex())]