pk/atst
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Maintain static list of CRL URIs and issuers.

Browse Source 
The previous solution (ad-hoc stream-parsing the CRLs to obtain their
issuers and nextUpdate) was too cute. It began breaking on CRLs that had
an addition hex 0x30 byte somewhere in their header. I thought that 0x30
was a reserved character only to be used for tags in ASN1 encoded with
DER; turns out that's not true. Rather than write a full-fledged ASN1
stream-parser, the simplest solution is to just maintain the list of
issuers as a constant in the codebase. This is fine because the issuer
for a specific CRL URI should not change. If it does, we've probably got
bigger problems.

This also removes the Flask app's functionality for updating the local
CRL cache. This is being handled out-of-band by a Kubernetes CronJob
and is not a concern of the app's. This means that instances of the
CRLCache do not have to explicitly track expirations for CRLs.
Previously, the in-memory dictionary or CRL issuers and locations
included expirations; now it is flattened to not include that
information.

The CRLCache class has been updated to accept a crl_list kwargs so that
unit tests can provide their own alternative CRL lists, since we now
hard-code the expected CRLs and issuers. The nightly CRL check job has
been updated to check that the hard-coded list of issuers matches what
we get when we actually sync the CRLs.
This commit is contained in:
dandds 2019-11-11 09:49:01 -05:00
parent 3d92ac4840
commit 1b6239893b
6 changed files with 307 additions and 274 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
atst/domain/authnid/crl/__init__.py
View File

@ -7,7 +7,7 @@ from OpenSSL import crypto, SSL
from datetime import datetime from datetime import datetime
from flask import current_app as app from flask import current_app as app
from .util import load_crl_locations_cache, serialize_crl_locations_cache from .util import load_crl_locations_cache, serialize_crl_locations_cache, CRL_LIST
# error codes from OpenSSL: https://github.com/openssl/openssl/blob/2c75f03b39de2fa7d006bc0f0d7c58235a54d9bb/include/openssl/x509_vfy.h#L111 # error codes from OpenSSL: https://github.com/openssl/openssl/blob/2c75f03b39de2fa7d006bc0f0d7c58235a54d9bb/include/openssl/x509_vfy.h#L111
CRL_EXPIRED_ERROR_CODE = 12 CRL_EXPIRED_ERROR_CODE = 12
@ -75,13 +75,13 @@ class CRLCache(CRLInterface):
crl_dir, crl_dir,
store_class=crypto.X509Store, store_class=crypto.X509Store,
logger=None, logger=None,
crl_update_func=None, crl_list=CRL_LIST,
): ):
self._crl_dir = crl_dir self._crl_dir = crl_dir
self.logger = logger self.logger = logger
self.store_class = store_class self.store_class = store_class
self.certificate_authorities = {} self.certificate_authorities = {}
self._crl_update_func = crl_update_func self.crl_list = crl_list
self._load_roots(root_location) self._load_roots(root_location)
self._build_crl_cache() self._build_crl_cache()
@ -101,7 +101,9 @@ class CRLCache(CRLInterface):
try: try:
self.crl_cache = load_crl_locations_cache(self._crl_dir) self.crl_cache = load_crl_locations_cache(self._crl_dir)
except FileNotFoundError: except FileNotFoundError:
self.crl_cache = serialize_crl_locations_cache(self._crl_dir) self.crl_cache = serialize_crl_locations_cache(
self._crl_dir, crl_list=self.crl_list
)
def _load_crl(self, crl_location): def _load_crl(self, crl_location):
with open(crl_location, "rb") as crl_file: with open(crl_location, "rb") as crl_file:
@ -117,19 +119,17 @@ class CRLCache(CRLInterface):
store = self.store_class() store = self.store_class()
self._log("STORE ID: {}. Building store.".format(id(store))) self._log("STORE ID: {}. Building store.".format(id(store)))
store.set_flags(crypto.X509StoreFlags.CRL_CHECK) store.set_flags(crypto.X509StoreFlags.CRL_CHECK)
crl_info = self.crl_cache.get(issuer.der(), {}) crl_location = self.crl_cache.get(issuer.der())
issuer_name = get_common_name(issuer) issuer_name = get_common_name(issuer)
if not crl_info: if not crl_location:
raise CRLInvalidException( raise CRLInvalidException(
"Could not find matching CRL for issuer with Common Name {}".format( "Could not find matching CRL for issuer with Common Name {}".format(
issuer_name issuer_name
) )
) )
self._manage_expiring(crl_info["expires"]) crl = self._load_crl(crl_location)
crl = self._load_crl(crl_info["location"])
store.add_crl(crl) store.add_crl(crl)
self._log( self._log(
@ -141,17 +141,6 @@ class CRLCache(CRLInterface):
store = self._add_certificate_chain_to_store(store, crl.get_issuer()) store = self._add_certificate_chain_to_store(store, crl.get_issuer())
return store return store
def _manage_expiring(self, crl_expiry):
"""
If a CRL is expired and CRLCache has been given a function for updating
the physical CRL locations, run the update function and then rebuild
the CRL cache.
"""
current = datetime.now(crl_expiry.tzinfo)
if self._crl_update_func and current > crl_expiry:
self._crl_update_func()
self._build_crl_cache()
# this _should_ happen just twice for the DoD PKI (intermediary, root) but # this _should_ happen just twice for the DoD PKI (intermediary, root) but
# theoretically it can build a longer certificate chain # theoretically it can build a longer certificate chain

426
atst/domain/authnid/crl/util.py
View File

@ -1,4 +1,3 @@
from datetime import datetime
import json import json
import os import os
import re import re
@ -19,161 +18,230 @@ MODIFIED_TIME_BUFFER = 15 * 60
CRL_LIST = [ CRL_LIST = [
"http://crl.disa.mil/crl/DODROOTCA2.crl", (
"http://crl.disa.mil/crl/DODROOTCA3.crl", "https://crl.gds.disa.mil/crl/DODROOTCA2.crl",
"http://crl.disa.mil/crl/DODROOTCA4.crl", "305b310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311630140603550403130d446f4420526f6f742043412032", # pragma: allowlist secret
"http://crl.disa.mil/crl/DODROOTCA5.crl", ),
"http://crl.disa.mil/crl/DODIDCA_33.crl", (
"http://crl.disa.mil/crl/DODIDCA_34.crl", "https://crl.gds.disa.mil/crl/DODROOTCA3.crl",
"http://crl.disa.mil/crl/DODIDSWCA_35.crl", "305b310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311630140603550403130d446f4420526f6f742043412033", # pragma: allowlist secret
"http://crl.disa.mil/crl/DODIDSWCA_36.crl", ),
"http://crl.disa.mil/crl/DODIDSWCA_37.crl", (
"http://crl.disa.mil/crl/DODIDSWCA_38.crl", "https://crl.gds.disa.mil/crl/DODROOTCA4.crl",
"http://crl.disa.mil/crl/DODIDCA_39.crl", "305b310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311630140603550403130d446f4420526f6f742043412034", # pragma: allowlist secret
"http://crl.disa.mil/crl/DODIDCA_40.crl", ),
"http://crl.disa.mil/crl/DODIDCA_41.crl", (
"http://crl.disa.mil/crl/DODIDCA_42.crl", "https://crl.gds.disa.mil/crl/DODROOTCA5.crl",
"http://crl.disa.mil/crl/DODIDCA_43.crl", "305b310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311630140603550403130d446f4420526f6f742043412035", # pragma: allowlist secret
"http://crl.disa.mil/crl/DODIDCA_44.crl", ),
"http://crl.disa.mil/crl/DODIDSWCA_45.crl", (
"http://crl.disa.mil/crl/DODIDSWCA_46.crl", "https://crl.gds.disa.mil/crl/DODIDCA_33.crl",
"http://crl.disa.mil/crl/DODIDSWCA_47.crl", "305a310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311530130603550403130c444f442049442043412d3333", # pragma: allowlist secret
"http://crl.disa.mil/crl/DODIDSWCA_48.crl", ),
"http://crl.disa.mil/crl/DODIDCA_49.crl", (
"http://crl.disa.mil/crl/DODIDCA_50.crl", "https://crl.gds.disa.mil/crl/DODIDCA_34.crl",
"http://crl.disa.mil/crl/DODIDCA_51.crl", "305a310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311530130603550403130c444f442049442043412d3334", # pragma: allowlist secret
"http://crl.disa.mil/crl/DODIDCA_52.crl", ),
"http://crl.disa.mil/crl/DODIDCA_59.crl", (
"http://crl.disa.mil/crl/DODSWCA_53.crl", "https://crl.gds.disa.mil/crl/DODIDSWCA_35.crl",
"http://crl.disa.mil/crl/DODSWCA_54.crl", "305d310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311830160603550403130f444f442049442053572043412d3335", # pragma: allowlist secret
"http://crl.disa.mil/crl/DODSWCA_55.crl", ),
"http://crl.disa.mil/crl/DODSWCA_56.crl", (
"http://crl.disa.mil/crl/DODSWCA_57.crl", "https://crl.gds.disa.mil/crl/DODIDSWCA_36.crl",
"http://crl.disa.mil/crl/DODSWCA_58.crl", "305d310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311830160603550403130f444f442049442053572043412d3336", # pragma: allowlist secret
"http://crl.disa.mil/crl/DODSWCA_60.crl", ),
"http://crl.disa.mil/crl/DODSWCA_61.crl", (
"http://crl.disa.mil/crl/DODEMAILCA_33.crl", "https://crl.gds.disa.mil/crl/DODIDSWCA_37.crl",
"http://crl.disa.mil/crl/DODEMAILCA_34.crl", "305d310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311830160603550403130f444f442049442053572043412d3337", # pragma: allowlist secret
"http://crl.disa.mil/crl/DODEMAILCA_39.crl", ),
"http://crl.disa.mil/crl/DODEMAILCA_40.crl", (
"http://crl.disa.mil/crl/DODEMAILCA_41.crl", "https://crl.gds.disa.mil/crl/DODIDSWCA_38.crl",
"http://crl.disa.mil/crl/DODEMAILCA_42.crl", "305d310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311830160603550403130f444f442049442053572043412d3338", # pragma: allowlist secret
"http://crl.disa.mil/crl/DODEMAILCA_43.crl", ),
"http://crl.disa.mil/crl/DODEMAILCA_44.crl", (
"http://crl.disa.mil/crl/DODEMAILCA_49.crl", "https://crl.gds.disa.mil/crl/DODIDCA_39.crl",
"http://crl.disa.mil/crl/DODEMAILCA_50.crl", "305a310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311530130603550403130c444f442049442043412d3339", # pragma: allowlist secret
"http://crl.disa.mil/crl/DODEMAILCA_51.crl", ),
"http://crl.disa.mil/crl/DODEMAILCA_52.crl", (
"http://crl.disa.mil/crl/DODEMAILCA_59.crl", "https://crl.gds.disa.mil/crl/DODIDCA_40.crl",
"http://crl.disa.mil/crl/DODINTEROPERABILITYROOTCA1.crl", "305a310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311530130603550403130c444f442049442043412d3430", # pragma: allowlist secret
"http://crl.disa.mil/crl/DODINTEROPERABILITYROOTCA2.crl", ),
"http://crl.disa.mil/crl/USDODCCEBINTEROPERABILITYROOTCA1.crl", (
"http://crl.disa.mil/crl/USDODCCEBINTEROPERABILITYROOTCA2.crl", "https://crl.gds.disa.mil/crl/DODIDCA_41.crl",
"http://crl.disa.mil/crl/DODNIPRINTERNALNPEROOTCA1.crl", "305a310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311530130603550403130c444f442049442043412d3431", # pragma: allowlist secret
"http://crl.disa.mil/crl/DODNPEROOTCA1.crl", ),
"http://crl.disa.mil/crl/DMDNSIGNINGCA_1.crl", (
"http://crl.disa.mil/crl/DODWCFROOTCA1.crl", "https://crl.gds.disa.mil/crl/DODIDCA_42.crl",
"305a310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311530130603550403130c444f442049442043412d3432", # pragma: allowlist secret
),
(
"https://crl.gds.disa.mil/crl/DODIDCA_43.crl",
"305a310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311530130603550403130c444f442049442043412d3433", # pragma: allowlist secret
),
(
"https://crl.gds.disa.mil/crl/DODIDCA_44.crl",
"305a310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311530130603550403130c444f442049442043412d3434", # pragma: allowlist secret
),
(
"https://crl.gds.disa.mil/crl/DODIDSWCA_45.crl",
"305d310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311830160603550403130f444f442049442053572043412d3435", # pragma: allowlist secret
),
(
"https://crl.gds.disa.mil/crl/DODIDSWCA_46.crl",
"305d310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311830160603550403130f444f442049442053572043412d3436", # pragma: allowlist secret
),
(
"https://crl.gds.disa.mil/crl/DODIDSWCA_47.crl",
"305d310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311830160603550403130f444f442049442053572043412d3437", # pragma: allowlist secret
),
(
"https://crl.gds.disa.mil/crl/DODIDSWCA_48.crl",
"305d310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311830160603550403130f444f442049442053572043412d3438", # pragma: allowlist secret
),
(
"https://crl.gds.disa.mil/crl/DODIDCA_49.crl",
"305a310b300906035504061302555331183016060355040a0c0f552e532e20476f7665726e6d656e74310c300a060355040b0c03446f44310c300a060355040b0c03504b493115301306035504030c0c444f442049442043412d3439", # pragma: allowlist secret
),
(
"https://crl.gds.disa.mil/crl/DODIDCA_50.crl",
"305a310b300906035504061302555331183016060355040a0c0f552e532e20476f7665726e6d656e74310c300a060355040b0c03446f44310c300a060355040b0c03504b493115301306035504030c0c444f442049442043412d3530", # pragma: allowlist secret
),
(
"https://crl.gds.disa.mil/crl/DODIDCA_51.crl",
"305a310b300906035504061302555331183016060355040a0c0f552e532e20476f7665726e6d656e74310c300a060355040b0c03446f44310c300a060355040b0c03504b493115301306035504030c0c444f442049442043412d3531", # pragma: allowlist secret
),
(
"https://crl.gds.disa.mil/crl/DODIDCA_52.crl",
"305a310b300906035504061302555331183016060355040a0c0f552e532e20476f7665726e6d656e74310c300a060355040b0c03446f44310c300a060355040b0c03504b493115301306035504030c0c444f442049442043412d3532", # pragma: allowlist secret
),
(
"https://crl.gds.disa.mil/crl/DODIDCA_59.crl",
"305a310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311530130603550403130c444f442049442043412d3539", # pragma: allowlist secret
),
(
"https://crl.gds.disa.mil/crl/DODSWCA_53.crl",
"305a310b300906035504061302555331183016060355040a0c0f552e532e20476f7665726e6d656e74310c300a060355040b0c03446f44310c300a060355040b0c03504b493115301306035504030c0c444f442053572043412d3533", # pragma: allowlist secret
),
(
"https://crl.gds.disa.mil/crl/DODSWCA_54.crl",
"305a310b300906035504061302555331183016060355040a0c0f552e532e20476f7665726e6d656e74310c300a060355040b0c03446f44310c300a060355040b0c03504b493115301306035504030c0c444f442053572043412d3534", # pragma: allowlist secret
),
(
"https://crl.gds.disa.mil/crl/DODSWCA_55.crl",
"305a310b300906035504061302555331183016060355040a0c0f552e532e20476f7665726e6d656e74310c300a060355040b0c03446f44310c300a060355040b0c03504b493115301306035504030c0c444f442053572043412d3535", # pragma: allowlist secret
),
(
"https://crl.gds.disa.mil/crl/DODSWCA_56.crl",
"305a310b300906035504061302555331183016060355040a0c0f552e532e20476f7665726e6d656e74310c300a060355040b0c03446f44310c300a060355040b0c03504b493115301306035504030c0c444f442053572043412d3536", # pragma: allowlist secret
),
(
"https://crl.gds.disa.mil/crl/DODSWCA_57.crl",
"305a310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311530130603550403130c444f442053572043412d3537", # pragma: allowlist secret
),
(
"https://crl.gds.disa.mil/crl/DODSWCA_58.crl",
"305a310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311530130603550403130c444f442053572043412d3538", # pragma: allowlist secret
),
(
"https://crl.gds.disa.mil/crl/DODSWCA_60.crl",
"305a310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311530130603550403130c444f442053572043412d3630", # pragma: allowlist secret
),
(
"https://crl.gds.disa.mil/crl/DODSWCA_61.crl",
"305a310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311530130603550403130c444f442053572043412d3631", # pragma: allowlist secret
),
(
"https://crl.gds.disa.mil/crl/DODEMAILCA_33.crl",
"305d310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311830160603550403130f444f4420454d41494c2043412d3333", # pragma: allowlist secret
),
(
"https://crl.gds.disa.mil/crl/DODEMAILCA_34.crl",
"305d310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311830160603550403130f444f4420454d41494c2043412d3334", # pragma: allowlist secret
),
(
"https://crl.gds.disa.mil/crl/DODEMAILCA_39.crl",
"305d310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311830160603550403130f444f4420454d41494c2043412d3339", # pragma: allowlist secret
),
(
"https://crl.gds.disa.mil/crl/DODEMAILCA_40.crl",
"305d310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311830160603550403130f444f4420454d41494c2043412d3430", # pragma: allowlist secret
),
(
"https://crl.gds.disa.mil/crl/DODEMAILCA_41.crl",
"305d310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311830160603550403130f444f4420454d41494c2043412d3431", # pragma: allowlist secret
),
(
"https://crl.gds.disa.mil/crl/DODEMAILCA_42.crl",
"305d310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311830160603550403130f444f4420454d41494c2043412d3432", # pragma: allowlist secret
),
(
"https://crl.gds.disa.mil/crl/DODEMAILCA_43.crl",
"305d310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311830160603550403130f444f4420454d41494c2043412d3433", # pragma: allowlist secret
),
(
"https://crl.gds.disa.mil/crl/DODEMAILCA_44.crl",
"305d310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311830160603550403130f444f4420454d41494c2043412d3434", # pragma: allowlist secret
),
(
"https://crl.gds.disa.mil/crl/DODEMAILCA_49.crl",
"305d310b300906035504061302555331183016060355040a0c0f552e532e20476f7665726e6d656e74310c300a060355040b0c03446f44310c300a060355040b0c03504b493118301606035504030c0f444f4420454d41494c2043412d3439", # pragma: allowlist secret
),
(
"https://crl.gds.disa.mil/crl/DODEMAILCA_50.crl",
"305d310b300906035504061302555331183016060355040a0c0f552e532e20476f7665726e6d656e74310c300a060355040b0c03446f44310c300a060355040b0c03504b493118301606035504030c0f444f4420454d41494c2043412d3530", # pragma: allowlist secret
),
(
"https://crl.gds.disa.mil/crl/DODEMAILCA_51.crl",
"305d310b300906035504061302555331183016060355040a0c0f552e532e20476f7665726e6d656e74310c300a060355040b0c03446f44310c300a060355040b0c03504b493118301606035504030c0f444f4420454d41494c2043412d3531", # pragma: allowlist secret
),
(
"https://crl.gds.disa.mil/crl/DODEMAILCA_52.crl",
"305d310b300906035504061302555331183016060355040a0c0f552e532e20476f7665726e6d656e74310c300a060355040b0c03446f44310c300a060355040b0c03504b493118301606035504030c0f444f4420454d41494c2043412d3532", # pragma: allowlist secret
),
(
"https://crl.gds.disa.mil/crl/DODEMAILCA_59.crl",
"305d310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311830160603550403130f444f4420454d41494c2043412d3539", # pragma: allowlist secret
),
(
"https://crl.gds.disa.mil/crl/DODINTEROPERABILITYROOTCA1.crl",
"306c310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49312730250603550403131e446f4420496e7465726f7065726162696c69747920526f6f742043412031", # pragma: allowlist secret
),
(
"https://crl.gds.disa.mil/crl/DODINTEROPERABILITYROOTCA2.crl",
"306c310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49312730250603550403131e446f4420496e7465726f7065726162696c69747920526f6f742043412032", # pragma: allowlist secret
),
(
"https://crl.gds.disa.mil/crl/USDODCCEBINTEROPERABILITYROOTCA1.crl",
"3074310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49312f302d06035504031326555320446f44204343454220496e7465726f7065726162696c69747920526f6f742043412031", # pragma: allowlist secret
),
(
"https://crl.gds.disa.mil/crl/USDODCCEBINTEROPERABILITYROOTCA2.crl",
"3074310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49312f302d06035504031326555320446f44204343454220496e7465726f7065726162696c69747920526f6f742043412032", # pragma: allowlist secret
),
(
"https://crl.gds.disa.mil/crl/DODNIPRINTERNALNPEROOTCA1.crl",
"3075310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f4431143012060355040b130b496e7465726e616c4e5045312830260603550403131f446f44204e49505220496e7465726e616c204e504520526f6f742043412031", # pragma: allowlist secret
),
(
"https://crl.gds.disa.mil/crl/DODNPEROOTCA1.crl",
"305f310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311a301806035504031311446f44204e504520526f6f742043412031", # pragma: allowlist secret
),
(
"https://crl.gds.disa.mil/crl/DMDNSIGNINGCA_1.crl",
"305f310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f44310c300a060355040b1303504b49311a301806035504031311444d444e205369676e696e672043412d31", # pragma: allowlist secret
),
(
"https://crl.gds.disa.mil/crl/DODWCFROOTCA1.crl",
"3063310b300906035504061302555331183016060355040a130f552e532e20476f7665726e6d656e74310c300a060355040b1303446f443110300e060355040b130757434620504b49311a301806035504031311446f442057434620526f6f742043412031", # pragma: allowlist secret
),
] ]
def scan_for_issuer_and_next_update(crl):
"""
Scans a CRL file byte-by-byte to find the issuer and nextUpdate fields.
Per RFC 5280, the issuer is the fourth ASN1 sequence element to occur in a
DER-encoded CRL file (https://tools.ietf.org/html/rfc5280#section-5.1).
This function takes a brute-force approach and scans the file until if find
the fourth sequence, then begins collecting that and the following bytes to
construct the issuer. It stop doing this when it finds \x17, which begins
the thisUpdate field. It then scans for the next UTCTime element (the next
occurrence of \x17) and grabs the 13 following bytes. It parses the ASN1
UTCTime byte string to derive a datetime object.
:param crl:
The path to a CRL file on-disk.
:return:
A two-element tuple. The first element is the raw DER bytes of the
issuer, the second is the parsed Python datetime object for nextUpdate.
"""
with open(crl, "rb") as f:
byte = f.read(1)
sequences = 0
issuer_finished = False
issuer = b""
while byte:
if not issuer_finished:
if byte == b"0" and sequences < 4:
sequences += 1
if byte == b"\x17" and sequences == 4:
issuer_finished = True
if sequences == 4 and not issuer_finished:
issuer += byte
else:
if byte == b"\x17":
byte_str = f.read(13)
next_update = datetime.strptime(
byte_str[1:].decode(), "%y%m%d%H%M%S"
)
return (issuer, next_update)
byte = f.read(1)
raise CRLParseError("CRL could not be scanned.")
def build_crl_locations_cache(crl_locations, logger=None):
crl_cache = {}
for crl_location in crl_locations:
try:
issuer_der, next_update = scan_for_issuer_and_next_update(crl_location)
crl_cache[issuer_der] = {"location": crl_location, "expires": next_update}
except CRLParseError:
if logger:
logger.warning(
"CRL could not be scanned for caching: {}".format(crl_location)
)
continue
return crl_cache
JSON_CACHE = "crl_locations.json" JSON_CACHE = "crl_locations.json"
def _serialize_cache_items(cache):
return {
der.hex(): {
k: v.timestamp() if hasattr(v, "timestamp") else v
for (k, v) in data.items()
}
for (der, data) in cache.items()
}
def _deserialize_cache_items(cache): def _deserialize_cache_items(cache):
return { return {bytes.fromhex(der): data for (der, data) in cache.items()}
bytes.fromhex(der): {
k: datetime.fromtimestamp(v) if isinstance(v, float) else v
for (k, v) in data.items()
}
for (der, data) in cache.items()
}
def serialize_crl_locations_cache(crl_dir, logger=None):
crl_locations = [
"{}/{}".format(crl_dir, crl_path) for crl_path in os.listdir(crl_dir)
]
location_cache = build_crl_locations_cache(crl_locations, logger=logger)
json_location = "{}/{}".format(crl_dir, JSON_CACHE)
with open(json_location, "w") as json_file:
json_ready = _serialize_cache_items(location_cache)
json.dump(json_ready, json_file)
return location_cache
def load_crl_locations_cache(crl_dir): def load_crl_locations_cache(crl_dir):
@ -183,6 +251,20 @@ def load_crl_locations_cache(crl_dir):
return _deserialize_cache_items(cache) return _deserialize_cache_items(cache)
def serialize_crl_locations_cache(crl_dir, crl_list=CRL_LIST):
crl_cache = {}
for crl_uri, crl_issuer in crl_list:
crl_path = crl_local_path(crl_dir, crl_uri)
if os.path.isfile(crl_path):
crl_cache[crl_issuer] = crl_path
json_location = "{}/{}".format(crl_dir, JSON_CACHE)
with open(json_location, "w") as json_file:
json.dump(crl_cache, json_file)
return {bytes.fromhex(k): v for k, v in crl_cache.items()}
def crl_local_path(out_dir, crl_location): def crl_local_path(out_dir, crl_location):
name = re.split("/", crl_location)[-1] name = re.split("/", crl_location)[-1]
crl = os.path.join(out_dir, name) crl = os.path.join(out_dir, name)
@ -214,14 +296,14 @@ def write_crl(out_dir, target_dir, crl_location):
raise CRLNotFoundError() raise CRLNotFoundError()
if response.status_code == 304: if response.status_code == 304:
return False return (False, existing)
with open(crl, "wb") as crl_file: with open(crl, "wb") as crl_file:
for chunk in response.iter_content(chunk_size=1024): for chunk in response.iter_content(chunk_size=1024):
if chunk: if chunk:
crl_file.write(chunk) crl_file.write(chunk)
return True return (True, existing)
def remove_bad_crl(out_dir, crl_location): def remove_bad_crl(out_dir, crl_location):
@ -238,19 +320,32 @@ def log_error(logger, crl_location):
) )
def refresh_crls(out_dir, target_dir, logger): def refresh_crl(out_dir, target_dir, crl_uri, logger):
for crl_location in CRL_LIST: logger.info("updating CRL from {}".format(crl_uri))
logger.info("updating CRL from {}".format(crl_location)) try:
try: was_updated, crl_path = write_crl(out_dir, target_dir, crl_uri)
if write_crl(out_dir, target_dir, crl_location): if was_updated:
logger.info("successfully synced CRL from {}".format(crl_location)) logger.info("successfully synced CRL from {}".format(crl_uri))
else: else:
logger.info("no updates for CRL from {}".format(crl_location)) logger.info("no updates for CRL from {}".format(crl_uri))
except requests.exceptions.ChunkedEncodingError:
log_error(logger, crl_location) return crl_path
remove_bad_crl(out_dir, crl_location) except requests.exceptions.ChunkedEncodingError:
except CRLNotFoundError: log_error(logger, crl_uri)
log_error(logger, crl_location) remove_bad_crl(out_dir, crl_uri)
except CRLNotFoundError:
log_error(logger, crl_uri)
def sync_crls(tmp_location, final_location):
crl_cache = {}
for crl_uri, crl_issuer in CRL_LIST:
crl_path = refresh_crl(tmp_location, final_location, crl_uri, logger)
crl_cache[crl_issuer] = crl_path
json_location = "{}/{}".format(final_location, JSON_CACHE)
with open(json_location, "w") as json_file:
json.dump(crl_cache, json_file)
if __name__ == "__main__": if __name__ == "__main__":
@ -265,8 +360,7 @@ if __name__ == "__main__":
try: try:
tmp_location = sys.argv[1] tmp_location = sys.argv[1]
final_location = sys.argv[2] final_location = sys.argv[2]
refresh_crls(tmp_location, final_location, logger) sync_crls(tmp_location, final_location)
serialize_crl_locations_cache(tmp_location, logger=logger)
except Exception as err: except Exception as err:
logger.exception("Fatal error encountered, stopping") logger.exception("Fatal error encountered, stopping")
sys.exit(1) sys.exit(1)

13
tests/check_crl_parse.py
View File

@ -1,19 +1,16 @@
import os import os
import pytest import pytest
from atst.domain.authnid.crl.util import scan_for_issuer_and_next_update from atst.domain.authnid.crl.util import crl_local_path, CRL_LIST
from tests.utils import parse_for_issuer_and_next_update from tests.utils import parse_for_issuer_and_next_update
CRL_DIR = "crls" CRL_DIR = "crls"
_CRLS = [
"{}/{}".format(CRL_DIR, file_) for file_ in os.listdir(CRL_DIR) if ".crl" in file_
]
@pytest.mark.parametrize("crl_path", _CRLS) @pytest.mark.parametrize("crl_uri, issuer", CRL_LIST)
def test_crl_scan_against_parse(crl_path): def test_crl_scan_against_parse(crl_uri, issuer):
crl_path = crl_local_path(CRL_DIR, crl_uri)
parsed_der = parse_for_issuer_and_next_update(crl_path) parsed_der = parse_for_issuer_and_next_update(crl_path)
scanned_der = scan_for_issuer_and_next_update(crl_path) assert issuer == parsed_der.hex()
assert parsed_der == scanned_der

89
tests/domain/authnid/test_crl.py
View File

@ -14,16 +14,14 @@ from atst.domain.authnid.crl import (
NoOpCRLCache, NoOpCRLCache,
) )
from atst.domain.authnid.crl.util import ( from atst.domain.authnid.crl.util import (
scan_for_issuer_and_next_update,
build_crl_locations_cache,
serialize_crl_locations_cache,
load_crl_locations_cache, load_crl_locations_cache,
serialize_crl_locations_cache,
CRLParseError, CRLParseError,
JSON_CACHE, JSON_CACHE,
) )
from tests.mocks import FIXTURE_EMAIL_ADDRESS, DOD_CN from tests.mocks import FIXTURE_EMAIL_ADDRESS, DOD_CN
from tests.utils import FakeLogger, parse_for_issuer_and_next_update from tests.utils import FakeLogger, parse_for_issuer_and_next_update, make_crl_list
class MockX509Store: class MockX509Store:
@ -43,14 +41,13 @@ class MockX509Store:
def test_can_build_crl_list(crl_file, ca_key, ca_file, make_crl, tmpdir): def test_can_build_crl_list(crl_file, ca_key, ca_file, make_crl, tmpdir):
crl = make_crl(ca_key) crl = make_crl(ca_key)
dir_ = os.path.dirname(crl_file)
serialize_crl_locations_cache(dir_)
cache = CRLCache(ca_file, dir_, store_class=MockX509Store)
issuer_der = crl.issuer.public_bytes(default_backend()) issuer_der = crl.issuer.public_bytes(default_backend())
dir_ = os.path.dirname(crl_file)
serialize_crl_locations_cache(dir_, crl_list=[(str(crl_file), issuer_der.hex())])
cache = CRLCache(ca_file, dir_, store_class=MockX509Store)
assert len(cache.crl_cache.keys()) == 1 assert len(cache.crl_cache.keys()) == 1
assert issuer_der in cache.crl_cache assert issuer_der in cache.crl_cache
assert cache.crl_cache[issuer_der]["location"] == crl_file assert cache.crl_cache[issuer_der] == crl_file
assert cache.crl_cache[issuer_der]["expires"] == crl.next_update
def test_can_build_trusted_root_list(app): def test_can_build_trusted_root_list(app):
@ -81,7 +78,8 @@ def test_crl_validation_on_login(
serialize_pki_object_to_disk(crl, crl_file, encoding=Encoding.DER) serialize_pki_object_to_disk(crl, crl_file, encoding=Encoding.DER)
crl_dir = os.path.dirname(crl_file) crl_dir = os.path.dirname(crl_file)
cache = CRLCache(ca_file, crl_dir) crl_list = make_crl_list(good_cert, crl_file)
cache = CRLCache(ca_file, crl_dir, crl_list=crl_list)
assert cache.crl_check(good_cert.public_bytes(Encoding.PEM).decode()) assert cache.crl_check(good_cert.public_bytes(Encoding.PEM).decode())
with pytest.raises(CRLRevocationException): with pytest.raises(CRLRevocationException):
cache.crl_check(bad_cert.public_bytes(Encoding.PEM).decode()) cache.crl_check(bad_cert.public_bytes(Encoding.PEM).decode())
@ -97,9 +95,10 @@ def test_can_dynamically_update_crls(
serialize_pki_object_to_disk, serialize_pki_object_to_disk,
): ):
crl_dir = os.path.dirname(crl_file) crl_dir = os.path.dirname(crl_file)
cache = CRLCache(ca_file, crl_dir)
client_cert = make_x509(rsa_key(), signer_key=ca_key, cn="chewbacca") client_cert = make_x509(rsa_key(), signer_key=ca_key, cn="chewbacca")
client_pem = client_cert.public_bytes(Encoding.PEM) client_pem = client_cert.public_bytes(Encoding.PEM)
crl_list = make_crl_list(client_cert, crl_file)
cache = CRLCache(ca_file, crl_dir, crl_list=crl_list)
assert cache.crl_check(client_pem) assert cache.crl_check(client_pem)
revoked_crl = make_crl(ca_key, expired_serials=[client_cert.serial_number]) revoked_crl = make_crl(ca_key, expired_serials=[client_cert.serial_number])
@ -147,7 +146,8 @@ def test_expired_crl_raises_CRLInvalidException_with_failover_config_false(
client_cert = make_x509(rsa_key(), signer_key=ca_key, cn="chewbacca") client_cert = make_x509(rsa_key(), signer_key=ca_key, cn="chewbacca")
client_pem = client_cert.public_bytes(Encoding.PEM) client_pem = client_cert.public_bytes(Encoding.PEM)
crl_dir = os.path.dirname(expired_crl_file) crl_dir = os.path.dirname(expired_crl_file)
crl_cache = CRLCache(ca_file, crl_dir) crl_list = make_crl_list(client_cert, expired_crl_file)
crl_cache = CRLCache(ca_file, crl_dir, crl_list=crl_list)
with pytest.raises(CRLInvalidException): with pytest.raises(CRLInvalidException):
crl_cache.crl_check(client_pem) crl_cache.crl_check(client_pem)
@ -158,72 +158,13 @@ def test_expired_crl_passes_with_failover_config_true(
client_cert = make_x509(rsa_key(), signer_key=ca_key, cn="chewbacca") client_cert = make_x509(rsa_key(), signer_key=ca_key, cn="chewbacca")
client_pem = client_cert.public_bytes(Encoding.PEM) client_pem = client_cert.public_bytes(Encoding.PEM)
crl_dir = os.path.dirname(expired_crl_file) crl_dir = os.path.dirname(expired_crl_file)
crl_cache = CRLCache(ca_file, crl_dir) crl_list = make_crl_list(client_cert, expired_crl_file)
crl_cache = CRLCache(ca_file, crl_dir, crl_list=crl_list)
assert crl_cache.crl_check(client_pem) assert crl_cache.crl_check(client_pem)
def test_updates_expired_certs( def test_load_crl_locations_cache(crl_file):
rsa_key, ca_file, expired_crl_file, crl_file, ca_key, make_x509
):
"""
Given a CRLCache object with an expired CRL and a function for updating the
CRLs, the CRLCache should run the update function before checking a
certificate that requires the expired CRL.
"""
client_cert = make_x509(rsa_key(), signer_key=ca_key, cn="chewbacca")
client_pem = client_cert.public_bytes(Encoding.PEM)
def _crl_update_func():
shutil.copyfile(crl_file, expired_crl_file)
crl_dir = os.path.dirname(expired_crl_file)
crl_cache = CRLCache(ca_file, crl_dir, crl_update_func=_crl_update_func)
crl_cache.crl_check(client_pem)
def test_scan_for_issuer_and_next_update(crl_file):
parsed = parse_for_issuer_and_next_update(crl_file)
scanned = scan_for_issuer_and_next_update(crl_file)
assert parsed == scanned
@pytest.fixture
def bad_crl(tmpdir):
bad_file = tmpdir.join("bad.crl")
with open(bad_file, "wb") as bad:
bad.write(b"definitely not a crl")
return bad_file
def test_scan_for_issuer_and_next_update_with_bad_data(bad_crl):
with pytest.raises(CRLParseError):
scan_for_issuer_and_next_update(bad_crl)
def test_build_crl_locations_cache(crl_file):
issuer_der, next_update = parse_for_issuer_and_next_update(crl_file)
cache = build_crl_locations_cache([crl_file])
assert cache == {issuer_der: {"location": crl_file, "expires": next_update}}
def test_build_crl_locations_cache_with_bad_data(crl_file, bad_crl):
logger = FakeLogger()
issuer_der, next_update = parse_for_issuer_and_next_update(crl_file)
cache = build_crl_locations_cache([crl_file, bad_crl], logger=logger)
assert cache == {issuer_der: {"location": crl_file, "expires": next_update}}
assert logger.messages
assert str(bad_crl) in logger.messages[0]
def test_serialize_crl_locations_cache(crl_file, bad_crl):
dir_ = os.path.dirname(crl_file)
serialize_crl_locations_cache(dir_)
assert os.path.isfile("{}/{}".format(dir_, JSON_CACHE))
def test_load_crl_locations_cache(crl_file, bad_crl):
dir_ = os.path.dirname(crl_file) dir_ = os.path.dirname(crl_file)
serialize_crl_locations_cache(dir_) serialize_crl_locations_cache(dir_)
cache = load_crl_locations_cache(dir_) cache = load_crl_locations_cache(dir_)

10
tests/test_auth.py
View File

@ -6,14 +6,16 @@ from datetime import datetime
from flask import session, url_for from flask import session, url_for
from cryptography.hazmat.primitives.serialization import Encoding from cryptography.hazmat.primitives.serialization import Encoding
from .mocks import DOD_SDN_INFO, DOD_SDN, FIXTURE_EMAIL_ADDRESS
from atst.domain.users import Users from atst.domain.users import Users
from atst.domain.permission_sets import PermissionSets from atst.domain.permission_sets import PermissionSets
from atst.domain.exceptions import NotFoundError from atst.domain.exceptions import NotFoundError
from atst.domain.authnid.crl import CRLInvalidException from atst.domain.authnid.crl import CRLInvalidException
from atst.domain.auth import UNPROTECTED_ROUTES from atst.domain.auth import UNPROTECTED_ROUTES
from atst.domain.authnid.crl import CRLCache from atst.domain.authnid.crl import CRLCache
from .factories import UserFactory from .factories import UserFactory
from .mocks import DOD_SDN_INFO, DOD_SDN, FIXTURE_EMAIL_ADDRESS
from .utils import make_crl_list
MOCK_USER = {"id": "438567dd-25fa-4d83-a8cc-8aa8366cb24a"} MOCK_USER = {"id": "438567dd-25fa-4d83-a8cc-8aa8366cb24a"}
@ -149,7 +151,8 @@ def swap_crl_cache(
crl = make_crl(ca_key) crl = make_crl(ca_key)
serialize_pki_object_to_disk(crl, crl_file, encoding=Encoding.DER) serialize_pki_object_to_disk(crl, crl_file, encoding=Encoding.DER)
crl_dir = os.path.dirname(crl_file) crl_dir = os.path.dirname(crl_file)
app.crl_cache = CRLCache(ca_file, crl_dir) crl_list = make_crl_list(crl, crl_file)
app.crl_cache = CRLCache(ca_file, crl_dir, crl_list=crl_list)
yield _swap_crl_cache yield _swap_crl_cache
@ -175,7 +178,8 @@ def test_crl_validation_on_login(
serialize_pki_object_to_disk(crl, crl_file, encoding=Encoding.DER) serialize_pki_object_to_disk(crl, crl_file, encoding=Encoding.DER)
crl_dir = os.path.dirname(crl_file) crl_dir = os.path.dirname(crl_file)
cache = CRLCache(ca_file, crl_dir) crl_list = make_crl_list(good_cert, crl_file)
cache = CRLCache(ca_file, crl_dir, crl_list=crl_list)
swap_crl_cache(cache) swap_crl_cache(cache)
# bad cert is on the test CRL # bad cert is on the test CRL

14
tests/utils.py
View File

@ -1,7 +1,10 @@
from flask import template_rendered
from contextlib import contextmanager from contextlib import contextmanager
import os
from unittest.mock import Mock from unittest.mock import Mock
from OpenSSL import crypto from OpenSSL import crypto
from cryptography.hazmat.backends import default_backend
from flask import template_rendered
from atst.utils.notification_sender import NotificationSender from atst.utils.notification_sender import NotificationSender
@ -49,5 +52,10 @@ FakeNotificationSender = lambda: Mock(spec=NotificationSender)
def parse_for_issuer_and_next_update(crl): def parse_for_issuer_and_next_update(crl):
with open(crl, "rb") as crl_file: with open(crl, "rb") as crl_file:
parsed = crypto.load_crl(crypto.FILETYPE_ASN1, crl_file.read()) parsed = crypto.load_crl(crypto.FILETYPE_ASN1, crl_file.read())
next_update = parsed.to_cryptography().next_update return parsed.get_issuer().der()
return (parsed.get_issuer().der(), next_update)
def make_crl_list(x509_obj, x509_path):
issuer = x509_obj.issuer.public_bytes(default_backend())
filename = os.path.basename(x509_path)
return [(filename, issuer.hex())]