CRL Provider for syncing CRLs from cached source
This commit is contained in:
parent
beea7d11da
commit
9aa15d57e8
1
.gitignore
vendored
1
.gitignore
vendored
@ -36,6 +36,7 @@ config/dev.ini
|
||||
|
||||
# CRLs
|
||||
/crl
|
||||
/crls
|
||||
/crl-tmp
|
||||
*.bk
|
||||
|
||||
|
@ -200,7 +200,7 @@ def make_crl_validator(app):
|
||||
app.crl_cache = NoOpCRLCache(logger=app.logger)
|
||||
else:
|
||||
crl_locations = []
|
||||
for filename in pathlib.Path(app.config["CRL_DIRECTORY"]).glob("*.crl"):
|
||||
for filename in pathlib.Path(app.config["CRL_CONTAINER"]).glob("*.crl"):
|
||||
crl_locations.append(filename.absolute())
|
||||
app.crl_cache = CRLCache(
|
||||
app.config["CA_CHAIN"], crl_locations, logger=app.logger
|
||||
|
@ -1,113 +0,0 @@
|
||||
import requests
|
||||
import re
|
||||
import os
|
||||
import pendulum
|
||||
from html.parser import HTMLParser
|
||||
|
||||
_DISA_CRLS = "https://iasecontent.disa.mil/pki-pke/data/crls/dod_crldps.htm"
|
||||
|
||||
MODIFIED_TIME_BUFFER = 15 * 60
|
||||
|
||||
|
||||
def fetch_disa():
|
||||
response = requests.get(_DISA_CRLS)
|
||||
return response.text
|
||||
|
||||
|
||||
class DISAParser(HTMLParser):
|
||||
crl_list = []
|
||||
_CRL_MATCH = re.compile("DOD(ROOT|EMAIL|ID)?CA")
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
if tag == "a":
|
||||
href = [pair[1] for pair in attrs if pair[0] == "href"].pop()
|
||||
if re.search(self._CRL_MATCH, href):
|
||||
self.crl_list.append(href)
|
||||
|
||||
|
||||
def crl_list_from_disa_html(html):
|
||||
parser = DISAParser()
|
||||
parser.reset()
|
||||
parser.feed(html)
|
||||
return parser.crl_list
|
||||
|
||||
|
||||
def crl_local_path(out_dir, crl_location):
|
||||
name = re.split("/", crl_location)[-1]
|
||||
crl = os.path.join(out_dir, name)
|
||||
return crl
|
||||
|
||||
|
||||
def existing_crl_modification_time(crl):
|
||||
if os.path.exists(crl):
|
||||
prev_time = os.path.getmtime(crl)
|
||||
buffered = prev_time + MODIFIED_TIME_BUFFER
|
||||
mod_time = prev_time if pendulum.now().timestamp() < buffered else buffered
|
||||
dt = pendulum.from_timestamp(mod_time, tz="GMT")
|
||||
return dt.format("ddd, DD MMM YYYY HH:mm:ss zz")
|
||||
|
||||
else:
|
||||
return False
|
||||
|
||||
|
||||
def write_crl(out_dir, target_dir, crl_location):
|
||||
crl = crl_local_path(out_dir, crl_location)
|
||||
existing = crl_local_path(target_dir, crl_location)
|
||||
options = {"stream": True}
|
||||
mod_time = existing_crl_modification_time(existing)
|
||||
if mod_time:
|
||||
options["headers"] = {"If-Modified-Since": mod_time}
|
||||
|
||||
with requests.get(crl_location, **options) as response:
|
||||
if response.status_code == 304:
|
||||
return False
|
||||
|
||||
with open(crl, "wb") as crl_file:
|
||||
for chunk in response.iter_content(chunk_size=1024):
|
||||
if chunk:
|
||||
crl_file.write(chunk)
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def remove_bad_crl(out_dir, crl_location):
|
||||
crl = crl_local_path(out_dir, crl_location)
|
||||
if os.path.isfile(crl):
|
||||
os.remove(crl)
|
||||
|
||||
|
||||
def refresh_crls(out_dir, target_dir, logger):
|
||||
disa_html = fetch_disa()
|
||||
crl_list = crl_list_from_disa_html(disa_html)
|
||||
for crl_location in crl_list:
|
||||
logger.info("updating CRL from {}".format(crl_location))
|
||||
try:
|
||||
if write_crl(out_dir, target_dir, crl_location):
|
||||
logger.info("successfully synced CRL from {}".format(crl_location))
|
||||
else:
|
||||
logger.info("no updates for CRL from {}".format(crl_location))
|
||||
except requests.exceptions.RequestException:
|
||||
if logger:
|
||||
logger.error(
|
||||
"Error downloading {}, removing file and continuing anyway".format(
|
||||
crl_location
|
||||
)
|
||||
)
|
||||
remove_bad_crl(out_dir, crl_location)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
import logging
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO, format="[%(asctime)s]:%(levelname)s: %(message)s"
|
||||
)
|
||||
logger = logging.getLogger()
|
||||
logger.info("Updating CRLs")
|
||||
try:
|
||||
refresh_crls(sys.argv[1], sys.argv[2], logger)
|
||||
except Exception as err:
|
||||
logger.exception("Fatal error encountered, stopping")
|
||||
sys.exit(1)
|
||||
logger.info("Finished updating CRLs")
|
@ -1,5 +1,5 @@
|
||||
from .cloud import MockCloudProvider
|
||||
from .files import RackspaceFileProvider
|
||||
from .files import RackspaceFileProvider, RackspaceCRLProvider
|
||||
from .reports import MockReportingProvider
|
||||
|
||||
|
||||
@ -8,6 +8,7 @@ class MockCSP:
|
||||
self.cloud = MockCloudProvider()
|
||||
self.files = RackspaceFileProvider(app)
|
||||
self.reports = MockReportingProvider()
|
||||
self.crls = RackspaceCRLProvider(app)
|
||||
|
||||
|
||||
def make_csp_provider(app):
|
||||
|
@ -1,4 +1,6 @@
|
||||
from tempfile import NamedTemporaryFile
|
||||
import os
|
||||
import tarfile
|
||||
from tempfile import NamedTemporaryFile, TemporaryDirectory
|
||||
from uuid import uuid4
|
||||
|
||||
from libcloud.storage.types import Provider
|
||||
@ -34,23 +36,24 @@ class FileProviderInterface:
|
||||
raise NotImplementedError()
|
||||
|
||||
|
||||
def get_rackspace_container(provider, container=None, **kwargs):
|
||||
if provider == "LOCAL": # pragma: no branch
|
||||
kwargs["key"] = container
|
||||
container = ""
|
||||
|
||||
driver = get_driver(getattr(Provider, provider))(**kwargs)
|
||||
return driver.get_container(container)
|
||||
|
||||
|
||||
class RackspaceFileProvider(FileProviderInterface):
|
||||
def __init__(self, app):
|
||||
self.container = self._get_container(
|
||||
self.container = get_rackspace_container(
|
||||
provider=app.config.get("STORAGE_PROVIDER"),
|
||||
container=app.config.get("STORAGE_CONTAINER"),
|
||||
key=app.config.get("STORAGE_KEY"),
|
||||
secret=app.config.get("STORAGE_SECRET"),
|
||||
)
|
||||
|
||||
def _get_container(self, provider, container=None, key=None, secret=None):
|
||||
if provider == "LOCAL": # pragma: no branch
|
||||
key = container
|
||||
container = ""
|
||||
|
||||
driver = get_driver(getattr(Provider, provider))(key=key, secret=secret)
|
||||
return driver.get_container(container)
|
||||
|
||||
def upload(self, fyle):
|
||||
self._enforce_mimetype(fyle)
|
||||
|
||||
@ -70,3 +73,35 @@ class RackspaceFileProvider(FileProviderInterface):
|
||||
with NamedTemporaryFile() as tempfile:
|
||||
obj.download(tempfile.name, overwrite_existing=True)
|
||||
return open(tempfile.name, "rb")
|
||||
|
||||
|
||||
class CRLProviderInterface:
|
||||
def sync_crls(self): # pragma: no cover
|
||||
"""
|
||||
Retrieve copies of the CRLs and unpack them to disk.
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
|
||||
class RackspaceCRLProvider(CRLProviderInterface):
|
||||
def __init__(self, app):
|
||||
self.container = get_rackspace_container(
|
||||
provider=app.config.get("STORAGE_PROVIDER"),
|
||||
container=app.config.get("CRL_CONTAINER"),
|
||||
key=app.config.get("STORAGE_KEY"),
|
||||
secret=app.config.get("STORAGE_SECRET"),
|
||||
region=app.config.get("CRL_REGION"),
|
||||
)
|
||||
self._crl_dir = app.config.get("CRL_CONTAINER")
|
||||
self._object_name = app.config.get("STORAGE_CRL_ARCHIVE_NAME")
|
||||
|
||||
def sync_crls(self):
|
||||
if not os.path.exists(self._crl_dir):
|
||||
os.mkdir(self._crl_dir)
|
||||
|
||||
obj = self.container.get_object(object_name=self._object_name)
|
||||
with TemporaryDirectory() as tempdir:
|
||||
dl_path = os.path.join(tempdir, self._object_name)
|
||||
obj.download(dl_path, overwrite_existing=True)
|
||||
archive = tarfile.open(dl_path, "r:bz2")
|
||||
archive.extractall(self._crl_dir)
|
||||
|
@ -3,7 +3,8 @@ CAC_URL = http://localhost:8000/login-redirect
|
||||
CA_CHAIN = ssl/server-certs/ca-chain.pem
|
||||
CLASSIFIED = false
|
||||
COOKIE_SECRET = some-secret-please-replace
|
||||
CRL_DIRECTORY = crl
|
||||
CRL_CONTAINER = crls
|
||||
CRL_REGION = iad
|
||||
DISABLE_CRL_CHECK = false
|
||||
DEBUG = true
|
||||
ENVIRONMENT = dev
|
||||
@ -25,4 +26,5 @@ SESSION_TYPE = redis
|
||||
SESSION_USE_SIGNER = True
|
||||
STORAGE_CONTAINER=uploads
|
||||
STORAGE_PROVIDER=LOCAL
|
||||
STORAGE_CRL_ARCHIVE_NAME = dod_crls.tar.bz
|
||||
WTF_CSRF_ENABLED = true
|
||||
|
@ -3,5 +3,5 @@ DEBUG = false
|
||||
PGHOST = postgreshost
|
||||
PGDATABASE = atat_test
|
||||
REDIS_URI = redis://redishost:6379
|
||||
CRL_DIRECTORY = tests/fixtures/crl
|
||||
CRL_CONTAINER = tests/fixtures/crl
|
||||
WTF_CSRF_ENABLED = false
|
||||
|
@ -1,3 +1,3 @@
|
||||
[default]
|
||||
PGDATABASE = atat_selenium
|
||||
CRL_DIRECTORY = tests/fixtures/crl
|
||||
CRL_CONTAINER = tests/fixtures/crl
|
||||
|
@ -2,6 +2,6 @@
|
||||
DEBUG = false
|
||||
ENVIRONMENT = test
|
||||
PGDATABASE = atat_test
|
||||
CRL_DIRECTORY = tests/fixtures/crl
|
||||
CRL_CONTAINER = tests/fixtures/crl
|
||||
WTF_CSRF_ENABLED = false
|
||||
STORAGE_PROVIDER=LOCAL
|
||||
|
@ -1,22 +1,14 @@
|
||||
#!/bin/bash
|
||||
#! .venv/bin/python
|
||||
# Add root application dir to the python path
|
||||
import os
|
||||
import sys
|
||||
|
||||
# script/sync-crls: update the DOD CRLs and place them where authnid expects them
|
||||
set -e
|
||||
cd "$(dirname "$0")/.."
|
||||
parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
|
||||
sys.path.append(parent_dir)
|
||||
|
||||
if [[ $# -eq 0 ]]; then
|
||||
TMP_DIR=crl-tmp
|
||||
else
|
||||
TMP_DIR=$1
|
||||
fi
|
||||
from atst.app import make_config, make_app
|
||||
|
||||
mkdir -p $TMP_DIR
|
||||
pipenv run python ./atst/domain/authnid/crl/util.py $TMP_DIR crl
|
||||
mkdir -p crl
|
||||
rsync -rq --min-size 400 $TMP_DIR/. crl/.
|
||||
rm -rf $TMP_DIR
|
||||
|
||||
if [[ $FLASK_ENV != "prod" ]]; then
|
||||
# place our test CRL there
|
||||
cp ssl/client-certs/client-ca.der.crl crl/
|
||||
fi
|
||||
if __name__ == "__main__":
|
||||
config = make_config({"DISABLE_CRL_CHECK": True})
|
||||
app = make_app(config)
|
||||
app.csp.crls.sync_crls()
|
||||
|
@ -6,7 +6,6 @@ import shutil
|
||||
from OpenSSL import crypto, SSL
|
||||
|
||||
from atst.domain.authnid.crl import CRLCache, CRLRevocationException, NoOpCRLCache
|
||||
import atst.domain.authnid.crl.util as util
|
||||
|
||||
from tests.mocks import FIXTURE_EMAIL_ADDRESS, DOD_CN
|
||||
|
||||
@ -104,47 +103,6 @@ def test_multistep_certificate_chain():
|
||||
assert cache.crl_check(cert)
|
||||
|
||||
|
||||
def test_parse_disa_pki_list():
|
||||
with open("tests/fixtures/disa-pki.html") as disa:
|
||||
disa_html = disa.read()
|
||||
crl_list = util.crl_list_from_disa_html(disa_html)
|
||||
href_matches = re.findall("DOD(ROOT|EMAIL|ID)?CA", disa_html)
|
||||
assert len(crl_list) > 0
|
||||
assert len(crl_list) == len(href_matches)
|
||||
|
||||
|
||||
class MockStreamingResponse:
|
||||
def __init__(self, content_chunks, code=200):
|
||||
self.content_chunks = content_chunks
|
||||
self.status_code = code
|
||||
|
||||
def iter_content(self, chunk_size=0):
|
||||
return self.content_chunks
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *args):
|
||||
pass
|
||||
|
||||
|
||||
def test_write_crl(tmpdir, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"requests.get", lambda u, **kwargs: MockStreamingResponse([b"it worked"])
|
||||
)
|
||||
crl = "crl_1"
|
||||
assert util.write_crl(tmpdir, "random_target_dir", crl)
|
||||
assert [p.basename for p in tmpdir.listdir()] == [crl]
|
||||
assert [p.read() for p in tmpdir.listdir()] == ["it worked"]
|
||||
|
||||
|
||||
def test_skips_crl_if_it_has_not_been_modified(tmpdir, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"requests.get", lambda u, **kwargs: MockStreamingResponse([b"it worked"], 304)
|
||||
)
|
||||
assert not util.write_crl(tmpdir, "random_target_dir", "crl_file_name")
|
||||
|
||||
|
||||
class FakeLogger:
|
||||
def __init__(self):
|
||||
self.messages = []
|
||||
@ -159,26 +117,6 @@ class FakeLogger:
|
||||
self.messages.append(msg)
|
||||
|
||||
|
||||
def test_refresh_crls_with_error(tmpdir, monkeypatch):
|
||||
def _mock_create_connection(*args, **kwargs):
|
||||
raise TimeoutError
|
||||
|
||||
fake_crl = "https://fakecrl.com/fake.crl"
|
||||
|
||||
monkeypatch.setattr(
|
||||
"urllib3.util.connection.create_connection", _mock_create_connection
|
||||
)
|
||||
monkeypatch.setattr("atst.domain.authnid.crl.util.fetch_disa", lambda *args: None)
|
||||
monkeypatch.setattr(
|
||||
"atst.domain.authnid.crl.util.crl_list_from_disa_html", lambda *args: [fake_crl]
|
||||
)
|
||||
|
||||
logger = FakeLogger()
|
||||
util.refresh_crls(tmpdir, tmpdir, logger)
|
||||
|
||||
assert "Error downloading {}".format(fake_crl) in logger.messages[-1]
|
||||
|
||||
|
||||
def test_no_op_crl_cache_logs_common_name():
|
||||
logger = FakeLogger()
|
||||
cert = open("ssl/client-certs/atat.mil.crt", "rb").read()
|
||||
|
Loading…
x
Reference in New Issue
Block a user