CRL Provider for syncing CRLs from cached source

This commit is contained in:
dandds
2019-02-24 13:47:38 -05:00
parent beea7d11da
commit 9aa15d57e8
11 changed files with 66 additions and 210 deletions

View File

@@ -200,7 +200,7 @@ def make_crl_validator(app):
app.crl_cache = NoOpCRLCache(logger=app.logger)
else:
crl_locations = []
for filename in pathlib.Path(app.config["CRL_DIRECTORY"]).glob("*.crl"):
for filename in pathlib.Path(app.config["CRL_CONTAINER"]).glob("*.crl"):
crl_locations.append(filename.absolute())
app.crl_cache = CRLCache(
app.config["CA_CHAIN"], crl_locations, logger=app.logger

View File

@@ -1,113 +0,0 @@
import requests
import re
import os
import pendulum
from html.parser import HTMLParser
_DISA_CRLS = "https://iasecontent.disa.mil/pki-pke/data/crls/dod_crldps.htm"
MODIFIED_TIME_BUFFER = 15 * 60
def fetch_disa():
response = requests.get(_DISA_CRLS)
return response.text
class DISAParser(HTMLParser):
crl_list = []
_CRL_MATCH = re.compile("DOD(ROOT|EMAIL|ID)?CA")
def handle_starttag(self, tag, attrs):
if tag == "a":
href = [pair[1] for pair in attrs if pair[0] == "href"].pop()
if re.search(self._CRL_MATCH, href):
self.crl_list.append(href)
def crl_list_from_disa_html(html):
parser = DISAParser()
parser.reset()
parser.feed(html)
return parser.crl_list
def crl_local_path(out_dir, crl_location):
name = re.split("/", crl_location)[-1]
crl = os.path.join(out_dir, name)
return crl
def existing_crl_modification_time(crl):
if os.path.exists(crl):
prev_time = os.path.getmtime(crl)
buffered = prev_time + MODIFIED_TIME_BUFFER
mod_time = prev_time if pendulum.now().timestamp() < buffered else buffered
dt = pendulum.from_timestamp(mod_time, tz="GMT")
return dt.format("ddd, DD MMM YYYY HH:mm:ss zz")
else:
return False
def write_crl(out_dir, target_dir, crl_location):
crl = crl_local_path(out_dir, crl_location)
existing = crl_local_path(target_dir, crl_location)
options = {"stream": True}
mod_time = existing_crl_modification_time(existing)
if mod_time:
options["headers"] = {"If-Modified-Since": mod_time}
with requests.get(crl_location, **options) as response:
if response.status_code == 304:
return False
with open(crl, "wb") as crl_file:
for chunk in response.iter_content(chunk_size=1024):
if chunk:
crl_file.write(chunk)
return True
def remove_bad_crl(out_dir, crl_location):
crl = crl_local_path(out_dir, crl_location)
if os.path.isfile(crl):
os.remove(crl)
def refresh_crls(out_dir, target_dir, logger):
disa_html = fetch_disa()
crl_list = crl_list_from_disa_html(disa_html)
for crl_location in crl_list:
logger.info("updating CRL from {}".format(crl_location))
try:
if write_crl(out_dir, target_dir, crl_location):
logger.info("successfully synced CRL from {}".format(crl_location))
else:
logger.info("no updates for CRL from {}".format(crl_location))
except requests.exceptions.RequestException:
if logger:
logger.error(
"Error downloading {}, removing file and continuing anyway".format(
crl_location
)
)
remove_bad_crl(out_dir, crl_location)
if __name__ == "__main__":
import sys
import logging
logging.basicConfig(
level=logging.INFO, format="[%(asctime)s]:%(levelname)s: %(message)s"
)
logger = logging.getLogger()
logger.info("Updating CRLs")
try:
refresh_crls(sys.argv[1], sys.argv[2], logger)
except Exception as err:
logger.exception("Fatal error encountered, stopping")
sys.exit(1)
logger.info("Finished updating CRLs")

View File

@@ -1,5 +1,5 @@
from .cloud import MockCloudProvider
from .files import RackspaceFileProvider
from .files import RackspaceFileProvider, RackspaceCRLProvider
from .reports import MockReportingProvider
@@ -8,6 +8,7 @@ class MockCSP:
self.cloud = MockCloudProvider()
self.files = RackspaceFileProvider(app)
self.reports = MockReportingProvider()
self.crls = RackspaceCRLProvider(app)
def make_csp_provider(app):

View File

@@ -1,4 +1,6 @@
from tempfile import NamedTemporaryFile
import os
import tarfile
from tempfile import NamedTemporaryFile, TemporaryDirectory
from uuid import uuid4
from libcloud.storage.types import Provider
@@ -34,23 +36,24 @@ class FileProviderInterface:
raise NotImplementedError()
def get_rackspace_container(provider, container=None, **kwargs):
if provider == "LOCAL": # pragma: no branch
kwargs["key"] = container
container = ""
driver = get_driver(getattr(Provider, provider))(**kwargs)
return driver.get_container(container)
class RackspaceFileProvider(FileProviderInterface):
def __init__(self, app):
self.container = self._get_container(
self.container = get_rackspace_container(
provider=app.config.get("STORAGE_PROVIDER"),
container=app.config.get("STORAGE_CONTAINER"),
key=app.config.get("STORAGE_KEY"),
secret=app.config.get("STORAGE_SECRET"),
)
def _get_container(self, provider, container=None, key=None, secret=None):
if provider == "LOCAL": # pragma: no branch
key = container
container = ""
driver = get_driver(getattr(Provider, provider))(key=key, secret=secret)
return driver.get_container(container)
def upload(self, fyle):
self._enforce_mimetype(fyle)
@@ -70,3 +73,35 @@ class RackspaceFileProvider(FileProviderInterface):
with NamedTemporaryFile() as tempfile:
obj.download(tempfile.name, overwrite_existing=True)
return open(tempfile.name, "rb")
class CRLProviderInterface:
def sync_crls(self): # pragma: no cover
"""
Retrieve copies of the CRLs and unpack them to disk.
"""
raise NotImplementedError()
class RackspaceCRLProvider(CRLProviderInterface):
def __init__(self, app):
self.container = get_rackspace_container(
provider=app.config.get("STORAGE_PROVIDER"),
container=app.config.get("CRL_CONTAINER"),
key=app.config.get("STORAGE_KEY"),
secret=app.config.get("STORAGE_SECRET"),
region=app.config.get("CRL_REGION"),
)
self._crl_dir = app.config.get("CRL_CONTAINER")
self._object_name = app.config.get("STORAGE_CRL_ARCHIVE_NAME")
def sync_crls(self):
if not os.path.exists(self._crl_dir):
os.mkdir(self._crl_dir)
obj = self.container.get_object(object_name=self._object_name)
with TemporaryDirectory() as tempdir:
dl_path = os.path.join(tempdir, self._object_name)
obj.download(dl_path, overwrite_existing=True)
archive = tarfile.open(dl_path, "r:bz2")
archive.extractall(self._crl_dir)