diff --git a/atst/app.py b/atst/app.py
index e58c43ec..0ba56929 100644
--- a/atst/app.py
+++ b/atst/app.py
@@ -246,11 +246,11 @@ def make_crl_validator(app):
     if app.config.get("DISABLE_CRL_CHECK"):
         app.crl_cache = NoOpCRLCache(logger=app.logger)
     else:
-        app.crl_cache = CRLCache(
-            app.config["CA_CHAIN"],
-            app.config["CRL_STORAGE_CONTAINER"],
-            logger=app.logger,
-        )
+        crl_dir = app.config["CRL_STORAGE_CONTAINER"]
+        if not os.path.isdir(crl_dir):
+            os.makedirs(crl_dir, exist_ok=True)
+
+        app.crl_cache = CRLCache(app.config["CA_CHAIN"], crl_dir, logger=app.logger,)
 
 
 def make_mailer(app):
diff --git a/tests/test_app.py b/tests/test_app.py
new file mode 100644
index 00000000..222f4a4f
--- /dev/null
+++ b/tests/test_app.py
@@ -0,0 +1,24 @@
+import os
+
+import pytest
+
+from atst.app import make_crl_validator
+
+
+@pytest.fixture
+def replace_crl_dir_config(app):
+    original = app.config.get("CRL_STORAGE_CONTAINER")
+
+    def _replace_crl_dir_config(crl_dir):
+        app.config.update({"CRL_STORAGE_CONTAINER": crl_dir})
+
+    yield _replace_crl_dir_config
+
+    app.config.update({"CRL_STORAGE_CONTAINER": original})
+
+
+def test_make_crl_validator_creates_crl_dir(app, tmpdir, replace_crl_dir_config):
+    crl_dir = tmpdir.join("new_crl_dir")
+    replace_crl_dir_config(crl_dir)
+    make_crl_validator(app)
+    assert os.path.isdir(crl_dir)