pk/atst
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Clean up defunct upload and CRL logic.

Browse Source 
- Applies our previous CSP namespacing pattern to the upload classes.
- Removes code and config for previous uploader implementation.
- Removes Attachment model's ability to upload files directly and
  adjusts tests that expected that behavior.
This commit is contained in:
dandds
2019-08-08 13:20:27 -04:00
parent 334a280610
commit 9fc6514d80
13 changed files with 39 additions and 292 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
atst/app.py
View File

@@ -30,7 +30,6 @@ from atst.utils.json import CustomJSONEncoder
from atst.queue import queue
from atst.utils.notification_sender import NotificationSender
from atst.utils.session_limiter import SessionLimiter
from atst.domain.csp.file_uploads import build_uploader
from logging.config import dictConfig
from atst.utils.logging import JsonFormatter, RequestContextFilter
@@ -61,7 +60,7 @@ def make_app(config):
make_flask_callbacks(app)
register_filters(app)
make_csp_provider(app)
make_csp_provider(app, config.get("CSP", "mock"))
make_crl_validator(app)
make_mailer(app)
make_notification_sender(app)
@@ -79,7 +78,6 @@ def make_app(config):
app.register_blueprint(task_orders_bp)
app.register_blueprint(applications_bp)
app.register_blueprint(user_routes)
app.uploader = build_uploader(app.config)
if ENV != "prod":
app.register_blueprint(dev_routes)
@@ -228,10 +226,7 @@ def make_crl_validator(app):
for filename in pathlib.Path(app.config["CRL_STORAGE_CONTAINER"]).glob("*.crl"):
crl_locations.append(filename.absolute())
app.crl_cache = CRLCache(
app.config["CA_CHAIN"],
crl_locations,
logger=app.logger,
crl_update_func=app.csp.crls.sync_crls,
app.config["CA_CHAIN"], crl_locations, logger=app.logger
)

28
atst/domain/csp/__init__.py
View File

@@ -1,15 +1,33 @@
from .cloud import MockCloudProvider
from .files import RackspaceFileProvider, RackspaceCRLProvider
from .file_uploads import AwsUploader, AzureUploader, MockUploader
from .reports import MockReportingProvider
class MockCSP:
def __init__(self, app):
self.cloud = MockCloudProvider()
self.files = RackspaceFileProvider(app)
self.files = MockUploader(app)
self.reports = MockReportingProvider()
self.crls = RackspaceCRLProvider(app)
def make_csp_provider(app):
app.csp = MockCSP(app)
class AzureCSP:
def __init__(self, app):
self.cloud = MockCloudProvider()
self.files = AzureUploader(app.config)
self.reports = MockReportingProvider()
class AwsCSP:
def __init__(self, app):
self.cloud = MockCloudProvider()
self.files = AwsUploader(app.config)
self.reports = MockReportingProvider()
def make_csp_provider(app, csp=None):
if csp == "aws":
app.csp = AwsCSP(app)
elif csp == "azure":
app.csp = AzureCSP(app)
else:
app.csp = MockCSP(app)

10
atst/domain/csp/file_uploads.py
View File

@@ -2,16 +2,6 @@ from datetime import datetime, timedelta
from uuid import uuid4
def build_uploader(config):
csp = config.get("CSP")
if csp == "aws":
return AwsUploader(config)
elif csp == "azure":
return AzureUploader(config)
else:
return MockUploader(config)
class Uploader:
def generate_token(self):
pass

125
atst/domain/csp/files.py
View File

@@ -1,125 +0,0 @@
import os
import tarfile
from tempfile import NamedTemporaryFile, TemporaryDirectory
from uuid import uuid4
from libcloud.storage.types import Provider
from libcloud.storage.providers import get_driver
from atst.domain.exceptions import UploadError
class CSPFileError(Exception):
pass
class FileProviderInterface:
_PERMITTED_MIMETYPES = ["application/pdf"]
def _enforce_mimetype(self, fyle):
# TODO: for hardening, we should probably use a better library for
# determining mimetype and not rely on FileUpload's determination
# TODO: we should set MAX_CONTENT_LENGTH in the config to prevent large
# uploads
if not fyle.mimetype in self._PERMITTED_MIMETYPES:
raise UploadError(
"could not upload {} with mimetype {}".format(
fyle.filename, fyle.mimetype
)
)
def upload(self, fyle): # pragma: no cover
"""Store the file object `fyle` in the CSP. This method returns the
object name that can be used to later look up the file."""
raise NotImplementedError()
def download(self, object_name): # pragma: no cover
"""Retrieve the stored file represented by `object_name`. Returns a
file object.
"""
raise NotImplementedError()
def get_rackspace_container(provider, container=None, **kwargs):
if provider == "LOCAL": # pragma: no branch
kwargs["key"] = container
if not os.path.exists(container):
os.mkdir(container)
container = ""
driver = get_driver(getattr(Provider, provider))(**kwargs)
return driver.get_container(container)
class RackspaceFileProvider(FileProviderInterface):
def __init__(self, app):
self.container = get_rackspace_container(
provider=app.config.get("STORAGE_PROVIDER"),
container=app.config.get("STORAGE_CONTAINER"),
key=app.config.get("STORAGE_KEY"),
secret=app.config.get("STORAGE_SECRET"),
)
def upload(self, fyle):
self._enforce_mimetype(fyle)
object_name = uuid4().hex
with NamedTemporaryFile() as tempfile:
tempfile.write(fyle.stream.read())
tempfile.seek(0)
self.container.upload_object(
file_path=tempfile.name,
object_name=object_name,
extra={"acl": "private"},
)
return object_name
def download(self, object_name):
obj = self.container.get_object(object_name=object_name)
with NamedTemporaryFile() as tempfile:
obj.download(tempfile.name, overwrite_existing=True)
return open(tempfile.name, "rb")
class CRLProviderInterface:
def sync_crls(self): # pragma: no cover
"""
Retrieve copies of the CRLs and unpack them to disk.
"""
raise NotImplementedError()
class RackspaceCRLProvider(CRLProviderInterface):
def __init__(self, app):
provider = app.config.get("CRL_STORAGE_PROVIDER") or app.config.get(
"STORAGE_PROVIDER"
)
self.container = get_rackspace_container(
provider=provider,
container=app.config.get("CRL_STORAGE_CONTAINER"),
key=app.config.get("STORAGE_KEY"),
secret=app.config.get("STORAGE_SECRET"),
region=app.config.get("CRL_STORAGE_REGION"),
)
self._crl_dir = app.config.get("CRL_STORAGE_CONTAINER")
self._object_name = app.config.get("STORAGE_CRL_ARCHIVE_NAME")
self._object = None
@property
def object(self):
if self._object is None:
self._object = self.container.get_object(object_name=self._object_name)
return self._object
def sync_crls(self):
if not os.path.exists(self._crl_dir):
os.mkdir(self._crl_dir)
with TemporaryDirectory() as tempdir:
dl_path = os.path.join(tempdir, self._object_name)
success = self.object.download(dl_path, overwrite_existing=True)
if not success:
raise CSPFileError("The CRL package was not downloaded")
archive = tarfile.open(dl_path, "r:bz2")
archive.extractall(self._crl_dir)

22
atst/models/attachment.py
View File

@@ -1,11 +1,10 @@
from sqlalchemy import Column, String
from sqlalchemy.dialects.postgresql import UUID
from sqlalchemy.orm.exc import NoResultFound
from flask import current_app as app
from atst.models import Base, types, mixins
from atst.database import db
from atst.domain.exceptions import NotFoundError, UploadError
from atst.domain.exceptions import NotFoundError
class AttachmentError(Exception):
@@ -21,25 +20,6 @@ class Attachment(Base, mixins.TimestampsMixin):
resource = Column(String)
resource_id = Column(UUID(as_uuid=True), index=True)
@classmethod
def attach(cls, fyle, resource=None, resource_id=None):
try:
object_name = app.csp.files.upload(fyle)
except UploadError as e:
raise AttachmentError("Could not add attachment. " + str(e))
attachment = Attachment(
filename=fyle.filename,
object_name=object_name,
resource=resource,
resource_id=resource_id,
)
db.session.add(attachment)
db.session.commit()
return attachment
@classmethod
def get_or_create(cls, object_name, params):
try:

3
atst/models/task_order.py
View File

@@ -4,7 +4,6 @@ from enum import Enum
from sqlalchemy import Column, DateTime, ForeignKey, String
from sqlalchemy.ext.hybrid import hybrid_property
from sqlalchemy.orm import relationship
from werkzeug.datastructures import FileStorage
from atst.models import Attachment, Base, mixins, types
from atst.models.clin import JEDICLINType
@@ -57,8 +56,6 @@ class TaskOrder(Base, mixins.TimestampsMixin):
def _set_attachment(self, new_attachment, attribute):
if isinstance(new_attachment, Attachment):
return new_attachment
elif isinstance(new_attachment, FileStorage):
return Attachment.attach(new_attachment, "task_order", self.id)
elif isinstance(new_attachment, dict):
if new_attachment["filename"] and new_attachment["object_name"]:
attachment = Attachment.get_or_create(

2
atst/routes/task_orders/new.py
View File

@@ -17,7 +17,7 @@ from atst.utils.flash import formatted_flash as flash
def render_task_orders_edit(template, portfolio_id=None, task_order_id=None, form=None):
(token, object_name) = current_app.uploader.get_token()
(token, object_name) = current_app.csp.files.get_token()
render_args = {"token": token, "object_name": object_name}
if task_order_id: