Merge pull request #219 from dod-ccpo/pdf-uploads-#159940565

Pdf uploads

atst/app.py

@@ -19,6 +19,7 @@ from atst.routes.errors import make_error_pages
 from atst.domain.authnid.crl import CRLCache
 from atst.domain.auth import apply_authentication
 from atst.eda_client import MockEDAClient
+from atst.uploader import Uploader
 
 
 ENV = os.getenv("FLASK_ENV", "dev")
@@ -43,6 +44,7 @@ def make_app(config):
     make_crl_validator(app)
     register_filters(app)
     make_eda_client(app)
+    make_upload_storage(app)
 
     db.init_app(app)
     csrf.init_app(app)
@@ -143,3 +145,13 @@ def make_crl_validator(app):
 
 def make_eda_client(app):
     app.eda_client = MockEDAClient()
+
+
+def make_upload_storage(app):
+    uploader = Uploader(
+        provider=app.config.get("STORAGE_PROVIDER"),
+        container=app.config.get("STORAGE_CONTAINER"),
+        key=app.config.get("STORAGE_KEY"),
+        secret=app.config.get("STORAGE_SECRET"),
+    )
+    app.uploader = uploader

atst/domain/requests.py

@@ -3,6 +3,7 @@ from sqlalchemy import exists, and_, exc
 from sqlalchemy.sql import text
 from sqlalchemy.orm.exc import NoResultFound
 from sqlalchemy.orm.attributes import flag_modified
+from werkzeug.datastructures import FileStorage
 
 from atst.models.request import Request
 from atst.models.request_status_event import RequestStatusEvent, RequestStatus
@@ -244,11 +245,17 @@ WHERE requests_with_status.status = :status
             for (k, v) in financial_data.items()
             if k in TaskOrders.TASK_ORDER_DATA
         }
+
         if task_order_data:
             task_order_number = request_data.pop("task_order_number")
         else:
             task_order_number = request_data.get("task_order_number")
 
+        if "task_order" in request_data and isinstance(
+            request_data["task_order"], FileStorage
+        ):
+            task_order_data["pdf"] = request_data.pop("task_order")
+
         task_order = TaskOrders.get_or_create_task_order(
             task_order_number, task_order_data
         )

atst/domain/task_orders.py

@@ -3,6 +3,7 @@ from flask import current_app as app
 
 from atst.database import db
 from atst.models.task_order import TaskOrder, Source
+from atst.models.attachment import Attachment
 from .exceptions import NotFoundError
 
 
@@ -53,6 +54,12 @@ class TaskOrders(object):
 
         except NotFoundError:
             if task_order_data:
+                pdf_file = task_order_data.pop("pdf")
+                # should catch the error here
+                attachment = Attachment.attach(pdf_file)
                 return TaskOrders.create(
-                    **task_order_data, number=number, source=Source.MANUAL
+                    **task_order_data,
+                    number=number,
+                    source=Source.MANUAL,
+                    pdf=attachment,
                 )

atst/filters.py

@@ -1,4 +1,6 @@
 import re
+from flask import current_app as app
+from werkzeug.datastructures import FileStorage
 
 
 def iconSvg(name):
@@ -31,9 +33,24 @@ def getOptionLabel(value, options):
     return next(tup[1] for tup in options if tup[0] == value)
 
 
+def mixedContentToJson(value):
+    """
+    This coerces the file upload in form data to its filename
+    so that the data can be JSON serialized.
+    """
+    if (
+        isinstance(value, dict)
+        and "task_order" in value
+        and isinstance(value["task_order"], FileStorage)
+    ):
+        value["task_order"] = value["task_order"].filename
+    return app.jinja_env.filters["tojson"](value)
+
+
 def register_filters(app):
     app.jinja_env.filters["iconSvg"] = iconSvg
     app.jinja_env.filters["dollars"] = dollars
     app.jinja_env.filters["usPhone"] = usPhone
     app.jinja_env.filters["readableInteger"] = readableInteger
     app.jinja_env.filters["getOptionLabel"] = getOptionLabel
+    app.jinja_env.filters["mixedContentToJson"] = mixedContentToJson

atst/forms/financial.py

@@ -1,7 +1,8 @@
 import re
 from wtforms.fields.html5 import EmailField
-from wtforms.fields import StringField
+from wtforms.fields import StringField, FileField
 from wtforms.validators import Required, Email, Regexp
+from flask_wtf.file import FileAllowed
 
 from atst.domain.exceptions import NotFoundError
 from atst.domain.pe_numbers import PENumbers
@@ -214,3 +215,11 @@ class ExtendedFinancialForm(BaseFinancialForm):
         description="Review your task order document, the amounts for each CLIN must match exactly here",
         filters=[number_to_int],
     )
+
+    task_order = FileField(
+        "Upload a copy of your Task Order",
+        validators=[
+            FileAllowed(["pdf"], "Only PDF documents can be uploaded."),
+            Required(),
+        ],
+    )

atst/models/__init__.py

@@ -13,3 +13,4 @@ from .task_order import TaskOrder
 from .workspace import Workspace
 from .project import Project
 from .environment import Environment
+from .attachment import Attachment

atst/models/attachment.py

@@ -0,0 +1,32 @@
+from sqlalchemy import Column, Integer, String
+from flask import current_app as app
+
+from atst.models import Base
+from atst.database import db
+from atst.uploader import UploadError
+
+
+class AttachmentError(Exception):
+    pass
+
+
+class Attachment(Base):
+    __tablename__ = "attachments"
+
+    id = Column(Integer, primary_key=True)
+    filename = Column(String)
+    object_name = Column(String, unique=True)
+
+    @classmethod
+    def attach(cls, fyle):
+        try:
+            filename, object_name = app.uploader.upload(fyle)
+        except UploadError as e:
+            raise AttachmentError("Could not add attachment. " + str(e))
+
+        attachment = Attachment(filename=filename, object_name=object_name)
+
+        db.session.add(attachment)
+        db.session.commit()
+
+        return attachment

atst/models/task_order.py

@@ -1,6 +1,7 @@
 from enum import Enum
 
-from sqlalchemy import Column, Integer, String, Enum as SQLAEnum
+from sqlalchemy import Column, Integer, String, ForeignKey, Enum as SQLAEnum
+from sqlalchemy.orm import relationship
 
 from atst.models import Base
 
@@ -31,3 +32,6 @@ class TaskOrder(Base):
     clin_1003 = Column(Integer)
     clin_2001 = Column(Integer)
     clin_2003 = Column(Integer)
+
+    attachment_id = Column(ForeignKey("attachments.id"))
+    pdf = relationship("Attachment")

atst/routes/requests/financial_verification.py

@@ -30,7 +30,6 @@ def update_financial_verification(request_id):
     post_data = http_request.form
     existing_request = Requests.get(request_id)
     form = financial_form(post_data)
-
     rerender_args = dict(
         request_id=request_id, f=form, extended=http_request.args.get("extended")
     )

atst/uploader.py

@@ -0,0 +1,45 @@
+from uuid import uuid4
+from libcloud.storage.types import Provider
+from libcloud.storage.providers import get_driver
+
+
+class UploadError(Exception):
+    pass
+
+
+class Uploader:
+    _PERMITTED_MIMETYPES = ["application/pdf"]
+
+    def __init__(self, provider, container=None, key=None, secret=None):
+        self.container = self._get_container(provider, container, key, secret)
+
+    def upload(self, fyle):
+        # TODO: for hardening, we should probably use a better library for
+        # determining mimetype and not rely on FileUpload's determination
+        # TODO: we should set MAX_CONTENT_LENGTH in the config to prevent large
+        # uploads
+        if not fyle.mimetype in self._PERMITTED_MIMETYPES:
+            raise UploadError(
+                "could not upload {} with mimetype {}".format(
+                    fyle.filename, fyle.mimetype
+                )
+            )
+
+        object_name = uuid4().hex
+        self.container.upload_object_via_stream(
+            iterator=fyle.stream.__iter__(),
+            object_name=object_name,
+            extra={"acl": "private"},
+        )
+        return (fyle.filename, object_name)
+
+    def download(self, path):
+        pass
+
+    def _get_container(self, provider, container, key, secret):
+        if provider == "LOCAL":
+            key = container
+            container = ""
+
+        driver = get_driver(getattr(Provider, provider))(key=key, secret=secret)
+        return driver.get_container(container)