Merge pull request #994 from dod-ccpo/cloud-pdf-uploads

CSP PDF uploads

atst/app.py

@@ -30,6 +30,7 @@ from atst.utils.json import CustomJSONEncoder
 from atst.queue import queue
 from atst.utils.notification_sender import NotificationSender
 from atst.utils.session_limiter import SessionLimiter
+from atst.domain.csp.file_uploads import build_uploader
 
 from logging.config import dictConfig
 from atst.utils.logging import JsonFormatter, RequestContextFilter
@@ -78,6 +79,7 @@ def make_app(config):
     app.register_blueprint(task_orders_bp)
     app.register_blueprint(applications_bp)
     app.register_blueprint(user_routes)
+    app.uploader = build_uploader(app.config)
 
     if ENV != "prod":
         app.register_blueprint(dev_routes)

atst/domain/csp/file_uploads.py

@@ -0,0 +1,107 @@
+from datetime import datetime, timedelta
+from uuid import uuid4
+
+
+def build_uploader(config):
+    csp = config.get("CSP")
+    if csp == "aws":
+        return AwsUploader(config)
+    elif csp == "azure":
+        return AzureUploader(config)
+    else:
+        return MockUploader(config)
+
+
+class Uploader:
+    def generate_token(self):
+        pass
+
+    def object_name(self):
+        return str(uuid4())
+
+
+class MockUploader(Uploader):
+    def __init__(self, config):
+        self.config = config
+
+    def get_token(self):
+        return ({}, self.object_name())
+
+
+class AzureUploader(Uploader):
+    def __init__(self, config):
+        self.account_name = config["AZURE_ACCOUNT_NAME"]
+        self.storage_key = config["AZURE_STORAGE_KEY"]
+        self.container_name = config["AZURE_TO_BUCKET_NAME"]
+        self.timeout = timedelta(seconds=config["PERMANENT_SESSION_LIFETIME"])
+
+        from azure.storage.common import CloudStorageAccount
+        from azure.storage.blob import BlobPermissions
+
+        self.CloudStorageAccount = CloudStorageAccount
+        self.BlobPermissions = BlobPermissions
+
+    def get_token(self):
+        """
+        Generates an Azure SAS token for pre-authorizing a file upload.
+
+        Returns a tuple in the following format: (token_dict, object_name), where
+            - token_dict has a `token` key which contains the SAS token as a string
+            - object_name is a string
+        """
+        account = self.CloudStorageAccount(
+            account_name=self.account_name, account_key=self.storage_key
+        )
+        bbs = account.create_block_blob_service()
+        object_name = self.object_name()
+        sas_token = bbs.generate_blob_shared_access_signature(
+            self.container_name,
+            object_name,
+            permission=self.BlobPermissions.CREATE,
+            expiry=datetime.utcnow() + self.timeout,
+            protocol="https",
+        )
+        return ({"token": sas_token}, object_name)
+
+
+class AwsUploader(Uploader):
+    def __init__(self, config):
+        self.access_key_id = config["AWS_ACCESS_KEY_ID"]
+        self.secret_key = config["AWS_SECRET_KEY"]
+        self.region_name = config["AWS_REGION_NAME"]
+        self.bucket_name = config["AWS_BUCKET_NAME"]
+        self.timeout_secs = config["PERMANENT_SESSION_LIFETIME"]
+
+        import boto3
+
+        self.boto3 = boto3
+
+    def get_token(self):
+        """
+        Generates an AWS presigned post for pre-authorizing a file upload.
+
+        Returns a tuple in the following format: (token_dict, object_name), where
+            - token_dict contains several fields that will be passed directly into the
+              form before being sent to S3
+            - object_name is a string
+        """
+        s3_client = self.boto3.client(
+            "s3",
+            aws_access_key_id=self.access_key_id,
+            aws_secret_access_key=self.secret_key,
+            config=boto3.session.Config(
+                signature_version="s3v4", region_name=self.region_name
+            ),
+        )
+        object_name = self.object_name()
+        presigned_post = s3_client.generate_presigned_post(
+            self.bucket_name,
+            object_name,
+            ExpiresIn=3600,
+            Conditions=[
+                ("eq", "$Content-Type", "application/pdf"),
+                ("starts-with", "$x-amz-meta-filename", ""),
+            ],
+            Fields={"Content-Type": "application/pdf", "x-amz-meta-filename": ""},
+        )
+        return (presigned_post, object_name)

atst/forms/task_order.py

@@ -2,19 +2,17 @@ from wtforms.fields import (
     BooleanField,
     DecimalField,
     FieldList,
-    FileField,
     FormField,
     StringField,
+    HiddenField,
 )
 from wtforms.fields.html5 import DateField
 from wtforms.validators import Required, Optional
-from flask_wtf.file import FileAllowed
 from flask_wtf import FlaskForm
 
 from .data import JEDI_CLIN_TYPES
 from .fields import SelectField
 from .forms import BaseForm
-from atst.forms.validators import FileLength
 from atst.utils.localization import translate
 
 
@@ -66,16 +64,18 @@ class CLINForm(FlaskForm):
             return valid
 
 
+class AttachmentForm(BaseForm):
+    filename = HiddenField(id="attachment_filename")
+    object_name = HiddenField(id="attachment_object_name")
+    accept = ".pdf,application/pdf"
+
+
 class TaskOrderForm(BaseForm):
     number = StringField(label=translate("forms.task_order.number_description"))
-    pdf = FileField(
-        None,
+    pdf = FormField(
+        AttachmentForm,
+        label=translate("task_orders.form.supporting_docs_size_limit"),
         description=translate("task_orders.form.supporting_docs_size_limit"),
-        validators=[
-            FileAllowed(["pdf"], translate("forms.task_order.file_format_not_allowed")),
-            FileLength(message=translate("forms.validators.file_length")),
-        ],
-        render_kw={"accept": ".pdf,application/pdf"},
     )
     clins = FieldList(FormField(CLINForm))
 

atst/models/attachment.py

@@ -40,6 +40,16 @@ class Attachment(Base, mixins.TimestampsMixin):
 
         return attachment
 
+    @classmethod
+    def get_or_create(cls, object_name, params):
+        try:
+            return db.session.query(Attachment).filter_by(object_name=object_name).one()
+        except NoResultFound:
+            new_attachment = cls(**params)
+            db.session.add(new_attachment)
+            db.session.commit()
+            return new_attachment
+
     @classmethod
     def get(cls, id_):
         try:

atst/models/task_order.py

@@ -59,6 +59,14 @@ class TaskOrder(Base, mixins.TimestampsMixin):
             return new_attachment
         elif isinstance(new_attachment, FileStorage):
             return Attachment.attach(new_attachment, "task_order", self.id)
+        elif isinstance(new_attachment, dict):
+            if new_attachment["filename"] and new_attachment["object_name"]:
+                attachment = Attachment.get_or_create(
+                    new_attachment["object_name"], new_attachment
+                )
+                return attachment
+            else:
+                return None
         elif not new_attachment and hasattr(self, attribute):
             return None
         else:

atst/routes/task_orders/new.py

@@ -1,4 +1,11 @@
-from flask import g, redirect, render_template, request as http_request, url_for
+from flask import (
+    g,
+    redirect,
+    render_template,
+    request as http_request,
+    url_for,
+    current_app,
+)
 
 from . import task_orders_bp
 from atst.domain.authz.decorator import user_can_access_decorator as user_can
@@ -10,7 +17,8 @@ from atst.utils.flash import formatted_flash as flash
 
 
 def render_task_orders_edit(template, portfolio_id=None, task_order_id=None, form=None):
-    render_args = {}
+    (token, object_name) = current_app.uploader.get_token()
+    render_args = {"token": token, "object_name": object_name}
 
     if task_order_id:
         task_order = TaskOrders.get(task_order_id)
@@ -110,7 +118,7 @@ def form_step_one_add_pdf(portfolio_id=None, task_order_id=None):
 @task_orders_bp.route("/task_orders/<task_order_id>/form/step_1", methods=["POST"])
 @user_can(Permissions.CREATE_TASK_ORDER, message="update task order form")
 def submit_form_step_one_add_pdf(portfolio_id=None, task_order_id=None):
-    form_data = {**http_request.form, **http_request.files}
+    form_data = {**http_request.form}
     next_page = "task_orders.form_step_two_add_number"
     current_template = "task_orders/step_1.html"