diff --git a/Pipfile b/Pipfile index 83fb25c8..4e169d3a 100644 --- a/Pipfile +++ b/Pipfile @@ -23,6 +23,9 @@ lockfile = "*" "flask-rq2" = "*" werkzeug = "*" PyYAML = "*" +azure-storage = "*" +azure-storage-common = "*" +boto3 = "*" [dev-packages] bandit = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 8d27c525..6fe43d84 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "9b0fa418945fbb89208d11a4d44ce4c03e6810cd536628a7b2e17a58e5fd377a" + "sha256": "d697cd7c279a761283ea2ffefab08d5854e4c0341b5646839b269ab0d999bf87" }, "pipfile-spec": 6, "requires": { @@ -38,6 +38,52 @@ ], "version": "==0.24.0" }, + "azure-common": { + "hashes": [ + "sha256:53b1195b8f20943ccc0e71a17849258f7781bc6db1c72edc7d6c055f79bd54e3", + "sha256:99ef36e74b6395329aada288764ce80504da16ecc8206cb9a72f55fb02e8b484" + ], + "version": "==1.1.23" + }, + "azure-nspkg": { + "hashes": [ + "sha256:1d0bbb2157cf57b1bef6c8c8e5b41133957364456c43b0a43599890023cca0a8", + "sha256:31a060caca00ed1ebd369fc7fe01a56768c927e404ebc92268f4d9d636435e28", + "sha256:e7d3cea6af63e667d87ba1ca4f8cd7cb4dfca678e4c55fc1cedb320760e39dd0" + ], + "version": "==3.0.2" + }, + "azure-storage": { + "hashes": [ + "sha256:4c406422e3edd41920bb1f0c3930c34fee3eb0d55258ef7ec7308ccbb9385ad5", + "sha256:fb6212dcbed91b49d9637aa5e8888eafdfcd523b7e560c8044d2d838bbd3ca5f" + ], + "index": "pypi", + "version": "==0.36.0" + }, + "azure-storage-common": { + "hashes": [ + "sha256:b01a491a18839b9d05a4fe3421458a0ddb5ab9443c14e487f40d16f9a1dc2fbe", + "sha256:ccedef5c67227bc4d6670ffd37cec18fb529a1b7c3a5e53e4096eb0cf23dc73f" + ], + "index": "pypi", + "version": "==2.1.0" + }, + "boto3": { + "hashes": [ + "sha256:666f37c5852f71925494fc2103b189deafe6702c1d9ae60bead5b1b6466de857", + "sha256:7b77b507221ec15550b02d492804166bcc61ef3a81312968065515a76aa1791b" + ], + "index": "pypi", + "version": "==1.9.202" + }, + "botocore": { + "hashes": [ + "sha256:71ca578701e746fe947c098e5dee06128d0f6ba98217ba7e29aff0dab8caf82f", + "sha256:e55003c46e71396a551d4b70f39286f8fc4094ac6cf90f5db8d7a68bb4af1f9d" + ], + "version": "==1.12.202" + }, "certifi": { "hashes": [ "sha256:046832c04d4e752f37383b628bc601a7ea7211496b4638f6514d0e5b9acc4939", @@ -120,6 +166,14 @@ ], "version": "==2.7" }, + "docutils": { + "hashes": [ + "sha256:02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6", + "sha256:51e64ef2ebfb29cae1faa133b3710143496eca21c530f3f71424d77687764274", + "sha256:7a4bd47eaf6596e1295ecb11361139febe29b084a87bf005bf899f9a42edc3c6" + ], + "version": "==0.14" + }, "flask": { "hashes": [ "sha256:13f9f196f330c7c2c5d7a5cf91af894110ca0215ac051b5844701f2bfd934d52", @@ -188,6 +242,13 @@ ], "version": "==2.10.1" }, + "jmespath": { + "hashes": [ + "sha256:3720a4b1bd659dd2eecad0666459b9788813e032b83e7ba58578e48254e0a0e6", + "sha256:bde2aef6f44302dfb30320115b17d030798de8c4110e28d5cf6cf91a7a31074c" + ], + "version": "==0.9.4" + }, "lockfile": { "hashes": [ "sha256:6aed02de03cba24efabcd600b30540140634fc06cfa603822d508d5361e9f799", @@ -198,9 +259,9 @@ }, "mako": { "hashes": [ - "sha256:f5a642d8c5699269ab62a68b296ff990767eb120f51e2e8f3d6afb16bdb57f4b" + "sha256:a36919599a9b7dc5d86a7a8988f23a9a3a3d083070023bab23d64f7f1d1e0a4b" ], - "version": "==1.0.14" + "version": "==1.1.0" }, "markupsafe": { "hashes": [ @@ -300,6 +361,7 @@ "sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb", "sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e" ], + "markers": "python_version >= '2.7'", "version": "==2.8.0" }, "python-editor": { @@ -319,28 +381,30 @@ }, "pyyaml": { "hashes": [ - "sha256:57acc1d8533cbe51f6662a55434f0dbecfa2b9eaf115bede8f6fd00115a0c0d3", - "sha256:588c94b3d16b76cfed8e0be54932e5729cc185caffaa5a451e7ad2f7ed8b4043", - "sha256:68c8dd247f29f9a0d09375c9c6b8fdc64b60810ebf07ba4cdd64ceee3a58c7b7", - "sha256:70d9818f1c9cd5c48bb87804f2efc8692f1023dac7f1a1a5c61d454043c1d265", - "sha256:86a93cccd50f8c125286e637328ff4eef108400dd7089b46a7be3445eecfa391", - "sha256:a0f329125a926876f647c9fa0ef32801587a12328b4a3c741270464e3e4fa778", - "sha256:a3c252ab0fa1bb0d5a3f6449a4826732f3eb6c0270925548cac342bc9b22c225", - "sha256:b4bb4d3f5e232425e25dda21c070ce05168a786ac9eda43768ab7f3ac2770955", - "sha256:cd0618c5ba5bda5f4039b9398bb7fb6a317bb8298218c3de25c47c4740e4b95e", - "sha256:ceacb9e5f8474dcf45b940578591c7f3d960e82f926c707788a570b51ba59190", - "sha256:fe6a88094b64132c4bb3b631412e90032e8cfe9745a58370462240b8cb7553cd" + "sha256:0113bc0ec2ad727182326b61326afa3d1d8280ae1122493553fd6f4397f33df9", + "sha256:01adf0b6c6f61bd11af6e10ca52b7d4057dd0be0343eb9283c878cf3af56aee4", + "sha256:5124373960b0b3f4aa7df1707e63e9f109b5263eca5976c66e08b1c552d4eaf8", + "sha256:5ca4f10adbddae56d824b2c09668e91219bb178a1eee1faa56af6f99f11bf696", + "sha256:7907be34ffa3c5a32b60b95f4d95ea25361c951383a894fec31be7252b2b6f34", + "sha256:7ec9b2a4ed5cad025c2278a1e6a19c011c80a3caaac804fd2d329e9cc2c287c9", + "sha256:87ae4c829bb25b9fe99cf71fbb2140c448f534e24c998cc60f39ae4f94396a73", + "sha256:9de9919becc9cc2ff03637872a440195ac4241c80536632fffeb6a1e25a74299", + "sha256:a5a85b10e450c66b49f98846937e8cfca1db3127a9d5d1e31ca45c3d0bef4c5b", + "sha256:b0997827b4f6a7c286c01c5f60384d218dca4ed7d9efa945c3e1aa623d5709ae", + "sha256:b631ef96d3222e62861443cc89d6563ba3eeb816eeb96b2629345ab795e53681", + "sha256:bf47c0607522fdbca6c9e817a6e81b08491de50f3766a7a0e6a5be7905961b41", + "sha256:f81025eddd0327c7d4cfe9b62cf33190e1e736cc6e97502b3ec425f574b3e7a8" ], "index": "pypi", - "version": "==5.1.1" + "version": "==5.1.2" }, "redis": { "hashes": [ - "sha256:1f2493b72f669a096c59ca7cf7f4067b1ab58a0a3c6bef51d5d8fd384298c6a2", - "sha256:b851b0ef53b6d416f1dc692dc168f3d390b37995925bfe29351b3a869976598c" + "sha256:45682ecf226c7611efe731974c4fa3390170ba045b9cdb26f0051114a5c2a68b", + "sha256:f2609a85e5f37f489ba3b5652e1175dc3711c4d7a7818c4f657615810afd23df" ], "index": "pypi", - "version": "==3.3.4" + "version": "==3.3.6" }, "requests": { "hashes": [ @@ -364,6 +428,13 @@ ], "version": "==0.9" }, + "s3transfer": { + "hashes": [ + "sha256:6efc926738a3cd576c2a79725fed9afde92378aa5c6a957e3af010cb019fac9d", + "sha256:b780f2411b824cb541dbcd2c713d0cb61c7d1bcadae204cdddda2b35cef493ba" + ], + "version": "==0.2.1" + }, "six": { "hashes": [ "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", @@ -391,6 +462,7 @@ "sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1", "sha256:dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232" ], + "markers": "python_version >= '3.4'", "version": "==1.25.3" }, "webassets": { @@ -736,10 +808,10 @@ }, "pbr": { "hashes": [ - "sha256:0ca44dc9fd3b04a22297c2a91082d8df2894862e8f4c86a49dac69eae9e85ca0", - "sha256:4aed6c1b1fa5020def0f22aed663d87b81bb3235f112490b07d2643d7a98c5b5" + "sha256:56e52299170b9492513c64be44736d27a512fa7e606f21942160b68ce510b4bc", + "sha256:9b321c204a88d8ab5082699469f52cc94c5da45c51f114113d01b3d993c24cdf" ], - "version": "==5.4.1" + "version": "==5.4.2" }, "pexpect": { "hashes": [ @@ -851,24 +923,27 @@ "sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb", "sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e" ], + "markers": "python_version >= '2.7'", "version": "==2.8.0" }, "pyyaml": { "hashes": [ - "sha256:57acc1d8533cbe51f6662a55434f0dbecfa2b9eaf115bede8f6fd00115a0c0d3", - "sha256:588c94b3d16b76cfed8e0be54932e5729cc185caffaa5a451e7ad2f7ed8b4043", - "sha256:68c8dd247f29f9a0d09375c9c6b8fdc64b60810ebf07ba4cdd64ceee3a58c7b7", - "sha256:70d9818f1c9cd5c48bb87804f2efc8692f1023dac7f1a1a5c61d454043c1d265", - "sha256:86a93cccd50f8c125286e637328ff4eef108400dd7089b46a7be3445eecfa391", - "sha256:a0f329125a926876f647c9fa0ef32801587a12328b4a3c741270464e3e4fa778", - "sha256:a3c252ab0fa1bb0d5a3f6449a4826732f3eb6c0270925548cac342bc9b22c225", - "sha256:b4bb4d3f5e232425e25dda21c070ce05168a786ac9eda43768ab7f3ac2770955", - "sha256:cd0618c5ba5bda5f4039b9398bb7fb6a317bb8298218c3de25c47c4740e4b95e", - "sha256:ceacb9e5f8474dcf45b940578591c7f3d960e82f926c707788a570b51ba59190", - "sha256:fe6a88094b64132c4bb3b631412e90032e8cfe9745a58370462240b8cb7553cd" + "sha256:0113bc0ec2ad727182326b61326afa3d1d8280ae1122493553fd6f4397f33df9", + "sha256:01adf0b6c6f61bd11af6e10ca52b7d4057dd0be0343eb9283c878cf3af56aee4", + "sha256:5124373960b0b3f4aa7df1707e63e9f109b5263eca5976c66e08b1c552d4eaf8", + "sha256:5ca4f10adbddae56d824b2c09668e91219bb178a1eee1faa56af6f99f11bf696", + "sha256:7907be34ffa3c5a32b60b95f4d95ea25361c951383a894fec31be7252b2b6f34", + "sha256:7ec9b2a4ed5cad025c2278a1e6a19c011c80a3caaac804fd2d329e9cc2c287c9", + "sha256:87ae4c829bb25b9fe99cf71fbb2140c448f534e24c998cc60f39ae4f94396a73", + "sha256:9de9919becc9cc2ff03637872a440195ac4241c80536632fffeb6a1e25a74299", + "sha256:a5a85b10e450c66b49f98846937e8cfca1db3127a9d5d1e31ca45c3d0bef4c5b", + "sha256:b0997827b4f6a7c286c01c5f60384d218dca4ed7d9efa945c3e1aa623d5709ae", + "sha256:b631ef96d3222e62861443cc89d6563ba3eeb816eeb96b2629345ab795e53681", + "sha256:bf47c0607522fdbca6c9e817a6e81b08491de50f3766a7a0e6a5be7905961b41", + "sha256:f81025eddd0327c7d4cfe9b62cf33190e1e736cc6e97502b3ec425f574b3e7a8" ], "index": "pypi", - "version": "==5.1.1" + "version": "==5.1.2" }, "selenium": { "hashes": [ @@ -946,6 +1021,7 @@ "sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1", "sha256:dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232" ], + "markers": "python_version >= '3.4'", "version": "==1.25.3" }, "watchdog": { diff --git a/atst/app.py b/atst/app.py index af9f1a0c..e72a9cdd 100644 --- a/atst/app.py +++ b/atst/app.py @@ -30,6 +30,7 @@ from atst.utils.json import CustomJSONEncoder from atst.queue import queue from atst.utils.notification_sender import NotificationSender from atst.utils.session_limiter import SessionLimiter +from atst.domain.csp.file_uploads import build_uploader from logging.config import dictConfig from atst.utils.logging import JsonFormatter, RequestContextFilter @@ -78,6 +79,7 @@ def make_app(config): app.register_blueprint(task_orders_bp) app.register_blueprint(applications_bp) app.register_blueprint(user_routes) + app.uploader = build_uploader(app.config) if ENV != "prod": app.register_blueprint(dev_routes) diff --git a/atst/domain/csp/file_uploads.py b/atst/domain/csp/file_uploads.py new file mode 100644 index 00000000..e1c1cb9d --- /dev/null +++ b/atst/domain/csp/file_uploads.py @@ -0,0 +1,107 @@ +from datetime import datetime, timedelta +from uuid import uuid4 + + +def build_uploader(config): + csp = config.get("CSP") + if csp == "aws": + return AwsUploader(config) + elif csp == "azure": + return AzureUploader(config) + else: + return MockUploader(config) + + +class Uploader: + def generate_token(self): + pass + + def object_name(self): + return str(uuid4()) + + +class MockUploader(Uploader): + def __init__(self, config): + self.config = config + + def get_token(self): + return ({}, self.object_name()) + + +class AzureUploader(Uploader): + def __init__(self, config): + self.account_name = config["AZURE_ACCOUNT_NAME"] + self.storage_key = config["AZURE_STORAGE_KEY"] + self.container_name = config["AZURE_TO_BUCKET_NAME"] + self.timeout = timedelta(seconds=config["PERMANENT_SESSION_LIFETIME"]) + + from azure.storage.common import CloudStorageAccount + from azure.storage.blob import BlobPermissions + + self.CloudStorageAccount = CloudStorageAccount + self.BlobPermissions = BlobPermissions + + def get_token(self): + """ + Generates an Azure SAS token for pre-authorizing a file upload. + + Returns a tuple in the following format: (token_dict, object_name), where + - token_dict has a `token` key which contains the SAS token as a string + - object_name is a string + """ + account = self.CloudStorageAccount( + account_name=self.account_name, account_key=self.storage_key + ) + bbs = account.create_block_blob_service() + object_name = self.object_name() + sas_token = bbs.generate_blob_shared_access_signature( + self.container_name, + object_name, + permission=self.BlobPermissions.CREATE, + expiry=datetime.utcnow() + self.timeout, + protocol="https", + ) + return ({"token": sas_token}, object_name) + + +class AwsUploader(Uploader): + def __init__(self, config): + self.access_key_id = config["AWS_ACCESS_KEY_ID"] + self.secret_key = config["AWS_SECRET_KEY"] + self.region_name = config["AWS_REGION_NAME"] + self.bucket_name = config["AWS_BUCKET_NAME"] + self.timeout_secs = config["PERMANENT_SESSION_LIFETIME"] + + import boto3 + + self.boto3 = boto3 + + def get_token(self): + """ + Generates an AWS presigned post for pre-authorizing a file upload. + + Returns a tuple in the following format: (token_dict, object_name), where + - token_dict contains several fields that will be passed directly into the + form before being sent to S3 + - object_name is a string + """ + s3_client = self.boto3.client( + "s3", + aws_access_key_id=self.access_key_id, + aws_secret_access_key=self.secret_key, + config=boto3.session.Config( + signature_version="s3v4", region_name=self.region_name + ), + ) + object_name = self.object_name() + presigned_post = s3_client.generate_presigned_post( + self.bucket_name, + object_name, + ExpiresIn=3600, + Conditions=[ + ("eq", "$Content-Type", "application/pdf"), + ("starts-with", "$x-amz-meta-filename", ""), + ], + Fields={"Content-Type": "application/pdf", "x-amz-meta-filename": ""}, + ) + return (presigned_post, object_name) diff --git a/atst/forms/task_order.py b/atst/forms/task_order.py index 80a31085..c9c9e19b 100644 --- a/atst/forms/task_order.py +++ b/atst/forms/task_order.py @@ -2,19 +2,17 @@ from wtforms.fields import ( BooleanField, DecimalField, FieldList, - FileField, FormField, StringField, + HiddenField, ) from wtforms.fields.html5 import DateField from wtforms.validators import Required, Optional -from flask_wtf.file import FileAllowed from flask_wtf import FlaskForm from .data import JEDI_CLIN_TYPES from .fields import SelectField from .forms import BaseForm -from atst.forms.validators import FileLength from atst.utils.localization import translate @@ -66,16 +64,18 @@ class CLINForm(FlaskForm): return valid +class AttachmentForm(BaseForm): + filename = HiddenField(id="attachment_filename") + object_name = HiddenField(id="attachment_object_name") + accept = ".pdf,application/pdf" + + class TaskOrderForm(BaseForm): number = StringField(label=translate("forms.task_order.number_description")) - pdf = FileField( - None, + pdf = FormField( + AttachmentForm, + label=translate("task_orders.form.supporting_docs_size_limit"), description=translate("task_orders.form.supporting_docs_size_limit"), - validators=[ - FileAllowed(["pdf"], translate("forms.task_order.file_format_not_allowed")), - FileLength(message=translate("forms.validators.file_length")), - ], - render_kw={"accept": ".pdf,application/pdf"}, ) clins = FieldList(FormField(CLINForm)) diff --git a/atst/models/attachment.py b/atst/models/attachment.py index e4a9d6c2..4edb3441 100644 --- a/atst/models/attachment.py +++ b/atst/models/attachment.py @@ -40,6 +40,16 @@ class Attachment(Base, mixins.TimestampsMixin): return attachment + @classmethod + def get_or_create(cls, object_name, params): + try: + return db.session.query(Attachment).filter_by(object_name=object_name).one() + except NoResultFound: + new_attachment = cls(**params) + db.session.add(new_attachment) + db.session.commit() + return new_attachment + @classmethod def get(cls, id_): try: diff --git a/atst/models/task_order.py b/atst/models/task_order.py index 247ff2c8..8b060164 100644 --- a/atst/models/task_order.py +++ b/atst/models/task_order.py @@ -59,6 +59,14 @@ class TaskOrder(Base, mixins.TimestampsMixin): return new_attachment elif isinstance(new_attachment, FileStorage): return Attachment.attach(new_attachment, "task_order", self.id) + elif isinstance(new_attachment, dict): + if new_attachment["filename"] and new_attachment["object_name"]: + attachment = Attachment.get_or_create( + new_attachment["object_name"], new_attachment + ) + return attachment + else: + return None elif not new_attachment and hasattr(self, attribute): return None else: diff --git a/atst/routes/task_orders/new.py b/atst/routes/task_orders/new.py index 9e88f8d4..c838f6b0 100644 --- a/atst/routes/task_orders/new.py +++ b/atst/routes/task_orders/new.py @@ -1,4 +1,11 @@ -from flask import g, redirect, render_template, request as http_request, url_for +from flask import ( + g, + redirect, + render_template, + request as http_request, + url_for, + current_app, +) from . import task_orders_bp from atst.domain.authz.decorator import user_can_access_decorator as user_can @@ -10,7 +17,8 @@ from atst.utils.flash import formatted_flash as flash def render_task_orders_edit(template, portfolio_id=None, task_order_id=None, form=None): - render_args = {} + (token, object_name) = current_app.uploader.get_token() + render_args = {"token": token, "object_name": object_name} if task_order_id: task_order = TaskOrders.get(task_order_id) @@ -110,7 +118,7 @@ def form_step_one_add_pdf(portfolio_id=None, task_order_id=None): @task_orders_bp.route("/task_orders//form/step_1", methods=["POST"]) @user_can(Permissions.CREATE_TASK_ORDER, message="update task order form") def submit_form_step_one_add_pdf(portfolio_id=None, task_order_id=None): - form_data = {**http_request.form, **http_request.files} + form_data = {**http_request.form} next_page = "task_orders.form_step_two_add_number" current_template = "task_orders/step_1.html" diff --git a/js/components/__tests__/checkbox_input.test.js b/js/components/__tests__/checkbox_input.test.js index 6be9c1e4..4c6b15f8 100644 --- a/js/components/__tests__/checkbox_input.test.js +++ b/js/components/__tests__/checkbox_input.test.js @@ -9,6 +9,9 @@ const WrapperComponent = makeTestWrapper({ checkboxinput, }, templatePath: 'checkbox_input_template.html', + data: function() { + return { initialvalue: this.initialData } + }, }) describe('CheckboxInput Renders Correctly', () => { diff --git a/js/components/__tests__/upload_input.test.js b/js/components/__tests__/upload_input.test.js index 96a1be89..c867eea3 100644 --- a/js/components/__tests__/upload_input.test.js +++ b/js/components/__tests__/upload_input.test.js @@ -7,18 +7,24 @@ import { makeTestWrapper } from '../../test_utils/component_test_helpers' const UploadWrapper = makeTestWrapper({ components: { uploadinput }, templatePath: 'upload_input_template.html', + data: function() { + return { initialvalue: this.initialData.initialvalue, token: this.token } + }, }) const UploadErrorWrapper = makeTestWrapper({ components: { uploadinput }, templatePath: 'upload_input_error_template.html', + data: function() { + return { initialvalue: null, token: null } + }, }) describe('UploadInput Test', () => { it('should show input and button when no attachment present', () => { const wrapper = mount(UploadWrapper, { propsData: { - initialData: null, + initialData: { initialvalue: null, token: 'token' }, }, }) @@ -29,7 +35,7 @@ describe('UploadInput Test', () => { it('should show file name and hide input', () => { const wrapper = mount(UploadWrapper, { propsData: { - initialData: 'somepdf.pdf', + initialData: { initialvalue: 'somepdf.pdf', token: 'token' }, }, }) @@ -41,7 +47,11 @@ describe('UploadInput Test', () => { }) it('should correctly display error treatment', () => { - const wrapper = mount(UploadErrorWrapper) + const wrapper = mount(UploadErrorWrapper, { + propsData: { + initialData: { initialvalue: 'somepdf.pdf', token: 'token' }, + }, + }) const messageArea = wrapper.find('.usa-input__message') expect(messageArea.html()).toContain('Test Error Message') diff --git a/js/components/upload_input.js b/js/components/upload_input.js index 9e848b4f..22a7c81c 100644 --- a/js/components/upload_input.js +++ b/js/components/upload_input.js @@ -6,6 +6,8 @@ import FormMixin from '../mixins/form' import textinput from './text_input' import optionsinput from './options_input' +import { buildUploader } from '../lib/upload' + export default { name: 'uploadinput', @@ -18,6 +20,12 @@ export default { props: { name: String, + token: { + type: Object, + }, + objectName: { + type: String, + }, initialData: { type: String, }, @@ -44,6 +52,7 @@ export default { }, created: function() { + this.uploader = buildUploader(this.token) emitEvent('field-mount', this, { optional: this.optional, name: this.name, @@ -52,9 +61,19 @@ export default { }, methods: { - addAttachment: function(e) { - this.attachment = e.target.value - this.showErrors = false + addAttachment: async function(e) { + const file = e.target.files[0] + try { + await this.uploader.upload(file, this.objectName) + this.attachment = e.target.value + this.showErrors = false + this.$refs.attachmentFilename.value = file.name + this.$refs.attachmentObjectName.value = this.objectName + } catch (err) { + console.log(err) + this.showErrors = true + } + this.changed = true emitEvent('field-change', this, { diff --git a/js/lib/upload.js b/js/lib/upload.js new file mode 100644 index 00000000..f102c851 --- /dev/null +++ b/js/lib/upload.js @@ -0,0 +1,90 @@ +import Azure from 'azure-storage' +import 'whatwg-fetch' + +class AzureUploader { + constructor(accountName, containerName, sasToken) { + this.accountName = accountName + this.containerName = containerName + this.sasToken = sasToken.token + } + + async upload(file, objectName) { + const blobService = Azure.createBlobServiceWithSas( + `https://${this.accountName}.blob.core.windows.net`, + this.sasToken + ) + const fileReader = new FileReader() + const options = { + contentSettings: { + contentType: 'application/pdf', + }, + metadata: { + filename: file.name, + }, + } + + return new Promise((resolve, reject) => { + fileReader.addEventListener('load', f => { + blobService.createBlockBlobFromText( + this.containerName, + `${objectName}`, + f.target.result, + options, + function(err, result) { + if (err) { + reject(err) + } else { + resolve(result) + } + } + ) + }) + fileReader.readAsText(file) + }) + } +} + +class AwsUploader { + constructor(presignedPost) { + this.presignedPost = presignedPost + } + + async upload(file, objectName) { + const form = new FormData() + Object.entries(this.presignedPost.fields).forEach(([k, v]) => { + form.append(k, v) + }) + form.append('file', file) + form.set('x-amz-meta-filename', file.name) + + return fetch(this.presignedPost.url, { + method: 'POST', + body: form, + }) + } +} + +class MockUploader { + constructor(token) { + this.token = token + } + + async upload(file, objectName) { + return Promise.resolve({}) + } +} + +export const buildUploader = token => { + const cloudProvider = process.env.CLOUD_PROVIDER || 'mock' + if (cloudProvider === 'aws') { + return new AwsUploader(token) + } else if (cloudProvider === 'azure') { + return new AzureUploader( + process.env.AZURE_ACCOUNT_NAME, + process.env.AZURE_CONTAINER_NAME, + token + ) + } else { + return new MockUploader(token) + } +} diff --git a/js/test_templates/upload_input_error_template.html b/js/test_templates/upload_input_error_template.html index 077eaeb2..efb83e29 100644 --- a/js/test_templates/upload_input_error_template.html +++ b/js/test_templates/upload_input_error_template.html @@ -4,8 +4,10 @@ v-bind:initial-errors='true' v-bind:watch='false' - name='errorfield' + name='pdf' :optional='false' + v-bind:token='token' + v-bind:object-name='"object_name"' >
@@ -19,7 +21,7 @@
-
- Test Error Message + ['Test Error Message']
diff --git a/js/test_templates/upload_input_template.html b/js/test_templates/upload_input_template.html index 2179385f..37412a66 100644 --- a/js/test_templates/upload_input_template.html +++ b/js/test_templates/upload_input_template.html @@ -4,8 +4,10 @@ v-bind:initial-data='initialvalue' v-bind:watch='false' - name='datafield' + name='pdf' :optional='false' + v-bind:token='token' + v-bind:object-name='"object_name"' >
@@ -19,7 +21,7 @@
-
diff --git a/js/test_utils/component_test_helpers.js b/js/test_utils/component_test_helpers.js index e81d9603..b8b28cc2 100644 --- a/js/test_utils/component_test_helpers.js +++ b/js/test_utils/component_test_helpers.js @@ -16,7 +16,7 @@ to be passed as a prop to checkboxinput at mount time v-bind:initial-checked='initialvalue' > */ -const makeTestWrapper = ({ components, templatePath }) => { +const makeTestWrapper = ({ components, templatePath, data }) => { const templateString = fs.readFileSync( `js/test_templates/${templatePath}`, 'utf-8' @@ -27,11 +27,7 @@ const makeTestWrapper = ({ components, templatePath }) => { components, template: templateString, props: ['initialData'], - data: function() { - return { - initialvalue: this.initialData, - } - }, + data, } return WrapperComponent diff --git a/package.json b/package.json index d9561c62..8fea9ca3 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "dependencies": { "ally.js": "^1.4.1", "autoprefixer": "^9.1.3", + "azure-storage": "^2.10.3", "babel-polyfill": "^6.26.0", "date-fns": "^1.29.0", "npm": "^6.0.1", @@ -25,7 +26,8 @@ "uswds": "^1.6.9", "v-tooltip": "^2.0.0-rc.33", "vue": "2.5.15", - "vue-text-mask": "^6.1.2" + "vue-text-mask": "^6.1.2", + "whatwg-fetch": "^3.0.0" }, "devDependencies": { "@vue/test-utils": "^1.0.0-beta.25", diff --git a/templates/components/upload_input.html b/templates/components/upload_input.html index 3a80a834..6e71d157 100644 --- a/templates/components/upload_input.html +++ b/templates/components/upload_input.html @@ -1,16 +1,18 @@ {% from "components/icon.html" import Icon %} -{% macro UploadInput(field, show_label=False, watch=False) -%} +{% macro UploadInput(field, show_label=False, watch=False, token="", object_name="") -%}
@@ -41,9 +43,11 @@ aria-label="Task Order Upload" v-bind:value="attachment" type="file"> + +
- {% for error in field.errors %} - {{error}} + {% for error, error_message in field.errors.items() %} + {{error_message}} {% endfor %}
diff --git a/templates/task_orders/step_1.html b/templates/task_orders/step_1.html index e81ad47c..843b8c0b 100644 --- a/templates/task_orders/step_1.html +++ b/templates/task_orders/step_1.html @@ -17,5 +17,5 @@ {% block to_builder_form_field %} {{ TOFormStepHeader('task_orders.form.supporting_docs_header' | translate, 'task_orders.form.supporting_docs_text' | translate) }} - {{ UploadInput(form.pdf, watch=True) }} + {{ UploadInput(form.pdf, watch=True, token=token, object_name=object_name) }} {% endblock %} diff --git a/tests/factories.py b/tests/factories.py index 4dd2a7f8..91fe0566 100644 --- a/tests/factories.py +++ b/tests/factories.py @@ -269,7 +269,9 @@ class TaskOrderFactory(Base): class Meta: model = TaskOrder - portfolio = factory.SubFactory(PortfolioFactory) + portfolio = factory.SubFactory( + PortfolioFactory, owner=factory.SelfAttribute("..creator") + ) number = factory.LazyFunction(random_task_order_number) creator = factory.SubFactory(UserFactory) _pdf = factory.SubFactory(AttachmentFactory) diff --git a/tests/render_vue_component.py b/tests/render_vue_component.py index 5d9f3b08..b502ea93 100644 --- a/tests/render_vue_component.py +++ b/tests/render_vue_component.py @@ -2,8 +2,8 @@ import pytest from wtforms.widgets import CheckboxInput from wtforms.fields import StringField -from wtforms.validators import InputRequired -from wtforms import Form +from wtforms.validators import InputRequired, URL +from wtforms import Form, FormField class InitialValueForm(Form): @@ -14,6 +14,19 @@ class InitialValueForm(Form): ) +class TaskOrderPdfForm(Form): + filename = StringField(default="initialvalue") + object_name = StringField() + + errorfield = StringField( + label="error", validators=[InputRequired(message="Test Error Message")] + ) + + +class TaskOrderForm(Form): + pdf = FormField(TaskOrderPdfForm, label="task_order_pdf") + + @pytest.fixture def env(app, scope="function"): return app.jinja_env @@ -39,6 +52,16 @@ def initial_value_form(scope="function"): return InitialValueForm() +@pytest.fixture +def task_order_form(scope="function"): + return TaskOrderForm() + + +@pytest.fixture +def error_task_order_form(scope="function"): + return ErrorTaskOrderForm() + + def write_template(content, name): with open("js/test_templates/{}".format(name), "w") as fh: fh.write(content) @@ -50,12 +73,16 @@ def test_make_checkbox_input_template(checkbox_input_macro, initial_value_form): write_template(rendered_checkbox_macro, "checkbox_input_template.html") -def test_make_upload_input_template(upload_input_macro, initial_value_form): - rendered_upload_macro = upload_input_macro(initial_value_form.datafield) +def test_make_upload_input_template(upload_input_macro, task_order_form): + rendered_upload_macro = upload_input_macro( + task_order_form.pdf, token="token", object_name="object_name" + ) write_template(rendered_upload_macro, "upload_input_template.html") -def test_make_upload_input_error_template(upload_input_macro, initial_value_form): - initial_value_form.validate() - rendered_upload_macro = upload_input_macro(initial_value_form.errorfield) +def test_make_upload_input_error_template(upload_input_macro, task_order_form): + task_order_form.validate() + rendered_upload_macro = upload_input_macro( + task_order_form.pdf, token="token", object_name="object_name" + ) write_template(rendered_upload_macro, "upload_input_error_template.html") diff --git a/tests/routes/task_orders/test_new.py b/tests/routes/task_orders/test_new.py index 0ab8a2bc..798fb4d7 100644 --- a/tests/routes/task_orders/test_new.py +++ b/tests/routes/task_orders/test_new.py @@ -9,6 +9,10 @@ from atst.models import TaskOrder from tests.factories import CLINFactory, PortfolioFactory, TaskOrderFactory, UserFactory +def build_pdf_form_data(filename="sample.pdf", object_name="object_name"): + return {"pdf-filename": filename, "pdf-object_name": object_name} + + @pytest.fixture def task_order(): user = UserFactory.create() @@ -47,19 +51,16 @@ def test_task_orders_form_step_one_add_pdf(client, user_session, portfolio): assert response.status_code == 200 -def test_task_orders_submit_form_step_one_add_pdf( - client, user_session, portfolio, pdf_upload, session -): +def test_task_orders_submit_form_step_one_add_pdf(client, user_session, portfolio): user_session(portfolio.owner) - form_data = {"pdf": pdf_upload} response = client.post( url_for("task_orders.submit_form_step_one_add_pdf", portfolio_id=portfolio.id), - data=form_data, + data=build_pdf_form_data(), ) assert response.status_code == 302 task_order = portfolio.task_orders[0] - assert task_order.pdf.filename == pdf_upload.filename + assert task_order.pdf.filename == "sample.pdf" def test_task_orders_form_step_one_add_pdf_existing_to( @@ -72,35 +73,29 @@ def test_task_orders_form_step_one_add_pdf_existing_to( assert response.status_code == 200 -def test_task_orders_submit_form_step_one_add_pdf_existing_to( - client, user_session, task_order, pdf_upload, pdf_upload2 -): - task_order.pdf = pdf_upload - assert task_order.pdf.filename == pdf_upload.filename - +def test_task_orders_submit_form_step_one_add_pdf_existing_to(client, user_session): + task_order = TaskOrderFactory.create() user_session(task_order.creator) - form_data = {"pdf": pdf_upload2} response = client.post( url_for( "task_orders.submit_form_step_one_add_pdf", task_order_id=task_order.id ), - data=form_data, + data=build_pdf_form_data(), ) assert response.status_code == 302 - assert task_order.pdf.filename == pdf_upload2.filename + assert task_order.pdf.filename == "sample.pdf" def test_task_orders_submit_form_step_one_add_pdf_delete_pdf( client, user_session, portfolio, pdf_upload ): user_session(portfolio.owner) - task_order = TaskOrderFactory.create(pdf=pdf_upload, portfolio=portfolio) - data = {"pdf": ""} + task_order = TaskOrderFactory.create(portfolio=portfolio) response = client.post( url_for( "task_orders.submit_form_step_one_add_pdf", task_order_id=task_order.id ), - data=data, + data=build_pdf_form_data(filename="", object_name=""), ) assert task_order.pdf is None assert response.status_code == 302 diff --git a/tests/test_access.py b/tests/test_access.py index 3a311894..141fed18 100644 --- a/tests/test_access.py +++ b/tests/test_access.py @@ -448,9 +448,7 @@ def test_task_orders_download_task_order_pdf_access(get_url_assert_status, monke rando = user_with() portfolio = PortfolioFactory.create(owner=owner) - task_order = TaskOrderFactory.create( - portfolio=portfolio, pdf=AttachmentFactory.create() - ) + task_order = TaskOrderFactory.create(portfolio=portfolio) url = url_for("task_orders.download_task_order_pdf", task_order_id=task_order.id) get_url_assert_status(owner, url, 200) diff --git a/yarn.lock b/yarn.lock index f9ec04c5..449ced8d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1163,6 +1163,23 @@ aws4@^1.6.0, aws4@^1.8.0: resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.8.0.tgz#f0e003d9ca9e7f59c7a508945d7b2ef9a04a542f" integrity sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ== +azure-storage@^2.10.3: + version "2.10.3" + resolved "https://registry.yarnpkg.com/azure-storage/-/azure-storage-2.10.3.tgz#c5966bf929d87587d78f6847040ea9a4b1d4a50a" + integrity sha512-IGLs5Xj6kO8Ii90KerQrrwuJKexLgSwYC4oLWmc11mzKe7Jt2E5IVg+ZQ8K53YWZACtVTMBNO3iGuA+4ipjJxQ== + dependencies: + browserify-mime "~1.2.9" + extend "^3.0.2" + json-edm-parser "0.1.2" + md5.js "1.3.4" + readable-stream "~2.0.0" + request "^2.86.0" + underscore "~1.8.3" + uuid "^3.0.0" + validator "~9.4.1" + xml2js "0.2.8" + xmlbuilder "^9.0.7" + babel-code-frame@^6.26.0: version "6.26.0" resolved "https://registry.yarnpkg.com/babel-code-frame/-/babel-code-frame-6.26.0.tgz#63fd43f7dc1e3bb7ce35947db8fe369a3f58c74b" @@ -1871,6 +1888,11 @@ browserify-des@^1.0.0: inherits "^2.0.1" safe-buffer "^5.1.2" +browserify-mime@~1.2.9: + version "1.2.9" + resolved "https://registry.yarnpkg.com/browserify-mime/-/browserify-mime-1.2.9.tgz#aeb1af28de6c0d7a6a2ce40adb68ff18422af31f" + integrity sha1-rrGvKN5sDXpqLOQK22j/GEIq8x8= + browserify-rsa@^4.0.0: version "4.0.1" resolved "https://registry.yarnpkg.com/browserify-rsa/-/browserify-rsa-4.0.1.tgz#21e0abfaf6f2029cf2fafb133567a701d4135524" @@ -3467,7 +3489,7 @@ extend-shallow@^3.0.0, extend-shallow@^3.0.2: assign-symbols "^1.0.0" is-extendable "^1.0.1" -extend@~3.0.0, extend@~3.0.1, extend@~3.0.2: +extend@^3.0.2, extend@~3.0.0, extend@~3.0.1, extend@~3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.2.tgz#f8b1136b4071fbd8eb140aff858b1019ec2915fa" @@ -5262,6 +5284,13 @@ jsesc@~0.5.0: resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-0.5.0.tgz#e7dee66e35d6fc16f710fe91d5cf69f70f08911d" integrity sha1-597mbjXW/Bb3EP6R1c9p9w8IkR0= +json-edm-parser@0.1.2: + version "0.1.2" + resolved "https://registry.yarnpkg.com/json-edm-parser/-/json-edm-parser-0.1.2.tgz#1e60b0fef1bc0af67bc0d146dfdde5486cd615b4" + integrity sha1-HmCw/vG8CvZ7wNFG393lSGzWFbQ= + dependencies: + jsonparse "~1.2.0" + json-parse-better-errors@^1.0.0, json-parse-better-errors@^1.0.1, json-parse-better-errors@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz#bb867cfb3450e69107c131d1c514bab3dc8bcaa9" @@ -5325,6 +5354,11 @@ jsonparse@^1.2.0: version "1.3.1" resolved "https://registry.yarnpkg.com/jsonparse/-/jsonparse-1.3.1.tgz#3f4dae4a91fac315f71062f8521cc239f1366280" +jsonparse@~1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/jsonparse/-/jsonparse-1.2.0.tgz#5c0c5685107160e72fe7489bddea0b44c2bc67bd" + integrity sha1-XAxWhRBxYOcv50ib3eoLRMK8Z70= + jsprim@^1.2.2: version "1.4.1" resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-1.4.1.tgz#313e66bc1e5cc06e438bc1b7499c2e5c56acb6a2" @@ -5687,6 +5721,14 @@ math-random@^1.0.1: resolved "https://registry.yarnpkg.com/math-random/-/math-random-1.0.4.tgz#5dd6943c938548267016d4e34f057583080c514c" integrity sha512-rUxjysqif/BZQH2yhd5Aaq7vXMSx9NdEsQcyA07uEzIvxgI7zIr33gGsh+RU0/XjmQpCW7RsVof1vlkvQVCK5A== +md5.js@1.3.4: + version "1.3.4" + resolved "https://registry.yarnpkg.com/md5.js/-/md5.js-1.3.4.tgz#e9bdbde94a20a5ac18b04340fc5764d5b09d901d" + integrity sha1-6b296UogpawYsENA/Fdk1bCdkB0= + dependencies: + hash-base "^3.0.0" + inherits "^2.0.1" + md5.js@^1.3.4: version "1.3.5" resolved "https://registry.yarnpkg.com/md5.js/-/md5.js-1.3.5.tgz#b5d07b8e3216e3e27cd728d72f70d1e6a342005f" @@ -7995,7 +8037,7 @@ request-promise-native@^1.0.5: tunnel-agent "^0.6.0" uuid "^3.0.0" -request@^2.72.0, request@^2.87.0, request@^2.88.0: +request@^2.72.0, request@^2.86.0, request@^2.87.0, request@^2.88.0: version "2.88.0" resolved "https://registry.yarnpkg.com/request/-/request-2.88.0.tgz#9c2fca4f7d35b592efe57c7f0a55e81052124fef" integrity sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg== @@ -8212,6 +8254,11 @@ sass-graph@^2.2.4: scss-tokenizer "^0.2.3" yargs "^7.0.0" +sax@0.5.x: + version "0.5.8" + resolved "https://registry.yarnpkg.com/sax/-/sax-0.5.8.tgz#d472db228eb331c2506b0e8c15524adb939d12c1" + integrity sha1-1HLbIo6zMcJQaw6MFVJK25OdEsE= + sax@^1.2.4, sax@~1.2.4: version "1.2.4" resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" @@ -9192,6 +9239,11 @@ undeclared-identifiers@^1.1.2: simple-concat "^1.0.0" xtend "^4.0.1" +underscore@~1.8.3: + version "1.8.3" + resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.8.3.tgz#4f3fb53b106e6097fcf9cb4109f2a5e9bdfa5022" + integrity sha1-Tz+1OxBuYJf8+ctBCfKl6b36UCI= + unicode-canonical-property-names-ecmascript@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-1.0.4.tgz#2619800c4c825800efdd8343af7dd9933cbe2818" @@ -9424,6 +9476,11 @@ validate-npm-package-name@^3.0.0, validate-npm-package-name@~3.0.0: dependencies: builtins "^1.0.3" +validator@~9.4.1: + version "9.4.1" + resolved "https://registry.yarnpkg.com/validator/-/validator-9.4.1.tgz#abf466d398b561cd243050112c6ff1de6cc12663" + integrity sha512-YV5KjzvRmSyJ1ee/Dm5UED0G+1L4GZnLN3w6/T+zZm8scVua4sOhYKWTUrKa0H/tMiJyO9QLHMPN+9mB/aMunA== + vendors@^1.0.0: version "1.0.3" resolved "https://registry.yarnpkg.com/vendors/-/vendors-1.0.3.tgz#a6467781abd366217c050f8202e7e50cc9eef8c0" @@ -9508,6 +9565,11 @@ whatwg-encoding@^1.0.1, whatwg-encoding@^1.0.3: dependencies: iconv-lite "0.4.24" +whatwg-fetch@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/whatwg-fetch/-/whatwg-fetch-3.0.0.tgz#fc804e458cc460009b1a2b966bc8817d2578aefb" + integrity sha512-9GSJUgz1D4MfyKU7KRqwOjXCXTqWdFNvEr7eUBYchQiVc744mqK/MzXPNR2WsPkmkOa4ywfg8C2n8h+13Bey1Q== + whatwg-mimetype@^2.1.0, whatwg-mimetype@^2.2.0: version "2.3.0" resolved "https://registry.yarnpkg.com/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz#3d4b1e0312d2079879f826aff18dbeeca5960fbf" @@ -9621,6 +9683,18 @@ xml-name-validator@^3.0.0: resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-3.0.0.tgz#6ae73e06de4d8c6e47f9fb181f78d648ad457c6a" integrity sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw== +xml2js@0.2.8: + version "0.2.8" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.2.8.tgz#9b81690931631ff09d1957549faf54f4f980b3c2" + integrity sha1-m4FpCTFjH/CdGVdUn69U9PmAs8I= + dependencies: + sax "0.5.x" + +xmlbuilder@^9.0.7: + version "9.0.7" + resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d" + integrity sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0= + xtend@^4.0.0, xtend@^4.0.1, xtend@~4.0.1: version "4.0.1" resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.1.tgz#a5c6d532be656e23db820efb943a1f04998d63af"