From 8eba9a097dcd4c28c846cde1ee73799c68f98495 Mon Sep 17 00:00:00 2001 From: richard-dds Date: Tue, 30 Jul 2019 16:51:58 -0400 Subject: [PATCH 01/20] Add CSP file uploads --- Pipfile | 4 + Pipfile.lock | 169 +++++++++++++++++++++---- atst/app.py | 2 + atst/domain/csp/file_uploads.py | 66 ++++++++++ atst/routes/task_orders/new.py | 16 ++- js/components/upload_input.js | 23 +++- js/lib/upload.js | 64 ++++++++++ package.json | 1 + templates/components/upload_input.html | 4 +- templates/task_orders/step_1.html | 2 +- yarn.lock | 73 ++++++++++- 11 files changed, 390 insertions(+), 34 deletions(-) create mode 100644 atst/domain/csp/file_uploads.py create mode 100644 js/lib/upload.js diff --git a/Pipfile b/Pipfile index 83fb25c8..1add2b71 100644 --- a/Pipfile +++ b/Pipfile @@ -23,6 +23,10 @@ lockfile = "*" "flask-rq2" = "*" werkzeug = "*" PyYAML = "*" +azure-storage = "*" +azure-storage-common = "*" +azure-storage-file = "*" +boto3 = "*" [dev-packages] bandit = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 8d27c525..988b3d10 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "9b0fa418945fbb89208d11a4d44ce4c03e6810cd536628a7b2e17a58e5fd377a" + "sha256": "a29cf3d28fc5257bdab859827271fd1e7bf444df88111b7e0f19731b1cd6c956" }, "pipfile-spec": 6, "requires": { @@ -38,6 +38,67 @@ ], "version": "==0.24.0" }, + "azure-common": { + "hashes": [ + "sha256:53b1195b8f20943ccc0e71a17849258f7781bc6db1c72edc7d6c055f79bd54e3", + "sha256:99ef36e74b6395329aada288764ce80504da16ecc8206cb9a72f55fb02e8b484" + ], + "version": "==1.1.23" + }, + "azure-core": { + "hashes": [ + "sha256:c49ab53efcebb40f35f0d0e524670057e7ad003ebd230fbc111335892a863e72", + "sha256:d4b2e5448125e753315e5b3fd9a18e49dcd7ac0fbfbbc0abe577043eb590aa7f" + ], + "version": "==1.0.0b1" + }, + "azure-nspkg": { + "hashes": [ + "sha256:1d0bbb2157cf57b1bef6c8c8e5b41133957364456c43b0a43599890023cca0a8", + "sha256:31a060caca00ed1ebd369fc7fe01a56768c927e404ebc92268f4d9d636435e28", + "sha256:e7d3cea6af63e667d87ba1ca4f8cd7cb4dfca678e4c55fc1cedb320760e39dd0" + ], + "version": "==3.0.2" + }, + "azure-storage": { + "hashes": [ + "sha256:4c406422e3edd41920bb1f0c3930c34fee3eb0d55258ef7ec7308ccbb9385ad5", + "sha256:fb6212dcbed91b49d9637aa5e8888eafdfcd523b7e560c8044d2d838bbd3ca5f" + ], + "index": "pypi", + "version": "==0.36.0" + }, + "azure-storage-common": { + "hashes": [ + "sha256:1fce4505880c345c83c06b2ae449e403be46dca039b9ef16d122fb1e4b2741f5", + "sha256:4390cd5f6fb50fbff37ba154258ea010291a84f128917141e1d7060597bd4708" + ], + "index": "pypi", + "version": "==2.0.0" + }, + "azure-storage-file": { + "hashes": [ + "sha256:d938d71459b108740d896220c0ed17530ca83fd0f82781a044b80ed7e03bd3be", + "sha256:e996a228cd632f6f586228381eaa2823b52884899dce2259876e5eb78f0eab6f" + ], + "index": "pypi", + "version": "==12.0.0b1" + }, + "boto3": { + "hashes": [ + "sha256:0cd4a3e158f40eedb54b36b3fbe60d135db74a245f0ca8eead1af2eb6d46a649", + "sha256:68e9eba6f846cf8e01973ec565afdb1adfb9612b531c15bb5c5524394db4df5b" + ], + "index": "pypi", + "version": "==1.9.199" + }, + "botocore": { + "hashes": [ + "sha256:25d87047241b7b775443570c0e790ca952f9f7491d4d6472430a4b006383a257", + "sha256:e4729c1acaa936d4c5c948a18d279f92bbf61fad9b5fb03942c753ec405e427d" + ], + "version": "==1.12.199" + }, "certifi": { "hashes": [ "sha256:046832c04d4e752f37383b628bc601a7ea7211496b4638f6514d0e5b9acc4939", @@ -120,6 +181,14 @@ ], "version": "==2.7" }, + "docutils": { + "hashes": [ + "sha256:02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6", + "sha256:51e64ef2ebfb29cae1faa133b3710143496eca21c530f3f71424d77687764274", + "sha256:7a4bd47eaf6596e1295ecb11361139febe29b084a87bf005bf899f9a42edc3c6" + ], + "version": "==0.14" + }, "flask": { "hashes": [ "sha256:13f9f196f330c7c2c5d7a5cf91af894110ca0215ac051b5844701f2bfd934d52", @@ -174,6 +243,13 @@ ], "version": "==2.8" }, + "isodate": { + "hashes": [ + "sha256:2e364a3d5759479cdb2d37cce6b9376ea504db2ff90252a2e5b7cc89cc9ff2d8", + "sha256:aa4d33c06640f5352aca96e4b81afd8ab3b47337cc12089822d6f322ac772c81" + ], + "version": "==0.6.0" + }, "itsdangerous": { "hashes": [ "sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19", @@ -188,6 +264,13 @@ ], "version": "==2.10.1" }, + "jmespath": { + "hashes": [ + "sha256:3720a4b1bd659dd2eecad0666459b9788813e032b83e7ba58578e48254e0a0e6", + "sha256:bde2aef6f44302dfb30320115b17d030798de8c4110e28d5cf6cf91a7a31074c" + ], + "version": "==0.9.4" + }, "lockfile": { "hashes": [ "sha256:6aed02de03cba24efabcd600b30540140634fc06cfa603822d508d5361e9f799", @@ -235,6 +318,20 @@ ], "version": "==1.1.1" }, + "msrest": { + "hashes": [ + "sha256:27589fb400da7e1a98778688f70a0099e4fc6fea59d0f4835b4fbdad3bb8a6d9", + "sha256:cda706a2ccfb032cf41fa8cc6575cbca29634fed2d226fc789e4a8daf44ab7c1" + ], + "version": "==0.6.9" + }, + "oauthlib": { + "hashes": [ + "sha256:40a63637707e9163eda62d0f5345120c65e001a790480b8256448543c1f78f66", + "sha256:b4d99ae8ccfb7d33ba9591b59355c64eef5241534aa3da2e4c0435346b84bc8e" + ], + "version": "==3.0.2" + }, "pendulum": { "hashes": [ "sha256:1cde6e3c6310fb882c98f373795f807cb2bd6af01f34d2857e6e283b5ee91e09", @@ -300,6 +397,7 @@ "sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb", "sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e" ], + "markers": "python_version >= '2.7'", "version": "==2.8.0" }, "python-editor": { @@ -319,20 +417,22 @@ }, "pyyaml": { "hashes": [ - "sha256:57acc1d8533cbe51f6662a55434f0dbecfa2b9eaf115bede8f6fd00115a0c0d3", - "sha256:588c94b3d16b76cfed8e0be54932e5729cc185caffaa5a451e7ad2f7ed8b4043", - "sha256:68c8dd247f29f9a0d09375c9c6b8fdc64b60810ebf07ba4cdd64ceee3a58c7b7", - "sha256:70d9818f1c9cd5c48bb87804f2efc8692f1023dac7f1a1a5c61d454043c1d265", - "sha256:86a93cccd50f8c125286e637328ff4eef108400dd7089b46a7be3445eecfa391", - "sha256:a0f329125a926876f647c9fa0ef32801587a12328b4a3c741270464e3e4fa778", - "sha256:a3c252ab0fa1bb0d5a3f6449a4826732f3eb6c0270925548cac342bc9b22c225", - "sha256:b4bb4d3f5e232425e25dda21c070ce05168a786ac9eda43768ab7f3ac2770955", - "sha256:cd0618c5ba5bda5f4039b9398bb7fb6a317bb8298218c3de25c47c4740e4b95e", - "sha256:ceacb9e5f8474dcf45b940578591c7f3d960e82f926c707788a570b51ba59190", - "sha256:fe6a88094b64132c4bb3b631412e90032e8cfe9745a58370462240b8cb7553cd" + "sha256:0113bc0ec2ad727182326b61326afa3d1d8280ae1122493553fd6f4397f33df9", + "sha256:01adf0b6c6f61bd11af6e10ca52b7d4057dd0be0343eb9283c878cf3af56aee4", + "sha256:5124373960b0b3f4aa7df1707e63e9f109b5263eca5976c66e08b1c552d4eaf8", + "sha256:5ca4f10adbddae56d824b2c09668e91219bb178a1eee1faa56af6f99f11bf696", + "sha256:7907be34ffa3c5a32b60b95f4d95ea25361c951383a894fec31be7252b2b6f34", + "sha256:7ec9b2a4ed5cad025c2278a1e6a19c011c80a3caaac804fd2d329e9cc2c287c9", + "sha256:87ae4c829bb25b9fe99cf71fbb2140c448f534e24c998cc60f39ae4f94396a73", + "sha256:9de9919becc9cc2ff03637872a440195ac4241c80536632fffeb6a1e25a74299", + "sha256:a5a85b10e450c66b49f98846937e8cfca1db3127a9d5d1e31ca45c3d0bef4c5b", + "sha256:b0997827b4f6a7c286c01c5f60384d218dca4ed7d9efa945c3e1aa623d5709ae", + "sha256:b631ef96d3222e62861443cc89d6563ba3eeb816eeb96b2629345ab795e53681", + "sha256:bf47c0607522fdbca6c9e817a6e81b08491de50f3766a7a0e6a5be7905961b41", + "sha256:f81025eddd0327c7d4cfe9b62cf33190e1e736cc6e97502b3ec425f574b3e7a8" ], "index": "pypi", - "version": "==5.1.1" + "version": "==5.1.2" }, "redis": { "hashes": [ @@ -350,6 +450,13 @@ "index": "pypi", "version": "==2.22.0" }, + "requests-oauthlib": { + "hashes": [ + "sha256:bd6533330e8748e94bf0b214775fed487d309b8b8fe823dc45641ebcd9a32f57", + "sha256:d3ed0c8f2e3bbc6b344fa63d6f933745ab394469da38db16bdddb461c7e25140" + ], + "version": "==1.2.0" + }, "rq": { "hashes": [ "sha256:2798d26a7b850e759f23f69695a389d676a9c08f2c14f96f0d34d9648c9d5616", @@ -364,6 +471,13 @@ ], "version": "==0.9" }, + "s3transfer": { + "hashes": [ + "sha256:6efc926738a3cd576c2a79725fed9afde92378aa5c6a957e3af010cb019fac9d", + "sha256:b780f2411b824cb541dbcd2c713d0cb61c7d1bcadae204cdddda2b35cef493ba" + ], + "version": "==0.2.1" + }, "six": { "hashes": [ "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", @@ -391,6 +505,7 @@ "sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1", "sha256:dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232" ], + "markers": "python_version >= '3.4'", "version": "==1.25.3" }, "webassets": { @@ -851,24 +966,27 @@ "sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb", "sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e" ], + "markers": "python_version >= '2.7'", "version": "==2.8.0" }, "pyyaml": { "hashes": [ - "sha256:57acc1d8533cbe51f6662a55434f0dbecfa2b9eaf115bede8f6fd00115a0c0d3", - "sha256:588c94b3d16b76cfed8e0be54932e5729cc185caffaa5a451e7ad2f7ed8b4043", - "sha256:68c8dd247f29f9a0d09375c9c6b8fdc64b60810ebf07ba4cdd64ceee3a58c7b7", - "sha256:70d9818f1c9cd5c48bb87804f2efc8692f1023dac7f1a1a5c61d454043c1d265", - "sha256:86a93cccd50f8c125286e637328ff4eef108400dd7089b46a7be3445eecfa391", - "sha256:a0f329125a926876f647c9fa0ef32801587a12328b4a3c741270464e3e4fa778", - "sha256:a3c252ab0fa1bb0d5a3f6449a4826732f3eb6c0270925548cac342bc9b22c225", - "sha256:b4bb4d3f5e232425e25dda21c070ce05168a786ac9eda43768ab7f3ac2770955", - "sha256:cd0618c5ba5bda5f4039b9398bb7fb6a317bb8298218c3de25c47c4740e4b95e", - "sha256:ceacb9e5f8474dcf45b940578591c7f3d960e82f926c707788a570b51ba59190", - "sha256:fe6a88094b64132c4bb3b631412e90032e8cfe9745a58370462240b8cb7553cd" + "sha256:0113bc0ec2ad727182326b61326afa3d1d8280ae1122493553fd6f4397f33df9", + "sha256:01adf0b6c6f61bd11af6e10ca52b7d4057dd0be0343eb9283c878cf3af56aee4", + "sha256:5124373960b0b3f4aa7df1707e63e9f109b5263eca5976c66e08b1c552d4eaf8", + "sha256:5ca4f10adbddae56d824b2c09668e91219bb178a1eee1faa56af6f99f11bf696", + "sha256:7907be34ffa3c5a32b60b95f4d95ea25361c951383a894fec31be7252b2b6f34", + "sha256:7ec9b2a4ed5cad025c2278a1e6a19c011c80a3caaac804fd2d329e9cc2c287c9", + "sha256:87ae4c829bb25b9fe99cf71fbb2140c448f534e24c998cc60f39ae4f94396a73", + "sha256:9de9919becc9cc2ff03637872a440195ac4241c80536632fffeb6a1e25a74299", + "sha256:a5a85b10e450c66b49f98846937e8cfca1db3127a9d5d1e31ca45c3d0bef4c5b", + "sha256:b0997827b4f6a7c286c01c5f60384d218dca4ed7d9efa945c3e1aa623d5709ae", + "sha256:b631ef96d3222e62861443cc89d6563ba3eeb816eeb96b2629345ab795e53681", + "sha256:bf47c0607522fdbca6c9e817a6e81b08491de50f3766a7a0e6a5be7905961b41", + "sha256:f81025eddd0327c7d4cfe9b62cf33190e1e736cc6e97502b3ec425f574b3e7a8" ], "index": "pypi", - "version": "==5.1.1" + "version": "==5.1.2" }, "selenium": { "hashes": [ @@ -946,6 +1064,7 @@ "sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1", "sha256:dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232" ], + "markers": "python_version >= '3.4'", "version": "==1.25.3" }, "watchdog": { diff --git a/atst/app.py b/atst/app.py index af9f1a0c..e72a9cdd 100644 --- a/atst/app.py +++ b/atst/app.py @@ -30,6 +30,7 @@ from atst.utils.json import CustomJSONEncoder from atst.queue import queue from atst.utils.notification_sender import NotificationSender from atst.utils.session_limiter import SessionLimiter +from atst.domain.csp.file_uploads import build_uploader from logging.config import dictConfig from atst.utils.logging import JsonFormatter, RequestContextFilter @@ -78,6 +79,7 @@ def make_app(config): app.register_blueprint(task_orders_bp) app.register_blueprint(applications_bp) app.register_blueprint(user_routes) + app.uploader = build_uploader(app.config) if ENV != "prod": app.register_blueprint(dev_routes) diff --git a/atst/domain/csp/file_uploads.py b/atst/domain/csp/file_uploads.py new file mode 100644 index 00000000..d9c2a367 --- /dev/null +++ b/atst/domain/csp/file_uploads.py @@ -0,0 +1,66 @@ +from azure.storage.common import CloudStorageAccount +from azure.storage.blob import ContainerPermissions +from datetime import datetime, timedelta +from uuid import uuid4 + +import boto3 + + +def build_uploader(config): + if config["CSP"] == "aws": + return AwsUploader(config) + elif config["CSP"] == "azure": + return AzureUploader(config) + + +class Uploader: + def generate_token(self): + pass + + def object_name(self): + return str(uuid4()) + + +class AzureUploader(Uploader): + def __init__(self, config): + self.config = config + + def get_token(self): + account = CloudStorageAccount( + account_name="atat", account_key=self.config["AZURE_STORAGE_KEY"] + ) + bbs = account.create_block_blob_service() + object_name = self.object_name() + sas_token = bbs.generate_container_shared_access_signature( + "task-order-pdfs", + ContainerPermissions.WRITE, + datetime.utcnow() + timedelta(minutes=15), + ) + return ({"token": sas_token}, object_name) + + +class AwsUploader(Uploader): + def __init__(self, config): + self.config = config + + def get_token(self): + s3_client = boto3.client( + "s3", + aws_access_key_id=self.config["AWS_ACCESS_KEY_ID"], + aws_secret_access_key=self.config["AWS_SECRET_KEY"], + config=boto3.session.Config( + signature_version="s3v4", region_name=self.config["AWS_REGION_NAME"] + ), + ) + object_name = self.object_name() + presigned_post = s3_client.generate_presigned_post( + self.config["AWS_BUCKET_NAME"], + object_name, + ExpiresIn=3600, + Conditions=[ + ("eq", "$Content-Type", "application/pdf"), + ("starts-with", "$x-amz-meta-filename", ""), + ], + Fields={"Content-Type": "application/pdf", "x-amz-meta-filename": ""}, + ) + return (presigned_post, object_name) diff --git a/atst/routes/task_orders/new.py b/atst/routes/task_orders/new.py index 43736e1a..cb5b89f2 100644 --- a/atst/routes/task_orders/new.py +++ b/atst/routes/task_orders/new.py @@ -1,4 +1,14 @@ -from flask import g, redirect, render_template, request as http_request, url_for +from flask import ( + g, + redirect, + render_template, + request as http_request, + url_for, + current_app, +) +from azure.storage.common import CloudStorageAccount +from azure.storage.blob import ContainerPermissions +from datetime import datetime, timedelta from . import task_orders_bp from atst.domain.authz.decorator import user_can_access_decorator as user_can @@ -10,7 +20,8 @@ from atst.utils.flash import formatted_flash as flash def render_task_orders_edit(template, portfolio_id=None, task_order_id=None, form=None): - render_args = {} + result = current_app.uploader.get_token() + render_args = {"token": result[0], "object_name": result[1]} if task_order_id: task_order = TaskOrders.get(task_order_id) @@ -89,6 +100,7 @@ def edit(task_order_id): @task_orders_bp.route("/task_orders//form/step_1") @user_can(Permissions.CREATE_TASK_ORDER, message="view task order form") def form_step_one_add_pdf(portfolio_id=None, task_order_id=None): + return render_task_orders_edit( "task_orders/step_1.html", portfolio_id=portfolio_id, diff --git a/js/components/upload_input.js b/js/components/upload_input.js index 9e848b4f..4763b010 100644 --- a/js/components/upload_input.js +++ b/js/components/upload_input.js @@ -6,6 +6,8 @@ import FormMixin from '../mixins/form' import textinput from './text_input' import optionsinput from './options_input' +import { buildUploader } from '../lib/upload' + export default { name: 'uploadinput', @@ -18,6 +20,12 @@ export default { props: { name: String, + token: { + type: Object, + }, + objectName: { + type: String, + }, initialData: { type: String, }, @@ -44,6 +52,7 @@ export default { }, created: function() { + this.uploader = buildUploader(this.token) emitEvent('field-mount', this, { optional: this.optional, name: this.name, @@ -52,9 +61,17 @@ export default { }, methods: { - addAttachment: function(e) { - this.attachment = e.target.value - this.showErrors = false + addAttachment: async function(e) { + const file = e.target.files[0] + try { + await this.uploader.upload(file, this.objectName) + this.attachment = e.target.value + this.showErrors = false + } catch (err) { + console.log(err) + this.showErrors = true + } + this.changed = true emitEvent('field-change', this, { diff --git a/js/lib/upload.js b/js/lib/upload.js new file mode 100644 index 00000000..e0a8369b --- /dev/null +++ b/js/lib/upload.js @@ -0,0 +1,64 @@ +import Azure from 'azure-storage' + +class AzureUploader { + constructor(sasToken) { + this.sasToken = sasToken.token + } + + async upload(file, objectName) { + const blobService = Azure.createBlobServiceWithSas( + 'https://atat.blob.core.windows.net', + this.sasToken + ) + const fileReader = new FileReader() + const options = { + contentSettings: { + contentType: 'application/pdf', + }, + metadata: { + filename: file.name, + }, + } + + return new Promise((resolve, reject) => { + fileReader.addEventListener('load', function(f) { + blobService.createBlockBlobFromText( + 'task-order-pdfs', + `${objectName}.pdf`, + f.target.result, + options, + function(err, result) { + if (err) { + reject(err) + } else { + resolve(result) + } + } + ) + }) + fileReader.readAsText(file) + }) + } +} + +class AwsUploader { + constructor(presignedPost) { + this.presignedPost = presignedPost + } + + async upload(file, objectName) { + const form = new FormData() + Object.entries(this.presignedPost.fields).forEach(([k, v]) => { + form.append(k, v) + }) + form.append('file', file) + form.set('x-amz-meta-filename', file.name) + + return fetch(this.presignedPost.url, { + method: 'POST', + body: form, + }) + } +} + +export const buildUploader = token => new AzureUploader(token) diff --git a/package.json b/package.json index d9561c62..61b6eaa2 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "dependencies": { "ally.js": "^1.4.1", "autoprefixer": "^9.1.3", + "azure-storage": "^2.10.3", "babel-polyfill": "^6.26.0", "date-fns": "^1.29.0", "npm": "^6.0.1", diff --git a/templates/components/upload_input.html b/templates/components/upload_input.html index 3a80a834..cacb66c3 100644 --- a/templates/components/upload_input.html +++ b/templates/components/upload_input.html @@ -1,6 +1,6 @@ {% from "components/icon.html" import Icon %} -{% macro UploadInput(field, show_label=False, watch=False) -%} +{% macro UploadInput(field, show_label=False, watch=False, token="", object_name="") -%}
diff --git a/templates/task_orders/step_1.html b/templates/task_orders/step_1.html index 21b38aac..1da36762 100644 --- a/templates/task_orders/step_1.html +++ b/templates/task_orders/step_1.html @@ -18,5 +18,5 @@ {% block to_builder_form_field %} {{ TOFormStepHeader('task_orders.form.supporting_docs_header' | translate, 'task_orders.form.supporting_docs_text' | translate) }} - {{ UploadInput(form.pdf, watch=True) }} + {{ UploadInput(form.pdf, watch=True, token=token, object_name=object_name) }} {% endblock %} diff --git a/yarn.lock b/yarn.lock index c080d1c1..c7f25c21 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1163,6 +1163,23 @@ aws4@^1.6.0, aws4@^1.8.0: resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.8.0.tgz#f0e003d9ca9e7f59c7a508945d7b2ef9a04a542f" integrity sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ== +azure-storage@^2.10.3: + version "2.10.3" + resolved "https://registry.yarnpkg.com/azure-storage/-/azure-storage-2.10.3.tgz#c5966bf929d87587d78f6847040ea9a4b1d4a50a" + integrity sha512-IGLs5Xj6kO8Ii90KerQrrwuJKexLgSwYC4oLWmc11mzKe7Jt2E5IVg+ZQ8K53YWZACtVTMBNO3iGuA+4ipjJxQ== + dependencies: + browserify-mime "~1.2.9" + extend "^3.0.2" + json-edm-parser "0.1.2" + md5.js "1.3.4" + readable-stream "~2.0.0" + request "^2.86.0" + underscore "~1.8.3" + uuid "^3.0.0" + validator "~9.4.1" + xml2js "0.2.8" + xmlbuilder "^9.0.7" + babel-code-frame@^6.26.0: version "6.26.0" resolved "https://registry.yarnpkg.com/babel-code-frame/-/babel-code-frame-6.26.0.tgz#63fd43f7dc1e3bb7ce35947db8fe369a3f58c74b" @@ -1871,6 +1888,11 @@ browserify-des@^1.0.0: inherits "^2.0.1" safe-buffer "^5.1.2" +browserify-mime@~1.2.9: + version "1.2.9" + resolved "https://registry.yarnpkg.com/browserify-mime/-/browserify-mime-1.2.9.tgz#aeb1af28de6c0d7a6a2ce40adb68ff18422af31f" + integrity sha1-rrGvKN5sDXpqLOQK22j/GEIq8x8= + browserify-rsa@^4.0.0: version "4.0.1" resolved "https://registry.yarnpkg.com/browserify-rsa/-/browserify-rsa-4.0.1.tgz#21e0abfaf6f2029cf2fafb133567a701d4135524" @@ -3467,7 +3489,7 @@ extend-shallow@^3.0.0, extend-shallow@^3.0.2: assign-symbols "^1.0.0" is-extendable "^1.0.1" -extend@~3.0.0, extend@~3.0.1, extend@~3.0.2: +extend@^3.0.2, extend@~3.0.0, extend@~3.0.1, extend@~3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.2.tgz#f8b1136b4071fbd8eb140aff858b1019ec2915fa" @@ -5262,6 +5284,13 @@ jsesc@~0.5.0: resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-0.5.0.tgz#e7dee66e35d6fc16f710fe91d5cf69f70f08911d" integrity sha1-597mbjXW/Bb3EP6R1c9p9w8IkR0= +json-edm-parser@0.1.2: + version "0.1.2" + resolved "https://registry.yarnpkg.com/json-edm-parser/-/json-edm-parser-0.1.2.tgz#1e60b0fef1bc0af67bc0d146dfdde5486cd615b4" + integrity sha1-HmCw/vG8CvZ7wNFG393lSGzWFbQ= + dependencies: + jsonparse "~1.2.0" + json-parse-better-errors@^1.0.0, json-parse-better-errors@^1.0.1, json-parse-better-errors@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz#bb867cfb3450e69107c131d1c514bab3dc8bcaa9" @@ -5325,6 +5354,11 @@ jsonparse@^1.2.0: version "1.3.1" resolved "https://registry.yarnpkg.com/jsonparse/-/jsonparse-1.3.1.tgz#3f4dae4a91fac315f71062f8521cc239f1366280" +jsonparse@~1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/jsonparse/-/jsonparse-1.2.0.tgz#5c0c5685107160e72fe7489bddea0b44c2bc67bd" + integrity sha1-XAxWhRBxYOcv50ib3eoLRMK8Z70= + jsprim@^1.2.2: version "1.4.1" resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-1.4.1.tgz#313e66bc1e5cc06e438bc1b7499c2e5c56acb6a2" @@ -5686,6 +5720,14 @@ math-random@^1.0.1: resolved "https://registry.yarnpkg.com/math-random/-/math-random-1.0.4.tgz#5dd6943c938548267016d4e34f057583080c514c" integrity sha512-rUxjysqif/BZQH2yhd5Aaq7vXMSx9NdEsQcyA07uEzIvxgI7zIr33gGsh+RU0/XjmQpCW7RsVof1vlkvQVCK5A== +md5.js@1.3.4: + version "1.3.4" + resolved "https://registry.yarnpkg.com/md5.js/-/md5.js-1.3.4.tgz#e9bdbde94a20a5ac18b04340fc5764d5b09d901d" + integrity sha1-6b296UogpawYsENA/Fdk1bCdkB0= + dependencies: + hash-base "^3.0.0" + inherits "^2.0.1" + md5.js@^1.3.4: version "1.3.5" resolved "https://registry.yarnpkg.com/md5.js/-/md5.js-1.3.5.tgz#b5d07b8e3216e3e27cd728d72f70d1e6a342005f" @@ -7994,7 +8036,7 @@ request-promise-native@^1.0.5: tunnel-agent "^0.6.0" uuid "^3.0.0" -request@^2.72.0, request@^2.87.0, request@^2.88.0: +request@^2.72.0, request@^2.86.0, request@^2.87.0, request@^2.88.0: version "2.88.0" resolved "https://registry.yarnpkg.com/request/-/request-2.88.0.tgz#9c2fca4f7d35b592efe57c7f0a55e81052124fef" integrity sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg== @@ -8211,6 +8253,11 @@ sass-graph@^2.2.4: scss-tokenizer "^0.2.3" yargs "^7.0.0" +sax@0.5.x: + version "0.5.8" + resolved "https://registry.yarnpkg.com/sax/-/sax-0.5.8.tgz#d472db228eb331c2506b0e8c15524adb939d12c1" + integrity sha1-1HLbIo6zMcJQaw6MFVJK25OdEsE= + sax@^1.2.4, sax@~1.2.4: version "1.2.4" resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" @@ -9191,6 +9238,11 @@ undeclared-identifiers@^1.1.2: simple-concat "^1.0.0" xtend "^4.0.1" +underscore@~1.8.3: + version "1.8.3" + resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.8.3.tgz#4f3fb53b106e6097fcf9cb4109f2a5e9bdfa5022" + integrity sha1-Tz+1OxBuYJf8+ctBCfKl6b36UCI= + unicode-canonical-property-names-ecmascript@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-1.0.4.tgz#2619800c4c825800efdd8343af7dd9933cbe2818" @@ -9423,6 +9475,11 @@ validate-npm-package-name@^3.0.0, validate-npm-package-name@~3.0.0: dependencies: builtins "^1.0.3" +validator@~9.4.1: + version "9.4.1" + resolved "https://registry.yarnpkg.com/validator/-/validator-9.4.1.tgz#abf466d398b561cd243050112c6ff1de6cc12663" + integrity sha512-YV5KjzvRmSyJ1ee/Dm5UED0G+1L4GZnLN3w6/T+zZm8scVua4sOhYKWTUrKa0H/tMiJyO9QLHMPN+9mB/aMunA== + vendors@^1.0.0: version "1.0.3" resolved "https://registry.yarnpkg.com/vendors/-/vendors-1.0.3.tgz#a6467781abd366217c050f8202e7e50cc9eef8c0" @@ -9620,6 +9677,18 @@ xml-name-validator@^3.0.0: resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-3.0.0.tgz#6ae73e06de4d8c6e47f9fb181f78d648ad457c6a" integrity sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw== +xml2js@0.2.8: + version "0.2.8" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.2.8.tgz#9b81690931631ff09d1957549faf54f4f980b3c2" + integrity sha1-m4FpCTFjH/CdGVdUn69U9PmAs8I= + dependencies: + sax "0.5.x" + +xmlbuilder@^9.0.7: + version "9.0.7" + resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d" + integrity sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0= + xtend@^4.0.0, xtend@^4.0.1, xtend@~4.0.1: version "4.0.1" resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.1.tgz#a5c6d532be656e23db820efb943a1f04998d63af" From f5c8424b463e6dbe024a2298d8872702bc7afc8b Mon Sep 17 00:00:00 2001 From: richard-dds Date: Thu, 1 Aug 2019 14:07:09 -0400 Subject: [PATCH 02/20] Wire up FE uploads to TO form --- atst/forms/task_order.py | 16 +++++++--------- atst/models/task_order.py | 6 ++++++ atst/routes/task_orders/new.py | 2 +- js/components/upload_input.js | 2 ++ templates/components/upload_input.html | 4 +++- 5 files changed, 19 insertions(+), 11 deletions(-) diff --git a/atst/forms/task_order.py b/atst/forms/task_order.py index fca7bcfd..3ab8684c 100644 --- a/atst/forms/task_order.py +++ b/atst/forms/task_order.py @@ -5,6 +5,7 @@ from wtforms.fields import ( FileField, FormField, StringField, + HiddenField ) from wtforms.fields.html5 import DateField from wtforms.validators import Required, Optional @@ -62,17 +63,14 @@ class CLINForm(FlaskForm): return valid +class AttachmentForm(BaseForm): + filename = HiddenField(id="attachment_filename") + object_name = HiddenField(id="attachment_object_name") + + class TaskOrderForm(BaseForm): number = StringField(label=translate("forms.task_order.number_description")) - pdf = FileField( - None, - description=translate("task_orders.form.supporting_docs_size_limit"), - validators=[ - FileAllowed(["pdf"], translate("forms.task_order.file_format_not_allowed")), - FileLength(message=translate("forms.validators.file_length")), - ], - render_kw={"accept": ".pdf,application/pdf"}, - ) + pdf = FormField(AttachmentForm, label=translate("task_orders.form.supporting_docs_size_limit"), description=translate("task_orders.form.supporting_docs_size_limit")) clins = FieldList(FormField(CLINForm)) diff --git a/atst/models/task_order.py b/atst/models/task_order.py index 247ff2c8..5b873d71 100644 --- a/atst/models/task_order.py +++ b/atst/models/task_order.py @@ -9,6 +9,7 @@ from werkzeug.datastructures import FileStorage from atst.models import Attachment, Base, mixins, types from atst.models.clin import JEDICLINType from atst.utils.clock import Clock +from atst.database import db class Status(Enum): @@ -55,6 +56,11 @@ class TaskOrder(Base, mixins.TimestampsMixin): self._pdf = self._set_attachment(new_pdf, "_pdf") def _set_attachment(self, new_attachment, attribute): + if isinstance(new_attachment, dict): + attachment = Attachment(**new_attachment) + db.session.add(attachment) + db.session.commit + return attachment if isinstance(new_attachment, Attachment): return new_attachment elif isinstance(new_attachment, FileStorage): diff --git a/atst/routes/task_orders/new.py b/atst/routes/task_orders/new.py index cb5b89f2..375fc845 100644 --- a/atst/routes/task_orders/new.py +++ b/atst/routes/task_orders/new.py @@ -114,7 +114,7 @@ def form_step_one_add_pdf(portfolio_id=None, task_order_id=None): @task_orders_bp.route("/task_orders//form/step_1", methods=["POST"]) @user_can(Permissions.CREATE_TASK_ORDER, message="update task order form") def submit_form_step_one_add_pdf(portfolio_id=None, task_order_id=None): - form_data = {**http_request.form, **http_request.files} + form_data = {**http_request.form} next_page = "task_orders.form_step_two_add_number" current_template = "task_orders/step_1.html" diff --git a/js/components/upload_input.js b/js/components/upload_input.js index 4763b010..22a7c81c 100644 --- a/js/components/upload_input.js +++ b/js/components/upload_input.js @@ -67,6 +67,8 @@ export default { await this.uploader.upload(file, this.objectName) this.attachment = e.target.value this.showErrors = false + this.$refs.attachmentFilename.value = file.name + this.$refs.attachmentObjectName.value = this.objectName } catch (err) { console.log(err) this.showErrors = true diff --git a/templates/components/upload_input.html b/templates/components/upload_input.html index cacb66c3..a7203bcc 100644 --- a/templates/components/upload_input.html +++ b/templates/components/upload_input.html @@ -4,7 +4,7 @@ + +
{% for error in field.errors %} {{error}} From 3d414e1a60c550206f4afb53276ff96a28fd88ed Mon Sep 17 00:00:00 2001 From: richard-dds Date: Thu, 1 Aug 2019 15:20:17 -0400 Subject: [PATCH 03/20] Fix TO upload tests --- atst/domain/csp/file_uploads.py | 15 ++++++++++++-- atst/forms/task_order.py | 11 +++++----- atst/models/attachment.py | 10 +++++++++ atst/models/task_order.py | 14 +++++++------ atst/routes/task_orders/new.py | 4 ---- tests/factories.py | 4 +++- tests/routes/task_orders/test_new.py | 31 ++++++++++++---------------- tests/test_access.py | 4 +--- 8 files changed, 54 insertions(+), 39 deletions(-) diff --git a/atst/domain/csp/file_uploads.py b/atst/domain/csp/file_uploads.py index d9c2a367..57ea8f0e 100644 --- a/atst/domain/csp/file_uploads.py +++ b/atst/domain/csp/file_uploads.py @@ -7,10 +7,13 @@ import boto3 def build_uploader(config): - if config["CSP"] == "aws": + csp = config.get("CSP") + if csp == "aws": return AwsUploader(config) - elif config["CSP"] == "azure": + elif csp == "azure": return AzureUploader(config) + else: + return MockUploader(config) class Uploader: @@ -21,6 +24,14 @@ class Uploader: return str(uuid4()) +class MockUploader(Uploader): + def __init__(self, config): + self.config = config + + def get_token(self): + return ({}, self.object_name()) + + class AzureUploader(Uploader): def __init__(self, config): self.config = config diff --git a/atst/forms/task_order.py b/atst/forms/task_order.py index 3ab8684c..c8fcbdb9 100644 --- a/atst/forms/task_order.py +++ b/atst/forms/task_order.py @@ -2,20 +2,17 @@ from wtforms.fields import ( BooleanField, DecimalField, FieldList, - FileField, FormField, StringField, - HiddenField + HiddenField, ) from wtforms.fields.html5 import DateField from wtforms.validators import Required, Optional -from flask_wtf.file import FileAllowed from flask_wtf import FlaskForm from .data import JEDI_CLIN_TYPES from .fields import SelectField from .forms import BaseForm -from atst.forms.validators import FileLength from atst.utils.localization import translate @@ -70,7 +67,11 @@ class AttachmentForm(BaseForm): class TaskOrderForm(BaseForm): number = StringField(label=translate("forms.task_order.number_description")) - pdf = FormField(AttachmentForm, label=translate("task_orders.form.supporting_docs_size_limit"), description=translate("task_orders.form.supporting_docs_size_limit")) + pdf = FormField( + AttachmentForm, + label=translate("task_orders.form.supporting_docs_size_limit"), + description=translate("task_orders.form.supporting_docs_size_limit"), + ) clins = FieldList(FormField(CLINForm)) diff --git a/atst/models/attachment.py b/atst/models/attachment.py index e4a9d6c2..4edb3441 100644 --- a/atst/models/attachment.py +++ b/atst/models/attachment.py @@ -40,6 +40,16 @@ class Attachment(Base, mixins.TimestampsMixin): return attachment + @classmethod + def get_or_create(cls, object_name, params): + try: + return db.session.query(Attachment).filter_by(object_name=object_name).one() + except NoResultFound: + new_attachment = cls(**params) + db.session.add(new_attachment) + db.session.commit() + return new_attachment + @classmethod def get(cls, id_): try: diff --git a/atst/models/task_order.py b/atst/models/task_order.py index 5b873d71..8b060164 100644 --- a/atst/models/task_order.py +++ b/atst/models/task_order.py @@ -9,7 +9,6 @@ from werkzeug.datastructures import FileStorage from atst.models import Attachment, Base, mixins, types from atst.models.clin import JEDICLINType from atst.utils.clock import Clock -from atst.database import db class Status(Enum): @@ -56,15 +55,18 @@ class TaskOrder(Base, mixins.TimestampsMixin): self._pdf = self._set_attachment(new_pdf, "_pdf") def _set_attachment(self, new_attachment, attribute): - if isinstance(new_attachment, dict): - attachment = Attachment(**new_attachment) - db.session.add(attachment) - db.session.commit - return attachment if isinstance(new_attachment, Attachment): return new_attachment elif isinstance(new_attachment, FileStorage): return Attachment.attach(new_attachment, "task_order", self.id) + elif isinstance(new_attachment, dict): + if new_attachment["filename"] and new_attachment["object_name"]: + attachment = Attachment.get_or_create( + new_attachment["object_name"], new_attachment + ) + return attachment + else: + return None elif not new_attachment and hasattr(self, attribute): return None else: diff --git a/atst/routes/task_orders/new.py b/atst/routes/task_orders/new.py index 375fc845..f8a9a3a9 100644 --- a/atst/routes/task_orders/new.py +++ b/atst/routes/task_orders/new.py @@ -6,9 +6,6 @@ from flask import ( url_for, current_app, ) -from azure.storage.common import CloudStorageAccount -from azure.storage.blob import ContainerPermissions -from datetime import datetime, timedelta from . import task_orders_bp from atst.domain.authz.decorator import user_can_access_decorator as user_can @@ -100,7 +97,6 @@ def edit(task_order_id): @task_orders_bp.route("/task_orders//form/step_1") @user_can(Permissions.CREATE_TASK_ORDER, message="view task order form") def form_step_one_add_pdf(portfolio_id=None, task_order_id=None): - return render_task_orders_edit( "task_orders/step_1.html", portfolio_id=portfolio_id, diff --git a/tests/factories.py b/tests/factories.py index 4dd2a7f8..91fe0566 100644 --- a/tests/factories.py +++ b/tests/factories.py @@ -269,7 +269,9 @@ class TaskOrderFactory(Base): class Meta: model = TaskOrder - portfolio = factory.SubFactory(PortfolioFactory) + portfolio = factory.SubFactory( + PortfolioFactory, owner=factory.SelfAttribute("..creator") + ) number = factory.LazyFunction(random_task_order_number) creator = factory.SubFactory(UserFactory) _pdf = factory.SubFactory(AttachmentFactory) diff --git a/tests/routes/task_orders/test_new.py b/tests/routes/task_orders/test_new.py index b88bbf7b..9cb5a6b4 100644 --- a/tests/routes/task_orders/test_new.py +++ b/tests/routes/task_orders/test_new.py @@ -17,6 +17,10 @@ from tests.factories import ( ) +def build_pdf_form_data(filename="sample.pdf", object_name="object_name"): + return {"pdf-filename": filename, "pdf-object_name": object_name} + + @pytest.fixture def task_order(): user = UserFactory.create() @@ -55,19 +59,16 @@ def test_task_orders_form_step_one_add_pdf(client, user_session, portfolio): assert response.status_code == 200 -def test_task_orders_submit_form_step_one_add_pdf( - client, user_session, portfolio, pdf_upload, session -): +def test_task_orders_submit_form_step_one_add_pdf(client, user_session, portfolio): user_session(portfolio.owner) - form_data = {"pdf": pdf_upload} response = client.post( url_for("task_orders.submit_form_step_one_add_pdf", portfolio_id=portfolio.id), - data=form_data, + data=build_pdf_form_data(), ) assert response.status_code == 302 task_order = portfolio.task_orders[0] - assert task_order.pdf.filename == pdf_upload.filename + assert task_order.pdf.filename == "sample.pdf" def test_task_orders_form_step_one_add_pdf_existing_to( @@ -80,35 +81,29 @@ def test_task_orders_form_step_one_add_pdf_existing_to( assert response.status_code == 200 -def test_task_orders_submit_form_step_one_add_pdf_existing_to( - client, user_session, task_order, pdf_upload, pdf_upload2 -): - task_order.pdf = pdf_upload - assert task_order.pdf.filename == pdf_upload.filename - +def test_task_orders_submit_form_step_one_add_pdf_existing_to(client, user_session): + task_order = TaskOrderFactory.create() user_session(task_order.creator) - form_data = {"pdf": pdf_upload2} response = client.post( url_for( "task_orders.submit_form_step_one_add_pdf", task_order_id=task_order.id ), - data=form_data, + data=build_pdf_form_data(), ) assert response.status_code == 302 - assert task_order.pdf.filename == pdf_upload2.filename + assert task_order.pdf.filename == "sample.pdf" def test_task_orders_submit_form_step_one_add_pdf_delete_pdf( client, user_session, portfolio, pdf_upload ): user_session(portfolio.owner) - task_order = TaskOrderFactory.create(pdf=pdf_upload, portfolio=portfolio) - data = {"pdf": ""} + task_order = TaskOrderFactory.create(portfolio=portfolio) response = client.post( url_for( "task_orders.submit_form_step_one_add_pdf", task_order_id=task_order.id ), - data=data, + data=build_pdf_form_data(filename="", object_name=""), ) assert task_order.pdf is None assert response.status_code == 302 diff --git a/tests/test_access.py b/tests/test_access.py index eefa2160..a47c7b02 100644 --- a/tests/test_access.py +++ b/tests/test_access.py @@ -438,9 +438,7 @@ def test_task_orders_download_task_order_pdf_access(get_url_assert_status, monke rando = user_with() portfolio = PortfolioFactory.create(owner=owner) - task_order = TaskOrderFactory.create( - portfolio=portfolio, pdf=AttachmentFactory.create() - ) + task_order = TaskOrderFactory.create(portfolio=portfolio) url = url_for("task_orders.download_task_order_pdf", task_order_id=task_order.id) get_url_assert_status(owner, url, 200) From 81a478915de3ceebb40250208b22367b24d1e686 Mon Sep 17 00:00:00 2001 From: richard-dds Date: Mon, 5 Aug 2019 11:19:21 -0400 Subject: [PATCH 04/20] Fix uploader test and refactor makeTestWrapper to accept data fn --- .../__tests__/checkbox_input.test.js | 3 ++ js/components/__tests__/upload_input.test.js | 16 ++++++-- .../upload_input_error_template.html | 15 +++++--- js/test_templates/upload_input_template.html | 13 +++++-- js/test_utils/component_test_helpers.js | 8 +--- templates/components/upload_input.html | 4 +- tests/render_vue_component.py | 37 +++++++++++++++---- 7 files changed, 69 insertions(+), 27 deletions(-) diff --git a/js/components/__tests__/checkbox_input.test.js b/js/components/__tests__/checkbox_input.test.js index 6be9c1e4..e7d0fb5a 100644 --- a/js/components/__tests__/checkbox_input.test.js +++ b/js/components/__tests__/checkbox_input.test.js @@ -9,6 +9,9 @@ const WrapperComponent = makeTestWrapper({ checkboxinput, }, templatePath: 'checkbox_input_template.html', + data: function() { + return { initialvalue: this.initialData } + } }) describe('CheckboxInput Renders Correctly', () => { diff --git a/js/components/__tests__/upload_input.test.js b/js/components/__tests__/upload_input.test.js index 96a1be89..7022ce41 100644 --- a/js/components/__tests__/upload_input.test.js +++ b/js/components/__tests__/upload_input.test.js @@ -7,18 +7,24 @@ import { makeTestWrapper } from '../../test_utils/component_test_helpers' const UploadWrapper = makeTestWrapper({ components: { uploadinput }, templatePath: 'upload_input_template.html', + data: function() { + return { initialvalue: this.initialData.initialvalue, token: this.token } + } }) const UploadErrorWrapper = makeTestWrapper({ components: { uploadinput }, templatePath: 'upload_input_error_template.html', + data: function() { + return { initialvalue: null, token: null } + } }) describe('UploadInput Test', () => { it('should show input and button when no attachment present', () => { const wrapper = mount(UploadWrapper, { propsData: { - initialData: null, + initialData: { initialvalue: null, token: "token" }, }, }) @@ -29,7 +35,7 @@ describe('UploadInput Test', () => { it('should show file name and hide input', () => { const wrapper = mount(UploadWrapper, { propsData: { - initialData: 'somepdf.pdf', + initialData: { initialvalue: "somepdf.pdf", token: "token" } }, }) @@ -41,7 +47,11 @@ describe('UploadInput Test', () => { }) it('should correctly display error treatment', () => { - const wrapper = mount(UploadErrorWrapper) + const wrapper = mount(UploadErrorWrapper, { + propsData: { + initialData: { initialvalue: "somepdf.pdf", token: "token" } + } + }) const messageArea = wrapper.find('.usa-input__message') expect(messageArea.html()).toContain('Test Error Message') diff --git a/js/test_templates/upload_input_error_template.html b/js/test_templates/upload_input_error_template.html index 077eaeb2..efb83e29 100644 --- a/js/test_templates/upload_input_error_template.html +++ b/js/test_templates/upload_input_error_template.html @@ -4,8 +4,10 @@ v-bind:initial-errors='true' v-bind:watch='false' - name='errorfield' + name='pdf' :optional='false' + v-bind:token='token' + v-bind:object-name='"object_name"' >
@@ -19,7 +21,7 @@
-
- Test Error Message + ['Test Error Message']
diff --git a/js/test_templates/upload_input_template.html b/js/test_templates/upload_input_template.html index 2179385f..37412a66 100644 --- a/js/test_templates/upload_input_template.html +++ b/js/test_templates/upload_input_template.html @@ -4,8 +4,10 @@ v-bind:initial-data='initialvalue' v-bind:watch='false' - name='datafield' + name='pdf' :optional='false' + v-bind:token='token' + v-bind:object-name='"object_name"' >
@@ -19,7 +21,7 @@
-
diff --git a/js/test_utils/component_test_helpers.js b/js/test_utils/component_test_helpers.js index e81d9603..b8b28cc2 100644 --- a/js/test_utils/component_test_helpers.js +++ b/js/test_utils/component_test_helpers.js @@ -16,7 +16,7 @@ to be passed as a prop to checkboxinput at mount time v-bind:initial-checked='initialvalue' > */ -const makeTestWrapper = ({ components, templatePath }) => { +const makeTestWrapper = ({ components, templatePath, data }) => { const templateString = fs.readFileSync( `js/test_templates/${templatePath}`, 'utf-8' @@ -27,11 +27,7 @@ const makeTestWrapper = ({ components, templatePath }) => { components, template: templateString, props: ['initialData'], - data: function() { - return { - initialvalue: this.initialData, - } - }, + data, } return WrapperComponent diff --git a/templates/components/upload_input.html b/templates/components/upload_input.html index a7203bcc..6e71d157 100644 --- a/templates/components/upload_input.html +++ b/templates/components/upload_input.html @@ -46,8 +46,8 @@
- {% for error in field.errors %} - {{error}} + {% for error, error_message in field.errors.items() %} + {{error_message}} {% endfor %}
diff --git a/tests/render_vue_component.py b/tests/render_vue_component.py index 5d9f3b08..c09f6eb2 100644 --- a/tests/render_vue_component.py +++ b/tests/render_vue_component.py @@ -2,8 +2,8 @@ import pytest from wtforms.widgets import CheckboxInput from wtforms.fields import StringField -from wtforms.validators import InputRequired -from wtforms import Form +from wtforms.validators import InputRequired, URL +from wtforms import Form, FormField class InitialValueForm(Form): @@ -14,6 +14,19 @@ class InitialValueForm(Form): ) +class TaskOrderPdfForm(Form): + filename = StringField(default="initialvalue") + object_name = StringField() + + errorfield = StringField( + label="error", validators=[InputRequired(message="Test Error Message")] + ) + + +class TaskOrderForm(Form): + pdf = FormField(TaskOrderPdfForm, label="task_order_pdf") + + @pytest.fixture def env(app, scope="function"): return app.jinja_env @@ -39,6 +52,16 @@ def initial_value_form(scope="function"): return InitialValueForm() +@pytest.fixture +def task_order_form(scope="function"): + return TaskOrderForm() + + +@pytest.fixture +def error_task_order_form(scope="function"): + return ErrorTaskOrderForm() + + def write_template(content, name): with open("js/test_templates/{}".format(name), "w") as fh: fh.write(content) @@ -50,12 +73,12 @@ def test_make_checkbox_input_template(checkbox_input_macro, initial_value_form): write_template(rendered_checkbox_macro, "checkbox_input_template.html") -def test_make_upload_input_template(upload_input_macro, initial_value_form): - rendered_upload_macro = upload_input_macro(initial_value_form.datafield) +def test_make_upload_input_template(upload_input_macro, task_order_form): + rendered_upload_macro = upload_input_macro(task_order_form.pdf, token="token", object_name="object_name") write_template(rendered_upload_macro, "upload_input_template.html") -def test_make_upload_input_error_template(upload_input_macro, initial_value_form): - initial_value_form.validate() - rendered_upload_macro = upload_input_macro(initial_value_form.errorfield) +def test_make_upload_input_error_template(upload_input_macro, task_order_form): + task_order_form.validate() + rendered_upload_macro = upload_input_macro(task_order_form.pdf, token="token", object_name="object_name") write_template(rendered_upload_macro, "upload_input_error_template.html") From 94b6b320fb5674ac21b5510ffb1f08aafe253017 Mon Sep 17 00:00:00 2001 From: richard-dds Date: Mon, 5 Aug 2019 11:19:48 -0400 Subject: [PATCH 05/20] Provide mock implementation of uploader --- js/lib/upload.js | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/js/lib/upload.js b/js/lib/upload.js index e0a8369b..53844003 100644 --- a/js/lib/upload.js +++ b/js/lib/upload.js @@ -61,4 +61,14 @@ class AwsUploader { } } -export const buildUploader = token => new AzureUploader(token) +class MockUploader { + constructor(token) { + this.token = token + } + + async upload(file, objectName) { + return Promise.resolve({}) + } +} + +export const buildUploader = token => new MockUploader(token) From 6405d9b958d29ef8a31ff93fd95e81bb226a8a18 Mon Sep 17 00:00:00 2001 From: richard-dds Date: Mon, 5 Aug 2019 11:36:19 -0400 Subject: [PATCH 06/20] Make Azure account name and bucket name configurable --- atst/domain/csp/file_uploads.py | 9 +++++++-- js/components/__tests__/checkbox_input.test.js | 2 +- js/components/__tests__/upload_input.test.js | 12 ++++++------ tests/render_vue_component.py | 8 ++++++-- 4 files changed, 20 insertions(+), 11 deletions(-) diff --git a/atst/domain/csp/file_uploads.py b/atst/domain/csp/file_uploads.py index 57ea8f0e..80d6cb70 100644 --- a/atst/domain/csp/file_uploads.py +++ b/atst/domain/csp/file_uploads.py @@ -29,6 +29,10 @@ class MockUploader(Uploader): self.config = config def get_token(self): + """ + Generates a pre-authenticated token and object name which we'll use to + upload the file to the CSP. + """ return ({}, self.object_name()) @@ -38,12 +42,13 @@ class AzureUploader(Uploader): def get_token(self): account = CloudStorageAccount( - account_name="atat", account_key=self.config["AZURE_STORAGE_KEY"] + account_name=self.config["AZURE_ACCOUNT_NAME"], + account_key=self.config["AZURE_STORAGE_KEY"], ) bbs = account.create_block_blob_service() object_name = self.object_name() sas_token = bbs.generate_container_shared_access_signature( - "task-order-pdfs", + self.config["AZURE_TO_BUCKET_NAME"], ContainerPermissions.WRITE, datetime.utcnow() + timedelta(minutes=15), ) diff --git a/js/components/__tests__/checkbox_input.test.js b/js/components/__tests__/checkbox_input.test.js index e7d0fb5a..4c6b15f8 100644 --- a/js/components/__tests__/checkbox_input.test.js +++ b/js/components/__tests__/checkbox_input.test.js @@ -11,7 +11,7 @@ const WrapperComponent = makeTestWrapper({ templatePath: 'checkbox_input_template.html', data: function() { return { initialvalue: this.initialData } - } + }, }) describe('CheckboxInput Renders Correctly', () => { diff --git a/js/components/__tests__/upload_input.test.js b/js/components/__tests__/upload_input.test.js index 7022ce41..c867eea3 100644 --- a/js/components/__tests__/upload_input.test.js +++ b/js/components/__tests__/upload_input.test.js @@ -9,7 +9,7 @@ const UploadWrapper = makeTestWrapper({ templatePath: 'upload_input_template.html', data: function() { return { initialvalue: this.initialData.initialvalue, token: this.token } - } + }, }) const UploadErrorWrapper = makeTestWrapper({ @@ -17,14 +17,14 @@ const UploadErrorWrapper = makeTestWrapper({ templatePath: 'upload_input_error_template.html', data: function() { return { initialvalue: null, token: null } - } + }, }) describe('UploadInput Test', () => { it('should show input and button when no attachment present', () => { const wrapper = mount(UploadWrapper, { propsData: { - initialData: { initialvalue: null, token: "token" }, + initialData: { initialvalue: null, token: 'token' }, }, }) @@ -35,7 +35,7 @@ describe('UploadInput Test', () => { it('should show file name and hide input', () => { const wrapper = mount(UploadWrapper, { propsData: { - initialData: { initialvalue: "somepdf.pdf", token: "token" } + initialData: { initialvalue: 'somepdf.pdf', token: 'token' }, }, }) @@ -49,8 +49,8 @@ describe('UploadInput Test', () => { it('should correctly display error treatment', () => { const wrapper = mount(UploadErrorWrapper, { propsData: { - initialData: { initialvalue: "somepdf.pdf", token: "token" } - } + initialData: { initialvalue: 'somepdf.pdf', token: 'token' }, + }, }) const messageArea = wrapper.find('.usa-input__message') diff --git a/tests/render_vue_component.py b/tests/render_vue_component.py index c09f6eb2..b502ea93 100644 --- a/tests/render_vue_component.py +++ b/tests/render_vue_component.py @@ -74,11 +74,15 @@ def test_make_checkbox_input_template(checkbox_input_macro, initial_value_form): def test_make_upload_input_template(upload_input_macro, task_order_form): - rendered_upload_macro = upload_input_macro(task_order_form.pdf, token="token", object_name="object_name") + rendered_upload_macro = upload_input_macro( + task_order_form.pdf, token="token", object_name="object_name" + ) write_template(rendered_upload_macro, "upload_input_template.html") def test_make_upload_input_error_template(upload_input_macro, task_order_form): task_order_form.validate() - rendered_upload_macro = upload_input_macro(task_order_form.pdf, token="token", object_name="object_name") + rendered_upload_macro = upload_input_macro( + task_order_form.pdf, token="token", object_name="object_name" + ) write_template(rendered_upload_macro, "upload_input_error_template.html") From 0b201809a117ad47e5c416455a018b718de15d2f Mon Sep 17 00:00:00 2001 From: richard-dds Date: Mon, 5 Aug 2019 15:59:27 -0400 Subject: [PATCH 07/20] Configure js uploader based on CLOUD_PROVIDER environment var --- js/lib/upload.js | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/js/lib/upload.js b/js/lib/upload.js index 53844003..beccaf1a 100644 --- a/js/lib/upload.js +++ b/js/lib/upload.js @@ -71,4 +71,13 @@ class MockUploader { } } -export const buildUploader = token => new MockUploader(token) +export const buildUploader = token => { + const cloudProvider = process.env.CLOUD_PROVIDER || "mock" + if (cloudProvider === "aws") { + return new AwsUploader(token) + } else if (cloudProvider === "azure") { + return new AzureUploader(token) + } else { + return new MockUploader(token) + } +} From e5a0318310a964c21781d89c2b26add50eef17a2 Mon Sep 17 00:00:00 2001 From: richard-dds Date: Mon, 5 Aug 2019 16:04:38 -0400 Subject: [PATCH 08/20] Formatting --- js/lib/upload.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/js/lib/upload.js b/js/lib/upload.js index beccaf1a..3a3c9881 100644 --- a/js/lib/upload.js +++ b/js/lib/upload.js @@ -72,10 +72,10 @@ class MockUploader { } export const buildUploader = token => { - const cloudProvider = process.env.CLOUD_PROVIDER || "mock" - if (cloudProvider === "aws") { + const cloudProvider = process.env.CLOUD_PROVIDER || 'mock' + if (cloudProvider === 'aws') { return new AwsUploader(token) - } else if (cloudProvider === "azure") { + } else if (cloudProvider === 'azure') { return new AzureUploader(token) } else { return new MockUploader(token) From 72275421d02c452c9f1f3589a1078f8c080334c8 Mon Sep 17 00:00:00 2001 From: richard-dds Date: Mon, 5 Aug 2019 16:14:01 -0400 Subject: [PATCH 09/20] More CSP-specific docstrings for get_token --- atst/domain/csp/file_uploads.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/atst/domain/csp/file_uploads.py b/atst/domain/csp/file_uploads.py index 80d6cb70..5cda782a 100644 --- a/atst/domain/csp/file_uploads.py +++ b/atst/domain/csp/file_uploads.py @@ -29,10 +29,6 @@ class MockUploader(Uploader): self.config = config def get_token(self): - """ - Generates a pre-authenticated token and object name which we'll use to - upload the file to the CSP. - """ return ({}, self.object_name()) @@ -41,6 +37,13 @@ class AzureUploader(Uploader): self.config = config def get_token(self): + """ + Generates an Azure SAS token for pre-authorizing a file upload. + + Returns a tuple in the following format: (token_dict, object_name), where + - token_dict has a `token` key which contains the SAS token as a string + - object_name is a string + """ account = CloudStorageAccount( account_name=self.config["AZURE_ACCOUNT_NAME"], account_key=self.config["AZURE_STORAGE_KEY"], @@ -60,6 +63,14 @@ class AwsUploader(Uploader): self.config = config def get_token(self): + """ + Generates an AWS presigned post for pre-authorizing a file upload. + + Returns a tuple in the following format: (token_dict, object_name), where + - token_dict contains several fields that will be passed directly into the + form before being sent to S3 + - object_name is a string + """ s3_client = boto3.client( "s3", aws_access_key_id=self.config["AWS_ACCESS_KEY_ID"], From 68a1004bf17650dd1c208d625d33693951a3976c Mon Sep 17 00:00:00 2001 From: richard-dds Date: Mon, 5 Aug 2019 16:51:28 -0400 Subject: [PATCH 10/20] Make Azure FE config injectable --- js/lib/upload.js | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/js/lib/upload.js b/js/lib/upload.js index 3a3c9881..b3ff730f 100644 --- a/js/lib/upload.js +++ b/js/lib/upload.js @@ -1,13 +1,16 @@ import Azure from 'azure-storage' class AzureUploader { - constructor(sasToken) { + constructor(accountName, containerName, sasToken) { + this.accountName = accountName + this.containerName = containerName this.sasToken = sasToken.token } async upload(file, objectName) { + console.log(this) const blobService = Azure.createBlobServiceWithSas( - 'https://atat.blob.core.windows.net', + `https://${this.accountName}.blob.core.windows.net`, this.sasToken ) const fileReader = new FileReader() @@ -23,7 +26,7 @@ class AzureUploader { return new Promise((resolve, reject) => { fileReader.addEventListener('load', function(f) { blobService.createBlockBlobFromText( - 'task-order-pdfs', + this.containerName, `${objectName}.pdf`, f.target.result, options, @@ -76,7 +79,11 @@ export const buildUploader = token => { if (cloudProvider === 'aws') { return new AwsUploader(token) } else if (cloudProvider === 'azure') { - return new AzureUploader(token) + return new AzureUploader( + process.env.AZURE_ACCOUNT_NAME, + process.env.AZURE_CONTAINER_NAME, + token + ) } else { return new MockUploader(token) } From b18e2915114f8fee6341cc7ce1a404189e18702f Mon Sep 17 00:00:00 2001 From: richard-dds Date: Tue, 6 Aug 2019 09:58:44 -0400 Subject: [PATCH 11/20] Set Azure protocol to https - Default allows https/http --- atst/domain/csp/file_uploads.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/atst/domain/csp/file_uploads.py b/atst/domain/csp/file_uploads.py index 5cda782a..749dac1b 100644 --- a/atst/domain/csp/file_uploads.py +++ b/atst/domain/csp/file_uploads.py @@ -53,7 +53,7 @@ class AzureUploader(Uploader): sas_token = bbs.generate_container_shared_access_signature( self.config["AZURE_TO_BUCKET_NAME"], ContainerPermissions.WRITE, - datetime.utcnow() + timedelta(minutes=15), + protocol="https" ) return ({"token": sas_token}, object_name) From 80831c2d85240f2db26620bcf978853543fa2352 Mon Sep 17 00:00:00 2001 From: richard-dds Date: Tue, 6 Aug 2019 09:59:12 -0400 Subject: [PATCH 12/20] Set uploader configs in __init__ Now missing configs cause the app to crash right at startup --- atst/domain/csp/file_uploads.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/atst/domain/csp/file_uploads.py b/atst/domain/csp/file_uploads.py index 749dac1b..6c4f04f7 100644 --- a/atst/domain/csp/file_uploads.py +++ b/atst/domain/csp/file_uploads.py @@ -34,7 +34,10 @@ class MockUploader(Uploader): class AzureUploader(Uploader): def __init__(self, config): - self.config = config + self.account_name = config["AZURE_ACCOUNT_NAME"] + self.storage_key = config["AZURE_STORAGE_KEY"] + self.container_name = config["AZURE_TO_BUCKET_NAME"] + self.timeout = timedelta(seconds=config["PERMANENT_SESSION_LIFETIME"]) def get_token(self): """ @@ -45,22 +48,26 @@ class AzureUploader(Uploader): - object_name is a string """ account = CloudStorageAccount( - account_name=self.config["AZURE_ACCOUNT_NAME"], - account_key=self.config["AZURE_STORAGE_KEY"], + account_name=self.account_name, account_key=self.storage_key ) bbs = account.create_block_blob_service() object_name = self.object_name() sas_token = bbs.generate_container_shared_access_signature( - self.config["AZURE_TO_BUCKET_NAME"], + self.container_name, ContainerPermissions.WRITE, - protocol="https" + datetime.utcnow() + self.timeout, + protocol="https", ) return ({"token": sas_token}, object_name) class AwsUploader(Uploader): def __init__(self, config): - self.config = config + self.access_key_id = config["AWS_ACCESS_KEY_ID"] + self.secret_key = config["AWS_SECRET_KEY"] + self.region_name = config["AWS_REGION_NAME"] + self.bucket_name = config["AWS_BUCKET_NAME"] + self.timeout_secs = config["PERMANENT_SESSION_LIFETIME"] def get_token(self): """ @@ -73,15 +80,15 @@ class AwsUploader(Uploader): """ s3_client = boto3.client( "s3", - aws_access_key_id=self.config["AWS_ACCESS_KEY_ID"], - aws_secret_access_key=self.config["AWS_SECRET_KEY"], + aws_access_key_id=self.access_key_id, + aws_secret_access_key=self.secret_key, config=boto3.session.Config( - signature_version="s3v4", region_name=self.config["AWS_REGION_NAME"] + signature_version="s3v4", region_name=self.region_name ), ) object_name = self.object_name() presigned_post = s3_client.generate_presigned_post( - self.config["AWS_BUCKET_NAME"], + self.bucket_name, object_name, ExpiresIn=3600, Conditions=[ From a4581e7a7c6fa8721f760357a40db1a924a2491d Mon Sep 17 00:00:00 2001 From: richard-dds Date: Tue, 6 Aug 2019 10:23:04 -0400 Subject: [PATCH 13/20] js `this` woes --- js/lib/upload.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/js/lib/upload.js b/js/lib/upload.js index b3ff730f..e77d403c 100644 --- a/js/lib/upload.js +++ b/js/lib/upload.js @@ -8,7 +8,6 @@ class AzureUploader { } async upload(file, objectName) { - console.log(this) const blobService = Azure.createBlobServiceWithSas( `https://${this.accountName}.blob.core.windows.net`, this.sasToken @@ -24,7 +23,7 @@ class AzureUploader { } return new Promise((resolve, reject) => { - fileReader.addEventListener('load', function(f) { + fileReader.addEventListener('load', f => { blobService.createBlockBlobFromText( this.containerName, `${objectName}.pdf`, From 5329937eb9018ef34bb2d9cdb3f893e9be1867b9 Mon Sep 17 00:00:00 2001 From: richard-dds Date: Tue, 6 Aug 2019 10:39:46 -0400 Subject: [PATCH 14/20] Polyfill fetch --- js/lib/upload.js | 1 + package.json | 3 ++- yarn.lock | 5 +++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/js/lib/upload.js b/js/lib/upload.js index e77d403c..d5512ef1 100644 --- a/js/lib/upload.js +++ b/js/lib/upload.js @@ -1,4 +1,5 @@ import Azure from 'azure-storage' +import 'whatwg-fetch' class AzureUploader { constructor(accountName, containerName, sasToken) { diff --git a/package.json b/package.json index 61b6eaa2..8fea9ca3 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,8 @@ "uswds": "^1.6.9", "v-tooltip": "^2.0.0-rc.33", "vue": "2.5.15", - "vue-text-mask": "^6.1.2" + "vue-text-mask": "^6.1.2", + "whatwg-fetch": "^3.0.0" }, "devDependencies": { "@vue/test-utils": "^1.0.0-beta.25", diff --git a/yarn.lock b/yarn.lock index c7f25c21..e3b3556a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9564,6 +9564,11 @@ whatwg-encoding@^1.0.1, whatwg-encoding@^1.0.3: dependencies: iconv-lite "0.4.24" +whatwg-fetch@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/whatwg-fetch/-/whatwg-fetch-3.0.0.tgz#fc804e458cc460009b1a2b966bc8817d2578aefb" + integrity sha512-9GSJUgz1D4MfyKU7KRqwOjXCXTqWdFNvEr7eUBYchQiVc744mqK/MzXPNR2WsPkmkOa4ywfg8C2n8h+13Bey1Q== + whatwg-mimetype@^2.1.0, whatwg-mimetype@^2.2.0: version "2.3.0" resolved "https://registry.yarnpkg.com/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz#3d4b1e0312d2079879f826aff18dbeeca5960fbf" From 989e28e5fbd8b8c60dc4da887386ec0dd3f495c9 Mon Sep 17 00:00:00 2001 From: richard-dds Date: Tue, 6 Aug 2019 10:41:17 -0400 Subject: [PATCH 15/20] Destructure get_token() result --- atst/routes/task_orders/new.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/atst/routes/task_orders/new.py b/atst/routes/task_orders/new.py index f8a9a3a9..d04dd52c 100644 --- a/atst/routes/task_orders/new.py +++ b/atst/routes/task_orders/new.py @@ -17,8 +17,8 @@ from atst.utils.flash import formatted_flash as flash def render_task_orders_edit(template, portfolio_id=None, task_order_id=None, form=None): - result = current_app.uploader.get_token() - render_args = {"token": result[0], "object_name": result[1]} + (token, object_name) = current_app.uploader.get_token() + render_args = {"token": token, "object_name": object_name} if task_order_id: task_order = TaskOrders.get(task_order_id) From 59de01031cd93cea5d154ec357165f2dadd4d7da Mon Sep 17 00:00:00 2001 From: richard-dds Date: Tue, 6 Aug 2019 13:43:57 -0400 Subject: [PATCH 16/20] More secure SAS permissions for Azure upload --- atst/domain/csp/file_uploads.py | 9 +++++---- js/lib/upload.js | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/atst/domain/csp/file_uploads.py b/atst/domain/csp/file_uploads.py index 6c4f04f7..b967e994 100644 --- a/atst/domain/csp/file_uploads.py +++ b/atst/domain/csp/file_uploads.py @@ -1,5 +1,5 @@ from azure.storage.common import CloudStorageAccount -from azure.storage.blob import ContainerPermissions +from azure.storage.blob import BlobPermissions from datetime import datetime, timedelta from uuid import uuid4 @@ -52,10 +52,11 @@ class AzureUploader(Uploader): ) bbs = account.create_block_blob_service() object_name = self.object_name() - sas_token = bbs.generate_container_shared_access_signature( + sas_token = bbs.generate_blob_shared_access_signature( self.container_name, - ContainerPermissions.WRITE, - datetime.utcnow() + self.timeout, + object_name, + permission=BlobPermissions.CREATE, + expiry=datetime.utcnow() + self.timeout, protocol="https", ) return ({"token": sas_token}, object_name) diff --git a/js/lib/upload.js b/js/lib/upload.js index d5512ef1..f102c851 100644 --- a/js/lib/upload.js +++ b/js/lib/upload.js @@ -27,7 +27,7 @@ class AzureUploader { fileReader.addEventListener('load', f => { blobService.createBlockBlobFromText( this.containerName, - `${objectName}.pdf`, + `${objectName}`, f.target.result, options, function(err, result) { From 86c1dfb74952dac68847da487cb18ab2b7029ad3 Mon Sep 17 00:00:00 2001 From: richard-dds Date: Tue, 6 Aug 2019 16:49:19 -0400 Subject: [PATCH 17/20] Enforce PDF in TO uploader file picker --- atst/forms/task_order.py | 1 + 1 file changed, 1 insertion(+) diff --git a/atst/forms/task_order.py b/atst/forms/task_order.py index c8fcbdb9..30a86c3b 100644 --- a/atst/forms/task_order.py +++ b/atst/forms/task_order.py @@ -63,6 +63,7 @@ class CLINForm(FlaskForm): class AttachmentForm(BaseForm): filename = HiddenField(id="attachment_filename") object_name = HiddenField(id="attachment_object_name") + accept = ".pdf,application/pdf" class TaskOrderForm(BaseForm): From 5a1c6b2b39eef671e34b32bae109392c214f3700 Mon Sep 17 00:00:00 2001 From: richard-dds Date: Tue, 6 Aug 2019 16:56:10 -0400 Subject: [PATCH 18/20] Remove azure-storage-file --- Pipfile | 1 - Pipfile.lock | 79 ++++++++++++---------------------------------------- 2 files changed, 18 insertions(+), 62 deletions(-) diff --git a/Pipfile b/Pipfile index 1add2b71..4e169d3a 100644 --- a/Pipfile +++ b/Pipfile @@ -25,7 +25,6 @@ werkzeug = "*" PyYAML = "*" azure-storage = "*" azure-storage-common = "*" -azure-storage-file = "*" boto3 = "*" [dev-packages] diff --git a/Pipfile.lock b/Pipfile.lock index 988b3d10..6fe43d84 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "a29cf3d28fc5257bdab859827271fd1e7bf444df88111b7e0f19731b1cd6c956" + "sha256": "d697cd7c279a761283ea2ffefab08d5854e4c0341b5646839b269ab0d999bf87" }, "pipfile-spec": 6, "requires": { @@ -45,13 +45,6 @@ ], "version": "==1.1.23" }, - "azure-core": { - "hashes": [ - "sha256:c49ab53efcebb40f35f0d0e524670057e7ad003ebd230fbc111335892a863e72", - "sha256:d4b2e5448125e753315e5b3fd9a18e49dcd7ac0fbfbbc0abe577043eb590aa7f" - ], - "version": "==1.0.0b1" - }, "azure-nspkg": { "hashes": [ "sha256:1d0bbb2157cf57b1bef6c8c8e5b41133957364456c43b0a43599890023cca0a8", @@ -70,34 +63,26 @@ }, "azure-storage-common": { "hashes": [ - "sha256:1fce4505880c345c83c06b2ae449e403be46dca039b9ef16d122fb1e4b2741f5", - "sha256:4390cd5f6fb50fbff37ba154258ea010291a84f128917141e1d7060597bd4708" + "sha256:b01a491a18839b9d05a4fe3421458a0ddb5ab9443c14e487f40d16f9a1dc2fbe", + "sha256:ccedef5c67227bc4d6670ffd37cec18fb529a1b7c3a5e53e4096eb0cf23dc73f" ], "index": "pypi", - "version": "==2.0.0" - }, - "azure-storage-file": { - "hashes": [ - "sha256:d938d71459b108740d896220c0ed17530ca83fd0f82781a044b80ed7e03bd3be", - "sha256:e996a228cd632f6f586228381eaa2823b52884899dce2259876e5eb78f0eab6f" - ], - "index": "pypi", - "version": "==12.0.0b1" + "version": "==2.1.0" }, "boto3": { "hashes": [ - "sha256:0cd4a3e158f40eedb54b36b3fbe60d135db74a245f0ca8eead1af2eb6d46a649", - "sha256:68e9eba6f846cf8e01973ec565afdb1adfb9612b531c15bb5c5524394db4df5b" + "sha256:666f37c5852f71925494fc2103b189deafe6702c1d9ae60bead5b1b6466de857", + "sha256:7b77b507221ec15550b02d492804166bcc61ef3a81312968065515a76aa1791b" ], "index": "pypi", - "version": "==1.9.199" + "version": "==1.9.202" }, "botocore": { "hashes": [ - "sha256:25d87047241b7b775443570c0e790ca952f9f7491d4d6472430a4b006383a257", - "sha256:e4729c1acaa936d4c5c948a18d279f92bbf61fad9b5fb03942c753ec405e427d" + "sha256:71ca578701e746fe947c098e5dee06128d0f6ba98217ba7e29aff0dab8caf82f", + "sha256:e55003c46e71396a551d4b70f39286f8fc4094ac6cf90f5db8d7a68bb4af1f9d" ], - "version": "==1.12.199" + "version": "==1.12.202" }, "certifi": { "hashes": [ @@ -243,13 +228,6 @@ ], "version": "==2.8" }, - "isodate": { - "hashes": [ - "sha256:2e364a3d5759479cdb2d37cce6b9376ea504db2ff90252a2e5b7cc89cc9ff2d8", - "sha256:aa4d33c06640f5352aca96e4b81afd8ab3b47337cc12089822d6f322ac772c81" - ], - "version": "==0.6.0" - }, "itsdangerous": { "hashes": [ "sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19", @@ -281,9 +259,9 @@ }, "mako": { "hashes": [ - "sha256:f5a642d8c5699269ab62a68b296ff990767eb120f51e2e8f3d6afb16bdb57f4b" + "sha256:a36919599a9b7dc5d86a7a8988f23a9a3a3d083070023bab23d64f7f1d1e0a4b" ], - "version": "==1.0.14" + "version": "==1.1.0" }, "markupsafe": { "hashes": [ @@ -318,20 +296,6 @@ ], "version": "==1.1.1" }, - "msrest": { - "hashes": [ - "sha256:27589fb400da7e1a98778688f70a0099e4fc6fea59d0f4835b4fbdad3bb8a6d9", - "sha256:cda706a2ccfb032cf41fa8cc6575cbca29634fed2d226fc789e4a8daf44ab7c1" - ], - "version": "==0.6.9" - }, - "oauthlib": { - "hashes": [ - "sha256:40a63637707e9163eda62d0f5345120c65e001a790480b8256448543c1f78f66", - "sha256:b4d99ae8ccfb7d33ba9591b59355c64eef5241534aa3da2e4c0435346b84bc8e" - ], - "version": "==3.0.2" - }, "pendulum": { "hashes": [ "sha256:1cde6e3c6310fb882c98f373795f807cb2bd6af01f34d2857e6e283b5ee91e09", @@ -436,11 +400,11 @@ }, "redis": { "hashes": [ - "sha256:1f2493b72f669a096c59ca7cf7f4067b1ab58a0a3c6bef51d5d8fd384298c6a2", - "sha256:b851b0ef53b6d416f1dc692dc168f3d390b37995925bfe29351b3a869976598c" + "sha256:45682ecf226c7611efe731974c4fa3390170ba045b9cdb26f0051114a5c2a68b", + "sha256:f2609a85e5f37f489ba3b5652e1175dc3711c4d7a7818c4f657615810afd23df" ], "index": "pypi", - "version": "==3.3.4" + "version": "==3.3.6" }, "requests": { "hashes": [ @@ -450,13 +414,6 @@ "index": "pypi", "version": "==2.22.0" }, - "requests-oauthlib": { - "hashes": [ - "sha256:bd6533330e8748e94bf0b214775fed487d309b8b8fe823dc45641ebcd9a32f57", - "sha256:d3ed0c8f2e3bbc6b344fa63d6f933745ab394469da38db16bdddb461c7e25140" - ], - "version": "==1.2.0" - }, "rq": { "hashes": [ "sha256:2798d26a7b850e759f23f69695a389d676a9c08f2c14f96f0d34d9648c9d5616", @@ -851,10 +808,10 @@ }, "pbr": { "hashes": [ - "sha256:0ca44dc9fd3b04a22297c2a91082d8df2894862e8f4c86a49dac69eae9e85ca0", - "sha256:4aed6c1b1fa5020def0f22aed663d87b81bb3235f112490b07d2643d7a98c5b5" + "sha256:56e52299170b9492513c64be44736d27a512fa7e606f21942160b68ce510b4bc", + "sha256:9b321c204a88d8ab5082699469f52cc94c5da45c51f114113d01b3d993c24cdf" ], - "version": "==5.4.1" + "version": "==5.4.2" }, "pexpect": { "hashes": [ From f1560d5d7f543c333be30aee32368be6a0d2f68c Mon Sep 17 00:00:00 2001 From: richard-dds Date: Tue, 6 Aug 2019 16:58:57 -0400 Subject: [PATCH 19/20] Import CSP deps in uploader implementations --- atst/domain/csp/file_uploads.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/atst/domain/csp/file_uploads.py b/atst/domain/csp/file_uploads.py index b967e994..d6bb6dd5 100644 --- a/atst/domain/csp/file_uploads.py +++ b/atst/domain/csp/file_uploads.py @@ -1,10 +1,6 @@ -from azure.storage.common import CloudStorageAccount -from azure.storage.blob import BlobPermissions from datetime import datetime, timedelta from uuid import uuid4 -import boto3 - def build_uploader(config): csp = config.get("CSP") @@ -39,6 +35,11 @@ class AzureUploader(Uploader): self.container_name = config["AZURE_TO_BUCKET_NAME"] self.timeout = timedelta(seconds=config["PERMANENT_SESSION_LIFETIME"]) + from azure.storage.common import CloudStorageAccount + from azure.storage.blob import BlobPermissions + self.CloudStorageAccount = CloudStorageAccount + self.BlobPermissions = BlobPermissions + def get_token(self): """ Generates an Azure SAS token for pre-authorizing a file upload. @@ -47,7 +48,7 @@ class AzureUploader(Uploader): - token_dict has a `token` key which contains the SAS token as a string - object_name is a string """ - account = CloudStorageAccount( + account = self.CloudStorageAccount( account_name=self.account_name, account_key=self.storage_key ) bbs = account.create_block_blob_service() @@ -55,7 +56,7 @@ class AzureUploader(Uploader): sas_token = bbs.generate_blob_shared_access_signature( self.container_name, object_name, - permission=BlobPermissions.CREATE, + permission=self.BlobPermissions.CREATE, expiry=datetime.utcnow() + self.timeout, protocol="https", ) @@ -70,6 +71,9 @@ class AwsUploader(Uploader): self.bucket_name = config["AWS_BUCKET_NAME"] self.timeout_secs = config["PERMANENT_SESSION_LIFETIME"] + import boto3 + self.boto3 = boto3 + def get_token(self): """ Generates an AWS presigned post for pre-authorizing a file upload. @@ -79,7 +83,7 @@ class AwsUploader(Uploader): form before being sent to S3 - object_name is a string """ - s3_client = boto3.client( + s3_client = self.boto3.client( "s3", aws_access_key_id=self.access_key_id, aws_secret_access_key=self.secret_key, From 0aaa3d542e62a6fc0606b724aeb0a284b788a292 Mon Sep 17 00:00:00 2001 From: richard-dds Date: Tue, 6 Aug 2019 17:17:38 -0400 Subject: [PATCH 20/20] Formatting --- atst/domain/csp/file_uploads.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/atst/domain/csp/file_uploads.py b/atst/domain/csp/file_uploads.py index d6bb6dd5..e1c1cb9d 100644 --- a/atst/domain/csp/file_uploads.py +++ b/atst/domain/csp/file_uploads.py @@ -37,6 +37,7 @@ class AzureUploader(Uploader): from azure.storage.common import CloudStorageAccount from azure.storage.blob import BlobPermissions + self.CloudStorageAccount = CloudStorageAccount self.BlobPermissions = BlobPermissions @@ -72,6 +73,7 @@ class AwsUploader(Uploader): self.timeout_secs = config["PERMANENT_SESSION_LIFETIME"] import boto3 + self.boto3 = boto3 def get_token(self):