diff --git a/.gitignore b/.gitignore index 7f2755cc..27a7244d 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,6 @@ config/dev.ini /crl /crl-tmp *.bk + +# uploads +/uploads diff --git a/Pipfile b/Pipfile index fef290df..d9cb9335 100644 --- a/Pipfile +++ b/Pipfile @@ -18,6 +18,8 @@ flask-session = "*" flask-wtf = "*" pyopenssl = "*" requests = "*" +apache-libcloud = "*" +lockfile = "*" [dev-packages] bandit = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 8a19a1a2..006dae19 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "41ad134816dae388385cfb15105e0eca436b25791ec4fbf67a2b36c4ae8056bd" + "sha256": "552b7ac6943559a1fc3be1c4e1c91f965cbfb97c115566051950450c7cd6f78b" }, "pipfile-spec": 6, "requires": { @@ -24,6 +24,14 @@ "index": "pypi", "version": "==1.0.0" }, + "apache-libcloud": { + "hashes": [ + "sha256:0e2eee3802163bd0605975ed1e284cafc23203919bfa80c0cc5d3cd2543aaf97", + "sha256:48d5d64790a5112cace1a8e28d228c3f1c5bd3ddbd986a5453172d2da19f47d5" + ], + "index": "pypi", + "version": "==2.3.0" + }, "asn1crypto": { "hashes": [ "sha256:2f1adbb7546ed199e3c90ef23ec95c5cf3585bac7d11fb7eb562a3fe89c64e87", @@ -33,10 +41,10 @@ }, "certifi": { "hashes": [ - "sha256:4c1d68a1408dd090d2f3a869aa94c3947cc1d967821d1ed303208c9f41f0f2f4", - "sha256:b6e8b28b2b7e771a41ecdd12d4d43262ecab52adebbafa42c77d6b57fb6ad3a4" + "sha256:376690d6f16d32f9d1fe8932551d80b23e9d393a8578c5633a2ed39a64861638", + "sha256:456048c7e371c089d0a77a5212fb37a2c2dce1e24146e3b7e0261736aaeaa22a" ], - "version": "==2018.8.13" + "version": "==2018.8.24" }, "cffi": { "hashes": [ @@ -172,6 +180,14 @@ ], "version": "==2.10" }, + "lockfile": { + "hashes": [ + "sha256:6aed02de03cba24efabcd600b30540140634fc06cfa603822d508d5361e9f799", + "sha256:6c3cb24f344923d30b2785d5ad75182c8ea7ac1b6171b08657258ec7429d50fa" + ], + "index": "pypi", + "version": "==0.12.2" + }, "mako": { "hashes": [ "sha256:4e02fde57bd4abb5ec400181e4c314f56ac3e49ba4fb8b0d50bba18cb27d25ae" @@ -271,7 +287,6 @@ "sha256:1d936da41ee06216d89fdc7ead1ee9a5da2811a8787515a976b646e110c3f622", "sha256:e4ef42e82b0b493c5849eed98b5ab49d6767caf982127e9a33167f1153b36cc5" ], - "markers": "python_version != '3.0.*' and python_version != '3.2.*' and python_version >= '2.7' and python_version != '3.1.*' and python_version != '3.3.*'", "version": "==2018.5" }, "redis": { @@ -317,7 +332,6 @@ "sha256:a68ac5e15e76e7e5dd2b8f94007233e01effe3e50e8daddf69acfd81cb686baf", "sha256:b5725a0bd4ba422ab0e66e89e030c806576753ea3ee08554382c14e685d117b5" ], - "markers": "python_version >= '2.6' and python_version != '3.2.*' and python_version != '3.0.*' and python_version < '4' and python_version != '3.1.*' and python_version != '3.3.*'", "version": "==1.23" }, "webassets": { @@ -350,6 +364,14 @@ ], "version": "==1.4.3" }, + "appnope": { + "hashes": [ + "sha256:5b26757dc6f79a3b7dc9fab95359328d5747fcb2409d331ea66d0272b90ab2a0", + "sha256:8b995ffe925347a2138d7ac0fe77155e4311a0ea6d6da4f5128fe4b3cbe5ed71" + ], + "markers": "sys_platform == 'darwin'", + "version": "==0.1.0" + }, "argh": { "hashes": [ "sha256:a9b3aaa1904eeb78e32394cd46c6f37ac0fb4af6dc488daa58971bdc7d7fcaf3", @@ -441,7 +463,6 @@ "sha256:ea7cfd3aeb1544732d08bd9cfba40c5b78e3a91e17b1a0698ab81bfc5554c628", "sha256:f6d67f04abfb2b4bea7afc7fa6c18cf4c523a67956e455668be9ae42bccc21ad" ], - "markers": "python_version != '3.2.*' and python_version != '3.0.*' and python_version != '3.1.*' and python_version >= '2.7'", "version": "==0.9.0" }, "flask": { @@ -494,7 +515,6 @@ "sha256:b9c40e9750f3d77e6e4d441d8b0266cf555e7cdabdcff33c4fd06366ca761ef8", "sha256:ec9ef8f4a9bc6f71eec99e1806bfa2de401650d996c59330782b89a5555c1497" ], - "markers": "python_version >= '2.7' and python_version != '3.2.*' and python_version != '3.1.*' and python_version != '3.0.*' and python_version != '3.3.*'", "version": "==4.3.4" }, "itsdangerous": { @@ -612,7 +632,6 @@ "sha256:6e3836e39f4d36ae72840833db137f7b7d35105079aee6ec4a62d9f80d594dd1", "sha256:95eb8364a4708392bae89035f45341871286a333f749c3141c20573d2b3876e1" ], - "markers": "python_version >= '2.7' and python_version != '3.2.*' and python_version != '3.1.*' and python_version != '3.0.*' and python_version != '3.3.*'", "version": "==0.7.1" }, "prompt-toolkit": { @@ -635,7 +654,6 @@ "sha256:3fd59af7435864e1a243790d322d763925431213b6b8529c6ca71081ace3bbf7", "sha256:e31fb2767eb657cbde86c454f02e99cb846d3cd9d61b318525140214fdc0e98e" ], - "markers": "python_version >= '2.7' and python_version != '3.2.*' and python_version != '3.1.*' and python_version != '3.0.*' and python_version != '3.3.*'", "version": "==1.5.4" }, "pygments": { @@ -692,11 +710,15 @@ }, "pyyaml": { "hashes": [ + "sha256:1cbc199009e78f92d9edf554be4fe40fb7b0bef71ba688602a00e97a51909110", "sha256:254bf6fda2b7c651837acb2c718e213df29d531eebf00edb54743d10bcb694eb", "sha256:3108529b78577327d15eec243f0ff348a0640b0c3478d67ad7f5648f93bac3e2", "sha256:3c17fb92c8ba2f525e4b5f7941d850e7a48c3a59b32d331e2502a3cdc6648e76", + "sha256:6f89b5c95e93945b597776163403d47af72d243f366bf4622ff08bdfd1c950b7", "sha256:8d6d96001aa7f0a6a4a95e8143225b5d06e41b1131044913fecb8f85a125714b", - "sha256:c8a88edd93ee29ede719080b2be6cb2333dfee1dccba213b422a9c8e97f2967b" + "sha256:be622cc81696e24d0836ba71f6272a2b5767669b0d79fdcf0295d51ac2e156c8", + "sha256:c8a88edd93ee29ede719080b2be6cb2333dfee1dccba213b422a9c8e97f2967b", + "sha256:f39411e380e2182ad33be039e8ee5770a5d9efe01a2bfb7ae58d9ba31c4a2a9d" ], "version": "==4.2b4" }, @@ -747,6 +769,35 @@ ], "version": "==4.3.2" }, + "typed-ast": { + "hashes": [ + "sha256:0948004fa228ae071054f5208840a1e88747a357ec1101c17217bfe99b299d58", + "sha256:10703d3cec8dcd9eef5a630a04056bbc898abc19bac5691612acba7d1325b66d", + "sha256:1f6c4bd0bdc0f14246fd41262df7dfc018d65bb05f6e16390b7ea26ca454a291", + "sha256:25d8feefe27eb0303b73545416b13d108c6067b846b543738a25ff304824ed9a", + "sha256:29464a177d56e4e055b5f7b629935af7f49c196be47528cc94e0a7bf83fbc2b9", + "sha256:2e214b72168ea0275efd6c884b114ab42e316de3ffa125b267e732ed2abda892", + "sha256:3e0d5e48e3a23e9a4d1a9f698e32a542a4a288c871d33ed8df1b092a40f3a0f9", + "sha256:519425deca5c2b2bdac49f77b2c5625781abbaf9a809d727d3a5596b30bb4ded", + "sha256:57fe287f0cdd9ceaf69e7b71a2e94a24b5d268b35df251a88fef5cc241bf73aa", + "sha256:668d0cec391d9aed1c6a388b0d5b97cd22e6073eaa5fbaa6d2946603b4871efe", + "sha256:68ba70684990f59497680ff90d18e756a47bf4863c604098f10de9716b2c0bdd", + "sha256:6de012d2b166fe7a4cdf505eee3aaa12192f7ba365beeefaca4ec10e31241a85", + "sha256:79b91ebe5a28d349b6d0d323023350133e927b4de5b651a8aa2db69c761420c6", + "sha256:8550177fa5d4c1f09b5e5f524411c44633c80ec69b24e0e98906dd761941ca46", + "sha256:898f818399cafcdb93cbbe15fc83a33d05f18e29fb498ddc09b0214cdfc7cd51", + "sha256:94b091dc0f19291adcb279a108f5d38de2430411068b219f41b343c03b28fb1f", + "sha256:a26863198902cda15ab4503991e8cf1ca874219e0118cbf07c126bce7c4db129", + "sha256:a8034021801bc0440f2e027c354b4eafd95891b573e12ff0418dec385c76785c", + "sha256:bc978ac17468fe868ee589c795d06777f75496b1ed576d308002c8a5756fb9ea", + "sha256:c05b41bc1deade9f90ddc5d988fe506208019ebba9f2578c622516fd201f5863", + "sha256:c9b060bd1e5a26ab6e8267fd46fc9e02b54eb15fffb16d112d4c7b1c12987559", + "sha256:edb04bdd45bfd76c8292c4d9654568efaedf76fe78eb246dde69bdb13b2dad87", + "sha256:f19f2a4f547505fe9072e15f6f4ae714af51b5a681a97f187971f50c283193b6" + ], + "markers": "python_version < '3.7' and implementation_name == 'cpython'", + "version": "==1.1.0" + }, "watchdog": { "hashes": [ "sha256:7e65882adb7746039b6f3876ee174952f8eaaa34491ba34333ddf1fe35de4162" diff --git a/atst/app.py b/atst/app.py index 153bdd42..de5c7baf 100644 --- a/atst/app.py +++ b/atst/app.py @@ -19,6 +19,7 @@ from atst.routes.errors import make_error_pages from atst.domain.authnid.crl import CRLCache from atst.domain.auth import apply_authentication from atst.eda_client import MockEDAClient +from atst.uploader import Uploader ENV = os.getenv("FLASK_ENV", "dev") @@ -43,6 +44,7 @@ def make_app(config): make_crl_validator(app) register_filters(app) make_eda_client(app) + make_upload_storage(app) db.init_app(app) csrf.init_app(app) @@ -143,3 +145,12 @@ def make_crl_validator(app): def make_eda_client(app): app.eda_client = MockEDAClient() + +def make_upload_storage(app): + uploader = Uploader( + provider=app.config.get("STORAGE_PROVIDER"), + container=app.config.get("STORAGE_CONTAINER"), + key=app.config.get("STORAGE_KEY"), + secret=app.config.get("STORAGE_SECRET") + ) + app.uploader = uploader diff --git a/atst/forms/financial.py b/atst/forms/financial.py index 6b231306..68bdce1a 100644 --- a/atst/forms/financial.py +++ b/atst/forms/financial.py @@ -1,7 +1,8 @@ import re from wtforms.fields.html5 import EmailField -from wtforms.fields import StringField +from wtforms.fields import StringField, FileField from wtforms.validators import Required, Email, Regexp +from flask_wtf.file import FileAllowed from atst.domain.exceptions import NotFoundError from atst.domain.pe_numbers import PENumbers @@ -214,3 +215,5 @@ class ExtendedFinancialForm(BaseFinancialForm): description="Review your task order document, the amounts for each CLIN must match exactly here", filters=[number_to_int], ) + + task_order = FileField("Upload a copy of your Task Order", validators=[FileAllowed(['pdf'], 'Only PDF documents can be uploaded.')]) diff --git a/atst/uploader.py b/atst/uploader.py new file mode 100644 index 00000000..9ec55287 --- /dev/null +++ b/atst/uploader.py @@ -0,0 +1,45 @@ +from uuid import uuid4 +from libcloud.storage.types import Provider +from libcloud.storage.providers import get_driver + + +class UploadError(Exception): + pass + + +class Uploader(): + _PERMITTED_MIMETYPES = ["application/pdf"] + + def __init__(self, provider, container=None, key=None, secret=None): + self.container = self._get_container(provider, container, key, secret) + + def upload(self, fyle): + # TODO: for hardening, we should probably use a better library for + # determining mimetype and not rely on FileUpload's determination + # TODO: we should set MAX_CONTENT_LENGTH in the config to prevent large + # uploads + if not fyle.mimetype in self._PERMITTED_MIMETYPES: + raise UploadError( + "could not upload {} with mimetype {}".format( + fyle.filename, fyle.mimetype + ) + ) + + object_name = uuid4().hex + self.container.upload_object_via_stream( + iterator=fyle.stream.__iter__(), + object_name=object_name, + extra={"acl": "private"}, + ) + return (fyle.filename, object_name) + + def download(self, path): + pass + + def _get_container(self, provider, container, key, secret): + if provider == "LOCAL": + key = container + container = "" + + driver = get_driver(getattr(Provider, provider))(key=key, secret=secret) + return driver.get_container(container) diff --git a/config/base.ini b/config/base.ini index 749b9b2b..5fa7ec6e 100644 --- a/config/base.ini +++ b/config/base.ini @@ -20,4 +20,6 @@ SECRET_KEY = change_me_into_something_secret SESSION_COOKIE_NAME=atat SESSION_TYPE = redis SESSION_USE_SIGNER = True +STORAGE_CONTAINER=uploads +STORAGE_PROVIDER=LOCAL WTF_CSRF_ENABLED = true diff --git a/script/test b/script/test index d1e22bbf..f5dbea41 100755 --- a/script/test +++ b/script/test @@ -6,6 +6,9 @@ source "$(dirname "${0}")"/../script/include/global_header.inc.sh export FLASK_ENV=test +# create upload directory for app +mkdir uploads | true + # Enable database resetting RESET_DB="true" diff --git a/templates/requests/financial_verification.html b/templates/requests/financial_verification.html index 2e7e094e..c3136541 100644 --- a/templates/requests/financial_verification.html +++ b/templates/requests/financial_verification.html @@ -94,6 +94,9 @@ f.clin_2003,placeholder="7,000", validation='integer' ) }} + + {{ f.task_order.label }} + {{ f.task_order(multiple="") }} {% endif %} diff --git a/tests/fixtures/sample.pdf b/tests/fixtures/sample.pdf new file mode 100644 index 00000000..c0e31a07 --- /dev/null +++ b/tests/fixtures/sample.pdf @@ -0,0 +1,198 @@ +%PDF-1.3 +%âãÏÓ + +1 0 obj +<< +/Type /Catalog +/Outlines 2 0 R +/Pages 3 0 R +>> +endobj + +2 0 obj +<< +/Type /Outlines +/Count 0 +>> +endobj + +3 0 obj +<< +/Type /Pages +/Count 2 +/Kids [ 4 0 R 6 0 R ] +>> +endobj + +4 0 obj +<< +/Type /Page +/Parent 3 0 R +/Resources << +/Font << +/F1 9 0 R +>> +/ProcSet 8 0 R +>> +/MediaBox [0 0 612.0000 792.0000] +/Contents 5 0 R +>> +endobj + +5 0 obj +<< /Length 1074 >> +stream +2 J +BT +0 0 0 rg +/F1 0027 Tf +57.3750 722.2800 Td +( A Simple PDF File ) Tj +ET +BT +/F1 0010 Tf +69.2500 688.6080 Td +( This is a small demonstration .pdf file - ) Tj +ET +BT +/F1 0010 Tf +69.2500 664.7040 Td +( just for use in the Virtual Mechanics tutorials. More text. And more ) Tj +ET +BT +/F1 0010 Tf +69.2500 652.7520 Td +( text. And more text. And more text. And more text. ) Tj +ET +BT +/F1 0010 Tf +69.2500 628.8480 Td +( And more text. And more text. And more text. And more text. And more ) Tj +ET +BT +/F1 0010 Tf +69.2500 616.8960 Td +( text. And more text. Boring, zzzzz. And more text. And more text. And ) Tj +ET +BT +/F1 0010 Tf +69.2500 604.9440 Td +( more text. And more text. And more text. And more text. And more text. ) Tj +ET +BT +/F1 0010 Tf +69.2500 592.9920 Td +( And more text. And more text. ) Tj +ET +BT +/F1 0010 Tf +69.2500 569.0880 Td +( And more text. And more text. And more text. And more text. And more ) Tj +ET +BT +/F1 0010 Tf +69.2500 557.1360 Td +( text. And more text. And more text. Even more. Continued on page 2 ...) Tj +ET +endstream +endobj + +6 0 obj +<< +/Type /Page +/Parent 3 0 R +/Resources << +/Font << +/F1 9 0 R +>> +/ProcSet 8 0 R +>> +/MediaBox [0 0 612.0000 792.0000] +/Contents 7 0 R +>> +endobj + +7 0 obj +<< /Length 676 >> +stream +2 J +BT +0 0 0 rg +/F1 0027 Tf +57.3750 722.2800 Td +( Simple PDF File 2 ) Tj +ET +BT +/F1 0010 Tf +69.2500 688.6080 Td +( ...continued from page 1. Yet more text. And more text. And more text. ) Tj +ET +BT +/F1 0010 Tf +69.2500 676.6560 Td +( And more text. And more text. And more text. And more text. And more ) Tj +ET +BT +/F1 0010 Tf +69.2500 664.7040 Td +( text. Oh, how boring typing this stuff. But not as boring as watching ) Tj +ET +BT +/F1 0010 Tf +69.2500 652.7520 Td +( paint dry. And more text. And more text. And more text. And more text. ) Tj +ET +BT +/F1 0010 Tf +69.2500 640.8000 Td +( Boring. More, a little more text. The end, and just as well. ) Tj +ET +endstream +endobj + +8 0 obj +[/PDF /Text] +endobj + +9 0 obj +<< +/Type /Font +/Subtype /Type1 +/Name /F1 +/BaseFont /Helvetica +/Encoding /WinAnsiEncoding +>> +endobj + +10 0 obj +<< +/Creator (Rave \(http://www.nevrona.com/rave\)) +/Producer (Nevrona Designs) +/CreationDate (D:20060301072826) +>> +endobj + +xref +0 11 +0000000000 65535 f +0000000019 00000 n +0000000093 00000 n +0000000147 00000 n +0000000222 00000 n +0000000390 00000 n +0000001522 00000 n +0000001690 00000 n +0000002423 00000 n +0000002456 00000 n +0000002574 00000 n + +trailer +<< +/Size 11 +/Root 1 0 R +/Info 10 0 R +>> + +startxref +2714 +%%EOF diff --git a/tests/test_uploader.py b/tests/test_uploader.py new file mode 100644 index 00000000..799cefb3 --- /dev/null +++ b/tests/test_uploader.py @@ -0,0 +1,38 @@ +import os +import pytest +from werkzeug.datastructures import FileStorage + +from atst.uploader import Uploader, UploadError + + +@pytest.fixture(scope="function") +def upload_dir(tmpdir): + return tmpdir.mkdir("uploads") + + +@pytest.fixture +def uploader(upload_dir): + return Uploader("LOCAL", container=upload_dir) + + +PDF_FILENAME = "tests/fixtures/sample.pdf" +NONPDF_FILENAME = "tests/fixtures/disa-pki.html" + +@pytest.fixture +def pdf(): + with open(PDF_FILENAME, "rb") as fp: + yield FileStorage(fp, content_type="application/pdf") + + +def test_upload(uploader, upload_dir, pdf): + filename, object_name = uploader.upload(pdf) + assert filename == PDF_FILENAME + assert os.path.isfile(os.path.join(upload_dir, object_name)) + + +def test_upload_fails_for_non_pdfs(uploader, pdf): + with open(NONPDF_FILENAME, "rb") as fp: + fs = FileStorage(fp, content_type="text/plain") + with pytest.raises(UploadError): + uploader.upload(fs) +