initial uploader and some form work

.gitignore

@@ -35,3 +35,6 @@ config/dev.ini
 /crl
 /crl-tmp
 *.bk
+
+# uploads
+/uploads

Pipfile

@@ -18,6 +18,8 @@ flask-session = "*"
 flask-wtf = "*"
 pyopenssl = "*"
 requests = "*"
+apache-libcloud = "*"
+lockfile = "*"
 
 [dev-packages]
 bandit = "*"

Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "41ad134816dae388385cfb15105e0eca436b25791ec4fbf67a2b36c4ae8056bd"
+            "sha256": "552b7ac6943559a1fc3be1c4e1c91f965cbfb97c115566051950450c7cd6f78b"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -24,6 +24,14 @@
             "index": "pypi",
             "version": "==1.0.0"
         },
+        "apache-libcloud": {
+            "hashes": [
+                "sha256:0e2eee3802163bd0605975ed1e284cafc23203919bfa80c0cc5d3cd2543aaf97",
+                "sha256:48d5d64790a5112cace1a8e28d228c3f1c5bd3ddbd986a5453172d2da19f47d5"
+            ],
+            "index": "pypi",
+            "version": "==2.3.0"
+        },
         "asn1crypto": {
             "hashes": [
                 "sha256:2f1adbb7546ed199e3c90ef23ec95c5cf3585bac7d11fb7eb562a3fe89c64e87",
@@ -33,10 +41,10 @@
         },
         "certifi": {
             "hashes": [
-                "sha256:4c1d68a1408dd090d2f3a869aa94c3947cc1d967821d1ed303208c9f41f0f2f4",
+                "sha256:376690d6f16d32f9d1fe8932551d80b23e9d393a8578c5633a2ed39a64861638",
-                "sha256:b6e8b28b2b7e771a41ecdd12d4d43262ecab52adebbafa42c77d6b57fb6ad3a4"
+                "sha256:456048c7e371c089d0a77a5212fb37a2c2dce1e24146e3b7e0261736aaeaa22a"
             ],
-            "version": "==2018.8.13"
+            "version": "==2018.8.24"
         },
         "cffi": {
             "hashes": [
@@ -172,6 +180,14 @@
             ],
             "version": "==2.10"
         },
+        "lockfile": {
+            "hashes": [
+                "sha256:6aed02de03cba24efabcd600b30540140634fc06cfa603822d508d5361e9f799",
+                "sha256:6c3cb24f344923d30b2785d5ad75182c8ea7ac1b6171b08657258ec7429d50fa"
+            ],
+            "index": "pypi",
+            "version": "==0.12.2"
+        },
         "mako": {
             "hashes": [
                 "sha256:4e02fde57bd4abb5ec400181e4c314f56ac3e49ba4fb8b0d50bba18cb27d25ae"
@@ -271,7 +287,6 @@
                 "sha256:1d936da41ee06216d89fdc7ead1ee9a5da2811a8787515a976b646e110c3f622",
                 "sha256:e4ef42e82b0b493c5849eed98b5ab49d6767caf982127e9a33167f1153b36cc5"
             ],
-            "markers": "python_version != '3.0.*' and python_version != '3.2.*' and python_version >= '2.7' and python_version != '3.1.*' and python_version != '3.3.*'",
             "version": "==2018.5"
         },
         "redis": {
@@ -317,7 +332,6 @@
                 "sha256:a68ac5e15e76e7e5dd2b8f94007233e01effe3e50e8daddf69acfd81cb686baf",
                 "sha256:b5725a0bd4ba422ab0e66e89e030c806576753ea3ee08554382c14e685d117b5"
             ],
-            "markers": "python_version >= '2.6' and python_version != '3.2.*' and python_version != '3.0.*' and python_version < '4' and python_version != '3.1.*' and python_version != '3.3.*'",
             "version": "==1.23"
         },
         "webassets": {
@@ -350,6 +364,14 @@
             ],
             "version": "==1.4.3"
         },
+        "appnope": {
+            "hashes": [
+                "sha256:5b26757dc6f79a3b7dc9fab95359328d5747fcb2409d331ea66d0272b90ab2a0",
+                "sha256:8b995ffe925347a2138d7ac0fe77155e4311a0ea6d6da4f5128fe4b3cbe5ed71"
+            ],
+            "markers": "sys_platform == 'darwin'",
+            "version": "==0.1.0"
+        },
         "argh": {
             "hashes": [
                 "sha256:a9b3aaa1904eeb78e32394cd46c6f37ac0fb4af6dc488daa58971bdc7d7fcaf3",
@@ -441,7 +463,6 @@
                 "sha256:ea7cfd3aeb1544732d08bd9cfba40c5b78e3a91e17b1a0698ab81bfc5554c628",
                 "sha256:f6d67f04abfb2b4bea7afc7fa6c18cf4c523a67956e455668be9ae42bccc21ad"
             ],
-            "markers": "python_version != '3.2.*' and python_version != '3.0.*' and python_version != '3.1.*' and python_version >= '2.7'",
             "version": "==0.9.0"
         },
         "flask": {
@@ -494,7 +515,6 @@
                 "sha256:b9c40e9750f3d77e6e4d441d8b0266cf555e7cdabdcff33c4fd06366ca761ef8",
                 "sha256:ec9ef8f4a9bc6f71eec99e1806bfa2de401650d996c59330782b89a5555c1497"
             ],
-            "markers": "python_version >= '2.7' and python_version != '3.2.*' and python_version != '3.1.*' and python_version != '3.0.*' and python_version != '3.3.*'",
             "version": "==4.3.4"
         },
         "itsdangerous": {
@@ -612,7 +632,6 @@
                 "sha256:6e3836e39f4d36ae72840833db137f7b7d35105079aee6ec4a62d9f80d594dd1",
                 "sha256:95eb8364a4708392bae89035f45341871286a333f749c3141c20573d2b3876e1"
             ],
-            "markers": "python_version >= '2.7' and python_version != '3.2.*' and python_version != '3.1.*' and python_version != '3.0.*' and python_version != '3.3.*'",
             "version": "==0.7.1"
         },
         "prompt-toolkit": {
@@ -635,7 +654,6 @@
                 "sha256:3fd59af7435864e1a243790d322d763925431213b6b8529c6ca71081ace3bbf7",
                 "sha256:e31fb2767eb657cbde86c454f02e99cb846d3cd9d61b318525140214fdc0e98e"
             ],
-            "markers": "python_version >= '2.7' and python_version != '3.2.*' and python_version != '3.1.*' and python_version != '3.0.*' and python_version != '3.3.*'",
             "version": "==1.5.4"
         },
         "pygments": {
@@ -692,11 +710,15 @@
         },
         "pyyaml": {
             "hashes": [
+                "sha256:1cbc199009e78f92d9edf554be4fe40fb7b0bef71ba688602a00e97a51909110",
                 "sha256:254bf6fda2b7c651837acb2c718e213df29d531eebf00edb54743d10bcb694eb",
                 "sha256:3108529b78577327d15eec243f0ff348a0640b0c3478d67ad7f5648f93bac3e2",
                 "sha256:3c17fb92c8ba2f525e4b5f7941d850e7a48c3a59b32d331e2502a3cdc6648e76",
+                "sha256:6f89b5c95e93945b597776163403d47af72d243f366bf4622ff08bdfd1c950b7",
                 "sha256:8d6d96001aa7f0a6a4a95e8143225b5d06e41b1131044913fecb8f85a125714b",
-                "sha256:c8a88edd93ee29ede719080b2be6cb2333dfee1dccba213b422a9c8e97f2967b"
+                "sha256:be622cc81696e24d0836ba71f6272a2b5767669b0d79fdcf0295d51ac2e156c8",
+                "sha256:c8a88edd93ee29ede719080b2be6cb2333dfee1dccba213b422a9c8e97f2967b",
+                "sha256:f39411e380e2182ad33be039e8ee5770a5d9efe01a2bfb7ae58d9ba31c4a2a9d"
             ],
             "version": "==4.2b4"
         },
@@ -747,6 +769,35 @@
             ],
             "version": "==4.3.2"
         },
+        "typed-ast": {
+            "hashes": [
+                "sha256:0948004fa228ae071054f5208840a1e88747a357ec1101c17217bfe99b299d58",
+                "sha256:10703d3cec8dcd9eef5a630a04056bbc898abc19bac5691612acba7d1325b66d",
+                "sha256:1f6c4bd0bdc0f14246fd41262df7dfc018d65bb05f6e16390b7ea26ca454a291",
+                "sha256:25d8feefe27eb0303b73545416b13d108c6067b846b543738a25ff304824ed9a",
+                "sha256:29464a177d56e4e055b5f7b629935af7f49c196be47528cc94e0a7bf83fbc2b9",
+                "sha256:2e214b72168ea0275efd6c884b114ab42e316de3ffa125b267e732ed2abda892",
+                "sha256:3e0d5e48e3a23e9a4d1a9f698e32a542a4a288c871d33ed8df1b092a40f3a0f9",
+                "sha256:519425deca5c2b2bdac49f77b2c5625781abbaf9a809d727d3a5596b30bb4ded",
+                "sha256:57fe287f0cdd9ceaf69e7b71a2e94a24b5d268b35df251a88fef5cc241bf73aa",
+                "sha256:668d0cec391d9aed1c6a388b0d5b97cd22e6073eaa5fbaa6d2946603b4871efe",
+                "sha256:68ba70684990f59497680ff90d18e756a47bf4863c604098f10de9716b2c0bdd",
+                "sha256:6de012d2b166fe7a4cdf505eee3aaa12192f7ba365beeefaca4ec10e31241a85",
+                "sha256:79b91ebe5a28d349b6d0d323023350133e927b4de5b651a8aa2db69c761420c6",
+                "sha256:8550177fa5d4c1f09b5e5f524411c44633c80ec69b24e0e98906dd761941ca46",
+                "sha256:898f818399cafcdb93cbbe15fc83a33d05f18e29fb498ddc09b0214cdfc7cd51",
+                "sha256:94b091dc0f19291adcb279a108f5d38de2430411068b219f41b343c03b28fb1f",
+                "sha256:a26863198902cda15ab4503991e8cf1ca874219e0118cbf07c126bce7c4db129",
+                "sha256:a8034021801bc0440f2e027c354b4eafd95891b573e12ff0418dec385c76785c",
+                "sha256:bc978ac17468fe868ee589c795d06777f75496b1ed576d308002c8a5756fb9ea",
+                "sha256:c05b41bc1deade9f90ddc5d988fe506208019ebba9f2578c622516fd201f5863",
+                "sha256:c9b060bd1e5a26ab6e8267fd46fc9e02b54eb15fffb16d112d4c7b1c12987559",
+                "sha256:edb04bdd45bfd76c8292c4d9654568efaedf76fe78eb246dde69bdb13b2dad87",
+                "sha256:f19f2a4f547505fe9072e15f6f4ae714af51b5a681a97f187971f50c283193b6"
+            ],
+            "markers": "python_version < '3.7' and implementation_name == 'cpython'",
+            "version": "==1.1.0"
+        },
         "watchdog": {
             "hashes": [
                 "sha256:7e65882adb7746039b6f3876ee174952f8eaaa34491ba34333ddf1fe35de4162"

atst/app.py

@@ -19,6 +19,7 @@ from atst.routes.errors import make_error_pages
 from atst.domain.authnid.crl import CRLCache
 from atst.domain.auth import apply_authentication
 from atst.eda_client import MockEDAClient
+from atst.uploader import Uploader
 
 
 ENV = os.getenv("FLASK_ENV", "dev")
@@ -43,6 +44,7 @@ def make_app(config):
     make_crl_validator(app)
     register_filters(app)
     make_eda_client(app)
+    make_upload_storage(app)
 
     db.init_app(app)
     csrf.init_app(app)
@@ -143,3 +145,12 @@ def make_crl_validator(app):
 
 def make_eda_client(app):
     app.eda_client = MockEDAClient()
+
+def make_upload_storage(app):
+    uploader = Uploader(
+            provider=app.config.get("STORAGE_PROVIDER"),
+            container=app.config.get("STORAGE_CONTAINER"),
+            key=app.config.get("STORAGE_KEY"),
+            secret=app.config.get("STORAGE_SECRET")
+            )
+    app.uploader = uploader

atst/forms/financial.py

@@ -1,7 +1,8 @@
 import re
 from wtforms.fields.html5 import EmailField
-from wtforms.fields import StringField
+from wtforms.fields import StringField, FileField
 from wtforms.validators import Required, Email, Regexp
+from flask_wtf.file import FileAllowed
 
 from atst.domain.exceptions import NotFoundError
 from atst.domain.pe_numbers import PENumbers
@@ -214,3 +215,5 @@ class ExtendedFinancialForm(BaseFinancialForm):
         description="Review your task order document, the amounts for each CLIN must match exactly here",
         filters=[number_to_int],
     )
+
+    task_order = FileField("Upload a copy of your Task Order", validators=[FileAllowed(['pdf'], 'Only PDF documents can be uploaded.')])

atst/uploader.py

@@ -0,0 +1,45 @@
+from uuid import uuid4
+from libcloud.storage.types import Provider
+from libcloud.storage.providers import get_driver
+
+
+class UploadError(Exception):
+    pass
+
+
+class Uploader():
+    _PERMITTED_MIMETYPES = ["application/pdf"]
+
+    def __init__(self, provider, container=None, key=None, secret=None):
+        self.container = self._get_container(provider, container, key, secret)
+
+    def upload(self, fyle):
+        # TODO: for hardening, we should probably use a better library for
+        # determining mimetype and not rely on FileUpload's determination
+        # TODO: we should set MAX_CONTENT_LENGTH in the config to prevent large
+        # uploads
+        if not fyle.mimetype in self._PERMITTED_MIMETYPES:
+            raise UploadError(
+                "could not upload {} with mimetype {}".format(
+                    fyle.filename, fyle.mimetype
+                )
+            )
+
+        object_name = uuid4().hex
+        self.container.upload_object_via_stream(
+            iterator=fyle.stream.__iter__(),
+            object_name=object_name,
+            extra={"acl": "private"},
+        )
+        return (fyle.filename, object_name)
+
+    def download(self, path):
+        pass
+
+    def _get_container(self, provider, container, key, secret):
+        if provider == "LOCAL":
+            key = container
+            container = ""
+
+        driver = get_driver(getattr(Provider, provider))(key=key, secret=secret)
+        return driver.get_container(container)

config/base.ini

@@ -20,4 +20,6 @@ SECRET_KEY = change_me_into_something_secret
 SESSION_COOKIE_NAME=atat
 SESSION_TYPE = redis
 SESSION_USE_SIGNER = True
+STORAGE_CONTAINER=uploads
+STORAGE_PROVIDER=LOCAL
 WTF_CSRF_ENABLED = true

script/test

@@ -6,6 +6,9 @@ source "$(dirname "${0}")"/../script/include/global_header.inc.sh
 
 export FLASK_ENV=test
 
+# create upload directory for app
+mkdir uploads | true
+
 # Enable database resetting
 RESET_DB="true"
 

templates/requests/financial_verification.html

@@ -94,6 +94,9 @@
               f.clin_2003,placeholder="7,000",
               validation='integer'
             ) }}
+
+            {{ f.task_order.label }}
+            {{ f.task_order(multiple="") }}
           </fieldset>
         {% endif %}
 

tests/fixtures/sample.pdf

@@ -0,0 +1,198 @@
+%PDF-1.3
+%âãÏÓ
+
+1 0 obj
+<<
+/Type /Catalog
+/Outlines 2 0 R
+/Pages 3 0 R
+>>
+endobj
+
+2 0 obj
+<<
+/Type /Outlines
+/Count 0
+>>
+endobj
+
+3 0 obj
+<<
+/Type /Pages
+/Count 2
+/Kids [ 4 0 R 6 0 R ] 
+>>
+endobj
+
+4 0 obj
+<<
+/Type /Page
+/Parent 3 0 R
+/Resources <<
+/Font <<
+/F1 9 0 R 
+>>
+/ProcSet 8 0 R
+>>
+/MediaBox [0 0 612.0000 792.0000]
+/Contents 5 0 R
+>>
+endobj
+
+5 0 obj
+<< /Length 1074 >>
+stream
+2 J
+BT
+0 0 0 rg
+/F1 0027 Tf
+57.3750 722.2800 Td
+( A Simple PDF File ) Tj
+ET
+BT
+/F1 0010 Tf
+69.2500 688.6080 Td
+( This is a small demonstration .pdf file - ) Tj
+ET
+BT
+/F1 0010 Tf
+69.2500 664.7040 Td
+( just for use in the Virtual Mechanics tutorials. More text. And more ) Tj
+ET
+BT
+/F1 0010 Tf
+69.2500 652.7520 Td
+( text. And more text. And more text. And more text. ) Tj
+ET
+BT
+/F1 0010 Tf
+69.2500 628.8480 Td
+( And more text. And more text. And more text. And more text. And more ) Tj
+ET
+BT
+/F1 0010 Tf
+69.2500 616.8960 Td
+( text. And more text. Boring, zzzzz. And more text. And more text. And ) Tj
+ET
+BT
+/F1 0010 Tf
+69.2500 604.9440 Td
+( more text. And more text. And more text. And more text. And more text. ) Tj
+ET
+BT
+/F1 0010 Tf
+69.2500 592.9920 Td
+( And more text. And more text. ) Tj
+ET
+BT
+/F1 0010 Tf
+69.2500 569.0880 Td
+( And more text. And more text. And more text. And more text. And more ) Tj
+ET
+BT
+/F1 0010 Tf
+69.2500 557.1360 Td
+( text. And more text. And more text. Even more. Continued on page 2 ...) Tj
+ET
+endstream
+endobj
+
+6 0 obj
+<<
+/Type /Page
+/Parent 3 0 R
+/Resources <<
+/Font <<
+/F1 9 0 R 
+>>
+/ProcSet 8 0 R
+>>
+/MediaBox [0 0 612.0000 792.0000]
+/Contents 7 0 R
+>>
+endobj
+
+7 0 obj
+<< /Length 676 >>
+stream
+2 J
+BT
+0 0 0 rg
+/F1 0027 Tf
+57.3750 722.2800 Td
+( Simple PDF File 2 ) Tj
+ET
+BT
+/F1 0010 Tf
+69.2500 688.6080 Td
+( ...continued from page 1. Yet more text. And more text. And more text. ) Tj
+ET
+BT
+/F1 0010 Tf
+69.2500 676.6560 Td
+( And more text. And more text. And more text. And more text. And more ) Tj
+ET
+BT
+/F1 0010 Tf
+69.2500 664.7040 Td
+( text. Oh, how boring typing this stuff. But not as boring as watching ) Tj
+ET
+BT
+/F1 0010 Tf
+69.2500 652.7520 Td
+( paint dry. And more text. And more text. And more text. And more text. ) Tj
+ET
+BT
+/F1 0010 Tf
+69.2500 640.8000 Td
+( Boring.  More, a little more text. The end, and just as well. ) Tj
+ET
+endstream
+endobj
+
+8 0 obj
+[/PDF /Text]
+endobj
+
+9 0 obj
+<<
+/Type /Font
+/Subtype /Type1
+/Name /F1
+/BaseFont /Helvetica
+/Encoding /WinAnsiEncoding
+>>
+endobj
+
+10 0 obj
+<<
+/Creator (Rave \(http://www.nevrona.com/rave\))
+/Producer (Nevrona Designs)
+/CreationDate (D:20060301072826)
+>>
+endobj
+
+xref
+0 11
+0000000000 65535 f
+0000000019 00000 n
+0000000093 00000 n
+0000000147 00000 n
+0000000222 00000 n
+0000000390 00000 n
+0000001522 00000 n
+0000001690 00000 n
+0000002423 00000 n
+0000002456 00000 n
+0000002574 00000 n
+
+trailer
+<<
+/Size 11
+/Root 1 0 R
+/Info 10 0 R
+>>
+
+startxref
+2714
+%%EOF

tests/test_uploader.py

@@ -0,0 +1,38 @@
+import os
+import pytest
+from werkzeug.datastructures import FileStorage
+
+from atst.uploader import Uploader, UploadError
+
+
+@pytest.fixture(scope="function")
+def upload_dir(tmpdir):
+    return tmpdir.mkdir("uploads")
+
+
+@pytest.fixture
+def uploader(upload_dir):
+    return Uploader("LOCAL", container=upload_dir)
+
+
+PDF_FILENAME = "tests/fixtures/sample.pdf"
+NONPDF_FILENAME = "tests/fixtures/disa-pki.html"
+
+@pytest.fixture
+def pdf():
+    with open(PDF_FILENAME, "rb") as fp:
+        yield FileStorage(fp, content_type="application/pdf")
+
+
+def test_upload(uploader, upload_dir, pdf):
+    filename, object_name = uploader.upload(pdf)
+    assert filename == PDF_FILENAME
+    assert os.path.isfile(os.path.join(upload_dir, object_name))
+
+
+def test_upload_fails_for_non_pdfs(uploader, pdf):
+    with open(NONPDF_FILENAME, "rb") as fp:
+        fs = FileStorage(fp, content_type="text/plain")
+        with pytest.raises(UploadError):
+            uploader.upload(fs)
+