initial uploader and some form work

This commit is contained in:
dandds 2018-08-23 17:00:24 -04:00 committed by Montana
parent 700d914a55
commit ef2e97713a
11 changed files with 371 additions and 12 deletions

3
.gitignore vendored
View File

@ -35,3 +35,6 @@ config/dev.ini
/crl
/crl-tmp
*.bk
# uploads
/uploads

View File

@ -18,6 +18,8 @@ flask-session = "*"
flask-wtf = "*"
pyopenssl = "*"
requests = "*"
apache-libcloud = "*"
lockfile = "*"
[dev-packages]
bandit = "*"

73
Pipfile.lock generated
View File

@ -1,7 +1,7 @@
{
"_meta": {
"hash": {
"sha256": "41ad134816dae388385cfb15105e0eca436b25791ec4fbf67a2b36c4ae8056bd"
"sha256": "552b7ac6943559a1fc3be1c4e1c91f965cbfb97c115566051950450c7cd6f78b"
},
"pipfile-spec": 6,
"requires": {
@ -24,6 +24,14 @@
"index": "pypi",
"version": "==1.0.0"
},
"apache-libcloud": {
"hashes": [
"sha256:0e2eee3802163bd0605975ed1e284cafc23203919bfa80c0cc5d3cd2543aaf97",
"sha256:48d5d64790a5112cace1a8e28d228c3f1c5bd3ddbd986a5453172d2da19f47d5"
],
"index": "pypi",
"version": "==2.3.0"
},
"asn1crypto": {
"hashes": [
"sha256:2f1adbb7546ed199e3c90ef23ec95c5cf3585bac7d11fb7eb562a3fe89c64e87",
@ -33,10 +41,10 @@
},
"certifi": {
"hashes": [
"sha256:4c1d68a1408dd090d2f3a869aa94c3947cc1d967821d1ed303208c9f41f0f2f4",
"sha256:b6e8b28b2b7e771a41ecdd12d4d43262ecab52adebbafa42c77d6b57fb6ad3a4"
"sha256:376690d6f16d32f9d1fe8932551d80b23e9d393a8578c5633a2ed39a64861638",
"sha256:456048c7e371c089d0a77a5212fb37a2c2dce1e24146e3b7e0261736aaeaa22a"
],
"version": "==2018.8.13"
"version": "==2018.8.24"
},
"cffi": {
"hashes": [
@ -172,6 +180,14 @@
],
"version": "==2.10"
},
"lockfile": {
"hashes": [
"sha256:6aed02de03cba24efabcd600b30540140634fc06cfa603822d508d5361e9f799",
"sha256:6c3cb24f344923d30b2785d5ad75182c8ea7ac1b6171b08657258ec7429d50fa"
],
"index": "pypi",
"version": "==0.12.2"
},
"mako": {
"hashes": [
"sha256:4e02fde57bd4abb5ec400181e4c314f56ac3e49ba4fb8b0d50bba18cb27d25ae"
@ -271,7 +287,6 @@
"sha256:1d936da41ee06216d89fdc7ead1ee9a5da2811a8787515a976b646e110c3f622",
"sha256:e4ef42e82b0b493c5849eed98b5ab49d6767caf982127e9a33167f1153b36cc5"
],
"markers": "python_version != '3.0.*' and python_version != '3.2.*' and python_version >= '2.7' and python_version != '3.1.*' and python_version != '3.3.*'",
"version": "==2018.5"
},
"redis": {
@ -317,7 +332,6 @@
"sha256:a68ac5e15e76e7e5dd2b8f94007233e01effe3e50e8daddf69acfd81cb686baf",
"sha256:b5725a0bd4ba422ab0e66e89e030c806576753ea3ee08554382c14e685d117b5"
],
"markers": "python_version >= '2.6' and python_version != '3.2.*' and python_version != '3.0.*' and python_version < '4' and python_version != '3.1.*' and python_version != '3.3.*'",
"version": "==1.23"
},
"webassets": {
@ -350,6 +364,14 @@
],
"version": "==1.4.3"
},
"appnope": {
"hashes": [
"sha256:5b26757dc6f79a3b7dc9fab95359328d5747fcb2409d331ea66d0272b90ab2a0",
"sha256:8b995ffe925347a2138d7ac0fe77155e4311a0ea6d6da4f5128fe4b3cbe5ed71"
],
"markers": "sys_platform == 'darwin'",
"version": "==0.1.0"
},
"argh": {
"hashes": [
"sha256:a9b3aaa1904eeb78e32394cd46c6f37ac0fb4af6dc488daa58971bdc7d7fcaf3",
@ -441,7 +463,6 @@
"sha256:ea7cfd3aeb1544732d08bd9cfba40c5b78e3a91e17b1a0698ab81bfc5554c628",
"sha256:f6d67f04abfb2b4bea7afc7fa6c18cf4c523a67956e455668be9ae42bccc21ad"
],
"markers": "python_version != '3.2.*' and python_version != '3.0.*' and python_version != '3.1.*' and python_version >= '2.7'",
"version": "==0.9.0"
},
"flask": {
@ -494,7 +515,6 @@
"sha256:b9c40e9750f3d77e6e4d441d8b0266cf555e7cdabdcff33c4fd06366ca761ef8",
"sha256:ec9ef8f4a9bc6f71eec99e1806bfa2de401650d996c59330782b89a5555c1497"
],
"markers": "python_version >= '2.7' and python_version != '3.2.*' and python_version != '3.1.*' and python_version != '3.0.*' and python_version != '3.3.*'",
"version": "==4.3.4"
},
"itsdangerous": {
@ -612,7 +632,6 @@
"sha256:6e3836e39f4d36ae72840833db137f7b7d35105079aee6ec4a62d9f80d594dd1",
"sha256:95eb8364a4708392bae89035f45341871286a333f749c3141c20573d2b3876e1"
],
"markers": "python_version >= '2.7' and python_version != '3.2.*' and python_version != '3.1.*' and python_version != '3.0.*' and python_version != '3.3.*'",
"version": "==0.7.1"
},
"prompt-toolkit": {
@ -635,7 +654,6 @@
"sha256:3fd59af7435864e1a243790d322d763925431213b6b8529c6ca71081ace3bbf7",
"sha256:e31fb2767eb657cbde86c454f02e99cb846d3cd9d61b318525140214fdc0e98e"
],
"markers": "python_version >= '2.7' and python_version != '3.2.*' and python_version != '3.1.*' and python_version != '3.0.*' and python_version != '3.3.*'",
"version": "==1.5.4"
},
"pygments": {
@ -692,11 +710,15 @@
},
"pyyaml": {
"hashes": [
"sha256:1cbc199009e78f92d9edf554be4fe40fb7b0bef71ba688602a00e97a51909110",
"sha256:254bf6fda2b7c651837acb2c718e213df29d531eebf00edb54743d10bcb694eb",
"sha256:3108529b78577327d15eec243f0ff348a0640b0c3478d67ad7f5648f93bac3e2",
"sha256:3c17fb92c8ba2f525e4b5f7941d850e7a48c3a59b32d331e2502a3cdc6648e76",
"sha256:6f89b5c95e93945b597776163403d47af72d243f366bf4622ff08bdfd1c950b7",
"sha256:8d6d96001aa7f0a6a4a95e8143225b5d06e41b1131044913fecb8f85a125714b",
"sha256:c8a88edd93ee29ede719080b2be6cb2333dfee1dccba213b422a9c8e97f2967b"
"sha256:be622cc81696e24d0836ba71f6272a2b5767669b0d79fdcf0295d51ac2e156c8",
"sha256:c8a88edd93ee29ede719080b2be6cb2333dfee1dccba213b422a9c8e97f2967b",
"sha256:f39411e380e2182ad33be039e8ee5770a5d9efe01a2bfb7ae58d9ba31c4a2a9d"
],
"version": "==4.2b4"
},
@ -747,6 +769,35 @@
],
"version": "==4.3.2"
},
"typed-ast": {
"hashes": [
"sha256:0948004fa228ae071054f5208840a1e88747a357ec1101c17217bfe99b299d58",
"sha256:10703d3cec8dcd9eef5a630a04056bbc898abc19bac5691612acba7d1325b66d",
"sha256:1f6c4bd0bdc0f14246fd41262df7dfc018d65bb05f6e16390b7ea26ca454a291",
"sha256:25d8feefe27eb0303b73545416b13d108c6067b846b543738a25ff304824ed9a",
"sha256:29464a177d56e4e055b5f7b629935af7f49c196be47528cc94e0a7bf83fbc2b9",
"sha256:2e214b72168ea0275efd6c884b114ab42e316de3ffa125b267e732ed2abda892",
"sha256:3e0d5e48e3a23e9a4d1a9f698e32a542a4a288c871d33ed8df1b092a40f3a0f9",
"sha256:519425deca5c2b2bdac49f77b2c5625781abbaf9a809d727d3a5596b30bb4ded",
"sha256:57fe287f0cdd9ceaf69e7b71a2e94a24b5d268b35df251a88fef5cc241bf73aa",
"sha256:668d0cec391d9aed1c6a388b0d5b97cd22e6073eaa5fbaa6d2946603b4871efe",
"sha256:68ba70684990f59497680ff90d18e756a47bf4863c604098f10de9716b2c0bdd",
"sha256:6de012d2b166fe7a4cdf505eee3aaa12192f7ba365beeefaca4ec10e31241a85",
"sha256:79b91ebe5a28d349b6d0d323023350133e927b4de5b651a8aa2db69c761420c6",
"sha256:8550177fa5d4c1f09b5e5f524411c44633c80ec69b24e0e98906dd761941ca46",
"sha256:898f818399cafcdb93cbbe15fc83a33d05f18e29fb498ddc09b0214cdfc7cd51",
"sha256:94b091dc0f19291adcb279a108f5d38de2430411068b219f41b343c03b28fb1f",
"sha256:a26863198902cda15ab4503991e8cf1ca874219e0118cbf07c126bce7c4db129",
"sha256:a8034021801bc0440f2e027c354b4eafd95891b573e12ff0418dec385c76785c",
"sha256:bc978ac17468fe868ee589c795d06777f75496b1ed576d308002c8a5756fb9ea",
"sha256:c05b41bc1deade9f90ddc5d988fe506208019ebba9f2578c622516fd201f5863",
"sha256:c9b060bd1e5a26ab6e8267fd46fc9e02b54eb15fffb16d112d4c7b1c12987559",
"sha256:edb04bdd45bfd76c8292c4d9654568efaedf76fe78eb246dde69bdb13b2dad87",
"sha256:f19f2a4f547505fe9072e15f6f4ae714af51b5a681a97f187971f50c283193b6"
],
"markers": "python_version < '3.7' and implementation_name == 'cpython'",
"version": "==1.1.0"
},
"watchdog": {
"hashes": [
"sha256:7e65882adb7746039b6f3876ee174952f8eaaa34491ba34333ddf1fe35de4162"

View File

@ -19,6 +19,7 @@ from atst.routes.errors import make_error_pages
from atst.domain.authnid.crl import CRLCache
from atst.domain.auth import apply_authentication
from atst.eda_client import MockEDAClient
from atst.uploader import Uploader
ENV = os.getenv("FLASK_ENV", "dev")
@ -43,6 +44,7 @@ def make_app(config):
make_crl_validator(app)
register_filters(app)
make_eda_client(app)
make_upload_storage(app)
db.init_app(app)
csrf.init_app(app)
@ -143,3 +145,12 @@ def make_crl_validator(app):
def make_eda_client(app):
app.eda_client = MockEDAClient()
def make_upload_storage(app):
uploader = Uploader(
provider=app.config.get("STORAGE_PROVIDER"),
container=app.config.get("STORAGE_CONTAINER"),
key=app.config.get("STORAGE_KEY"),
secret=app.config.get("STORAGE_SECRET")
)
app.uploader = uploader

View File

@ -1,7 +1,8 @@
import re
from wtforms.fields.html5 import EmailField
from wtforms.fields import StringField
from wtforms.fields import StringField, FileField
from wtforms.validators import Required, Email, Regexp
from flask_wtf.file import FileAllowed
from atst.domain.exceptions import NotFoundError
from atst.domain.pe_numbers import PENumbers
@ -214,3 +215,5 @@ class ExtendedFinancialForm(BaseFinancialForm):
description="Review your task order document, the amounts for each CLIN must match exactly here",
filters=[number_to_int],
)
task_order = FileField("Upload a copy of your Task Order", validators=[FileAllowed(['pdf'], 'Only PDF documents can be uploaded.')])

45
atst/uploader.py Normal file
View File

@ -0,0 +1,45 @@
from uuid import uuid4
from libcloud.storage.types import Provider
from libcloud.storage.providers import get_driver
class UploadError(Exception):
pass
class Uploader():
_PERMITTED_MIMETYPES = ["application/pdf"]
def __init__(self, provider, container=None, key=None, secret=None):
self.container = self._get_container(provider, container, key, secret)
def upload(self, fyle):
# TODO: for hardening, we should probably use a better library for
# determining mimetype and not rely on FileUpload's determination
# TODO: we should set MAX_CONTENT_LENGTH in the config to prevent large
# uploads
if not fyle.mimetype in self._PERMITTED_MIMETYPES:
raise UploadError(
"could not upload {} with mimetype {}".format(
fyle.filename, fyle.mimetype
)
)
object_name = uuid4().hex
self.container.upload_object_via_stream(
iterator=fyle.stream.__iter__(),
object_name=object_name,
extra={"acl": "private"},
)
return (fyle.filename, object_name)
def download(self, path):
pass
def _get_container(self, provider, container, key, secret):
if provider == "LOCAL":
key = container
container = ""
driver = get_driver(getattr(Provider, provider))(key=key, secret=secret)
return driver.get_container(container)

View File

@ -20,4 +20,6 @@ SECRET_KEY = change_me_into_something_secret
SESSION_COOKIE_NAME=atat
SESSION_TYPE = redis
SESSION_USE_SIGNER = True
STORAGE_CONTAINER=uploads
STORAGE_PROVIDER=LOCAL
WTF_CSRF_ENABLED = true

View File

@ -6,6 +6,9 @@ source "$(dirname "${0}")"/../script/include/global_header.inc.sh
export FLASK_ENV=test
# create upload directory for app
mkdir uploads | true
# Enable database resetting
RESET_DB="true"

View File

@ -94,6 +94,9 @@
f.clin_2003,placeholder="7,000",
validation='integer'
) }}
{{ f.task_order.label }}
{{ f.task_order(multiple="") }}
</fieldset>
{% endif %}

198
tests/fixtures/sample.pdf vendored Normal file
View File

@ -0,0 +1,198 @@
%PDF-1.3
%âãÏÓ
1 0 obj
<<
/Type /Catalog
/Outlines 2 0 R
/Pages 3 0 R
>>
endobj
2 0 obj
<<
/Type /Outlines
/Count 0
>>
endobj
3 0 obj
<<
/Type /Pages
/Count 2
/Kids [ 4 0 R 6 0 R ]
>>
endobj
4 0 obj
<<
/Type /Page
/Parent 3 0 R
/Resources <<
/Font <<
/F1 9 0 R
>>
/ProcSet 8 0 R
>>
/MediaBox [0 0 612.0000 792.0000]
/Contents 5 0 R
>>
endobj
5 0 obj
<< /Length 1074 >>
stream
2 J
BT
0 0 0 rg
/F1 0027 Tf
57.3750 722.2800 Td
( A Simple PDF File ) Tj
ET
BT
/F1 0010 Tf
69.2500 688.6080 Td
( This is a small demonstration .pdf file - ) Tj
ET
BT
/F1 0010 Tf
69.2500 664.7040 Td
( just for use in the Virtual Mechanics tutorials. More text. And more ) Tj
ET
BT
/F1 0010 Tf
69.2500 652.7520 Td
( text. And more text. And more text. And more text. ) Tj
ET
BT
/F1 0010 Tf
69.2500 628.8480 Td
( And more text. And more text. And more text. And more text. And more ) Tj
ET
BT
/F1 0010 Tf
69.2500 616.8960 Td
( text. And more text. Boring, zzzzz. And more text. And more text. And ) Tj
ET
BT
/F1 0010 Tf
69.2500 604.9440 Td
( more text. And more text. And more text. And more text. And more text. ) Tj
ET
BT
/F1 0010 Tf
69.2500 592.9920 Td
( And more text. And more text. ) Tj
ET
BT
/F1 0010 Tf
69.2500 569.0880 Td
( And more text. And more text. And more text. And more text. And more ) Tj
ET
BT
/F1 0010 Tf
69.2500 557.1360 Td
( text. And more text. And more text. Even more. Continued on page 2 ...) Tj
ET
endstream
endobj
6 0 obj
<<
/Type /Page
/Parent 3 0 R
/Resources <<
/Font <<
/F1 9 0 R
>>
/ProcSet 8 0 R
>>
/MediaBox [0 0 612.0000 792.0000]
/Contents 7 0 R
>>
endobj
7 0 obj
<< /Length 676 >>
stream
2 J
BT
0 0 0 rg
/F1 0027 Tf
57.3750 722.2800 Td
( Simple PDF File 2 ) Tj
ET
BT
/F1 0010 Tf
69.2500 688.6080 Td
( ...continued from page 1. Yet more text. And more text. And more text. ) Tj
ET
BT
/F1 0010 Tf
69.2500 676.6560 Td
( And more text. And more text. And more text. And more text. And more ) Tj
ET
BT
/F1 0010 Tf
69.2500 664.7040 Td
( text. Oh, how boring typing this stuff. But not as boring as watching ) Tj
ET
BT
/F1 0010 Tf
69.2500 652.7520 Td
( paint dry. And more text. And more text. And more text. And more text. ) Tj
ET
BT
/F1 0010 Tf
69.2500 640.8000 Td
( Boring. More, a little more text. The end, and just as well. ) Tj
ET
endstream
endobj
8 0 obj
[/PDF /Text]
endobj
9 0 obj
<<
/Type /Font
/Subtype /Type1
/Name /F1
/BaseFont /Helvetica
/Encoding /WinAnsiEncoding
>>
endobj
10 0 obj
<<
/Creator (Rave \(http://www.nevrona.com/rave\))
/Producer (Nevrona Designs)
/CreationDate (D:20060301072826)
>>
endobj
xref
0 11
0000000000 65535 f
0000000019 00000 n
0000000093 00000 n
0000000147 00000 n
0000000222 00000 n
0000000390 00000 n
0000001522 00000 n
0000001690 00000 n
0000002423 00000 n
0000002456 00000 n
0000002574 00000 n
trailer
<<
/Size 11
/Root 1 0 R
/Info 10 0 R
>>
startxref
2714
%%EOF

38
tests/test_uploader.py Normal file
View File

@ -0,0 +1,38 @@
import os
import pytest
from werkzeug.datastructures import FileStorage
from atst.uploader import Uploader, UploadError
@pytest.fixture(scope="function")
def upload_dir(tmpdir):
return tmpdir.mkdir("uploads")
@pytest.fixture
def uploader(upload_dir):
return Uploader("LOCAL", container=upload_dir)
PDF_FILENAME = "tests/fixtures/sample.pdf"
NONPDF_FILENAME = "tests/fixtures/disa-pki.html"
@pytest.fixture
def pdf():
with open(PDF_FILENAME, "rb") as fp:
yield FileStorage(fp, content_type="application/pdf")
def test_upload(uploader, upload_dir, pdf):
filename, object_name = uploader.upload(pdf)
assert filename == PDF_FILENAME
assert os.path.isfile(os.path.join(upload_dir, object_name))
def test_upload_fails_for_non_pdfs(uploader, pdf):
with open(NONPDF_FILENAME, "rb") as fp:
fs = FileStorage(fp, content_type="text/plain")
with pytest.raises(UploadError):
uploader.upload(fs)