From 00a5a98577aa4f685b442131d9878900a3100125 Mon Sep 17 00:00:00 2001 From: graham-dds Date: Thu, 30 Jan 2020 10:46:04 -0500 Subject: [PATCH 1/4] Add Mailer class method to email TOs to MSFT --- README.md | 1 + atst/utils/mailer.py | 29 +++++++++++++++++- config/base.ini | 1 + tests/conftest.py | 6 ++++ tests/utils/test_mailer.py | 61 +++++++++++++++++++++++++++++++++++++- translations.yaml | 4 ++- 6 files changed, 99 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index d846d486..b8530135 100644 --- a/README.md +++ b/README.md @@ -241,6 +241,7 @@ To generate coverage reports for the Javascript tests: - `MAIL_SENDER`: String. Email address to send outgoing mail from. - `MAIL_SERVER`: The SMTP host - `MAIL_TLS`: Boolean. Use TLS to connect to the SMTP server. +- `MICROSOFT_TASK_ORDER_EMAIL_ADDRESS`: String. Email address for Microsoft to receive PDFs of new and updated task orders. - `PERMANENT_SESSION_LIFETIME`: Integer specifying how many seconds a user's session can stay valid for. https://flask.palletsprojects.com/en/1.1.x/config/#PERMANENT_SESSION_LIFETIME - `PGDATABASE`: String specifying the name of the postgres database. - `PGHOST`: String specifying the hostname of the postgres database. diff --git a/atst/utils/mailer.py b/atst/utils/mailer.py index a67a851b..a5dbfc0b 100644 --- a/atst/utils/mailer.py +++ b/atst/utils/mailer.py @@ -1,5 +1,6 @@ from contextlib import contextmanager import smtplib +import io from email.message import EmailMessage @@ -76,8 +77,34 @@ class Mailer(object): return msg - def send(self, recipients, subject, body): + def _add_attachment(self, message, content, filename, maintype, subtype): + with io.BytesIO(content) as bytes_: + message.add_attachment( + bytes_.read(), filename=filename, maintype=maintype, subtype=subtype + ) + + def send(self, recipients, subject, body, attachments=[]): + """ + Send a message, optionally with attachments. + Attachments should be provided as a list of dictionaries of the form: + { + content: bytes, + maintype: string, + subtype: string, + filename: string, + } + """ message = self._build_message(recipients, subject, body) + if attachments: + message.make_mixed() + for attachment in attachments: + self._add_attachment( + message, + content=attachment["content"], + filename=attachment["filename"], + maintype=attachment.get("maintype", "application"), + subtype=attachment.get("subtype", "octet-stream"), + ) self.connection.send(message) @property diff --git a/config/base.ini b/config/base.ini index 71081e83..1f4c732a 100644 --- a/config/base.ini +++ b/config/base.ini @@ -26,6 +26,7 @@ MAIL_PORT MAIL_SENDER MAIL_SERVER MAIL_TLS +MICROSOFT_TASK_ORDER_EMAIL_ADDRESS = example@example.com PERMANENT_SESSION_LIFETIME = 1800 PGDATABASE = atat PGHOST = localhost diff --git a/tests/conftest.py b/tests/conftest.py index 8c22ddc3..03d07a12 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -164,6 +164,12 @@ def pdf_upload2(): yield FileStorage(fp, content_type="application/pdf") +@pytest.fixture +def downloaded_task_order(): + with open(PDF_FILENAME, "rb") as fp: + yield {"name": "mock.pdf", "content": fp.read()} + + @pytest.fixture def extended_financial_verification_data(pdf_upload): return { diff --git a/tests/utils/test_mailer.py b/tests/utils/test_mailer.py index 8b40ea48..41c68434 100644 --- a/tests/utils/test_mailer.py +++ b/tests/utils/test_mailer.py @@ -1,10 +1,17 @@ import pytest -from atst.utils.mailer import Mailer, Mailer, MailConnection, RedisConnection +from atst.utils.mailer import ( + Mailer, + MailConnection, + RedisConnection, +) +from atst.utils.localization import translate +from email.mime.base import MIMEBase class MockConnection(MailConnection): def __init__(self): self._messages = [] + self.sender = "mock@mock.com" def send(self, message): self._messages.append(message) @@ -46,3 +53,55 @@ def test_redis_mailer_can_save_messages(app): assert message_data["recipients"][0] in message assert message_data["subject"] in message assert message_data["body"] in message + + +def test_send_with_attachment(app, mailer, downloaded_task_order): + to_number = "11111111111111" + attachment = { + "maintype": "application", + "subtype": "pdf", + "filename": downloaded_task_order["name"], + "content": downloaded_task_order["content"], + } + mailer.send( + recipients=[app.config["MICROSOFT_TASK_ORDER_EMAIL_ADDRESS"]], + subject=translate("email.task_order_sent.subject", {"to_number": to_number}), + body=translate("email.task_order_sent.body", {"to_number": to_number}), + attachments=[attachment], + ) + # one email was sent + assert len(mailer.messages) == 1 + + # the email was sent to Microsoft with the correct subject line + message = mailer.messages[0] + assert message["To"] == app.config["MICROSOFT_TASK_ORDER_EMAIL_ADDRESS"] + assert message["Subject"] == translate( + "email.task_order_sent.subject", {"to_number": to_number} + ) + + # the email was sent as a multipart message with two parts -- the message + # body and the attachment + assert message.is_multipart() + message_payload = message.get_payload() + assert len(message_payload) == 2 + + # A body and attachment were in the email + body = next( + ( + part + for part in message_payload + if part["Content-Type"] == 'text/plain; charset="utf-8"' + ), + None, + ) + attachment = next( + (part for part in message_payload if part["Content-Type"] == "application/pdf"), + None, + ) + assert body + assert attachment + + assert ( + attachment["Content-Disposition"] + == f"attachment; filename=\"{downloaded_task_order['name']}\"" + ) diff --git a/translations.yaml b/translations.yaml index 4fdb3b3c..44dd2a92 100644 --- a/translations.yaml +++ b/translations.yaml @@ -84,6 +84,9 @@ email: application_invite: "{inviter_name} has invited you to a JEDI cloud application" portfolio_invite: "{inviter_name} has invited you to a JEDI cloud portfolio" environment_ready: JEDI cloud environment ready + task_order_sent: + subject: "Task Order {to_number}" + body: "Task Order number {to_number} updated." empty_state: applications: header: @@ -480,7 +483,6 @@ portfolios: "False": View Team "True": Edit Team perms_env_mgmt: - "False": View Environments "True": Edit Environments roles: From 0e49d038be385cbf0f3a2bfc4a3a3022e125efca Mon Sep 17 00:00:00 2001 From: graham-dds Date: Thu, 30 Jan 2020 15:06:46 -0500 Subject: [PATCH 2/4] Add azure-storage-blob and remove pytest-watch pytest-watch + pipenv were causing problems when building the app in CI, so pytest-watched was removed for the time being. --- Pipfile | 2 +- Pipfile.lock | 204 +++++++++++++++++++++------------------------------ 2 files changed, 85 insertions(+), 121 deletions(-) diff --git a/Pipfile b/Pipfile index 5d3a7920..4030b90d 100644 --- a/Pipfile +++ b/Pipfile @@ -37,6 +37,7 @@ azure-mgmt-consumption = "*" adal = "*" azure-identity = "*" azure-keyvault = "*" +azure-storage-blob = "*" [dev-packages] bandit = "*" @@ -45,7 +46,6 @@ ipython = "*" ipdb = "*" pylint = "*" black = "*" -pytest-watch = "*" factory-boy = "*" pytest-flask = "*" pytest-env = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 53a28b17..8b3379bb 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "4dbb023bcb860eb6dc56e1c201c91f272e1e67ad03e5e5eeb3a7a7fdff350eed" + "sha256": "faa5dab7bc6d13d39c0ef80f015f34a7fce2d66bec273ff38b7bd9ba232a3502" }, "pipfile-spec": 6, "requires": { @@ -116,11 +116,11 @@ }, "azure-mgmt-resource": { "hashes": [ - "sha256:20b3394e4dc76fbd9459723cb8c0300fb18a8c32100076f023b5470426b9f104", - "sha256:eaea8b5d05495d1b74220052275d46b6bed93b59245bcaa747279a52e41c3bdf" + "sha256:455a10bbae15673c7879d7515b38e1548cb1a8982dd35029ab3192565262c573", + "sha256:c2ad10cab63999c0a88ee498bc36200ee7f6e6e5d4bf82712bde882eda11146f" ], "index": "pypi", - "version": "==7.0.0" + "version": "==8.0.0" }, "azure-mgmt-subscription": { "hashes": [ @@ -146,6 +146,14 @@ "index": "pypi", "version": "==0.36.0" }, + "azure-storage-blob": { + "hashes": [ + "sha256:b628e2f8a8470a52895e96c43c3321542be3246c61ff9ed4f3856fe87a04a406", + "sha256:f6f39a2c297c0ed5db2fbf480ac73a00c1ace94a364d203d2e00081371c27ada" + ], + "index": "pypi", + "version": "==12.1.0" + }, "azure-storage-common": { "hashes": [ "sha256:b01a491a18839b9d05a4fe3421458a0ddb5ab9443c14e487f40d16f9a1dc2fbe", @@ -254,14 +262,6 @@ ], "version": "==2.8" }, - "dataclasses": { - "hashes": [ - "sha256:3459118f7ede7c8bea0fe795bff7c6c2ce287d01dd226202f7c9ebc0610a7836", - "sha256:494a6dcae3b8bcf80848eea2ef64c0cc5cd307ffc263e17cdf42f3e5420808e6" - ], - "markers": "python_version < '3.7'", - "version": "==0.7" - }, "flask": { "hashes": [ "sha256:13f9f196f330c7c2c5d7a5cf91af894110ca0215ac051b5844701f2bfd934d52", @@ -311,11 +311,11 @@ }, "importlib-metadata": { "hashes": [ - "sha256:bdd9b7c397c273bcc9a11d6629a38487cd07154fa255a467bf704cd2c258e359", - "sha256:f17c015735e1a88296994c0697ecea7e11db24290941983b08c9feb30921e6d8" + "sha256:06f5b3a99029c7134207dd882428a66992a9de2bef7c2b699b5641f9886c3302", + "sha256:b97607a1a18a5100839aec1dc26a1ea17ee0d93b20b0f008d80a5a050afb200b" ], "markers": "python_version < '3.8'", - "version": "==1.4.0" + "version": "==1.5.0" }, "isodate": { "hashes": [ @@ -333,10 +333,10 @@ }, "jinja2": { "hashes": [ - "sha256:74320bb91f31270f9551d46522e33af46a80c3d619f4a4bf42b3164d30b5911f", - "sha256:9fe95f19286cfefaa917656583d020be14e7859c6b0252588391e47db34527de" + "sha256:93187ffbc7808079673ef52771baa950426fd664d3aad1d0fa3e95644360e250", + "sha256:b0eaf100007721b5c16c1fc1eecb87409464edc10469ddc9a22a27a99123be49" ], - "version": "==2.10.3" + "version": "==2.11.1" }, "kombu": { "hashes": [ @@ -365,13 +365,16 @@ "sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161", "sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235", "sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5", + "sha256:13d3144e1e340870b25e7b10b98d779608c02016d5184cfb9927a9f10c689f42", "sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff", "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b", "sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1", "sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e", "sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183", "sha256:535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66", + "sha256:596510de112c685489095da617b5bcbbac7dd6384aeebeda4df6025d0256a81b", "sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1", + "sha256:6788b695d50a51edb699cb55e35487e430fa21f1ed838122d722e0ff0ac5ba15", "sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1", "sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e", "sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b", @@ -388,23 +391,18 @@ "sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6", "sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f", "sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f", - "sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7" + "sha256:cdb132fc825c38e1aeec2c8aa9338310d29d337bebbd7baa06889d09a60a1fa2", + "sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7", + "sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be" ], "version": "==1.1.1" }, - "more-itertools": { - "hashes": [ - "sha256:1a2a32c72400d365000412fe08eb4a24ebee89997c18d3d147544f70f5403b39", - "sha256:c468adec578380b6281a114cb8a5db34eb1116277da92d7c46f904f0b52d3288" - ], - "version": "==8.1.0" - }, "msal": { "hashes": [ - "sha256:c944b833bf686dfbc973e9affdef94b77e616cb52ab397e76cde82e26b8a3373", - "sha256:ecbe3f5ac77facad16abf08eb9d8562af3bc7184be5d4d90c9ef4db5bde26340" + "sha256:1a8df853f61a56a36332d6575a4ccae951e06145e494ad8259b4dd526b5d829a", + "sha256:aa24496c17edfbfb9143983fbd7525ba826a52ef00c2388a70b2c5ef7f6aed8e" ], - "version": "==1.0.0" + "version": "==1.1.0" }, "msal-extensions": { "hashes": [ @@ -415,10 +413,10 @@ }, "msrest": { "hashes": [ - "sha256:56b8b5b4556fb2a92cac640df267d560889bdc9e2921187772d4691d97bc4e8d", - "sha256:f5153bfe60ee757725816aedaa0772cbfe0bddb52cd2d6db4cb8b4c3c6c6f928" + "sha256:40faff88e151d393e29512e58b27d141974d6a963e63e4a340fc0ceb13c15f37", + "sha256:57eba26bd09d839d8f9133aea4b632d3216902efedf580b1a757c67b6538fb2c" ], - "version": "==0.6.10" + "version": "==0.6.11" }, "msrestazure": { "hashes": [ @@ -500,23 +498,23 @@ }, "pydantic": { "hashes": [ - "sha256:176885123dfdd8f7ab6e7ba1b66d4197de75ba830bb44d921af88b3d977b8aa5", - "sha256:2b32a5f14558c36e39aeefda0c550bfc0f47fc32b4ce16d80dc4df2b33838ed8", - "sha256:2eab7d548b0e530bf65bee7855ad8164c2f6a889975d5e9c4eefd1e7c98245dc", - "sha256:479ca8dc7cc41418751bf10302ee0a1b1f8eedb2de6c4f4c0f3cf8372b204f9a", - "sha256:59235324dd7dc5363a654cd14271ea8631f1a43de5d4fc29c782318fcc498002", - "sha256:87673d1de790c8d5282153cab0b09271be77c49aabcedf3ac5ab1a1fd4dcbac0", - "sha256:8a8e089aec18c26561e09ee6daf15a3cc06df05bdc67de60a8684535ef54562f", - "sha256:b60f2b3b0e0dd74f1800a57d1bbd597839d16faf267e45fa4a5407b15d311085", - "sha256:c0da48978382c83f9488c6bbe4350e065ea5c83e85ca5cfb8fa14ac11de3c296", - "sha256:cbe284bd5ad67333d49ecc0dc27fa52c25b4c2fe72802a5c060b5f922db58bef", - "sha256:d03df07b7611004140b0fef91548878c2b5f48c520a8cb76d11d20e9887a495e", - "sha256:d4bb6a75abc2f04f6993124f1ed4221724c9dc3bd9df5cb54132e0b68775d375", - "sha256:dacb79144bb3fdb57cf9435e1bd16c35586bc44256215cfaa33bf21565d926ae", - "sha256:dd9359db7644317898816f6142f378aa48848dcc5cf14a481236235fde11a148" + "sha256:012c422859bac2e03ab3151ea6624fecf0e249486be7eb8c6ee69c91740c6752", + "sha256:07911aab70f3bc52bb845ce1748569c5e70478ac977e106a150dd9d0465ebf04", + "sha256:47b8db7024ba3d46c3d4768535e1cf87b6c8cf92ccd81e76f4e1cb8ee47688b3", + "sha256:50e4e948892a6815649ad5a9a9379ad1e5f090f17842ac206535dfaed75c6f2f", + "sha256:51f11c8bbf794a68086540da099aae4a9107447c7a9d63151edbb7d50110cf21", + "sha256:6100d7862371115c40be55cc4b8d766a74b1d0dbaf99dbfe72bb4bac0faf89ed", + "sha256:61d22d36808087d3184ed6ac0d91dd71c533b66addb02e4a9930e1e30833202f", + "sha256:72184c1421103cca128300120f8f1185fb42a9ea73a1c9845b1c53db8c026a7d", + "sha256:831a0265a9e3933b3d0f04d1a81bba543bafbe4119c183ff2771871db70524ab", + "sha256:8848b4eb458469739126e4c1a202d723dd092e087f8dbe3104371335f87ba5df", + "sha256:bbbed364376f4a0aebb9ea452ff7968b306499a9e74f4db69b28ff2cd4043a11", + "sha256:e27559cedbd7f59d2375bfd6eea29a330ea1a5b0589c34d6b4e0d7bec6027bbf", + "sha256:f17ec336e64d4583311249fb179528e9a2c27c8a2eaf590ec6ec2c6dece7cb3f", + "sha256:f863456d3d4bf817f2e5248553dee3974c5dc796f48e6ddb599383570f4215ac" ], "index": "pypi", - "version": "==1.3" + "version": "==1.4" }, "pyjwt": { "extras": [ @@ -584,11 +582,11 @@ }, "redis": { "hashes": [ - "sha256:3613daad9ce5951e426f460deddd5caf469e08a3af633e9578fc77d362becf62", - "sha256:8d0fc278d3f5e1249967cba2eb4a5632d19e45ce5c09442b8422d15ee2c22cc2" + "sha256:7595976eb0b4e1fc3ad5478f1fd44215a814ee184a7820de92726f559bdff9cd", + "sha256:e933bdb504c69cbd5bdf4e2bb819a99644a36731cef4c59aa637cebfd5ddd4f9" ], "index": "pypi", - "version": "==3.3.11" + "version": "==3.4.0" }, "requests": { "hashes": [ @@ -672,11 +670,11 @@ }, "werkzeug": { "hashes": [ - "sha256:7280924747b5733b246fe23972186c6b348f9ae29724135a6dfc1e53cea433e7", - "sha256:e5f4a1f98b52b18a93da705a7458e55afb26f32bff83ff5d19189f92462d65c4" + "sha256:1e0dedc2acb1f46827daa2e399c1485c8fa17c0d8e70b6b875b4e7f54bf408d2", + "sha256:b353856d37dec59d6511359f97f6a4b2468442e454bd1c98298ddce53cac1f04" ], "index": "pypi", - "version": "==0.16.0" + "version": "==0.16.1" }, "wtforms": { "hashes": [ @@ -687,10 +685,10 @@ }, "zipp": { "hashes": [ - "sha256:b338014b9bc7102ca69e0fb96ed07215a8954d2989bc5d83658494ab2ba634af", - "sha256:e013e7800f60ec4dde789ebf4e9f7a54236e4bbf5df2a1a4e20ce9e1d9609d67" + "sha256:ccc94ed0909b58ffe34430ea5451f07bc0c76467d7081619a454bf5c98b89e28", + "sha256:feae2f18633c32fc71f2de629bfb3bd3c9325cd4419642b1f1da42ee488d9b98" ], - "version": "==2.0.1" + "version": "==2.1.0" } }, "develop": { @@ -709,13 +707,6 @@ "markers": "sys_platform == 'darwin'", "version": "==0.1.0" }, - "argh": { - "hashes": [ - "sha256:a9b3aaa1904eeb78e32394cd46c6f37ac0fb4af6dc488daa58971bdc7d7fcaf3", - "sha256:e9535b8c84dc9571a48999094fda7f33e63c3f1b74f3e5f3ac0105a58405bb65" - ], - "version": "==0.26.2" - }, "astroid": { "hashes": [ "sha256:71ea07f44df9568a75d0f354c49143a4575d90645e9fead6dfb52c26a85ed13a", @@ -797,13 +788,6 @@ ], "version": "==7.0" }, - "colorama": { - "hashes": [ - "sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff", - "sha256:e96da0d330793e2cb9485e9ddfd918d456036c7149416295932478192f4436a1" - ], - "version": "==0.4.3" - }, "coverage": { "hashes": [ "sha256:15cf13a6896048d6d947bf7d222f36e4809ab926894beb748fc9caa14605d9c3", @@ -855,12 +839,6 @@ "index": "pypi", "version": "==0.13.0" }, - "docopt": { - "hashes": [ - "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491" - ], - "version": "==0.6.2" - }, "factory-boy": { "hashes": [ "sha256:728df59b372c9588b83153facf26d3d28947fc750e8e3c95cefa9bed0e6394ee", @@ -915,11 +893,11 @@ }, "importlib-metadata": { "hashes": [ - "sha256:bdd9b7c397c273bcc9a11d6629a38487cd07154fa255a467bf704cd2c258e359", - "sha256:f17c015735e1a88296994c0697ecea7e11db24290941983b08c9feb30921e6d8" + "sha256:06f5b3a99029c7134207dd882428a66992a9de2bef7c2b699b5641f9886c3302", + "sha256:b97607a1a18a5100839aec1dc26a1ea17ee0d93b20b0f008d80a5a050afb200b" ], "markers": "python_version < '3.8'", - "version": "==1.4.0" + "version": "==1.5.0" }, "ipdb": { "hashes": [ @@ -959,17 +937,17 @@ }, "jedi": { "hashes": [ - "sha256:1349c1e8c107095a55386628bb3b2a79422f3a2cab8381e34ce19909e0cf5064", - "sha256:e909527104a903606dd63bea6e8e888833f0ef087057829b89a18364a856f807" + "sha256:b4f4052551025c6b0b0b193b29a6ff7bdb74c52450631206c262aef9f7159ad2", + "sha256:d5c871cb9360b414f981e7072c52c33258d598305280fef91c6cae34739d65d5" ], - "version": "==0.15.2" + "version": "==0.16.0" }, "jinja2": { "hashes": [ - "sha256:74320bb91f31270f9551d46522e33af46a80c3d619f4a4bf42b3164d30b5911f", - "sha256:9fe95f19286cfefaa917656583d020be14e7859c6b0252588391e47db34527de" + "sha256:93187ffbc7808079673ef52771baa950426fd664d3aad1d0fa3e95644360e250", + "sha256:b0eaf100007721b5c16c1fc1eecb87409464edc10469ddc9a22a27a99123be49" ], - "version": "==2.10.3" + "version": "==2.11.1" }, "lazy-object-proxy": { "hashes": [ @@ -1003,13 +981,16 @@ "sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161", "sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235", "sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5", + "sha256:13d3144e1e340870b25e7b10b98d779608c02016d5184cfb9927a9f10c689f42", "sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff", "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b", "sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1", "sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e", "sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183", "sha256:535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66", + "sha256:596510de112c685489095da617b5bcbbac7dd6384aeebeda4df6025d0256a81b", "sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1", + "sha256:6788b695d50a51edb699cb55e35487e430fa21f1ed838122d722e0ff0ac5ba15", "sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1", "sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e", "sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b", @@ -1026,7 +1007,9 @@ "sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6", "sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f", "sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f", - "sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7" + "sha256:cdb132fc825c38e1aeec2c8aa9338310d29d337bebbd7baa06889d09a60a1fa2", + "sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7", + "sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be" ], "version": "==1.1.1" }, @@ -1039,10 +1022,10 @@ }, "more-itertools": { "hashes": [ - "sha256:1a2a32c72400d365000412fe08eb4a24ebee89997c18d3d147544f70f5403b39", - "sha256:c468adec578380b6281a114cb8a5db34eb1116277da92d7c46f904f0b52d3288" + "sha256:5dd8bcf33e5f9513ffa06d5ad33d78f31e1931ac9a18f33d37e77a180d393a7c", + "sha256:b1ddb932186d8a6ac451e1d95844b382f55e12686d51ca0c68b6f61f2ab7a507" ], - "version": "==8.1.0" + "version": "==8.2.0" }, "mypy": { "hashes": [ @@ -1073,10 +1056,10 @@ }, "parso": { "hashes": [ - "sha256:55cf25df1a35fd88b878715874d2c4dc1ad3f0eebd1e0266a67e1f55efccfbe1", - "sha256:5c1f7791de6bd5dbbeac8db0ef5594b36799de198b3f7f7014643b0c5536b9d3" + "sha256:1376bdc8cb81377ca481976933773295218a2df47d3e1182ba76d372b1acb128", + "sha256:597f36de5102a8db05ffdf7ecdc761838b86565a4a111604c6e78beaedf1b045" ], - "version": "==0.5.2" + "version": "==0.6.0" }, "pathspec": { "hashes": [ @@ -1085,12 +1068,6 @@ ], "version": "==0.7.0" }, - "pathtools": { - "hashes": [ - "sha256:7c35c5421a39bb82e58018febd90e3b6e5db34c5443aaaf742b3f33d4655f1c0" - ], - "version": "==0.1.2" - }, "pbr": { "hashes": [ "sha256:139d2625547dbfa5fb0b81daebb39601c478c21956dc57e2e07b74450a8c506b", @@ -1122,10 +1099,10 @@ }, "prompt-toolkit": { "hashes": [ - "sha256:0278d2f51b5ceba6ea8da39f76d15684e84c996b325475f6e5720edc584326a7", - "sha256:63daee79aa8366c8f1c637f1a4876b890da5fc92a19ebd2f7080ebacb901e990" + "sha256:a402e9bf468b63314e37460b68ba68243d55b2f8c4d0192f85a019af3945050e", + "sha256:c93e53af97f630f12f5f62a3274e79527936ed466f038953dfa379d4941f651a" ], - "version": "==3.0.2" + "version": "==3.0.3" }, "ptyprocess": { "hashes": [ @@ -1195,13 +1172,6 @@ "index": "pypi", "version": "==2.0.0" }, - "pytest-watch": { - "hashes": [ - "sha256:06136f03d5b361718b8d0d234042f7b2f203910d8568f63df2f866b547b3d4b9" - ], - "index": "pypi", - "version": "==4.2.0" - }, "python-dateutil": { "hashes": [ "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c", @@ -1368,12 +1338,6 @@ ], "version": "==1.25.8" }, - "watchdog": { - "hashes": [ - "sha256:965f658d0732de3188211932aeb0bb457587f04f63ab4c1e33eab878e9de961d" - ], - "version": "==0.9.0" - }, "wcwidth": { "hashes": [ "sha256:8fd29383f539be45b20bd4df0dc29c20ba48654a41e661925e612311e9f3c603", @@ -1383,11 +1347,11 @@ }, "werkzeug": { "hashes": [ - "sha256:7280924747b5733b246fe23972186c6b348f9ae29724135a6dfc1e53cea433e7", - "sha256:e5f4a1f98b52b18a93da705a7458e55afb26f32bff83ff5d19189f92462d65c4" + "sha256:1e0dedc2acb1f46827daa2e399c1485c8fa17c0d8e70b6b875b4e7f54bf408d2", + "sha256:b353856d37dec59d6511359f97f6a4b2468442e454bd1c98298ddce53cac1f04" ], "index": "pypi", - "version": "==0.16.0" + "version": "==0.16.1" }, "wrapt": { "hashes": [ @@ -1397,10 +1361,10 @@ }, "zipp": { "hashes": [ - "sha256:b338014b9bc7102ca69e0fb96ed07215a8954d2989bc5d83658494ab2ba634af", - "sha256:e013e7800f60ec4dde789ebf4e9f7a54236e4bbf5df2a1a4e20ce9e1d9609d67" + "sha256:ccc94ed0909b58ffe34430ea5451f07bc0c76467d7081619a454bf5c98b89e28", + "sha256:feae2f18633c32fc71f2de629bfb3bd3c9325cd4419642b1f1da42ee488d9b98" ], - "version": "==2.0.1" + "version": "==2.1.0" } } } From a7770d8a6a1b5d8cfe4e1c688b09f529d3cd169c Mon Sep 17 00:00:00 2001 From: graham-dds Date: Thu, 30 Jan 2020 13:27:34 -0500 Subject: [PATCH 3/4] Add method to download TOs from Azure --- atst/domain/csp/file_uploads.py | 36 +++++++++++++++++++++---- tests/domain/cloud/test_file_uploads.py | 22 +++++++++++++++ 2 files changed, 53 insertions(+), 5 deletions(-) create mode 100644 tests/domain/cloud/test_file_uploads.py diff --git a/atst/domain/csp/file_uploads.py b/atst/domain/csp/file_uploads.py index 7916797d..ce18df63 100644 --- a/atst/domain/csp/file_uploads.py +++ b/atst/domain/csp/file_uploads.py @@ -4,14 +4,17 @@ from uuid import uuid4 class Uploader: def generate_token(self): - pass + raise NotImplementedError() def generate_download_link(self, object_name, filename) -> (dict, str): - pass + raise NotImplementedError() def object_name(self) -> str: return str(uuid4()) + def download_task_order(self, object_name): + raise NotImplementedError() + class MockUploader(Uploader): def __init__(self, config): @@ -23,6 +26,13 @@ class MockUploader(Uploader): def generate_download_link(self, object_name, filename): return "" + def download_task_order(self, object_name): + with open("tests/fixtures/sample.pdf", "rb") as some_bytes: + return { + "name": object_name, + "content": some_bytes, + } + class AzureUploader(Uploader): def __init__(self, config): @@ -32,10 +42,12 @@ class AzureUploader(Uploader): self.timeout = timedelta(seconds=config["PERMANENT_SESSION_LIFETIME"]) from azure.storage.common import CloudStorageAccount - from azure.storage.blob import BlobPermissions + from azure.storage.blob import BlobSasPermissions + from azure.storage.blob.blockblobservice import BlockBlobService self.CloudStorageAccount = CloudStorageAccount - self.BlobPermissions = BlobPermissions + self.BlobSasPermissions = BlobSasPermissions + self.BlockBlobService = BlockBlobService def get_token(self): """ @@ -53,7 +65,7 @@ class AzureUploader(Uploader): sas_token = bbs.generate_blob_shared_access_signature( self.container_name, object_name, - permission=self.BlobPermissions.CREATE, + permission=self.BlobSasPermissions(create=True), expiry=datetime.utcnow() + self.timeout, protocol="https", ) @@ -75,3 +87,17 @@ class AzureUploader(Uploader): return bbs.make_blob_url( self.container_name, object_name, protocol="https", sas_token=sas_token ) + + def download_task_order(self, object_name): + block_blob_service = self.BlockBlobService( + account_name=self.account_name, account_key=self.storage_key + ) + # TODO: We should downloading errors more gracefully + # - what happens when we try to request a TO that doesn't exist? + b = block_blob_service.get_blob_to_bytes( + container_name=self.container_name, blob_name=object_name, + ) + return { + "name": b.name, + "content": b.content, + } diff --git a/tests/domain/cloud/test_file_uploads.py b/tests/domain/cloud/test_file_uploads.py new file mode 100644 index 00000000..9b1b0932 --- /dev/null +++ b/tests/domain/cloud/test_file_uploads.py @@ -0,0 +1,22 @@ +from atst.domain.csp.file_uploads import AzureUploader +from azure.storage.blob.models import Blob + + +class MockBlockBlobService(object): + def __init__(self, exception=None, **kwargs): + self.exception = exception + + def get_blob_to_bytes(self, blob_name="test.pdf", **kwargs): + if self.exception: + raise self.exception + else: + return Blob(name=blob_name, content=b"mock content") + + +def test_download_task_order_success(app, monkeypatch): + uploader = AzureUploader(config=app.config) + uploader.BlockBlobService = MockBlockBlobService + + task_order = uploader.download_task_order("test.pdf") + assert task_order["name"] == "test.pdf" + assert task_order["content"] == b"mock content" From e32e211966323fb99e410f1db6023ed28fb296dd Mon Sep 17 00:00:00 2001 From: graham-dds Date: Thu, 30 Jan 2020 13:39:33 -0500 Subject: [PATCH 4/4] Make Upload related classes more generic Renames Python classes that refer to Upload to something to FileService. We added this change because these classes now handle downloading as well as uploading. --- atst/domain/csp/__init__.py | 6 +++--- atst/domain/csp/{file_uploads.py => files.py} | 6 +++--- .../{test_file_uploads.py => test_azure_file_service.py} | 8 ++++---- 3 files changed, 10 insertions(+), 10 deletions(-) rename atst/domain/csp/{file_uploads.py => files.py} (97%) rename tests/domain/cloud/{test_file_uploads.py => test_azure_file_service.py} (70%) diff --git a/atst/domain/csp/__init__.py b/atst/domain/csp/__init__.py index d886f8a2..69eac768 100644 --- a/atst/domain/csp/__init__.py +++ b/atst/domain/csp/__init__.py @@ -1,5 +1,5 @@ from .cloud import MockCloudProvider -from .file_uploads import AzureUploader, MockUploader +from .files import AzureFileService, MockFileService from .reports import MockReportingProvider @@ -11,14 +11,14 @@ class MockCSP: with_failure=(not test_mode), with_authorization=(not test_mode), ) - self.files = MockUploader(app) + self.files = MockFileService(app) self.reports = MockReportingProvider() class AzureCSP: def __init__(self, app): self.cloud = MockCloudProvider(app.config) - self.files = AzureUploader(app.config) + self.files = AzureFileService(app.config) self.reports = MockReportingProvider() diff --git a/atst/domain/csp/file_uploads.py b/atst/domain/csp/files.py similarity index 97% rename from atst/domain/csp/file_uploads.py rename to atst/domain/csp/files.py index ce18df63..837cecbb 100644 --- a/atst/domain/csp/file_uploads.py +++ b/atst/domain/csp/files.py @@ -2,7 +2,7 @@ from datetime import datetime, timedelta from uuid import uuid4 -class Uploader: +class FileService: def generate_token(self): raise NotImplementedError() @@ -16,7 +16,7 @@ class Uploader: raise NotImplementedError() -class MockUploader(Uploader): +class MockFileService(FileService): def __init__(self, config): self.config = config @@ -34,7 +34,7 @@ class MockUploader(Uploader): } -class AzureUploader(Uploader): +class AzureFileService(FileService): def __init__(self, config): self.account_name = config["AZURE_ACCOUNT_NAME"] self.storage_key = config["AZURE_STORAGE_KEY"] diff --git a/tests/domain/cloud/test_file_uploads.py b/tests/domain/cloud/test_azure_file_service.py similarity index 70% rename from tests/domain/cloud/test_file_uploads.py rename to tests/domain/cloud/test_azure_file_service.py index 9b1b0932..4eaf680c 100644 --- a/tests/domain/cloud/test_file_uploads.py +++ b/tests/domain/cloud/test_azure_file_service.py @@ -1,4 +1,4 @@ -from atst.domain.csp.file_uploads import AzureUploader +from atst.domain.csp.files import AzureFileService from azure.storage.blob.models import Blob @@ -14,9 +14,9 @@ class MockBlockBlobService(object): def test_download_task_order_success(app, monkeypatch): - uploader = AzureUploader(config=app.config) - uploader.BlockBlobService = MockBlockBlobService + file_service = AzureFileService(config=app.config) + file_service.BlockBlobService = MockBlockBlobService - task_order = uploader.download_task_order("test.pdf") + task_order = file_service.download_task_order("test.pdf") assert task_order["name"] == "test.pdf" assert task_order["content"] == b"mock content"