diff --git a/Pipfile b/Pipfile index bc5ed901..10d7ec29 100644 --- a/Pipfile +++ b/Pipfile @@ -21,6 +21,7 @@ requests = "*" apache-libcloud = "*" lockfile = "*" defusedxml = "*" +"flask-rq2" = "*" [dev-packages] bandit = "*" @@ -35,6 +36,7 @@ pytest-flask = "*" pytest-env = "*" pytest-cov = "*" selenium = "*" +honcho = "*" [requires] python_version = "3.6.6" diff --git a/Pipfile.lock b/Pipfile.lock index bf863dc6..cf9730d8 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "1e5e6a695229166aaa5e6c427fed07a903766e9b3d24981a19cc8e5ada8db978" + "sha256": "c67f5a847351d9d6e8ef165c380dd97fdf623f87cf8299a64109e453027e2458" }, "pipfile-spec": 6, "requires": { @@ -41,10 +41,10 @@ }, "certifi": { "hashes": [ - "sha256:376690d6f16d32f9d1fe8932551d80b23e9d393a8578c5633a2ed39a64861638", - "sha256:456048c7e371c089d0a77a5212fb37a2c2dce1e24146e3b7e0261736aaeaa22a" + "sha256:339dc09518b07e2fa7eda5450740925974815557727d6bd35d319c1524a04a4c", + "sha256:6d58c986d22b038c8c0df30d639f23a3e6d172a05c3583e766f4c0b785c0986a" ], - "version": "==2018.8.24" + "version": "==2018.10.15" }, "cffi": { "hashes": [ @@ -97,6 +97,13 @@ ], "version": "==7.0" }, + "croniter": { + "hashes": [ + "sha256:64d5f8c719249694265190810ef2f051345007246c99a3879a35b393d593d668", + "sha256:8ce5e4edd6f1956e70c8a31211cf86a7859aa1f0ff256107723582d79238e002" + ], + "version": "==0.3.25" + }, "cryptography": { "hashes": [ "sha256:02602e1672b62e803e08617ec286041cc453e8d43f093a5f4162095506bc0beb", @@ -144,6 +151,14 @@ "index": "pypi", "version": "==0.12" }, + "flask-rq2": { + "hashes": [ + "sha256:83e28f0279828198e64e1ed52a43fd6b530d9192d9944f4dea30e99e5688c9de", + "sha256:d513aa8d3b91eda34091ed8e40d55655e3acff6d37d57197417b14839a066185" + ], + "index": "pypi", + "version": "==18.1" + }, "flask-session": { "hashes": [ "sha256:a31c27e0c3287f00c825b3d9625aba585f4df4cccedb1e7dd5a69a215881a731", @@ -313,6 +328,20 @@ "index": "pypi", "version": "==2.19.1" }, + "rq": { + "hashes": [ + "sha256:5dd83625ca64b0dbf668ee65a8d38f3f5132aa9b64de4d813ff76f97db194b60", + "sha256:7ac5989a27bdff713dd40517498c1b3bf720f8ebc47305055496f653a29da899" + ], + "version": "==0.12.0" + }, + "rq-scheduler": { + "hashes": [ + "sha256:6cad6b6d29eae55d4585e2ac9be3b8a36b3f18c87a494fc508a4fa19b9c845d6", + "sha256:fc51da3d4ad1a047cada3b97a96afea21a3102ea5aa5b79ed2ea97d8ffdf8821" + ], + "version": "==0.8.3" + }, "six": { "hashes": [ "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", @@ -453,7 +482,6 @@ "sha256:2105ee183c51fed27e2b6801029b3903f5c2774c78e3f53bd920ca468d0f5679", "sha256:236505d15af6c7b7bfe2a9485db4b2bdea21d9239351483326184314418c79a8", "sha256:237284425271db4f30d458b355decf388ab20b05278bdf8dc9a65de0973726c6", - "sha256:2619f0369412e22f01ad8f5bea503f15fb099a5eef3c31f1edb81dcb29221bf7", "sha256:26d8eea4c840b73c61a1081d68bceb57b21a2d4f7afda6cac8ac38cb05226b00", "sha256:39a3740f7721155f4269aedf67b211101c07bd2111b334dfd69b807156ab15d9", "sha256:4bd0c42db8efc8a60965769796d43a5570906a870bc819f7388860aa72779d1b", @@ -462,26 +490,19 @@ "sha256:5415cafb082dad78935b3045c2e5d8907f436d15ad24c3fdb8e1839e084e4961", "sha256:5631f1983074b33c35dbb84607f337b9d7e9808116d7f0f2cb7b9d6d4381d50e", "sha256:5e9249bc361cd22565fd98590a53fd25a3dd666b74791ed7237fa99de938bbed", - "sha256:61ad080b78287e8a10ae485a194fc552625d4ed4196ab32cc8987e61bdcceb0f", "sha256:6a48746154f1331f28ef9e889c625b5b15a36cb86dd8021b4bdd1180a2186aa5", "sha256:71d376dbac64855ed693bc1ca121794570fe603e8783cdfa304ec6825d4e768f", "sha256:749ebd8a615337747592bd1523dfc4af7199b2bf6403b55f96c728668aeff91f", - "sha256:8575f3e1a12eae8d2fd3935dcc6fad2d5a7cf32bc15150a69d3bede229e970d5", "sha256:8ec528b585b95234e9c0c31dcd0a89152d8ed82b4567aa62dbcb3e9a0600deee", "sha256:a1a9ccd879811437ca0307c914f136d6edb85bd0470e6d4966c6397927bcabd9", "sha256:abd956c334752776230b779537d911a5a12fcb69d8fd3fe332ae63a140301ae6", "sha256:ad18f836017f2e8881145795f483636564807aaed54223459915a0d4735300cf", "sha256:b07ac0b1533298ddbc54c9bf3464664895f22899fec027b8d6c8d3ac59023283", - "sha256:c3ae3527c72581595952977c1b391b9e7313d236216581099ee38e4240d997fe", - "sha256:d5309c5c6750ff882d47c0d4d5952d2384232e522db56d2bb63beb01dcb07f46", "sha256:d9385f1445e30e8e42b75a36a7899ea1fd0f5784233a626625d70f9b087de404", "sha256:db2d1fcd32dbeeb914b2660af1838e9c178b75173f95fd221b1f9410b5d3ef1d", "sha256:e1dec211147f1fd7cb7a0f9a96aeeca467a5af02d38911307b3b8c2324f9917e", - "sha256:e20f11023ab77ad08dcdbf3a740e2512f73ebfbbfcb4f08f0b8a8f65f98210a2", - "sha256:e2cc3fc55566990059afb0f06141e136095898b55e977af66d0b498415098792", "sha256:e96dffc1fa57bb8c1c238f3d989341a97302492d09cb11f77df031112621c35c", - "sha256:ed4d97eb0ecdee29d0748acd84e6380729f78ce5ba0c7fe3401801634c25a1c5", - "sha256:eecc9d908a22a97356a1033d756281cd8c37285430f047cb35458d1bc8e6f8de" + "sha256:ed4d97eb0ecdee29d0748acd84e6380729f78ce5ba0c7fe3401801634c25a1c5" ], "version": "==5.0a3" }, @@ -535,6 +556,14 @@ ], "version": "==2.1.11" }, + "honcho": { + "hashes": [ + "sha256:af5806bf13e3b20acdcb9ff8c0beb91eee6fe07393c3448dfad89667e6ac7576", + "sha256:c189402ad2e337777283c6a12d0f4f61dc6dd20c254c9a3a4af5087fc66cea6e" + ], + "index": "pypi", + "version": "==1.0.1" + }, "ipdb": { "hashes": [ "sha256:7081c65ed7bfe7737f83fa4213ca8afd9617b42ff6b3f1daf9a3419839a2a00a" @@ -766,15 +795,11 @@ }, "pyyaml": { "hashes": [ - "sha256:1cbc199009e78f92d9edf554be4fe40fb7b0bef71ba688602a00e97a51909110", "sha256:254bf6fda2b7c651837acb2c718e213df29d531eebf00edb54743d10bcb694eb", "sha256:3108529b78577327d15eec243f0ff348a0640b0c3478d67ad7f5648f93bac3e2", "sha256:3c17fb92c8ba2f525e4b5f7941d850e7a48c3a59b32d331e2502a3cdc6648e76", - "sha256:6f89b5c95e93945b597776163403d47af72d243f366bf4622ff08bdfd1c950b7", "sha256:8d6d96001aa7f0a6a4a95e8143225b5d06e41b1131044913fecb8f85a125714b", - "sha256:be622cc81696e24d0836ba71f6272a2b5767669b0d79fdcf0295d51ac2e156c8", - "sha256:c8a88edd93ee29ede719080b2be6cb2333dfee1dccba213b422a9c8e97f2967b", - "sha256:f39411e380e2182ad33be039e8ee5770a5d9efe01a2bfb7ae58d9ba31c4a2a9d" + "sha256:c8a88edd93ee29ede719080b2be6cb2333dfee1dccba213b422a9c8e97f2967b" ], "version": "==4.2b4" }, @@ -823,8 +848,7 @@ "toml": { "hashes": [ "sha256:229f81c57791a41d65e399fc06bf0848bab550a9dfd5ed66df18ce5f05e73d5c", - "sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e", - "sha256:f1db651f9657708513243e61e6cc67d101a39bad662eaa9b5546f789338e07a3" + "sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e" ], "version": "==0.10.0" }, diff --git a/Procfile b/Procfile new file mode 100644 index 00000000..ba8ec7cf --- /dev/null +++ b/Procfile @@ -0,0 +1,3 @@ +assets: yarn watch +web: PORT=8000 python app.py +queue: ./script/dev_queue diff --git a/README.md b/README.md index a0c82d63..e5adfcfc 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,10 @@ locally: running on the default port of 6379. You can ensure that Redis is running by executing `redis-cli` with no options and ensuring a connection is succesfully made. +* [`entr`](http://www.entrproject.org/) + This dependency is optional. If present, the queue worker process will hot + reload in development. + ### Cloning This project contains git submodules. Here is an example clone command that will automatically initialize and update those modules: @@ -77,13 +81,9 @@ virtualenvs for you when you enter and leave the directory. To start the app locally in the foreground and watch for changes: - script/dev_server + script/server -To watch for changes to any js/css assets: - - yarn watch - -After running `script/dev_server`, the application is available at +After running `script/server`, the application is available at [`http://localhost:8000`](http://localhost:8000). @@ -111,6 +111,22 @@ projects for all of the test users: `pipenv run python script/seed_sample.py` +### Email Notifications + +To send email, the following configuration values must be set: + +``` +MAIL_SERVER = +MAIL_PORT = +MAIL_SENDER = +MAIL_PASSWORD = +MAIL_TLS = +``` + +When the `DEBUG` environment variable is enabled and the app environment is not +set to production, sent email messages are available at the `/messages` endpoint. +Emails are not sent in development and test modes. + ## Testing Tests require a test database: diff --git a/atst/app.py b/atst/app.py index 0cc2ca13..ac3ed9b3 100644 --- a/atst/app.py +++ b/atst/app.py @@ -23,6 +23,8 @@ from atst.domain.authz import Authorization from atst.models.permissions import Permissions from atst.eda_client import MockEDAClient from atst.uploader import Uploader +from atst.utils import mailer +from atst.queue import queue ENV = os.getenv("FLASK_ENV", "dev") @@ -37,17 +39,19 @@ def make_app(config): template_folder=parent_dir.child("templates").absolute(), static_folder=parent_dir.child("static").absolute(), ) - redis = make_redis(config) + make_redis(app, config) csrf = CSRFProtect() app.config.update(config) - app.config.update({"SESSION_REDIS": redis}) + app.config.update({"SESSION_REDIS": app.redis}) make_flask_callbacks(app) make_crl_validator(app) register_filters(app) make_eda_client(app) make_upload_storage(app) + make_mailer(app) + queue.init_app(app) db.init_app(app) csrf.init_app(app) @@ -87,7 +91,7 @@ def map_config(config): return { **config["default"], "ENV": config["default"]["ENVIRONMENT"], - "DEBUG": config["default"]["DEBUG"], + "DEBUG": config["default"].getboolean("DEBUG"), "PORT": int(config["default"]["PORT"]), "SQLALCHEMY_DATABASE_URI": config["default"]["DATABASE_URI"], "SQLALCHEMY_TRACK_MODIFICATIONS": False, @@ -95,6 +99,8 @@ def map_config(config): "PERMANENT_SESSION_LIFETIME": config.getint( "default", "PERMANENT_SESSION_LIFETIME" ), + "RQ_REDIS_URL": config["default"]["REDIS_URI"], + "RQ_QUEUES": ["atat_{}".format(ENV.lower())], } @@ -143,8 +149,9 @@ def make_config(): return map_config(config) -def make_redis(config): - return redis.Redis.from_url(config["REDIS_URI"]) +def make_redis(app, config): + r = redis.Redis.from_url(config["REDIS_URI"]) + app.redis = r def make_crl_validator(app): @@ -166,3 +173,18 @@ def make_upload_storage(app): secret=app.config.get("STORAGE_SECRET"), ) app.uploader = uploader + + +def make_mailer(app): + if app.config["DEBUG"]: + mailer_connection = mailer.RedisConnection(app.redis) + else: + mailer_connection = mailer.SMTPConnection( + server=app.config.get("MAIL_SERVER"), + port=app.config.get("MAIL_PORT"), + username=app.config.get("MAIL_SENDER"), + password=app.config.get("MAIL_PASSWORD"), + use_tls=app.config.get("MAIL_TLS"), + ) + sender = app.config.get("MAIL_SENDER") + app.mailer = mailer.Mailer(mailer_connection, sender) diff --git a/atst/queue.py b/atst/queue.py new file mode 100644 index 00000000..e036a642 --- /dev/null +++ b/atst/queue.py @@ -0,0 +1,45 @@ +from flask_rq2 import RQ +from flask import current_app as app + + +class ATSTQueue(RQ): + + """Internal helpers to get the queue that actually does the work. + + The RQ object always uses the "default" queue, unless we explicitly request + otherwise. These helpers allow us to use `.queue_name` to get the name of + the configured queue and `_queue_job` will use the appropriate queue. + + """ + + @property + def queue_name(self): + return self.queues[0] + + def get_queue(self, name=None): + if not name: + name = self.queue_name + return super().get_queue(name) + + def _queue_job(self, function, *args, **kwargs): + self.get_queue().enqueue(function, *args, **kwargs) + + # pylint: disable=pointless-string-statement + """Instance methods to queue up application-specific jobs.""" + + def send_mail(self, to, subject, body): + self._queue_job(ATSTQueue._send_mail, to, subject, body) + + # pylint: disable=pointless-string-statement + """Class methods to actually perform the work. + + Must be a class method (or a module-level function) because we being able + to pickle the class is more effort than its worth. + """ + + @classmethod + def _send_mail(self, to, subject, body): + app.mailer.send(to, subject, body) + + +queue = ATSTQueue() diff --git a/atst/routes/dev.py b/atst/routes/dev.py index e9b107e8..0d90397f 100644 --- a/atst/routes/dev.py +++ b/atst/routes/dev.py @@ -1,7 +1,16 @@ -from flask import Blueprint, request, session, redirect +from flask import ( + Blueprint, + request, + session, + redirect, + render_template, + url_for, + current_app as app, +) from . import redirect_after_login_url from atst.domain.users import Users +from atst.queue import queue bp = Blueprint("dev", __name__) @@ -65,3 +74,17 @@ def login_dev(): session["user_id"] = user.id return redirect(redirect_after_login_url()) + + +@bp.route("/test-email") +def test_email(): + queue.send_mail( + [request.args.get("to")], request.args.get("subject"), request.args.get("body") + ) + + return redirect(url_for("dev.messages")) + + +@bp.route("/messages") +def messages(): + return render_template("dev/emails.html", messages=app.mailer.messages) diff --git a/atst/utils.py b/atst/utils/__init__.py similarity index 100% rename from atst/utils.py rename to atst/utils/__init__.py diff --git a/atst/utils/mailer.py b/atst/utils/mailer.py new file mode 100644 index 00000000..a67a851b --- /dev/null +++ b/atst/utils/mailer.py @@ -0,0 +1,85 @@ +from contextlib import contextmanager +import smtplib +from email.message import EmailMessage + + +class MailConnection(object): + def send(self, message): + raise NotImplementedError() + + @property + def messages(self): + raise NotImplementedError() + + +class SMTPConnection(MailConnection): + def __init__(self, server, port, username, password, use_tls=False): + self.server = server + self.port = port + self.username = username + self.password = password + self.use_tls = use_tls + + @contextmanager + def _connected_host(self): + host = None + + if self.use_tls: + host = smtplib.SMTP(self.server, self.port) + host.starttls() + else: + host = smtplib.SMTP_SSL(self.server, self.port) + + host.login(self.username, self.password) + + yield host + + host.quit() + + @property + def messages(self): + return [] + + def send(self, message): + with self._connected_host() as host: + host.send_message(message) + + +class RedisConnection(MailConnection): + def __init__(self, redis, **kwargs): + super().__init__(**kwargs) + self.redis = redis + self._reset() + + def _reset(self): + self.redis.delete("atat_inbox") + + @property + def messages(self): + return [msg.decode() for msg in self.redis.lrange("atat_inbox", 0, -1)] + + def send(self, message): + self.redis.lpush("atat_inbox", str(message)) + + +class Mailer(object): + def __init__(self, connection, sender): + self.connection = connection + self.sender = sender + + def _build_message(self, recipients, subject, body): + msg = EmailMessage() + msg.set_content(body) + msg["From"] = self.sender + msg["To"] = ", ".join(recipients) + msg["Subject"] = subject + + return msg + + def send(self, recipients, subject, body): + message = self._build_message(recipients, subject, body) + self.connection.send(message) + + @property + def messages(self): + return self.connection.messages diff --git a/script/dev_queue b/script/dev_queue new file mode 100755 index 00000000..db171e3e --- /dev/null +++ b/script/dev_queue @@ -0,0 +1,11 @@ +#!/bin/bash + +# script/dev_queue: Run the queue with entr if available + +set -e + +if [[ `command -v entr` ]]; then + find atst | entr -r flask rq worker +else + flask rq worker +fi diff --git a/script/dev_server b/script/dev_server deleted file mode 100755 index 1f5f75a1..00000000 --- a/script/dev_server +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash - -# script/dev_server: Launch a local dev version of the server in the background - -# -# WIP -# - -source "$(dirname "${0}")"/../script/include/global_header.inc.sh - -# Create a function to run after a trap is triggered -reap() { - kill -s SIGTERM -- "-$$" - sleep 0.1 - exit -} - -# Register trapping of SIGTERM and SIGINT -trap reap SIGTERM SIGINT - -# Display the script PID, which will also be the process group ID for all -# child processes -echo "Process Group: $$" - -# Set server launch related environment variables -DEBUG=1 -LAUNCH_ARGS="$*" - - -# Launch the app -source ./script/server & -wait diff --git a/script/server b/script/server index c03cfdd5..1fc46774 100755 --- a/script/server +++ b/script/server @@ -4,8 +4,5 @@ source "$(dirname "${0}")"/../script/include/global_header.inc.sh -# Compile js/css assets -yarn build - # Launch the app -run_command "./app.py ${LAUNCH_ARGS}" +run_command "honcho start" diff --git a/templates/dev/emails.html b/templates/dev/emails.html new file mode 100644 index 00000000..b07f6b46 --- /dev/null +++ b/templates/dev/emails.html @@ -0,0 +1,6 @@ +{% for msg in messages %} +
+{{ msg }} +
+
+{% endfor %} diff --git a/tests/conftest.py b/tests/conftest.py index 947f054d..50a555e7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,6 +10,7 @@ from tempfile import TemporaryDirectory from atst.app import make_app, make_config from atst.database import db as _db from atst.domain.auth import logout +from atst.queue import queue import tests.factories as factories from tests.mocks import PDF_FILENAME diff --git a/tests/test_queue.py b/tests/test_queue.py new file mode 100644 index 00000000..da6069d4 --- /dev/null +++ b/tests/test_queue.py @@ -0,0 +1,17 @@ +import pytest +from atst.queue import queue + +# ensure queue is always empty for unit testing +@pytest.fixture(scope="function", autouse=True) +def reset_queue(): + queue.get_queue().empty() + yield + queue.get_queue().empty() + + +def test_send_mail(): + initial = len(queue.get_queue()) + queue.send_mail( + ["lordvader@geocities.net"], "death start", "how is it coming along?" + ) + assert len(queue.get_queue()) == initial + 1 diff --git a/tests/utils/test_mailer.py b/tests/utils/test_mailer.py new file mode 100644 index 00000000..8b40ea48 --- /dev/null +++ b/tests/utils/test_mailer.py @@ -0,0 +1,48 @@ +import pytest +from atst.utils.mailer import Mailer, Mailer, MailConnection, RedisConnection + + +class MockConnection(MailConnection): + def __init__(self): + self._messages = [] + + def send(self, message): + self._messages.append(message) + + @property + def messages(self): + return self._messages + + +@pytest.fixture +def mailer(): + return Mailer(MockConnection(), "test@atat.com") + + +def test_mailer_can_send_mail(mailer): + message_data = { + "recipients": ["ben@tattoine.org"], + "subject": "help", + "body": "you're my only hope", + } + mailer.send(**message_data) + assert len(mailer.messages) == 1 + message = mailer.messages[0] + assert message["To"] == message_data["recipients"][0] + assert message["Subject"] == message_data["subject"] + assert message.get_content().strip() == message_data["body"] + + +def test_redis_mailer_can_save_messages(app): + mailer = Mailer(RedisConnection(app.redis), "test@atat.com") + message_data = { + "recipients": ["ben@tattoine.org"], + "subject": "help", + "body": "you're my only hope", + } + mailer.send(**message_data) + assert len(mailer.messages) == 1 + message = mailer.messages[0] + assert message_data["recipients"][0] in message + assert message_data["subject"] in message + assert message_data["body"] in message