Merge pull request #384 from dod-ccpo/mailer

Mailer

Pipfile

@@ -21,6 +21,7 @@ requests = "*"
 apache-libcloud = "*"
 lockfile = "*"
 defusedxml = "*"
+"flask-rq2" = "*"
 
 [dev-packages]
 bandit = "*"
@@ -35,6 +36,7 @@ pytest-flask = "*"
 pytest-env = "*"
 pytest-cov = "*"
 selenium = "*"
+honcho = "*"
 
 [requires]
 python_version = "3.6.6"

Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "1e5e6a695229166aaa5e6c427fed07a903766e9b3d24981a19cc8e5ada8db978"
+            "sha256": "c67f5a847351d9d6e8ef165c380dd97fdf623f87cf8299a64109e453027e2458"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -41,10 +41,10 @@
         },
         "certifi": {
             "hashes": [
-                "sha256:376690d6f16d32f9d1fe8932551d80b23e9d393a8578c5633a2ed39a64861638",
+                "sha256:339dc09518b07e2fa7eda5450740925974815557727d6bd35d319c1524a04a4c",
-                "sha256:456048c7e371c089d0a77a5212fb37a2c2dce1e24146e3b7e0261736aaeaa22a"
+                "sha256:6d58c986d22b038c8c0df30d639f23a3e6d172a05c3583e766f4c0b785c0986a"
             ],
-            "version": "==2018.8.24"
+            "version": "==2018.10.15"
         },
         "cffi": {
             "hashes": [
@@ -97,6 +97,13 @@
             ],
             "version": "==7.0"
         },
+        "croniter": {
+            "hashes": [
+                "sha256:64d5f8c719249694265190810ef2f051345007246c99a3879a35b393d593d668",
+                "sha256:8ce5e4edd6f1956e70c8a31211cf86a7859aa1f0ff256107723582d79238e002"
+            ],
+            "version": "==0.3.25"
+        },
         "cryptography": {
             "hashes": [
                 "sha256:02602e1672b62e803e08617ec286041cc453e8d43f093a5f4162095506bc0beb",
@@ -144,6 +151,14 @@
             "index": "pypi",
             "version": "==0.12"
         },
+        "flask-rq2": {
+            "hashes": [
+                "sha256:83e28f0279828198e64e1ed52a43fd6b530d9192d9944f4dea30e99e5688c9de",
+                "sha256:d513aa8d3b91eda34091ed8e40d55655e3acff6d37d57197417b14839a066185"
+            ],
+            "index": "pypi",
+            "version": "==18.1"
+        },
         "flask-session": {
             "hashes": [
                 "sha256:a31c27e0c3287f00c825b3d9625aba585f4df4cccedb1e7dd5a69a215881a731",
@@ -313,6 +328,20 @@
             "index": "pypi",
             "version": "==2.19.1"
         },
+        "rq": {
+            "hashes": [
+                "sha256:5dd83625ca64b0dbf668ee65a8d38f3f5132aa9b64de4d813ff76f97db194b60",
+                "sha256:7ac5989a27bdff713dd40517498c1b3bf720f8ebc47305055496f653a29da899"
+            ],
+            "version": "==0.12.0"
+        },
+        "rq-scheduler": {
+            "hashes": [
+                "sha256:6cad6b6d29eae55d4585e2ac9be3b8a36b3f18c87a494fc508a4fa19b9c845d6",
+                "sha256:fc51da3d4ad1a047cada3b97a96afea21a3102ea5aa5b79ed2ea97d8ffdf8821"
+            ],
+            "version": "==0.8.3"
+        },
         "six": {
             "hashes": [
                 "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9",
@@ -453,7 +482,6 @@
                 "sha256:2105ee183c51fed27e2b6801029b3903f5c2774c78e3f53bd920ca468d0f5679",
                 "sha256:236505d15af6c7b7bfe2a9485db4b2bdea21d9239351483326184314418c79a8",
                 "sha256:237284425271db4f30d458b355decf388ab20b05278bdf8dc9a65de0973726c6",
-                "sha256:2619f0369412e22f01ad8f5bea503f15fb099a5eef3c31f1edb81dcb29221bf7",
                 "sha256:26d8eea4c840b73c61a1081d68bceb57b21a2d4f7afda6cac8ac38cb05226b00",
                 "sha256:39a3740f7721155f4269aedf67b211101c07bd2111b334dfd69b807156ab15d9",
                 "sha256:4bd0c42db8efc8a60965769796d43a5570906a870bc819f7388860aa72779d1b",
@@ -462,26 +490,19 @@
                 "sha256:5415cafb082dad78935b3045c2e5d8907f436d15ad24c3fdb8e1839e084e4961",
                 "sha256:5631f1983074b33c35dbb84607f337b9d7e9808116d7f0f2cb7b9d6d4381d50e",
                 "sha256:5e9249bc361cd22565fd98590a53fd25a3dd666b74791ed7237fa99de938bbed",
-                "sha256:61ad080b78287e8a10ae485a194fc552625d4ed4196ab32cc8987e61bdcceb0f",
                 "sha256:6a48746154f1331f28ef9e889c625b5b15a36cb86dd8021b4bdd1180a2186aa5",
                 "sha256:71d376dbac64855ed693bc1ca121794570fe603e8783cdfa304ec6825d4e768f",
                 "sha256:749ebd8a615337747592bd1523dfc4af7199b2bf6403b55f96c728668aeff91f",
-                "sha256:8575f3e1a12eae8d2fd3935dcc6fad2d5a7cf32bc15150a69d3bede229e970d5",
                 "sha256:8ec528b585b95234e9c0c31dcd0a89152d8ed82b4567aa62dbcb3e9a0600deee",
                 "sha256:a1a9ccd879811437ca0307c914f136d6edb85bd0470e6d4966c6397927bcabd9",
                 "sha256:abd956c334752776230b779537d911a5a12fcb69d8fd3fe332ae63a140301ae6",
                 "sha256:ad18f836017f2e8881145795f483636564807aaed54223459915a0d4735300cf",
                 "sha256:b07ac0b1533298ddbc54c9bf3464664895f22899fec027b8d6c8d3ac59023283",
-                "sha256:c3ae3527c72581595952977c1b391b9e7313d236216581099ee38e4240d997fe",
-                "sha256:d5309c5c6750ff882d47c0d4d5952d2384232e522db56d2bb63beb01dcb07f46",
                 "sha256:d9385f1445e30e8e42b75a36a7899ea1fd0f5784233a626625d70f9b087de404",
                 "sha256:db2d1fcd32dbeeb914b2660af1838e9c178b75173f95fd221b1f9410b5d3ef1d",
                 "sha256:e1dec211147f1fd7cb7a0f9a96aeeca467a5af02d38911307b3b8c2324f9917e",
-                "sha256:e20f11023ab77ad08dcdbf3a740e2512f73ebfbbfcb4f08f0b8a8f65f98210a2",
-                "sha256:e2cc3fc55566990059afb0f06141e136095898b55e977af66d0b498415098792",
                 "sha256:e96dffc1fa57bb8c1c238f3d989341a97302492d09cb11f77df031112621c35c",
-                "sha256:ed4d97eb0ecdee29d0748acd84e6380729f78ce5ba0c7fe3401801634c25a1c5",
+                "sha256:ed4d97eb0ecdee29d0748acd84e6380729f78ce5ba0c7fe3401801634c25a1c5"
-                "sha256:eecc9d908a22a97356a1033d756281cd8c37285430f047cb35458d1bc8e6f8de"
             ],
             "version": "==5.0a3"
         },
@@ -535,6 +556,14 @@
             ],
             "version": "==2.1.11"
         },
+        "honcho": {
+            "hashes": [
+                "sha256:af5806bf13e3b20acdcb9ff8c0beb91eee6fe07393c3448dfad89667e6ac7576",
+                "sha256:c189402ad2e337777283c6a12d0f4f61dc6dd20c254c9a3a4af5087fc66cea6e"
+            ],
+            "index": "pypi",
+            "version": "==1.0.1"
+        },
         "ipdb": {
             "hashes": [
                 "sha256:7081c65ed7bfe7737f83fa4213ca8afd9617b42ff6b3f1daf9a3419839a2a00a"
@@ -766,15 +795,11 @@
         },
         "pyyaml": {
             "hashes": [
-                "sha256:1cbc199009e78f92d9edf554be4fe40fb7b0bef71ba688602a00e97a51909110",
                 "sha256:254bf6fda2b7c651837acb2c718e213df29d531eebf00edb54743d10bcb694eb",
                 "sha256:3108529b78577327d15eec243f0ff348a0640b0c3478d67ad7f5648f93bac3e2",
                 "sha256:3c17fb92c8ba2f525e4b5f7941d850e7a48c3a59b32d331e2502a3cdc6648e76",
-                "sha256:6f89b5c95e93945b597776163403d47af72d243f366bf4622ff08bdfd1c950b7",
                 "sha256:8d6d96001aa7f0a6a4a95e8143225b5d06e41b1131044913fecb8f85a125714b",
-                "sha256:be622cc81696e24d0836ba71f6272a2b5767669b0d79fdcf0295d51ac2e156c8",
+                "sha256:c8a88edd93ee29ede719080b2be6cb2333dfee1dccba213b422a9c8e97f2967b"
-                "sha256:c8a88edd93ee29ede719080b2be6cb2333dfee1dccba213b422a9c8e97f2967b",
-                "sha256:f39411e380e2182ad33be039e8ee5770a5d9efe01a2bfb7ae58d9ba31c4a2a9d"
             ],
             "version": "==4.2b4"
         },
@@ -823,8 +848,7 @@
         "toml": {
             "hashes": [
                 "sha256:229f81c57791a41d65e399fc06bf0848bab550a9dfd5ed66df18ce5f05e73d5c",
-                "sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e",
+                "sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e"
-                "sha256:f1db651f9657708513243e61e6cc67d101a39bad662eaa9b5546f789338e07a3"
             ],
             "version": "==0.10.0"
         },

Procfile

@@ -0,0 +1,3 @@
+assets: yarn watch
+web: PORT=8000 python app.py
+queue: ./script/dev_queue

README.md

@@ -39,6 +39,10 @@ locally:
   running on the default port of 6379. You can ensure that Redis is running by
   executing `redis-cli` with no options and ensuring a connection is succesfully made.
 
+* [`entr`](http://www.entrproject.org/)
+  This dependency is optional. If present, the queue worker process will hot
+  reload in development.
+
 ### Cloning
 This project contains git submodules. Here is an example clone command that will
 automatically initialize and update those modules:
@@ -77,13 +81,9 @@ virtualenvs for you when you enter and leave the directory.
 
 To start the app locally in the foreground and watch for changes:
 
-    script/dev_server
+    script/server
 
-To watch for changes to any js/css assets:
+After running `script/server`, the application is available at
-
-    yarn watch
-
-After running `script/dev_server`, the application is available at
 [`http://localhost:8000`](http://localhost:8000).
 
 
@@ -111,6 +111,22 @@ projects for all of the test users:
 
 `pipenv run python script/seed_sample.py`
 
+### Email Notifications
+
+To send email, the following configuration values must be set:
+
+```
+MAIL_SERVER = <SMTP server URL>
+MAIL_PORT = <SMTP server port>
+MAIL_SENDER = <Login name for the email account and sender address>
+MAIL_PASSWORD = <login password for the email account>
+MAIL_TLS = <Boolean, whether TLS should be enabled for outgoing email. Defaults to false.>
+```
+
+When the `DEBUG` environment variable is enabled and the app environment is not
+set to production, sent email messages are available at the `/messages` endpoint.
+Emails are not sent in development and test modes.
+
 ## Testing
 
 Tests require a test database:

atst/app.py

@@ -23,6 +23,8 @@ from atst.domain.authz import Authorization
 from atst.models.permissions import Permissions
 from atst.eda_client import MockEDAClient
 from atst.uploader import Uploader
+from atst.utils import mailer
+from atst.queue import queue
 
 
 ENV = os.getenv("FLASK_ENV", "dev")
@@ -37,17 +39,19 @@ def make_app(config):
         template_folder=parent_dir.child("templates").absolute(),
         static_folder=parent_dir.child("static").absolute(),
     )
-    redis = make_redis(config)
+    make_redis(app, config)
     csrf = CSRFProtect()
 
     app.config.update(config)
-    app.config.update({"SESSION_REDIS": redis})
+    app.config.update({"SESSION_REDIS": app.redis})
 
     make_flask_callbacks(app)
     make_crl_validator(app)
     register_filters(app)
     make_eda_client(app)
     make_upload_storage(app)
+    make_mailer(app)
+    queue.init_app(app)
 
     db.init_app(app)
     csrf.init_app(app)
@@ -87,7 +91,7 @@ def map_config(config):
     return {
         **config["default"],
         "ENV": config["default"]["ENVIRONMENT"],
-        "DEBUG": config["default"]["DEBUG"],
+        "DEBUG": config["default"].getboolean("DEBUG"),
         "PORT": int(config["default"]["PORT"]),
         "SQLALCHEMY_DATABASE_URI": config["default"]["DATABASE_URI"],
         "SQLALCHEMY_TRACK_MODIFICATIONS": False,
@@ -95,6 +99,8 @@ def map_config(config):
         "PERMANENT_SESSION_LIFETIME": config.getint(
             "default", "PERMANENT_SESSION_LIFETIME"
         ),
+        "RQ_REDIS_URL": config["default"]["REDIS_URI"],
+        "RQ_QUEUES": ["atat_{}".format(ENV.lower())],
     }
 
 
@@ -143,8 +149,9 @@ def make_config():
     return map_config(config)
 
 
-def make_redis(config):
+def make_redis(app, config):
-    return redis.Redis.from_url(config["REDIS_URI"])
+    r = redis.Redis.from_url(config["REDIS_URI"])
+    app.redis = r
 
 
 def make_crl_validator(app):
@@ -166,3 +173,18 @@ def make_upload_storage(app):
         secret=app.config.get("STORAGE_SECRET"),
     )
     app.uploader = uploader
+
+
+def make_mailer(app):
+    if app.config["DEBUG"]:
+        mailer_connection = mailer.RedisConnection(app.redis)
+    else:
+        mailer_connection = mailer.SMTPConnection(
+            server=app.config.get("MAIL_SERVER"),
+            port=app.config.get("MAIL_PORT"),
+            username=app.config.get("MAIL_SENDER"),
+            password=app.config.get("MAIL_PASSWORD"),
+            use_tls=app.config.get("MAIL_TLS"),
+        )
+    sender = app.config.get("MAIL_SENDER")
+    app.mailer = mailer.Mailer(mailer_connection, sender)

atst/queue.py

@@ -0,0 +1,45 @@
+from flask_rq2 import RQ
+from flask import current_app as app
+
+
+class ATSTQueue(RQ):
+
+    """Internal helpers to get the queue that actually does the work.
+
+    The RQ object always uses the "default" queue, unless we explicitly request
+    otherwise. These helpers allow us to use `.queue_name` to get the name of
+    the configured queue and `_queue_job` will use the appropriate queue.
+
+    """
+
+    @property
+    def queue_name(self):
+        return self.queues[0]
+
+    def get_queue(self, name=None):
+        if not name:
+            name = self.queue_name
+        return super().get_queue(name)
+
+    def _queue_job(self, function, *args, **kwargs):
+        self.get_queue().enqueue(function, *args, **kwargs)
+
+    # pylint: disable=pointless-string-statement
+    """Instance methods to queue up application-specific jobs."""
+
+    def send_mail(self, to, subject, body):
+        self._queue_job(ATSTQueue._send_mail, to, subject, body)
+
+    # pylint: disable=pointless-string-statement
+    """Class methods to actually perform the work.
+
+    Must be a class method (or a module-level function) because we being able
+    to pickle the class is more effort than its worth.
+    """
+
+    @classmethod
+    def _send_mail(self, to, subject, body):
+        app.mailer.send(to, subject, body)
+
+
+queue = ATSTQueue()

atst/routes/dev.py

@@ -1,7 +1,16 @@
-from flask import Blueprint, request, session, redirect
+from flask import (
+    Blueprint,
+    request,
+    session,
+    redirect,
+    render_template,
+    url_for,
+    current_app as app,
+)
 
 from . import redirect_after_login_url
 from atst.domain.users import Users
+from atst.queue import queue
 
 bp = Blueprint("dev", __name__)
 
@@ -65,3 +74,17 @@ def login_dev():
     session["user_id"] = user.id
 
     return redirect(redirect_after_login_url())
+
+
+@bp.route("/test-email")
+def test_email():
+    queue.send_mail(
+        [request.args.get("to")], request.args.get("subject"), request.args.get("body")
+    )
+
+    return redirect(url_for("dev.messages"))
+
+
+@bp.route("/messages")
+def messages():
+    return render_template("dev/emails.html", messages=app.mailer.messages)

atst/utils/mailer.py

@@ -0,0 +1,85 @@
+from contextlib import contextmanager
+import smtplib
+from email.message import EmailMessage
+
+
+class MailConnection(object):
+    def send(self, message):
+        raise NotImplementedError()
+
+    @property
+    def messages(self):
+        raise NotImplementedError()
+
+
+class SMTPConnection(MailConnection):
+    def __init__(self, server, port, username, password, use_tls=False):
+        self.server = server
+        self.port = port
+        self.username = username
+        self.password = password
+        self.use_tls = use_tls
+
+    @contextmanager
+    def _connected_host(self):
+        host = None
+
+        if self.use_tls:
+            host = smtplib.SMTP(self.server, self.port)
+            host.starttls()
+        else:
+            host = smtplib.SMTP_SSL(self.server, self.port)
+
+        host.login(self.username, self.password)
+
+        yield host
+
+        host.quit()
+
+    @property
+    def messages(self):
+        return []
+
+    def send(self, message):
+        with self._connected_host() as host:
+            host.send_message(message)
+
+
+class RedisConnection(MailConnection):
+    def __init__(self, redis, **kwargs):
+        super().__init__(**kwargs)
+        self.redis = redis
+        self._reset()
+
+    def _reset(self):
+        self.redis.delete("atat_inbox")
+
+    @property
+    def messages(self):
+        return [msg.decode() for msg in self.redis.lrange("atat_inbox", 0, -1)]
+
+    def send(self, message):
+        self.redis.lpush("atat_inbox", str(message))
+
+
+class Mailer(object):
+    def __init__(self, connection, sender):
+        self.connection = connection
+        self.sender = sender
+
+    def _build_message(self, recipients, subject, body):
+        msg = EmailMessage()
+        msg.set_content(body)
+        msg["From"] = self.sender
+        msg["To"] = ", ".join(recipients)
+        msg["Subject"] = subject
+
+        return msg
+
+    def send(self, recipients, subject, body):
+        message = self._build_message(recipients, subject, body)
+        self.connection.send(message)
+
+    @property
+    def messages(self):
+        return self.connection.messages

script/dev_queue

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# script/dev_queue: Run the queue with entr if available
+
+set -e
+
+if [[ `command -v entr` ]]; then
+  find atst | entr -r flask rq worker
+else
+  flask rq worker
+fi

script/dev_server

@@ -1,32 +0,0 @@
-#!/bin/bash
-
-# script/dev_server: Launch a local dev version of the server in the background
-
-#
-# WIP
-#
-
-source "$(dirname "${0}")"/../script/include/global_header.inc.sh
-
-# Create a function to run after a trap is triggered
-reap() {
-   kill -s SIGTERM -- "-$$"
-   sleep 0.1
-   exit
-}
-
-# Register trapping of SIGTERM and SIGINT
-trap reap SIGTERM SIGINT
-
-# Display the script PID, which will also be the process group ID for all 
-# child processes
-echo "Process Group: $$"
-
-# Set server launch related environment variables
-DEBUG=1
-LAUNCH_ARGS="$*"
-
-
-# Launch the app
-source ./script/server &
-wait

script/server

@@ -4,8 +4,5 @@
 
 source "$(dirname "${0}")"/../script/include/global_header.inc.sh
 
-# Compile js/css assets
-yarn build
-
 # Launch the app
-run_command "./app.py ${LAUNCH_ARGS}"
+run_command "honcho start"

templates/dev/emails.html

@@ -0,0 +1,6 @@
+{% for msg in messages %}
+  <div style="white-space: pre-wrap">
+{{ msg }}
+  </div>
+  <hr>
+{% endfor %}

tests/conftest.py

@@ -10,6 +10,7 @@ from tempfile import TemporaryDirectory
 from atst.app import make_app, make_config
 from atst.database import db as _db
 from atst.domain.auth import logout
+from atst.queue import queue
 import tests.factories as factories
 from tests.mocks import PDF_FILENAME
 

tests/test_queue.py

@@ -0,0 +1,17 @@
+import pytest
+from atst.queue import queue
+
+# ensure queue is always empty for unit testing
+@pytest.fixture(scope="function", autouse=True)
+def reset_queue():
+    queue.get_queue().empty()
+    yield
+    queue.get_queue().empty()
+
+
+def test_send_mail():
+    initial = len(queue.get_queue())
+    queue.send_mail(
+        ["lordvader@geocities.net"], "death start", "how is it coming along?"
+    )
+    assert len(queue.get_queue()) == initial + 1

tests/utils/test_mailer.py

@@ -0,0 +1,48 @@
+import pytest
+from atst.utils.mailer import Mailer, Mailer, MailConnection, RedisConnection
+
+
+class MockConnection(MailConnection):
+    def __init__(self):
+        self._messages = []
+
+    def send(self, message):
+        self._messages.append(message)
+
+    @property
+    def messages(self):
+        return self._messages
+
+
+@pytest.fixture
+def mailer():
+    return Mailer(MockConnection(), "test@atat.com")
+
+
+def test_mailer_can_send_mail(mailer):
+    message_data = {
+        "recipients": ["ben@tattoine.org"],
+        "subject": "help",
+        "body": "you're my only hope",
+    }
+    mailer.send(**message_data)
+    assert len(mailer.messages) == 1
+    message = mailer.messages[0]
+    assert message["To"] == message_data["recipients"][0]
+    assert message["Subject"] == message_data["subject"]
+    assert message.get_content().strip() == message_data["body"]
+
+
+def test_redis_mailer_can_save_messages(app):
+    mailer = Mailer(RedisConnection(app.redis), "test@atat.com")
+    message_data = {
+        "recipients": ["ben@tattoine.org"],
+        "subject": "help",
+        "body": "you're my only hope",
+    }
+    mailer.send(**message_data)
+    assert len(mailer.messages) == 1
+    message = mailer.messages[0]
+    assert message_data["recipients"][0] in message
+    assert message_data["subject"] in message
+    assert message_data["body"] in message