diff --git a/Pipfile b/Pipfile index bc5ed901..937896a5 100644 --- a/Pipfile +++ b/Pipfile @@ -21,6 +21,7 @@ requests = "*" apache-libcloud = "*" lockfile = "*" defusedxml = "*" +"flask-rq2" = "*" [dev-packages] bandit = "*" diff --git a/Pipfile.lock b/Pipfile.lock index bf863dc6..581e1a7d 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "1e5e6a695229166aaa5e6c427fed07a903766e9b3d24981a19cc8e5ada8db978" + "sha256": "7162a0e3c45d05aff99adde9d75128d1772cf030d1e2a722f441b21f251a4645" }, "pipfile-spec": 6, "requires": { @@ -41,10 +41,10 @@ }, "certifi": { "hashes": [ - "sha256:376690d6f16d32f9d1fe8932551d80b23e9d393a8578c5633a2ed39a64861638", - "sha256:456048c7e371c089d0a77a5212fb37a2c2dce1e24146e3b7e0261736aaeaa22a" + "sha256:339dc09518b07e2fa7eda5450740925974815557727d6bd35d319c1524a04a4c", + "sha256:6d58c986d22b038c8c0df30d639f23a3e6d172a05c3583e766f4c0b785c0986a" ], - "version": "==2018.8.24" + "version": "==2018.10.15" }, "cffi": { "hashes": [ @@ -97,6 +97,13 @@ ], "version": "==7.0" }, + "croniter": { + "hashes": [ + "sha256:64d5f8c719249694265190810ef2f051345007246c99a3879a35b393d593d668", + "sha256:8ce5e4edd6f1956e70c8a31211cf86a7859aa1f0ff256107723582d79238e002" + ], + "version": "==0.3.25" + }, "cryptography": { "hashes": [ "sha256:02602e1672b62e803e08617ec286041cc453e8d43f093a5f4162095506bc0beb", @@ -144,6 +151,14 @@ "index": "pypi", "version": "==0.12" }, + "flask-rq2": { + "hashes": [ + "sha256:83e28f0279828198e64e1ed52a43fd6b530d9192d9944f4dea30e99e5688c9de", + "sha256:d513aa8d3b91eda34091ed8e40d55655e3acff6d37d57197417b14839a066185" + ], + "index": "pypi", + "version": "==18.1" + }, "flask-session": { "hashes": [ "sha256:a31c27e0c3287f00c825b3d9625aba585f4df4cccedb1e7dd5a69a215881a731", @@ -313,6 +328,20 @@ "index": "pypi", "version": "==2.19.1" }, + "rq": { + "hashes": [ + "sha256:5dd83625ca64b0dbf668ee65a8d38f3f5132aa9b64de4d813ff76f97db194b60", + "sha256:7ac5989a27bdff713dd40517498c1b3bf720f8ebc47305055496f653a29da899" + ], + "version": "==0.12.0" + }, + "rq-scheduler": { + "hashes": [ + "sha256:6cad6b6d29eae55d4585e2ac9be3b8a36b3f18c87a494fc508a4fa19b9c845d6", + "sha256:fc51da3d4ad1a047cada3b97a96afea21a3102ea5aa5b79ed2ea97d8ffdf8821" + ], + "version": "==0.8.3" + }, "six": { "hashes": [ "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", @@ -453,7 +482,6 @@ "sha256:2105ee183c51fed27e2b6801029b3903f5c2774c78e3f53bd920ca468d0f5679", "sha256:236505d15af6c7b7bfe2a9485db4b2bdea21d9239351483326184314418c79a8", "sha256:237284425271db4f30d458b355decf388ab20b05278bdf8dc9a65de0973726c6", - "sha256:2619f0369412e22f01ad8f5bea503f15fb099a5eef3c31f1edb81dcb29221bf7", "sha256:26d8eea4c840b73c61a1081d68bceb57b21a2d4f7afda6cac8ac38cb05226b00", "sha256:39a3740f7721155f4269aedf67b211101c07bd2111b334dfd69b807156ab15d9", "sha256:4bd0c42db8efc8a60965769796d43a5570906a870bc819f7388860aa72779d1b", @@ -462,26 +490,19 @@ "sha256:5415cafb082dad78935b3045c2e5d8907f436d15ad24c3fdb8e1839e084e4961", "sha256:5631f1983074b33c35dbb84607f337b9d7e9808116d7f0f2cb7b9d6d4381d50e", "sha256:5e9249bc361cd22565fd98590a53fd25a3dd666b74791ed7237fa99de938bbed", - "sha256:61ad080b78287e8a10ae485a194fc552625d4ed4196ab32cc8987e61bdcceb0f", "sha256:6a48746154f1331f28ef9e889c625b5b15a36cb86dd8021b4bdd1180a2186aa5", "sha256:71d376dbac64855ed693bc1ca121794570fe603e8783cdfa304ec6825d4e768f", "sha256:749ebd8a615337747592bd1523dfc4af7199b2bf6403b55f96c728668aeff91f", - "sha256:8575f3e1a12eae8d2fd3935dcc6fad2d5a7cf32bc15150a69d3bede229e970d5", "sha256:8ec528b585b95234e9c0c31dcd0a89152d8ed82b4567aa62dbcb3e9a0600deee", "sha256:a1a9ccd879811437ca0307c914f136d6edb85bd0470e6d4966c6397927bcabd9", "sha256:abd956c334752776230b779537d911a5a12fcb69d8fd3fe332ae63a140301ae6", "sha256:ad18f836017f2e8881145795f483636564807aaed54223459915a0d4735300cf", "sha256:b07ac0b1533298ddbc54c9bf3464664895f22899fec027b8d6c8d3ac59023283", - "sha256:c3ae3527c72581595952977c1b391b9e7313d236216581099ee38e4240d997fe", - "sha256:d5309c5c6750ff882d47c0d4d5952d2384232e522db56d2bb63beb01dcb07f46", "sha256:d9385f1445e30e8e42b75a36a7899ea1fd0f5784233a626625d70f9b087de404", "sha256:db2d1fcd32dbeeb914b2660af1838e9c178b75173f95fd221b1f9410b5d3ef1d", "sha256:e1dec211147f1fd7cb7a0f9a96aeeca467a5af02d38911307b3b8c2324f9917e", - "sha256:e20f11023ab77ad08dcdbf3a740e2512f73ebfbbfcb4f08f0b8a8f65f98210a2", - "sha256:e2cc3fc55566990059afb0f06141e136095898b55e977af66d0b498415098792", "sha256:e96dffc1fa57bb8c1c238f3d989341a97302492d09cb11f77df031112621c35c", - "sha256:ed4d97eb0ecdee29d0748acd84e6380729f78ce5ba0c7fe3401801634c25a1c5", - "sha256:eecc9d908a22a97356a1033d756281cd8c37285430f047cb35458d1bc8e6f8de" + "sha256:ed4d97eb0ecdee29d0748acd84e6380729f78ce5ba0c7fe3401801634c25a1c5" ], "version": "==5.0a3" }, @@ -766,15 +787,11 @@ }, "pyyaml": { "hashes": [ - "sha256:1cbc199009e78f92d9edf554be4fe40fb7b0bef71ba688602a00e97a51909110", "sha256:254bf6fda2b7c651837acb2c718e213df29d531eebf00edb54743d10bcb694eb", "sha256:3108529b78577327d15eec243f0ff348a0640b0c3478d67ad7f5648f93bac3e2", "sha256:3c17fb92c8ba2f525e4b5f7941d850e7a48c3a59b32d331e2502a3cdc6648e76", - "sha256:6f89b5c95e93945b597776163403d47af72d243f366bf4622ff08bdfd1c950b7", "sha256:8d6d96001aa7f0a6a4a95e8143225b5d06e41b1131044913fecb8f85a125714b", - "sha256:be622cc81696e24d0836ba71f6272a2b5767669b0d79fdcf0295d51ac2e156c8", - "sha256:c8a88edd93ee29ede719080b2be6cb2333dfee1dccba213b422a9c8e97f2967b", - "sha256:f39411e380e2182ad33be039e8ee5770a5d9efe01a2bfb7ae58d9ba31c4a2a9d" + "sha256:c8a88edd93ee29ede719080b2be6cb2333dfee1dccba213b422a9c8e97f2967b" ], "version": "==4.2b4" }, @@ -823,8 +840,7 @@ "toml": { "hashes": [ "sha256:229f81c57791a41d65e399fc06bf0848bab550a9dfd5ed66df18ce5f05e73d5c", - "sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e", - "sha256:f1db651f9657708513243e61e6cc67d101a39bad662eaa9b5546f789338e07a3" + "sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e" ], "version": "==0.10.0" }, diff --git a/atst/app.py b/atst/app.py index 36aca6d0..d709fb5e 100644 --- a/atst/app.py +++ b/atst/app.py @@ -22,7 +22,8 @@ from atst.domain.authz import Authorization from atst.models.permissions import Permissions from atst.eda_client import MockEDAClient from atst.uploader import Uploader -from atst.utils.mailer import make_mailer +from atst.utils.mailer import Mailer, RedisMailer +from atst.queue import queue ENV = os.getenv("FLASK_ENV", "dev") @@ -37,11 +38,11 @@ def make_app(config): template_folder=parent_dir.child("templates").absolute(), static_folder=parent_dir.child("static").absolute(), ) - redis = make_redis(config) + make_redis(app, config) csrf = CSRFProtect() app.config.update(config) - app.config.update({"SESSION_REDIS": redis}) + app.config.update({"SESSION_REDIS": app.redis}) make_flask_callbacks(app) make_crl_validator(app) @@ -49,6 +50,7 @@ def make_app(config): make_eda_client(app) make_upload_storage(app) make_mailer(app) + queue.init_app(app) db.init_app(app) csrf.init_app(app) @@ -95,6 +97,7 @@ def map_config(config): "PERMANENT_SESSION_LIFETIME": config.getint( "default", "PERMANENT_SESSION_LIFETIME" ), + "RQ_REDIS_URL": config["default"]["REDIS_URI"], } @@ -143,8 +146,9 @@ def make_config(): return map_config(config) -def make_redis(config): - return redis.Redis.from_url(config["REDIS_URI"]) +def make_redis(app, config): + r = redis.Redis.from_url(config["REDIS_URI"]) + app.redis = r def make_crl_validator(app): @@ -166,3 +170,22 @@ def make_upload_storage(app): secret=app.config.get("STORAGE_SECRET"), ) app.uploader = uploader + + +def _map_config(config): + return { + "server": config.get("MAIL_SERVER"), + "port": config.get("MAIL_PORT"), + "sender": config.get("MAIL_SENDER"), + "password": config.get("MAIL_PASSWORD"), + "use_tls": config.get("MAIL_TLS", False), + } + + +def make_mailer(app): + config = _map_config(app.config) + if app.config["DEBUG"]: + mailer = RedisMailer(redis=app.redis, **config) + else: + mailer = Mailer(**config) + app.mailer = mailer diff --git a/atst/queue.py b/atst/queue.py new file mode 100644 index 00000000..8c94bf8f --- /dev/null +++ b/atst/queue.py @@ -0,0 +1,9 @@ +from flask_rq2 import RQ +from flask import current_app as app + +queue = RQ() + + +@queue.job +def send_mail(to, subject, body): + app.mailer.send(to, subject, body) diff --git a/atst/routes/dev.py b/atst/routes/dev.py index a087d061..862d602e 100644 --- a/atst/routes/dev.py +++ b/atst/routes/dev.py @@ -10,6 +10,7 @@ from flask import ( from . import redirect_after_login_url from atst.domain.users import Users +from atst.queue import send_mail bp = Blueprint("dev", __name__) @@ -77,7 +78,7 @@ def login_dev(): @bp.route("/test-email") def test_email(): - app.mailer.send( + send_mail.queue( [request.args.get("to")], request.args.get("subject"), request.args.get("body") ) diff --git a/atst/utils/mailer.py b/atst/utils/mailer.py index e3a6a094..7f1ca6cf 100644 --- a/atst/utils/mailer.py +++ b/atst/utils/mailer.py @@ -1,6 +1,5 @@ import smtplib from email.message import EmailMessage -from collections import deque class _HostConnection: @@ -25,18 +24,13 @@ class _HostConnection: self.host.quit() -class Mailer: - def __init__(self, server, port, sender, password, use_tls=False, debug=False): +class BaseMailer: + def __init__(self, server, port, sender, password, use_tls=False): self.server = server self.port = port self.sender = sender self.password = password self.use_tls = use_tls - self.debug = debug - self.messages = deque(maxlen=50) - - def connection(self): - return _HostConnection(self.server, self.port, self.sender, self.password) def _message(self, recipients, subject, body): msg = EmailMessage() @@ -47,27 +41,33 @@ class Mailer: return msg + def send(self, recipients, subject, body): + pass + + +class Mailer(BaseMailer): + def connection(self): + return _HostConnection(self.server, self.port, self.sender, self.password) + def send(self, recipients, subject, body): message = self._message(recipients, subject, body) - if self.debug: - self.messages.appendleft(message) - else: - with self.connection() as conn: - conn.send_message(message) + with self.connection() as conn: + conn.send_message(message) -def _map_config(config): - return { - "server": config.get("MAIL_SERVER"), - "port": config.get("MAIL_PORT"), - "sender": config.get("MAIL_SENDER"), - "password": config.get("MAIL_PASSWORD"), - "use_tls": config.get("MAIL_TLS", False), - "debug": config.get("DEBUG", False), - } +class RedisMailer(BaseMailer): + def __init__(self, redis, **kwargs): + super().__init__(**kwargs) + self.redis = redis + self._reset() + def _reset(self): + self.redis.delete("atat_inbox") -def make_mailer(app): - config = _map_config(app.config) - mailer = Mailer(**config) - app.mailer = mailer + @property + def messages(self): + return [msg.decode() for msg in self.redis.lrange("atat_inbox", 0, -1)] + + def send(self, recipients, subject, body): + message = self._message(recipients, subject, body) + self.redis.lpush("atat_inbox", str(message)) diff --git a/tests/utils/test_mailer.py b/tests/utils/test_mailer.py index 1a820b6b..f06d6ee9 100644 --- a/tests/utils/test_mailer.py +++ b/tests/utils/test_mailer.py @@ -1,5 +1,5 @@ import pytest -from atst.utils.mailer import Mailer +from atst.utils.mailer import Mailer, RedisMailer class MockHost: @@ -21,7 +21,7 @@ def mail_host(): return MockHost() -def test_can_send_mail(monkeypatch, mail_host): +def test_mailer_can_send_mail(monkeypatch, mail_host): monkeypatch.setattr("atst.utils.mailer.Mailer.connection", lambda *args: mail_host) mailer = Mailer("localhost", 456, "leia@rebellion.net", "droidsyourelookingfor") message_data = { @@ -37,10 +37,8 @@ def test_can_send_mail(monkeypatch, mail_host): assert message.get_content().strip() == message_data["body"] -def test_can_save_messages(): - mailer = Mailer( - "localhost", 456, "leia@rebellion.net", "droidsyourelookingfor", debug=True - ) +def test_redis_mailer_can_save_messages(app): + mailer = RedisMailer(app.redis, server=None, port=None, sender=None, password=None) message_data = { "recipients": ["ben@tattoine.org"], "subject": "help", @@ -49,6 +47,3 @@ def test_can_save_messages(): mailer.send(**message_data) assert len(mailer.messages) == 1 message = mailer.messages[0] - assert message["To"] == message_data["recipients"][0] - assert message["Subject"] == message_data["subject"] - assert message.get_content().strip() == message_data["body"]