Use Celery instead of RQ.

Celery provides a more robust set of queueing options for both tasks and
worker processes. Updates include:
- infrastructure necessary to run Celery, including celery entrypoint
- backgrounded functions are now imported directly from atst.jobs
- update tests as-needed
- update kubernetes worker pod command

Pipfile

@@ -20,12 +20,12 @@ pyopenssl = "*"
 requests = "*"
 apache-libcloud = "*"
 lockfile = "*"
-"flask-rq2" = "*"
 werkzeug = "*"
 PyYAML = "*"
 azure-storage = "*"
 azure-storage-common = "*"
 boto3 = "*"
+celery = "*"
 
 [dev-packages]
 bandit = "*"

Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "f09d90a1b4b86eff2e0ed453ceefcce4f19d54c57fb50ebed1f02aae8ab83c2a"
+            "sha256": "bf3b598c052193f70249da97ac746bc53aeb72f45a4515e10945a4274aba7b18"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -18,10 +18,17 @@
     "default": {
         "alembic": {
             "hashes": [
-                "sha256:cdb7d98bd5cbf65acd38d70b1c05573c432e6473a82f955cdea541b5c153b0cc"
+                "sha256:4a4811119efbdc5259d1f4c8f6de977b36ad3bcc919f59a29c2960c5ef9149e4"
             ],
             "index": "pypi",
-            "version": "==1.0.11"
+            "version": "==1.1.0"
+        },
+        "amqp": {
+            "hashes": [
+                "sha256:19a917e260178b8d410122712bac69cb3e6db010d68f6101e7307508aded5e68",
+                "sha256:19d851b879a471fcfdcf01df9936cff924f422baa77653289f7095dedd5fb26a"
+            ],
+            "version": "==2.5.1"
         },
         "apache-libcloud": {
             "hashes": [
@@ -69,20 +76,35 @@
             "index": "pypi",
             "version": "==2.1.0"
         },
+        "billiard": {
+            "hashes": [
+                "sha256:01afcb4e7c4fd6480940cfbd4d9edc19d7a7509d6ada533984d0d0f49901ec82",
+                "sha256:b8809c74f648dfe69b973c8e660bcec00603758c9db8ba89d7719f88d5f01f26"
+            ],
+            "version": "==3.6.1.0"
+        },
         "boto3": {
             "hashes": [
-                "sha256:3ec5b520dbd0a430cdd581a8250991fb0f21ee7e668a8928f15006b312fa86dc",
+                "sha256:366a1f3ec37b9434f25247cbe876f9ca1b53d35e35af18f74c735445100b4bc4",
-                "sha256:8aec0247131a0db1e33d28ad13910e01e6dfa208e8ab8ee5a4095e92dbaabf45"
+                "sha256:e7718b48cd073ad59a99a33d14252319dfaf550be3682b0c6a58da052fb05fcc"
             ],
             "index": "pypi",
-            "version": "==1.9.204"
+            "version": "==1.9.217"
         },
         "botocore": {
             "hashes": [
-                "sha256:56cd1114e0ce35733e890b321160c8c438243f4fa54d3d074dfa6bdce4ee55aa",
+                "sha256:68a0a22ca4e0e7e7ab482f63e21debfe402841fc49b8503dec0a7307b565d774",
-                "sha256:f86504bcc9c44d5b2e7b019f2f279b70f17b1400d2fc4775bc009ec473530cad"
+                "sha256:7a213b876e58b1b5380cf30faa05ba45073692ad4a3cc803ba763082a36436bb"
             ],
-            "version": "==1.12.204"
+            "version": "==1.12.217"
+        },
+        "celery": {
+            "hashes": [
+                "sha256:821d11967f0f3f8fe24bd61ecfc7b6acbb5a926b719f1e8c4d5ff7bc09e18d81",
+                "sha256:ae4541fb3af5182bd4af749fee9b89c4858f2792d34bb5d034967e662cf9b55c"
+            ],
+            "index": "pypi",
+            "version": "==4.4.0rc3"
         },
         "certifi": {
             "hashes": [
@@ -138,13 +160,6 @@
             ],
             "version": "==7.0"
         },
-        "croniter": {
-            "hashes": [
-                "sha256:0d905dbe6f131a910fd3dde792f0129788cd2cb3a8048c5f7aaa212670b0cef2",
-                "sha256:538adeb3a7f7816c3cdec6db974c441620d764c25ff4ed0146ee7296b8a50590"
-            ],
-            "version": "==0.3.30"
-        },
         "cryptography": {
             "hashes": [
                 "sha256:24b61e5fcb506424d3ec4e18bca995833839bf13c59fc43e530e488f28d46b8c",
@@ -168,11 +183,11 @@
         },
         "docutils": {
             "hashes": [
-                "sha256:02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6",
+                "sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0",
-                "sha256:51e64ef2ebfb29cae1faa133b3710143496eca21c530f3f71424d77687764274",
+                "sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827",
-                "sha256:7a4bd47eaf6596e1295ecb11361139febe29b084a87bf005bf899f9a42edc3c6"
+                "sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99"
             ],
-            "version": "==0.14"
+            "version": "==0.15.2"
         },
         "flask": {
             "hashes": [
@@ -189,14 +204,6 @@
             "index": "pypi",
             "version": "==0.12"
         },
-        "flask-rq2": {
-            "hashes": [
-                "sha256:3ef6395065255447f8e1516ccca24858ba87da1d71a6975e0e3b55256bf04967",
-                "sha256:abe1e52d3b98abe37e85830a614ba6af864516f1b6cf2229f352f8500eafc5fd"
-            ],
-            "index": "pypi",
-            "version": "==18.3"
-        },
         "flask-session": {
             "hashes": [
                 "sha256:a31c27e0c3287f00c825b3d9625aba585f4df4cccedb1e7dd5a69a215881a731",
@@ -228,6 +235,13 @@
             ],
             "version": "==2.8"
         },
+        "importlib-metadata": {
+            "hashes": [
+                "sha256:23d3d873e008a513952355379d93cbcab874c58f4f034ff657c7a87422fa64e8",
+                "sha256:80d2de76188eabfbfcf27e6a37342c2827801e59c4cc14b0371c56fed43820e3"
+            ],
+            "version": "==0.19"
+        },
         "itsdangerous": {
             "hashes": [
                 "sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19",
@@ -249,6 +263,13 @@
             ],
             "version": "==0.9.4"
         },
+        "kombu": {
+            "hashes": [
+                "sha256:55274dc75eb3c3994538b0973a0fadddb236b698a4bc135b8aa4981e0a710b8f",
+                "sha256:e5f0312dfb9011bebbf528ccaf118a6c2b5c3b8244451f08381fb23e7715809b"
+            ],
+            "version": "==4.6.4"
+        },
         "lockfile": {
             "hashes": [
                 "sha256:6aed02de03cba24efabcd600b30540140634fc06cfa603822d508d5361e9f799",
@@ -296,6 +317,13 @@
             ],
             "version": "==1.1.1"
         },
+        "more-itertools": {
+            "hashes": [
+                "sha256:409cd48d4db7052af495b09dec721011634af3753ae1ef92d2b32f73a745f832",
+                "sha256:92b8c4b06dac4f0611c0729b2f2ede52b2e1bac1ab48f089c7ddc12e26bb60c4"
+            ],
+            "version": "==7.2.0"
+        },
         "pendulum": {
             "hashes": [
                 "sha256:1cde6e3c6310fb882c98f373795f807cb2bd6af01f34d2857e6e283b5ee91e09",
@@ -372,6 +400,13 @@
             ],
             "version": "==1.0.4"
         },
+        "pytz": {
+            "hashes": [
+                "sha256:26c0b32e437e54a18161324a2fca3c4b9846b74a8dccddd843113109e1116b32",
+                "sha256:c894d57500a4cd2d5c71114aaab77dbab5eabd9022308ce5ac9bb93a60a6f0c7"
+            ],
+            "version": "==2019.2"
+        },
         "pytzdata": {
             "hashes": [
                 "sha256:c0c8316eaf6c25ba45816390a1a45c39790767069b3275c5f7de3ddf773eb810",
@@ -400,11 +435,11 @@
         },
         "redis": {
             "hashes": [
-                "sha256:45682ecf226c7611efe731974c4fa3390170ba045b9cdb26f0051114a5c2a68b",
+                "sha256:98a22fb750c9b9bb46e75e945dc3f61d0ab30d06117cbb21ff9cd1d315fedd3b",
-                "sha256:f2609a85e5f37f489ba3b5652e1175dc3711c4d7a7818c4f657615810afd23df"
+                "sha256:c504251769031b0dd7dd5cf786050a6050197c6de0d37778c80c08cb04ae8275"
             ],
             "index": "pypi",
-            "version": "==3.3.6"
+            "version": "==3.3.8"
         },
         "requests": {
             "hashes": [
@@ -414,20 +449,6 @@
             "index": "pypi",
             "version": "==2.22.0"
         },
-        "rq": {
-            "hashes": [
-                "sha256:2798d26a7b850e759f23f69695a389d676a9c08f2c14f96f0d34d9648c9d5616",
-                "sha256:4f27c6a690d1bd02b9157e615d8819555b9b359c0c4ec8ff0013d160c31b40bb"
-            ],
-            "version": "==1.1.0"
-        },
-        "rq-scheduler": {
-            "hashes": [
-                "sha256:90eb3915e31cc2032c301e5ab1fd5ad6f23d9500435046995e009f098e18efbe",
-                "sha256:ff7d45b34a8a39c9c83634f642aeef950641c75c4eeea3fa140bf574bfc6aca2"
-            ],
-            "version": "==0.9"
-        },
         "s3transfer": {
             "hashes": [
                 "sha256:6efc926738a3cd576c2a79725fed9afde92378aa5c6a957e3af010cb019fac9d",
@@ -444,10 +465,10 @@
         },
         "sqlalchemy": {
             "hashes": [
-                "sha256:217e7fc52199a05851eee9b6a0883190743c4fb9c8ac4313ccfceaffd852b0ff"
+                "sha256:2f8ff566a4d3a92246d367f2e9cd6ed3edeef670dcd6dda6dfdc9efed88bcd80"
             ],
             "index": "pypi",
-            "version": "==1.3.6"
+            "version": "==1.3.8"
         },
         "unipath": {
             "hashes": [
@@ -465,6 +486,13 @@
             "markers": "python_version >= '3.4'",
             "version": "==1.25.3"
         },
+        "vine": {
+            "hashes": [
+                "sha256:133ee6d7a9016f177ddeaf191c1f58421a1dcc6ee9a42c58b34bed40e1d2cd87",
+                "sha256:ea4947cc56d1fd6f2095c8d543ee25dad966f78692528e68b4fada11ba3f98af"
+            ],
+            "version": "==1.3.0"
+        },
         "webassets": {
             "hashes": [
                 "sha256:e7d9c8887343123fd5b32309b33167428cb1318cdda97ece12d0907fd69d38db"
@@ -486,6 +514,13 @@
                 "sha256:e3ee092c827582c50877cdbd49e9ce6d2c5c1f6561f849b3b068c1b8029626f1"
             ],
             "version": "==2.2.1"
+        },
+        "zipp": {
+            "hashes": [
+                "sha256:3718b1cbcd963c7d4c5511a8240812904164b7f381b647143a89d3b98f9bcd8e",
+                "sha256:f06903e9f1f43b12d371004b4ac7b06ab39a44adc747266928ae6debfa7b3335"
+            ],
+            "version": "==0.6.0"
         }
     },
     "develop": {
@@ -653,10 +688,10 @@
         },
         "faker": {
             "hashes": [
-                "sha256:96ad7902706f2409a2d0c3de5132f69b413555a419bacec99d3f16e657895b47",
+                "sha256:1d3f700e8dfcefd6e657118d71405d53e86974448aba78884f119bbd84c0cddf",
-                "sha256:b3bb64aff9571510de6812df45122b633dbc6227e870edae3ed9430f94698521"
+                "sha256:d5366e120191c5610fceeebfe1c298dc46da0277096f639c6dd7e2eaee0fa547"
             ],
-            "version": "==2.0.0"
+            "version": "==2.0.1"
         },
         "flask": {
             "hashes": [
@@ -675,10 +710,10 @@
         },
         "gitpython": {
             "hashes": [
-                "sha256:c15c55ff890cd3a6a8330059e80885410a328f645551b55a91d858bfb3eb2573",
+                "sha256:947cc75913e7b6da108458136607e2ee0e40c20be1e12d4284e7c6c12956c276",
-                "sha256:df752b6b6f06f11213e91c4925aea7eaf9e37e88fb71c8a7a1aa0a5c10852120"
+                "sha256:d2f4945f8260f6981d724f5957bc076398ada55cb5d25aaee10108bcdc894100"
             ],
-            "version": "==2.1.13"
+            "version": "==3.0.2"
         },
         "honcho": {
             "hashes": [
@@ -740,10 +775,10 @@
         },
         "jedi": {
             "hashes": [
-                "sha256:53c850f1a7d3cfcd306cc513e2450a54bdf5cacd7604b74e42dd1f0758eaaf36",
+                "sha256:786b6c3d80e2f06fd77162a07fed81b8baa22dde5d62896a790a331d6ac21a27",
-                "sha256:e07457174ef7cb2342ff94fa56484fe41cec7ef69b0059f01d3f812379cb6f7c"
+                "sha256:ba859c74fa3c966a22f2aeebe1b74ee27e2a462f56d3f5f7ca4a59af61bfe42e"
             ],
-            "version": "==0.14.1"
+            "version": "==0.15.1"
         },
         "jinja2": {
             "hashes": [
@@ -754,26 +789,26 @@
         },
         "lazy-object-proxy": {
             "hashes": [
-                "sha256:159a745e61422217881c4de71f9eafd9d703b93af95618635849fe469a283661",
+                "sha256:02b260c8deb80db09325b99edf62ae344ce9bc64d68b7a634410b8e9a568edbf",
-                "sha256:23f63c0821cc96a23332e45dfaa83266feff8adc72b9bcaef86c202af765244f",
+                "sha256:18f9c401083a4ba6e162355873f906315332ea7035803d0fd8166051e3d402e3",
-                "sha256:3b11be575475db2e8a6e11215f5aa95b9ec14de658628776e10d96fa0b4dac13",
+                "sha256:1f2c6209a8917c525c1e2b55a716135ca4658a3042b5122d4e3413a4030c26ce",
-                "sha256:3f447aff8bc61ca8b42b73304f6a44fa0d915487de144652816f950a3f1ab821",
+                "sha256:2f06d97f0ca0f414f6b707c974aaf8829c2292c1c497642f63824119d770226f",
-                "sha256:4ba73f6089cd9b9478bc0a4fa807b47dbdb8fad1d8f31a0f0a5dbf26a4527a71",
+                "sha256:616c94f8176808f4018b39f9638080ed86f96b55370b5a9463b2ee5c926f6c5f",
-                "sha256:4f53eadd9932055eac465bd3ca1bd610e4d7141e1278012bd1f28646aebc1d0e",
+                "sha256:63b91e30ef47ef68a30f0c3c278fbfe9822319c15f34b7538a829515b84ca2a0",
-                "sha256:64483bd7154580158ea90de5b8e5e6fc29a16a9b4db24f10193f0c1ae3f9d1ea",
+                "sha256:77b454f03860b844f758c5d5c6e5f18d27de899a3db367f4af06bec2e6013a8e",
-                "sha256:6f72d42b0d04bfee2397aa1862262654b56922c20a9bb66bb76b6f0e5e4f9229",
+                "sha256:83fe27ba321e4cfac466178606147d3c0aa18e8087507caec78ed5a966a64905",
-                "sha256:7c7f1ec07b227bdc561299fa2328e85000f90179a2f44ea30579d38e037cb3d4",
+                "sha256:84742532d39f72df959d237912344d8a1764c2d03fe58beba96a87bfa11a76d8",
-                "sha256:7c8b1ba1e15c10b13cad4171cfa77f5bb5ec2580abc5a353907780805ebe158e",
+                "sha256:874ebf3caaf55a020aeb08acead813baf5a305927a71ce88c9377970fe7ad3c2",
-                "sha256:8559b94b823f85342e10d3d9ca4ba5478168e1ac5658a8a2f18c991ba9c52c20",
+                "sha256:9f5caf2c7436d44f3cec97c2fa7791f8a675170badbfa86e1992ca1b84c37009",
-                "sha256:a262c7dfb046f00e12a2bdd1bafaed2408114a89ac414b0af8755c696eb3fc16",
+                "sha256:a0c8758d01fcdfe7ae8e4b4017b13552efa7f1197dd7358dc9da0576f9d0328a",
-                "sha256:acce4e3267610c4fdb6632b3886fe3f2f7dd641158a843cf6b6a68e4ce81477b",
+                "sha256:a4def978d9d28cda2d960c279318d46b327632686d82b4917516c36d4c274512",
-                "sha256:be089bb6b83fac7f29d357b2dc4cf2b8eb8d98fe9d9ff89f9ea6012970a853c7",
+                "sha256:ad4f4be843dace866af5fc142509e9b9817ca0c59342fdb176ab6ad552c927f5",
-                "sha256:bfab710d859c779f273cc48fb86af38d6e9210f38287df0069a63e40b45a2f5c",
+                "sha256:ae33dd198f772f714420c5ab698ff05ff900150486c648d29951e9c70694338e",
-                "sha256:c10d29019927301d524a22ced72706380de7cfc50f767217485a912b4c8bd82a",
+                "sha256:b4a2b782b8a8c5522ad35c93e04d60e2ba7f7dcb9271ec8e8c3e08239be6c7b4",
-                "sha256:dd6e2b598849b3d7aee2295ac765a578879830fb8966f70be8cd472e6069932e",
+                "sha256:c462eb33f6abca3b34cdedbe84d761f31a60b814e173b98ede3c81bb48967c4f",
-                "sha256:e408f1eacc0a68fed0c08da45f31d0ebb38079f043328dce69ff133b95c29dc1"
+                "sha256:fd135b8d35dfdcdb984828c84d695937e58cc5f49e1c854eb311c4d6aa03f4f1"
             ],
-            "version": "==1.4.1"
+            "version": "==1.4.2"
         },
         "markupsafe": {
             "hashes": [
@@ -1090,10 +1125,10 @@
         },
         "zipp": {
             "hashes": [
-                "sha256:4970c3758f4e89a7857a973b1e2a5d75bcdc47794442f2e2dd4fe8e0466e809a",
+                "sha256:3718b1cbcd963c7d4c5511a8240812904164b7f381b647143a89d3b98f9bcd8e",
-                "sha256:8a5712cfd3bb4248015eb3b0b3c54a5f6ee3f2425963ef2a0125b8bc40aafaec"
+                "sha256:f06903e9f1f43b12d371004b4ac7b06ab39a44adc747266928ae6debfa7b3335"
             ],
-            "version": "==0.5.2"
+            "version": "==0.6.0"
         }
     }
 }

atst/app.py

@@ -25,10 +25,10 @@ from atst.domain.authz import Authorization
 from atst.domain.csp import make_csp_provider
 from atst.domain.portfolios import Portfolios
 from atst.models.permissions import Permissions
+from atst.queue import celery, update_celery
 from atst.utils import mailer
 from atst.utils.form_cache import FormCache
 from atst.utils.json import CustomJSONEncoder
-from atst.queue import queue
 from atst.utils.notification_sender import NotificationSender
 from atst.utils.session_limiter import SessionLimiter
 
@@ -59,13 +59,14 @@ def make_app(config):
     app.config.update(config)
     app.config.update({"SESSION_REDIS": app.redis})
 
+    update_celery(celery, app)
+
     make_flask_callbacks(app)
     register_filters(app)
     make_csp_provider(app, config.get("CSP", "mock"))
     make_crl_validator(app)
     make_mailer(app)
     make_notification_sender(app)
-    queue.init_app(app)
 
     db.init_app(app)
     csrf.init_app(app)
@@ -149,6 +150,7 @@ def map_config(config):
     return {
         **config["default"],
         "ENV": config["default"]["ENVIRONMENT"],
+        "BROKER_URL": config["default"]["REDIS_URI"],
         "DEBUG": config["default"].getboolean("DEBUG"),
         "SQLALCHEMY_ECHO": config["default"].getboolean("SQLALCHEMY_ECHO"),
         "CLASSIFIED": config["default"].getboolean("CLASSIFIED"),
@@ -248,7 +250,7 @@ def make_mailer(app):
 
 
 def make_notification_sender(app):
-    app.notification_sender = NotificationSender(queue)
+    app.notification_sender = NotificationSender()
 
 
 def make_session_limiter(app, session, config):

atst/jobs.py

@@ -0,0 +1,18 @@
+from flask import current_app as app
+
+from atst.queue import celery
+
+
+@celery.task()
+def send_mail(recipients, subject, body):
+    app.mailer.send(recipients, subject, body)
+
+
+@celery.task()
+def send_notification_mail(recipients, subject, body):
+    app.logger.info(
+        "Sending a notification to these recipients: {}\n\nSubject: {}\n\n{}".format(
+            recipients, subject, body
+        )
+    )
+    app.mailer.send(recipients, subject, body)

atst/queue.py

@@ -1,57 +1,15 @@
-from flask_rq2 import RQ
+from celery import Celery
-from flask import current_app as app
+
+celery = Celery(__name__)
 
 
-class ATSTQueue(RQ):
+def update_celery(celery, app):
+    celery.conf.update(app.config)
 
-    """Internal helpers to get the queue that actually does the work.
+    class ContextTask(celery.Task):
+        def __call__(self, *args, **kwargs):
+            with app.app_context():
+                return self.run(*args, **kwargs)
 
-    The RQ object always uses the "default" queue, unless we explicitly request
+    celery.Task = ContextTask
-    otherwise. These helpers allow us to use `.queue_name` to get the name of
+    return celery
-    the configured queue and `_queue_job` will use the appropriate queue.
-
-    """
-
-    @property
-    def queue_name(self):
-        return self.queues[0]
-
-    def get_queue(self, name=None):
-        if not name:
-            name = self.queue_name
-        return super().get_queue(name)
-
-    def _queue_job(self, function, *args, **kwargs):
-        self.get_queue().enqueue(function, *args, **kwargs)
-
-    # pylint: disable=pointless-string-statement
-    """Instance methods to queue up application-specific jobs."""
-
-    def send_mail(self, recipients, subject, body):
-        self._queue_job(ATSTQueue._send_mail, recipients, subject, body)
-
-    def send_notification_mail(self, recipients, subject, body):
-        self._queue_job(ATSTQueue._send_notification_mail, recipients, subject, body)
-
-    # pylint: disable=pointless-string-statement
-    """Class methods to actually perform the work.
-
-    Must be a class method (or a module-level function) because we being able
-    to pickle the class is more effort than its worth.
-    """
-
-    @classmethod
-    def _send_mail(self, recipients, subject, body):
-        app.mailer.send(recipients, subject, body)
-
-    @classmethod
-    def _send_notification_mail(self, recipients, subject, body):
-        app.logger.info(
-            "Sending a notification to these recipients: {}\n\nSubject: {}\n\n{}".format(
-                recipients, subject, body
-            )
-        )
-        app.mailer.send(recipients, subject, body)
-
-
-queue = ATSTQueue()

atst/routes/applications/team.py

@@ -14,7 +14,7 @@ from atst.forms.team import TeamForm
 from atst.models import Permissions
 from atst.utils.flash import formatted_flash as flash
 from atst.utils.localization import translate
-from atst.queue import queue
+from atst.jobs import send_mail
 
 
 def get_form_permission_value(member, edit_perm_set):
@@ -129,7 +129,7 @@ def send_application_invitation(invitee_email, inviter_name, token):
     body = render_template(
         "emails/application/invitation.txt", owner=inviter_name, token=token
     )
-    queue.send_mail(
+    send_mail.delay(
         [invitee_email],
         translate("email.application_invite", {"inviter_name": inviter_name}),
         body,

atst/routes/dev.py

@@ -15,7 +15,7 @@ from atst.domain.exceptions import AlreadyExistsError, NotFoundError
 from atst.domain.users import Users
 from atst.domain.permission_sets import PermissionSets
 from atst.forms.data import SERVICE_BRANCHES
-from atst.queue import queue
+from atst.jobs import send_mail
 from atst.utils import pick
 
 
@@ -174,7 +174,7 @@ def dev_new_user():
 
 @bp.route("/test-email")
 def test_email():
-    queue.send_mail(
+    send_mail.delay(
         [request.args.get("to")], request.args.get("subject"), request.args.get("body")
     )
 

atst/routes/portfolios/invitations.py

@@ -6,7 +6,7 @@ from atst.domain.exceptions import AlreadyExistsError
 from atst.domain.invitations import PortfolioInvitations
 from atst.domain.portfolios import Portfolios
 from atst.models import Permissions
-from atst.queue import queue
+from atst.jobs import send_mail
 from atst.utils.flash import formatted_flash as flash
 from atst.utils.localization import translate
 import atst.forms.portfolio_member as member_forms
@@ -16,7 +16,7 @@ def send_portfolio_invitation(invitee_email, inviter_name, token):
     body = render_template(
         "emails/portfolio/invitation.txt", owner=inviter_name, token=token
     )
-    queue.send_mail(
+    send_mail.delay(
         [invitee_email],
         translate("email.portfolio_invite", {"inviter_name": inviter_name}),
         body,

atst/utils/notification_sender.py

@@ -1,6 +1,6 @@
 from sqlalchemy import select
 
-from atst.queue import ATSTQueue
+from atst.jobs import send_notification_mail
 from atst.database import db
 from atst.models import NotificationRecipient
 
@@ -8,12 +8,9 @@ from atst.models import NotificationRecipient
 class NotificationSender(object):
     EMAIL_SUBJECT = "ATST notification"
 
-    def __init__(self, queue: ATSTQueue):
-        self.queue = queue
-
     def send(self, body, type_=None):
         recipients = self._get_recipients(type_)
-        self.queue.send_notification_mail(recipients, self.EMAIL_SUBJECT, body)
+        send_notification_mail.delay(recipients, self.EMAIL_SUBJECT, body)
 
     def _get_recipients(self, type_):
         query = select([NotificationRecipient.email])

celery_worker.py

@@ -0,0 +1,7 @@
+#!/usr/bin/env python
+
+from atst.app import celery, make_app, make_config
+
+config = make_config()
+app = make_app(config)
+app.app_context().push()

deploy/aws/aws.yml

@@ -142,9 +142,10 @@ spec:
           image: 904153757533.dkr.ecr.us-east-2.amazonaws.com/atat:884d95ada21a5097f5c07f305d8e4e24d0f2a03f
           args: [
             "/opt/atat/atst/.venv/bin/python",
-            "/opt/atat/atst/.venv/bin/flask",
+            "/opt/atat/atst/.venv/bin/celery",
-            "rq",
+            "-A",
-            "worker"
+            "celery_worker.celery",
+            "worker",
           ]
           resources:
             requests:

deploy/azure/azure.yml

@@ -143,9 +143,10 @@ spec:
           image: pwatat.azurecr.io/atat:884d95ada21a5097f5c07f305d8e4e24d0f2a03f
           args: [
             "/opt/atat/atst/.venv/bin/python",
-            "/opt/atat/atst/.venv/bin/flask",
+            "/opt/atat/atst/.venv/bin/celery",
-            "rq",
+            "-A",
-            "worker"
+            "celery_worker.celery",
+            "worker",
           ]
           resources:
             requests:

script/dev_queue

@@ -4,8 +4,10 @@
 
 set -e
 
+WORKER="pipenv run celery -A celery_worker.celery worker --loglevel=info"
+
 if [[ `command -v entr` ]]; then
-  find atst | entr -r flask rq worker
+  find atst | entr -r $WORKER
 else
-  flask rq worker
+  $WORKER
 fi

tests/conftest.py

@@ -9,7 +9,6 @@ from collections import OrderedDict
 
 from atst.app import make_app, make_config
 from atst.database import db as _db
-from atst.queue import queue as atst_queue
 import tests.factories as factories
 from tests.mocks import PDF_FILENAME, PDF_FILENAME2
 from tests.utils import FakeLogger, FakeNotificationSender
@@ -158,12 +157,6 @@ def extended_financial_verification_data(pdf_upload):
     }
 
 
-@pytest.fixture(scope="function", autouse=True)
-def queue():
-    yield atst_queue
-    atst_queue.get_queue().empty()
-
-
 @pytest.fixture
 def crl_failover_open_app(app):
     app.config.update({"CRL_FAIL_OPEN": True})

tests/routes/applications/test_team.py

@@ -1,10 +1,11 @@
 import uuid
+from unittest.mock import Mock
+
 from flask import url_for
 
 from atst.domain.permission_sets import PermissionSets
 from atst.models import CSPRole
 from atst.forms.data import ENV_ROLE_NO_ACCESS as NO_ACCESS
-from atst.queue import queue
 
 from tests.factories import *
 
@@ -149,8 +150,9 @@ def test_update_team_revoke_environment_access(client, user_session, db, session
     assert user not in environment.users
 
 
-def test_create_member(client, user_session, session):
+def test_create_member(monkeypatch, client, user_session, session):
-    queue_length = len(queue.get_queue())
+    job_mock = Mock()
+    monkeypatch.setattr("atst.jobs.send_mail.delay", job_mock)
     user = UserFactory.create()
     application = ApplicationFactory.create(
         environments=[{"name": "Naboo"}, {"name": "Endor"}]
@@ -198,7 +200,7 @@ def test_create_member(client, user_session, session):
     )
     assert invitation.role.application == application
 
-    assert len(queue.get_queue()) == queue_length + 1
+    assert job_mock.called
 
 
 def test_remove_member_success(client, user_session):

tests/routes/portfolios/test_invitations.py

@@ -1,10 +1,11 @@
 import datetime
+from unittest.mock import Mock
+
 from flask import url_for
 
 from atst.domain.portfolios import Portfolios
 from atst.models import InvitationStatus, PortfolioRoleStatus
 from atst.domain.permission_sets import PermissionSets
-from atst.queue import queue
 
 from tests.factories import *
 
@@ -183,7 +184,11 @@ def test_user_can_only_revoke_invites_in_their_portfolio(client, user_session):
     assert not invite.is_revoked
 
 
-def test_user_can_only_resend_invites_in_their_portfolio(client, user_session, queue):
+def test_user_can_only_resend_invites_in_their_portfolio(
+    monkeypatch, client, user_session
+):
+    job_mock = Mock()
+    monkeypatch.setattr("atst.jobs.send_mail.delay", job_mock)
     portfolio = PortfolioFactory.create()
     other_portfolio = PortfolioFactory.create()
     user = UserFactory.create()
@@ -206,10 +211,12 @@ def test_user_can_only_resend_invites_in_their_portfolio(client, user_session, q
     )
 
     assert response.status_code == 404
-    assert len(queue.get_queue()) == 0
+    assert not job_mock.called
 
 
-def test_resend_invitation_sends_email(client, user_session, queue):
+def test_resend_invitation_sends_email(monkeypatch, client, user_session):
+    job_mock = Mock()
+    monkeypatch.setattr("atst.jobs.send_mail.delay", job_mock)
     user = UserFactory.create()
     portfolio = PortfolioFactory.create()
     ws_role = PortfolioRoleFactory.create(
@@ -227,12 +234,14 @@ def test_resend_invitation_sends_email(client, user_session, queue):
         )
     )
 
-    assert len(queue.get_queue()) == 1
+    assert job_mock.called
 
 
 def test_existing_member_invite_resent_to_email_submitted_in_form(
-    client, user_session, queue
+    monkeypatch, client, user_session
 ):
+    job_mock = Mock()
+    monkeypatch.setattr("atst.jobs.send_mail.delay", job_mock)
     portfolio = PortfolioFactory.create()
     user = UserFactory.create()
     ws_role = PortfolioRoleFactory.create(
@@ -253,10 +262,10 @@ def test_existing_member_invite_resent_to_email_submitted_in_form(
         )
     )
 
-    send_mail_job = queue.get_queue().jobs[0]
     assert user.email != "example@example.com"
-    assert send_mail_job.func.__func__.__name__ == "_send_mail"
+    ordered_args, _unordered_args = job_mock.call_args
-    assert send_mail_job.args[0] == ["example@example.com"]
+    recipients, _subject, _message = ordered_args
+    assert recipients[0] == "example@example.com"
 
 
 _DEFAULT_PERMS_FORM_DATA = {
@@ -278,11 +287,12 @@ def test_user_with_permission_has_add_member_link(client, user_session):
     )
 
 
-def test_invite_member(client, user_session, session):
+def test_invite_member(monkeypatch, client, user_session, session):
+    job_mock = Mock()
+    monkeypatch.setattr("atst.jobs.send_mail.delay", job_mock)
     user_data = UserFactory.dictionary()
     portfolio = PortfolioFactory.create()
     user_session(portfolio.owner)
-    queue_length = len(queue.get_queue())
 
     response = client.post(
         url_for("portfolios.invite_member", portfolio_id=portfolio.id),
@@ -307,5 +317,5 @@ def test_invite_member(client, user_session, session):
     )
     assert invitation.role.portfolio == portfolio
 
-    assert len(queue.get_queue()) == queue_length + 1
+    assert job_mock.called
     assert len(invitation.role.permission_sets) == 5

tests/test_queue.py

@@ -1,5 +0,0 @@
-def test_send_mail(queue):
-    queue.send_mail(
-        ["lordvader@geocities.net"], "death start", "how is it coming along?"
-    )
-    assert len(queue.get_queue()) == 1

tests/utils/test_notification_sender.py

@@ -6,22 +6,19 @@ from atst.utils.notification_sender import NotificationSender
 
 
 @pytest.fixture
-def mock_queue(queue):
+def notification_sender():
-    return Mock(spec=queue)
+    return NotificationSender()
 
 
-@pytest.fixture
+def test_can_send_notification(monkeypatch, notification_sender):
-def notification_sender(mock_queue):
+    job_mock = Mock()
-    return NotificationSender(mock_queue)
+    monkeypatch.setattr("atst.jobs.send_notification_mail.delay", job_mock)
-
-
-def test_can_send_notification(mock_queue, notification_sender):
     recipient_email = "test@example.com"
     email_body = "This is a test"
 
     NotificationRecipientFactory.create(email=recipient_email)
     notification_sender.send(email_body)
 
-    mock_queue.send_notification_mail.assert_called_once_with(
+    job_mock.assert_called_once_with(
         ("test@example.com",), notification_sender.EMAIL_SUBJECT, email_body
     )