Merge pull request #1078 from dod-ccpo/env-beat-schedule

Add a beat processing schedule for environment provisioning jobs.

.circleci/config.yml

@@ -185,6 +185,20 @@ workflows:
             branches:
               only:
                 - master
+      - azure-aks/update-container-image:
+          cluster-name: atat-cluster
+          container-image-updates: "atst-beat=${AZURE_SERVER_NAME}/atat:atat-${CIRCLE_SHA1}"
+          namespace: atat
+          resource-name: deployment.apps/atst-beat
+          resource-group: atat
+          # uncomment below for debugging
+          # show-kubectl-command: true
+          requires:
+            - azure-migration
+          filters:
+            branches:
+              only:
+                - master
       - aws-ecr/build-and-push-image:
           extra-build-args: "--build-arg CSP=aws"
           repo: atat
@@ -230,3 +244,17 @@ workflows:
             branches:
               only:
                 - master
+      - aws-eks/update-container-image:
+          cluster-name: atat
+          container-image-updates: "atst-beat=${AWS_ECR_ACCOUNT_URL}/atat:atat-${CIRCLE_SHA1}"
+          namespace: atat
+          resource-name: deployment.apps/atst-beat
+          aws-region: "${AWS_REGION}"
+          # uncomment below for debugging
+          # show-kubectl-command: true
+          requires:
+            - aws-migration
+          filters:
+            branches:
+              only:
+                - master

atst/domain/environments.py

@@ -1,4 +1,4 @@
-from sqlalchemy import text, func, or_
+from sqlalchemy import func, or_
 from sqlalchemy.orm.exc import NoResultFound
 from typing import List
 from uuid import UUID
@@ -130,7 +130,7 @@ class Environments(object):
         results = (
             cls.base_provision_query(now)
             .filter(Environment.cloud_id != None)
-            .filter(Environment.root_user_info == text("'null'"))
+            .filter(Environment.root_user_info == None)
         ).all()
         return [id_ for id_, in results]
 
@@ -143,7 +143,7 @@ class Environments(object):
         results = (
             cls.base_provision_query(now)
             .filter(Environment.cloud_id != None)
-            .filter(Environment.root_user_info != text("'null'"))
+            .filter(Environment.root_user_info != None)
-            .filter(Environment.baseline_info == text("'null'"))
+            .filter(Environment.baseline_info == None)
         ).all()
         return [id_ for id_, in results]

atst/models/environment.py

@@ -26,8 +26,8 @@ class Environment(
     creator = relationship("User")
 
     cloud_id = Column(String)
-    root_user_info = Column(JSONB)
+    root_user_info = Column(JSONB(none_as_null=True))
-    baseline_info = Column(JSONB)
+    baseline_info = Column(JSONB(none_as_null=True))
 
     claimed_until = Column(TIMESTAMP(timezone=True))
 
@@ -68,6 +68,10 @@ class Environment(
         else:
             return self.ProvisioningStatus.COMPLETED
 
+    @property
+    def is_pending(self):
+        return self.provisioning_status == self.ProvisioningStatus.PENDING
+
     def __repr__(self):
         return "<Environment(name='{}', num_users='{}', application='{}', portfolio='{}', id='{}')>".format(
             self.name,

atst/models/utils.py

@@ -47,3 +47,4 @@ def claim_for_update(resource, minutes=30):
         db.session.query(Model).filter(Model.id == resource.id).filter(
             Model.claimed_until != None
         ).update({"claimed_until": None}, synchronize_session="fetch")
+        db.session.commit()

atst/queue.py

@@ -5,7 +5,20 @@ celery = Celery(__name__)
 
 def update_celery(celery, app):
     celery.conf.update(app.config)
-    celery.conf.CELERYBEAT_SCHEDULE = {}
+    celery.conf.CELERYBEAT_SCHEDULE = {
+        "beat-dispatch_create_environment": {
+            "task": "atst.jobs.dispatch_create_environment",
+            "schedule": 60,
+        },
+        "beat-dispatch_create_atat_admin_user": {
+            "task": "atst.jobs.dispatch_create_atat_admin_user",
+            "schedule": 60,
+        },
+        "beat-dispatch_create_environment_baseline": {
+            "task": "atst.jobs.dispatch_create_environment_baseline",
+            "schedule": 60,
+        },
+    }
 
     class ContextTask(celery.Task):
         def __call__(self, *args, **kwargs):

atst/routes/applications/settings.py

@@ -25,6 +25,7 @@ def get_environments_obj_for_app(application):
         env_data = {
             "id": env.id,
             "name": env.name,
+            "pending": env.is_pending,
             "edit_form": EditEnvironmentForm(obj=env),
             "member_count": len(env.roles),
             "members": [env_role.application_role.user_name for env_role in env.roles],

deploy/aws/aws.yml

@@ -29,9 +29,6 @@ spec:
       containers:
         - name: atst
           image: 904153757533.dkr.ecr.us-east-2.amazonaws.com/atat:latest
-          resources:
-            requests:
-               memory: "500Mi"
           envFrom:
           - configMapRef:
               name: atst-envvars
@@ -148,9 +145,57 @@ spec:
             "worker",
             "--loglevel=info"
           ]
-          resources:
+          envFrom:
-            requests:
+          - configMapRef:
-               memory: "500Mi"
+              name: atst-envvars
+          - configMapRef:
+              name: atst-worker-envvars
+          volumeMounts:
+            - name: atst-config
+              mountPath: "/opt/atat/atst/atst-overrides.ini"
+              subPath: atst-overrides.ini
+      volumes:
+        - name: atst-config
+          secret:
+            secretName: atst-config-ini
+            items:
+            - key: override.ini
+              path: atst-overrides.ini
+              mode: 0644
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  labels:
+    app: atst
+  name: atst-beat
+  namespace: atat
+spec:
+  selector:
+    matchLabels:
+      role: beat
+  replicas: 1
+  strategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: atst
+        role: beat
+    spec:
+      securityContext:
+        fsGroup: 101
+      containers:
+        - name: atst-beat
+          image: 904153757533.dkr.ecr.us-east-2.amazonaws.com/atat:latest
+          args: [
+            "/opt/atat/atst/.venv/bin/python",
+            "/opt/atat/atst/.venv/bin/celery",
+            "-A",
+            "celery_worker.celery",
+            "beat",
+            "--loglevel=info"
+          ]
           envFrom:
           - configMapRef:
               name: atst-envvars

deploy/azure/azure.yml

@@ -29,9 +29,6 @@ spec:
       containers:
         - name: atst
           image: pwatat.azurecr.io/atat:latest
-          resources:
-            requests:
-               memory: "500Mi"
           envFrom:
           - configMapRef:
               name: atst-envvars
@@ -149,9 +146,57 @@ spec:
             "worker",
             "--loglevel=info"
           ]
-          resources:
+          envFrom:
-            requests:
+          - configMapRef:
-               memory: "500Mi"
+              name: atst-envvars
+          - configMapRef:
+              name: atst-worker-envvars
+          volumeMounts:
+            - name: atst-config
+              mountPath: "/opt/atat/atst/atst-overrides.ini"
+              subPath: atst-overrides.ini
+      volumes:
+        - name: atst-config
+          secret:
+            secretName: atst-config-ini
+            items:
+            - key: override.ini
+              path: atst-overrides.ini
+              mode: 0644
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  labels:
+    app: atst
+  name: atst-beat
+  namespace: atat
+spec:
+  selector:
+    matchLabels:
+      role: beat
+  replicas: 1
+  strategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: atst
+        role: beat
+    spec:
+      securityContext:
+        fsGroup: 101
+      containers:
+        - name: atst-beat
+          image: pwatat.azurecr.io/atat:latest
+          args: [
+            "/opt/atat/atst/.venv/bin/python",
+            "/opt/atat/atst/.venv/bin/celery",
+            "-A",
+            "celery_worker.celery",
+            "beat",
+            "--loglevel=info"
+          ]
           envFrom:
           - configMapRef:
               name: atst-envvars

script/dev_queue

@@ -4,7 +4,7 @@
 
 set -e
 
-WORKER="pipenv run celery -A celery_worker.celery worker --loglevel=info -B"
+WORKER="pipenv run celery -A celery_worker.celery worker --loglevel=info -B -c 1"
 
 if [[ `command -v entr` ]]; then
   find atst | entr -r $WORKER

templates/portfolios/applications/settings.html

@@ -218,7 +218,6 @@
         <ul class="accordion-table__items">
           {% for env in environments_obj %}
             {% set edit_form = env['edit_form'] %}
-            {% set testing_env_name = 'qa' %}
             <toggler inline-template>
               <li class="accordion-table__item">
                 <div class="accordion-table__item-content form-row">
@@ -227,10 +226,10 @@
                       <span>
                         {{ env['name'] }}
                       </span>
-                      {% if env['name'].lower() == testing_env_name %}
+                      {% if env['pending'] %}
-                        <span class='label'>PROCESSING</span>
+                        <span class='usa-label'>PROCESSING</span>
                       {% endif %}
-                      {% if env['name'].lower() != testing_env_name %}
+                      {% if env['pending'] %}
                         <span class="icon-link">
                           {% set edit_environment_button %}
                             {{ Icon('edit') }}
@@ -258,7 +257,7 @@
                     </div>
                   </div>
                   <div class="form-col form-col--third">
-                    {% if env['name'].lower() == testing_env_name %}
+                    {% if env['pending'] %}
                       <em>Cloud service provider link unavailable</em>
                     {% else %}
                       <a href='{{ url_for("applications.access_environment", environment_id=env.id)}}' target='_blank' rel='noopener noreferrer' class='application-list-item__environment__csp_link icon-link'>

tests/test_jobs.py

@@ -3,8 +3,8 @@ import pytest
 from uuid import uuid4
 from unittest.mock import Mock
 from threading import Thread
+from time import sleep
 
-from atst.models import Environment
 from atst.domain.csp.cloud import MockCloudProvider
 from atst.jobs import (
     RecordEnvironmentFailure,
@@ -19,12 +19,7 @@ from atst.jobs import (
 )
 from atst.models.utils import claim_for_update
 from atst.domain.exceptions import ClaimFailedException
-from tests.factories import (
+from tests.factories import EnvironmentFactory, EnvironmentRoleFactory, PortfolioFactory
-    EnvironmentFactory,
-    EnvironmentRoleFactory,
-    UserFactory,
-    PortfolioFactory,
-)
 
 
 def test_environment_job_failure(celery_app, celery_worker):
@@ -262,6 +257,7 @@ def test_claim_for_update(session):
     class FirstThread(Thread):
         def run(self):
             with claim_for_update(environment):
+                sleep(0.1)  # doing some work
                 satisfied_claims.append("FirstThread")
 
     class SecondThread(Thread):