Merge branch 'staging' into azure-initial-mgmt-grp

atst/app.py

@@ -157,7 +157,6 @@ def map_config(config):
         **config["default"],
         "USE_AUDIT_LOG": config["default"].getboolean("USE_AUDIT_LOG"),
         "ENV": config["default"]["ENVIRONMENT"],
-        "BROKER_URL": config["default"]["REDIS_URI"],
         "DEBUG": config["default"].getboolean("DEBUG"),
         "DEBUG_MAILER": config["default"].getboolean("DEBUG_MAILER"),
         "SQLALCHEMY_ECHO": config["default"].getboolean("SQLALCHEMY_ECHO"),
@@ -240,12 +239,27 @@ def make_config(direct_config=None):
         (config.get("default", "REDIS_PASSWORD") or ""),
         config.get("default", "REDIS_HOST"),
     )
+    celery_uri = redis_uri
     if redis_use_tls:
         tls_mode = config.get("default", "REDIS_SSLMODE")
         tls_mode_str = tls_mode.lower() if tls_mode else "none"
         redis_uri = f"{redis_uri}/?ssl_cert_reqs={tls_mode_str}"
 
+        # TODO: Kombu, one of Celery's dependencies, still requires
+        # that ssl_cert_reqs be passed as the string version of an
+        # option on the ssl module. We can clean this up and use
+        # the REDIS_URI for both when this PR to Kombu is released:
+        # https://github.com/celery/kombu/pull/1139
+        kombu_modes = {
+            "none": "CERT_NONE",
+            "required": "CERT_REQUIRED",
+            "optional": "CERT_OPTIONAL",
+        }
+        celery_tls_mode_str = kombu_modes[tls_mode_str]
+        celery_uri = f"{celery_uri}/?ssl_cert_reqs={celery_tls_mode_str}"
+
     config.set("default", "REDIS_URI", redis_uri)
+    config.set("default", "BROKER_URL", celery_uri)
 
     return map_config(config)
 

atst/models/task_order.py

@@ -25,7 +25,6 @@ SORT_ORDERING = [
     Status.DRAFT,
     Status.UPCOMING,
     Status.EXPIRED,
-    Status.UNSIGNED,
 ]
 
 
@@ -148,7 +147,10 @@ class TaskOrder(Base, mixins.TimestampsMixin):
 
     @property
     def display_status(self):
-        return self.status.value
+        if self.status == Status.UNSIGNED:
+            return Status.DRAFT.value
+        else:
+            return self.status.value
 
     @property
     def portfolio_name(self):

deploy/azure/kustomization.yaml

@@ -10,6 +10,5 @@ resources:
   - volume-claim.yml
   - nginx-client-ca-bundle.yml
   - acme-challenges.yml
-  - aadpodidentity.yml
   - nginx-snippets.yml
   - autoscaling.yml

deploy/overlays/cloudzero-dev/envvars.yml

@@ -4,19 +4,30 @@ kind: ConfigMap
 metadata:
   name: atst-worker-envvars
 data:
+  AZURE_ACCOUNT_NAME: jeditasksatat
   CELERY_DEFAULT_QUEUE: celery-staging
-  SERVER_NAME: staging.atat.code.mil
   FLASK_ENV: staging
+  PGDATABASE: cloudzero_jedidev_atat
+  PGHOST: 191.238.6.43
+  PGUSER: atat@cloudzero-jedidev-sql
+  PGSSLMODE: require
+  REDIS_HOST: 10.1.3.34:6380
+  SERVER_NAME: dev.atat.cloud.mil
 ---
 apiVersion: v1
 kind: ConfigMap
 metadata:
   name: atst-envvars
 data:
-  ASSETS_URL: https://atat-cdn-staging.azureedge.net/
+  ASSETS_URL: ""
-  CDN_ORIGIN: https://staging.atat.code.mil
+  AZURE_ACCOUNT_NAME: jeditasksatat
+  CAC_URL: https://auth-dev.atat.cloud.mil
+  CDN_ORIGIN: https://dev.atat.cloud.mil
   CELERY_DEFAULT_QUEUE: celery-staging
   FLASK_ENV: staging
-  STATIC_URL: https://atat-cdn-staging.azureedge.net/static/
+  PGDATABASE: cloudzero_jedidev_atat
-  PGHOST: cloudzero-dev-sql.postgres.database.azure.com
+  PGHOST: 191.238.6.43
-  REDIS_HOST: cloudzero-dev-redis.redis.cache.windows.net:6380
+  PGUSER: atat@cloudzero-jedidev-sql
+  PGSSLMODE: require
+  REDIS_HOST: 10.1.3.34:6380
+  SESSION_COOKIE_DOMAIN: atat.cloud.mil

deploy/overlays/cloudzero-dev/flex_vol.yml

@@ -9,23 +9,19 @@ spec:
         - name: nginx-secret
           flexVolume:
             options:
-              keyvaultname: "cloudzero-dev-keyvault"
-              # keyvaultobjectnames: "dhparam4096;cert;cert"
-              keyvaultobjectnames: "foo"
-              keyvaultobjectaliases: "FOO"
-              keyvaultobjecttypes: "secret"
-              usevmmanagedidentity: "true"
               usepodidentity: "false"
+              usevmmanagedidentity: "true"
+              vmmanagedidentityclientid: $VMSS_CLIENT_ID
+              keyvaultname: "cz-jedidev-keyvault"
+              keyvaultobjectnames: "dhparam4096;ATATCERT;ATATCERT"
         - name: flask-secret
           flexVolume:
             options:
-              keyvaultname: "cloudzero-dev-keyvault"
-              # keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY"
-              keyvaultobjectnames: "master-PGPASSWORD"
-              keyvaultobjectaliases: "PGPASSWORD"
-              keyvaultobjecttypes: "secret"
-              usevmmanagedidentity: "true"
               usepodidentity: "false"
+              usevmmanagedidentity: "true"
+              vmmanagedidentityclientid: $VMSS_CLIENT_ID
+              keyvaultname: "cz-jedidev-keyvault"
+              keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY"
 ---
 apiVersion: extensions/v1beta1
 kind: Deployment
@@ -38,10 +34,11 @@ spec:
         - name: flask-secret
           flexVolume:
             options:
-              keyvaultname: "cloudzero-dev-keyvault"
-              keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY"
-              usevmmanagedidentity: "true"
               usepodidentity: "false"
+              usevmmanagedidentity: "true"
+              vmmanagedidentityclientid: $VMSS_CLIENT_ID
+              keyvaultname: "cz-jedidev-keyvault"
+              keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY"
 ---
 apiVersion: extensions/v1beta1
 kind: Deployment
@@ -54,10 +51,11 @@ spec:
         - name: flask-secret
           flexVolume:
             options:
-              keyvaultname: "cloudzero-dev-keyvault"
-              keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY"
-              usevmmanagedidentity: "true"
               usepodidentity: "false"
+              usevmmanagedidentity: "true"
+              vmmanagedidentityclientid: $VMSS_CLIENT_ID
+              keyvaultname: "cz-jedidev-keyvault"
+              keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY"
 ---
 apiVersion: batch/v1beta1
 kind: CronJob
@@ -72,7 +70,8 @@ spec:
             - name: flask-secret
               flexVolume:
                 options:
-                  keyvaultname: "cloudzero-dev-keyvault"
-                  keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY"
-                  usevmmanagedidentity: "true"
                   usepodidentity: "false"
+                  usevmmanagedidentity: "true"
+                  vmmanagedidentityclientid: $VMSS_CLIENT_ID
+                  keyvaultname: "cz-jedidev-keyvault"
+                  keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY"

deploy/overlays/cloudzero-dev/kustomization.yaml

@@ -1,9 +1,8 @@
-namespace: staging
+namespace: cloudzero-dev
 bases:
   - ../../azure/
 resources:
   - namespace.yml
-  - reset-cron-job.yml
 patchesStrategicMerge:
   - ports.yml
   - envvars.yml

deploy/overlays/cloudzero-dev/namespace.yml

@@ -1,4 +1,4 @@
 apiVersion: v1
 kind: Namespace
 metadata:
-  name: staging
+  name: cloudzero-dev

deploy/overlays/cloudzero-dev/ports.yml

@@ -5,7 +5,7 @@ metadata:
   name: atst-main
   annotations:
     service.beta.kubernetes.io/azure-load-balancer-internal: "true"
-    service.beta.kubernetes.io/azure-load-balancer-internal-subnet: "cloudzero-dev-public"
+    service.beta.kubernetes.io/azure-load-balancer-internal-subnet: "cloudzero-jedidev-public"
 spec:
   loadBalancerIP: ""
   ports:
@@ -22,7 +22,7 @@ metadata:
   name: atst-auth
   annotations:
     service.beta.kubernetes.io/azure-load-balancer-internal: "true"
-    service.beta.kubernetes.io/azure-load-balancer-internal-subnet: "cloudzero-dev-public"
+    service.beta.kubernetes.io/azure-load-balancer-internal-subnet: "cloudzero-jedidev-public"
 spec:
   loadBalancerIP: ""
   ports:

deploy/overlays/cloudzero-dev/reset-cron-job.yml

@@ -1,46 +0,0 @@
-apiVersion: batch/v1beta1
-kind: CronJob
-metadata:
-  name: reset-db
-  namespace: atat
-spec:
-  schedule: "0 4 * * *"
-  concurrencyPolicy: Replace
-  successfulJobsHistoryLimit: 1
-  jobTemplate:
-    spec:
-      template:
-        metadata:
-          labels:
-            app: atst
-            role: reset-db
-            aadpodidbinding: atat-kv-id-binding
-        spec:
-          restartPolicy: OnFailure
-          containers:
-          - name: reset
-            image: $CONTAINER_IMAGE
-            command: [
-              "/bin/sh", "-c"
-            ]
-            args: [
-              "/opt/atat/atst/.venv/bin/python",
-              "/opt/atat/atst/script/reset_database.py"
-            ]
-            envFrom:
-            - configMapRef:
-                name: atst-worker-envvars
-            volumeMounts:
-              - name: flask-secret
-                mountPath: "/config"
-          volumes:
-            - name: flask-secret
-              flexVolume:
-                driver: "azure/kv"
-                options:
-                  usepodidentity: "true"
-                  keyvaultname: "atat-vault-test"
-                  keyvaultobjectnames: "staging-AZURE-STORAGE-KEY;staging-MAIL-PASSWORD;staging-PGPASSWORD;staging-REDIS-PASSWORD;staging-SECRET-KEY"
-                  keyvaultobjectaliases: "AZURE_STORAGE_KEY;MAIL_PASSWORD;PGPASSWORD;REDIS_PASSWORD;SECRET_KEY"
-                  keyvaultobjecttypes: "secret;secret;secret;secret;key"
-                  tenantid: $TENANT_ID

deploy/overlays/migration-cloudzero-dev/kustomization.yaml

@@ -0,0 +1,5 @@
+namespace: cloudzero-dev
+bases:
+  - ../../shared/
+patchesStrategicMerge:
+  - migration.yaml

deploy/overlays/migration-cloudzero-dev/migration.yaml

@@ -0,0 +1,16 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: migration
+spec:
+  template:
+    spec:
+      volumes:
+        - name: flask-secret
+          flexVolume:
+            options:
+              usepodidentity: "false"
+              usevmmanagedidentity: "true"
+              vmmanagedidentityclientid: $VMSS_CLIENT_ID
+              keyvaultname: "cz-jedidev-keyvault"
+              keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY"

deploy/shared/kustomization.yaml

@@ -0,0 +1,3 @@
+namespace: atat
+resources:
+  - migration.yaml

js/components/forms/multi_step_modal_form.js

@@ -34,8 +34,10 @@ export default {
 
   methods: {
     next: function() {
+      this.submitted = true
       if (this.validateFields()) {
         this.step += 1
+        this.submitted = false
       }
     },
     previous: function() {

js/components/sidenav_toggler.js

@@ -1,5 +1,6 @@
 import ExpandSidenavMixin from '../mixins/expand_sidenav'
 import ToggleMixin from '../mixins/toggle'
+import { sidenavCookieName } from '../lib/constants'
 
 export default {
   name: 'sidenav-toggler',
@@ -14,7 +15,7 @@ export default {
     toggle: function(e) {
       e.preventDefault()
       this.isVisible = !this.isVisible
-      document.cookie = this.cookieName + '=' + this.isVisible + '; path=/'
+      document.cookie = sidenavCookieName + '=' + this.isVisible + '; path=/'
       this.$parent.$emit('sidenavToggle', this.isVisible)
     },
   },

js/lib/constants.js

@@ -0,0 +1 @@
+export const sidenavCookieName = 'expandSidenav'

js/mixins/expand_sidenav.js

@@ -1,11 +1,12 @@
+import { sidenavCookieName } from '../lib/constants'
+
 export default {
   props: {
-    cookieName: 'expandSidenav',
     defaultVisible: {
       type: Boolean,
       default: function() {
-        if (document.cookie.match(this.cookieName)) {
+        if (document.cookie.match(sidenavCookieName)) {
-          return !!document.cookie.match(this.cookieName + ' *= *true')
+          return !!document.cookie.match(sidenavCookieName + ' *= *true')
         } else {
           return true
         }

script/database_setup.py

@@ -16,16 +16,14 @@ from reset_database import reset_database
 
 
 def database_setup(username, password, dbname, ccpo_users):
+    print("Applying schema and seeding roles and permissions.")
+    reset_database()
+
     print(
         f"Creating Postgres user role for '{username}' and granting all privileges to database '{dbname}'."
     )
-    try:
+    _create_database_user(username, password, dbname)
-        _create_database_user(username, password, dbname)
-    except sqlalchemy.exc.ProgrammingError as err:
-        print(f"Postgres user role '{username}' already exists.")
 
-    print("Applying schema and seeding roles and permissions.")
-    reset_database()
     print("Creating initial set of CCPO users.")
     _add_ccpo_users(ccpo_users)
 
@@ -47,6 +45,22 @@ def _create_database_user(username, password, dbname):
         f"ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL PRIVILEGES ON FUNCTIONS TO {username}; \n"
     )
 
+    try:
+        # TODO: make this more configurable
+        engine.execute(f"GRANT {username} TO azure_pg_admin;")
+    except sqlalchemy.exc.ProgrammingError as err:
+        print(f"Cannot grant new role {username} to azure_pg_admin")
+
+    for table in meta.tables:
+        engine.execute(f"ALTER TABLE {table} OWNER TO {username};\n")
+
+    sequence_results = engine.execute(
+        "SELECT c.relname FROM pg_class c WHERE c.relkind = 'S';"
+    ).fetchall()
+    sequences = [p[0] for p in sequence_results]
+    for sequence in sequences:
+        engine.execute(f"ALTER SEQUENCE {sequence} OWNER TO {username};\n")
+
     trans.commit()
 
 

script/k8s_config

@@ -13,6 +13,7 @@ SETTINGS=(
   AUTH_DOMAIN
   KV_MI_ID
   KV_MI_CLIENT_ID
+  VMSS_CLIENT_ID
   TENANT_ID
 )
 

templates/components/accordion.html

@@ -6,8 +6,12 @@
   heading_tag="h2",
   heading_classes="",
   content_tag="div",
-  content_classes="") %}
+  content_classes="",
-  <accordion v-cloak inline-template>
+  default_visible=False) %}
+  <accordion
+    v-cloak
+    inline-template
+    v-bind:default-visible='{{ default_visible | string | lower }}'>
     <{{wrapper_tag}} class="{{ wrapper_classes }}">
       <{{heading_tag}} class="accordion__button {{ heading_classes }}">
         <button

templates/task_orders/index.html

@@ -14,9 +14,15 @@
 
 
 {% macro TaskOrderList(task_orders, status) %}
+  {% set show_task_orders = task_orders|length > 0 %}
   <div class="accordion">
-    {% call Accordion(title=("task_orders.status_list_title"|translate({'status': status})), id=status, heading_tag="h4") %}
+    {% call Accordion(
-      {% if task_orders|length > 0 %}
+      title=("task_orders.status_list_title"|translate({'status': status})),
+      id=status,
+      heading_tag="h4",
+      default_visible=show_task_orders
+    ) %}
+      {% if show_task_orders %}
         {% for task_order in task_orders %}
           {% set to_number %}
             {% if task_order.number != None %}

tests/domain/test_task_orders.py

@@ -149,11 +149,12 @@ def test_task_order_sort_by_status():
     ]
 
     sorted_by_status = TaskOrders.sort_by_status(initial_to_list)
-    assert len(sorted_by_status["Draft"]) == 3
+    assert len(sorted_by_status["Draft"]) == 4
     assert len(sorted_by_status["Active"]) == 1
     assert len(sorted_by_status["Upcoming"]) == 1
     assert len(sorted_by_status["Expired"]) == 2
-    assert len(sorted_by_status["Unsigned"]) == 1
+    with pytest.raises(KeyError):
+        sorted_by_status["Unsigned"]
     assert list(sorted_by_status.keys()) == [status.value for status in SORT_ORDERING]