Merge pull request #1108 from dod-ccpo/delete-user-task

Delete CSP users

alembic/versions/1497926ddec1_add_deleted_environment_role_status.py

@@ -0,0 +1,59 @@
+"""add deleted environment_role status
+
+Revision ID: 1497926ddec1
+Revises: e3d93f9caba7
+Create Date: 2019-10-04 10:44:54.198368
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "1497926ddec1"  # pragma: allowlist secret
+down_revision = "f50596c5ffbb"  # pragma: allowlist secret
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.alter_column(
+        "environment_roles",
+        "status",
+        type_=sa.Enum(
+            "PENDING",
+            "COMPLETED",
+            "PENDING_DELETE",
+            "DELETED",
+            name="status",
+            native_enum=False,
+        ),
+        existing_type=sa.Enum(
+            "PENDING", "COMPLETED", "PENDING_DELETE", name="status", native_enum=False
+        ),
+    )
+
+
+def downgrade():
+    conn = op.get_bind()
+    conn.execute(
+        """
+        UPDATE environment_roles
+        SET status = (CASE WHEN status = 'DELETED' THEN 'PENDING_DELETE' ELSE status END)
+        """
+    )
+    op.alter_column(
+        "environment_roles",
+        "status",
+        type_=sa.Enum(
+            "PENDING", "COMPLETED", "PENDING_DELETE", name="status", native_enum=False
+        ),
+        existing_type=sa.Enum(
+            "PENDING",
+            "COMPLETED",
+            "PENDING_DELETE",
+            "DELETED",
+            name="status",
+            native_enum=False,
+        ),
+    )

atst/app.py

@@ -29,7 +29,7 @@ from atst.models.permissions import Permissions
 from atst.queue import celery, update_celery
 from atst.utils import mailer
 from atst.utils.form_cache import FormCache
-from atst.utils.json import CustomJSONEncoder
+from atst.utils.json import CustomJSONEncoder, sqlalchemy_dumps
 from atst.utils.notification_sender import NotificationSender
 from atst.utils.session_limiter import SessionLimiter
 
@@ -158,6 +158,7 @@ def map_config(config):
         "PORT": int(config["default"]["PORT"]),
         "SQLALCHEMY_DATABASE_URI": config["default"]["DATABASE_URI"],
         "SQLALCHEMY_TRACK_MODIFICATIONS": False,
+        "SQLALCHEMY_ENGINE_OPTIONS": {"json_serializer": sqlalchemy_dumps},
         "WTF_CSRF_ENABLED": config.getboolean("default", "WTF_CSRF_ENABLED"),
         "PERMANENT_SESSION_LIFETIME": config.getint(
             "default", "PERMANENT_SESSION_LIFETIME"

atst/domain/environment_roles.py

@@ -56,10 +56,20 @@ class EnvironmentRoles(object):
 
     @classmethod
     def delete(cls, application_role_id, environment_id):
-        existing_env_role = EnvironmentRoles.get(application_role_id, environment_id)
+        existing_env_role = (
+            db.session.query(EnvironmentRole)
+            .join(ApplicationRole)
+            .filter(
+                ApplicationRole.id == application_role_id,
+                EnvironmentRole.environment_id == environment_id,
+                EnvironmentRole.status != EnvironmentRole.Status.PENDING_DELETE,
+            )
+            .one_or_none()
+        )
+
         if existing_env_role:
-            # TODO: Set status to pending_delete
+            existing_env_role.status = EnvironmentRole.Status.PENDING_DELETE
-            db.session.delete(existing_env_role)
+            existing_env_role.role = "deleted"
             db.session.commit()
             return True
         else:
@@ -87,3 +97,17 @@ class EnvironmentRoles(object):
             .all()
         )
         return [id_ for id_, in results]
+
+    @classmethod
+    def get_environment_roles_pending_deletion(cls) -> List[UUID]:
+        results = (
+            db.session.query(EnvironmentRole.id)
+            .join(Environment)
+            .join(ApplicationRole)
+            .filter(Environment.deleted == False)
+            .filter(Environment.baseline_info != None)
+            .filter(EnvironmentRole.status == EnvironmentRole.Status.PENDING_DELETE)
+            .filter(ApplicationRole.status == ApplicationRoleStatus.ACTIVE)
+            .all()
+        )
+        return [id_ for id_, in results]

atst/jobs.py

@@ -3,7 +3,11 @@ import pendulum
 
 from atst.database import db
 from atst.queue import celery
-from atst.models import EnvironmentJobFailure, EnvironmentRoleJobFailure
+from atst.models import (
+    EnvironmentJobFailure,
+    EnvironmentRoleJobFailure,
+    EnvironmentRole,
+)
 from atst.domain.csp.cloud import CloudProviderInterface, GeneralCSPException
 from atst.domain.environments import Environments
 from atst.domain.environment_roles import EnvironmentRoles
@@ -112,6 +116,20 @@ def do_provision_user(csp: CloudProviderInterface, environment_role_id=None):
             credentials, environment_role, environment_role.role
         )
         environment_role.csp_user_id = csp_user_id
+        environment_role.status = EnvironmentRole.Status.COMPLETED
+        db.session.add(environment_role)
+        db.session.commit()
+
+
+def do_delete_user(csp: CloudProviderInterface, environment_role_id=None):
+    environment_role = EnvironmentRoles.get_by_id(environment_role_id)
+
+    with claim_for_update(environment_role) as environment_role:
+        credentials = environment_role.environment.csp_credentials
+
+        csp.delete_user(credentials, environment_role.csp_user_id)
+        environment_role.status = EnvironmentRole.Status.DELETED
+        environment_role.deleted = True
         db.session.add(environment_role)
         db.session.commit()
 
@@ -152,6 +170,13 @@ def provision_user(self, environment_role_id=None):
     )
 
 
+@celery.task(bind=True)
+def delete_user(self, environment_role_id=None):
+    do_work(
+        do_delete_user, self, app.csp.cloud, environment_role_id=environment_role_id
+    )
+
+
 @celery.task(bind=True)
 def dispatch_create_environment(self):
     for environment_id in Environments.get_environments_pending_creation(
@@ -182,3 +207,11 @@ def dispatch_provision_user(self):
         environment_role_id
     ) in EnvironmentRoles.get_environment_roles_pending_creation():
         provision_user.delay(environment_role_id=environment_role_id)
+
+
+@celery.task(bind=True)
+def dispatch_delete_user(self):
+    for (
+        environment_role_id
+    ) in EnvironmentRoles.get_environment_roles_pending_deletion():
+        delete_user.delay(environment_role_id=environment_role_id)

atst/models/environment_role.py

@@ -40,6 +40,7 @@ class EnvironmentRole(
         PENDING = "pending"
         COMPLETED = "completed"
         PENDING_DELETE = "pending_delete"
+        DELETED = "deleted"
 
     status = Column(SQLAEnum(Status, native_enum=False), default=Status.PENDING)
 

atst/queue.py

@@ -22,6 +22,10 @@ def update_celery(celery, app):
             "task": "atst.jobs.dispatch_provision_user",
             "schedule": 60,
         },
+        "beat-dispatch_delete_user": {
+            "task": "atst.jobs.dispatch_delete_user",
+            "schedule": 60,
+        },
     }
 
     class ContextTask(celery.Task):

atst/utils/json.py

@@ -1,6 +1,9 @@
 from flask.json import JSONEncoder
+import json
 from werkzeug.datastructures import FileStorage
 from datetime import date
+from enum import Enum
+
 from atst.models.attachment import Attachment
 
 
@@ -13,3 +16,13 @@ class CustomJSONEncoder(JSONEncoder):
         elif isinstance(obj, FileStorage):
             return obj.filename
         return JSONEncoder.default(self, obj)
+
+
+def sqlalchemy_dumps(dct):
+    def _default(obj):
+        if isinstance(obj, Enum):
+            return obj.name
+        else:
+            raise TypeError()
+
+    return json.dumps(dct, default=_default)

js/test_templates/clin_fields.html

@@ -312,6 +312,8 @@
         </div>
         
           
+  
+  
   <pop-date-range
     initial-min-start-date="2019-09-14"
     initial-max-end-date="2022-09-14"
@@ -454,7 +456,6 @@
                   End Date
                 </div>
                 
-                
   
 
   <div class='usa-alert usa-alert-info' role='alert' aria-live='polite'>
@@ -481,10 +482,10 @@
                 </p>
 
                 <div v-if='outsideRange && !maxError' class="usa-input-error-message">
-                  PoP end date must be on or after September 14, 2019.
+                  PoP end date must be after or on September 14, 2019.
                 </div>
                 <div v-if='minError && !outsideRange' class="usa-input-error-message">
-                  PoP end date must be after start date.
+                  PoP start date must be before end date.
                 </div>
                 <div v-if='maxError' class="usa-input-error-message">
                   PoP end date must be before or on September 14, 2022.

js/test_templates/pop_date_range.html

@@ -1,16 +1,18 @@
 
+  
+  
   <pop-date-range
     initial-min-start-date="2019-09-14"
     initial-max-end-date="2022-09-14"
-
+    
       v-bind:clin-index="1"
-
+    
-
+    
       initial-start-date="None"
-
+    
-
+    
       initial-end-date="None"
-
+    
     inline-template>
 
     <div>
@@ -21,12 +23,12 @@
             :maxdate="maxStartProp"
             :minrange='initialMinStartDate'
             :maxrange='initialMaxEndDate'
-
+            
               name-tag='start_date'
               initialmonth=""
               initialday=""
               initialyear=""
-
+            
             :optional='true'
             v-on:date-change='handleDateChange'
             inline-template>
@@ -42,7 +44,6 @@
                 </p>
 
                 <div v-if='outsideRange && !minError' class="usa-input-error-message">
-                  {{ ""}}
                   PoP start date must be before or on September 14, 2022.
                 </div>
                 <div v-if='minError' class="usa-input-error-message">
@@ -100,17 +101,17 @@
 
                 <div v-if="isDateComplete">
                   <div class="usa-form-group-date-ok" v-if="isDateValid">
-
+                    
       <span class="icon icon--ok icon--green" aria-hidden="true"><svg aria-hidden="true" focusable="false" data-prefix="fas" data-icon="check-circle" class="svg-inline--fa fa-check-circle fa-w-16" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><path fill="currentColor" d="M504 256c0 136.967-111.033 248-248 248S8 392.967 8 256 119.033 8 256 8s248 111.033 248 248zM227.314 387.314l184-184c6.248-6.248 6.248-16.379 0-22.627l-22.627-22.627c-6.248-6.249-16.379-6.249-22.628 0L216 308.118l-70.059-70.059c-6.248-6.248-16.379-6.248-22.628 0l-22.627 22.627c-6.248 6.248-6.248 16.379 0 22.627l104 104c6.249 6.249 16.379 6.249 22.628.001z"></path></svg></span>
-
+  
                   </div>
                   <div class="usa-form-group-date-ok" v-else>
-
+                    
       <span class="icon icon--alert icon--red" aria-hidden="true"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16" fill="#fdb81e">
   <path d="M8 16c-4.411 0-8-3.589-8-8s3.589-8 8-8 8 3.589 8 8-3.589 8-8 8zM8 2C4.691 2 2 4.691 2 8s2.691 6 6 6 6-2.691 6-6-2.691-6-6-6zm0 8c-.552 0-1-.447-1-1V4c0-.552.448-1 1-1s1 .448 1 1v5c0 .553-.448 1-1 1zm0 3c-.26 0-.52-.11-.71-.29-.18-.19-.29-.45-.29-.71 0-.271.11-.521.29-.71.38-.37 1.05-.37 1.42 0 .18.189.29.45.29.71s-.11.52-.29.71c-.19.18-.45.29-.71.29z"/>
 </svg>
 </span>
-
+  
                   </div>
                 </div>
               </div>
@@ -126,12 +127,12 @@
             :maxdate="initialMaxEndDate"
             :minrange='initialMinStartDate'
             :maxrange='initialMaxEndDate'
-
+            
               name-tag='end_date'
               initialmonth=""
               initialday=""
               initialyear=""
-
+            
             :optional='true'
             v-on:date-change='handleDateChange'
             inline-template>
@@ -141,26 +142,25 @@
                 <div class="usa-input__title">
                   End Date
                 </div>
-
+                
-
+  
-
 
   <div class='usa-alert usa-alert-info' role='alert' aria-live='polite'>
 
     <div class='usa-alert-body'>
+      
 
-
+      
-
         <p class='usa-alert-text'>
           A CLIN's period of performance must end before September 14, 2022.
         </p>
+      
 
+      
 
+      
 
-
+      
-
-
-
     </div>
   </div>
 
@@ -169,10 +169,10 @@
                 </p>
 
                 <div v-if='outsideRange && !maxError' class="usa-input-error-message">
-                  PoP end date must be on or after September 14, 2019.
+                  PoP end date must be after or on September 14, 2019.
                 </div>
                 <div v-if='minError && !outsideRange' class="usa-input-error-message">
-                  PoP end date must be after start date.
+                  PoP start date must be before end date.
                 </div>
                 <div v-if='maxError' class="usa-input-error-message">
                   PoP end date must be before or on September 14, 2022.
@@ -223,17 +223,17 @@
 
                 <div v-if="isDateComplete">
                   <div class="usa-form-group-date-ok" v-if="isDateValid">
-
+                    
       <span class="icon icon--ok icon--green" aria-hidden="true"><svg aria-hidden="true" focusable="false" data-prefix="fas" data-icon="check-circle" class="svg-inline--fa fa-check-circle fa-w-16" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><path fill="currentColor" d="M504 256c0 136.967-111.033 248-248 248S8 392.967 8 256 119.033 8 256 8s248 111.033 248 248zM227.314 387.314l184-184c6.248-6.248 6.248-16.379 0-22.627l-22.627-22.627c-6.248-6.249-16.379-6.249-22.628 0L216 308.118l-70.059-70.059c-6.248-6.248-16.379-6.248-22.628 0l-22.627 22.627c-6.248 6.248-6.248 16.379 0 22.627l104 104c6.249 6.249 16.379 6.249 22.628.001z"></path></svg></span>
-
+  
                   </div>
                   <div class="usa-form-group-date-ok" v-else>
-
+                    
       <span class="icon icon--alert icon--red" aria-hidden="true"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16" fill="#fdb81e">
   <path d="M8 16c-4.411 0-8-3.589-8-8s3.589-8 8-8 8 3.589 8 8-3.589 8-8 8zM8 2C4.691 2 2 4.691 2 8s2.691 6 6 6 6-2.691 6-6-2.691-6-6-6zm0 8c-.552 0-1-.447-1-1V4c0-.552.448-1 1-1s1 .448 1 1v5c0 .553-.448 1-1 1zm0 3c-.26 0-.52-.11-.71-.29-.18-.19-.29-.45-.29-.71 0-.271.11-.521.29-.71.38-.37 1.05-.37 1.42 0 .18.189.29.45.29.71s-.11.52-.29.71c-.19.18-.45.29-.71.29z"/>
 </svg>
 </span>
-
+  
                   </div>
                 </div>
               </div>

js/test_templates/to_form.html

@@ -271,10 +271,10 @@
                   For example: 07 04 1776
                 </p>
 <div class="usa-input-error-message" v-if="outsideRange &amp;&amp; !maxError">
-                  PoP end date must be on or after September 14, 2019.
+                  PoP end date must be after or on September 14, 2019.
                 </div>
 <div class="usa-input-error-message" v-if="minError &amp;&amp; !outsideRange">
-                  PoP end date must be after start date.
+                  PoP start date must be before end date.
                 </div>
 <div class="usa-input-error-message" v-if="maxError">
                   PoP end date must be before or on September 14, 2022.

tests/domain/test_application_roles.py

@@ -83,5 +83,6 @@ def test_disable(session):
 
     ApplicationRoles.disable(member_role)
     session.refresh(member_role)
+    session.refresh(environment_role)
     assert member_role.status == ApplicationRoleStatus.DISABLED
-    assert not EnvironmentRoles.get_by_user_and_environment(user.id, environment.id)
+    assert environment_role.status == EnvironmentRole.Status.PENDING_DELETE

tests/domain/test_environments.py

@@ -5,7 +5,7 @@ from uuid import uuid4
 from atst.domain.environments import Environments
 from atst.domain.environment_roles import EnvironmentRoles
 from atst.domain.exceptions import NotFoundError
-from atst.models.environment_role import CSPRole
+from atst.models.environment_role import CSPRole, EnvironmentRole
 
 from tests.factories import (
     ApplicationFactory,
@@ -35,15 +35,15 @@ def test_update_env_role():
     assert env_role.role == new_role
 
 
-def test_update_env_role_no_access():
+def test_update_env_role_no_access(session):
     env_role = EnvironmentRoleFactory.create(role=CSPRole.BASIC_ACCESS.value)
 
     assert Environments.update_env_role(
         env_role.environment, env_role.application_role, None
     )
-    assert not EnvironmentRoles.get(
+
-        env_role.application_role.id, env_role.environment.id
+    session.refresh(env_role)
-    )
+    assert env_role.status == EnvironmentRole.Status.PENDING_DELETE
 
 
 def test_update_env_role_no_change():

tests/routes/applications/test_settings.py

@@ -1,4 +1,3 @@
-import pytest
 import uuid
 from flask import url_for, get_flashed_messages
 from unittest.mock import Mock
@@ -9,16 +8,10 @@ from tests.factories import *
 from atst.domain.applications import Applications
 from atst.domain.application_roles import ApplicationRoles
 from atst.domain.environment_roles import EnvironmentRoles
-from atst.domain.environments import Environments
-from atst.domain.environment_roles import EnvironmentRoles
 from atst.domain.common import Paginator
 from atst.domain.permission_sets import PermissionSets
-from atst.domain.portfolios import Portfolios
-from atst.domain.exceptions import NotFoundError
-from atst.models.application_role import Status as ApplicationRoleStatus
 from atst.models.environment_role import CSPRole
 from atst.models.permissions import Permissions
-from atst.models.portfolio_role import Status as PortfolioRoleStatus
 from atst.forms.application import EditEnvironmentForm
 from atst.forms.application_member import UpdateMemberForm
 from atst.forms.data import ENV_ROLE_NO_ACCESS as NO_ACCESS
@@ -483,7 +476,7 @@ def test_remove_member_failure(client, user_session):
     assert response.status_code == 404
 
 
-def test_update_member(client, user_session):
+def test_update_member(client, user_session, session):
     role = PermissionSets.get(PermissionSets.EDIT_APPLICATION_TEAM)
     # create an app role with only edit team perms
     app_role = ApplicationRoleFactory.create(permission_sets=[role])
@@ -544,13 +537,6 @@ def test_update_member(client, user_session):
         app_role.has_permission_set(PermissionSets.DELETE_APPLICATION_ENVIRONMENTS)
     )
 
-    environment_roles = application.roles[0].environment_roles
-    # make sure that old env role was deleted and there are only 2 env roles
-    assert len(environment_roles) == 2
-    # check that the user has roles in the correct envs
-    assert environment_roles[0].environment in [env, env_2]
-    assert environment_roles[1].environment in [env, env_2]
-
 
 def test_revoke_invite(client, user_session):
     invite = ApplicationInvitationFactory.create()

tests/test_jobs.py

@@ -3,7 +3,6 @@ import pytest
 from uuid import uuid4
 from unittest.mock import Mock
 from threading import Thread
-from time import sleep
 
 from atst.domain.csp.cloud import MockCloudProvider
 from atst.jobs import (
@@ -18,6 +17,8 @@ from atst.jobs import (
     create_environment,
     dispatch_provision_user,
     do_provision_user,
+    do_delete_user,
+    dispatch_delete_user,
 )
 from atst.models.utils import claim_for_update
 from atst.domain.exceptions import ClaimFailedException
@@ -310,7 +311,7 @@ def test_claim_for_update(session):
 
 
 def test_dispatch_provision_user(csp, session, celery_app, celery_worker, monkeypatch):
-    # Given that I have three environment roles:
+    # Given that I have four environment roles:
     #   (A) one of which has a completed status
     #   (B) one of which has an environment that has not been provisioned
     #   (C) one of which is pending, has a provisioned environment but an inactive application role
@@ -370,3 +371,68 @@ def test_do_provision_user(csp, session):
     )
     # I expect that the EnvironmentRole now has a csp_user_id
     assert environment_role.csp_user_id
+
+
+def test_do_delete_user(csp, session):
+    credentials = MockCloudProvider(())._auth_credentials
+    provisioned_environment = EnvironmentFactory.create(
+        cloud_id="cloud_id",
+        root_user_info={"credentials": credentials},
+        baseline_info={},
+    )
+
+    environment_role = EnvironmentRoleFactory.create(
+        environment=provisioned_environment,
+        status=EnvironmentRole.Status.PENDING_DELETE,
+        role="my_role",
+    )
+
+    do_delete_user(csp=csp, environment_role_id=environment_role.id)
+
+    session.refresh(environment_role)
+
+    assert environment_role.status == EnvironmentRole.Status.DELETED
+    assert environment_role.deleted == True
+
+
+def test_dispatch_delete_user(csp, session, monkeypatch):
+    # Given that I have five environment roles:
+    #   (A) one of which has a completed status
+    #   (B) one of which has an environment that has not been provisioned
+    #   (C) one of which is pending, has a provisioned environment but an inactive application role
+    #   (D) one of which is pending, has a provisioned environment and has an active application role
+    #   (E) one of which is pending delete, has a provisioned environment and has an active application role
+    provisioned_environment = EnvironmentFactory.create(
+        cloud_id="cloud_id", root_user_info={}, baseline_info={}
+    )
+    unprovisioned_environment = EnvironmentFactory.create()
+    _er_a = EnvironmentRoleFactory.create(
+        environment=provisioned_environment, status=EnvironmentRole.Status.COMPLETED
+    )
+    _er_b = EnvironmentRoleFactory.create(
+        environment=unprovisioned_environment, status=EnvironmentRole.Status.PENDING
+    )
+    _er_c = EnvironmentRoleFactory.create(
+        environment=unprovisioned_environment,
+        status=EnvironmentRole.Status.PENDING,
+        application_role=ApplicationRoleFactory(status=ApplicationRoleStatus.PENDING),
+    )
+    _er_d = EnvironmentRoleFactory.create(
+        environment=unprovisioned_environment,
+        status=EnvironmentRole.Status.PENDING_DELETE,
+        application_role=ApplicationRoleFactory(status=ApplicationRoleStatus.PENDING),
+    )
+    er_e = EnvironmentRoleFactory.create(
+        environment=provisioned_environment,
+        status=EnvironmentRole.Status.PENDING_DELETE,
+        application_role=ApplicationRoleFactory(status=ApplicationRoleStatus.ACTIVE),
+    )
+
+    mock = Mock()
+    monkeypatch.setattr("atst.jobs.delete_user", mock)
+
+    # When I dispatch the user deletion task
+    dispatch_delete_user.run()
+
+    # I expect it to dispatch only one call, to EnvironmentRole E
+    mock.delay.assert_called_once_with(environment_role_id=er_e.id)