Merge pull request #1083 from dod-ccpo/user-provisioning-task

Add create_user task

alembic/versions/e3d93f9caba7_add_environment_role_provisioning_fields.py

@@ -0,0 +1,32 @@
+"""add environment_role provisioning fields
+
+Revision ID: e3d93f9caba7
+Revises: 691b04ecd85e
+Create Date: 2019-09-18 16:35:47.554060
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'e3d93f9caba7' # pragma: allowlist secret
+down_revision = '691b04ecd85e' # pragma: allowlist secret
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('environment_roles', sa.Column('claimed_until', sa.TIMESTAMP(timezone=True), nullable=True))
+    op.add_column('environment_roles', sa.Column('csp_user_id', sa.String(), nullable=True))
+    op.add_column('environment_roles', sa.Column('status', sa.Enum('PENDING', 'COMPLETED', 'PENDING_DELETE', name='status', native_enum=False), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('environment_roles', 'csp_user_id')
+    op.drop_column('environment_roles', 'claimed_until')
+    op.drop_column('environment_roles', 'status')
+    # ### end Alembic commands ###

atst/domain/csp/cloud.py

@@ -194,7 +194,7 @@ class MockCloudProvider(CloudProviderInterface):
             GeneralCSPException("Could not create user."),
         )
 
-        return {"id": self._id()}
+        return self._id()
 
     def suspend_user(self, auth_credentials, csp_user_id):
         self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION)

atst/domain/environment_roles.py

@@ -1,5 +1,15 @@
+from sqlalchemy.orm.exc import NoResultFound
+
 from atst.database import db
-from atst.models import EnvironmentRole, ApplicationRole
+from atst.models import (
+    EnvironmentRole,
+    ApplicationRole,
+    Environment,
+    ApplicationRoleStatus,
+)
+from atst.domain.exceptions import NotFoundError
+from uuid import UUID
+from typing import List
 
 
 class EnvironmentRoles(object):
@@ -22,6 +32,15 @@ class EnvironmentRoles(object):
         )
         return existing_env_role
 
+    @classmethod
+    def get_by_id(cls, id_) -> EnvironmentRole:
+        try:
+            return (
+                db.session.query(EnvironmentRole).filter(EnvironmentRole.id == id_)
+            ).one()
+        except NoResultFound:
+            raise NotFoundError(cls.resource_name)
+
     @classmethod
     def get_by_user_and_environment(cls, user_id, environment_id):
         existing_env_role = (
@@ -54,3 +73,17 @@ class EnvironmentRoles(object):
             .filter(EnvironmentRole.deleted != True)
             .all()
         )
+
+    @classmethod
+    def get_environment_roles_pending_creation(cls) -> List[UUID]:
+        results = (
+            db.session.query(EnvironmentRole.id)
+            .join(Environment)
+            .join(ApplicationRole)
+            .filter(Environment.deleted == False)
+            .filter(Environment.baseline_info != None)
+            .filter(EnvironmentRole.status == EnvironmentRole.Status.PENDING)
+            .filter(ApplicationRole.status == ApplicationRoleStatus.ACTIVE)
+            .all()
+        )
+        return [id_ for id_, in results]

atst/jobs.py

@@ -6,6 +6,7 @@ from atst.queue import celery
 from atst.models import EnvironmentJobFailure, EnvironmentRoleJobFailure
 from atst.domain.csp.cloud import CloudProviderInterface, GeneralCSPException
 from atst.domain.environments import Environments
+from atst.domain.environment_roles import EnvironmentRoles
 from atst.models.utils import claim_for_update
 
 
@@ -101,6 +102,20 @@ def do_create_environment_baseline(csp: CloudProviderInterface, environment_id=N
         db.session.commit()
 
 
+def do_provision_user(csp: CloudProviderInterface, environment_role_id=None):
+    environment_role = EnvironmentRoles.get_by_id(environment_role_id)
+
+    with claim_for_update(environment_role) as environment_role:
+        credentials = environment_role.environment.csp_credentials
+
+        csp_user_id = csp.create_or_update_user(
+            credentials, environment_role, environment_role.role
+        )
+        environment_role.csp_user_id = csp_user_id
+        db.session.add(environment_role)
+        db.session.commit()
+
+
 def do_work(fn, task, csp, **kwargs):
     try:
         fn(csp, **kwargs)
@@ -130,6 +145,13 @@ def create_environment_baseline(self, environment_id=None):
     )
 
 
+@celery.task(bind=True)
+def provision_user(self, environment_role_id=None):
+    do_work(
+        do_provision_user, self, app.csp.cloud, environment_role_id=environment_role_id
+    )
+
+
 @celery.task(bind=True)
 def dispatch_create_environment(self):
     for environment_id in Environments.get_environments_pending_creation(
@@ -152,3 +174,11 @@ def dispatch_create_environment_baseline(self):
         pendulum.now()
     ):
         create_environment_baseline.delay(environment_id=environment_id)
+
+
+@celery.task(bind=True)
+def dispatch_provision_user(self):
+    for (
+        environment_role_id
+    ) in EnvironmentRoles.get_environment_roles_pending_creation():
+        provision_user.delay(environment_role_id=environment_role_id)

atst/models/environment.py

@@ -84,3 +84,11 @@ class Environment(
     @property
     def history(self):
         return self.get_changes()
+
+    @property
+    def csp_credentials(self):
+        return (
+            self.root_user_info.get("credentials")
+            if self.root_user_info is not None
+            else None
+        )

atst/models/environment_role.py

@@ -1,5 +1,5 @@
 from enum import Enum
-from sqlalchemy import Index, ForeignKey, Column, String
+from sqlalchemy import Index, ForeignKey, Column, String, TIMESTAMP, Enum as SQLAEnum
 from sqlalchemy.dialects.postgresql import UUID
 from sqlalchemy.orm import relationship
 
@@ -33,6 +33,16 @@ class EnvironmentRole(
 
     job_failures = relationship("EnvironmentRoleJobFailure")
 
+    csp_user_id = Column(String())
+    claimed_until = Column(TIMESTAMP(timezone=True))
+
+    class Status(Enum):
+        PENDING = "pending"
+        COMPLETED = "completed"
+        PENDING_DELETE = "pending_delete"
+
+    status = Column(SQLAEnum(Status, native_enum=False), default=Status.PENDING)
+
     def __repr__(self):
         return "<EnvironmentRole(role='{}', user='{}', environment='{}', id='{}')>".format(
             self.role, self.application_role.user_name, self.environment.name, self.id

atst/queue.py

@@ -18,6 +18,10 @@ def update_celery(celery, app):
             "task": "atst.jobs.dispatch_create_environment_baseline",
             "schedule": 60,
         },
+        "beat-dispatch_provision_user": {
+            "task": "atst.jobs.dispatch_provision_user",
+            "schedule": 60,
+        },
     }
 
     class ContextTask(celery.Task):

tests/domain/test_mock_csp.py

@@ -27,8 +27,8 @@ def test_create_environment_baseline(mock_csp: MockCloudProvider):
 
 
 def test_create_or_update_user(mock_csp: MockCloudProvider):
-    user_dict = mock_csp.create_or_update_user(CREDENTIALS, {}, "csp_role_id")
+    csp_user_id = mock_csp.create_or_update_user(CREDENTIALS, {}, "csp_role_id")
-    assert isinstance(user_dict["id"], str)
+    assert isinstance(csp_user_id, str)
 
 
 def test_suspend_user(mock_csp: MockCloudProvider):

tests/test_jobs.py

@@ -16,10 +16,23 @@ from atst.jobs import (
     dispatch_create_atat_admin_user,
     dispatch_create_environment_baseline,
     create_environment,
+    dispatch_provision_user,
+    do_provision_user,
 )
 from atst.models.utils import claim_for_update
 from atst.domain.exceptions import ClaimFailedException
-from tests.factories import EnvironmentFactory, EnvironmentRoleFactory, PortfolioFactory
+from tests.factories import (
+    EnvironmentFactory,
+    EnvironmentRoleFactory,
+    PortfolioFactory,
+    ApplicationRoleFactory,
+)
+from atst.models import EnvironmentRole, ApplicationRoleStatus
+
+
+@pytest.fixture(autouse=True, scope="function")
+def csp():
+    return Mock(wraps=MockCloudProvider({}, with_delay=False, with_failure=False))
 
 
 def test_environment_job_failure(celery_app, celery_worker):
@@ -63,11 +76,6 @@ yesterday = now.subtract(days=1)
 tomorrow = now.add(days=1)
 
 
-@pytest.fixture(autouse=True, scope="function")
-def csp():
-    return Mock(wraps=MockCloudProvider({}, with_delay=False, with_failure=False))
-
-
 def test_create_environment_job(session, csp):
     environment = EnvironmentFactory.create()
     do_create_environment(csp, environment.id)
@@ -299,3 +307,66 @@ def test_claim_for_update(session):
 
     # The claim is released
     assert environment.claimed_until is None
+
+
+def test_dispatch_provision_user(csp, session, celery_app, celery_worker, monkeypatch):
+    # Given that I have three environment roles:
+    #   (A) one of which has a completed status
+    #   (B) one of which has an environment that has not been provisioned
+    #   (C) one of which is pending, has a provisioned environment but an inactive application role
+    #   (D) one of which is pending, has a provisioned environment and has an active application role
+    provisioned_environment = EnvironmentFactory.create(
+        cloud_id="cloud_id", root_user_info={}, baseline_info={}
+    )
+    unprovisioned_environment = EnvironmentFactory.create()
+    _er_a = EnvironmentRoleFactory.create(
+        environment=provisioned_environment, status=EnvironmentRole.Status.COMPLETED
+    )
+    _er_b = EnvironmentRoleFactory.create(
+        environment=unprovisioned_environment, status=EnvironmentRole.Status.PENDING
+    )
+    _er_c = EnvironmentRoleFactory.create(
+        environment=unprovisioned_environment,
+        status=EnvironmentRole.Status.PENDING,
+        application_role=ApplicationRoleFactory(status=ApplicationRoleStatus.PENDING),
+    )
+    er_d = EnvironmentRoleFactory.create(
+        environment=provisioned_environment,
+        status=EnvironmentRole.Status.PENDING,
+        application_role=ApplicationRoleFactory(status=ApplicationRoleStatus.ACTIVE),
+    )
+
+    mock = Mock()
+    monkeypatch.setattr("atst.jobs.provision_user", mock)
+
+    # When I dispatch the user provisioning task
+    dispatch_provision_user.run()
+
+    # I expect it to dispatch only one call, to EnvironmentRole D
+    mock.delay.assert_called_once_with(environment_role_id=er_d.id)
+
+
+def test_do_provision_user(csp, session):
+    # Given that I have an EnvironmentRole with a provisioned environment
+    credentials = MockCloudProvider(())._auth_credentials
+    provisioned_environment = EnvironmentFactory.create(
+        cloud_id="cloud_id",
+        root_user_info={"credentials": credentials},
+        baseline_info={},
+    )
+    environment_role = EnvironmentRoleFactory.create(
+        environment=provisioned_environment,
+        status=EnvironmentRole.Status.PENDING,
+        role="my_role",
+    )
+
+    # When I call the user provisoning task
+    do_provision_user(csp=csp, environment_role_id=environment_role.id)
+
+    session.refresh(environment_role)
+    # I expect that the CSP create_or_update_user method will be called
+    csp.create_or_update_user.assert_called_once_with(
+        credentials, environment_role, "my_role"
+    )
+    # I expect that the EnvironmentRole now has a csp_user_id
+    assert environment_role.csp_user_id