diff --git a/alembic/versions/e3d93f9caba7_add_environment_role_provisioning_fields.py b/alembic/versions/e3d93f9caba7_add_environment_role_provisioning_fields.py new file mode 100644 index 00000000..98257583 --- /dev/null +++ b/alembic/versions/e3d93f9caba7_add_environment_role_provisioning_fields.py @@ -0,0 +1,32 @@ +"""add environment_role provisioning fields + +Revision ID: e3d93f9caba7 +Revises: 691b04ecd85e +Create Date: 2019-09-18 16:35:47.554060 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'e3d93f9caba7' # pragma: allowlist secret +down_revision = '691b04ecd85e' # pragma: allowlist secret +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('environment_roles', sa.Column('claimed_until', sa.TIMESTAMP(timezone=True), nullable=True)) + op.add_column('environment_roles', sa.Column('csp_user_id', sa.String(), nullable=True)) + op.add_column('environment_roles', sa.Column('status', sa.Enum('PENDING', 'COMPLETED', 'PENDING_DELETE', name='status', native_enum=False), nullable=True)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('environment_roles', 'csp_user_id') + op.drop_column('environment_roles', 'claimed_until') + op.drop_column('environment_roles', 'status') + # ### end Alembic commands ### diff --git a/atst/domain/csp/cloud.py b/atst/domain/csp/cloud.py index 5f8689c1..4a6961e5 100644 --- a/atst/domain/csp/cloud.py +++ b/atst/domain/csp/cloud.py @@ -194,7 +194,7 @@ class MockCloudProvider(CloudProviderInterface): GeneralCSPException("Could not create user."), ) - return {"id": self._id()} + return self._id() def suspend_user(self, auth_credentials, csp_user_id): self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) diff --git a/atst/domain/environment_roles.py b/atst/domain/environment_roles.py index b28e94b2..9dd59365 100644 --- a/atst/domain/environment_roles.py +++ b/atst/domain/environment_roles.py @@ -1,5 +1,15 @@ +from sqlalchemy.orm.exc import NoResultFound + from atst.database import db -from atst.models import EnvironmentRole, ApplicationRole +from atst.models import ( + EnvironmentRole, + ApplicationRole, + Environment, + ApplicationRoleStatus, +) +from atst.domain.exceptions import NotFoundError +from uuid import UUID +from typing import List class EnvironmentRoles(object): @@ -22,6 +32,15 @@ class EnvironmentRoles(object): ) return existing_env_role + @classmethod + def get_by_id(cls, id_) -> EnvironmentRole: + try: + return ( + db.session.query(EnvironmentRole).filter(EnvironmentRole.id == id_) + ).one() + except NoResultFound: + raise NotFoundError(cls.resource_name) + @classmethod def get_by_user_and_environment(cls, user_id, environment_id): existing_env_role = ( @@ -54,3 +73,17 @@ class EnvironmentRoles(object): .filter(EnvironmentRole.deleted != True) .all() ) + + @classmethod + def get_environment_roles_pending_creation(cls) -> List[UUID]: + results = ( + db.session.query(EnvironmentRole.id) + .join(Environment) + .join(ApplicationRole) + .filter(Environment.deleted == False) + .filter(Environment.baseline_info != None) + .filter(EnvironmentRole.status == EnvironmentRole.Status.PENDING) + .filter(ApplicationRole.status == ApplicationRoleStatus.ACTIVE) + .all() + ) + return [id_ for id_, in results] diff --git a/atst/jobs.py b/atst/jobs.py index 46717192..e861d8b5 100644 --- a/atst/jobs.py +++ b/atst/jobs.py @@ -6,6 +6,7 @@ from atst.queue import celery from atst.models import EnvironmentJobFailure, EnvironmentRoleJobFailure from atst.domain.csp.cloud import CloudProviderInterface, GeneralCSPException from atst.domain.environments import Environments +from atst.domain.environment_roles import EnvironmentRoles from atst.models.utils import claim_for_update @@ -101,6 +102,20 @@ def do_create_environment_baseline(csp: CloudProviderInterface, environment_id=N db.session.commit() +def do_provision_user(csp: CloudProviderInterface, environment_role_id=None): + environment_role = EnvironmentRoles.get_by_id(environment_role_id) + + with claim_for_update(environment_role) as environment_role: + credentials = environment_role.environment.csp_credentials + + csp_user_id = csp.create_or_update_user( + credentials, environment_role, environment_role.role + ) + environment_role.csp_user_id = csp_user_id + db.session.add(environment_role) + db.session.commit() + + def do_work(fn, task, csp, **kwargs): try: fn(csp, **kwargs) @@ -130,6 +145,13 @@ def create_environment_baseline(self, environment_id=None): ) +@celery.task(bind=True) +def provision_user(self, environment_role_id=None): + do_work( + do_provision_user, self, app.csp.cloud, environment_role_id=environment_role_id + ) + + @celery.task(bind=True) def dispatch_create_environment(self): for environment_id in Environments.get_environments_pending_creation( @@ -152,3 +174,11 @@ def dispatch_create_environment_baseline(self): pendulum.now() ): create_environment_baseline.delay(environment_id=environment_id) + + +@celery.task(bind=True) +def dispatch_provision_user(self): + for ( + environment_role_id + ) in EnvironmentRoles.get_environment_roles_pending_creation(): + provision_user.delay(environment_role_id=environment_role_id) diff --git a/atst/models/environment.py b/atst/models/environment.py index f3d85502..b8129fe5 100644 --- a/atst/models/environment.py +++ b/atst/models/environment.py @@ -84,3 +84,11 @@ class Environment( @property def history(self): return self.get_changes() + + @property + def csp_credentials(self): + return ( + self.root_user_info.get("credentials") + if self.root_user_info is not None + else None + ) diff --git a/atst/models/environment_role.py b/atst/models/environment_role.py index 9f6754f8..988d19ab 100644 --- a/atst/models/environment_role.py +++ b/atst/models/environment_role.py @@ -1,5 +1,5 @@ from enum import Enum -from sqlalchemy import Index, ForeignKey, Column, String +from sqlalchemy import Index, ForeignKey, Column, String, TIMESTAMP, Enum as SQLAEnum from sqlalchemy.dialects.postgresql import UUID from sqlalchemy.orm import relationship @@ -33,6 +33,16 @@ class EnvironmentRole( job_failures = relationship("EnvironmentRoleJobFailure") + csp_user_id = Column(String()) + claimed_until = Column(TIMESTAMP(timezone=True)) + + class Status(Enum): + PENDING = "pending" + COMPLETED = "completed" + PENDING_DELETE = "pending_delete" + + status = Column(SQLAEnum(Status, native_enum=False), default=Status.PENDING) + def __repr__(self): return "".format( self.role, self.application_role.user_name, self.environment.name, self.id diff --git a/atst/queue.py b/atst/queue.py index 5fc12a3f..c7d117c6 100644 --- a/atst/queue.py +++ b/atst/queue.py @@ -18,6 +18,10 @@ def update_celery(celery, app): "task": "atst.jobs.dispatch_create_environment_baseline", "schedule": 60, }, + "beat-dispatch_provision_user": { + "task": "atst.jobs.dispatch_provision_user", + "schedule": 60, + }, } class ContextTask(celery.Task): diff --git a/tests/domain/test_mock_csp.py b/tests/domain/test_mock_csp.py index 27495f4c..941479b7 100644 --- a/tests/domain/test_mock_csp.py +++ b/tests/domain/test_mock_csp.py @@ -27,8 +27,8 @@ def test_create_environment_baseline(mock_csp: MockCloudProvider): def test_create_or_update_user(mock_csp: MockCloudProvider): - user_dict = mock_csp.create_or_update_user(CREDENTIALS, {}, "csp_role_id") - assert isinstance(user_dict["id"], str) + csp_user_id = mock_csp.create_or_update_user(CREDENTIALS, {}, "csp_role_id") + assert isinstance(csp_user_id, str) def test_suspend_user(mock_csp: MockCloudProvider): diff --git a/tests/test_jobs.py b/tests/test_jobs.py index 9fc4aa27..c4d17b5d 100644 --- a/tests/test_jobs.py +++ b/tests/test_jobs.py @@ -16,10 +16,23 @@ from atst.jobs import ( dispatch_create_atat_admin_user, dispatch_create_environment_baseline, create_environment, + dispatch_provision_user, + do_provision_user, ) from atst.models.utils import claim_for_update from atst.domain.exceptions import ClaimFailedException -from tests.factories import EnvironmentFactory, EnvironmentRoleFactory, PortfolioFactory +from tests.factories import ( + EnvironmentFactory, + EnvironmentRoleFactory, + PortfolioFactory, + ApplicationRoleFactory, +) +from atst.models import EnvironmentRole, ApplicationRoleStatus + + +@pytest.fixture(autouse=True, scope="function") +def csp(): + return Mock(wraps=MockCloudProvider({}, with_delay=False, with_failure=False)) def test_environment_job_failure(celery_app, celery_worker): @@ -63,11 +76,6 @@ yesterday = now.subtract(days=1) tomorrow = now.add(days=1) -@pytest.fixture(autouse=True, scope="function") -def csp(): - return Mock(wraps=MockCloudProvider({}, with_delay=False, with_failure=False)) - - def test_create_environment_job(session, csp): environment = EnvironmentFactory.create() do_create_environment(csp, environment.id) @@ -299,3 +307,66 @@ def test_claim_for_update(session): # The claim is released assert environment.claimed_until is None + + +def test_dispatch_provision_user(csp, session, celery_app, celery_worker, monkeypatch): + # Given that I have three environment roles: + # (A) one of which has a completed status + # (B) one of which has an environment that has not been provisioned + # (C) one of which is pending, has a provisioned environment but an inactive application role + # (D) one of which is pending, has a provisioned environment and has an active application role + provisioned_environment = EnvironmentFactory.create( + cloud_id="cloud_id", root_user_info={}, baseline_info={} + ) + unprovisioned_environment = EnvironmentFactory.create() + _er_a = EnvironmentRoleFactory.create( + environment=provisioned_environment, status=EnvironmentRole.Status.COMPLETED + ) + _er_b = EnvironmentRoleFactory.create( + environment=unprovisioned_environment, status=EnvironmentRole.Status.PENDING + ) + _er_c = EnvironmentRoleFactory.create( + environment=unprovisioned_environment, + status=EnvironmentRole.Status.PENDING, + application_role=ApplicationRoleFactory(status=ApplicationRoleStatus.PENDING), + ) + er_d = EnvironmentRoleFactory.create( + environment=provisioned_environment, + status=EnvironmentRole.Status.PENDING, + application_role=ApplicationRoleFactory(status=ApplicationRoleStatus.ACTIVE), + ) + + mock = Mock() + monkeypatch.setattr("atst.jobs.provision_user", mock) + + # When I dispatch the user provisioning task + dispatch_provision_user.run() + + # I expect it to dispatch only one call, to EnvironmentRole D + mock.delay.assert_called_once_with(environment_role_id=er_d.id) + + +def test_do_provision_user(csp, session): + # Given that I have an EnvironmentRole with a provisioned environment + credentials = MockCloudProvider(())._auth_credentials + provisioned_environment = EnvironmentFactory.create( + cloud_id="cloud_id", + root_user_info={"credentials": credentials}, + baseline_info={}, + ) + environment_role = EnvironmentRoleFactory.create( + environment=provisioned_environment, + status=EnvironmentRole.Status.PENDING, + role="my_role", + ) + + # When I call the user provisoning task + do_provision_user(csp=csp, environment_role_id=environment_role.id) + + session.refresh(environment_role) + # I expect that the CSP create_or_update_user method will be called + csp.create_or_update_user.assert_called_once_with( + credentials, environment_role, "my_role" + ) + # I expect that the EnvironmentRole now has a csp_user_id + assert environment_role.csp_user_id