Merge pull request #1393 from dod-ccpo/environment-role-creation

Environment role creation

.secrets.baseline

@@ -82,7 +82,7 @@
         "hashed_secret": "afc848c316af1a89d49826c5ae9d00ed769415f3",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 33,
+        "line_number": 43,
         "type": "Secret Keyword"
       }
     ],

alembic/versions/418b52c1cedf_change_to_environment_roles_cloud_id.py

@@ -0,0 +1,30 @@
+"""change to environment_roles.cloud_Id
+
+Revision ID: 418b52c1cedf
+Revises: 542bd3215dec
+Create Date: 2020-02-05 13:40:37.870183
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '418b52c1cedf' # pragma: allowlist secret
+down_revision = '542bd3215dec' # pragma: allowlist secret
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('environment_roles', sa.Column('cloud_id', sa.String(), nullable=True))
+    op.drop_column('environment_roles', 'csp_user_id')
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('environment_roles', sa.Column('csp_user_id', sa.VARCHAR(), autoincrement=False, nullable=True))
+    op.drop_column('environment_roles', 'cloud_id')
+    # ### end Alembic commands ###

atst/domain/csp/cloud/azure_cloud_provider.py

@@ -60,6 +60,8 @@ from .models import (
     TenantPrincipalOwnershipCSPResult,
     UserCSPPayload,
     UserCSPResult,
+    UserRoleCSPPayload,
+    UserRoleCSPResult,
 )
 from .policy import AzurePolicyManager
 
@@ -106,10 +108,14 @@ class AzureCloudProvider(CloudProviderInterface):
         self.secret_key = config["AZURE_SECRET_KEY"]
         self.tenant_id = config["AZURE_TENANT_ID"]
         self.vault_url = config["AZURE_VAULT_URL"]
-        self.ps_client_id = config["POWERSHELL_CLIENT_ID"]
+        self.ps_client_id = config["AZURE_POWERSHELL_CLIENT_ID"]
-        self.owner_role_def_id = config["AZURE_OWNER_ROLE_DEF_ID"]
         self.graph_resource = config["AZURE_GRAPH_RESOURCE"]
         self.default_aadp_qty = config["AZURE_AADP_QTY"]
+        self.roles = {
+            "owner": config["AZURE_ROLE_DEF_ID_OWNER"],
+            "contributor": config["AZURE_ROLE_DEF_ID_CONTRIBUTOR"],
+            "billing": config["AZURE_ROLE_DEF_ID_BILLING_READER"],
+        }
 
         if azure_sdk_provider is None:
             self.sdk = AzureSDKProvider()
@@ -620,7 +626,7 @@ class AzureCloudProvider(CloudProviderInterface):
     def create_tenant_admin_ownership(self, payload: TenantAdminOwnershipCSPPayload):
         mgmt_token = self._get_elevated_management_token(payload.tenant_id)
 
-        role_definition_id = f"/providers/Microsoft.Management/managementGroups/{payload.tenant_id}/providers/Microsoft.Authorization/roleDefinitions/{self.owner_role_def_id}"
+        role_definition_id = f"/providers/Microsoft.Management/managementGroups/{payload.tenant_id}/providers/Microsoft.Authorization/roleDefinitions/{self.roles['owner']}"
 
         request_body = {
             "properties": {
@@ -648,7 +654,7 @@ class AzureCloudProvider(CloudProviderInterface):
         mgmt_token = self._get_elevated_management_token(payload.tenant_id)
 
         # NOTE: the tenant_id is also the id of the root management group, once it is created
-        role_definition_id = f"/providers/Microsoft.Management/managementGroups/{payload.tenant_id}/providers/Microsoft.Authorization/roleDefinitions/{self.owner_role_def_id}"
+        role_definition_id = f"/providers/Microsoft.Management/managementGroups/{payload.tenant_id}/providers/Microsoft.Authorization/roleDefinitions/{self.roles['owner']}"
 
         request_body = {
             "properties": {
@@ -935,6 +941,40 @@ class AzureCloudProvider(CloudProviderInterface):
                 f"Failed update user email: {response.json()}"
             )
 
+    def create_user_role(self, payload: UserRoleCSPPayload):
+        graph_token = self._get_tenant_principal_token(payload.tenant_id)
+        if graph_token is None:
+            raise AuthenticationException(
+                "Could not resolve graph token for tenant admin"
+            )
+
+        role_guid = self.roles[payload.role]
+        role_definition_id = f"/providers/Microsoft.Management/managementGroups/{payload.management_group_id}/providers/Microsoft.Authorization/roleDefinitions/{role_guid}"
+
+        request_body = {
+            "properties": {
+                "roleDefinitionId": role_definition_id,
+                "principalId": payload.user_object_id,
+            }
+        }
+
+        auth_header = {
+            "Authorization": f"Bearer {graph_token}",
+        }
+
+        assignment_guid = str(uuid4())
+
+        url = f"{self.sdk.cloud.endpoints.resource_manager}/providers/Microsoft.Management/managementGroups/{payload.management_group_id}/providers/Microsoft.Authorization/roleAssignments/{assignment_guid}?api-version=2015-07-01"
+
+        response = self.sdk.requests.put(url, headers=auth_header, json=request_body)
+
+        if response.ok:
+            return UserRoleCSPResult(**response.json())
+        else:
+            raise UserProvisioningException(
+                f"Failed to create user role assignment: {response.json()}"
+            )
+
     def _extract_subscription_id(self, subscription_url):
         sub_id_match = SUBSCRIPTION_ID_REGEX.match(subscription_url)
 

atst/domain/csp/cloud/cloud_provider_interface.py

@@ -1,7 +1,7 @@
 from typing import Dict
 
 
-class CloudProviderInterface:
+class CloudProviderInterface:  # pragma: no cover
     def set_secret(self, secret_key: str, secret_value: str):
         raise NotImplementedError()
 

atst/domain/csp/cloud/models.py

@@ -1,3 +1,4 @@
+from enum import Enum
 from secrets import token_urlsafe
 from typing import Dict, List, Optional
 from uuid import uuid4
@@ -543,6 +544,21 @@ class UserCSPResult(AliasModel):
     id: str
 
 
+class UserRoleCSPPayload(BaseCSPPayload):
+    class Roles(str, Enum):
+        owner = "owner"
+        contributor = "contributor"
+        billing = "billing"
+
+    management_group_id: str
+    role: Roles
+    user_object_id: str
+
+
+class UserRoleCSPResult(AliasModel):
+    id: str
+
+
 class QueryColumn(AliasModel):
     name: str
     type: str

atst/domain/environment_roles.py

@@ -90,14 +90,18 @@ class EnvironmentRoles(object):
         )
 
     @classmethod
-    def get_environment_roles_pending_creation(cls) -> List[UUID]:
+    def get_pending_creation(cls) -> List[UUID]:
         results = (
             db.session.query(EnvironmentRole.id)
             .join(Environment)
             .join(ApplicationRole)
             .filter(Environment.deleted == False)
-            .filter(EnvironmentRole.status == EnvironmentRole.Status.PENDING)
+            .filter(EnvironmentRole.deleted == False)
-            .filter(ApplicationRole.status == ApplicationRoleStatus.ACTIVE)
+            .filter(ApplicationRole.deleted == False)
+            .filter(ApplicationRole.cloud_id != None)
+            .filter(ApplicationRole.status != ApplicationRoleStatus.DISABLED)
+            .filter(EnvironmentRole.status != EnvironmentRole.Status.DISABLED)
+            .filter(EnvironmentRole.cloud_id.is_(None))
             .all()
         )
         return [id_ for id_, in results]
@@ -106,7 +110,7 @@ class EnvironmentRoles(object):
     def disable(cls, environment_role_id):
         environment_role = EnvironmentRoles.get_by_id(environment_role_id)
 
-        if environment_role.csp_user_id and not environment_role.environment.cloud_id:
+        if environment_role.cloud_id and not environment_role.environment.cloud_id:
             tenant_id = environment_role.environment.application.portfolio.csp_data.get(
                 "tenant_id"
             )

atst/jobs.py

@@ -12,10 +12,12 @@ from atst.domain.csp.cloud.models import (
     ApplicationCSPPayload,
     EnvironmentCSPPayload,
     UserCSPPayload,
+    UserRoleCSPPayload,
 )
 from atst.domain.environments import Environments
+from atst.domain.environment_roles import EnvironmentRoles
 from atst.domain.portfolios import Portfolios
-from atst.models import JobFailure
+from atst.models import CSPRole, JobFailure
 from atst.domain.task_orders import TaskOrders
 from atst.models.utils import claim_for_update, claim_many_for_update
 from atst.queue import celery
@@ -124,13 +126,46 @@ def do_create_environment(csp: CloudProviderInterface, environment_id=None):
         payload = EnvironmentCSPPayload(
             tenant_id=tenant_id, display_name=environment.name, parent_id=parent_id
         )
-
         env_result = csp.create_environment(payload)
         environment.cloud_id = env_result.id
         db.session.add(environment)
         db.session.commit()
 
 
+def do_create_environment_role(csp: CloudProviderInterface, environment_role_id=None):
+    env_role = EnvironmentRoles.get_by_id(environment_role_id)
+
+    with claim_for_update(env_role) as env_role:
+
+        if env_role.cloud_id is not None:
+            return
+
+        env = env_role.environment
+        csp_details = env.application.portfolio.csp_data
+        app_role = env_role.application_role
+
+        role = None
+        if env_role.role == CSPRole.ADMIN:
+            role = UserRoleCSPPayload.Roles.owner
+        elif env_role.role == CSPRole.BILLING_READ:
+            role = UserRoleCSPPayload.Roles.billing
+        elif env_role.role == CSPRole.CONTRIBUTOR:
+            role = UserRoleCSPPayload.Roles.contributor
+
+        payload = UserRoleCSPPayload(
+            tenant_id=csp_details.get("tenant_id"),
+            management_group_id=env.cloud_id,
+            user_object_id=app_role.cloud_id,
+            role=role,
+        )
+        result = csp.create_user_role(payload)
+
+        env_role.cloud_id = result.id
+        db.session.add(env_role)
+        db.session.commit()
+        # TODO: should send notification email to the user, maybe with their portal login name
+
+
 def render_email(template_path, context):
     return app.jinja_env.get_template(template_path).render(context)
 
@@ -165,6 +200,16 @@ def create_user(self, application_role_ids=None):
     )
 
 
+@celery.task(bind=True, base=RecordFailure)
+def create_environment_role(self, environment_role_id=None):
+    do_work(
+        do_create_environment_role,
+        self,
+        app.csp.cloud,
+        environment_role_id=environment_role_id,
+    )
+
+
 @celery.task(bind=True, base=RecordFailure)
 def create_environment(self, environment_id=None):
     do_work(do_create_environment, self, app.csp.cloud, environment_id=environment_id)
@@ -191,6 +236,12 @@ def dispatch_create_user(self):
         create_user.delay(application_role_ids=application_role_ids)
 
 
+@celery.task(bind=True)
+def dispatch_create_environment_role(self):
+    for environment_role_id in EnvironmentRoles.get_pending_creation():
+        create_environment_role.delay(environment_role_id=environment_role_id)
+
+
 @celery.task(bind=True)
 def dispatch_create_environment(self):
     for environment_id in Environments.get_environments_pending_creation(

atst/models/environment_role.py

@@ -36,7 +36,7 @@ class EnvironmentRole(
     )
     application_role = relationship("ApplicationRole")
 
-    csp_user_id = Column(String())
+    cloud_id = Column(String())
 
     class Status(Enum):
         PENDING = "pending"

atst/queue.py

@@ -23,6 +23,10 @@ def update_celery(celery, app):
             "task": "atst.jobs.dispatch_create_user",
             "schedule": 60,
         },
+        "beat-dispatch_create_environment_role": {
+            "task": "atst.jobs.dispatch_create_environment_role",
+            "schedule": 60,
+        },
     }
 
     class ContextTask(celery.Task):

config/base.ini

@@ -1,9 +1,19 @@
 [default]
 ASSETS_URL
+AZURE_AADP_QTY=5
 AZURE_ACCOUNT_NAME
-AZURE_STORAGE_KEY
+AZURE_CLIENT_ID
-AZURE_TO_BUCKET_NAME
+AZURE_GRAPH_RESOURCE="https://graph.microsoft.com/"
 AZURE_POLICY_LOCATION=policies
+AZURE_POWERSHELL_CLIENT_ID
+AZURE_ROLE_DEF_ID_BILLING_READER="fa23ad8b-c56e-40d8-ac0c-ce449e1d2c64"
+AZURE_ROLE_DEF_ID_CONTRIBUTOR="b24988ac-6180-42a0-ab88-20f7382dd24c"
+AZURE_ROLE_DEF_ID_OWNER="8e3af657-a8ff-443c-a75c-2fe8c4bcb635"
+AZURE_SECRET_KEY
+AZURE_STORAGE_KEY
+AZURE_TENANT_ID
+AZURE_TO_BUCKET_NAME
+AZURE_VAULT_URL
 BLOB_STORAGE_URL=http://localhost:8000/
 CAC_URL = http://localhost:8000/login-redirect
 CA_CHAIN = ssl/server-certs/ca-chain.pem
@@ -43,10 +53,10 @@ REDIS_TLS=False
 REDIS_USER
 SECRET_KEY = change_me_into_something_secret
 SERVER_NAME
-SESSION_COOKIE_NAME=atat
 SESSION_COOKIE_DOMAIN
-SESSION_KEY_PREFIX=session:
+SESSION_COOKIE_NAME=atat
 SESSION_COOKIE_SECURE=false
+SESSION_KEY_PREFIX=session:
 SESSION_TYPE = redis
 SESSION_USE_SIGNER = True
 SQLALCHEMY_ECHO = False

tests/domain/cloud/test_azure_csp.py

@@ -58,7 +58,9 @@ from atst.domain.csp.cloud.models import (
     TenantPrincipalOwnershipCSPPayload,
     TenantPrincipalOwnershipCSPResult,
     UserCSPPayload,
+    UserRoleCSPPayload,
 )
+from atst.domain.csp.cloud.exceptions import UserProvisioningException
 
 BILLING_ACCOUNT_NAME = "52865e4c-52e8-5a6c-da6b-c58f0814f06f:7ea5de9d-b8ce-4901-b1c5-d864320c7b03_2019-05-31"
 
@@ -986,6 +988,54 @@ def test_create_user(mock_azure: AzureCloudProvider):
         assert result.id == "id"
 
 
+def test_create_user_role(mock_azure: AzureCloudProvider):
+    with patch.object(
+        AzureCloudProvider,
+        "_get_tenant_principal_token",
+        wraps=mock_azure._get_tenant_principal_token,
+    ) as _get_tenant_principal_token:
+        _get_tenant_principal_token.return_value = "token"
+
+        mock_result_create = Mock()
+        mock_result_create.ok = True
+        mock_result_create.json.return_value = {"id": "id"}
+        mock_azure.sdk.requests.put.return_value = mock_result_create
+
+        payload = UserRoleCSPPayload(
+            tenant_id=uuid4().hex,
+            user_object_id=str(uuid4()),
+            management_group_id=str(uuid4()),
+            role="owner",
+        )
+
+        result = mock_azure.create_user_role(payload)
+
+        assert result.id == "id"
+
+
+def test_create_user_role_failure(mock_azure: AzureCloudProvider):
+    with patch.object(
+        AzureCloudProvider,
+        "_get_tenant_principal_token",
+        wraps=mock_azure._get_tenant_principal_token,
+    ) as _get_tenant_principal_token:
+        _get_tenant_principal_token.return_value = "token"
+
+        mock_result_create = Mock()
+        mock_result_create.ok = False
+        mock_azure.sdk.requests.put.return_value = mock_result_create
+
+        payload = UserRoleCSPPayload(
+            tenant_id=uuid4().hex,
+            user_object_id=str(uuid4()),
+            management_group_id=str(uuid4()),
+            role="owner",
+        )
+
+        with pytest.raises(UserProvisioningException):
+            mock_azure.create_user_role(payload)
+
+
 def test_update_tenant_creds(mock_azure: AzureCloudProvider):
     with patch.object(
         AzureCloudProvider, "set_secret", wraps=mock_azure.set_secret,

tests/domain/test_environment_roles.py

@@ -1,7 +1,7 @@
 import pytest
 
 from atst.domain.environment_roles import EnvironmentRoles
-from atst.models.environment_role import EnvironmentRole
+from atst.models import EnvironmentRole, ApplicationRoleStatus
 
 from tests.factories import *
 
@@ -113,14 +113,14 @@ def test_disable_checks_env_role_provisioning_status():
     environment = EnvironmentFactory.create(cloud_id="cloud-id")
     environment.application.portfolio.csp_data = {"tenant_id": uuid4().hex}
     env_role1 = EnvironmentRoleFactory.create(environment=environment)
-    assert not env_role1.csp_user_id
+    assert not env_role1.cloud_id
     env_role1 = EnvironmentRoles.disable(env_role1.id)
     assert env_role1.disabled
 
     env_role2 = EnvironmentRoleFactory.create(
-        environment=environment, csp_user_id="123456"
+        environment=environment, cloud_id="123456"
     )
-    assert env_role2.csp_user_id
+    assert env_role2.cloud_id
     env_role2 = EnvironmentRoles.disable(env_role2.id)
     assert env_role2.disabled
 
@@ -159,3 +159,34 @@ def test_for_user(application_role):
     assert len(env_roles) == 3
     assert env_roles == [env_role_1, env_role_2, env_role_3]
     assert not rando_env_role in env_roles
+
+
+class TestPendingCreation:
+    def test_pending_role(self):
+        appr = ApplicationRoleFactory.create(cloud_id="123")
+        envr = EnvironmentRoleFactory.create(application_role=appr)
+        assert EnvironmentRoles.get_pending_creation() == [envr.id]
+
+    def test_deleted_role(self):
+        appr = ApplicationRoleFactory.create(cloud_id="123")
+        envr = EnvironmentRoleFactory.create(application_role=appr, deleted=True)
+        assert EnvironmentRoles.get_pending_creation() == []
+
+    def test_not_ready_role(self):
+        appr = ApplicationRoleFactory.create(cloud_id=None)
+        envr = EnvironmentRoleFactory.create(application_role=appr)
+        assert EnvironmentRoles.get_pending_creation() == []
+
+    def test_disabled_app_role(self):
+        appr = ApplicationRoleFactory.create(
+            cloud_id="123", status=ApplicationRoleStatus.DISABLED
+        )
+        envr = EnvironmentRoleFactory.create(application_role=appr)
+        assert EnvironmentRoles.get_pending_creation() == []
+
+    def test_disabled_env_role(self):
+        appr = ApplicationRoleFactory.create(cloud_id="123")
+        envr = EnvironmentRoleFactory.create(
+            application_role=appr, status=EnvironmentRole.Status.DISABLED
+        )
+        assert EnvironmentRoles.get_pending_creation() == []

tests/mock_azure.py

@@ -9,8 +9,10 @@ AZURE_CONFIG = {
     "AZURE_TENANT_ID": "MOCK",
     "AZURE_POLICY_LOCATION": "policies",
     "AZURE_VAULT_URL": "http://vault",
-    "POWERSHELL_CLIENT_ID": "MOCK",
+    "AZURE_POWERSHELL_CLIENT_ID": "MOCK",
-    "AZURE_OWNER_ROLE_DEF_ID": "MOCK",
+    "AZURE_ROLE_DEF_ID_OWNER": "MOCK",
+    "AZURE_ROLE_DEF_ID_CONTRIBUTOR": "MOCK",
+    "AZURE_ROLE_DEF_ID_BILLING_READER": "MOCK",
     "AZURE_GRAPH_RESOURCE": "MOCK",
     "AZURE_AADP_QTY": 5,
 }

tests/test_jobs.py

@@ -1,11 +1,12 @@
 import pendulum
 import pytest
 from uuid import uuid4
-from unittest.mock import Mock
+from unittest.mock import Mock, MagicMock
 from smtplib import SMTPException
 from azure.core.exceptions import AzureError
 
 from atst.domain.csp.cloud import MockCloudProvider
+from atst.domain.csp.cloud.models import UserRoleCSPResult
 from atst.domain.portfolios import Portfolios
 from atst.models import ApplicationRoleStatus
 
@@ -14,12 +15,14 @@ from atst.jobs import (
     dispatch_create_environment,
     dispatch_create_application,
     dispatch_create_user,
+    dispatch_create_environment_role,
     dispatch_provision_portfolio,
     dispatch_send_task_order_files,
     create_environment,
     do_create_user,
     do_provision_portfolio,
     do_create_environment,
+    do_create_environment_role,
     do_create_application,
 )
 from tests.factories import (
@@ -294,6 +297,42 @@ def test_provision_portfolio_create_tenant(
     # mock.delay.assert_called_once_with(portfolio_id=portfolio.id)
 
 
+def test_dispatch_create_environment_role(monkeypatch):
+    portfolio = PortfolioFactory.create(csp_data={"tenant_id": "123"})
+    app_role = ApplicationRoleFactory.create(
+        application=ApplicationFactory.create(portfolio=portfolio),
+        status=ApplicationRoleStatus.ACTIVE,
+        cloud_id="123",
+    )
+    env_role = EnvironmentRoleFactory.create(application_role=app_role)
+
+    mock = Mock()
+    monkeypatch.setattr("atst.jobs.create_environment_role", mock)
+
+    dispatch_create_environment_role.run()
+
+    mock.delay.assert_called_once_with(environment_role_id=env_role.id)
+
+
+def test_create_environment_role():
+    portfolio = PortfolioFactory.create(csp_data={"tenant_id": "123"})
+    app = ApplicationFactory.create(portfolio=portfolio)
+    app_role = ApplicationRoleFactory.create(
+        application=app, status=ApplicationRoleStatus.ACTIVE, cloud_id="123",
+    )
+    env = EnvironmentFactory.create(application=app, cloud_id="123")
+    env_role = EnvironmentRoleFactory.create(
+        environment=env, application_role=app_role, cloud_id=None
+    )
+
+    csp = Mock()
+    result = UserRoleCSPResult(id="a-cloud-id")
+    csp.create_user_role = MagicMock(return_value=result)
+    do_create_environment_role(csp, environment_role_id=env_role.id)
+
+    assert env_role.cloud_id == "a-cloud-id"
+
+
 # TODO: Refactor the tests related to dispatch_send_task_order_files() into a class
 # and separate the success test into two tests
 def test_dispatch_send_task_order_files(monkeypatch, app):