From 02438dc39bc3e09acaa2480e1f6f30dfdf1eb0f6 Mon Sep 17 00:00:00 2001 From: dandds Date: Sat, 25 Jan 2020 14:30:17 -0500 Subject: [PATCH 1/7] Query for applications that need to be provisioned. Adds a method to the Applications domain class that can return a list of UUIDs for applications that are ready to be provisioned. It requires that: - the associated portfolio and state machine have a state of COMPLETED - the application not have been marked deleted - the application not have an existing cloud_id - the application does not have an existing claim on it --- ...598199f6_add_applications_claimed_until.py | 30 +++++++++++++++++++ atst/domain/applications.py | 27 ++++++++++++++++- atst/models/application.py | 5 +++- tests/domain/test_applications.py | 18 +++++++++++ tests/factories.py | 8 +++++ 5 files changed, 86 insertions(+), 2 deletions(-) create mode 100644 alembic/versions/07e0598199f6_add_applications_claimed_until.py diff --git a/alembic/versions/07e0598199f6_add_applications_claimed_until.py b/alembic/versions/07e0598199f6_add_applications_claimed_until.py new file mode 100644 index 00000000..ada20eaf --- /dev/null +++ b/alembic/versions/07e0598199f6_add_applications_claimed_until.py @@ -0,0 +1,30 @@ +"""add applications.claimed_until + +Revision ID: 07e0598199f6 +Revises: 26319c44a8d5 +Create Date: 2020-01-25 13:33:17.711548 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = '07e0598199f6' # pragma: allowlist secret +down_revision = '26319c44a8d5' # pragma: allowlist secret +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('applications', sa.Column('claimed_until', sa.TIMESTAMP(timezone=True), nullable=True)) + op.add_column('applications', sa.Column('cloud_id', sa.String(), nullable=True)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('applications', 'claimed_until') + op.drop_column('applications', 'cloud_id') + # ### end Alembic commands ### diff --git a/atst/domain/applications.py b/atst/domain/applications.py index 3dbb9953..4e7f3c8a 100644 --- a/atst/domain/applications.py +++ b/atst/domain/applications.py @@ -1,5 +1,9 @@ -from . import BaseDomainClass from flask import g +from sqlalchemy import func, or_ +from typing import List +from uuid import UUID + +from . import BaseDomainClass from atst.database import db from atst.domain.application_roles import ApplicationRoles from atst.domain.environments import Environments @@ -10,7 +14,10 @@ from atst.models import ( ApplicationRole, ApplicationRoleStatus, EnvironmentRole, + Portfolio, + PortfolioStateMachine, ) +from atst.models.mixins.state_machines import FSMStates from atst.utils import first_or_none, commit_or_raise_already_exists_error @@ -118,3 +125,21 @@ class Applications(BaseDomainClass): db.session.commit() return invitation + + @classmethod + def get_applications_pending_creation(cls) -> List[UUID]: + results = ( + db.session.query(Application.id) + .join(Portfolio) + .join(PortfolioStateMachine) + .filter(PortfolioStateMachine.state == FSMStates.COMPLETED) + .filter(Application.deleted == False) + .filter(Application.cloud_id == None) + .filter( + or_( + Application.claimed_until == None, + Application.claimed_until >= func.now(), + ) + ) + ).all() + return [id_ for id_, in results] diff --git a/atst/models/application.py b/atst/models/application.py index a7bdadba..1af9e39f 100644 --- a/atst/models/application.py +++ b/atst/models/application.py @@ -1,4 +1,4 @@ -from sqlalchemy import and_, Column, ForeignKey, String, UniqueConstraint +from sqlalchemy import and_, Column, ForeignKey, String, UniqueConstraint, TIMESTAMP from sqlalchemy.orm import relationship, synonym from atst.models.base import Base @@ -40,6 +40,9 @@ class Application( ), ) + cloud_id = Column(String) + claimed_until = Column(TIMESTAMP(timezone=True)) + @property def users(self): return set(role.user for role in self.members) diff --git a/tests/domain/test_applications.py b/tests/domain/test_applications.py index 9fda3114..8ddc0867 100644 --- a/tests/domain/test_applications.py +++ b/tests/domain/test_applications.py @@ -1,3 +1,4 @@ +from datetime import datetime, timedelta import pytest from uuid import uuid4 @@ -196,3 +197,20 @@ def test_update_does_not_duplicate_names_within_portfolio(): with pytest.raises(AlreadyExistsError): Applications.update(dupe_application, {"name": name}) + + +def test_get_applications_pending_creation(): + now = datetime.now() + later = now + timedelta(minutes=30) + + portfolio1 = PortfolioFactory.create(state="COMPLETED") + app_ready = ApplicationFactory.create(portfolio=portfolio1) + + app_claimed = ApplicationFactory.create(portfolio=portfolio1, claimed_until=later) + + portfolio2 = PortfolioFactory.create(state="UNSTARTED") + app_not_ready = ApplicationFactory.create(portfolio=portfolio2) + + uuids = Applications.get_applications_pending_creation() + + assert [app_ready.id] == uuids diff --git a/tests/factories.py b/tests/factories.py index d9af7c40..b7a63243 100644 --- a/tests/factories.py +++ b/tests/factories.py @@ -7,6 +7,7 @@ import datetime from atst.forms import data from atst.models import * +from atst.models.mixins.state_machines import FSMStates from atst.domain.invitations import PortfolioInvitations from atst.domain.permission_sets import PermissionSets @@ -121,6 +122,7 @@ class PortfolioFactory(Base): owner = kwargs.pop("owner", UserFactory.create()) members = kwargs.pop("members", []) with_task_orders = kwargs.pop("task_orders", []) + state = kwargs.pop("state", None) portfolio = super()._create(model_class, *args, **kwargs) @@ -161,6 +163,12 @@ class PortfolioFactory(Base): permission_sets=perms_set, ) + if state: + state = getattr(FSMStates, state) + fsm = PortfolioStateMachineFactory.create(state=state, portfolio=portfolio) + # setting it in the factory is not working for some reason + fsm.state = state + portfolio.applications = applications portfolio.task_orders = task_orders return portfolio From bfc06920630442099318889722105c568d0b58ed Mon Sep 17 00:00:00 2001 From: dandds Date: Sat, 25 Jan 2020 15:50:01 -0500 Subject: [PATCH 2/7] Remove multiple job failure tables in favor of one. We don't know yet how useful the job failue tables will be, and maintaining multiple failure tables--one for every entity involved in CSP provisioning--is burdensome. This collapses them all into a single table that track the entity type (environment, portfolio, etc.) and the entity ID. That way we can construct queries when needed to find task results. --- .../508957112ed6_combine_job_failures.py | 60 +++++++++++++++++++ atst/jobs.py | 57 +++++++----------- atst/models/__init__.py | 6 +- atst/models/environment.py | 2 - atst/models/environment_role.py | 2 - atst/models/job_failure.py | 27 ++++----- atst/models/mixins/__init__.py | 1 - atst/models/mixins/job_failure.py | 14 ----- tests/test_jobs.py | 28 +++++---- 9 files changed, 114 insertions(+), 83 deletions(-) create mode 100644 alembic/versions/508957112ed6_combine_job_failures.py delete mode 100644 atst/models/mixins/job_failure.py diff --git a/alembic/versions/508957112ed6_combine_job_failures.py b/alembic/versions/508957112ed6_combine_job_failures.py new file mode 100644 index 00000000..9d40bb12 --- /dev/null +++ b/alembic/versions/508957112ed6_combine_job_failures.py @@ -0,0 +1,60 @@ +"""combine job failures + +Revision ID: 508957112ed6 +Revises: 07e0598199f6 +Create Date: 2020-01-25 15:03:06.377442 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = '508957112ed6' # pragma: allowlist secret +down_revision = '07e0598199f6' # pragma: allowlist secret +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('job_failures', + sa.Column('time_created', sa.TIMESTAMP(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('time_updated', sa.TIMESTAMP(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('task_id', sa.String(), nullable=False), + sa.Column('entity', sa.String(), nullable=False), + sa.Column('entity_id', sa.String(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.drop_table('environment_job_failures') + op.drop_table('environment_role_job_failures') + op.drop_table('portfolio_job_failures') + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('portfolio_job_failures', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('task_id', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column('portfolio_id', postgresql.UUID(), autoincrement=False, nullable=False), + sa.ForeignKeyConstraint(['portfolio_id'], ['portfolios.id'], name='portfolio_job_failures_portfolio_id_fkey'), + sa.PrimaryKeyConstraint('id', name='portfolio_job_failures_pkey') + ) + op.create_table('environment_role_job_failures', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('task_id', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column('environment_role_id', postgresql.UUID(), autoincrement=False, nullable=False), + sa.ForeignKeyConstraint(['environment_role_id'], ['environment_roles.id'], name='environment_role_job_failures_environment_role_id_fkey'), + sa.PrimaryKeyConstraint('id', name='environment_role_job_failures_pkey') + ) + op.create_table('environment_job_failures', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('task_id', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column('environment_id', postgresql.UUID(), autoincrement=False, nullable=False), + sa.ForeignKeyConstraint(['environment_id'], ['environments.id'], name='environment_job_failures_environment_id_fkey'), + sa.PrimaryKeyConstraint('id', name='environment_job_failures_pkey') + ) + op.drop_table('job_failures') + # ### end Alembic commands ### diff --git a/atst/jobs.py b/atst/jobs.py index ab52cf17..47fefb71 100644 --- a/atst/jobs.py +++ b/atst/jobs.py @@ -3,12 +3,7 @@ import pendulum from atst.database import db from atst.queue import celery -from atst.models import ( - EnvironmentJobFailure, - EnvironmentRoleJobFailure, - EnvironmentRole, - PortfolioJobFailure, -) +from atst.models import EnvironmentRole, JobFailure from atst.domain.csp.cloud import CloudProviderInterface, GeneralCSPException from atst.domain.environments import Environments from atst.domain.portfolios import Portfolios @@ -17,32 +12,26 @@ from atst.models.utils import claim_for_update from atst.utils.localization import translate -class RecordPortfolioFailure(celery.Task): +class RecordFailure(celery.Task): + _ENTITIES = [ + "portfolio_id", + "application_id", + "environment_id", + "environment_role_id", + ] + + def _derive_entity_info(self, kwargs): + matches = [e for e in self._ENTITIES if e in kwargs.keys()] + if matches: + match = matches[0] + return {"entity": match.replace("_id", ""), "entity_id": kwargs[match]} + else: + return None + def on_failure(self, exc, task_id, args, kwargs, einfo): - if "portfolio_id" in kwargs: - failure = PortfolioJobFailure( - portfolio_id=kwargs["portfolio_id"], task_id=task_id - ) - db.session.add(failure) - db.session.commit() - - -class RecordEnvironmentFailure(celery.Task): - def on_failure(self, exc, task_id, args, kwargs, einfo): - if "environment_id" in kwargs: - failure = EnvironmentJobFailure( - environment_id=kwargs["environment_id"], task_id=task_id - ) - db.session.add(failure) - db.session.commit() - - -class RecordEnvironmentRoleFailure(celery.Task): - def on_failure(self, exc, task_id, args, kwargs, einfo): - if "environment_role_id" in kwargs: - failure = EnvironmentRoleJobFailure( - environment_role_id=kwargs["environment_role_id"], task_id=task_id - ) + info = self._derive_entity_info(kwargs) + if info: + failure = JobFailure(**info, task_id=task_id) db.session.add(failure) db.session.commit() @@ -143,17 +132,17 @@ def do_provision_portfolio(csp: CloudProviderInterface, portfolio_id=None): fsm.trigger_next_transition() -@celery.task(bind=True, base=RecordPortfolioFailure) +@celery.task(bind=True, base=RecordFailure) def provision_portfolio(self, portfolio_id=None): do_work(do_provision_portfolio, self, app.csp.cloud, portfolio_id=portfolio_id) -@celery.task(bind=True, base=RecordEnvironmentFailure) +@celery.task(bind=True, base=RecordFailure) def create_environment(self, environment_id=None): do_work(do_create_environment, self, app.csp.cloud, environment_id=environment_id) -@celery.task(bind=True, base=RecordEnvironmentFailure) +@celery.task(bind=True, base=RecordFailure) def create_atat_admin_user(self, environment_id=None): do_work( do_create_atat_admin_user, self, app.csp.cloud, environment_id=environment_id diff --git a/atst/models/__init__.py b/atst/models/__init__.py index f6c48306..dfb1c19d 100644 --- a/atst/models/__init__.py +++ b/atst/models/__init__.py @@ -7,11 +7,7 @@ from .audit_event import AuditEvent from .clin import CLIN, JEDICLINType from .environment import Environment from .environment_role import EnvironmentRole, CSPRole -from .job_failure import ( - EnvironmentJobFailure, - EnvironmentRoleJobFailure, - PortfolioJobFailure, -) +from .job_failure import JobFailure from .notification_recipient import NotificationRecipient from .permissions import Permissions from .permission_set import PermissionSet diff --git a/atst/models/environment.py b/atst/models/environment.py index 115f3ed7..a0713c63 100644 --- a/atst/models/environment.py +++ b/atst/models/environment.py @@ -30,8 +30,6 @@ class Environment( claimed_until = Column(TIMESTAMP(timezone=True)) - job_failures = relationship("EnvironmentJobFailure") - roles = relationship( "EnvironmentRole", back_populates="environment", diff --git a/atst/models/environment_role.py b/atst/models/environment_role.py index 21f033e0..24aaeb7e 100644 --- a/atst/models/environment_role.py +++ b/atst/models/environment_role.py @@ -32,8 +32,6 @@ class EnvironmentRole( ) application_role = relationship("ApplicationRole") - job_failures = relationship("EnvironmentRoleJobFailure") - csp_user_id = Column(String()) claimed_until = Column(TIMESTAMP(timezone=True)) diff --git a/atst/models/job_failure.py b/atst/models/job_failure.py index 7a7f010a..5f9eee6c 100644 --- a/atst/models/job_failure.py +++ b/atst/models/job_failure.py @@ -1,22 +1,21 @@ -from sqlalchemy import Column, ForeignKey +from celery.result import AsyncResult +from sqlalchemy import Column, String, Integer from atst.models.base import Base import atst.models.mixins as mixins -class EnvironmentJobFailure(Base, mixins.JobFailureMixin): - __tablename__ = "environment_job_failures" +class JobFailure(Base, mixins.TimestampsMixin): + __tablename__ = "job_failures" - environment_id = Column(ForeignKey("environments.id"), nullable=False) + id = Column(Integer(), primary_key=True) + task_id = Column(String(), nullable=False) + entity = Column(String(), nullable=False) + entity_id = Column(String(), nullable=False) + @property + def task(self): + if not hasattr(self, "_task"): + self._task = AsyncResult(self.task_id) -class EnvironmentRoleJobFailure(Base, mixins.JobFailureMixin): - __tablename__ = "environment_role_job_failures" - - environment_role_id = Column(ForeignKey("environment_roles.id"), nullable=False) - - -class PortfolioJobFailure(Base, mixins.JobFailureMixin): - __tablename__ = "portfolio_job_failures" - - portfolio_id = Column(ForeignKey("portfolios.id"), nullable=False) + return self._task diff --git a/atst/models/mixins/__init__.py b/atst/models/mixins/__init__.py index 955171ab..e95b2516 100644 --- a/atst/models/mixins/__init__.py +++ b/atst/models/mixins/__init__.py @@ -3,5 +3,4 @@ from .auditable import AuditableMixin from .permissions import PermissionsMixin from .deletable import DeletableMixin from .invites import InvitesMixin -from .job_failure import JobFailureMixin from .state_machines import FSMMixin diff --git a/atst/models/mixins/job_failure.py b/atst/models/mixins/job_failure.py deleted file mode 100644 index c4f4cfa4..00000000 --- a/atst/models/mixins/job_failure.py +++ /dev/null @@ -1,14 +0,0 @@ -from celery.result import AsyncResult -from sqlalchemy import Column, String, Integer - - -class JobFailureMixin(object): - id = Column(Integer(), primary_key=True) - task_id = Column(String(), nullable=False) - - @property - def task(self): - if not hasattr(self, "_task"): - self._task = AsyncResult(self.task_id) - - return self._task diff --git a/tests/test_jobs.py b/tests/test_jobs.py index ff8e4602..9734bd75 100644 --- a/tests/test_jobs.py +++ b/tests/test_jobs.py @@ -8,8 +8,7 @@ from atst.domain.csp.cloud import MockCloudProvider from atst.domain.portfolios import Portfolios from atst.jobs import ( - RecordEnvironmentFailure, - RecordEnvironmentRoleFailure, + RecordFailure, dispatch_create_environment, dispatch_create_atat_admin_user, dispatch_provision_portfolio, @@ -29,7 +28,7 @@ from tests.factories import ( PortfolioStateMachineFactory, ApplicationRoleFactory, ) -from atst.models import CSPRole, EnvironmentRole, ApplicationRoleStatus +from atst.models import CSPRole, EnvironmentRole, ApplicationRoleStatus, JobFailure @pytest.fixture(autouse=True, scope="function") @@ -43,8 +42,17 @@ def portfolio(): return portfolio -def test_environment_job_failure(celery_app, celery_worker): - @celery_app.task(bind=True, base=RecordEnvironmentFailure) +def _find_failure(session, entity, id_): + return ( + session.query(JobFailure) + .filter(JobFailure.entity == entity) + .filter(JobFailure.entity_id == id_) + .one() + ) + + +def test_environment_job_failure(session, celery_app, celery_worker): + @celery_app.task(bind=True, base=RecordFailure) def _fail_hard(self, environment_id=None): raise ValueError("something bad happened") @@ -56,13 +64,12 @@ def test_environment_job_failure(celery_app, celery_worker): with pytest.raises(ValueError): task.get() - assert environment.job_failures - job_failure = environment.job_failures[0] + job_failure = _find_failure(session, "environment", str(environment.id)) assert job_failure.task == task -def test_environment_role_job_failure(celery_app, celery_worker): - @celery_app.task(bind=True, base=RecordEnvironmentRoleFailure) +def test_environment_role_job_failure(session, celery_app, celery_worker): + @celery_app.task(bind=True, base=RecordFailure) def _fail_hard(self, environment_role_id=None): raise ValueError("something bad happened") @@ -74,8 +81,7 @@ def test_environment_role_job_failure(celery_app, celery_worker): with pytest.raises(ValueError): task.get() - assert role.job_failures - job_failure = role.job_failures[0] + job_failure = _find_failure(session, "environment_role", str(role.id)) assert job_failure.task == task From 8810a59e0adc62736777986592063570287d834a Mon Sep 17 00:00:00 2001 From: dandds Date: Sat, 25 Jan 2020 17:29:17 -0500 Subject: [PATCH 3/7] Orchestration for creating app management groups. This adds: - A Celery beat task for enqueuing application creation tasks - A Celery task for creating the application - Payload and Response dataclasses for creating management groups It also does some incidental cleanup. --- atst/domain/csp/cloud.py | 113 +++++++++++++++++-- atst/jobs.py | 35 ++++++ atst/queue.py | 4 + tests/domain/cloud/test_azure_csp.py | 12 +- tests/domain/cloud/test_payloads.py | 72 ++++++++++++ tests/domain/test_portfolio_state_machine.py | 1 - tests/test_jobs.py | 36 ++++++ 7 files changed, 258 insertions(+), 15 deletions(-) create mode 100644 tests/domain/cloud/test_payloads.py diff --git a/atst/domain/csp/cloud.py b/atst/domain/csp/cloud.py index d22f9475..eff5a6d8 100644 --- a/atst/domain/csp/cloud.py +++ b/atst/domain/csp/cloud.py @@ -7,7 +7,6 @@ from pydantic import BaseModel, validator from flask import current_app as app from atst.models.user import User -from atst.models.application import Application from atst.models.environment import Environment from atst.models.environment_role import EnvironmentRole from atst.utils import snake_to_camel @@ -376,6 +375,60 @@ class BillingInstructionCSPResult(AliasModel): } +AZURE_MGMNT_PATH = "/providers/Microsoft.Management/managementGroups/" + +MANAGEMENT_GROUP_NAME_REGEX = "^[a-zA-Z0-9\-_\(\)\.]+$" + + +class ManagementGroupCSPPayload(BaseCSPPayload): + """ + :param: management_group_name: Just pass a UUID for this. + :param: display_name: This can contain any character and + spaces, but should be 90 characters or fewer long. + :param: parent_id: This should be the fully qualified Azure ID, + i.e. /providers/Microsoft.Management/managementGroups/[management group ID] + """ + + management_group_name: Optional[str] + display_name: str + parent_id: str + + @validator("management_group_name", pre=True, always=True) + def supply_management_group_name_default(cls, name): + if name: + if re.match(MANAGEMENT_GROUP_NAME_REGEX, name) is None: + raise ValueError( + f"Management group name must match {MANAGEMENT_GROUP_NAME_REGEX}" + ) + + return name[0:90] + else: + return str(uuid4()) + + @validator("display_name", pre=True, always=True) + def enforce_display_name_length(cls, name): + return name[0:90] + + @validator("parent_id", pre=True, always=True) + def enforce_parent_id_pattern(cls, id_): + if AZURE_MGMNT_PATH not in id_: + return f"{AZURE_MGMNT_PATH}{id_}" + else: + return id_ + + +class ManagementGroupCSPResponse(AliasModel): + id: str + + +class ApplicationCSPPayload(ManagementGroupCSPPayload): + pass + + +class ApplicationCSPResult(ManagementGroupCSPResponse): + pass + + class CloudProviderInterface: def set_secret(self, secret_key: str, secret_value: str): raise NotImplementedError() @@ -806,6 +859,15 @@ class MockCloudProvider(CloudProviderInterface): if self._with_authorization and credentials != self._auth_credentials: raise self.AUTHENTICATION_EXCEPTION + def create_application(self, payload: ApplicationCSPPayload): + self._maybe_raise(self.UNAUTHORIZED_RATE, GeneralCSPException) + + id_ = f"{AZURE_MGMNT_PATH}{payload.management_group_name}" + return ApplicationCSPResult(id=id_) + + def get_credentials(self, scope="portfolio", tenant_id=None): + return self.root_creds() + AZURE_ENVIRONMENT = "AZURE_PUBLIC_CLOUD" # TBD AZURE_SKU_ID = "?" # probably a static sku specific to ATAT/JEDI @@ -840,7 +902,7 @@ class AzureSDKProvider(object): self.graphrbac = graphrbac self.credentials = credentials self.identity = identity - self.exceptions = exceptions + # self.exceptions = exceptions self.secrets = secrets self.requests = requests # may change to a JEDI cloud @@ -908,7 +970,7 @@ class AzureCloudProvider(CloudProviderInterface): credentials, management_group_id, display_name, parent_id, ) - return management_group + return ManagementGroupCSPResponse(**management_group) def create_atat_admin_user( self, auth_credentials: Dict, csp_environment_id: str @@ -947,16 +1009,19 @@ class AzureCloudProvider(CloudProviderInterface): "role_name": role_assignment_id, } - def _create_application(self, auth_credentials: Dict, application: Application): - management_group_name = str(uuid4()) # can be anything, not just uuid - display_name = application.name # Does this need to be unique? - credentials = self._get_credential_obj(auth_credentials) - parent_id = "?" # application.portfolio.csp_details.management_group_id + def create_application(self, payload: ApplicationCSPPayload): + creds = payload.creds + credentials = self._get_credential_obj(creds, resource=AZURE_MANAGEMENT_API) - return self._create_management_group( - credentials, management_group_name, display_name, parent_id, + response = self._create_management_group( + credentials, + payload.management_group_name, + payload.display_name, + payload.parent_id, ) + return ApplicationCSPResult(**response) + def _create_management_group( self, credentials, management_group_id, display_name, parent_id=None, ): @@ -978,6 +1043,9 @@ class AzureCloudProvider(CloudProviderInterface): # result is a synchronous wait, might need to do a poll instead to handle first mgmt group create # since we were told it could take 10+ minutes to complete, unless this handles that polling internally + # TODO: what to do is status is not 'Succeeded' on the + # response object? Will it always raise its own error + # instead? return create_request.result() def _create_subscription( @@ -1289,6 +1357,7 @@ class AzureCloudProvider(CloudProviderInterface): # we likely only want the budget ID, can be updated or replaced? response = {"id": "id"} + return self._ok({"budget_id": response["id"]}) def _get_management_service_principal(self): @@ -1406,3 +1475,27 @@ class AzureCloudProvider(CloudProviderInterface): "secret_key": self.secret_key, "tenant_id": self.tenant_id, } + + def get_credentials(self, scope="portfolio", tenant_id=None): + """ + This could be implemented to determine, based on type, whether to return creds for: + - scope="atat": the ATAT main app registration in ATAT's home tenant + - scope="tenantadmin": the tenant administrator credentials + - scope="portfolio": the credentials for the ATAT SP in the portfolio tenant + """ + if scope == "atat": + return self._root_creds + elif scope == "tenantadmin": + # magic with key vault happens + return { + "client_id": "some id", + "secret_key": "very secret", + "tenant_id": tenant_id, + } + elif scope == "portfolio": + # magic with key vault happens + return { + "client_id": "some id", + "secret_key": "very secret", + "tenant_id": tenant_id, + } diff --git a/atst/jobs.py b/atst/jobs.py index 47fefb71..37f73450 100644 --- a/atst/jobs.py +++ b/atst/jobs.py @@ -5,11 +5,13 @@ from atst.database import db from atst.queue import celery from atst.models import EnvironmentRole, JobFailure from atst.domain.csp.cloud import CloudProviderInterface, GeneralCSPException +from atst.domain.applications import Applications from atst.domain.environments import Environments from atst.domain.portfolios import Portfolios from atst.domain.environment_roles import EnvironmentRoles from atst.models.utils import claim_for_update from atst.utils.localization import translate +from atst.domain.csp.cloud import ApplicationCSPPayload class RecordFailure(celery.Task): @@ -51,6 +53,28 @@ def send_notification_mail(recipients, subject, body): app.mailer.send(recipients, subject, body) +def do_create_application(csp: CloudProviderInterface, application_id=None): + application = Applications.get(application_id) + + with claim_for_update(application) as application: + + if application.cloud_id is not None: + return + + csp_details = application.portfolio.csp_data + parent_id = csp_details.get("root_management_group_id") + tenant_id = csp_details.get("tenant_id") + creds = csp.get_credentials(tenant_id) + payload = ApplicationCSPPayload( + creds=creds, display_name=application.name, parent_id=parent_id + ) + + app_result = csp.create_application(payload) + application.cloud_id = app_result.id + db.session.add(application) + db.session.commit() + + def do_create_environment(csp: CloudProviderInterface, environment_id=None): environment = Environments.get(environment_id) @@ -137,6 +161,11 @@ def provision_portfolio(self, portfolio_id=None): do_work(do_provision_portfolio, self, app.csp.cloud, portfolio_id=portfolio_id) +@celery.task(bind=True, base=RecordFailure) +def create_application(self, application_id=None): + do_work(do_create_application, self, app.csp.cloud, application_id=application_id) + + @celery.task(bind=True, base=RecordFailure) def create_environment(self, environment_id=None): do_work(do_create_environment, self, app.csp.cloud, environment_id=environment_id) @@ -165,6 +194,12 @@ def dispatch_provision_portfolio(self): provision_portfolio.delay(portfolio_id=portfolio_id) +@celery.task(bind=True) +def dispatch_create_application(self): + for application_id in Applications.get_applications_pending_creation(): + create_application.delay(application_id=application_id) + + @celery.task(bind=True) def dispatch_create_environment(self): for environment_id in Environments.get_environments_pending_creation( diff --git a/atst/queue.py b/atst/queue.py index 1dce690c..70718150 100644 --- a/atst/queue.py +++ b/atst/queue.py @@ -11,6 +11,10 @@ def update_celery(celery, app): "task": "atst.jobs.dispatch_provision_portfolio", "schedule": 60, }, + "beat-dispatch_create_application": { + "task": "atst.jobs.dispatch_create_application", + "schedule": 60, + }, "beat-dispatch_create_environment": { "task": "atst.jobs.dispatch_create_environment", "schedule": 60, diff --git a/tests/domain/cloud/test_azure_csp.py b/tests/domain/cloud/test_azure_csp.py index 9e87e7a7..c98a43ee 100644 --- a/tests/domain/cloud/test_azure_csp.py +++ b/tests/domain/cloud/test_azure_csp.py @@ -18,6 +18,7 @@ from atst.domain.csp.cloud import ( TaskOrderBillingVerificationCSPResult, TenantCSPPayload, TenantCSPResult, + ApplicationCSPPayload, ) from tests.mock_azure import mock_azure, AUTH_CREDENTIALS @@ -67,8 +68,8 @@ def test_create_subscription_succeeds(mock_azure: AzureCloudProvider): def mock_management_group_create(mock_azure, spec_dict): - mock_azure.sdk.managementgroups.ManagementGroupsAPI.return_value.management_groups.create_or_update.return_value.result.return_value = Mock( - **spec_dict + mock_azure.sdk.managementgroups.ManagementGroupsAPI.return_value.management_groups.create_or_update.return_value.result.return_value = ( + spec_dict ) @@ -89,7 +90,10 @@ def test_create_application_succeeds(mock_azure: AzureCloudProvider): mock_management_group_create(mock_azure, {"id": "Test Id"}) - result = mock_azure._create_application(AUTH_CREDENTIALS, application) + payload = ApplicationCSPPayload( + creds={}, display_name=application.name, parent_id=str(uuid4()) + ) + result = mock_azure.create_application(payload) assert result.id == "Test Id" @@ -150,7 +154,7 @@ def test_create_tenant(mock_azure: AzureCloudProvider): **dict( creds=creds, user_id="admin", - password="JediJan13$coot", + password="JediJan13$coot", # pragma: allowlist secret domain_name="jediccpospawnedtenant2", first_name="Tedry", last_name="Tenet", diff --git a/tests/domain/cloud/test_payloads.py b/tests/domain/cloud/test_payloads.py new file mode 100644 index 00000000..08ca147c --- /dev/null +++ b/tests/domain/cloud/test_payloads.py @@ -0,0 +1,72 @@ +import pytest + +from pydantic import ValidationError + +from atst.domain.csp.cloud import ( + AZURE_MGMNT_PATH, + ManagementGroupCSPPayload, + ManagementGroupCSPResponse, +) + + +def test_ManagementGroupCSPPayload_management_group_name(): + # supplies management_group_name when absent + payload = ManagementGroupCSPPayload( + creds={}, display_name="Council of Naboo", parent_id="Galactic_Senate" + ) + assert payload.management_group_name + # validates management_group_name + with pytest.raises(ValidationError): + payload = ManagementGroupCSPPayload( + creds={}, + management_group_name="council of Naboo 1%^&", + display_name="Council of Naboo", + parent_id="Galactic_Senate", + ) + # shortens management_group_name to fit + name = "council_of_naboo" + for _ in range(90): + name = f"{name}1" + + assert len(name) > 90 + payload = ManagementGroupCSPPayload( + creds={}, + management_group_name=name, + display_name="Council of Naboo", + parent_id="Galactic_Senate", + ) + assert len(payload.management_group_name) == 90 + + +def test_ManagementGroupCSPPayload_display_name(): + # shortens display_name to fit + name = "Council of Naboo" + for _ in range(90): + name = f"{name}1" + assert len(name) > 90 + payload = ManagementGroupCSPPayload( + creds={}, display_name=name, parent_id="Galactic_Senate" + ) + assert len(payload.display_name) == 90 + + +def test_ManagementGroupCSPPayload_parent_id(): + full_path = f"{AZURE_MGMNT_PATH}Galactic_Senate" + # adds full path + payload = ManagementGroupCSPPayload( + creds={}, display_name="Council of Naboo", parent_id="Galactic_Senate" + ) + assert payload.parent_id == full_path + # keeps full path + payload = ManagementGroupCSPPayload( + creds={}, display_name="Council of Naboo", parent_id=full_path + ) + assert payload.parent_id == full_path + + +def test_ManagementGroupCSPResponse_id(): + full_id = "/path/to/naboo-123" + response = ManagementGroupCSPResponse( + **{"id": "/path/to/naboo-123", "other": "stuff"} + ) + assert response.id == full_id diff --git a/tests/domain/test_portfolio_state_machine.py b/tests/domain/test_portfolio_state_machine.py index 0d37c9c7..330d5195 100644 --- a/tests/domain/test_portfolio_state_machine.py +++ b/tests/domain/test_portfolio_state_machine.py @@ -141,7 +141,6 @@ def test_fsm_transition_start(portfolio: Portfolio): config = {"billing_account_name": "billing_account_name"} for expected_state in expected_states: - print(expected_state) collected_data = dict( list(csp_data.items()) + list(portfolio_data.items()) + list(config.items()) ) diff --git a/tests/test_jobs.py b/tests/test_jobs.py index 9734bd75..2ac5f408 100644 --- a/tests/test_jobs.py +++ b/tests/test_jobs.py @@ -10,6 +10,7 @@ from atst.domain.portfolios import Portfolios from atst.jobs import ( RecordFailure, dispatch_create_environment, + dispatch_create_application, dispatch_create_atat_admin_user, dispatch_provision_portfolio, dispatch_provision_user, @@ -17,6 +18,7 @@ from atst.jobs import ( do_provision_user, do_provision_portfolio, do_create_environment, + do_create_application, do_create_atat_admin_user, ) from atst.models.utils import claim_for_update @@ -26,6 +28,7 @@ from tests.factories import ( EnvironmentRoleFactory, PortfolioFactory, PortfolioStateMachineFactory, + ApplicationFactory, ApplicationRoleFactory, ) from atst.models import CSPRole, EnvironmentRole, ApplicationRoleStatus, JobFailure @@ -105,6 +108,24 @@ def test_create_environment_job_is_idempotent(csp, session): csp.create_environment.assert_not_called() +def test_create_application_job(session, csp): + portfolio = PortfolioFactory.create( + csp_data={"tenant_id": str(uuid4()), "root_management_group_id": str(uuid4())} + ) + application = ApplicationFactory.create(portfolio=portfolio, cloud_id=None) + do_create_application(csp, application.id) + session.refresh(application) + + assert application.cloud_id + + +def test_create_application_job_is_idempotent(csp): + application = ApplicationFactory.create(cloud_id=uuid4()) + do_create_application(csp, application.id) + + csp.create_application.assert_not_called() + + def test_create_atat_admin_user(csp, session): environment = EnvironmentFactory.create(cloud_id="something") do_create_atat_admin_user(csp, environment.id) @@ -145,6 +166,21 @@ def test_dispatch_create_environment(session, monkeypatch): mock.delay.assert_called_once_with(environment_id=e1.id) +def test_dispatch_create_application(monkeypatch): + portfolio = PortfolioFactory.create(state="COMPLETED") + app = ApplicationFactory.create(portfolio=portfolio) + + mock = Mock() + monkeypatch.setattr("atst.jobs.create_application", mock) + + # When dispatch_create_application is called + dispatch_create_application.run() + + # It should cause the create_application task to be called once + # with the application id + mock.delay.assert_called_once_with(application_id=app.id) + + def test_dispatch_create_atat_admin_user(session, monkeypatch): portfolio = PortfolioFactory.create( applications=[ From ff172b43b257b7d66a5da8d67cb83ffd9c3bbd78 Mon Sep 17 00:00:00 2001 From: dandds Date: Sun, 26 Jan 2020 12:45:18 -0500 Subject: [PATCH 4/7] Fix some import errors. There is an issue with circular imports because the PortfolioStateMachine model imports some error classes from the cloud module. The cloud module was importing some other models in turn, which was causing the issue. Since we plan to pass all data as dataclass payloads to the cloud interfacem, I removed the type hints that referenced specific SQLAlchemy models and removed the imports. --- atst/domain/csp/cloud.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/atst/domain/csp/cloud.py b/atst/domain/csp/cloud.py index eff5a6d8..b106e9ee 100644 --- a/atst/domain/csp/cloud.py +++ b/atst/domain/csp/cloud.py @@ -6,9 +6,6 @@ from pydantic import BaseModel, validator from flask import current_app as app -from atst.models.user import User -from atst.models.environment import Environment -from atst.models.environment_role import EnvironmentRole from atst.utils import snake_to_camel from .policy import AzurePolicyManager @@ -439,9 +436,7 @@ class CloudProviderInterface: def root_creds(self) -> Dict: raise NotImplementedError() - def create_environment( - self, auth_credentials: Dict, user: User, environment: Environment - ) -> str: + def create_environment(self, auth_credentials: Dict, user, environment) -> str: """Create a new environment in the CSP. Arguments: @@ -489,7 +484,7 @@ class CloudProviderInterface: raise NotImplementedError() def create_or_update_user( - self, auth_credentials: Dict, user_info: EnvironmentRole, csp_role_id: str + self, auth_credentials: Dict, user_info, csp_role_id: str ) -> str: """Creates a user or updates an existing user's role. @@ -889,6 +884,7 @@ class AzureSDKProvider(object): import azure.common.credentials as credentials import azure.identity as identity from azure.keyvault import secrets + from azure.core import exceptions from msrestazure.azure_cloud import AZURE_PUBLIC_CLOUD import adal @@ -902,7 +898,7 @@ class AzureSDKProvider(object): self.graphrbac = graphrbac self.credentials = credentials self.identity = identity - # self.exceptions = exceptions + self.exceptions = exceptions self.secrets = secrets self.requests = requests # may change to a JEDI cloud @@ -951,9 +947,7 @@ class AzureCloudProvider(CloudProviderInterface): exc_info=1, ) - def create_environment( - self, auth_credentials: Dict, user: User, environment: Environment - ): + def create_environment(self, auth_credentials: Dict, user, environment): # since this operation would only occur within a tenant, should we source the tenant # via lookup from environment once we've created the portfolio csp data schema # something like this: From 37a5218a1d40cb04545d862c58fba2b3e1d633da Mon Sep 17 00:00:00 2001 From: dandds Date: Sun, 26 Jan 2020 18:38:02 -0500 Subject: [PATCH 5/7] Split and barrel cloud module for merge. --- atst/domain/csp/__init__.py | 4 +- atst/domain/csp/cloud.py | 1495 ----------------- atst/domain/csp/cloud/__init__.py | 3 + atst/domain/csp/cloud/azure_cloud_provider.py | 657 ++++++++ .../csp/cloud/cloud_provider_interface.py | 120 ++ atst/domain/csp/cloud/exceptions.py | 131 ++ atst/domain/csp/cloud/mock_cloud_provider.py | 354 ++++ atst/domain/csp/cloud/models.py | 290 ++++ atst/domain/csp/{ => cloud}/policy.py | 0 atst/jobs.py | 5 +- atst/models/portfolio_state_machine.py | 2 +- atst/routes/applications/settings.py | 2 +- script/include/test_functions.inc.sh | 2 +- tests/domain/cloud/test_azure_csp.py | 18 +- tests/domain/cloud/test_payloads.py | 2 +- tests/domain/cloud/test_policy.py | 2 +- tests/routes/applications/test_settings.py | 2 +- 17 files changed, 1577 insertions(+), 1512 deletions(-) delete mode 100644 atst/domain/csp/cloud.py create mode 100644 atst/domain/csp/cloud/__init__.py create mode 100644 atst/domain/csp/cloud/azure_cloud_provider.py create mode 100644 atst/domain/csp/cloud/cloud_provider_interface.py create mode 100644 atst/domain/csp/cloud/exceptions.py create mode 100644 atst/domain/csp/cloud/mock_cloud_provider.py create mode 100644 atst/domain/csp/cloud/models.py rename atst/domain/csp/{ => cloud}/policy.py (100%) diff --git a/atst/domain/csp/__init__.py b/atst/domain/csp/__init__.py index f15ac1cd..62b28f94 100644 --- a/atst/domain/csp/__init__.py +++ b/atst/domain/csp/__init__.py @@ -45,6 +45,8 @@ def get_stage_csp_class(stage, class_type): """ cls_name = f"{_stage_to_classname(stage)}CSP{class_type.capitalize()}" try: - return getattr(importlib.import_module("atst.domain.csp.cloud"), cls_name) + return getattr( + importlib.import_module("atst.domain.csp.cloud.models"), cls_name + ) except AttributeError: print("could not import CSP Result class <%s>" % cls_name) diff --git a/atst/domain/csp/cloud.py b/atst/domain/csp/cloud.py deleted file mode 100644 index b106e9ee..00000000 --- a/atst/domain/csp/cloud.py +++ /dev/null @@ -1,1495 +0,0 @@ -import re -from typing import Dict, List, Optional -from uuid import uuid4 - -from pydantic import BaseModel, validator - -from flask import current_app as app - -from atst.utils import snake_to_camel -from .policy import AzurePolicyManager - - -class GeneralCSPException(Exception): - pass - - -class OperationInProgressException(GeneralCSPException): - """Throw this for instances when the CSP reports that the current entity is already - being operated on/created/deleted/etc - """ - - def __init__(self, operation_desc): - self.operation_desc = operation_desc - - @property - def message(self): - return "An operation for this entity is already in progress: {}".format( - self.operation_desc - ) - - -class AuthenticationException(GeneralCSPException): - """Throw this for instances when there is a problem with the auth credentials: - * Missing credentials - * Incorrect credentials - * Other credential problems - """ - - def __init__(self, auth_error): - self.auth_error = auth_error - - @property - def message(self): - return "An error occurred with authentication: {}".format(self.auth_error) - - -class AuthorizationException(GeneralCSPException): - """Throw this for instances when the current credentials are not authorized - for the current action. - """ - - def __init__(self, auth_error): - self.auth_error = auth_error - - @property - def message(self): - return "An error occurred with authorization: {}".format(self.auth_error) - - -class ConnectionException(GeneralCSPException): - """A general problem with the connection, timeouts or unresolved endpoints - """ - - def __init__(self, connection_error): - self.connection_error = connection_error - - @property - def message(self): - return "Could not connect to cloud provider: {}".format(self.connection_error) - - -class UnknownServerException(GeneralCSPException): - """An error occured on the CSP side (5xx) and we don't know why - """ - - def __init__(self, server_error): - self.server_error = server_error - - @property - def message(self): - return "A server error occured: {}".format(self.server_error) - - -class EnvironmentCreationException(GeneralCSPException): - """If there was an error in creating the environment - """ - - def __init__(self, env_identifier, reason): - self.env_identifier = env_identifier - self.reason = reason - - @property - def message(self): - return "The envionment {} couldn't be created: {}".format( - self.env_identifier, self.reason - ) - - -class UserProvisioningException(GeneralCSPException): - """Failed to provision a user - """ - - def __init__(self, env_identifier, user_identifier, reason): - self.env_identifier = env_identifier - self.user_identifier = user_identifier - self.reason = reason - - @property - def message(self): - return "Failed to create user {} for environment {}: {}".format( - self.user_identifier, self.env_identifier, self.reason - ) - - -class UserRemovalException(GeneralCSPException): - """Failed to remove a user - """ - - def __init__(self, user_csp_id, reason): - self.user_csp_id = user_csp_id - self.reason = reason - - @property - def message(self): - return "Failed to suspend or delete user {}: {}".format( - self.user_csp_id, self.reason - ) - - -class BaselineProvisionException(GeneralCSPException): - """If there's any issues standing up whatever is required - for an environment baseline - """ - - def __init__(self, env_identifier, reason): - self.env_identifier = env_identifier - self.reason = reason - - @property - def message(self): - return "Could not complete baseline provisioning for environment ({}): {}".format( - self.env_identifier, self.reason - ) - - -class AliasModel(BaseModel): - """ - This provides automatic camel <-> snake conversion for serializing to/from json - You can override the alias generation in subclasses by providing a Config that defines - a fields property with a dict mapping variables to their cast names, for cases like: - * some_url:someURL - * user_object_id:objectId - """ - - class Config: - alias_generator = snake_to_camel - allow_population_by_field_name = True - - -class BaseCSPPayload(AliasModel): - # {"username": "mock-cloud", "pass": "shh"} - creds: Dict - - def dict(self, *args, **kwargs): - exclude = {"creds"} - if "exclude" not in kwargs: - kwargs["exclude"] = exclude - else: - kwargs["exclude"].update(exclude) - - return super().dict(*args, **kwargs) - - -class TenantCSPPayload(BaseCSPPayload): - user_id: str - password: str - domain_name: str - first_name: str - last_name: str - country_code: str - password_recovery_email_address: str - - -class TenantCSPResult(AliasModel): - user_id: str - tenant_id: str - user_object_id: str - - tenant_admin_username: Optional[str] - tenant_admin_password: Optional[str] - - class Config: - fields = { - "user_object_id": "objectId", - } - - def dict(self, *args, **kwargs): - exclude = {"tenant_admin_username", "tenant_admin_password"} - if "exclude" not in kwargs: - kwargs["exclude"] = exclude - else: - kwargs["exclude"].update(exclude) - - return super().dict(*args, **kwargs) - - def get_creds(self): - return { - "tenant_admin_username": self.tenant_admin_username, - "tenant_admin_password": self.tenant_admin_password, - "tenant_id": self.tenant_id, - } - - -class BillingProfileAddress(AliasModel): - company_name: str - address_line_1: str - city: str - region: str - country: str - postal_code: str - - -class BillingProfileCLINBudget(AliasModel): - clin_budget: Dict - """ - "clinBudget": { - "amount": 0, - "startDate": "2019-12-18T16:47:40.909Z", - "endDate": "2019-12-18T16:47:40.909Z", - "externalReferenceId": "string" - } - """ - - -class BillingProfileCreationCSPPayload(BaseCSPPayload): - tenant_id: str - billing_profile_display_name: str - billing_account_name: str - enabled_azure_plans: Optional[List[str]] - address: BillingProfileAddress - - @validator("enabled_azure_plans", pre=True, always=True) - def default_enabled_azure_plans(cls, v): - """ - Normally you'd implement this by setting the field with a value of: - dataclasses.field(default_factory=list) - but that prevents the object from being correctly pickled, so instead we need - to rely on a validator to ensure this has an empty value when not specified - """ - return v or [] - - class Config: - fields = {"billing_profile_display_name": "displayName"} - - -class BillingProfileCreationCSPResult(AliasModel): - billing_profile_verify_url: str - billing_profile_retry_after: int - - class Config: - fields = { - "billing_profile_verify_url": "Location", - "billing_profile_retry_after": "Retry-After", - } - - -class BillingProfileVerificationCSPPayload(BaseCSPPayload): - billing_profile_verify_url: str - - -class BillingInvoiceSection(AliasModel): - invoice_section_id: str - invoice_section_name: str - - class Config: - fields = {"invoice_section_id": "id", "invoice_section_name": "name"} - - -class BillingProfileProperties(AliasModel): - address: BillingProfileAddress - billing_profile_display_name: str - invoice_sections: List[BillingInvoiceSection] - - class Config: - fields = {"billing_profile_display_name": "displayName"} - - -class BillingProfileVerificationCSPResult(AliasModel): - billing_profile_id: str - billing_profile_name: str - billing_profile_properties: BillingProfileProperties - - class Config: - fields = { - "billing_profile_id": "id", - "billing_profile_name": "name", - "billing_profile_properties": "properties", - } - - -class BillingProfileTenantAccessCSPPayload(BaseCSPPayload): - tenant_id: str - user_object_id: str - billing_account_name: str - billing_profile_name: str - - -class BillingProfileTenantAccessCSPResult(AliasModel): - billing_role_assignment_id: str - billing_role_assignment_name: str - - class Config: - fields = { - "billing_role_assignment_id": "id", - "billing_role_assignment_name": "name", - } - - -class TaskOrderBillingCreationCSPPayload(BaseCSPPayload): - billing_account_name: str - billing_profile_name: str - - -class TaskOrderBillingCreationCSPResult(AliasModel): - task_order_billing_verify_url: str - task_order_retry_after: int - - class Config: - fields = { - "task_order_billing_verify_url": "Location", - "task_order_retry_after": "Retry-After", - } - - -class TaskOrderBillingVerificationCSPPayload(BaseCSPPayload): - task_order_billing_verify_url: str - - -class BillingProfileEnabledPlanDetails(AliasModel): - enabled_azure_plans: List[Dict] - - -class TaskOrderBillingVerificationCSPResult(AliasModel): - billing_profile_id: str - billing_profile_name: str - billing_profile_enabled_plan_details: BillingProfileEnabledPlanDetails - - class Config: - fields = { - "billing_profile_id": "id", - "billing_profile_name": "name", - "billing_profile_enabled_plan_details": "properties", - } - - -class BillingInstructionCSPPayload(BaseCSPPayload): - initial_clin_amount: float - initial_clin_start_date: str - initial_clin_end_date: str - initial_clin_type: str - initial_task_order_id: str - billing_account_name: str - billing_profile_name: str - - -class BillingInstructionCSPResult(AliasModel): - reported_clin_name: str - - class Config: - fields = { - "reported_clin_name": "name", - } - - -AZURE_MGMNT_PATH = "/providers/Microsoft.Management/managementGroups/" - -MANAGEMENT_GROUP_NAME_REGEX = "^[a-zA-Z0-9\-_\(\)\.]+$" - - -class ManagementGroupCSPPayload(BaseCSPPayload): - """ - :param: management_group_name: Just pass a UUID for this. - :param: display_name: This can contain any character and - spaces, but should be 90 characters or fewer long. - :param: parent_id: This should be the fully qualified Azure ID, - i.e. /providers/Microsoft.Management/managementGroups/[management group ID] - """ - - management_group_name: Optional[str] - display_name: str - parent_id: str - - @validator("management_group_name", pre=True, always=True) - def supply_management_group_name_default(cls, name): - if name: - if re.match(MANAGEMENT_GROUP_NAME_REGEX, name) is None: - raise ValueError( - f"Management group name must match {MANAGEMENT_GROUP_NAME_REGEX}" - ) - - return name[0:90] - else: - return str(uuid4()) - - @validator("display_name", pre=True, always=True) - def enforce_display_name_length(cls, name): - return name[0:90] - - @validator("parent_id", pre=True, always=True) - def enforce_parent_id_pattern(cls, id_): - if AZURE_MGMNT_PATH not in id_: - return f"{AZURE_MGMNT_PATH}{id_}" - else: - return id_ - - -class ManagementGroupCSPResponse(AliasModel): - id: str - - -class ApplicationCSPPayload(ManagementGroupCSPPayload): - pass - - -class ApplicationCSPResult(ManagementGroupCSPResponse): - pass - - -class CloudProviderInterface: - def set_secret(self, secret_key: str, secret_value: str): - raise NotImplementedError() - - def get_secret(self, secret_key: str): - raise NotImplementedError() - - def root_creds(self) -> Dict: - raise NotImplementedError() - - def create_environment(self, auth_credentials: Dict, user, environment) -> str: - """Create a new environment in the CSP. - - Arguments: - auth_credentials -- Object containing CSP account credentials - user -- ATAT user authorizing the environment creation - environment -- ATAT Environment model - - Returns: - string: ID of created environment - - Raises: - AuthenticationException: Problem with the credentials - AuthorizationException: Credentials not authorized for current action(s) - ConnectionException: Issue with the CSP API connection - UnknownServerException: Unknown issue on the CSP side - EnvironmentExistsException: Environment already exists and has been created - """ - raise NotImplementedError() - - def create_atat_admin_user( - self, auth_credentials: Dict, csp_environment_id: str - ) -> Dict: - """Creates a new, programmatic user in the CSP. Grants this user full permissions to administer - the CSP. - - Arguments: - auth_credentials -- Object containing CSP account credentials - csp_environment_id -- ID of the CSP Environment the admin user should be created in - - Returns: - object: Object representing new remote admin user, including credentials - Something like: - { - "user_id": string, - "credentials": dict, # structure TBD based on csp - } - - Raises: - AuthenticationException: Problem with the credentials - AuthorizationException: Credentials not authorized for current action(s) - ConnectionException: Issue with the CSP API connection - UnknownServerException: Unknown issue on the CSP side - UserProvisioningException: Problem creating the root user - """ - raise NotImplementedError() - - def create_or_update_user( - self, auth_credentials: Dict, user_info, csp_role_id: str - ) -> str: - """Creates a user or updates an existing user's role. - - Arguments: - auth_credentials -- Object containing CSP account credentials - user_info -- instance of EnvironmentRole containing user data - if it has a csp_user_id it will try to update that user - csp_role_id -- The id of the role the user should be given in the CSP - - Returns: - string: Returns the interal csp_user_id of the created/updated user account - - Raises: - AuthenticationException: Problem with the credentials - AuthorizationException: Credentials not authorized for current action(s) - ConnectionException: Issue with the CSP API connection - UnknownServerException: Unknown issue on the CSP side - UserProvisioningException: User couldn't be created or modified - """ - raise NotImplementedError() - - def disable_user(self, auth_credentials: Dict, csp_user_id: str) -> bool: - """Revoke all privileges for a user. Used to prevent user access while a full - delete is being processed. - - Arguments: - auth_credentials -- Object containing CSP account credentials - csp_user_id -- CSP internal user identifier - - Returns: - bool -- True on success - - Raises: - AuthenticationException: Problem with the credentials - AuthorizationException: Credentials not authorized for current action(s) - ConnectionException: Issue with the CSP API connection - UnknownServerException: Unknown issue on the CSP side - UserRemovalException: User couldn't be suspended - """ - raise NotImplementedError() - - def get_calculator_url(self) -> str: - """Returns the calculator url for the CSP. - This will likely be a static property elsewhere once a CSP is chosen. - """ - raise NotImplementedError() - - def get_environment_login_url(self, environment) -> str: - """Returns the login url for a given environment - This may move to be a computed property on the Environment domain object - """ - raise NotImplementedError() - - def create_subscription(self, environment): - """Returns True if a new subscription has been created or raises an - exception if an error occurs while creating a subscription. - """ - raise NotImplementedError() - - -class MockCloudProvider(CloudProviderInterface): - - # TODO: All of these constants - AUTHENTICATION_EXCEPTION = AuthenticationException("Authentication failure.") - AUTHORIZATION_EXCEPTION = AuthorizationException("Not authorized.") - NETWORK_EXCEPTION = ConnectionException("Network failure.") - SERVER_EXCEPTION = UnknownServerException("Not our fault.") - - SERVER_FAILURE_PCT = 1 - NETWORK_FAILURE_PCT = 7 - ENV_CREATE_FAILURE_PCT = 12 - ATAT_ADMIN_CREATE_FAILURE_PCT = 12 - UNAUTHORIZED_RATE = 2 - - def __init__( - self, config, with_delay=True, with_failure=True, with_authorization=True - ): - from time import sleep - import random - - self._with_delay = with_delay - self._with_failure = with_failure - self._with_authorization = with_authorization - self._sleep = sleep - self._random = random - - def root_creds(self): - return self._auth_credentials - - def set_secret(self, secret_key: str, secret_value: str): - pass - - def get_secret(self, secret_key: str): - return {} - - def create_environment(self, auth_credentials, user, environment): - self._authorize(auth_credentials) - - self._delay(1, 5) - self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) - self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) - self._maybe_raise( - self.ENV_CREATE_FAILURE_PCT, - EnvironmentCreationException( - environment.id, "Could not create environment." - ), - ) - - csp_environment_id = self._id() - - self._delay(1, 5) - self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) - self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) - self._maybe_raise( - self.ATAT_ADMIN_CREATE_FAILURE_PCT, - BaselineProvisionException( - csp_environment_id, "Could not create environment baseline." - ), - ) - self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) - - return csp_environment_id - - def create_atat_admin_user(self, auth_credentials, csp_environment_id): - self._authorize(auth_credentials) - - self._delay(1, 5) - self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) - self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) - self._maybe_raise( - self.ATAT_ADMIN_CREATE_FAILURE_PCT, - UserProvisioningException( - csp_environment_id, "atat_admin", "Could not create admin user." - ), - ) - - self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) - - return {"id": self._id(), "credentials": self._auth_credentials} - - def create_tenant(self, payload: TenantCSPPayload): - """ - payload is an instance of TenantCSPPayload data class - """ - - self._authorize(payload.creds) - - self._delay(1, 5) - - self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) - self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) - self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) - - return TenantCSPResult( - **{ - "tenant_id": "", - "user_id": "", - "user_object_id": "", - "tenant_admin_username": "test", - "tenant_admin_password": "test", - } - ).dict() - - def create_billing_profile_creation( - self, payload: BillingProfileCreationCSPPayload - ): - # response will be mostly the same as the body, but we only really care about the id - self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) - self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) - self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) - - return BillingProfileCreationCSPResult( - **dict( - billing_profile_verify_url="https://zombo.com", - billing_profile_retry_after=10, - ) - ).dict() - - def create_billing_profile_verification( - self, payload: BillingProfileVerificationCSPPayload - ): - self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) - self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) - self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) - return BillingProfileVerificationCSPResult( - **{ - "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB", - "name": "KQWI-W2SU-BG7-TGB", - "properties": { - "address": { - "addressLine1": "123 S Broad Street, Suite 2400", - "city": "Philadelphia", - "companyName": "Promptworks", - "country": "US", - "postalCode": "19109", - "region": "PA", - }, - "currency": "USD", - "displayName": "Test Billing Profile", - "enabledAzurePlans": [], - "hasReadAccess": True, - "invoiceDay": 5, - "invoiceEmailOptIn": False, - "invoiceSections": [ - { - "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB/invoiceSections/CHCO-BAAR-PJA-TGB", - "name": "CHCO-BAAR-PJA-TGB", - "properties": {"displayName": "Test Billing Profile"}, - "type": "Microsoft.Billing/billingAccounts/billingProfiles/invoiceSections", - } - ], - }, - "type": "Microsoft.Billing/billingAccounts/billingProfiles", - } - ).dict() - - def create_billing_profile_tenant_access(self, payload): - self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) - self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) - self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) - - return BillingProfileTenantAccessCSPResult( - **{ - "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB/billingRoleAssignments/40000000-aaaa-bbbb-cccc-100000000000_0a5f4926-e3ee-4f47-a6e3-8b0a30a40e3d", - "name": "40000000-aaaa-bbbb-cccc-100000000000_0a5f4926-e3ee-4f47-a6e3-8b0a30a40e3d", - "properties": { - "createdOn": "2020-01-14T14:39:26.3342192+00:00", - "createdByPrincipalId": "82e2b376-3297-4096-8743-ed65b3be0b03", - "principalId": "0a5f4926-e3ee-4f47-a6e3-8b0a30a40e3d", - "principalTenantId": "60ff9d34-82bf-4f21-b565-308ef0533435", - "roleDefinitionId": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB/billingRoleDefinitions/40000000-aaaa-bbbb-cccc-100000000000", - "scope": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB", - }, - "type": "Microsoft.Billing/billingRoleAssignments", - } - ).dict() - - def create_task_order_billing_creation( - self, payload: TaskOrderBillingCreationCSPPayload - ): - self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) - self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) - self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) - - return TaskOrderBillingCreationCSPResult( - **{"Location": "https://somelocation", "Retry-After": "10"} - ).dict() - - def create_task_order_billing_verification( - self, payload: TaskOrderBillingVerificationCSPPayload - ): - self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) - self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) - self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) - - return TaskOrderBillingVerificationCSPResult( - **{ - "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/XC36-GRNZ-BG7-TGB", - "name": "XC36-GRNZ-BG7-TGB", - "properties": { - "address": { - "addressLine1": "123 S Broad Street, Suite 2400", - "city": "Philadelphia", - "companyName": "Promptworks", - "country": "US", - "postalCode": "19109", - "region": "PA", - }, - "currency": "USD", - "displayName": "First Portfolio Billing Profile", - "enabledAzurePlans": [ - { - "productId": "DZH318Z0BPS6", - "skuId": "0001", - "skuDescription": "Microsoft Azure Plan", - } - ], - "hasReadAccess": True, - "invoiceDay": 5, - "invoiceEmailOptIn": False, - }, - "type": "Microsoft.Billing/billingAccounts/billingProfiles", - } - ).dict() - - def create_billing_instruction(self, payload: BillingInstructionCSPPayload): - self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) - self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) - self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) - - return BillingInstructionCSPResult( - **{ - "name": "TO1:CLIN001", - "properties": { - "amount": 1000.0, - "endDate": "2020-03-01T00:00:00+00:00", - "startDate": "2020-01-01T00:00:00+00:00", - }, - "type": "Microsoft.Billing/billingAccounts/billingProfiles/billingInstructions", - } - ).dict() - - def create_or_update_user(self, auth_credentials, user_info, csp_role_id): - self._authorize(auth_credentials) - - self._delay(1, 5) - self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) - self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) - self._maybe_raise( - self.ATAT_ADMIN_CREATE_FAILURE_PCT, - UserProvisioningException( - user_info.environment.id, - user_info.application_role.user_id, - "Could not create user.", - ), - ) - - self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) - return self._id() - - def disable_user(self, auth_credentials, csp_user_id): - self._authorize(auth_credentials) - self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) - self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) - - self._maybe_raise( - self.ATAT_ADMIN_CREATE_FAILURE_PCT, - UserRemovalException(csp_user_id, "Could not disable user."), - ) - - return self._maybe(12) - - def create_subscription(self, environment): - self._maybe_raise(self.UNAUTHORIZED_RATE, GeneralCSPException) - - return True - - def get_calculator_url(self): - return "https://www.rackspace.com/en-us/calculator" - - def get_environment_login_url(self, environment): - """Returns the login url for a given environment - """ - return "https://www.mycloud.com/my-env-login" - - def _id(self): - return uuid4().hex - - def _delay(self, min_secs, max_secs): - if self._with_delay: - duration = self._random.randrange(min_secs, max_secs) - self._sleep(duration) - - def _maybe(self, pct): - return not self._with_failure or self._random.randrange(0, 100) < pct - - def _maybe_raise(self, pct, exc): - if self._with_failure and self._maybe(pct): - raise exc - - @property - def _auth_credentials(self): - return {"username": "mock-cloud", "password": "shh"} # pragma: allowlist secret - - def _authorize(self, credentials): - self._delay(1, 5) - if self._with_authorization and credentials != self._auth_credentials: - raise self.AUTHENTICATION_EXCEPTION - - def create_application(self, payload: ApplicationCSPPayload): - self._maybe_raise(self.UNAUTHORIZED_RATE, GeneralCSPException) - - id_ = f"{AZURE_MGMNT_PATH}{payload.management_group_name}" - return ApplicationCSPResult(id=id_) - - def get_credentials(self, scope="portfolio", tenant_id=None): - return self.root_creds() - - -AZURE_ENVIRONMENT = "AZURE_PUBLIC_CLOUD" # TBD -AZURE_SKU_ID = "?" # probably a static sku specific to ATAT/JEDI -SUBSCRIPTION_ID_REGEX = re.compile( - "subscriptions\/([0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})", - re.I, -) - -# This needs to be a fully pathed role definition identifier, not just a UUID -REMOTE_ROOT_ROLE_DEF_ID = "/providers/Microsoft.Authorization/roleDefinitions/00000000-0000-4000-8000-000000000000" -AZURE_MANAGEMENT_API = "https://management.azure.com" - - -class AzureSDKProvider(object): - def __init__(self): - from azure.mgmt import subscription, authorization, managementgroups - from azure.mgmt.resource import policy - import azure.graphrbac as graphrbac - import azure.common.credentials as credentials - import azure.identity as identity - from azure.keyvault import secrets - from azure.core import exceptions - - from msrestazure.azure_cloud import AZURE_PUBLIC_CLOUD - import adal - import requests - - self.subscription = subscription - self.policy = policy - self.managementgroups = managementgroups - self.authorization = authorization - self.adal = adal - self.graphrbac = graphrbac - self.credentials = credentials - self.identity = identity - self.exceptions = exceptions - self.secrets = secrets - self.requests = requests - # may change to a JEDI cloud - self.cloud = AZURE_PUBLIC_CLOUD - - -class AzureCloudProvider(CloudProviderInterface): - def __init__(self, config, azure_sdk_provider=None): - self.config = config - - self.client_id = config["AZURE_CLIENT_ID"] - self.secret_key = config["AZURE_SECRET_KEY"] - self.tenant_id = config["AZURE_TENANT_ID"] - self.vault_url = config["AZURE_VAULT_URL"] - - if azure_sdk_provider is None: - self.sdk = AzureSDKProvider() - else: - self.sdk = azure_sdk_provider - - self.policy_manager = AzurePolicyManager(config["AZURE_POLICY_LOCATION"]) - - def set_secret(self, secret_key, secret_value): - credential = self._get_client_secret_credential_obj({}) - secret_client = self.secrets.SecretClient( - vault_url=self.vault_url, credential=credential, - ) - try: - return secret_client.set_secret(secret_key, secret_value) - except self.exceptions.HttpResponseError: - app.logger.error( - f"Could not SET secret in Azure keyvault for key {secret_key}.", - exc_info=1, - ) - - def get_secret(self, secret_key): - credential = self._get_client_secret_credential_obj({}) - secret_client = self.secrets.SecretClient( - vault_url=self.vault_url, credential=credential, - ) - try: - return secret_client.get_secret(secret_key).value - except self.exceptions.HttpResponseError: - app.logger.error( - f"Could not GET secret in Azure keyvault for key {secret_key}.", - exc_info=1, - ) - - def create_environment(self, auth_credentials: Dict, user, environment): - # since this operation would only occur within a tenant, should we source the tenant - # via lookup from environment once we've created the portfolio csp data schema - # something like this: - # environment_tenant = environment.application.portfolio.csp_data.get('tenant_id', None) - # though we'd probably source the whole credentials for these calls from the portfolio csp - # data, as it would have to be where we store the creds for the at-at user within the portfolio tenant - # credentials = self._get_credential_obj(environment.application.portfolio.csp_data.get_creds()) - credentials = self._get_credential_obj(self._root_creds) - display_name = f"{environment.application.name}_{environment.name}_{environment.id}" # proposed format - management_group_id = "?" # management group id chained from environment - parent_id = "?" # from environment.application - - management_group = self._create_management_group( - credentials, management_group_id, display_name, parent_id, - ) - - return ManagementGroupCSPResponse(**management_group) - - def create_atat_admin_user( - self, auth_credentials: Dict, csp_environment_id: str - ) -> Dict: - root_creds = self._root_creds - credentials = self._get_credential_obj(root_creds) - - sub_client = self.sdk.subscription.SubscriptionClient(credentials) - subscription = sub_client.subscriptions.get(csp_environment_id) - - managment_principal = self._get_management_service_principal() - - auth_client = self.sdk.authorization.AuthorizationManagementClient( - credentials, - # TODO: Determine which subscription this needs to point at - # Once we're in a multi-sub environment - subscription.id, - ) - - # Create role assignment for - role_assignment_id = str(uuid4()) - role_assignment_create_params = auth_client.role_assignments.models.RoleAssignmentCreateParameters( - role_definition_id=REMOTE_ROOT_ROLE_DEF_ID, - principal_id=managment_principal.id, - ) - - auth_client.role_assignments.create( - scope=f"/subscriptions/{subscription.id}/", - role_assignment_name=role_assignment_id, - parameters=role_assignment_create_params, - ) - - return { - "csp_user_id": managment_principal.object_id, - "credentials": managment_principal.password_credentials, - "role_name": role_assignment_id, - } - - def create_application(self, payload: ApplicationCSPPayload): - creds = payload.creds - credentials = self._get_credential_obj(creds, resource=AZURE_MANAGEMENT_API) - - response = self._create_management_group( - credentials, - payload.management_group_name, - payload.display_name, - payload.parent_id, - ) - - return ApplicationCSPResult(**response) - - def _create_management_group( - self, credentials, management_group_id, display_name, parent_id=None, - ): - mgmgt_group_client = self.sdk.managementgroups.ManagementGroupsAPI(credentials) - create_parent_grp_info = self.sdk.managementgroups.models.CreateParentGroupInfo( - id=parent_id - ) - create_mgmt_grp_details = self.sdk.managementgroups.models.CreateManagementGroupDetails( - parent=create_parent_grp_info - ) - mgmt_grp_create = self.sdk.managementgroups.models.CreateManagementGroupRequest( - name=management_group_id, - display_name=display_name, - details=create_mgmt_grp_details, - ) - create_request = mgmgt_group_client.management_groups.create_or_update( - management_group_id, mgmt_grp_create - ) - - # result is a synchronous wait, might need to do a poll instead to handle first mgmt group create - # since we were told it could take 10+ minutes to complete, unless this handles that polling internally - # TODO: what to do is status is not 'Succeeded' on the - # response object? Will it always raise its own error - # instead? - return create_request.result() - - def _create_subscription( - self, - credentials, - display_name, - billing_profile_id, - sku_id, - management_group_id, - billing_account_name, - invoice_section_name, - ): - sub_client = self.sdk.subscription.SubscriptionClient(credentials) - - billing_profile_id = "?" # where do we source this? - sku_id = AZURE_SKU_ID - # These 2 seem like something that might be worthwhile to allow tiebacks to - # TOs filed for the environment - billing_account_name = "?" # from TO? - invoice_section_name = "?" # from TO? - - body = self.sdk.subscription.models.ModernSubscriptionCreationParameters( - display_name=display_name, - billing_profile_id=billing_profile_id, - sku_id=sku_id, - management_group_id=management_group_id, - ) - - # We may also want to create billing sections in the enrollment account - sub_creation_operation = sub_client.subscription_factory.create_subscription( - billing_account_name, invoice_section_name, body - ) - - # the resulting object from this process is a link to the new subscription - # not a subscription model, so we'll have to unpack the ID - new_sub = sub_creation_operation.result() - - subscription_id = self._extract_subscription_id(new_sub.subscription_link) - if subscription_id: - return subscription_id - else: - # troublesome error, subscription should exist at this point - # but we just don't have a valid ID - pass - - def _create_policy_definition( - self, credentials, subscription_id, management_group_id, properties, - ): - """ - Requires credentials that have AZURE_MANAGEMENT_API - specified as the resource. The Service Principal - specified in the credentials must have the "Resource - Policy Contributor" role assigned with a scope at least - as high as the management group specified by - management_group_id. - - Arguments: - credentials -- ServicePrincipalCredentials - subscription_id -- str, ID of the subscription (just the UUID, not the path) - management_group_id -- str, ID of the management group (just the UUID, not the path) - properties -- dictionary, the "properties" section of a valid Azure policy definition document - - Returns: - azure.mgmt.resource.policy.[api version].models.PolicyDefinition: the PolicyDefinition object provided to Azure - - Raises: - TBD - """ - # TODO: which subscription would this be? - client = self.sdk.policy.PolicyClient(credentials, subscription_id) - - definition = client.policy_definitions.models.PolicyDefinition( - policy_type=properties.get("policyType"), - mode=properties.get("mode"), - display_name=properties.get("displayName"), - description=properties.get("description"), - policy_rule=properties.get("policyRule"), - parameters=properties.get("parameters"), - ) - - name = properties.get("displayName") - - return client.policy_definitions.create_or_update_at_management_group( - policy_definition_name=name, - parameters=definition, - management_group_id=management_group_id, - ) - - def create_tenant(self, payload: TenantCSPPayload): - sp_token = self._get_sp_token(payload.creds) - if sp_token is None: - raise AuthenticationException("Could not resolve token for tenant creation") - - create_tenant_body = payload.dict(by_alias=True) - - create_tenant_headers = { - "Authorization": f"Bearer {sp_token}", - } - - result = self.sdk.requests.post( - "https://management.azure.com/providers/Microsoft.SignUp/createTenant?api-version=2020-01-01-preview", - json=create_tenant_body, - headers=create_tenant_headers, - ) - - if result.status_code == 200: - return self._ok( - TenantCSPResult( - **result.json(), - tenant_admin_password=payload.password, - tenant_admin_username=payload.user_id, - ) - ) - else: - return self._error(result.json()) - - def create_billing_profile_creation( - self, payload: BillingProfileCreationCSPPayload - ): - sp_token = self._get_sp_token(payload.creds) - if sp_token is None: - raise AuthenticationException( - "Could not resolve token for billing profile creation" - ) - - create_billing_account_body = payload.dict(by_alias=True) - - create_billing_account_headers = { - "Authorization": f"Bearer {sp_token}", - } - - billing_account_create_url = f"https://management.azure.com/providers/Microsoft.Billing/billingAccounts/{payload.billing_account_name}/billingProfiles?api-version=2019-10-01-preview" - - result = self.sdk.requests.post( - billing_account_create_url, - json=create_billing_account_body, - headers=create_billing_account_headers, - ) - - if result.status_code == 202: - # 202 has location/retry after headers - return self._ok(BillingProfileCreationCSPResult(**result.headers)) - elif result.status_code == 200: - # NB: Swagger docs imply call can sometimes resolve immediately - return self._ok(BillingProfileVerificationCSPResult(**result.json())) - else: - return self._error(result.json()) - - def create_billing_profile_verification( - self, payload: BillingProfileVerificationCSPPayload - ): - sp_token = self._get_sp_token(payload.creds) - if sp_token is None: - raise AuthenticationException( - "Could not resolve token for billing profile validation" - ) - - auth_header = { - "Authorization": f"Bearer {sp_token}", - } - - result = self.sdk.requests.get( - payload.billing_profile_verify_url, headers=auth_header - ) - - if result.status_code == 202: - # 202 has location/retry after headers - return self._ok(BillingProfileCreationCSPResult(**result.headers)) - elif result.status_code == 200: - return self._ok(BillingProfileVerificationCSPResult(**result.json())) - else: - return self._error(result.json()) - - def create_billing_profile_tenant_access( - self, payload: BillingProfileTenantAccessCSPPayload - ): - sp_token = self._get_sp_token(payload.creds) - request_body = { - "properties": { - "principalTenantId": payload.tenant_id, # from tenant creation - "principalId": payload.user_object_id, # from tenant creationn - "roleDefinitionId": f"/providers/Microsoft.Billing/billingAccounts/{payload.billing_account_name}/billingProfiles/{payload.billing_profile_name}/billingRoleDefinitions/40000000-aaaa-bbbb-cccc-100000000000", - } - } - - headers = { - "Authorization": f"Bearer {sp_token}", - } - - url = f"https://management.azure.com/providers/Microsoft.Billing/billingAccounts/{payload.billing_account_name}/billingProfiles/{payload.billing_profile_name}/createBillingRoleAssignment?api-version=2019-10-01-preview" - - result = self.sdk.requests.post(url, headers=headers, json=request_body) - if result.status_code == 201: - return self._ok(BillingProfileTenantAccessCSPResult(**result.json())) - else: - return self._error(result.json()) - - def create_task_order_billing_creation( - self, payload: TaskOrderBillingCreationCSPPayload - ): - sp_token = self._get_sp_token(payload.creds) - request_body = [ - { - "op": "replace", - "path": "/enabledAzurePlans", - "value": [{"skuId": "0001"}], - } - ] - - request_headers = { - "Authorization": f"Bearer {sp_token}", - } - - url = f"https://management.azure.com/providers/Microsoft.Billing/billingAccounts/{payload.billing_account_name}/billingProfiles/{payload.billing_profile_name}?api-version=2019-10-01-preview" - - result = self.sdk.requests.patch( - url, headers=request_headers, json=request_body - ) - - if result.status_code == 202: - # 202 has location/retry after headers - return self._ok(TaskOrderBillingCreationCSPResult(**result.headers)) - elif result.status_code == 200: - return self._ok(TaskOrderBillingVerificationCSPResult(**result.json())) - else: - return self._error(result.json()) - - def create_task_order_billing_verification( - self, payload: TaskOrderBillingVerificationCSPPayload - ): - sp_token = self._get_sp_token(payload.creds) - if sp_token is None: - raise AuthenticationException( - "Could not resolve token for task order billing validation" - ) - - auth_header = { - "Authorization": f"Bearer {sp_token}", - } - - result = self.sdk.requests.get( - payload.task_order_billing_verify_url, headers=auth_header - ) - - if result.status_code == 202: - # 202 has location/retry after headers - return self._ok(TaskOrderBillingCreationCSPResult(**result.headers)) - elif result.status_code == 200: - return self._ok(TaskOrderBillingVerificationCSPResult(**result.json())) - else: - return self._error(result.json()) - - def create_billing_instruction(self, payload: BillingInstructionCSPPayload): - sp_token = self._get_sp_token(payload.creds) - if sp_token is None: - raise AuthenticationException( - "Could not resolve token for task order billing validation" - ) - - request_body = { - "properties": { - "amount": payload.initial_clin_amount, - "startDate": payload.initial_clin_start_date, - "endDate": payload.initial_clin_end_date, - } - } - - url = f"https://management.azure.com/providers/Microsoft.Billing/billingAccounts/{payload.billing_account_name}/billingProfiles/{payload.billing_profile_name}/instructions/{payload.initial_task_order_id}:CLIN00{payload.initial_clin_type}?api-version=2019-10-01-preview" - - auth_header = { - "Authorization": f"Bearer {sp_token}", - } - - result = self.sdk.requests.put(url, headers=auth_header, json=request_body) - - if result.status_code == 200: - return self._ok(BillingInstructionCSPResult(**result.json())) - else: - return self._error(result.json()) - - def create_remote_admin(self, creds, tenant_details): - # create app/service principal within tenant, with name constructed from tenant details - # assign principal global admin - - # needs to call out to CLI with tenant owner username/password, prototyping for that underway - - # return identifier and creds to consumer for storage - response = {"clientId": "string", "secretKey": "string", "tenantId": "string"} - return self._ok( - { - "client_id": response["clientId"], - "secret_key": response["secret_key"], - "tenant_id": response["tenantId"], - } - ) - - def force_tenant_admin_pw_update(self, creds, tenant_owner_id): - # use creds to update to force password recovery? - # not sure what the endpoint/method for this is, yet - - return self._ok() - - def create_billing_alerts(self, TBD): - # TODO: Add azure-mgmt-consumption for Budget and Notification entities/operations - # TODO: Determine how to auth against that API using the SDK, doesn't seeem possible at the moment - # TODO: billing alerts are registered as Notifications on Budget objects, which have start/end dates - # TODO: determine what the keys in the Notifications dict are supposed to be - # we may need to rotate budget objects when new TOs/CLINs are reported? - - # we likely only want the budget ID, can be updated or replaced? - response = {"id": "id"} - - return self._ok({"budget_id": response["id"]}) - - def _get_management_service_principal(self): - # we really should be using graph.microsoft.com, but i'm getting - # "expired token" errors for that - # graph_resource = "https://graph.microsoft.com" - graph_resource = "https://graph.windows.net" - graph_creds = self._get_credential_obj( - self._root_creds, resource=graph_resource - ) - # I needed to set permissions for the graph.windows.net API before I - # could get this to work. - - # how do we scope the graph client to the new subscription rather than - # the cloud0 subscription? tenant id seems to be separate from subscription id - graph_client = self.sdk.graphrbac.GraphRbacManagementClient( - graph_creds, self._root_creds.get("tenant_id") - ) - - # do we need to create a new application to manage each subscripition - # or should we manage access to each subscription from a single service - # principal with multiple role assignments? - app_display_name = "?" # name should reflect the subscription it exists - app_create_param = self.sdk.graphrbac.models.ApplicationCreateParameters( - display_name=app_display_name - ) - - # we need the appropriate perms here: - # https://docs.microsoft.com/en-us/graph/api/application-post-applications?view=graph-rest-beta&tabs=http - # https://docs.microsoft.com/en-us/graph/permissions-reference#microsoft-graph-permission-names - # set app perms in app registration portal - # https://docs.microsoft.com/en-us/graph/auth-v2-service#2-configure-permissions-for-microsoft-graph - app: self.sdk.graphrbac.models.Application = graph_client.applications.create( - app_create_param - ) - - # create a new service principle for the new application, which should be scoped - # to the new subscription - app_id = app.app_id - sp_create_params = self.sdk.graphrbac.models.ServicePrincipalCreateParameters( - app_id=app_id, account_enabled=True - ) - - service_principal = graph_client.service_principals.create(sp_create_params) - - return service_principal - - def _extract_subscription_id(self, subscription_url): - sub_id_match = SUBSCRIPTION_ID_REGEX.match(subscription_url) - - if sub_id_match: - return sub_id_match.group(1) - - def _get_sp_token(self, creds): - home_tenant_id = creds.get("home_tenant_id") - client_id = creds.get("client_id") - secret_key = creds.get("secret_key") - - # TODO: Make endpoints consts or configs - authentication_endpoint = "https://login.microsoftonline.com/" - resource = "https://management.azure.com/" - - context = self.sdk.adal.AuthenticationContext( - authentication_endpoint + home_tenant_id - ) - - # TODO: handle failure states here - token_response = context.acquire_token_with_client_credentials( - resource, client_id, secret_key - ) - - return token_response.get("accessToken", None) - - def _get_credential_obj(self, creds, resource=None): - return self.sdk.credentials.ServicePrincipalCredentials( - client_id=creds.get("client_id"), - secret=creds.get("secret_key"), - tenant=creds.get("tenant_id"), - resource=resource, - cloud_environment=self.sdk.cloud, - ) - - def _get_client_secret_credential_obj(self, creds): - return self.sdk.identity.ClientSecretCredential( - tenant_id=creds.get("tenant_id"), - client_id=creds.get("client_id"), - client_secret=creds.get("secret_key"), - ) - - def _make_tenant_admin_cred_obj(self, username, password): - return self.sdk.credentials.UserPassCredentials(username, password) - - def _ok(self, body=None): - return self._make_response("ok", body) - - def _error(self, body=None): - return self._make_response("error", body) - - def _make_response(self, status, body=dict()): - """Create body for responses from API - - Arguments: - status {string} -- "ok" or "error" - body {dict} -- dict containing details of response or error, if applicable - - Returns: - dict -- status of call with body containing details - """ - return {"status": status, "body": body} - - @property - def _root_creds(self): - return { - "client_id": self.client_id, - "secret_key": self.secret_key, - "tenant_id": self.tenant_id, - } - - def get_credentials(self, scope="portfolio", tenant_id=None): - """ - This could be implemented to determine, based on type, whether to return creds for: - - scope="atat": the ATAT main app registration in ATAT's home tenant - - scope="tenantadmin": the tenant administrator credentials - - scope="portfolio": the credentials for the ATAT SP in the portfolio tenant - """ - if scope == "atat": - return self._root_creds - elif scope == "tenantadmin": - # magic with key vault happens - return { - "client_id": "some id", - "secret_key": "very secret", - "tenant_id": tenant_id, - } - elif scope == "portfolio": - # magic with key vault happens - return { - "client_id": "some id", - "secret_key": "very secret", - "tenant_id": tenant_id, - } diff --git a/atst/domain/csp/cloud/__init__.py b/atst/domain/csp/cloud/__init__.py new file mode 100644 index 00000000..99128d9c --- /dev/null +++ b/atst/domain/csp/cloud/__init__.py @@ -0,0 +1,3 @@ +from .azure_cloud_provider import AzureCloudProvider +from .cloud_provider_interface import CloudProviderInterface +from .mock_cloud_provider import MockCloudProvider diff --git a/atst/domain/csp/cloud/azure_cloud_provider.py b/atst/domain/csp/cloud/azure_cloud_provider.py new file mode 100644 index 00000000..5bbcdd54 --- /dev/null +++ b/atst/domain/csp/cloud/azure_cloud_provider.py @@ -0,0 +1,657 @@ +import re +from typing import Dict +from uuid import uuid4 + +from .cloud_provider_interface import CloudProviderInterface +from .exceptions import AuthenticationException +from .models import ( + ApplicationCSPPayload, + ApplicationCSPResult, + BillingInstructionCSPPayload, + BillingInstructionCSPResult, + BillingProfileCreationCSPPayload, + BillingProfileCreationCSPResult, + BillingProfileTenantAccessCSPPayload, + BillingProfileTenantAccessCSPResult, + BillingProfileVerificationCSPPayload, + BillingProfileVerificationCSPResult, + ManagementGroupCSPResponse, + TaskOrderBillingCreationCSPPayload, + TaskOrderBillingCreationCSPResult, + TaskOrderBillingVerificationCSPPayload, + TaskOrderBillingVerificationCSPResult, + TenantCSPPayload, + TenantCSPResult, +) +from .policy import AzurePolicyManager + + +AZURE_ENVIRONMENT = "AZURE_PUBLIC_CLOUD" # TBD +AZURE_SKU_ID = "?" # probably a static sku specific to ATAT/JEDI +SUBSCRIPTION_ID_REGEX = re.compile( + "subscriptions\/([0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})", + re.I, +) + +# This needs to be a fully pathed role definition identifier, not just a UUID +REMOTE_ROOT_ROLE_DEF_ID = "/providers/Microsoft.Authorization/roleDefinitions/00000000-0000-4000-8000-000000000000" +AZURE_MANAGEMENT_API = "https://management.azure.com" + + +class AzureSDKProvider(object): + def __init__(self): + from azure.mgmt import subscription, authorization, managementgroups + from azure.mgmt.resource import policy + import azure.graphrbac as graphrbac + import azure.common.credentials as credentials + import azure.identity as identity + from azure.keyvault import secrets + from azure.core import exceptions + + from msrestazure.azure_cloud import AZURE_PUBLIC_CLOUD + import adal + import requests + + self.subscription = subscription + self.policy = policy + self.managementgroups = managementgroups + self.authorization = authorization + self.adal = adal + self.graphrbac = graphrbac + self.credentials = credentials + self.identity = identity + self.exceptions = exceptions + self.secrets = secrets + self.requests = requests + # may change to a JEDI cloud + self.cloud = AZURE_PUBLIC_CLOUD + + +class AzureCloudProvider(CloudProviderInterface): + def __init__(self, config, azure_sdk_provider=None): + self.config = config + + self.client_id = config["AZURE_CLIENT_ID"] + self.secret_key = config["AZURE_SECRET_KEY"] + self.tenant_id = config["AZURE_TENANT_ID"] + self.vault_url = config["AZURE_VAULT_URL"] + + if azure_sdk_provider is None: + self.sdk = AzureSDKProvider() + else: + self.sdk = azure_sdk_provider + + self.policy_manager = AzurePolicyManager(config["AZURE_POLICY_LOCATION"]) + + def set_secret(self, secret_key, secret_value): + credential = self._get_client_secret_credential_obj({}) + secret_client = self.secrets.SecretClient( + vault_url=self.vault_url, credential=credential, + ) + try: + return secret_client.set_secret(secret_key, secret_value) + except self.exceptions.HttpResponseError: + app.logger.error( + f"Could not SET secret in Azure keyvault for key {secret_key}.", + exc_info=1, + ) + + def get_secret(self, secret_key): + credential = self._get_client_secret_credential_obj({}) + secret_client = self.secrets.SecretClient( + vault_url=self.vault_url, credential=credential, + ) + try: + return secret_client.get_secret(secret_key).value + except self.exceptions.HttpResponseError: + app.logger.error( + f"Could not GET secret in Azure keyvault for key {secret_key}.", + exc_info=1, + ) + + def create_environment(self, auth_credentials: Dict, user, environment): + # since this operation would only occur within a tenant, should we source the tenant + # via lookup from environment once we've created the portfolio csp data schema + # something like this: + # environment_tenant = environment.application.portfolio.csp_data.get('tenant_id', None) + # though we'd probably source the whole credentials for these calls from the portfolio csp + # data, as it would have to be where we store the creds for the at-at user within the portfolio tenant + # credentials = self._get_credential_obj(environment.application.portfolio.csp_data.get_creds()) + credentials = self._get_credential_obj(self._root_creds) + display_name = f"{environment.application.name}_{environment.name}_{environment.id}" # proposed format + management_group_id = "?" # management group id chained from environment + parent_id = "?" # from environment.application + + management_group = self._create_management_group( + credentials, management_group_id, display_name, parent_id, + ) + + return ManagementGroupCSPResponse(**management_group) + + def create_atat_admin_user( + self, auth_credentials: Dict, csp_environment_id: str + ) -> Dict: + root_creds = self._root_creds + credentials = self._get_credential_obj(root_creds) + + sub_client = self.sdk.subscription.SubscriptionClient(credentials) + subscription = sub_client.subscriptions.get(csp_environment_id) + + managment_principal = self._get_management_service_principal() + + auth_client = self.sdk.authorization.AuthorizationManagementClient( + credentials, + # TODO: Determine which subscription this needs to point at + # Once we're in a multi-sub environment + subscription.id, + ) + + # Create role assignment for + role_assignment_id = str(uuid4()) + role_assignment_create_params = auth_client.role_assignments.models.RoleAssignmentCreateParameters( + role_definition_id=REMOTE_ROOT_ROLE_DEF_ID, + principal_id=managment_principal.id, + ) + + auth_client.role_assignments.create( + scope=f"/subscriptions/{subscription.id}/", + role_assignment_name=role_assignment_id, + parameters=role_assignment_create_params, + ) + + return { + "csp_user_id": managment_principal.object_id, + "credentials": managment_principal.password_credentials, + "role_name": role_assignment_id, + } + + def create_application(self, payload: ApplicationCSPPayload): + creds = payload.creds + credentials = self._get_credential_obj(creds, resource=AZURE_MANAGEMENT_API) + + response = self._create_management_group( + credentials, + payload.management_group_name, + payload.display_name, + payload.parent_id, + ) + + return ApplicationCSPResult(**response) + + def _create_management_group( + self, credentials, management_group_id, display_name, parent_id=None, + ): + mgmgt_group_client = self.sdk.managementgroups.ManagementGroupsAPI(credentials) + create_parent_grp_info = self.sdk.managementgroups.models.CreateParentGroupInfo( + id=parent_id + ) + create_mgmt_grp_details = self.sdk.managementgroups.models.CreateManagementGroupDetails( + parent=create_parent_grp_info + ) + mgmt_grp_create = self.sdk.managementgroups.models.CreateManagementGroupRequest( + name=management_group_id, + display_name=display_name, + details=create_mgmt_grp_details, + ) + create_request = mgmgt_group_client.management_groups.create_or_update( + management_group_id, mgmt_grp_create + ) + + # result is a synchronous wait, might need to do a poll instead to handle first mgmt group create + # since we were told it could take 10+ minutes to complete, unless this handles that polling internally + # TODO: what to do is status is not 'Succeeded' on the + # response object? Will it always raise its own error + # instead? + return create_request.result() + + def _create_subscription( + self, + credentials, + display_name, + billing_profile_id, + sku_id, + management_group_id, + billing_account_name, + invoice_section_name, + ): + sub_client = self.sdk.subscription.SubscriptionClient(credentials) + + billing_profile_id = "?" # where do we source this? + sku_id = AZURE_SKU_ID + # These 2 seem like something that might be worthwhile to allow tiebacks to + # TOs filed for the environment + billing_account_name = "?" # from TO? + invoice_section_name = "?" # from TO? + + body = self.sdk.subscription.models.ModernSubscriptionCreationParameters( + display_name=display_name, + billing_profile_id=billing_profile_id, + sku_id=sku_id, + management_group_id=management_group_id, + ) + + # We may also want to create billing sections in the enrollment account + sub_creation_operation = sub_client.subscription_factory.create_subscription( + billing_account_name, invoice_section_name, body + ) + + # the resulting object from this process is a link to the new subscription + # not a subscription model, so we'll have to unpack the ID + new_sub = sub_creation_operation.result() + + subscription_id = self._extract_subscription_id(new_sub.subscription_link) + if subscription_id: + return subscription_id + else: + # troublesome error, subscription should exist at this point + # but we just don't have a valid ID + pass + + def _create_policy_definition( + self, credentials, subscription_id, management_group_id, properties, + ): + """ + Requires credentials that have AZURE_MANAGEMENT_API + specified as the resource. The Service Principal + specified in the credentials must have the "Resource + Policy Contributor" role assigned with a scope at least + as high as the management group specified by + management_group_id. + + Arguments: + credentials -- ServicePrincipalCredentials + subscription_id -- str, ID of the subscription (just the UUID, not the path) + management_group_id -- str, ID of the management group (just the UUID, not the path) + properties -- dictionary, the "properties" section of a valid Azure policy definition document + + Returns: + azure.mgmt.resource.policy.[api version].models.PolicyDefinition: the PolicyDefinition object provided to Azure + + Raises: + TBD + """ + # TODO: which subscription would this be? + client = self.sdk.policy.PolicyClient(credentials, subscription_id) + + definition = client.policy_definitions.models.PolicyDefinition( + policy_type=properties.get("policyType"), + mode=properties.get("mode"), + display_name=properties.get("displayName"), + description=properties.get("description"), + policy_rule=properties.get("policyRule"), + parameters=properties.get("parameters"), + ) + + name = properties.get("displayName") + + return client.policy_definitions.create_or_update_at_management_group( + policy_definition_name=name, + parameters=definition, + management_group_id=management_group_id, + ) + + def create_tenant(self, payload: TenantCSPPayload): + sp_token = self._get_sp_token(payload.creds) + if sp_token is None: + raise AuthenticationException("Could not resolve token for tenant creation") + + create_tenant_body = payload.dict(by_alias=True) + + create_tenant_headers = { + "Authorization": f"Bearer {sp_token}", + } + + result = self.sdk.requests.post( + "https://management.azure.com/providers/Microsoft.SignUp/createTenant?api-version=2020-01-01-preview", + json=create_tenant_body, + headers=create_tenant_headers, + ) + + if result.status_code == 200: + return self._ok( + TenantCSPResult( + **result.json(), + tenant_admin_password=payload.password, + tenant_admin_username=payload.user_id, + ) + ) + else: + return self._error(result.json()) + + def create_billing_profile_creation( + self, payload: BillingProfileCreationCSPPayload + ): + sp_token = self._get_sp_token(payload.creds) + if sp_token is None: + raise AuthenticationException( + "Could not resolve token for billing profile creation" + ) + + create_billing_account_body = payload.dict(by_alias=True) + + create_billing_account_headers = { + "Authorization": f"Bearer {sp_token}", + } + + billing_account_create_url = f"https://management.azure.com/providers/Microsoft.Billing/billingAccounts/{payload.billing_account_name}/billingProfiles?api-version=2019-10-01-preview" + + result = self.sdk.requests.post( + billing_account_create_url, + json=create_billing_account_body, + headers=create_billing_account_headers, + ) + + if result.status_code == 202: + # 202 has location/retry after headers + return self._ok(BillingProfileCreationCSPResult(**result.headers)) + elif result.status_code == 200: + # NB: Swagger docs imply call can sometimes resolve immediately + return self._ok(BillingProfileVerificationCSPResult(**result.json())) + else: + return self._error(result.json()) + + def create_billing_profile_verification( + self, payload: BillingProfileVerificationCSPPayload + ): + sp_token = self._get_sp_token(payload.creds) + if sp_token is None: + raise AuthenticationException( + "Could not resolve token for billing profile validation" + ) + + auth_header = { + "Authorization": f"Bearer {sp_token}", + } + + result = self.sdk.requests.get( + payload.billing_profile_verify_url, headers=auth_header + ) + + if result.status_code == 202: + # 202 has location/retry after headers + return self._ok(BillingProfileCreationCSPResult(**result.headers)) + elif result.status_code == 200: + return self._ok(BillingProfileVerificationCSPResult(**result.json())) + else: + return self._error(result.json()) + + def create_billing_profile_tenant_access( + self, payload: BillingProfileTenantAccessCSPPayload + ): + sp_token = self._get_sp_token(payload.creds) + request_body = { + "properties": { + "principalTenantId": payload.tenant_id, # from tenant creation + "principalId": payload.user_object_id, # from tenant creationn + "roleDefinitionId": f"/providers/Microsoft.Billing/billingAccounts/{payload.billing_account_name}/billingProfiles/{payload.billing_profile_name}/billingRoleDefinitions/40000000-aaaa-bbbb-cccc-100000000000", + } + } + + headers = { + "Authorization": f"Bearer {sp_token}", + } + + url = f"https://management.azure.com/providers/Microsoft.Billing/billingAccounts/{payload.billing_account_name}/billingProfiles/{payload.billing_profile_name}/createBillingRoleAssignment?api-version=2019-10-01-preview" + + result = self.sdk.requests.post(url, headers=headers, json=request_body) + if result.status_code == 201: + return self._ok(BillingProfileTenantAccessCSPResult(**result.json())) + else: + return self._error(result.json()) + + def create_task_order_billing_creation( + self, payload: TaskOrderBillingCreationCSPPayload + ): + sp_token = self._get_sp_token(payload.creds) + request_body = [ + { + "op": "replace", + "path": "/enabledAzurePlans", + "value": [{"skuId": "0001"}], + } + ] + + request_headers = { + "Authorization": f"Bearer {sp_token}", + } + + url = f"https://management.azure.com/providers/Microsoft.Billing/billingAccounts/{payload.billing_account_name}/billingProfiles/{payload.billing_profile_name}?api-version=2019-10-01-preview" + + result = self.sdk.requests.patch( + url, headers=request_headers, json=request_body + ) + + if result.status_code == 202: + # 202 has location/retry after headers + return self._ok(TaskOrderBillingCreationCSPResult(**result.headers)) + elif result.status_code == 200: + return self._ok(TaskOrderBillingVerificationCSPResult(**result.json())) + else: + return self._error(result.json()) + + def create_task_order_billing_verification( + self, payload: TaskOrderBillingVerificationCSPPayload + ): + sp_token = self._get_sp_token(payload.creds) + if sp_token is None: + raise AuthenticationException( + "Could not resolve token for task order billing validation" + ) + + auth_header = { + "Authorization": f"Bearer {sp_token}", + } + + result = self.sdk.requests.get( + payload.task_order_billing_verify_url, headers=auth_header + ) + + if result.status_code == 202: + # 202 has location/retry after headers + return self._ok(TaskOrderBillingCreationCSPResult(**result.headers)) + elif result.status_code == 200: + return self._ok(TaskOrderBillingVerificationCSPResult(**result.json())) + else: + return self._error(result.json()) + + def create_billing_instruction(self, payload: BillingInstructionCSPPayload): + sp_token = self._get_sp_token(payload.creds) + if sp_token is None: + raise AuthenticationException( + "Could not resolve token for task order billing validation" + ) + + request_body = { + "properties": { + "amount": payload.initial_clin_amount, + "startDate": payload.initial_clin_start_date, + "endDate": payload.initial_clin_end_date, + } + } + + url = f"https://management.azure.com/providers/Microsoft.Billing/billingAccounts/{payload.billing_account_name}/billingProfiles/{payload.billing_profile_name}/instructions/{payload.initial_task_order_id}:CLIN00{payload.initial_clin_type}?api-version=2019-10-01-preview" + + auth_header = { + "Authorization": f"Bearer {sp_token}", + } + + result = self.sdk.requests.put(url, headers=auth_header, json=request_body) + + if result.status_code == 200: + return self._ok(BillingInstructionCSPResult(**result.json())) + else: + return self._error(result.json()) + + def create_remote_admin(self, creds, tenant_details): + # create app/service principal within tenant, with name constructed from tenant details + # assign principal global admin + + # needs to call out to CLI with tenant owner username/password, prototyping for that underway + + # return identifier and creds to consumer for storage + response = {"clientId": "string", "secretKey": "string", "tenantId": "string"} + return self._ok( + { + "client_id": response["clientId"], + "secret_key": response["secret_key"], + "tenant_id": response["tenantId"], + } + ) + + def force_tenant_admin_pw_update(self, creds, tenant_owner_id): + # use creds to update to force password recovery? + # not sure what the endpoint/method for this is, yet + + return self._ok() + + def create_billing_alerts(self, TBD): + # TODO: Add azure-mgmt-consumption for Budget and Notification entities/operations + # TODO: Determine how to auth against that API using the SDK, doesn't seeem possible at the moment + # TODO: billing alerts are registered as Notifications on Budget objects, which have start/end dates + # TODO: determine what the keys in the Notifications dict are supposed to be + # we may need to rotate budget objects when new TOs/CLINs are reported? + + # we likely only want the budget ID, can be updated or replaced? + response = {"id": "id"} + + return self._ok({"budget_id": response["id"]}) + + def _get_management_service_principal(self): + # we really should be using graph.microsoft.com, but i'm getting + # "expired token" errors for that + # graph_resource = "https://graph.microsoft.com" + graph_resource = "https://graph.windows.net" + graph_creds = self._get_credential_obj( + self._root_creds, resource=graph_resource + ) + # I needed to set permissions for the graph.windows.net API before I + # could get this to work. + + # how do we scope the graph client to the new subscription rather than + # the cloud0 subscription? tenant id seems to be separate from subscription id + graph_client = self.sdk.graphrbac.GraphRbacManagementClient( + graph_creds, self._root_creds.get("tenant_id") + ) + + # do we need to create a new application to manage each subscripition + # or should we manage access to each subscription from a single service + # principal with multiple role assignments? + app_display_name = "?" # name should reflect the subscription it exists + app_create_param = self.sdk.graphrbac.models.ApplicationCreateParameters( + display_name=app_display_name + ) + + # we need the appropriate perms here: + # https://docs.microsoft.com/en-us/graph/api/application-post-applications?view=graph-rest-beta&tabs=http + # https://docs.microsoft.com/en-us/graph/permissions-reference#microsoft-graph-permission-names + # set app perms in app registration portal + # https://docs.microsoft.com/en-us/graph/auth-v2-service#2-configure-permissions-for-microsoft-graph + app: self.sdk.graphrbac.models.Application = graph_client.applications.create( + app_create_param + ) + + # create a new service principle for the new application, which should be scoped + # to the new subscription + app_id = app.app_id + sp_create_params = self.sdk.graphrbac.models.ServicePrincipalCreateParameters( + app_id=app_id, account_enabled=True + ) + + service_principal = graph_client.service_principals.create(sp_create_params) + + return service_principal + + def _extract_subscription_id(self, subscription_url): + sub_id_match = SUBSCRIPTION_ID_REGEX.match(subscription_url) + + if sub_id_match: + return sub_id_match.group(1) + + def _get_sp_token(self, creds): + home_tenant_id = creds.get("home_tenant_id") + client_id = creds.get("client_id") + secret_key = creds.get("secret_key") + + # TODO: Make endpoints consts or configs + authentication_endpoint = "https://login.microsoftonline.com/" + resource = "https://management.azure.com/" + + context = self.sdk.adal.AuthenticationContext( + authentication_endpoint + home_tenant_id + ) + + # TODO: handle failure states here + token_response = context.acquire_token_with_client_credentials( + resource, client_id, secret_key + ) + + return token_response.get("accessToken", None) + + def _get_credential_obj(self, creds, resource=None): + return self.sdk.credentials.ServicePrincipalCredentials( + client_id=creds.get("client_id"), + secret=creds.get("secret_key"), + tenant=creds.get("tenant_id"), + resource=resource, + cloud_environment=self.sdk.cloud, + ) + + def _get_client_secret_credential_obj(self, creds): + return self.sdk.identity.ClientSecretCredential( + tenant_id=creds.get("tenant_id"), + client_id=creds.get("client_id"), + client_secret=creds.get("secret_key"), + ) + + def _make_tenant_admin_cred_obj(self, username, password): + return self.sdk.credentials.UserPassCredentials(username, password) + + def _ok(self, body=None): + return self._make_response("ok", body) + + def _error(self, body=None): + return self._make_response("error", body) + + def _make_response(self, status, body=dict()): + """Create body for responses from API + + Arguments: + status {string} -- "ok" or "error" + body {dict} -- dict containing details of response or error, if applicable + + Returns: + dict -- status of call with body containing details + """ + return {"status": status, "body": body} + + @property + def _root_creds(self): + return { + "client_id": self.client_id, + "secret_key": self.secret_key, + "tenant_id": self.tenant_id, + } + + def get_credentials(self, scope="portfolio", tenant_id=None): + """ + This could be implemented to determine, based on type, whether to return creds for: + - scope="atat": the ATAT main app registration in ATAT's home tenant + - scope="tenantadmin": the tenant administrator credentials + - scope="portfolio": the credentials for the ATAT SP in the portfolio tenant + """ + if scope == "atat": + return self._root_creds + elif scope == "tenantadmin": + # magic with key vault happens + return { + "client_id": "some id", + "secret_key": "very secret", + "tenant_id": tenant_id, + } + elif scope == "portfolio": + # magic with key vault happens + return { + "client_id": "some id", + "secret_key": "very secret", + "tenant_id": tenant_id, + } diff --git a/atst/domain/csp/cloud/cloud_provider_interface.py b/atst/domain/csp/cloud/cloud_provider_interface.py new file mode 100644 index 00000000..5f4b9ab5 --- /dev/null +++ b/atst/domain/csp/cloud/cloud_provider_interface.py @@ -0,0 +1,120 @@ +from typing import Dict + + +class CloudProviderInterface: + def set_secret(self, secret_key: str, secret_value: str): + raise NotImplementedError() + + def get_secret(self, secret_key: str): + raise NotImplementedError() + + def root_creds(self) -> Dict: + raise NotImplementedError() + + def create_environment(self, auth_credentials: Dict, user, environment) -> str: + """Create a new environment in the CSP. + + Arguments: + auth_credentials -- Object containing CSP account credentials + user -- ATAT user authorizing the environment creation + environment -- ATAT Environment model + + Returns: + string: ID of created environment + + Raises: + AuthenticationException: Problem with the credentials + AuthorizationException: Credentials not authorized for current action(s) + ConnectionException: Issue with the CSP API connection + UnknownServerException: Unknown issue on the CSP side + EnvironmentExistsException: Environment already exists and has been created + """ + raise NotImplementedError() + + def create_atat_admin_user( + self, auth_credentials: Dict, csp_environment_id: str + ) -> Dict: + """Creates a new, programmatic user in the CSP. Grants this user full permissions to administer + the CSP. + + Arguments: + auth_credentials -- Object containing CSP account credentials + csp_environment_id -- ID of the CSP Environment the admin user should be created in + + Returns: + object: Object representing new remote admin user, including credentials + Something like: + { + "user_id": string, + "credentials": dict, # structure TBD based on csp + } + + Raises: + AuthenticationException: Problem with the credentials + AuthorizationException: Credentials not authorized for current action(s) + ConnectionException: Issue with the CSP API connection + UnknownServerException: Unknown issue on the CSP side + UserProvisioningException: Problem creating the root user + """ + raise NotImplementedError() + + def create_or_update_user( + self, auth_credentials: Dict, user_info, csp_role_id: str + ) -> str: + """Creates a user or updates an existing user's role. + + Arguments: + auth_credentials -- Object containing CSP account credentials + user_info -- instance of EnvironmentRole containing user data + if it has a csp_user_id it will try to update that user + csp_role_id -- The id of the role the user should be given in the CSP + + Returns: + string: Returns the interal csp_user_id of the created/updated user account + + Raises: + AuthenticationException: Problem with the credentials + AuthorizationException: Credentials not authorized for current action(s) + ConnectionException: Issue with the CSP API connection + UnknownServerException: Unknown issue on the CSP side + UserProvisioningException: User couldn't be created or modified + """ + raise NotImplementedError() + + def disable_user(self, auth_credentials: Dict, csp_user_id: str) -> bool: + """Revoke all privileges for a user. Used to prevent user access while a full + delete is being processed. + + Arguments: + auth_credentials -- Object containing CSP account credentials + csp_user_id -- CSP internal user identifier + + Returns: + bool -- True on success + + Raises: + AuthenticationException: Problem with the credentials + AuthorizationException: Credentials not authorized for current action(s) + ConnectionException: Issue with the CSP API connection + UnknownServerException: Unknown issue on the CSP side + UserRemovalException: User couldn't be suspended + """ + raise NotImplementedError() + + def get_calculator_url(self) -> str: + """Returns the calculator url for the CSP. + This will likely be a static property elsewhere once a CSP is chosen. + """ + raise NotImplementedError() + + def get_environment_login_url(self, environment) -> str: + """Returns the login url for a given environment + This may move to be a computed property on the Environment domain object + """ + raise NotImplementedError() + + def create_subscription(self, environment): + """Returns True if a new subscription has been created or raises an + exception if an error occurs while creating a subscription. + """ + raise NotImplementedError() diff --git a/atst/domain/csp/cloud/exceptions.py b/atst/domain/csp/cloud/exceptions.py new file mode 100644 index 00000000..6ed47dff --- /dev/null +++ b/atst/domain/csp/cloud/exceptions.py @@ -0,0 +1,131 @@ +class GeneralCSPException(Exception): + pass + + +class OperationInProgressException(GeneralCSPException): + """Throw this for instances when the CSP reports that the current entity is already + being operated on/created/deleted/etc + """ + + def __init__(self, operation_desc): + self.operation_desc = operation_desc + + @property + def message(self): + return "An operation for this entity is already in progress: {}".format( + self.operation_desc + ) + + +class AuthenticationException(GeneralCSPException): + """Throw this for instances when there is a problem with the auth credentials: + * Missing credentials + * Incorrect credentials + * Other credential problems + """ + + def __init__(self, auth_error): + self.auth_error = auth_error + + @property + def message(self): + return "An error occurred with authentication: {}".format(self.auth_error) + + +class AuthorizationException(GeneralCSPException): + """Throw this for instances when the current credentials are not authorized + for the current action. + """ + + def __init__(self, auth_error): + self.auth_error = auth_error + + @property + def message(self): + return "An error occurred with authorization: {}".format(self.auth_error) + + +class ConnectionException(GeneralCSPException): + """A general problem with the connection, timeouts or unresolved endpoints + """ + + def __init__(self, connection_error): + self.connection_error = connection_error + + @property + def message(self): + return "Could not connect to cloud provider: {}".format(self.connection_error) + + +class UnknownServerException(GeneralCSPException): + """An error occured on the CSP side (5xx) and we don't know why + """ + + def __init__(self, server_error): + self.server_error = server_error + + @property + def message(self): + return "A server error occured: {}".format(self.server_error) + + +class EnvironmentCreationException(GeneralCSPException): + """If there was an error in creating the environment + """ + + def __init__(self, env_identifier, reason): + self.env_identifier = env_identifier + self.reason = reason + + @property + def message(self): + return "The envionment {} couldn't be created: {}".format( + self.env_identifier, self.reason + ) + + +class UserProvisioningException(GeneralCSPException): + """Failed to provision a user + """ + + def __init__(self, env_identifier, user_identifier, reason): + self.env_identifier = env_identifier + self.user_identifier = user_identifier + self.reason = reason + + @property + def message(self): + return "Failed to create user {} for environment {}: {}".format( + self.user_identifier, self.env_identifier, self.reason + ) + + +class UserRemovalException(GeneralCSPException): + """Failed to remove a user + """ + + def __init__(self, user_csp_id, reason): + self.user_csp_id = user_csp_id + self.reason = reason + + @property + def message(self): + return "Failed to suspend or delete user {}: {}".format( + self.user_csp_id, self.reason + ) + + +class BaselineProvisionException(GeneralCSPException): + """If there's any issues standing up whatever is required + for an environment baseline + """ + + def __init__(self, env_identifier, reason): + self.env_identifier = env_identifier + self.reason = reason + + @property + def message(self): + return "Could not complete baseline provisioning for environment ({}): {}".format( + self.env_identifier, self.reason + ) diff --git a/atst/domain/csp/cloud/mock_cloud_provider.py b/atst/domain/csp/cloud/mock_cloud_provider.py new file mode 100644 index 00000000..39bc6da3 --- /dev/null +++ b/atst/domain/csp/cloud/mock_cloud_provider.py @@ -0,0 +1,354 @@ +from uuid import uuid4 + +from atst.domain.csp.cloud.exceptions import ( + BaselineProvisionException, + EnvironmentCreationException, + GeneralCSPException, + UserProvisioningException, + UserRemovalException, +) +from atst.domain.csp.cloud.models import BillingProfileTenantAccessCSPResult + +from .cloud_provider_interface import CloudProviderInterface +from .exceptions import ( + AuthenticationException, + AuthorizationException, + ConnectionException, + UnknownServerException, +) +from .models import ( + AZURE_MGMNT_PATH, + ApplicationCSPPayload, + ApplicationCSPResult, + BillingInstructionCSPPayload, + BillingInstructionCSPResult, + BillingProfileCreationCSPPayload, + BillingProfileCreationCSPResult, + BillingProfileVerificationCSPPayload, + BillingProfileVerificationCSPResult, + TaskOrderBillingCreationCSPPayload, + TaskOrderBillingCreationCSPResult, + TaskOrderBillingVerificationCSPPayload, + TaskOrderBillingVerificationCSPResult, + TenantCSPPayload, + TenantCSPResult, +) + + +class MockCloudProvider(CloudProviderInterface): + + # TODO: All of these constants + AUTHENTICATION_EXCEPTION = AuthenticationException("Authentication failure.") + AUTHORIZATION_EXCEPTION = AuthorizationException("Not authorized.") + NETWORK_EXCEPTION = ConnectionException("Network failure.") + SERVER_EXCEPTION = UnknownServerException("Not our fault.") + + SERVER_FAILURE_PCT = 1 + NETWORK_FAILURE_PCT = 7 + ENV_CREATE_FAILURE_PCT = 12 + ATAT_ADMIN_CREATE_FAILURE_PCT = 12 + UNAUTHORIZED_RATE = 2 + + def __init__( + self, config, with_delay=True, with_failure=True, with_authorization=True + ): + from time import sleep + import random + + self._with_delay = with_delay + self._with_failure = with_failure + self._with_authorization = with_authorization + self._sleep = sleep + self._random = random + + def root_creds(self): + return self._auth_credentials + + def set_secret(self, secret_key: str, secret_value: str): + pass + + def get_secret(self, secret_key: str): + return {} + + def create_environment(self, auth_credentials, user, environment): + self._authorize(auth_credentials) + + self._delay(1, 5) + self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) + self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) + self._maybe_raise( + self.ENV_CREATE_FAILURE_PCT, + EnvironmentCreationException( + environment.id, "Could not create environment." + ), + ) + + csp_environment_id = self._id() + + self._delay(1, 5) + self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) + self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) + self._maybe_raise( + self.ATAT_ADMIN_CREATE_FAILURE_PCT, + BaselineProvisionException( + csp_environment_id, "Could not create environment baseline." + ), + ) + self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) + + return csp_environment_id + + def create_atat_admin_user(self, auth_credentials, csp_environment_id): + self._authorize(auth_credentials) + + self._delay(1, 5) + self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) + self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) + self._maybe_raise( + self.ATAT_ADMIN_CREATE_FAILURE_PCT, + UserProvisioningException( + csp_environment_id, "atat_admin", "Could not create admin user." + ), + ) + + self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) + + return {"id": self._id(), "credentials": self._auth_credentials} + + def create_tenant(self, payload: TenantCSPPayload): + """ + payload is an instance of TenantCSPPayload data class + """ + + self._authorize(payload.creds) + + self._delay(1, 5) + + self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) + self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) + self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) + + return TenantCSPResult( + **{ + "tenant_id": "", + "user_id": "", + "user_object_id": "", + "tenant_admin_username": "test", + "tenant_admin_password": "test", + } + ).dict() + + def create_billing_profile_creation( + self, payload: BillingProfileCreationCSPPayload + ): + # response will be mostly the same as the body, but we only really care about the id + self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) + self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) + self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) + + return BillingProfileCreationCSPResult( + **dict( + billing_profile_verify_url="https://zombo.com", + billing_profile_retry_after=10, + ) + ).dict() + + def create_billing_profile_verification( + self, payload: BillingProfileVerificationCSPPayload + ): + self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) + self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) + self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) + return BillingProfileVerificationCSPResult( + **{ + "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB", + "name": "KQWI-W2SU-BG7-TGB", + "properties": { + "address": { + "addressLine1": "123 S Broad Street, Suite 2400", + "city": "Philadelphia", + "companyName": "Promptworks", + "country": "US", + "postalCode": "19109", + "region": "PA", + }, + "currency": "USD", + "displayName": "Test Billing Profile", + "enabledAzurePlans": [], + "hasReadAccess": True, + "invoiceDay": 5, + "invoiceEmailOptIn": False, + "invoiceSections": [ + { + "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB/invoiceSections/CHCO-BAAR-PJA-TGB", + "name": "CHCO-BAAR-PJA-TGB", + "properties": {"displayName": "Test Billing Profile"}, + "type": "Microsoft.Billing/billingAccounts/billingProfiles/invoiceSections", + } + ], + }, + "type": "Microsoft.Billing/billingAccounts/billingProfiles", + } + ).dict() + + def create_billing_profile_tenant_access(self, payload): + self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) + self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) + self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) + + return BillingProfileTenantAccessCSPResult( + **{ + "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB/billingRoleAssignments/40000000-aaaa-bbbb-cccc-100000000000_0a5f4926-e3ee-4f47-a6e3-8b0a30a40e3d", + "name": "40000000-aaaa-bbbb-cccc-100000000000_0a5f4926-e3ee-4f47-a6e3-8b0a30a40e3d", + "properties": { + "createdOn": "2020-01-14T14:39:26.3342192+00:00", + "createdByPrincipalId": "82e2b376-3297-4096-8743-ed65b3be0b03", + "principalId": "0a5f4926-e3ee-4f47-a6e3-8b0a30a40e3d", + "principalTenantId": "60ff9d34-82bf-4f21-b565-308ef0533435", + "roleDefinitionId": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB/billingRoleDefinitions/40000000-aaaa-bbbb-cccc-100000000000", + "scope": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB", + }, + "type": "Microsoft.Billing/billingRoleAssignments", + } + ).dict() + + def create_task_order_billing_creation( + self, payload: TaskOrderBillingCreationCSPPayload + ): + self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) + self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) + self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) + + return TaskOrderBillingCreationCSPResult( + **{"Location": "https://somelocation", "Retry-After": "10"} + ).dict() + + def create_task_order_billing_verification( + self, payload: TaskOrderBillingVerificationCSPPayload + ): + self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) + self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) + self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) + + return TaskOrderBillingVerificationCSPResult( + **{ + "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/XC36-GRNZ-BG7-TGB", + "name": "XC36-GRNZ-BG7-TGB", + "properties": { + "address": { + "addressLine1": "123 S Broad Street, Suite 2400", + "city": "Philadelphia", + "companyName": "Promptworks", + "country": "US", + "postalCode": "19109", + "region": "PA", + }, + "currency": "USD", + "displayName": "First Portfolio Billing Profile", + "enabledAzurePlans": [ + { + "productId": "DZH318Z0BPS6", + "skuId": "0001", + "skuDescription": "Microsoft Azure Plan", + } + ], + "hasReadAccess": True, + "invoiceDay": 5, + "invoiceEmailOptIn": False, + }, + "type": "Microsoft.Billing/billingAccounts/billingProfiles", + } + ).dict() + + def create_billing_instruction(self, payload: BillingInstructionCSPPayload): + self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) + self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) + self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) + + return BillingInstructionCSPResult( + **{ + "name": "TO1:CLIN001", + "properties": { + "amount": 1000.0, + "endDate": "2020-03-01T00:00:00+00:00", + "startDate": "2020-01-01T00:00:00+00:00", + }, + "type": "Microsoft.Billing/billingAccounts/billingProfiles/billingInstructions", + } + ).dict() + + def create_or_update_user(self, auth_credentials, user_info, csp_role_id): + self._authorize(auth_credentials) + + self._delay(1, 5) + self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) + self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) + self._maybe_raise( + self.ATAT_ADMIN_CREATE_FAILURE_PCT, + UserProvisioningException( + user_info.environment.id, + user_info.application_role.user_id, + "Could not create user.", + ), + ) + + self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) + return self._id() + + def disable_user(self, auth_credentials, csp_user_id): + self._authorize(auth_credentials) + self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) + self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) + + self._maybe_raise( + self.ATAT_ADMIN_CREATE_FAILURE_PCT, + UserRemovalException(csp_user_id, "Could not disable user."), + ) + + return self._maybe(12) + + def create_subscription(self, environment): + self._maybe_raise(self.UNAUTHORIZED_RATE, GeneralCSPException) + + return True + + def get_calculator_url(self): + return "https://www.rackspace.com/en-us/calculator" + + def get_environment_login_url(self, environment): + """Returns the login url for a given environment + """ + return "https://www.mycloud.com/my-env-login" + + def _id(self): + return uuid4().hex + + def _delay(self, min_secs, max_secs): + if self._with_delay: + duration = self._random.randrange(min_secs, max_secs) + self._sleep(duration) + + def _maybe(self, pct): + return not self._with_failure or self._random.randrange(0, 100) < pct + + def _maybe_raise(self, pct, exc): + if self._with_failure and self._maybe(pct): + raise exc + + @property + def _auth_credentials(self): + return {"username": "mock-cloud", "password": "shh"} # pragma: allowlist secret + + def _authorize(self, credentials): + self._delay(1, 5) + if self._with_authorization and credentials != self._auth_credentials: + raise self.AUTHENTICATION_EXCEPTION + + def create_application(self, payload: ApplicationCSPPayload): + self._maybe_raise(self.UNAUTHORIZED_RATE, GeneralCSPException) + + id_ = f"{AZURE_MGMNT_PATH}{payload.management_group_name}" + return ApplicationCSPResult(id=id_) + + def get_credentials(self, scope="portfolio", tenant_id=None): + return self.root_creds() diff --git a/atst/domain/csp/cloud/models.py b/atst/domain/csp/cloud/models.py new file mode 100644 index 00000000..93ac7d8d --- /dev/null +++ b/atst/domain/csp/cloud/models.py @@ -0,0 +1,290 @@ +from typing import Dict, List, Optional +import re +from uuid import uuid4 + +from pydantic import BaseModel, validator + +from atst.utils import snake_to_camel + + +class AliasModel(BaseModel): + """ + This provides automatic camel <-> snake conversion for serializing to/from json + You can override the alias generation in subclasses by providing a Config that defines + a fields property with a dict mapping variables to their cast names, for cases like: + * some_url:someURL + * user_object_id:objectId + """ + + class Config: + alias_generator = snake_to_camel + allow_population_by_field_name = True + + +class BaseCSPPayload(AliasModel): + # {"username": "mock-cloud", "pass": "shh"} + creds: Dict + + def dict(self, *args, **kwargs): + exclude = {"creds"} + if "exclude" not in kwargs: + kwargs["exclude"] = exclude + else: + kwargs["exclude"].update(exclude) + + return super().dict(*args, **kwargs) + + +class TenantCSPPayload(BaseCSPPayload): + user_id: str + password: str + domain_name: str + first_name: str + last_name: str + country_code: str + password_recovery_email_address: str + + +class TenantCSPResult(AliasModel): + user_id: str + tenant_id: str + user_object_id: str + + tenant_admin_username: Optional[str] + tenant_admin_password: Optional[str] + + class Config: + fields = { + "user_object_id": "objectId", + } + + def dict(self, *args, **kwargs): + exclude = {"tenant_admin_username", "tenant_admin_password"} + if "exclude" not in kwargs: + kwargs["exclude"] = exclude + else: + kwargs["exclude"].update(exclude) + + return super().dict(*args, **kwargs) + + def get_creds(self): + return { + "tenant_admin_username": self.tenant_admin_username, + "tenant_admin_password": self.tenant_admin_password, + "tenant_id": self.tenant_id, + } + + +class BillingProfileAddress(AliasModel): + company_name: str + address_line_1: str + city: str + region: str + country: str + postal_code: str + + +class BillingProfileCLINBudget(AliasModel): + clin_budget: Dict + """ + "clinBudget": { + "amount": 0, + "startDate": "2019-12-18T16:47:40.909Z", + "endDate": "2019-12-18T16:47:40.909Z", + "externalReferenceId": "string" + } + """ + + +class BillingProfileCreationCSPPayload(BaseCSPPayload): + tenant_id: str + billing_profile_display_name: str + billing_account_name: str + enabled_azure_plans: Optional[List[str]] + address: BillingProfileAddress + + @validator("enabled_azure_plans", pre=True, always=True) + def default_enabled_azure_plans(cls, v): + """ + Normally you'd implement this by setting the field with a value of: + dataclasses.field(default_factory=list) + but that prevents the object from being correctly pickled, so instead we need + to rely on a validator to ensure this has an empty value when not specified + """ + return v or [] + + class Config: + fields = {"billing_profile_display_name": "displayName"} + + +class BillingProfileCreationCSPResult(AliasModel): + billing_profile_verify_url: str + billing_profile_retry_after: int + + class Config: + fields = { + "billing_profile_verify_url": "Location", + "billing_profile_retry_after": "Retry-After", + } + + +class BillingProfileVerificationCSPPayload(BaseCSPPayload): + billing_profile_verify_url: str + + +class BillingInvoiceSection(AliasModel): + invoice_section_id: str + invoice_section_name: str + + class Config: + fields = {"invoice_section_id": "id", "invoice_section_name": "name"} + + +class BillingProfileProperties(AliasModel): + address: BillingProfileAddress + billing_profile_display_name: str + invoice_sections: List[BillingInvoiceSection] + + class Config: + fields = {"billing_profile_display_name": "displayName"} + + +class BillingProfileVerificationCSPResult(AliasModel): + billing_profile_id: str + billing_profile_name: str + billing_profile_properties: BillingProfileProperties + + class Config: + fields = { + "billing_profile_id": "id", + "billing_profile_name": "name", + "billing_profile_properties": "properties", + } + + +class BillingProfileTenantAccessCSPPayload(BaseCSPPayload): + tenant_id: str + user_object_id: str + billing_account_name: str + billing_profile_name: str + + +class BillingProfileTenantAccessCSPResult(AliasModel): + billing_role_assignment_id: str + billing_role_assignment_name: str + + class Config: + fields = { + "billing_role_assignment_id": "id", + "billing_role_assignment_name": "name", + } + + +class TaskOrderBillingCreationCSPPayload(BaseCSPPayload): + billing_account_name: str + billing_profile_name: str + + +class TaskOrderBillingCreationCSPResult(AliasModel): + task_order_billing_verify_url: str + task_order_retry_after: int + + class Config: + fields = { + "task_order_billing_verify_url": "Location", + "task_order_retry_after": "Retry-After", + } + + +class TaskOrderBillingVerificationCSPPayload(BaseCSPPayload): + task_order_billing_verify_url: str + + +class BillingProfileEnabledPlanDetails(AliasModel): + enabled_azure_plans: List[Dict] + + +class TaskOrderBillingVerificationCSPResult(AliasModel): + billing_profile_id: str + billing_profile_name: str + billing_profile_enabled_plan_details: BillingProfileEnabledPlanDetails + + class Config: + fields = { + "billing_profile_id": "id", + "billing_profile_name": "name", + "billing_profile_enabled_plan_details": "properties", + } + + +class BillingInstructionCSPPayload(BaseCSPPayload): + initial_clin_amount: float + initial_clin_start_date: str + initial_clin_end_date: str + initial_clin_type: str + initial_task_order_id: str + billing_account_name: str + billing_profile_name: str + + +class BillingInstructionCSPResult(AliasModel): + reported_clin_name: str + + class Config: + fields = { + "reported_clin_name": "name", + } + + +AZURE_MGMNT_PATH = "/providers/Microsoft.Management/managementGroups/" + +MANAGEMENT_GROUP_NAME_REGEX = "^[a-zA-Z0-9\-_\(\)\.]+$" + + +class ManagementGroupCSPPayload(BaseCSPPayload): + """ + :param: management_group_name: Just pass a UUID for this. + :param: display_name: This can contain any character and + spaces, but should be 90 characters or fewer long. + :param: parent_id: This should be the fully qualified Azure ID, + i.e. /providers/Microsoft.Management/managementGroups/[management group ID] + """ + + management_group_name: Optional[str] + display_name: str + parent_id: str + + @validator("management_group_name", pre=True, always=True) + def supply_management_group_name_default(cls, name): + if name: + if re.match(MANAGEMENT_GROUP_NAME_REGEX, name) is None: + raise ValueError( + f"Management group name must match {MANAGEMENT_GROUP_NAME_REGEX}" + ) + + return name[0:90] + else: + return str(uuid4()) + + @validator("display_name", pre=True, always=True) + def enforce_display_name_length(cls, name): + return name[0:90] + + @validator("parent_id", pre=True, always=True) + def enforce_parent_id_pattern(cls, id_): + if AZURE_MGMNT_PATH not in id_: + return f"{AZURE_MGMNT_PATH}{id_}" + else: + return id_ + + +class ManagementGroupCSPResponse(AliasModel): + id: str + + +class ApplicationCSPPayload(ManagementGroupCSPPayload): + pass + + +class ApplicationCSPResult(ManagementGroupCSPResponse): + pass diff --git a/atst/domain/csp/policy.py b/atst/domain/csp/cloud/policy.py similarity index 100% rename from atst/domain/csp/policy.py rename to atst/domain/csp/cloud/policy.py diff --git a/atst/jobs.py b/atst/jobs.py index 37f73450..7a4a3792 100644 --- a/atst/jobs.py +++ b/atst/jobs.py @@ -4,14 +4,15 @@ import pendulum from atst.database import db from atst.queue import celery from atst.models import EnvironmentRole, JobFailure -from atst.domain.csp.cloud import CloudProviderInterface, GeneralCSPException +from atst.domain.csp.cloud.exceptions import GeneralCSPException +from atst.domain.csp.cloud import CloudProviderInterface from atst.domain.applications import Applications from atst.domain.environments import Environments from atst.domain.portfolios import Portfolios from atst.domain.environment_roles import EnvironmentRoles from atst.models.utils import claim_for_update from atst.utils.localization import translate -from atst.domain.csp.cloud import ApplicationCSPPayload +from atst.domain.csp.cloud.models import ApplicationCSPPayload class RecordFailure(celery.Task): diff --git a/atst/models/portfolio_state_machine.py b/atst/models/portfolio_state_machine.py index a0cc77cd..cdd82da9 100644 --- a/atst/models/portfolio_state_machine.py +++ b/atst/models/portfolio_state_machine.py @@ -8,7 +8,7 @@ from transitions.extensions.states import add_state_features, Tags from flask import current_app as app -from atst.domain.csp.cloud import ConnectionException, UnknownServerException +from atst.domain.csp.cloud.exceptions import ConnectionException, UnknownServerException from atst.domain.csp import MockCSP, AzureCSP, get_stage_csp_class from atst.database import db from atst.models.types import Id diff --git a/atst/routes/applications/settings.py b/atst/routes/applications/settings.py index b4e75fc1..443989db 100644 --- a/atst/routes/applications/settings.py +++ b/atst/routes/applications/settings.py @@ -13,7 +13,7 @@ from atst.domain.environments import Environments from atst.domain.applications import Applications from atst.domain.application_roles import ApplicationRoles from atst.domain.audit_log import AuditLog -from atst.domain.csp.cloud import GeneralCSPException +from atst.domain.csp.cloud.exceptions import GeneralCSPException from atst.domain.common import Paginator from atst.domain.environment_roles import EnvironmentRoles from atst.domain.invitations import ApplicationInvitations diff --git a/script/include/test_functions.inc.sh b/script/include/test_functions.inc.sh index d6bf0b4d..bfd4d343 100644 --- a/script/include/test_functions.inc.sh +++ b/script/include/test_functions.inc.sh @@ -8,7 +8,7 @@ run_python_lint() { } run_python_typecheck() { - run_command "mypy --ignore-missing-imports --follow-imports=skip atst/domain/csp/cloud.py" + run_command "mypy --ignore-missing-imports --follow-imports=skip atst/domain/csp/cloud/__init__.py" return $? } diff --git a/tests/domain/cloud/test_azure_csp.py b/tests/domain/cloud/test_azure_csp.py index c98a43ee..228b78e4 100644 --- a/tests/domain/cloud/test_azure_csp.py +++ b/tests/domain/cloud/test_azure_csp.py @@ -1,11 +1,17 @@ +from uuid import uuid4 from unittest.mock import Mock -from uuid import uuid4 +from tests.factories import ApplicationFactory, EnvironmentFactory +from tests.mock_azure import AUTH_CREDENTIALS, mock_azure -from atst.domain.csp.cloud import ( - AzureCloudProvider, - BillingProfileCreationCSPResult, +from atst.domain.csp.cloud import AzureCloudProvider +from atst.domain.csp.cloud.models import ( + ApplicationCSPPayload, + ApplicationCSPResult, + BillingInstructionCSPPayload, + BillingInstructionCSPResult, BillingProfileCreationCSPPayload, + BillingProfileCreationCSPResult, BillingProfileTenantAccessCSPPayload, BillingProfileTenantAccessCSPResult, BillingProfileVerificationCSPPayload, @@ -18,12 +24,8 @@ from atst.domain.csp.cloud import ( TaskOrderBillingVerificationCSPResult, TenantCSPPayload, TenantCSPResult, - ApplicationCSPPayload, ) -from tests.mock_azure import mock_azure, AUTH_CREDENTIALS -from tests.factories import EnvironmentFactory, ApplicationFactory - creds = { "home_tenant_id": "tenant_id", diff --git a/tests/domain/cloud/test_payloads.py b/tests/domain/cloud/test_payloads.py index 08ca147c..d92a4840 100644 --- a/tests/domain/cloud/test_payloads.py +++ b/tests/domain/cloud/test_payloads.py @@ -2,7 +2,7 @@ import pytest from pydantic import ValidationError -from atst.domain.csp.cloud import ( +from atst.domain.csp.cloud.models import ( AZURE_MGMNT_PATH, ManagementGroupCSPPayload, ManagementGroupCSPResponse, diff --git a/tests/domain/cloud/test_policy.py b/tests/domain/cloud/test_policy.py index c0189262..18f0a7ab 100644 --- a/tests/domain/cloud/test_policy.py +++ b/tests/domain/cloud/test_policy.py @@ -1,4 +1,4 @@ -from atst.domain.csp.policy import AzurePolicyManager, AzurePolicy +from atst.domain.csp.cloud.policy import AzurePolicyManager, AzurePolicy def test_portfolio_definitions(): diff --git a/tests/routes/applications/test_settings.py b/tests/routes/applications/test_settings.py index 37c71878..8f2595f2 100644 --- a/tests/routes/applications/test_settings.py +++ b/tests/routes/applications/test_settings.py @@ -12,7 +12,7 @@ from atst.domain.application_roles import ApplicationRoles from atst.domain.environment_roles import EnvironmentRoles from atst.domain.invitations import ApplicationInvitations from atst.domain.common import Paginator -from atst.domain.csp.cloud import GeneralCSPException +from atst.domain.csp.cloud.exceptions import GeneralCSPException from atst.domain.permission_sets import PermissionSets from atst.models.application_role import Status as ApplicationRoleStatus from atst.models.environment_role import CSPRole, EnvironmentRole From a10d733fb7e400c3cede347585a0acdeb8c5c3c0 Mon Sep 17 00:00:00 2001 From: dandds Date: Mon, 27 Jan 2020 06:27:18 -0500 Subject: [PATCH 6/7] Fix test and LGTM warnings. - Fixes LGTM warnings for an unused import and equality comparisons to None in SQLAlchemy filters. - Removes part of a unit test asserting that the claimed_until locking mechanism works correctly. If I recall correctly, this does not work in unit tests because the test takes place inside a transaction, and the database provider does evaluate the current time until the transaction is written. --- .../versions/07e0598199f6_add_applications_claimed_until.py | 1 - atst/domain/applications.py | 6 +++--- tests/domain/test_applications.py | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/alembic/versions/07e0598199f6_add_applications_claimed_until.py b/alembic/versions/07e0598199f6_add_applications_claimed_until.py index ada20eaf..9c5d3abc 100644 --- a/alembic/versions/07e0598199f6_add_applications_claimed_until.py +++ b/alembic/versions/07e0598199f6_add_applications_claimed_until.py @@ -7,7 +7,6 @@ Create Date: 2020-01-25 13:33:17.711548 """ from alembic import op import sqlalchemy as sa -from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = '07e0598199f6' # pragma: allowlist secret diff --git a/atst/domain/applications.py b/atst/domain/applications.py index 4e7f3c8a..b9df260e 100644 --- a/atst/domain/applications.py +++ b/atst/domain/applications.py @@ -134,11 +134,11 @@ class Applications(BaseDomainClass): .join(PortfolioStateMachine) .filter(PortfolioStateMachine.state == FSMStates.COMPLETED) .filter(Application.deleted == False) - .filter(Application.cloud_id == None) + .filter(Application.cloud_id.is_(None)) .filter( or_( - Application.claimed_until == None, - Application.claimed_until >= func.now(), + Application.claimed_until.is_(None), + Application.claimed_until <= func.now(), ) ) ).all() diff --git a/tests/domain/test_applications.py b/tests/domain/test_applications.py index 8ddc0867..02dd3124 100644 --- a/tests/domain/test_applications.py +++ b/tests/domain/test_applications.py @@ -206,7 +206,7 @@ def test_get_applications_pending_creation(): portfolio1 = PortfolioFactory.create(state="COMPLETED") app_ready = ApplicationFactory.create(portfolio=portfolio1) - app_claimed = ApplicationFactory.create(portfolio=portfolio1, claimed_until=later) + app_done = ApplicationFactory.create(portfolio=portfolio1, cloud_id="123456") portfolio2 = PortfolioFactory.create(state="UNSTARTED") app_not_ready = ApplicationFactory.create(portfolio=portfolio2) From abd03be806e8ca2f61373c147eb6019ca6f5c0fd Mon Sep 17 00:00:00 2001 From: dandds Date: Mon, 27 Jan 2020 15:00:20 -0500 Subject: [PATCH 7/7] Store and pull tenant creds from Key Vault. The tenant ID should be hashed and used as the key for the JSON blob of relevant creds for any given tenant. Azure CSP interface methods that need to source creds should call the internal `_source_creds` method, either with a `tenant_id` or no parameters. That method will source the creds. If a tenant ID is provided, it will source them from the Key Vault. If not provided, it will return the default creds for the app registration in the home tenant. --- atst/domain/csp/cloud/azure_cloud_provider.py | 61 +++++++++++-------- atst/domain/csp/cloud/mock_cloud_provider.py | 8 ++- atst/domain/csp/cloud/models.py | 57 ++++++++++++++++- atst/jobs.py | 5 +- atst/models/portfolio_state_machine.py | 2 +- atst/utils/__init__.py | 6 ++ tests/domain/cloud/test_azure_csp.py | 23 ++++++- .../{test_payloads.py => test_models.py} | 51 ++++++++++++---- tests/mock_azure.py | 7 +++ tests/utils/test_hash.py | 16 +++++ 10 files changed, 186 insertions(+), 50 deletions(-) rename tests/domain/cloud/{test_payloads.py => test_models.py} (54%) create mode 100644 tests/utils/test_hash.py diff --git a/atst/domain/csp/cloud/azure_cloud_provider.py b/atst/domain/csp/cloud/azure_cloud_provider.py index 0ed18d9d..84a9238c 100644 --- a/atst/domain/csp/cloud/azure_cloud_provider.py +++ b/atst/domain/csp/cloud/azure_cloud_provider.py @@ -1,3 +1,4 @@ +import json import re from secrets import token_urlsafe from typing import Dict @@ -16,6 +17,7 @@ from .models import ( BillingProfileTenantAccessCSPResult, BillingProfileVerificationCSPPayload, BillingProfileVerificationCSPResult, + KeyVaultCredentials, ManagementGroupCSPResponse, TaskOrderBillingCreationCSPPayload, TaskOrderBillingCreationCSPResult, @@ -25,6 +27,7 @@ from .models import ( TenantCSPResult, ) from .policy import AzurePolicyManager +from atst.utils import sha256_hex AZURE_ENVIRONMENT = "AZURE_PUBLIC_CLOUD" # TBD AZURE_SKU_ID = "?" # probably a static sku specific to ATAT/JEDI @@ -85,7 +88,7 @@ class AzureCloudProvider(CloudProviderInterface): def set_secret(self, secret_key, secret_value): credential = self._get_client_secret_credential_obj({}) - secret_client = self.secrets.SecretClient( + secret_client = self.sdk.secrets.SecretClient( vault_url=self.vault_url, credential=credential, ) try: @@ -98,7 +101,7 @@ class AzureCloudProvider(CloudProviderInterface): def get_secret(self, secret_key): credential = self._get_client_secret_credential_obj({}) - secret_client = self.secrets.SecretClient( + secret_client = self.sdk.secrets.SecretClient( vault_url=self.vault_url, credential=credential, ) try: @@ -166,8 +169,15 @@ class AzureCloudProvider(CloudProviderInterface): } def create_application(self, payload: ApplicationCSPPayload): - creds = payload.creds - credentials = self._get_credential_obj(creds, resource=AZURE_MANAGEMENT_API) + creds = self._source_creds(payload.tenant_id) + credentials = self._get_credential_obj( + { + "client_id": creds.root_sp_client_id, + "secret_key": creds.root_sp_key, + "tenant_id": creds.root_tenant_id, + }, + resource=AZURE_MANAGEMENT_API, + ) response = self._create_management_group( credentials, @@ -632,26 +642,23 @@ class AzureCloudProvider(CloudProviderInterface): "tenant_id": self.tenant_id, } - def get_credentials(self, scope="portfolio", tenant_id=None): - """ - This could be implemented to determine, based on type, whether to return creds for: - - scope="atat": the ATAT main app registration in ATAT's home tenant - - scope="tenantadmin": the tenant administrator credentials - - scope="portfolio": the credentials for the ATAT SP in the portfolio tenant - """ - if scope == "atat": - return self._root_creds - elif scope == "tenantadmin": - # magic with key vault happens - return { - "client_id": "some id", - "secret_key": "very secret", - "tenant_id": tenant_id, - } - elif scope == "portfolio": - # magic with key vault happens - return { - "client_id": "some id", - "secret_key": "very secret", - "tenant_id": tenant_id, - } + def _source_creds(self, tenant_id=None) -> KeyVaultCredentials: + if tenant_id: + return self._source_tenant_creds(tenant_id) + else: + return KeyVaultCredentials( + root_tenant_id=self._root_creds.get("tenant_id"), + root_sp_client_id=self._root_creds.get("client_id"), + root_sp_key=self._root_creds.get("secret_key"), + ) + + def update_tenant_creds(self, tenant_id, secret): + hashed = sha256_hex(tenant_id) + self.set_secret(hashed, json.dumps(secret)) + + return secret + + def _source_tenant_creds(self, tenant_id): + hashed = sha256_hex(tenant_id) + raw_creds = self.get_secret(hashed) + return KeyVaultCredentials(**json.loads(raw_creds)) diff --git a/atst/domain/csp/cloud/mock_cloud_provider.py b/atst/domain/csp/cloud/mock_cloud_provider.py index 6df61003..10d62e15 100644 --- a/atst/domain/csp/cloud/mock_cloud_provider.py +++ b/atst/domain/csp/cloud/mock_cloud_provider.py @@ -347,8 +347,12 @@ class MockCloudProvider(CloudProviderInterface): def create_application(self, payload: ApplicationCSPPayload): self._maybe_raise(self.UNAUTHORIZED_RATE, GeneralCSPException) - id_ = f"{AZURE_MGMNT_PATH}{payload.management_group_name}" - return ApplicationCSPResult(id=id_) + return ApplicationCSPResult( + id=f"{AZURE_MGMNT_PATH}{payload.management_group_name}" + ) def get_credentials(self, scope="portfolio", tenant_id=None): return self.root_creds() + + def update_tenant_creds(self, tenant_id, secret): + return secret diff --git a/atst/domain/csp/cloud/models.py b/atst/domain/csp/cloud/models.py index 369bed31..b4ff9232 100644 --- a/atst/domain/csp/cloud/models.py +++ b/atst/domain/csp/cloud/models.py @@ -2,7 +2,7 @@ from typing import Dict, List, Optional import re from uuid import uuid4 -from pydantic import BaseModel, validator +from pydantic import BaseModel, validator, root_validator from atst.utils import snake_to_camel @@ -241,7 +241,7 @@ AZURE_MGMNT_PATH = "/providers/Microsoft.Management/managementGroups/" MANAGEMENT_GROUP_NAME_REGEX = "^[a-zA-Z0-9\-_\(\)\.]+$" -class ManagementGroupCSPPayload(BaseCSPPayload): +class ManagementGroupCSPPayload(AliasModel): """ :param: management_group_name: Just pass a UUID for this. :param: display_name: This can contain any character and @@ -250,6 +250,7 @@ class ManagementGroupCSPPayload(BaseCSPPayload): i.e. /providers/Microsoft.Management/managementGroups/[management group ID] """ + tenant_id: str management_group_name: Optional[str] display_name: str parent_id: str @@ -288,3 +289,55 @@ class ApplicationCSPPayload(ManagementGroupCSPPayload): class ApplicationCSPResult(ManagementGroupCSPResponse): pass + + +class KeyVaultCredentials(BaseModel): + root_sp_client_id: Optional[str] + root_sp_key: Optional[str] + root_tenant_id: Optional[str] + + tenant_id: Optional[str] + + tenant_admin_username: Optional[str] + tenant_admin_password: Optional[str] + + tenant_sp_client_id: Optional[str] + tenant_sp_key: Optional[str] + + @root_validator(pre=True) + def enforce_admin_creds(cls, values): + tenant_id = values.get("tenant_id") + username = values.get("tenant_admin_username") + password = values.get("tenant_admin_password") + if any([username, password]) and not all([tenant_id, username, password]): + raise ValueError( + "tenant_id, tenant_admin_username, and tenant_admin_password must all be set if any one is" + ) + + return values + + @root_validator(pre=True) + def enforce_sp_creds(cls, values): + tenant_id = values.get("tenant_id") + client_id = values.get("tenant_sp_client_id") + key = values.get("tenant_sp_key") + if any([client_id, key]) and not all([tenant_id, client_id, key]): + raise ValueError( + "tenant_id, tenant_sp_client_id, and tenant_sp_key must all be set if any one is" + ) + + return values + + @root_validator(pre=True) + def enforce_root_creds(cls, values): + sp_creds = [ + values.get("root_tenant_id"), + values.get("root_sp_client_id"), + values.get("root_sp_key"), + ] + if any(sp_creds) and not all(sp_creds): + raise ValueError( + "root_tenant_id, root_sp_client_id, and root_sp_key must all be set if any one is" + ) + + return values diff --git a/atst/jobs.py b/atst/jobs.py index 7a4a3792..14256336 100644 --- a/atst/jobs.py +++ b/atst/jobs.py @@ -59,15 +59,14 @@ def do_create_application(csp: CloudProviderInterface, application_id=None): with claim_for_update(application) as application: - if application.cloud_id is not None: + if application.cloud_id: return csp_details = application.portfolio.csp_data parent_id = csp_details.get("root_management_group_id") tenant_id = csp_details.get("tenant_id") - creds = csp.get_credentials(tenant_id) payload = ApplicationCSPPayload( - creds=creds, display_name=application.name, parent_id=parent_id + tenant_id=tenant_id, display_name=application.name, parent_id=parent_id ) app_result = csp.create_application(payload) diff --git a/atst/models/portfolio_state_machine.py b/atst/models/portfolio_state_machine.py index cf42710b..be9324b1 100644 --- a/atst/models/portfolio_state_machine.py +++ b/atst/models/portfolio_state_machine.py @@ -175,7 +175,7 @@ class PortfolioStateMachine( tenant_id = new_creds.get("tenant_id") secret = self.csp.get_secret(tenant_id, new_creds) secret.update(new_creds) - self.csp.set_secret(tenant_id, secret) + self.csp.update_tenant_creds(tenant_id, secret) except PydanticValidationError as exc: app.logger.error( f"Failed to cast response to valid result class {self.__repr__()}:", diff --git a/atst/utils/__init__.py b/atst/utils/__init__.py index 09c63dea..79d5362a 100644 --- a/atst/utils/__init__.py +++ b/atst/utils/__init__.py @@ -1,3 +1,4 @@ +import hashlib import re from sqlalchemy.exc import IntegrityError @@ -41,3 +42,8 @@ def commit_or_raise_already_exists_error(message): except IntegrityError: db.session.rollback() raise AlreadyExistsError(message) + + +def sha256_hex(string): + hsh = hashlib.sha256(string.encode()) + return hsh.digest().hex() diff --git a/tests/domain/cloud/test_azure_csp.py b/tests/domain/cloud/test_azure_csp.py index 0d23d6c0..39fa2f77 100644 --- a/tests/domain/cloud/test_azure_csp.py +++ b/tests/domain/cloud/test_azure_csp.py @@ -1,5 +1,7 @@ +import pytest +import json from uuid import uuid4 -from unittest.mock import Mock +from unittest.mock import Mock, patch from tests.factories import ApplicationFactory, EnvironmentFactory from tests.mock_azure import AUTH_CREDENTIALS, mock_azure @@ -84,13 +86,28 @@ def test_create_environment_succeeds(mock_azure: AzureCloudProvider): assert result.id == "Test Id" +# mock the get_secret so it returns a JSON string +MOCK_CREDS = { + "tenant_id": str(uuid4()), + "tenant_sp_client_id": str(uuid4()), + "tenant_sp_key": "1234", +} + + +def mock_get_secret(azure, func): + azure.get_secret = func + + return azure + + def test_create_application_succeeds(mock_azure: AzureCloudProvider): application = ApplicationFactory.create() - mock_management_group_create(mock_azure, {"id": "Test Id"}) + mock_azure = mock_get_secret(mock_azure, lambda *a, **k: json.dumps(MOCK_CREDS)) + payload = ApplicationCSPPayload( - creds={}, display_name=application.name, parent_id=str(uuid4()) + tenant_id="1234", display_name=application.name, parent_id=str(uuid4()) ) result = mock_azure.create_application(payload) diff --git a/tests/domain/cloud/test_payloads.py b/tests/domain/cloud/test_models.py similarity index 54% rename from tests/domain/cloud/test_payloads.py rename to tests/domain/cloud/test_models.py index d92a4840..d9fc963d 100644 --- a/tests/domain/cloud/test_payloads.py +++ b/tests/domain/cloud/test_models.py @@ -4,6 +4,7 @@ from pydantic import ValidationError from atst.domain.csp.cloud.models import ( AZURE_MGMNT_PATH, + KeyVaultCredentials, ManagementGroupCSPPayload, ManagementGroupCSPResponse, ) @@ -12,25 +13,25 @@ from atst.domain.csp.cloud.models import ( def test_ManagementGroupCSPPayload_management_group_name(): # supplies management_group_name when absent payload = ManagementGroupCSPPayload( - creds={}, display_name="Council of Naboo", parent_id="Galactic_Senate" + tenant_id="any-old-id", + display_name="Council of Naboo", + parent_id="Galactic_Senate", ) assert payload.management_group_name # validates management_group_name with pytest.raises(ValidationError): payload = ManagementGroupCSPPayload( - creds={}, + tenant_id="any-old-id", management_group_name="council of Naboo 1%^&", display_name="Council of Naboo", parent_id="Galactic_Senate", ) # shortens management_group_name to fit - name = "council_of_naboo" - for _ in range(90): - name = f"{name}1" + name = "council_of_naboo".ljust(95, "1") assert len(name) > 90 payload = ManagementGroupCSPPayload( - creds={}, + tenant_id="any-old-id", management_group_name=name, display_name="Council of Naboo", parent_id="Galactic_Senate", @@ -40,12 +41,10 @@ def test_ManagementGroupCSPPayload_management_group_name(): def test_ManagementGroupCSPPayload_display_name(): # shortens display_name to fit - name = "Council of Naboo" - for _ in range(90): - name = f"{name}1" + name = "Council of Naboo".ljust(95, "1") assert len(name) > 90 payload = ManagementGroupCSPPayload( - creds={}, display_name=name, parent_id="Galactic_Senate" + tenant_id="any-old-id", display_name=name, parent_id="Galactic_Senate" ) assert len(payload.display_name) == 90 @@ -54,12 +53,14 @@ def test_ManagementGroupCSPPayload_parent_id(): full_path = f"{AZURE_MGMNT_PATH}Galactic_Senate" # adds full path payload = ManagementGroupCSPPayload( - creds={}, display_name="Council of Naboo", parent_id="Galactic_Senate" + tenant_id="any-old-id", + display_name="Council of Naboo", + parent_id="Galactic_Senate", ) assert payload.parent_id == full_path # keeps full path payload = ManagementGroupCSPPayload( - creds={}, display_name="Council of Naboo", parent_id=full_path + tenant_id="any-old-id", display_name="Council of Naboo", parent_id=full_path ) assert payload.parent_id == full_path @@ -70,3 +71,29 @@ def test_ManagementGroupCSPResponse_id(): **{"id": "/path/to/naboo-123", "other": "stuff"} ) assert response.id == full_id + + +def test_KeyVaultCredentials_enforce_admin_creds(): + with pytest.raises(ValidationError): + KeyVaultCredentials(tenant_id="an id", tenant_admin_username="C3PO") + assert KeyVaultCredentials( + tenant_id="an id", + tenant_admin_username="C3PO", + tenant_admin_password="beep boop", + ) + + +def test_KeyVaultCredentials_enforce_sp_creds(): + with pytest.raises(ValidationError): + KeyVaultCredentials(tenant_id="an id", tenant_sp_client_id="C3PO") + assert KeyVaultCredentials( + tenant_id="an id", tenant_sp_client_id="C3PO", tenant_sp_key="beep boop" + ) + + +def test_KeyVaultCredentials_enforce_root_creds(): + with pytest.raises(ValidationError): + KeyVaultCredentials(root_tenant_id="an id", root_sp_client_id="C3PO") + assert KeyVaultCredentials( + root_tenant_id="an id", root_sp_client_id="C3PO", root_sp_key="beep boop" + ) diff --git a/tests/mock_azure.py b/tests/mock_azure.py index 7fa67667..4f37848e 100644 --- a/tests/mock_azure.py +++ b/tests/mock_azure.py @@ -72,6 +72,12 @@ def mock_secrets(): return Mock(spec=secrets) +def mock_identity(): + import azure.identity as identity + + return Mock(spec=identity) + + class MockAzureSDK(object): def __init__(self): from msrestazure.azure_cloud import AZURE_PUBLIC_CLOUD @@ -88,6 +94,7 @@ class MockAzureSDK(object): self.requests = mock_requests() # may change to a JEDI cloud self.cloud = AZURE_PUBLIC_CLOUD + self.identity = mock_identity() @pytest.fixture(scope="function") diff --git a/tests/utils/test_hash.py b/tests/utils/test_hash.py new file mode 100644 index 00000000..5cfb8489 --- /dev/null +++ b/tests/utils/test_hash.py @@ -0,0 +1,16 @@ +import random +import re +import string + +from atst.utils import sha256_hex + + +def test_sha256_hex(): + sample = "".join( + random.choices(string.ascii_uppercase + string.digits, k=random.randrange(200)) + ) + hashed = sha256_hex(sample) + assert re.match("^[a-zA-Z0-9]+$", hashed) + assert len(hashed) == 64 + hashed_again = sha256_hex(sample) + assert hashed == hashed_again