From ba47053a1c98f809967e2052e310a1adbd2338ce Mon Sep 17 00:00:00 2001 From: Philip Kalinsky Date: Wed, 8 Jan 2020 11:01:55 -0500 Subject: [PATCH 01/46] provision portfolio state machine --- atst/jobs.py | 1 - 1 file changed, 1 deletion(-) diff --git a/atst/jobs.py b/atst/jobs.py index f4611a9a..ab52cf17 100644 --- a/atst/jobs.py +++ b/atst/jobs.py @@ -12,7 +12,6 @@ from atst.models import ( from atst.domain.csp.cloud import CloudProviderInterface, GeneralCSPException from atst.domain.environments import Environments from atst.domain.portfolios import Portfolios - from atst.domain.environment_roles import EnvironmentRoles from atst.models.utils import claim_for_update from atst.utils.localization import translate From 2ac333e0b738d5fa5ebc94e988a53ab40327cfdb Mon Sep 17 00:00:00 2001 From: tomdds Date: Thu, 9 Jan 2020 17:29:34 -0500 Subject: [PATCH 02/46] Sample create tenant itegration This integration works with the happy path, we'll need to expand some fields and handle error states more coherently. --- Pipfile | 2 + Pipfile.lock | 11 +++- atst/domain/csp/cloud.py | 76 ++++++++++++++++++++++++---- tests/domain/cloud/test_azure_csp.py | 21 +++++++- tests/mock_azure.py | 14 +++++ 5 files changed, 111 insertions(+), 13 deletions(-) diff --git a/Pipfile b/Pipfile index 0de14fa2..f1d852b9 100644 --- a/Pipfile +++ b/Pipfile @@ -33,6 +33,8 @@ azure-mgmt-authorization = "*" azure-mgmt-managementgroups = "*" azure-mgmt-resource = "*" transitions = "*" +azure-mgmt-consumption = "*" +adal = "*" [dev-packages] bandit = "*" diff --git a/Pipfile.lock b/Pipfile.lock index ef6d0203..681fddb1 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "63b8f9d203f306a6f0ff20514b024909aa7e64917e1befcc9ea79931b5b4bd34" + "sha256": "a127b88e6c64842786f1868cb93bb1cdc828aa78040ea8ba4079bb3de0316dab" }, "pipfile-spec": 6, "requires": { @@ -21,6 +21,7 @@ "sha256:5a7f1e037c6290c6d7609cab33a9e5e988c2fbec5c51d1c4c649ee3faff37eaf", "sha256:fd17e5661f60634ddf96a569b95d34ccb8a98de60593d729c28bdcfe360eaad1" ], + "index": "pypi", "version": "==1.2.2" }, "alembic": { @@ -60,6 +61,14 @@ "index": "pypi", "version": "==0.60.0" }, + "azure-mgmt-consumption": { + "hashes": [ + "sha256:035d4b74ca7c47e2683bea17105fd9014c27060336fb6255324ac86b27f70f5b", + "sha256:af319ad6e3ec162a7578563f149e3cdd7d833a62ec80761cfd93caf79467610b" + ], + "index": "pypi", + "version": "==3.0.0" + }, "azure-mgmt-managementgroups": { "hashes": [ "sha256:3d5237947458dc94b4a392141174b1c1258d26611241ee104e9006d1d798f682", diff --git a/atst/domain/csp/cloud.py b/atst/domain/csp/cloud.py index 9f764da0..b9323585 100644 --- a/atst/domain/csp/cloud.py +++ b/atst/domain/csp/cloud.py @@ -156,12 +156,31 @@ class TenantCSPPayload(BaseCSPPayload): country_code: str password_recovery_email_address: str + class Config: + fields = { + "user_id": "userId", + "domain_name": "domainName", + "first_name": "firstName", + "last_name": "lastName", + "country_code": "countryCode", + "password_recovery_email_address": "passwordRecoveryEmailAddress", + } + allow_population_by_field_name = True + class TenantCSPResult(BaseModel): user_id: str tenant_id: str user_object_id: str + class Config: + allow_population_by_field_name = True + fields = { + "user_id": "userId", + "tenant_id": "tenantId", + "user_object_id": "objectId", + } + class BillingProfileAddress(BaseModel): address: Dict @@ -558,11 +577,15 @@ class AzureSDKProvider(object): import azure.graphrbac as graphrbac import azure.common.credentials as credentials from msrestazure.azure_cloud import AZURE_PUBLIC_CLOUD + import adal + import requests self.subscription = subscription self.authorization = authorization + self.adal = adal self.graphrbac = graphrbac self.credentials = credentials + self.requests = requests # may change to a JEDI cloud self.cloud = AZURE_PUBLIC_CLOUD @@ -657,20 +680,31 @@ class AzureCloudProvider(CloudProviderInterface): "role_name": role_assignment_id, } - def create_tenant(self, payload): - # auth as SP that is allowed to create tenant? (tenant creation sp creds) - # create tenant with owner details (populated from portfolio point of contact, pw is generated) + def create_tenant(self, payload: TenantCSPPayload): + sp_token = self._get_sp_token(payload.creds) + if sp_token is None: + raise AuthenticationException("Could not resolve token for tenant creation") - # return tenant id, tenant owner id and tenant owner object id from: - response = {"tenantId": "string", "userId": "string", "objectId": "string"} - return self._ok( - { - "tenant_id": response["tenantId"], - "user_id": response["userId"], - "user_object_id": response["objectId"], - } + create_tenant_body = payload.dict(by_alias=True) + + print(create_tenant_body) + + create_tenant_headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {sp_token}", + } + + result = self.sdk.requests.post( + "https://management.azure.com/providers/Microsoft.SignUp/createTenant?api-version=2020-01-01-preview", + json=create_tenant_body, + headers=create_tenant_headers, ) + if result.status_code == 200: + return self._ok(TenantCSPResult(**result.json())) + else: + return self._error(result.json()) + def create_billing_owner(self, creds, tenant_admin_details): # authenticate as tenant_admin # create billing owner identity @@ -838,6 +872,26 @@ class AzureCloudProvider(CloudProviderInterface): if sub_id_match: return sub_id_match.group(1) + def _get_sp_token(self, creds): + home_tenant_id = creds.get("home_tenant_id") + client_id = creds.get("client_id") + secret_key = creds.get("secret_key") + + # TODO: Make endpoints consts or configs + authentication_endpoint = "https://login.microsoftonline.com/" + resource = "https://management.azure.com/" + + context = self.sdk.adal.AuthenticationContext( + authentication_endpoint + home_tenant_id + ) + + # TODO: handle failure states here + token_response = context.acquire_token_with_client_credentials( + resource, client_id, secret_key + ) + + return token_response.get("accessToken", None) + def _get_credential_obj(self, creds, resource=None): return self.sdk.credentials.ServicePrincipalCredentials( diff --git a/tests/domain/cloud/test_azure_csp.py b/tests/domain/cloud/test_azure_csp.py index cc6beb5b..b6e0d380 100644 --- a/tests/domain/cloud/test_azure_csp.py +++ b/tests/domain/cloud/test_azure_csp.py @@ -3,7 +3,7 @@ from unittest.mock import Mock from uuid import uuid4 -from atst.domain.csp.cloud import AzureCloudProvider +from atst.domain.csp.cloud import AzureCloudProvider, TenantCSPResult from tests.mock_azure import mock_azure, AUTH_CREDENTIALS from tests.factories import EnvironmentFactory, ApplicationFactory @@ -121,3 +121,22 @@ def test_create_policy_definition_succeeds(mock_azure: AzureCloudProvider): policy_definition_name=properties.get("displayName"), parameters=mock_policy_definition, ) + + +def test_create_tenant(mock_azure: AzureCloudProvider): + mock_azure.sdk.adal.AuthenticationContext.return_value.context.acquire_token_with_client_credentials.return_value = { + "accessToken": "TOKEN" + } + + mock_result = Mock() + mock_result.json.return_value = { + "objectId": "0a5f4926-e3ee-4f47-a6e3-8b0a30a40e3d", + "tenantId": "60ff9d34-82bf-4f21-b565-308ef0533435", + "userId": "1153801116406515559", + } + mock_result.status_code = 200 + mock_azure.sdk.requests.post.return_value = mock_result + result = mock_azure.create_tenant(None, suffix=2) + print(result) + body: TenantCSPResult = result.get("body") + assert body.tenant_id == "60ff9d34-82bf-4f21-b565-308ef0533435" diff --git a/tests/mock_azure.py b/tests/mock_azure.py index 417e69fb..ecfafaac 100644 --- a/tests/mock_azure.py +++ b/tests/mock_azure.py @@ -53,16 +53,30 @@ def mock_policy(): return Mock(spec=policy) +def mock_adal(): + import adal + + return Mock(spec=adal) + + +def mock_requests(): + import requests + + return Mock(spec=requests) + + class MockAzureSDK(object): def __init__(self): from msrestazure.azure_cloud import AZURE_PUBLIC_CLOUD self.subscription = mock_subscription() self.authorization = mock_authorization() + self.adal = mock_adal() self.managementgroups = mock_managementgroups() self.graphrbac = mock_graphrbac() self.credentials = mock_credentials() self.policy = mock_policy() + self.requests = mock_requests() # may change to a JEDI cloud self.cloud = AZURE_PUBLIC_CLOUD From 7c22922d6d7489cde4d7611086b75da0b4ba20ce Mon Sep 17 00:00:00 2001 From: tomdds Date: Mon, 13 Jan 2020 16:40:17 -0500 Subject: [PATCH 03/46] Create new AliasModel for CSP datalcasses, ignore credentials when converting to dictionary.This will allow all of our dataclasses to convert automatically between python style snake_case and the camelCase that the Azure APIs use. This also allows us to default to that behavior while specifying aliases for any fields as necessary.Additionally, any dataclass including the creds schema will have those creds removed from their dict representation. This can help keep creds out of logs as well as making the dataclasses more consumable for API usage. --- atst/domain/csp/cloud.py | 63 +++++++++++++--------------- atst/utils/__init__.py | 5 +++ tests/domain/cloud/test_azure_csp.py | 20 ++++++++- 3 files changed, 52 insertions(+), 36 deletions(-) diff --git a/atst/domain/csp/cloud.py b/atst/domain/csp/cloud.py index b9323585..39f11d36 100644 --- a/atst/domain/csp/cloud.py +++ b/atst/domain/csp/cloud.py @@ -7,6 +7,7 @@ from pydantic import BaseModel from atst.models.user import User from atst.models.environment import Environment from atst.models.environment_role import EnvironmentRole +from atst.utils import snake_to_camel class GeneralCSPException(Exception): @@ -142,10 +143,33 @@ class BaselineProvisionException(GeneralCSPException): ) -class BaseCSPPayload(BaseModel): +class AliasModel(BaseModel): + """ + This provides automatic camel <-> snake conversion for serializing to/from json + You can override the alias generation in subclasses by providing a Config that defines + a fields property with a dict mapping variables to their cast names, for cases like: + * some_url:someURL + * user_object_id:objectId + """ + + class Config: + alias_generator = snake_to_camel + allow_population_by_field_name = True + + +class BaseCSPPayload(AliasModel): # {"username": "mock-cloud", "pass": "shh"} creds: Dict + def dict(self, *args, **kwargs): + exclude = {"creds"} + if "exclude" not in kwargs: + kwargs["exclude"] = exclude + else: + kwargs["exclude"].update(exclude) + + return super().dict(*args, **kwargs) + class TenantCSPPayload(BaseCSPPayload): user_id: str @@ -156,52 +180,23 @@ class TenantCSPPayload(BaseCSPPayload): country_code: str password_recovery_email_address: str - class Config: - fields = { - "user_id": "userId", - "domain_name": "domainName", - "first_name": "firstName", - "last_name": "lastName", - "country_code": "countryCode", - "password_recovery_email_address": "passwordRecoveryEmailAddress", - } - allow_population_by_field_name = True - -class TenantCSPResult(BaseModel): +class TenantCSPResult(AliasModel): user_id: str tenant_id: str user_object_id: str class Config: - allow_population_by_field_name = True fields = { - "user_id": "userId", - "tenant_id": "tenantId", "user_object_id": "objectId", } -class BillingProfileAddress(BaseModel): - address: Dict - """ - "address": { - "firstName": "string", - "lastName": "string", - "companyName": "string", - "addressLine1": "string", - "addressLine2": "string", - "addressLine3": "string", - "city": "string", - "region": "string", - "country": "string", - "postalCode": "string" - }, - """ +class BillingProfileAddress(AliasModel): -class BillingProfileCLINBudget(BaseModel): - clinBudget: Dict +class BillingProfileCLINBudget(AliasModel): + clin_budget: Dict """ "clinBudget": { "amount": 0, diff --git a/atst/utils/__init__.py b/atst/utils/__init__.py index d3f284cc..09c63dea 100644 --- a/atst/utils/__init__.py +++ b/atst/utils/__init__.py @@ -25,6 +25,11 @@ def camel_to_snake(camel_cased): return re.sub("([a-z0-9])([A-Z])", r"\1_\2", s1).lower() +def snake_to_camel(snake_cased): + parts = snake_cased.split("_") + return f"{parts[0]}{''.join([w.capitalize() for w in parts[1:]])}" + + def pick(keys, dct): _keys = set(keys) return {k: v for (k, v) in dct.items() if k in _keys} diff --git a/tests/domain/cloud/test_azure_csp.py b/tests/domain/cloud/test_azure_csp.py index b6e0d380..1a9b5c56 100644 --- a/tests/domain/cloud/test_azure_csp.py +++ b/tests/domain/cloud/test_azure_csp.py @@ -3,7 +3,11 @@ from unittest.mock import Mock from uuid import uuid4 -from atst.domain.csp.cloud import AzureCloudProvider, TenantCSPResult +from atst.domain.csp.cloud import ( + AzureCloudProvider, + TenantCSPResult, + TenantCSPPayload, +) from tests.mock_azure import mock_azure, AUTH_CREDENTIALS from tests.factories import EnvironmentFactory, ApplicationFactory @@ -136,7 +140,19 @@ def test_create_tenant(mock_azure: AzureCloudProvider): } mock_result.status_code = 200 mock_azure.sdk.requests.post.return_value = mock_result - result = mock_azure.create_tenant(None, suffix=2) + payload = TenantCSPPayload( + **dict( + creds={"username": "mock-cloud", "pass": "shh"}, + user_id="123", + password="123", + domain_name="123", + first_name="john", + last_name="doe", + country_code="US", + password_recovery_email_address="password@email.com", + ) + ) + result = mock_azure.create_tenant(payload) print(result) body: TenantCSPResult = result.get("body") assert body.tenant_id == "60ff9d34-82bf-4f21-b565-308ef0533435" From 161462f3cb9cae2479cecfc21648ed5b0e9fc46f Mon Sep 17 00:00:00 2001 From: tomdds Date: Mon, 13 Jan 2020 16:48:05 -0500 Subject: [PATCH 04/46] Sample create and validate billing profile integration Adds 2 methods to the azure csp interface to create and subsequently validate creation of the billing profile. --- atst/domain/csp/cloud.py | 186 +++++++++++++++------------ tests/domain/cloud/test_azure_csp.py | 97 ++++++++++++++ 2 files changed, 204 insertions(+), 79 deletions(-) diff --git a/atst/domain/csp/cloud.py b/atst/domain/csp/cloud.py index 39f11d36..86c436e5 100644 --- a/atst/domain/csp/cloud.py +++ b/atst/domain/csp/cloud.py @@ -1,8 +1,8 @@ import re -from typing import Dict +from typing import Dict, List, Optional from uuid import uuid4 -from pydantic import BaseModel +from pydantic import BaseModel, validator from atst.models.user import User from atst.models.environment import Environment @@ -193,6 +193,12 @@ class TenantCSPResult(AliasModel): class BillingProfileAddress(AliasModel): + company_name: str + address_line_1: str + city: str + region: str + country: str + postal_code: str class BillingProfileCLINBudget(AliasModel): @@ -207,44 +213,60 @@ class BillingProfileCLINBudget(AliasModel): """ -class BillingProfileCSPPayload( - BaseCSPPayload, BillingProfileAddress, BillingProfileCLINBudget -): - displayName: str - poNumber: str - invoiceEmailOptIn: str +class BillingProfileCSPPayload(BaseCSPPayload): + tenant_id: str + display_name: str + enabled_azure_plans: Optional[List[str]] + address: BillingProfileAddress - """ - { - "displayName": "string", - "poNumber": "string", - "address": { - "firstName": "string", - "lastName": "string", - "companyName": "string", - "addressLine1": "string", - "addressLine2": "string", - "addressLine3": "string", - "city": "string", - "region": "string", - "country": "string", - "postalCode": "string" - }, - "invoiceEmailOptIn": true, - Note: These last 2 are also the body for adding/updating new TOs/clins - "enabledAzurePlans": [ - { - "skuId": "string" - } - ], - "clinBudget": { - "amount": 0, - "startDate": "2019-12-18T16:47:40.909Z", - "endDate": "2019-12-18T16:47:40.909Z", - "externalReferenceId": "string" + @validator("enabled_azure_plans", pre=True, always=True) + def default_enabled_azure_plans(cls, v): + """ + Normally you'd implement this by setting the field with a value of: + dataclasses.field(default_factory=list) + but that prevents the object from being correctly pickled, so instead we need + to rely on a validator to ensure this has an empty value when not specified + """ + return v or [] + + +class BillingProfileCreateCSPResult(AliasModel): + location: str + retry_after: int + + class Config: + fields = {"location": "Location", "retry_after": "Retry-After"} + + +class BillingProfileVerifyCSPPayload(BaseCSPPayload): + location: str + + +class BillingInvoiceSection(AliasModel): + invoice_section_id: str + invoice_section_name: str + + class Config: + fields = {"invoice_section_id": "id", "invoice_section_name": "name"} + + +class BillingProfileProperties(AliasModel): + address: BillingProfileAddress + display_name: str + invoice_sections: List[BillingInvoiceSection] + + +class BillingProfileCSPResult(AliasModel): + billing_profile_id: str + billing_profile_name: str + billing_profile_properties: BillingProfileProperties + + class Config: + fields = { + "billing_profile_id": "id", + "billing_profile_name": "name", + "billing_profile_properties": "properties", } - } - """ class CloudProviderInterface: @@ -682,8 +704,6 @@ class AzureCloudProvider(CloudProviderInterface): create_tenant_body = payload.dict(by_alias=True) - print(create_tenant_body) - create_tenant_headers = { "Content-Type": "application/json", "Authorization": f"Bearer {sp_token}", @@ -700,6 +720,53 @@ class AzureCloudProvider(CloudProviderInterface): else: return self._error(result.json()) + def create_billing_profile(self, payload: BillingProfileCSPPayload): + sp_token = self._get_sp_token(payload.creds) + if sp_token is None: + raise AuthenticationException( + "Could not resolve token for billing profile creation" + ) + + create_billing_account_body = payload.dict(by_alias=True) + + create_billing_account_headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {sp_token}", + } + + # TODO: unsure if this is a static value or needs to be constructed/configurable + BILLING_ACCOUT_NAME = "7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31" + billing_account_create_url = f"https://management.azure.com/providers/Microsoft.Billing/billingAccounts/{BILLING_ACCOUT_NAME}/billingProfiles?api-version=2019-10-01-preview" + + result = self.sdk.requests.post( + billing_account_create_url, + json=create_billing_account_body, + headers=create_billing_account_headers, + ) + + if result.status_code == 202: + return self._ok(BillingProfileCreateCSPResult(**result.headers)) + else: + return self._error(result.json()) + + def validate_billing_profile_created(self, payload: BillingProfileVerifyCSPPayload): + sp_token = self._get_sp_token(payload.creds) + if sp_token is None: + raise AuthenticationException( + "Could not resolve token for billing profile validation" + ) + + auth_header = { + "Authorization": f"Bearer {sp_token}", + } + + result = self.sdk.requests.get(payload.location, headers=auth_header) + + if result.status_code == 200: + return self._ok(BillingProfileCSPResult(**result.json())) + else: + return self._error(result.json()) + def create_billing_owner(self, creds, tenant_admin_details): # authenticate as tenant_admin # create billing owner identity @@ -722,45 +789,6 @@ class AzureCloudProvider(CloudProviderInterface): return self.ok() - def create_billing_profile(self, creds, tenant_admin_details, billing_owner_id): - # call billing profile creation endpoint, specifying owner - # Payload: - """ - { - "displayName": "string", - "poNumber": "string", - "address": { - "firstName": "string", - "lastName": "string", - "companyName": "string", - "addressLine1": "string", - "addressLine2": "string", - "addressLine3": "string", - "city": "string", - "region": "string", - "country": "string", - "postalCode": "string" - }, - "invoiceEmailOptIn": true, - Note: These last 2 are also the body for adding/updating new TOs/clins - "enabledAzurePlans": [ - { - "skuId": "string" - } - ], - "clinBudget": { - "amount": 0, - "startDate": "2019-12-18T16:47:40.909Z", - "endDate": "2019-12-18T16:47:40.909Z", - "externalReferenceId": "string" - } - } - """ - - # response will be mostly the same as the body, but we only really care about the id - response = {"id": "string"} - return self._ok({"billing_profile_id": response["id"]}) - def report_clin(self, creds, clin_id, clin_amount, clin_start, clin_end, clin_to): # should consumer be responsible for reporting each clin or # should this take a list and manage the sequential reporting? diff --git a/tests/domain/cloud/test_azure_csp.py b/tests/domain/cloud/test_azure_csp.py index 1a9b5c56..6b9a8b12 100644 --- a/tests/domain/cloud/test_azure_csp.py +++ b/tests/domain/cloud/test_azure_csp.py @@ -7,6 +7,11 @@ from atst.domain.csp.cloud import ( AzureCloudProvider, TenantCSPResult, TenantCSPPayload, + BillingProfileCSPPayload, + BillingProfileAddress, + BillingProfileCreateCSPResult, + BillingProfileVerifyCSPPayload, + BillingProfileCSPResult, ) from tests.mock_azure import mock_azure, AUTH_CREDENTIALS @@ -156,3 +161,95 @@ def test_create_tenant(mock_azure: AzureCloudProvider): print(result) body: TenantCSPResult = result.get("body") assert body.tenant_id == "60ff9d34-82bf-4f21-b565-308ef0533435" + + +def test_create_billing_profile(mock_azure: AzureCloudProvider): + # mock_azure.sdk.adal.AuthenticationContext.return_value.context.acquire_token_with_client_credentials.return_value = { + # "accessToken": "TOKEN" + # } + + # mock_result = Mock() + # mock_result.headers = { + # "Location": "http://retry-url", + # "Retry-After": "10", + # } + # mock_result.status_code = 202 + # mock_azure.sdk.requests.post.return_value = mock_result + payload = BillingProfileCSPPayload( + **dict( + address=dict( + address_line_1="123 S Broad Street, Suite 2400", + company_name="Promptworks", + city="Philadelphia", + region="PA", + country="US", + postal_code="19109", + ), + creds={"username": "mock-cloud", "pass": "shh"}, + tenant_id="60ff9d34-82bf-4f21-b565-308ef0533435", + display_name="Test Billing Profile", + ) + ) + result = mock_azure.create_billing_profile(payload) + print(result) + body: BillingProfileCreateCSPResult = result.get("body") + assert body.retry_after == 10 + + +def test_validate_billing_profile_creation(mock_azure: AzureCloudProvider): + mock_azure.sdk.adal.AuthenticationContext.return_value.context.acquire_token_with_client_credentials.return_value = { + "accessToken": "TOKEN" + } + + mock_result = Mock() + mock_result.status_code = 200 + mock_result.json.return_value = { + "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/XC36-GRNZ-BG7-TGB", + "name": "XC36-GRNZ-BG7-TGB", + "properties": { + "address": { + "addressLine1": "123 S Broad Street, Suite 2400", + "city": "Philadelphia", + "companyName": "Promptworks", + "country": "US", + "postalCode": "19109", + "region": "PA", + }, + "currency": "USD", + "displayName": "First Portfolio Billing Profile", + "enabledAzurePlans": [], + "hasReadAccess": True, + "invoiceDay": 5, + "invoiceEmailOptIn": False, + "invoiceSections": [ + { + "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/XC36-GRNZ-BG7-TGB/invoiceSections/6HMZ-2HLO-PJA-TGB", + "name": "6HMZ-2HLO-PJA-TGB", + "properties": {"displayName": "First Portfolio Billing Profile"}, + "type": "Microsoft.Billing/billingAccounts/billingProfiles/invoiceSections", + } + ], + }, + "type": "Microsoft.Billing/billingAccounts/billingProfiles", + } + mock_azure.sdk.requests.get.return_value = mock_result + + payload = BillingProfileVerifyCSPPayload( + **dict( + creds={ + "username": "username", + "password": "password", + "tenant_id": "tenant_id", + }, + location="https://management.azure.com/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/operationResults/createBillingProfile_478d5706-71f9-4a8b-8d4e-2cbaca27a668?api-version=2019-10-01-preview", + ) + ) + + result = mock_azure.validate_billing_profile_created(payload) + body: BillingProfileCreateCSPResult = result.get("body") + assert body.billing_profile_name == "XC36-GRNZ-BG7-TGB" + assert ( + body.billing_profile_properties.display_name + == "First Portfolio Billing Profile" + ) + From 81f23ebc22cc405a70d1bf4c557558822253694a Mon Sep 17 00:00:00 2001 From: tomdds Date: Tue, 14 Jan 2020 15:04:13 -0500 Subject: [PATCH 05/46] Finish first passes at baseline tenant integration Add last of the integrations for setting up billing and reporting a CLIN. --- atst/domain/csp/cloud.py | 220 ++++++++++++++++++------ tests/domain/cloud/test_azure_csp.py | 241 +++++++++++++++++++++++---- 2 files changed, 385 insertions(+), 76 deletions(-) diff --git a/atst/domain/csp/cloud.py b/atst/domain/csp/cloud.py index 86c436e5..703dec53 100644 --- a/atst/domain/csp/cloud.py +++ b/atst/domain/csp/cloud.py @@ -231,15 +231,15 @@ class BillingProfileCSPPayload(BaseCSPPayload): class BillingProfileCreateCSPResult(AliasModel): - location: str + billing_profile_validate_url: str retry_after: int class Config: - fields = {"location": "Location", "retry_after": "Retry-After"} + fields = {"billing_profile_validate_url": "Location", "retry_after": "Retry-After"} class BillingProfileVerifyCSPPayload(BaseCSPPayload): - location: str + billing_profile_validate_url: str class BillingInvoiceSection(AliasModel): @@ -269,6 +269,70 @@ class BillingProfileCSPResult(AliasModel): } +class BillingRoleAssignmentCSPPayload(BaseCSPPayload): + tenant_id: str + user_object_id: str + billing_account_name: str + billing_profile_name: str + + +class BillingRoleAssignmentCSPResult(AliasModel): + billing_role_assignment_id: str + billing_role_assignment_name: str + + class Config: + fields = { + "billing_role_assignment_id": "id", + "billing_role_assignment_name": "name", + } + +class EnableTaskOrderBillingCSPPayload(BaseCSPPayload): + billing_account_name: str + billing_profile_name: str + +class EnableTaskOrderBillingCSPResult(AliasModel): + task_order_billing_validation_url: str + retry_after: int + + class Config: + fields = {"task_order_billing_validation_url": "Location", "retry_after": "Retry-After"} + +class VerifyTaskOrderBillingCSPPayload(BaseCSPPayload): + task_order_billing_validation_url: str + +class BillingProfileEnabledPlanDetails(AliasModel): + enabled_azure_plans: List[Dict] + + +class BillingProfileEnabledCSPResult(AliasModel): + billing_profile_id: str + billing_profile_name: str + billing_profile_enabled_plan_details: BillingProfileEnabledPlanDetails + + class Config: + fields = { + "billing_profile_id": "id", + "billing_profile_name": "name", + "billing_profile_enabled_plan_details": "properties", + } + +class ReportCLINCSPPayload(BaseCSPPayload): + amount: float + start_date: str + end_date: str + clin_type: str + task_order_id: str + billing_account_name: str + billing_profile_name: str + +class ReportCLINCSPResult(AliasModel): + reported_clin_name: str + + class Config: + fields = { + "reported_clin_name": "name", + } + class CloudProviderInterface: def root_creds(self) -> Dict: raise NotImplementedError() @@ -705,7 +769,6 @@ class AzureCloudProvider(CloudProviderInterface): create_tenant_body = payload.dict(by_alias=True) create_tenant_headers = { - "Content-Type": "application/json", "Authorization": f"Bearer {sp_token}", } @@ -730,7 +793,6 @@ class AzureCloudProvider(CloudProviderInterface): create_billing_account_body = payload.dict(by_alias=True) create_billing_account_headers = { - "Content-Type": "application/json", "Authorization": f"Bearer {sp_token}", } @@ -745,7 +807,11 @@ class AzureCloudProvider(CloudProviderInterface): ) if result.status_code == 202: + # 202 has location/retry after headers return self._ok(BillingProfileCreateCSPResult(**result.headers)) + elif result.status_code == 200: + # NB: Swagger docs imply call can sometimes resolve immediately + return self._ok(BillingProfileCSPResult(**result.json())) else: return self._error(result.json()) @@ -760,56 +826,116 @@ class AzureCloudProvider(CloudProviderInterface): "Authorization": f"Bearer {sp_token}", } - result = self.sdk.requests.get(payload.location, headers=auth_header) + result = self.sdk.requests.get(payload.billing_profile_validate_url, headers=auth_header) - if result.status_code == 200: + if result.status_code == 202: + # 202 has location/retry after headers + return self._ok(BillingProfileCreateCSPResult(**result.headers)) + elif result.status_code == 200: return self._ok(BillingProfileCSPResult(**result.json())) else: return self._error(result.json()) - def create_billing_owner(self, creds, tenant_admin_details): - # authenticate as tenant_admin - # create billing owner identity - - # TODO: Lookup response format - # Managed service identity? - response = {"id": "string"} - return self._ok({"billing_owner_id": response["id"]}) - - def assign_billing_owner(self, creds, billing_owner_id, tenant_id): - # TODO: Do we source role definition ID from config, api or self-defined? - # TODO: If from api, - """ - { - "principalId": "string", - "principalTenantId": "string", - "billingRoleDefinitionId": "string" - } - """ - - return self.ok() - - def report_clin(self, creds, clin_id, clin_amount, clin_start, clin_end, clin_to): - # should consumer be responsible for reporting each clin or - # should this take a list and manage the sequential reporting? - """ Payload - { - "enabledAzurePlans": [ - { - "skuId": "string" - } - ], - "clinBudget": { - "amount": 0, - "startDate": "2019-12-18T16:47:40.909Z", - "endDate": "2019-12-18T16:47:40.909Z", - "externalReferenceId": "string" + def grant_billing_profile_tenant_access(self, payload: BillingRoleAssignmentCSPPayload): + sp_token = self._get_sp_token(payload.creds) + request_body = { + "properties": { + "principalTenantId": payload.tenant_id, # from tenant creation + "principalId": payload.user_object_id, # from tenant creationn + "roleDefinitionId": f"/providers/Microsoft.Billing/billingAccounts/{payload.billing_account_name}/billingProfiles/{payload.billing_profile_name}/billingRoleDefinitions/40000000-aaaa-bbbb-cccc-100000000000", } } - """ - # we don't need any of the returned info for this - return self._ok() + headers = { + "Authorization": f"Bearer {sp_token}", + } + + url = f"https://management.azure.com/providers/Microsoft.Billing/billingAccounts/{payload.billing_account_name}/billingProfiles/{payload.billing_profile_name}/createBillingRoleAssignment?api-version=2019-10-01-preview" + + result = self.sdk.requests.post(url, headers=headers, json=request_body) + if result.status_code == 201: + return self._ok(BillingRoleAssignmentCSPResult(**result.json())) + else: + return self._error(result.json()) + + def enable_task_order_billing(self, payload: EnableTaskOrderBillingCSPPayload): + sp_token = self._get_sp_token(payload.creds) + request_body = [ + { + "op": "replace", + "path": "/enabledAzurePlans", + "value": [ + { + "skuId": "0001" + } + ] + } + ] + + request_headers = { + "Authorization": f"Bearer {sp_token}", + } + + url = f"https://management.azure.com/providers/Microsoft.Billing/billingAccounts/{payload.billing_account_name}/billingProfiles/{payload.billing_profile_name}?api-version=2019-10-01-preview" + + result = self.sdk.requests.patch(url, headers=request_headers, json=request_body) + + if result.status_code == 202: + # 202 has location/retry after headers + return self._ok(BillingProfileCreateCSPResult(**result.headers)) + elif result.status_code == 200: + return self._ok(BillingProfileEnabledCSPResult(**result.json())) + else: + return self._error(result.json()) + + def validate_task_order_billing_enabled(self, payload: VerifyTaskOrderBillingCSPPayload): + sp_token = self._get_sp_token(payload.creds) + if sp_token is None: + raise AuthenticationException( + "Could not resolve token for task order billing validation" + ) + + auth_header = { + "Authorization": f"Bearer {sp_token}", + } + + result = self.sdk.requests.get(payload.task_order_billing_validation_url, headers=auth_header) + + if result.status_code == 202: + # 202 has location/retry after headers + return self._ok(EnableTaskOrderBillingCSPResult(**result.headers)) + elif result.status_code == 200: + return self._ok(BillingProfileEnabledCSPResult(**result.json())) + else: + return self._error(result.json()) + + def report_clin(self, payload: ReportCLINCSPPayload): + sp_token = self._get_sp_token(payload.creds) + if sp_token is None: + raise AuthenticationException( + "Could not resolve token for task order billing validation" + ) + + request_body = { + "properties": { + "amount": payload.amount, + "startDate": payload.start_date, + "endDate": payload.end_date + } + } + + url = f"https://management.azure.com/providers/Microsoft.Billing/billingAccounts/{payload.billing_account_name}/billingProfiles/{payload.billing_profile_name}/instructions/{payload.task_order_id}:CLIN00{payload.clin_type}?api-version=2019-10-01-preview" + + auth_header = { + "Authorization": f"Bearer {sp_token}", + } + + result = self.sdk.requests.put(url, headers=auth_header, json=request_body) + + if result.status_code == 200: + return self._ok(ReportCLINCSPResult(**result.json())) + else: + return self._error(result.json()) def create_remote_admin(self, creds, tenant_details): # create app/service principal within tenant, with name constructed from tenant details diff --git a/tests/domain/cloud/test_azure_csp.py b/tests/domain/cloud/test_azure_csp.py index 6b9a8b12..9e9809c7 100644 --- a/tests/domain/cloud/test_azure_csp.py +++ b/tests/domain/cloud/test_azure_csp.py @@ -12,6 +12,13 @@ from atst.domain.csp.cloud import ( BillingProfileCreateCSPResult, BillingProfileVerifyCSPPayload, BillingProfileCSPResult, + BillingRoleAssignmentCSPPayload, + BillingRoleAssignmentCSPResult, + EnableTaskOrderBillingCSPPayload, + VerifyTaskOrderBillingCSPPayload, + BillingProfileEnabledCSPResult, + ReportCLINCSPPayload, + ReportCLINCSPResult, ) from tests.mock_azure import mock_azure, AUTH_CREDENTIALS @@ -25,7 +32,7 @@ from tests.factories import EnvironmentFactory, ApplicationFactory # -@pytest.mark.skip() +@pytest.mark.skip("Skipping legacy azure integration tests") def test_create_subscription_succeeds(mock_azure: AzureCloudProvider): environment = EnvironmentFactory.create() @@ -60,14 +67,13 @@ def test_create_subscription_succeeds(mock_azure: AzureCloudProvider): assert result == subscription_id -@pytest.mark.skip() def mock_management_group_create(mock_azure, spec_dict): mock_azure.sdk.managementgroups.ManagementGroupsAPI.return_value.management_groups.create_or_update.return_value.result.return_value = Mock( **spec_dict ) -@pytest.mark.skip() +@pytest.mark.skip("Skipping legacy azure integration tests") def test_create_environment_succeeds(mock_azure: AzureCloudProvider): environment = EnvironmentFactory.create() @@ -80,7 +86,7 @@ def test_create_environment_succeeds(mock_azure: AzureCloudProvider): assert result.id == "Test Id" -@pytest.mark.skip() +@pytest.mark.skip("Skipping legacy azure integration tests") def test_create_application_succeeds(mock_azure: AzureCloudProvider): application = ApplicationFactory.create() @@ -91,7 +97,7 @@ def test_create_application_succeeds(mock_azure: AzureCloudProvider): assert result.id == "Test Id" -@pytest.mark.skip() +@pytest.mark.skip("Skipping legacy azure integration tests") def test_create_atat_admin_user_succeeds(mock_azure: AzureCloudProvider): environment_id = str(uuid4()) @@ -106,7 +112,7 @@ def test_create_atat_admin_user_succeeds(mock_azure: AzureCloudProvider): assert result.get("csp_user_id") == csp_user_id -@pytest.mark.skip() +@pytest.mark.skip("Skipping legacy azure integration tests") def test_create_policy_definition_succeeds(mock_azure: AzureCloudProvider): subscription_id = str(uuid4()) management_group_id = str(uuid4()) @@ -148,33 +154,32 @@ def test_create_tenant(mock_azure: AzureCloudProvider): payload = TenantCSPPayload( **dict( creds={"username": "mock-cloud", "pass": "shh"}, - user_id="123", - password="123", - domain_name="123", - first_name="john", - last_name="doe", + user_id="admin", + password="JediJan13$coot", + domain_name="jediccpospawnedtenant2", + first_name="Tedry", + last_name="Tenet", country_code="US", - password_recovery_email_address="password@email.com", + password_recovery_email_address="thomas@promptworks.com", ) ) result = mock_azure.create_tenant(payload) - print(result) body: TenantCSPResult = result.get("body") assert body.tenant_id == "60ff9d34-82bf-4f21-b565-308ef0533435" def test_create_billing_profile(mock_azure: AzureCloudProvider): - # mock_azure.sdk.adal.AuthenticationContext.return_value.context.acquire_token_with_client_credentials.return_value = { - # "accessToken": "TOKEN" - # } + mock_azure.sdk.adal.AuthenticationContext.return_value.context.acquire_token_with_client_credentials.return_value = { + "accessToken": "TOKEN" + } - # mock_result = Mock() - # mock_result.headers = { - # "Location": "http://retry-url", - # "Retry-After": "10", - # } - # mock_result.status_code = 202 - # mock_azure.sdk.requests.post.return_value = mock_result + mock_result = Mock() + mock_result.headers = { + "Location": "http://retry-url", + "Retry-After": "10", + } + mock_result.status_code = 202 + mock_azure.sdk.requests.post.return_value = mock_result payload = BillingProfileCSPPayload( **dict( address=dict( @@ -191,7 +196,6 @@ def test_create_billing_profile(mock_azure: AzureCloudProvider): ) ) result = mock_azure.create_billing_profile(payload) - print(result) body: BillingProfileCreateCSPResult = result.get("body") assert body.retry_after == 10 @@ -204,8 +208,8 @@ def test_validate_billing_profile_creation(mock_azure: AzureCloudProvider): mock_result = Mock() mock_result.status_code = 200 mock_result.json.return_value = { - "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/XC36-GRNZ-BG7-TGB", - "name": "XC36-GRNZ-BG7-TGB", + "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB", + "name": "KQWI-W2SU-BG7-TGB", "properties": { "address": { "addressLine1": "123 S Broad Street, Suite 2400", @@ -223,7 +227,7 @@ def test_validate_billing_profile_creation(mock_azure: AzureCloudProvider): "invoiceEmailOptIn": False, "invoiceSections": [ { - "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/XC36-GRNZ-BG7-TGB/invoiceSections/6HMZ-2HLO-PJA-TGB", + "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB/invoiceSections/6HMZ-2HLO-PJA-TGB", "name": "6HMZ-2HLO-PJA-TGB", "properties": {"displayName": "First Portfolio Billing Profile"}, "type": "Microsoft.Billing/billingAccounts/billingProfiles/invoiceSections", @@ -241,15 +245,194 @@ def test_validate_billing_profile_creation(mock_azure: AzureCloudProvider): "password": "password", "tenant_id": "tenant_id", }, - location="https://management.azure.com/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/operationResults/createBillingProfile_478d5706-71f9-4a8b-8d4e-2cbaca27a668?api-version=2019-10-01-preview", + billing_profile_validate_url="https://management.azure.com/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/operationResults/createBillingProfile_478d5706-71f9-4a8b-8d4e-2cbaca27a668?api-version=2019-10-01-preview", ) ) result = mock_azure.validate_billing_profile_created(payload) body: BillingProfileCreateCSPResult = result.get("body") - assert body.billing_profile_name == "XC36-GRNZ-BG7-TGB" + assert body.billing_profile_name == "KQWI-W2SU-BG7-TGB" assert ( body.billing_profile_properties.display_name == "First Portfolio Billing Profile" ) + +def test_grant_billing_profile_tenant_access(mock_azure: AzureCloudProvider): + mock_azure.sdk.adal.AuthenticationContext.return_value.context.acquire_token_with_client_credentials.return_value = { + "accessToken": "TOKEN" + } + + mock_result = Mock() + mock_result.status_code = 201 + mock_result.json.return_value = { + "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB/billingRoleAssignments/40000000-aaaa-bbbb-cccc-100000000000_0a5f4926-e3ee-4f47-a6e3-8b0a30a40e3d", + "name": "40000000-aaaa-bbbb-cccc-100000000000_0a5f4926-e3ee-4f47-a6e3-8b0a30a40e3d", + "properties": { + "createdOn": "2020-01-14T14:39:26.3342192+00:00", + "createdByPrincipalId": "82e2b376-3297-4096-8743-ed65b3be0b03", + "principalId": "0a5f4926-e3ee-4f47-a6e3-8b0a30a40e3d", + "principalTenantId": "60ff9d34-82bf-4f21-b565-308ef0533435", + "roleDefinitionId": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB/billingRoleDefinitions/40000000-aaaa-bbbb-cccc-100000000000", + "scope": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB", + }, + "type": "Microsoft.Billing/billingRoleAssignments", + } + + mock_azure.sdk.requests.post.return_value = mock_result + + payload = BillingRoleAssignmentCSPPayload( + **dict( + creds={ + "username": "username", + "password": "password", + "tenant_id": "tenant_id", + }, + tenant_id="60ff9d34-82bf-4f21-b565-308ef0533435", + user_object_id="0a5f4926-e3ee-4f47-a6e3-8b0a30a40e3d", + billing_account_name="7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31", + billing_profile_name="KQWI-W2SU-BG7-TGB", + ) + ) + + result = mock_azure.grant_billing_profile_tenant_access(payload) + body: BillingRoleAssignmentCSPResult = result.get("body") + assert ( + body.billing_role_assignment_name + == "40000000-aaaa-bbbb-cccc-100000000000_0a5f4926-e3ee-4f47-a6e3-8b0a30a40e3d" + ) + + +def test_enable_task_order_billing(mock_azure: AzureCloudProvider): + mock_azure.sdk.adal.AuthenticationContext.return_value.context.acquire_token_with_client_credentials.return_value = { + "accessToken": "TOKEN" + } + + mock_result = Mock() + mock_result.status_code = 202 + mock_result.headers = { + "Location": "https://management.azure.com/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/operationResults/patchBillingProfile_KQWI-W2SU-BG7-TGB:02715576-4118-466c-bca7-b1cd3169ff46?api-version=2019-10-01-preview", + "Retry-After": "10", + } + + mock_azure.sdk.requests.patch.return_value = mock_result + + payload = EnableTaskOrderBillingCSPPayload( + **dict( + creds={ + "username": "username", + "password": "password", + "tenant_id": "tenant_id", + }, + billing_account_name="7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31", + billing_profile_name="KQWI-W2SU-BG7-TGB", + ) + ) + + result = mock_azure.enable_task_order_billing(payload) + body: BillingProfileCreateCSPResult = result.get("body") + assert ( + body.billing_profile_validate_url + == "https://management.azure.com/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/operationResults/patchBillingProfile_KQWI-W2SU-BG7-TGB:02715576-4118-466c-bca7-b1cd3169ff46?api-version=2019-10-01-preview" + ) + + +def test_validate_task_order_billing_enabled(mock_azure): + mock_azure.sdk.adal.AuthenticationContext.return_value.context.acquire_token_with_client_credentials.return_value = { + "accessToken": "TOKEN" + } + + mock_result = Mock() + mock_result.status_code = 200 + mock_result.json.return_value = { + "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB", + "name": "KQWI-W2SU-BG7-TGB", + "properties": { + "address": { + "addressLine1": "123 S Broad Street, Suite 2400", + "city": "Philadelphia", + "companyName": "Promptworks", + "country": "US", + "postalCode": "19109", + "region": "PA", + }, + "currency": "USD", + "displayName": "Test Billing Profile", + "enabledAzurePlans": [ + { + "productId": "DZH318Z0BPS6", + "skuId": "0001", + "skuDescription": "Microsoft Azure Plan", + } + ], + "hasReadAccess": True, + "invoiceDay": 5, + "invoiceEmailOptIn": False, + "invoiceSections": [ + { + "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB/invoiceSections/CHCO-BAAR-PJA-TGB", + "name": "CHCO-BAAR-PJA-TGB", + "properties": {"displayName": "Test Billing Profile"}, + "type": "Microsoft.Billing/billingAccounts/billingProfiles/invoiceSections", + } + ], + }, + "type": "Microsoft.Billing/billingAccounts/billingProfiles", + } + mock_azure.sdk.requests.get.return_value = mock_result + + payload = VerifyTaskOrderBillingCSPPayload( + **dict( + creds={ + "username": "username", + "password": "password", + "tenant_id": "tenant_id", + }, + task_order_billing_validation_url="https://management.azure.com/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/operationResults/createBillingProfile_478d5706-71f9-4a8b-8d4e-2cbaca27a668?api-version=2019-10-01-preview", + ) + ) + + result = mock_azure.validate_task_order_billing_enabled(payload) + body: BillingProfileEnabledCSPResult = result.get("body") + assert body.billing_profile_name == "KQWI-W2SU-BG7-TGB" + assert ( + body.billing_profile_enabled_plan_details.enabled_azure_plans[0].get("skuId") + == "0001" + ) + + +def test_report_clin(mock_azure: AzureCloudProvider): + mock_azure.sdk.adal.AuthenticationContext.return_value.context.acquire_token_with_client_credentials.return_value = { + "accessToken": "TOKEN" + } + + mock_result = Mock() + mock_result.status_code = 200 + mock_result.json.return_value = { + "name": "TO1:CLIN001", + "properties": { + "amount": 1000.0, + "endDate": "2020-03-01T00:00:00+00:00", + "startDate": "2020-01-01T00:00:00+00:00", + }, + "type": "Microsoft.Billing/billingAccounts/billingProfiles/billingInstructions", + } + + mock_azure.sdk.requests.put.return_value = mock_result + + payload = ReportCLINCSPPayload( + **dict( + creds={}, + amount=1000.00, + start_date="2020/1/1", + end_date="2020/3/1", + clin_type="1", + task_order_id="TO1", + billing_account_name="7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31", + billing_profile_name="KQWI-W2SU-BG7-TGB", + ) + ) + result = mock_azure.report_clin(payload) + body: ReportCLINCSPResult = result.get("body") + assert body.reported_clin_name == "TO1:CLIN001" + From 187ee0033e4b7277243d0fbac3c39597f4bcb943 Mon Sep 17 00:00:00 2001 From: Philip Kalinsky Date: Wed, 15 Jan 2020 16:07:50 -0500 Subject: [PATCH 06/46] state machine unit tests --- atst/models/mixins/state_machines.py | 37 +++++++----- atst/models/portfolio_state_machine.py | 12 ++-- tests/domain/test_portfolio_state_machine.py | 61 +++++++++++++++++++- 3 files changed, 85 insertions(+), 25 deletions(-) diff --git a/atst/models/mixins/state_machines.py b/atst/models/mixins/state_machines.py index bc35209d..b2eda399 100644 --- a/atst/models/mixins/state_machines.py +++ b/atst/models/mixins/state_machines.py @@ -10,7 +10,6 @@ class StageStates(Enum): class AzureStages(Enum): TENANT = "tenant" BILLING_PROFILE = "billing profile" - ADMIN_SUBSCRIPTION = "admin subscription" def _build_csp_states(csp_stages): @@ -31,14 +30,14 @@ def _build_csp_states(csp_stages): FSMStates = Enum("FSMStates", _build_csp_states(AzureStages)) +compose_state = lambda csp_stage, state: getattr( + FSMStates, "_".join([csp_stage.name, state.name]) +) + def _build_transitions(csp_stages): transitions = [] states = [] - compose_state = lambda csp_stage, state: getattr( - FSMStates, "_".join([csp_stage.name, state.name]) - ) - for stage_i, csp_stage in enumerate(csp_stages): for state in StageStates: states.append( @@ -99,6 +98,24 @@ class FSMMixin: {"trigger": "fail", "source": "*", "dest": FSMStates.FAILED,}, ] + def fail_stage(self, stage): + fail_trigger = "fail" + stage + if fail_trigger in self.machine.get_triggers(self.current_state.name): + self.trigger(fail_trigger) + + def finish_stage(self, stage): + finish_trigger = "finish_" + stage + if finish_trigger in self.machine.get_triggers(self.current_state.name): + self.trigger(finish_trigger) + + def _get_first_stage_create_trigger(self): + return list( + filter( + lambda trigger: trigger.startswith("create_"), + self.machine.get_triggers(FSMStates.STARTED.name), + ) + )[0] + def prepare_init(self, event): pass @@ -125,13 +142,3 @@ class FSMMixin: def after_reset(self, event): pass - - def fail_stage(self, stage): - fail_trigger = "fail" + stage - if fail_trigger in self.machine.get_triggers(self.current_state.name): - self.trigger(fail_trigger) - - def finish_stage(self, stage): - finish_trigger = "finish_" + stage - if finish_trigger in self.machine.get_triggers(self.current_state.name): - self.trigger(finish_trigger) diff --git a/atst/models/portfolio_state_machine.py b/atst/models/portfolio_state_machine.py index 3c934197..d7b6a36e 100644 --- a/atst/models/portfolio_state_machine.py +++ b/atst/models/portfolio_state_machine.py @@ -84,13 +84,11 @@ class PortfolioStateMachine( elif self.current_state == FSMStates.STARTED: # get the first trigger that starts with 'create_' - create_trigger = list( - filter( - lambda trigger: trigger.startswith("create_"), - self.machine.get_triggers(FSMStates.STARTED.name), - ) - )[0] - self.trigger(create_trigger) + create_trigger = self._get_first_stage_create_trigger() + if create_trigger: + self.trigger(create_trigger) + else: + self.fail_stage(stage) elif state_obj.is_IN_PROGRESS: pass diff --git a/tests/domain/test_portfolio_state_machine.py b/tests/domain/test_portfolio_state_machine.py index 0aa90867..d0a78fa0 100644 --- a/tests/domain/test_portfolio_state_machine.py +++ b/tests/domain/test_portfolio_state_machine.py @@ -6,6 +6,8 @@ from tests.factories import ( ) from atst.models import FSMStates +from atst.models.mixins.state_machines import AzureStages, StageStates, compose_state +from atst.domain.csp import get_stage_csp_class @pytest.fixture(scope="function") @@ -19,14 +21,67 @@ def test_fsm_creation(portfolio): assert sm.portfolio +def test_state_machine_trigger_next_transition(portfolio): + sm = PortfolioStateMachineFactory.create(portfolio=portfolio) + + sm.trigger_next_transition() + assert sm.current_state == FSMStates.STARTING + + sm.trigger_next_transition() + assert sm.current_state == FSMStates.STARTED + + +def test_state_machine_compose_state(portfolio): + sm = PortfolioStateMachineFactory.create(portfolio=portfolio) + assert ( + compose_state(AzureStages.TENANT, StageStates.CREATED) + == FSMStates.TENANT_CREATED + ) + + +def test_state_machine_first_stage_create_trigger(portfolio): + sm = PortfolioStateMachineFactory.create(portfolio=portfolio) + first_stage_create_trigger = sm._get_first_stage_create_trigger() + first_stage_name = list(AzureStages)[0].name.lower() + assert "create_" + first_stage_name == first_stage_create_trigger + + +def test_state_machine_valid_data_classes_for_stages(portfolio): + sm = PortfolioStateMachineFactory.create(portfolio=portfolio) + for stage in AzureStages: + assert get_stage_csp_class(stage.name.lower(), "payload") is not None + assert get_stage_csp_class(stage.name.lower(), "result") is not None + + +def test_state_machine_initialization(portfolio): + + sm = PortfolioStateMachineFactory.create(portfolio=portfolio) + for stage in AzureStages: + + # check that all stages have a 'create' and 'fail' triggers + stage_name = stage.name.lower() + for trigger_prefix in ["create", "fail"]: + assert hasattr(sm, trigger_prefix + "_" + stage_name) + + # check that machine + in_progress_triggers = sm.machine.get_triggers(stage.name + "_IN_PROGRESS") + assert [ + "reset", + "fail", + "finish_" + stage_name, + "fail_" + stage_name, + ] == in_progress_triggers + + started_triggers = sm.machine.get_triggers("STARTED") + first_stage_create_trigger = sm._get_first_stage_create_trigger() + assert ["reset", "fail", first_stage_create_trigger] == started_triggers + + def test_fsm_transition_start(portfolio): sm = PortfolioStateMachineFactory.create(portfolio=portfolio) assert sm.portfolio assert sm.state == FSMStates.UNSTARTED - # next_state does not create the trigger callbacks !!! - # sm.next_state() - sm.init() assert sm.state == FSMStates.STARTING From b1adaf771d30efbd03b53bfbc81b67b87e4a0724 Mon Sep 17 00:00:00 2001 From: tomdds Date: Thu, 16 Jan 2020 13:44:10 -0500 Subject: [PATCH 07/46] state machine integration wip --- atst/domain/csp/__init__.py | 6 +- atst/domain/csp/cloud.py | 119 ++++++++++++++++--- atst/models/mixins/state_machines.py | 3 + atst/models/portfolio_state_machine.py | 71 ++++++----- tests/domain/cloud/test_azure_csp.py | 14 +-- tests/domain/test_portfolio_state_machine.py | 50 +++++++- 6 files changed, 204 insertions(+), 59 deletions(-) diff --git a/atst/domain/csp/__init__.py b/atst/domain/csp/__init__.py index fc452935..f15ac1cd 100644 --- a/atst/domain/csp/__init__.py +++ b/atst/domain/csp/__init__.py @@ -34,9 +34,7 @@ def make_csp_provider(app, csp=None): def _stage_to_classname(stage): - return "".join( - map(lambda word: word.capitalize(), stage.replace("_", " ").split(" ")) - ) + return "".join(map(lambda word: word.capitalize(), stage.split("_"))) def get_stage_csp_class(stage, class_type): @@ -45,7 +43,7 @@ def get_stage_csp_class(stage, class_type): class_type is either 'payload' or 'result' """ - cls_name = "".join([_stage_to_classname(stage), "CSP", class_type.capitalize()]) + cls_name = f"{_stage_to_classname(stage)}CSP{class_type.capitalize()}" try: return getattr(importlib.import_module("atst.domain.csp.cloud"), cls_name) except AttributeError: diff --git a/atst/domain/csp/cloud.py b/atst/domain/csp/cloud.py index 703dec53..87492f19 100644 --- a/atst/domain/csp/cloud.py +++ b/atst/domain/csp/cloud.py @@ -186,11 +186,29 @@ class TenantCSPResult(AliasModel): tenant_id: str user_object_id: str + tenant_admin_username: str + tenant_admin_password: str + class Config: fields = { "user_object_id": "objectId", } + def dict(self, *args, **kwargs): + exclude = {"tenant_admin_username", "tenant_admin_password"} + if "exclude" not in kwargs: + kwargs["exclude"] = exclude + else: + kwargs["exclude"].update(exclude) + + return super().dict(*args, **kwargs) + + def get_creds(self): + return { + "tenant_admin_username": self.tenant_admin_username, + "tenant_admin_password": self.tenant_admin_password, + "tenant_id": self.tenant_id + } class BillingProfileAddress(AliasModel): company_name: str @@ -215,7 +233,7 @@ class BillingProfileCLINBudget(AliasModel): class BillingProfileCSPPayload(BaseCSPPayload): tenant_id: str - display_name: str + billing_profile_display_name: str enabled_azure_plans: Optional[List[str]] address: BillingProfileAddress @@ -229,6 +247,11 @@ class BillingProfileCSPPayload(BaseCSPPayload): """ return v or [] + class Config: + fields = { + "billing_profile_display_name": "displayName" + } + class BillingProfileCreateCSPResult(AliasModel): billing_profile_validate_url: str @@ -252,9 +275,14 @@ class BillingInvoiceSection(AliasModel): class BillingProfileProperties(AliasModel): address: BillingProfileAddress - display_name: str + billing_profile_display_name: str invoice_sections: List[BillingInvoiceSection] + class Config: + fields = { + "billing_profile_display_name": "displayName" + } + class BillingProfileCSPResult(AliasModel): billing_profile_id: str @@ -269,14 +297,14 @@ class BillingProfileCSPResult(AliasModel): } -class BillingRoleAssignmentCSPPayload(BaseCSPPayload): +class BillingProfileTenantAccessCSPPayload(BaseCSPPayload): tenant_id: str user_object_id: str billing_account_name: str billing_profile_name: str -class BillingRoleAssignmentCSPResult(AliasModel): +class BillingProfileTenantAccessCSPResult(AliasModel): billing_role_assignment_id: str billing_role_assignment_name: str @@ -286,7 +314,7 @@ class BillingRoleAssignmentCSPResult(AliasModel): "billing_role_assignment_name": "name", } -class EnableTaskOrderBillingCSPPayload(BaseCSPPayload): +class TaskOrderBillingCSPPayload(BaseCSPPayload): billing_account_name: str billing_profile_name: str @@ -297,14 +325,14 @@ class EnableTaskOrderBillingCSPResult(AliasModel): class Config: fields = {"task_order_billing_validation_url": "Location", "retry_after": "Retry-After"} -class VerifyTaskOrderBillingCSPPayload(BaseCSPPayload): +class TaskOrderBillingCSPResult(BaseCSPPayload): task_order_billing_validation_url: str class BillingProfileEnabledPlanDetails(AliasModel): enabled_azure_plans: List[Dict] -class BillingProfileEnabledCSPResult(AliasModel): +class TaskOrderBillingCSPResult(AliasModel): billing_profile_id: str billing_profile_name: str billing_profile_enabled_plan_details: BillingProfileEnabledPlanDetails @@ -534,9 +562,11 @@ class MockCloudProvider(CloudProviderInterface): "tenant_id": response["tenantId"], "user_id": response["userId"], "user_object_id": response["objectId"], + "tenant_admin_username": "test", + "tenant_admin_password": "test" } - def create_billing_profile(self, creds, tenant_admin_details, billing_owner_id): + def create_billing_profile(self, payload): # call billing profile creation endpoint, specifying owner # Payload: """ @@ -576,7 +606,55 @@ class MockCloudProvider(CloudProviderInterface): self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) response = {"id": "string"} - return {"billing_profile_id": response["id"]} + # return {"billing_profile_id": response["id"]} + return { + 'id': '/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB', + 'name': 'KQWI-W2SU-BG7-TGB', + 'properties': { + 'address': { + 'addressLine1': '123 S Broad Street, Suite 2400', + 'city': 'Philadelphia', + 'companyName': 'Promptworks', + 'country': 'US', + 'postalCode': '19109', + 'region': 'PA' + }, + 'currency': 'USD', + 'displayName': 'Test Billing Profile', + 'enabledAzurePlans': [], + 'hasReadAccess': True, + 'invoiceDay': 5, + 'invoiceEmailOptIn': False, + 'invoiceSections': [{ + 'id': '/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB/invoiceSections/CHCO-BAAR-PJA-TGB', + 'name': 'CHCO-BAAR-PJA-TGB', + 'properties': { + 'displayName': 'Test Billing Profile' + }, + 'type': 'Microsoft.Billing/billingAccounts/billingProfiles/invoiceSections' + }] + }, + 'type': 'Microsoft.Billing/billingAccounts/billingProfiles' + } + + def create_billing_profile_tenant_access(self, payload): + self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) + self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) + self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) + + return { + "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB/billingRoleAssignments/40000000-aaaa-bbbb-cccc-100000000000_0a5f4926-e3ee-4f47-a6e3-8b0a30a40e3d", + "name": "40000000-aaaa-bbbb-cccc-100000000000_0a5f4926-e3ee-4f47-a6e3-8b0a30a40e3d", + "properties": { + "createdOn": "2020-01-14T14:39:26.3342192+00:00", + "createdByPrincipalId": "82e2b376-3297-4096-8743-ed65b3be0b03", + "principalId": "0a5f4926-e3ee-4f47-a6e3-8b0a30a40e3d", + "principalTenantId": "60ff9d34-82bf-4f21-b565-308ef0533435", + "roleDefinitionId": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB/billingRoleDefinitions/40000000-aaaa-bbbb-cccc-100000000000", + "scope": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB" + }, + "type": "Microsoft.Billing/billingRoleAssignments" + } def create_or_update_user(self, auth_credentials, user_info, csp_role_id): self._authorize(auth_credentials) @@ -633,7 +711,7 @@ class MockCloudProvider(CloudProviderInterface): @property def _auth_credentials(self): - return {"username": "mock-cloud", "pass": "shh"} + return {"username": "mock-cloud", "password": "shh"} def _authorize(self, credentials): self._delay(1, 5) @@ -778,6 +856,9 @@ class AzureCloudProvider(CloudProviderInterface): headers=create_tenant_headers, ) + print('create tenant result') + print(result.json()) + if result.status_code == 200: return self._ok(TenantCSPResult(**result.json())) else: @@ -836,7 +917,7 @@ class AzureCloudProvider(CloudProviderInterface): else: return self._error(result.json()) - def grant_billing_profile_tenant_access(self, payload: BillingRoleAssignmentCSPPayload): + def create_billing_profile_tenant_access(self, payload: BillingProfileTenantAccessCSPPayload): sp_token = self._get_sp_token(payload.creds) request_body = { "properties": { @@ -854,11 +935,11 @@ class AzureCloudProvider(CloudProviderInterface): result = self.sdk.requests.post(url, headers=headers, json=request_body) if result.status_code == 201: - return self._ok(BillingRoleAssignmentCSPResult(**result.json())) + return self._ok(BillingProfileTenantAccessCSPResult(**result.json())) else: return self._error(result.json()) - def enable_task_order_billing(self, payload: EnableTaskOrderBillingCSPPayload): + def enable_task_order_billing(self, payload: TaskOrderBillingCSPPayload): sp_token = self._get_sp_token(payload.creds) request_body = [ { @@ -884,7 +965,7 @@ class AzureCloudProvider(CloudProviderInterface): # 202 has location/retry after headers return self._ok(BillingProfileCreateCSPResult(**result.headers)) elif result.status_code == 200: - return self._ok(BillingProfileEnabledCSPResult(**result.json())) + return self._ok(TaskOrderBillingCSPResult(**result.json())) else: return self._error(result.json()) @@ -903,13 +984,13 @@ class AzureCloudProvider(CloudProviderInterface): if result.status_code == 202: # 202 has location/retry after headers - return self._ok(EnableTaskOrderBillingCSPResult(**result.headers)) + return self._ok(TaskOrderBillingCSPResult(**result.headers)) elif result.status_code == 200: - return self._ok(BillingProfileEnabledCSPResult(**result.json())) + return self._ok(TaskOrderBillingCSPResult(**result.json())) else: return self._error(result.json()) - def report_clin(self, payload: ReportCLINCSPPayload): + def create_billing_instruction(self, payload: ReportCLINCSPPayload): sp_token = self._get_sp_token(payload.creds) if sp_token is None: raise AuthenticationException( @@ -1022,7 +1103,9 @@ class AzureCloudProvider(CloudProviderInterface): return sub_id_match.group(1) def _get_sp_token(self, creds): - home_tenant_id = creds.get("home_tenant_id") + home_tenant_id = creds.get( + "home_tenant_id" + ) client_id = creds.get("client_id") secret_key = creds.get("secret_key") diff --git a/atst/models/mixins/state_machines.py b/atst/models/mixins/state_machines.py index b2eda399..43044be0 100644 --- a/atst/models/mixins/state_machines.py +++ b/atst/models/mixins/state_machines.py @@ -10,6 +10,9 @@ class StageStates(Enum): class AzureStages(Enum): TENANT = "tenant" BILLING_PROFILE = "billing profile" + BILLING_PROFILE_TENANT_ACCESS = "billing profile tenant access" + TASK_ORDER_BILLING = "task order billing" + BILLING_INSTRUCTION = "billing instruction" def _build_csp_states(csp_stages): diff --git a/atst/models/portfolio_state_machine.py b/atst/models/portfolio_state_machine.py index d7b6a36e..aef03b22 100644 --- a/atst/models/portfolio_state_machine.py +++ b/atst/models/portfolio_state_machine.py @@ -1,3 +1,7 @@ +from random import choice, choices +import re +import string + from sqlalchemy import Column, ForeignKey, Enum as SQLAEnum from sqlalchemy.orm import relationship, reconstructor from sqlalchemy.dialects.postgresql import UUID @@ -17,6 +21,16 @@ import atst.models.mixins as mixins from atst.models.mixins.state_machines import FSMStates, AzureStages, _build_transitions +def make_password(): + return choice(string.ascii_letters) + "".join( + choices(string.ascii_letters + string.digits + string.punctuation, k=15) + ) + + +def fetch_portfolio_creds(portfolio): + return dict(username="mock-cloud", password="shh") + + @add_state_features(Tags) class StateMachineWithTags(Machine): pass @@ -73,57 +87,49 @@ class PortfolioStateMachine( return getattr(FSMStates, self.state) return self.state - def trigger_next_transition(self): + def trigger_next_transition(self, **kwargs): state_obj = self.machine.get_state(self.state) if state_obj.is_system: if self.current_state in (FSMStates.UNSTARTED, FSMStates.STARTING): # call the first trigger availabe for these two system states trigger_name = self.machine.get_triggers(self.current_state.name)[0] - self.trigger(trigger_name) + self.trigger(trigger_name, **kwargs) elif self.current_state == FSMStates.STARTED: # get the first trigger that starts with 'create_' create_trigger = self._get_first_stage_create_trigger() if create_trigger: - self.trigger(create_trigger) + self.trigger(create_trigger, **kwargs) else: self.fail_stage(stage) - elif state_obj.is_IN_PROGRESS: - pass - - # elif state_obj.is_TENANT: - # pass - # elif state_obj.is_BILLING_PROFILE: - # pass + elif state_obj.is_CREATED: + triggers = self.machine.get_triggers(state_obj.name) + self.trigger(triggers[-1], **kwargs) # @with_payload def after_in_progress_callback(self, event): stage = self.current_state.name.split("_IN_PROGRESS")[0].lower() - if stage == "tenant": - payload = dict( # nosec - creds={"username": "mock-cloud", "pass": "shh"}, - user_id="123", - password="123", - domain_name="123", - first_name="john", - last_name="doe", - country_code="US", - password_recovery_email_address="password@email.com", - ) - elif stage == "billing_profile": - payload = dict(creds={"username": "mock-cloud", "pass": "shh"},) + + # Accumulate payload w/ creds + payload = event.kwargs.get("csp_data") + payload["creds"] = event.kwargs.get("creds") payload_data_cls = get_stage_csp_class(stage, "payload") if not payload_data_cls: + print("could not resolve payload data class") self.fail_stage(stage) try: payload_data = payload_data_cls(**payload) except PydanticValidationError as exc: + print("Payload Validation Error:") print(exc.json()) + print("got") + print(payload) self.fail_stage(stage) + # TODO: Determine best place to do this, maybe @reconstructor csp = event.kwargs.get("csp") if csp is not None: self.csp = AzureCSP(app).cloud @@ -132,7 +138,8 @@ class PortfolioStateMachine( for attempt in range(5): try: - response = getattr(self.csp, "create_" + stage)(payload_data) + func_name = f"create_{stage}" + response = getattr(self.csp, func_name)(payload_data) except (ConnectionException, UnknownServerException) as exc: print("caught exception. retry", attempt) continue @@ -140,14 +147,17 @@ class PortfolioStateMachine( break else: # failed all attempts + print("failed") self.fail_stage(stage) if self.portfolio.csp_data is None: self.portfolio.csp_data = {} - self.portfolio.csp_data[stage + "_data"] = response + self.portfolio.csp_data.update(response) db.session.add(self.portfolio) db.session.commit() + # store any updated creds, if necessary + self.finish_stage(stage) def is_csp_data_valid(self, event): @@ -156,16 +166,23 @@ class PortfolioStateMachine( if self.portfolio.csp_data is None or not isinstance( self.portfolio.csp_data, dict ): + print("no csp data") return False stage = self.current_state.name.split("_IN_PROGRESS")[0].lower() - stage_data = self.portfolio.csp_data.get(stage + "_data") + stage_data = self.portfolio.csp_data cls = get_stage_csp_class(stage, "result") if not cls: return False try: - cls(**stage_data) + dc = cls(**stage_data) + if getattr(dc, "get_creds", None) is not None: + new_creds = dc.get_creds() + # TODO: how/where to store these + # TODO: credential schema + # self.store_creds(self.portfolio, new_creds) + except PydanticValidationError as exc: print(exc.json()) return False diff --git a/tests/domain/cloud/test_azure_csp.py b/tests/domain/cloud/test_azure_csp.py index 9e9809c7..bf228874 100644 --- a/tests/domain/cloud/test_azure_csp.py +++ b/tests/domain/cloud/test_azure_csp.py @@ -153,7 +153,7 @@ def test_create_tenant(mock_azure: AzureCloudProvider): mock_azure.sdk.requests.post.return_value = mock_result payload = TenantCSPPayload( **dict( - creds={"username": "mock-cloud", "pass": "shh"}, + creds={"username": "mock-cloud", "password": "shh"}, user_id="admin", password="JediJan13$coot", domain_name="jediccpospawnedtenant2", @@ -190,7 +190,7 @@ def test_create_billing_profile(mock_azure: AzureCloudProvider): country="US", postal_code="19109", ), - creds={"username": "mock-cloud", "pass": "shh"}, + creds={"username": "mock-cloud", "password": "shh"}, tenant_id="60ff9d34-82bf-4f21-b565-308ef0533435", display_name="Test Billing Profile", ) @@ -258,7 +258,7 @@ def test_validate_billing_profile_creation(mock_azure: AzureCloudProvider): ) -def test_grant_billing_profile_tenant_access(mock_azure: AzureCloudProvider): +def test_create_billing_profile_tenant_access(mock_azure: AzureCloudProvider): mock_azure.sdk.adal.AuthenticationContext.return_value.context.acquire_token_with_client_credentials.return_value = { "accessToken": "TOKEN" } @@ -295,7 +295,7 @@ def test_grant_billing_profile_tenant_access(mock_azure: AzureCloudProvider): ) ) - result = mock_azure.grant_billing_profile_tenant_access(payload) + result = mock_azure.create_billing_profile_tenant_access(payload) body: BillingRoleAssignmentCSPResult = result.get("body") assert ( body.billing_role_assignment_name @@ -303,7 +303,7 @@ def test_grant_billing_profile_tenant_access(mock_azure: AzureCloudProvider): ) -def test_enable_task_order_billing(mock_azure: AzureCloudProvider): +def test_create_task_order_billing(mock_azure: AzureCloudProvider): mock_azure.sdk.adal.AuthenticationContext.return_value.context.acquire_token_with_client_credentials.return_value = { "accessToken": "TOKEN" } @@ -401,7 +401,7 @@ def test_validate_task_order_billing_enabled(mock_azure): ) -def test_report_clin(mock_azure: AzureCloudProvider): +def test_create_billing_instruction(mock_azure: AzureCloudProvider): mock_azure.sdk.adal.AuthenticationContext.return_value.context.acquire_token_with_client_credentials.return_value = { "accessToken": "TOKEN" } @@ -432,7 +432,7 @@ def test_report_clin(mock_azure: AzureCloudProvider): billing_profile_name="KQWI-W2SU-BG7-TGB", ) ) - result = mock_azure.report_clin(payload) + result = mock_azure.create_billing_instruction(payload) body: ReportCLINCSPResult = result.get("body") assert body.reported_clin_name == "TO1:CLIN001" diff --git a/tests/domain/test_portfolio_state_machine.py b/tests/domain/test_portfolio_state_machine.py index d0a78fa0..9bd2c842 100644 --- a/tests/domain/test_portfolio_state_machine.py +++ b/tests/domain/test_portfolio_state_machine.py @@ -1,11 +1,12 @@ import pytest +import re from tests.factories import ( PortfolioFactory, PortfolioStateMachineFactory, ) -from atst.models import FSMStates +from atst.models import FSMStates, PortfolioStateMachine from atst.models.mixins.state_machines import AzureStages, StageStates, compose_state from atst.domain.csp import get_stage_csp_class @@ -78,7 +79,7 @@ def test_state_machine_initialization(portfolio): def test_fsm_transition_start(portfolio): - sm = PortfolioStateMachineFactory.create(portfolio=portfolio) + sm: PortfolioStateMachine = PortfolioStateMachineFactory.create(portfolio=portfolio) assert sm.portfolio assert sm.state == FSMStates.UNSTARTED @@ -87,5 +88,48 @@ def test_fsm_transition_start(portfolio): sm.start() assert sm.state == FSMStates.STARTED - sm.create_tenant(a=1, b=2) + + # Should source all creds for portfolio? might be easier to manage than per-step specific ones + creds = {"username": "mock-cloud", "password": "shh"} + if portfolio.csp_data is not None: + csp_data = portfolio.csp_data + else: + csp_data = {} + + ppoc = portfolio.owner + user_id = f"{ppoc.first_name[0]}{ppoc.last_name}".lower() + domain_name = re.sub("[^0-9a-zA-Z]+", "", portfolio.name).lower() + + portfolio_data = { + "user_id": user_id, + "password": "jklfsdNCVD83nklds2#202", + "domain_name": domain_name, + "first_name": ppoc.first_name, + "last_name": ppoc.last_name, + "country_code": "US", + "password_recovery_email_address": ppoc.email, + "address": { + "company_name": "", + "address_line_1": "", + "city": "", + "region": "", + "country": "", + "postal_code": "", + }, + "billing_profile_display_name": "My Billing Profile", + } + + collected_data = dict(list(csp_data.items()) + list(portfolio_data.items())) + sm.trigger_next_transition(creds=creds, csp_data=collected_data) + assert sm.state == FSMStates.TENANT_CREATED + assert portfolio.csp_data.get("tenant_id", None) is not None + + if portfolio.csp_data is not None: + csp_data = portfolio.csp_data + else: + csp_data = {} + collected_data = dict(list(csp_data.items()) + list(portfolio_data.items())) + sm.trigger_next_transition(creds=creds, csp_data=collected_data) + assert sm.state == FSMStates.BILLING_PROFILE_CREATED + From 743a91d658d101938788c5fb8356eebea2a4d9ed Mon Sep 17 00:00:00 2001 From: Philip Kalinsky Date: Thu, 16 Jan 2020 14:50:16 -0500 Subject: [PATCH 08/46] trigger_next_transition method will call the create trigger of the next stage for machines in CREATED state --- atst/models/portfolio_state_machine.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/atst/models/portfolio_state_machine.py b/atst/models/portfolio_state_machine.py index aef03b22..c5a5bb2d 100644 --- a/atst/models/portfolio_state_machine.py +++ b/atst/models/portfolio_state_machine.py @@ -105,8 +105,17 @@ class PortfolioStateMachine( self.fail_stage(stage) elif state_obj.is_CREATED: + # the create trigger for the next stage should be in the available + # triggers for the current state triggers = self.machine.get_triggers(state_obj.name) - self.trigger(triggers[-1], **kwargs) + create_trigger = list( + filter( + lambda trigger: trigger.startswith("create_"), + self.machine.get_triggers(self.state.name), + ) + )[0] + if create_trigger: + self.trigger(create_trigger, **kwargs) # @with_payload def after_in_progress_callback(self, event): From dfaea2d937270b954c0ac5509d9b46ffa5e4f91f Mon Sep 17 00:00:00 2001 From: Philip Kalinsky Date: Thu, 16 Jan 2020 15:29:14 -0500 Subject: [PATCH 09/46] trigger_next_transition method will call the create trigger of the next stage for machines in CREATED state --- atst/models/portfolio_state_machine.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/atst/models/portfolio_state_machine.py b/atst/models/portfolio_state_machine.py index c5a5bb2d..d3794a04 100644 --- a/atst/models/portfolio_state_machine.py +++ b/atst/models/portfolio_state_machine.py @@ -108,13 +108,17 @@ class PortfolioStateMachine( # the create trigger for the next stage should be in the available # triggers for the current state triggers = self.machine.get_triggers(state_obj.name) - create_trigger = list( - filter( - lambda trigger: trigger.startswith("create_"), - self.machine.get_triggers(self.state.name), - ) - )[0] - if create_trigger: + try: + create_trigger = list( + filter( + lambda trigger: trigger.startswith("create_"), + self.machine.get_triggers(self.state.name), + ) + )[0] + except IndexError: + # are we done ? + pass + else: self.trigger(create_trigger, **kwargs) # @with_payload From 67842748b8e4950c402682fac2fcd3ad3a5503d4 Mon Sep 17 00:00:00 2001 From: Philip Kalinsky Date: Thu, 16 Jan 2020 16:14:01 -0500 Subject: [PATCH 10/46] call next on the results of filter function to get the first value --- atst/models/mixins/state_machines.py | 8 -------- atst/models/portfolio_state_machine.py | 28 ++++++++++++++------------ 2 files changed, 15 insertions(+), 21 deletions(-) diff --git a/atst/models/mixins/state_machines.py b/atst/models/mixins/state_machines.py index 43044be0..6938c8a9 100644 --- a/atst/models/mixins/state_machines.py +++ b/atst/models/mixins/state_machines.py @@ -111,14 +111,6 @@ class FSMMixin: if finish_trigger in self.machine.get_triggers(self.current_state.name): self.trigger(finish_trigger) - def _get_first_stage_create_trigger(self): - return list( - filter( - lambda trigger: trigger.startswith("create_"), - self.machine.get_triggers(FSMStates.STARTED.name), - ) - )[0] - def prepare_init(self, event): pass diff --git a/atst/models/portfolio_state_machine.py b/atst/models/portfolio_state_machine.py index d3794a04..b8f29f24 100644 --- a/atst/models/portfolio_state_machine.py +++ b/atst/models/portfolio_state_machine.py @@ -98,7 +98,13 @@ class PortfolioStateMachine( elif self.current_state == FSMStates.STARTED: # get the first trigger that starts with 'create_' - create_trigger = self._get_first_stage_create_trigger() + create_trigger = next( + filter( + lambda trigger: trigger.startswith("create_"), + self.machine.get_triggers(FSMStates.STARTED.name), + ), + None, + ) if create_trigger: self.trigger(create_trigger, **kwargs) else: @@ -108,20 +114,16 @@ class PortfolioStateMachine( # the create trigger for the next stage should be in the available # triggers for the current state triggers = self.machine.get_triggers(state_obj.name) - try: - create_trigger = list( - filter( - lambda trigger: trigger.startswith("create_"), - self.machine.get_triggers(self.state.name), - ) - )[0] - except IndexError: - # are we done ? - pass - else: + create_trigger = next( + filter( + lambda trigger: trigger.startswith("create_"), + self.machine.get_triggers(self.state.name), + ), + None, + ) + if create_trigger is not None: self.trigger(create_trigger, **kwargs) - # @with_payload def after_in_progress_callback(self, event): stage = self.current_state.name.split("_IN_PROGRESS")[0].lower() From becc3630c2f7e7d3de2c466e9b22f89b483213dc Mon Sep 17 00:00:00 2001 From: Philip Kalinsky Date: Mon, 20 Jan 2020 14:17:01 -0500 Subject: [PATCH 11/46] azure integration. methods to authenticate and set/get value in keyvault --- Pipfile | 1 + Pipfile.lock | 73 +++++++++++++------ atst/domain/csp/cloud.py | 149 ++++++++++++++++++++++++++------------- 3 files changed, 152 insertions(+), 71 deletions(-) diff --git a/Pipfile b/Pipfile index f1d852b9..06bd2675 100644 --- a/Pipfile +++ b/Pipfile @@ -35,6 +35,7 @@ azure-mgmt-resource = "*" transitions = "*" azure-mgmt-consumption = "*" adal = "*" +azure-identity = "*" [dev-packages] bandit = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 681fddb1..8a030e11 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "a127b88e6c64842786f1868cb93bb1cdc828aa78040ea8ba4079bb3de0316dab" + "sha256": "3760f0b1df1156211d671afa2eb417b7bf980aa33d2f74d390e8eed6a3ce8c8b" }, "pipfile-spec": 6, "requires": { @@ -45,6 +45,13 @@ ], "version": "==1.1.24" }, + "azure-core": { + "hashes": [ + "sha256:b8ccbd901d085048e4e3e72627b066923c5bd3780e4c43cf9cf9948aee9bdf9e", + "sha256:e2cd99f0c0aef12c168d498cb5bc47a3a45c8ab08112183e3ec97e4dcb33ceb9" + ], + "version": "==1.2.1" + }, "azure-graphrbac": { "hashes": [ "sha256:53e98ae2ca7c19b349e9e9bb1b6a824aeae8dcfcbe17190d20fe69c0f185b2e2", @@ -53,6 +60,14 @@ "index": "pypi", "version": "==0.61.1" }, + "azure-identity": { + "hashes": [ + "sha256:4ce65058461c277991763ed3f121efc6b9eb9c2edefb62c414dfa85c814690d3", + "sha256:b32acd1cdb6202bfe10d9a0858dc463d8960295da70ae18097eb3b85ab12cb91" + ], + "index": "pypi", + "version": "==1.2.0" + }, "azure-mgmt-authorization": { "hashes": [ "sha256:31e875a34ac2c5d6fefe77b4a8079a8b2bdbe9edb957e47e8b44222fb212d6a7", @@ -354,6 +369,20 @@ ], "version": "==8.1.0" }, + "msal": { + "hashes": [ + "sha256:c944b833bf686dfbc973e9affdef94b77e616cb52ab397e76cde82e26b8a3373", + "sha256:ecbe3f5ac77facad16abf08eb9d8562af3bc7184be5d4d90c9ef4db5bde26340" + ], + "version": "==1.0.0" + }, + "msal-extensions": { + "hashes": [ + "sha256:59e171a9a4baacdbf001c66915efeaef372fb424421f1a4397115a3ddd6205dc", + "sha256:c5a32b8e1dce1c67733dcdf8aa8bebcff5ab123e779ef7bc14e416bd0da90037" + ], + "version": "==0.1.3" + }, "msrest": { "hashes": [ "sha256:56b8b5b4556fb2a92cac640df267d560889bdc9e2921187772d4691d97bc4e8d", @@ -388,6 +417,13 @@ "index": "pypi", "version": "==2.0.5" }, + "portalocker": { + "hashes": [ + "sha256:6f57aabb25ba176462dc7c63b86c42ad6a9b5bd3d679a9d776d0536bfb803d54", + "sha256:dac62e53e5670cb40d2ee4cdc785e6b829665932c3ee75307ad677cf5f7d2e9f" + ], + "version": "==1.5.2" + }, "psycopg2-binary": { "hashes": [ "sha256:040234f8a4a8dfd692662a8308d78f63f31a97e1c42d2480e5e6810c48966a29", @@ -453,6 +489,9 @@ "version": "==1.3" }, "pyjwt": { + "extras": [ + "crypto" + ], "hashes": [ "sha256:5c6eca3c2940464d106b99ba83b00c6add741c9becaec087fb7ccdefea71350e", "sha256:8d59a976fb773f3e6a39c85636357c4f0e242707394cadadd9814f5cbaa20e96" @@ -538,10 +577,10 @@ }, "six": { "hashes": [ - "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd", - "sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66" + "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a", + "sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c" ], - "version": "==1.13.0" + "version": "==1.14.0" }, "sqlalchemy": { "hashes": [ @@ -618,10 +657,10 @@ }, "zipp": { "hashes": [ - "sha256:8dda78f06bd1674bd8720df8a50bb47b6e1233c503a4eed8e7810686bde37656", - "sha256:d38fbe01bbf7a3593a32bc35a9c4453c32bc42b98c377f9bff7e9f8da157786c" + "sha256:57147f6b0403b59f33fd357f169f860e031303415aeb7d04ede4839d23905ab8", + "sha256:7ae5ccaca427bafa9760ac3cd8f8c244bfc259794b5b6bb9db4dda2241575d09" ], - "version": "==1.0.0" + "version": "==2.0.0" } }, "develop": { @@ -632,14 +671,6 @@ ], "version": "==1.4.3" }, - "appnope": { - "hashes": [ - "sha256:5b26757dc6f79a3b7dc9fab95359328d5747fcb2409d331ea66d0272b90ab2a0", - "sha256:8b995ffe925347a2138d7ac0fe77155e4311a0ea6d6da4f5128fe4b3cbe5ed71" - ], - "markers": "sys_platform == 'darwin'", - "version": "==0.1.0" - }, "argh": { "hashes": [ "sha256:a9b3aaa1904eeb78e32394cd46c6f37ac0fb4af6dc488daa58971bdc7d7fcaf3", @@ -1210,10 +1241,10 @@ }, "six": { "hashes": [ - "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd", - "sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66" + "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a", + "sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c" ], - "version": "==1.13.0" + "version": "==1.14.0" }, "smmap2": { "hashes": [ @@ -1328,10 +1359,10 @@ }, "zipp": { "hashes": [ - "sha256:8dda78f06bd1674bd8720df8a50bb47b6e1233c503a4eed8e7810686bde37656", - "sha256:d38fbe01bbf7a3593a32bc35a9c4453c32bc42b98c377f9bff7e9f8da157786c" + "sha256:57147f6b0403b59f33fd357f169f860e031303415aeb7d04ede4839d23905ab8", + "sha256:7ae5ccaca427bafa9760ac3cd8f8c244bfc259794b5b6bb9db4dda2241575d09" ], - "version": "==1.0.0" + "version": "==2.0.0" } } } diff --git a/atst/domain/csp/cloud.py b/atst/domain/csp/cloud.py index 87492f19..f14e468d 100644 --- a/atst/domain/csp/cloud.py +++ b/atst/domain/csp/cloud.py @@ -207,9 +207,10 @@ class TenantCSPResult(AliasModel): return { "tenant_admin_username": self.tenant_admin_username, "tenant_admin_password": self.tenant_admin_password, - "tenant_id": self.tenant_id + "tenant_id": self.tenant_id, } + class BillingProfileAddress(AliasModel): company_name: str address_line_1: str @@ -248,9 +249,7 @@ class BillingProfileCSPPayload(BaseCSPPayload): return v or [] class Config: - fields = { - "billing_profile_display_name": "displayName" - } + fields = {"billing_profile_display_name": "displayName"} class BillingProfileCreateCSPResult(AliasModel): @@ -258,7 +257,10 @@ class BillingProfileCreateCSPResult(AliasModel): retry_after: int class Config: - fields = {"billing_profile_validate_url": "Location", "retry_after": "Retry-After"} + fields = { + "billing_profile_validate_url": "Location", + "retry_after": "Retry-After", + } class BillingProfileVerifyCSPPayload(BaseCSPPayload): @@ -279,9 +281,7 @@ class BillingProfileProperties(AliasModel): invoice_sections: List[BillingInvoiceSection] class Config: - fields = { - "billing_profile_display_name": "displayName" - } + fields = {"billing_profile_display_name": "displayName"} class BillingProfileCSPResult(AliasModel): @@ -314,20 +314,27 @@ class BillingProfileTenantAccessCSPResult(AliasModel): "billing_role_assignment_name": "name", } + class TaskOrderBillingCSPPayload(BaseCSPPayload): billing_account_name: str billing_profile_name: str + class EnableTaskOrderBillingCSPResult(AliasModel): task_order_billing_validation_url: str retry_after: int class Config: - fields = {"task_order_billing_validation_url": "Location", "retry_after": "Retry-After"} + fields = { + "task_order_billing_validation_url": "Location", + "retry_after": "Retry-After", + } + class TaskOrderBillingCSPResult(BaseCSPPayload): task_order_billing_validation_url: str + class BillingProfileEnabledPlanDetails(AliasModel): enabled_azure_plans: List[Dict] @@ -344,6 +351,7 @@ class TaskOrderBillingCSPResult(AliasModel): "billing_profile_enabled_plan_details": "properties", } + class ReportCLINCSPPayload(BaseCSPPayload): amount: float start_date: str @@ -353,6 +361,7 @@ class ReportCLINCSPPayload(BaseCSPPayload): billing_account_name: str billing_profile_name: str + class ReportCLINCSPResult(AliasModel): reported_clin_name: str @@ -361,7 +370,16 @@ class ReportCLINCSPResult(AliasModel): "reported_clin_name": "name", } + class CloudProviderInterface: + + + def set_secret(secret_key: str, secret_value: str): + raise NotImplementedError() + + def get_secret(secret_key: str, secret_value: str): + raise NotImplementedError() + def root_creds(self) -> Dict: raise NotImplementedError() @@ -563,7 +581,7 @@ class MockCloudProvider(CloudProviderInterface): "user_id": response["userId"], "user_object_id": response["objectId"], "tenant_admin_username": "test", - "tenant_admin_password": "test" + "tenant_admin_password": "test", } def create_billing_profile(self, payload): @@ -608,33 +626,33 @@ class MockCloudProvider(CloudProviderInterface): response = {"id": "string"} # return {"billing_profile_id": response["id"]} return { - 'id': '/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB', - 'name': 'KQWI-W2SU-BG7-TGB', - 'properties': { - 'address': { - 'addressLine1': '123 S Broad Street, Suite 2400', - 'city': 'Philadelphia', - 'companyName': 'Promptworks', - 'country': 'US', - 'postalCode': '19109', - 'region': 'PA' + "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB", + "name": "KQWI-W2SU-BG7-TGB", + "properties": { + "address": { + "addressLine1": "123 S Broad Street, Suite 2400", + "city": "Philadelphia", + "companyName": "Promptworks", + "country": "US", + "postalCode": "19109", + "region": "PA", }, - 'currency': 'USD', - 'displayName': 'Test Billing Profile', - 'enabledAzurePlans': [], - 'hasReadAccess': True, - 'invoiceDay': 5, - 'invoiceEmailOptIn': False, - 'invoiceSections': [{ - 'id': '/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB/invoiceSections/CHCO-BAAR-PJA-TGB', - 'name': 'CHCO-BAAR-PJA-TGB', - 'properties': { - 'displayName': 'Test Billing Profile' - }, - 'type': 'Microsoft.Billing/billingAccounts/billingProfiles/invoiceSections' - }] + "currency": "USD", + "displayName": "Test Billing Profile", + "enabledAzurePlans": [], + "hasReadAccess": True, + "invoiceDay": 5, + "invoiceEmailOptIn": False, + "invoiceSections": [ + { + "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB/invoiceSections/CHCO-BAAR-PJA-TGB", + "name": "CHCO-BAAR-PJA-TGB", + "properties": {"displayName": "Test Billing Profile"}, + "type": "Microsoft.Billing/billingAccounts/billingProfiles/invoiceSections", + } + ], }, - 'type': 'Microsoft.Billing/billingAccounts/billingProfiles' + "type": "Microsoft.Billing/billingAccounts/billingProfiles", } def create_billing_profile_tenant_access(self, payload): @@ -651,9 +669,9 @@ class MockCloudProvider(CloudProviderInterface): "principalId": "0a5f4926-e3ee-4f47-a6e3-8b0a30a40e3d", "principalTenantId": "60ff9d34-82bf-4f21-b565-308ef0533435", "roleDefinitionId": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB/billingRoleDefinitions/40000000-aaaa-bbbb-cccc-100000000000", - "scope": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB" + "scope": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB", }, - "type": "Microsoft.Billing/billingRoleAssignments" + "type": "Microsoft.Billing/billingRoleAssignments", } def create_or_update_user(self, auth_credentials, user_info, csp_role_id): @@ -735,6 +753,9 @@ class AzureSDKProvider(object): from azure.mgmt import subscription, authorization import azure.graphrbac as graphrbac import azure.common.credentials as credentials + import azure.identity as identity + from azure.keyvault import secrets import secrets + from msrestazure.azure_cloud import AZURE_PUBLIC_CLOUD import adal import requests @@ -744,6 +765,8 @@ class AzureSDKProvider(object): self.adal = adal self.graphrbac = graphrbac self.credentials = credentials + self.identity = identity + self.secrets = secrets self.requests = requests # may change to a JEDI cloud self.cloud = AZURE_PUBLIC_CLOUD @@ -756,12 +779,29 @@ class AzureCloudProvider(CloudProviderInterface): self.client_id = config["AZURE_CLIENT_ID"] self.secret_key = config["AZURE_SECRET_KEY"] self.tenant_id = config["AZURE_TENANT_ID"] + self.vault_url = config["AZURE_VAULT_URL"] if azure_sdk_provider is None: self.sdk = AzureSDKProvider() else: self.sdk = azure_sdk_provider + def set_secret(secret_key, secret_value): + credential = self._get_client_secret_credential_obj() + secret_client = self.secrets.SecretClient( + vault_url=self.vault_url, + credential=credential, + ) + return secret_client.set_secret(secret_key, secret_value) + + def get_secret(secret_key) + credential = self._get_client_secret_credential_obj() + secret_client = self.secrets.SecretClient( + vault_url=self.vault_url, + credential=credential, + ) + return secret_client.get_secret(secret_key).value + def create_environment( self, auth_credentials: Dict, user: User, environment: Environment ): @@ -856,7 +896,7 @@ class AzureCloudProvider(CloudProviderInterface): headers=create_tenant_headers, ) - print('create tenant result') + print("create tenant result") print(result.json()) if result.status_code == 200: @@ -907,7 +947,9 @@ class AzureCloudProvider(CloudProviderInterface): "Authorization": f"Bearer {sp_token}", } - result = self.sdk.requests.get(payload.billing_profile_validate_url, headers=auth_header) + result = self.sdk.requests.get( + payload.billing_profile_validate_url, headers=auth_header + ) if result.status_code == 202: # 202 has location/retry after headers @@ -917,7 +959,9 @@ class AzureCloudProvider(CloudProviderInterface): else: return self._error(result.json()) - def create_billing_profile_tenant_access(self, payload: BillingProfileTenantAccessCSPPayload): + def create_billing_profile_tenant_access( + self, payload: BillingProfileTenantAccessCSPPayload + ): sp_token = self._get_sp_token(payload.creds) request_body = { "properties": { @@ -945,11 +989,7 @@ class AzureCloudProvider(CloudProviderInterface): { "op": "replace", "path": "/enabledAzurePlans", - "value": [ - { - "skuId": "0001" - } - ] + "value": [{"skuId": "0001"}], } ] @@ -959,7 +999,9 @@ class AzureCloudProvider(CloudProviderInterface): url = f"https://management.azure.com/providers/Microsoft.Billing/billingAccounts/{payload.billing_account_name}/billingProfiles/{payload.billing_profile_name}?api-version=2019-10-01-preview" - result = self.sdk.requests.patch(url, headers=request_headers, json=request_body) + result = self.sdk.requests.patch( + url, headers=request_headers, json=request_body + ) if result.status_code == 202: # 202 has location/retry after headers @@ -969,7 +1011,7 @@ class AzureCloudProvider(CloudProviderInterface): else: return self._error(result.json()) - def validate_task_order_billing_enabled(self, payload: VerifyTaskOrderBillingCSPPayload): + def validate_task_order_billing_enabled(self, payload: TaskOrderBillingCSPPayload): sp_token = self._get_sp_token(payload.creds) if sp_token is None: raise AuthenticationException( @@ -980,7 +1022,9 @@ class AzureCloudProvider(CloudProviderInterface): "Authorization": f"Bearer {sp_token}", } - result = self.sdk.requests.get(payload.task_order_billing_validation_url, headers=auth_header) + result = self.sdk.requests.get( + payload.task_order_billing_validation_url, headers=auth_header + ) if result.status_code == 202: # 202 has location/retry after headers @@ -1001,7 +1045,7 @@ class AzureCloudProvider(CloudProviderInterface): "properties": { "amount": payload.amount, "startDate": payload.start_date, - "endDate": payload.end_date + "endDate": payload.end_date, } } @@ -1125,7 +1169,6 @@ class AzureCloudProvider(CloudProviderInterface): return token_response.get("accessToken", None) def _get_credential_obj(self, creds, resource=None): - return self.sdk.credentials.ServicePrincipalCredentials( client_id=creds.get("client_id"), secret=creds.get("secret_key"), @@ -1133,6 +1176,12 @@ class AzureCloudProvider(CloudProviderInterface): resource=resource, cloud_environment=self.sdk.cloud, ) + def _get_client_secret_credential_obj(): + return self.sdk.identity.ClientSecretCredential( + tenant_id=creds.get("tenant_id"), + client_id =creds.get("client_id"), + client_secret = creds.get("secret_key"), + ) def _make_tenant_admin_cred_obj(self, username, password): return self.sdk.credentials.UserPassCredentials(username, password) From d646c3c00fc71b4922db74a480639fb4d08d3499 Mon Sep 17 00:00:00 2001 From: tomdds Date: Mon, 20 Jan 2020 12:51:02 -0500 Subject: [PATCH 12/46] Updates from Production Scripts Made a bunch of tweaks when using these tests to run production scripts for initial setup, this brings over a bunch of those changes --- atst/domain/csp/cloud.py | 22 +++++-------- tests/domain/cloud/test_azure_csp.py | 46 ++++++++++++++-------------- 2 files changed, 31 insertions(+), 37 deletions(-) diff --git a/atst/domain/csp/cloud.py b/atst/domain/csp/cloud.py index f14e468d..cd958781 100644 --- a/atst/domain/csp/cloud.py +++ b/atst/domain/csp/cloud.py @@ -235,6 +235,7 @@ class BillingProfileCLINBudget(AliasModel): class BillingProfileCSPPayload(BaseCSPPayload): tenant_id: str billing_profile_display_name: str + billing_account_name: str enabled_azure_plans: Optional[List[str]] address: BillingProfileAddress @@ -321,7 +322,7 @@ class TaskOrderBillingCSPPayload(BaseCSPPayload): class EnableTaskOrderBillingCSPResult(AliasModel): - task_order_billing_validation_url: str + task_order_billing_validate_url: str retry_after: int class Config: @@ -331,8 +332,8 @@ class EnableTaskOrderBillingCSPResult(AliasModel): } -class TaskOrderBillingCSPResult(BaseCSPPayload): - task_order_billing_validation_url: str +class VerifyTaskOrderBillingCSPPayload(BaseCSPPayload): + task_order_billing_validate_url: str class BillingProfileEnabledPlanDetails(AliasModel): @@ -754,7 +755,7 @@ class AzureSDKProvider(object): import azure.graphrbac as graphrbac import azure.common.credentials as credentials import azure.identity as identity - from azure.keyvault import secrets import secrets + from azure.keyvault import secrets from msrestazure.azure_cloud import AZURE_PUBLIC_CLOUD import adal @@ -896,11 +897,8 @@ class AzureCloudProvider(CloudProviderInterface): headers=create_tenant_headers, ) - print("create tenant result") - print(result.json()) - if result.status_code == 200: - return self._ok(TenantCSPResult(**result.json())) + return self._ok(TenantCSPResult(**result.json(), tenant_admin_password=payload.password, tenant_admin_username=payload.user_id)) else: return self._error(result.json()) @@ -917,9 +915,7 @@ class AzureCloudProvider(CloudProviderInterface): "Authorization": f"Bearer {sp_token}", } - # TODO: unsure if this is a static value or needs to be constructed/configurable - BILLING_ACCOUT_NAME = "7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31" - billing_account_create_url = f"https://management.azure.com/providers/Microsoft.Billing/billingAccounts/{BILLING_ACCOUT_NAME}/billingProfiles?api-version=2019-10-01-preview" + billing_account_create_url = f"https://management.azure.com/providers/Microsoft.Billing/billingAccounts/{payload.billing_account_name}/billingProfiles?api-version=2019-10-01-preview" result = self.sdk.requests.post( billing_account_create_url, @@ -1022,9 +1018,7 @@ class AzureCloudProvider(CloudProviderInterface): "Authorization": f"Bearer {sp_token}", } - result = self.sdk.requests.get( - payload.task_order_billing_validation_url, headers=auth_header - ) + result = self.sdk.requests.get(payload.task_order_billing_validate_url, headers=auth_header) if result.status_code == 202: # 202 has location/retry after headers diff --git a/tests/domain/cloud/test_azure_csp.py b/tests/domain/cloud/test_azure_csp.py index bf228874..577582d2 100644 --- a/tests/domain/cloud/test_azure_csp.py +++ b/tests/domain/cloud/test_azure_csp.py @@ -5,31 +5,30 @@ from uuid import uuid4 from atst.domain.csp.cloud import ( AzureCloudProvider, - TenantCSPResult, - TenantCSPPayload, - BillingProfileCSPPayload, - BillingProfileAddress, BillingProfileCreateCSPResult, - BillingProfileVerifyCSPPayload, + BillingProfileCSPPayload, BillingProfileCSPResult, - BillingRoleAssignmentCSPPayload, - BillingRoleAssignmentCSPResult, - EnableTaskOrderBillingCSPPayload, - VerifyTaskOrderBillingCSPPayload, - BillingProfileEnabledCSPResult, + BillingProfileTenantAccessCSPPayload, + BillingProfileTenantAccessCSPResult, + BillingProfileVerifyCSPPayload, ReportCLINCSPPayload, ReportCLINCSPResult, + TaskOrderBillingCSPPayload, + TenantCSPPayload, + TenantCSPResult, + VerifyTaskOrderBillingCSPPayload, ) from tests.mock_azure import mock_azure, AUTH_CREDENTIALS from tests.factories import EnvironmentFactory, ApplicationFactory -# TODO: Directly test create subscription, provide all args √ -# TODO: Test create environment (create management group with parent) -# TODO: Test create application (create manageemnt group with parent) -# Create reusable mock for mocking the management group calls for multiple services -# +creds = { + "home_tenant_id": "", + "client_id": "", + "secret_key": "", +} +BILLING_ACCOUNT_NAME = "52865e4c-52e8-5a6c-da6b-c58f0814f06f:7ea5de9d-b8ce-4901-b1c5-d864320c7b03_2019-05-31" @pytest.mark.skip("Skipping legacy azure integration tests") @@ -153,7 +152,7 @@ def test_create_tenant(mock_azure: AzureCloudProvider): mock_azure.sdk.requests.post.return_value = mock_result payload = TenantCSPPayload( **dict( - creds={"username": "mock-cloud", "password": "shh"}, + creds=creds, user_id="admin", password="JediJan13$coot", domain_name="jediccpospawnedtenant2", @@ -192,7 +191,8 @@ def test_create_billing_profile(mock_azure: AzureCloudProvider): ), creds={"username": "mock-cloud", "password": "shh"}, tenant_id="60ff9d34-82bf-4f21-b565-308ef0533435", - display_name="Test Billing Profile", + billing_profile_display_name="Test Billing Profile", + billing_account_name=BILLING_ACCOUNT_NAME, ) ) result = mock_azure.create_billing_profile(payload) @@ -250,10 +250,10 @@ def test_validate_billing_profile_creation(mock_azure: AzureCloudProvider): ) result = mock_azure.validate_billing_profile_created(payload) - body: BillingProfileCreateCSPResult = result.get("body") + body: BillingProfileCSPResult = result.get("body") assert body.billing_profile_name == "KQWI-W2SU-BG7-TGB" assert ( - body.billing_profile_properties.display_name + body.billing_profile_properties.billing_profile_display_name == "First Portfolio Billing Profile" ) @@ -281,7 +281,7 @@ def test_create_billing_profile_tenant_access(mock_azure: AzureCloudProvider): mock_azure.sdk.requests.post.return_value = mock_result - payload = BillingRoleAssignmentCSPPayload( + payload = BillingProfileTenantAccessCSPPayload( **dict( creds={ "username": "username", @@ -296,7 +296,7 @@ def test_create_billing_profile_tenant_access(mock_azure: AzureCloudProvider): ) result = mock_azure.create_billing_profile_tenant_access(payload) - body: BillingRoleAssignmentCSPResult = result.get("body") + body: BillingProfileTenantAccessCSPResult = result.get("body") assert ( body.billing_role_assignment_name == "40000000-aaaa-bbbb-cccc-100000000000_0a5f4926-e3ee-4f47-a6e3-8b0a30a40e3d" @@ -317,7 +317,7 @@ def test_create_task_order_billing(mock_azure: AzureCloudProvider): mock_azure.sdk.requests.patch.return_value = mock_result - payload = EnableTaskOrderBillingCSPPayload( + payload = TaskOrderBillingCSPPayload( **dict( creds={ "username": "username", @@ -388,7 +388,7 @@ def test_validate_task_order_billing_enabled(mock_azure): "password": "password", "tenant_id": "tenant_id", }, - task_order_billing_validation_url="https://management.azure.com/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/operationResults/createBillingProfile_478d5706-71f9-4a8b-8d4e-2cbaca27a668?api-version=2019-10-01-preview", + task_order_billing_validate_url="https://management.azure.com/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/operationResults/createBillingProfile_478d5706-71f9-4a8b-8d4e-2cbaca27a668?api-version=2019-10-01-preview", ) ) From f5e4b603cb9b8b286f954fea667760f7c174a04e Mon Sep 17 00:00:00 2001 From: tomdds Date: Wed, 22 Jan 2020 09:46:03 -0500 Subject: [PATCH 13/46] Bring naming conventions for methods and classes related to CSP provisioning in line with state machine --- atst/domain/csp/cloud.py | 171 ++++++++----------- atst/models/mixins/state_machines.py | 6 +- atst/models/portfolio_state_machine.py | 2 + tests/domain/cloud/test_azure_csp.py | 61 +++---- tests/domain/test_portfolio_state_machine.py | 14 +- tests/mock_azure.py | 1 + 6 files changed, 120 insertions(+), 135 deletions(-) diff --git a/atst/domain/csp/cloud.py b/atst/domain/csp/cloud.py index cd958781..afee36d9 100644 --- a/atst/domain/csp/cloud.py +++ b/atst/domain/csp/cloud.py @@ -186,8 +186,8 @@ class TenantCSPResult(AliasModel): tenant_id: str user_object_id: str - tenant_admin_username: str - tenant_admin_password: str + tenant_admin_username: Optional[str] + tenant_admin_password: Optional[str] class Config: fields = { @@ -232,7 +232,7 @@ class BillingProfileCLINBudget(AliasModel): """ -class BillingProfileCSPPayload(BaseCSPPayload): +class BillingProfileCreationCSPPayload(BaseCSPPayload): tenant_id: str billing_profile_display_name: str billing_account_name: str @@ -253,19 +253,19 @@ class BillingProfileCSPPayload(BaseCSPPayload): fields = {"billing_profile_display_name": "displayName"} -class BillingProfileCreateCSPResult(AliasModel): - billing_profile_validate_url: str - retry_after: int +class BillingProfileCreationCSPResult(AliasModel): + billing_profile_verify_url: str + billing_profile_retry_after: int class Config: fields = { - "billing_profile_validate_url": "Location", - "retry_after": "Retry-After", + "billing_profile_verify_url": "Location", + "billing_profile_retry_after": "Retry-After", } -class BillingProfileVerifyCSPPayload(BaseCSPPayload): - billing_profile_validate_url: str +class BillingProfileVerificationCSPPayload(BaseCSPPayload): + billing_profile_verify_url: str class BillingInvoiceSection(AliasModel): @@ -285,7 +285,7 @@ class BillingProfileProperties(AliasModel): fields = {"billing_profile_display_name": "displayName"} -class BillingProfileCSPResult(AliasModel): +class BillingProfileVerificationCSPResult(AliasModel): billing_profile_id: str billing_profile_name: str billing_profile_properties: BillingProfileProperties @@ -316,31 +316,31 @@ class BillingProfileTenantAccessCSPResult(AliasModel): } -class TaskOrderBillingCSPPayload(BaseCSPPayload): +class TaskOrderBillingCreationCSPPayload(BaseCSPPayload): billing_account_name: str billing_profile_name: str -class EnableTaskOrderBillingCSPResult(AliasModel): - task_order_billing_validate_url: str +class TaskOrderBillingCreationCSPResult(AliasModel): + task_order_billing_verify_url: str retry_after: int class Config: fields = { - "task_order_billing_validation_url": "Location", + "task_order_billing_verify_url": "Location", "retry_after": "Retry-After", } -class VerifyTaskOrderBillingCSPPayload(BaseCSPPayload): - task_order_billing_validate_url: str +class TaskOrderBillingVerificationCSPPayload(BaseCSPPayload): + task_order_billing_verify_url: str class BillingProfileEnabledPlanDetails(AliasModel): enabled_azure_plans: List[Dict] -class TaskOrderBillingCSPResult(AliasModel): +class TaskOrderBillingVerificationCSPResult(AliasModel): billing_profile_id: str billing_profile_name: str billing_profile_enabled_plan_details: BillingProfileEnabledPlanDetails @@ -353,7 +353,7 @@ class TaskOrderBillingCSPResult(AliasModel): } -class ReportCLINCSPPayload(BaseCSPPayload): +class BillingInstructionCSPPayload(BaseCSPPayload): amount: float start_date: str end_date: str @@ -363,7 +363,7 @@ class ReportCLINCSPPayload(BaseCSPPayload): billing_profile_name: str -class ReportCLINCSPResult(AliasModel): +class BillingInstructionCSPResult(AliasModel): reported_clin_name: str class Config: @@ -373,12 +373,10 @@ class ReportCLINCSPResult(AliasModel): class CloudProviderInterface: - - - def set_secret(secret_key: str, secret_value: str): + def set_secret(self, secret_key: str, secret_value: str): raise NotImplementedError() - def get_secret(secret_key: str, secret_value: str): + def get_secret(self, secret_key: str, secret_value: str): raise NotImplementedError() def root_creds(self) -> Dict: @@ -563,7 +561,7 @@ class MockCloudProvider(CloudProviderInterface): return {"id": self._id(), "credentials": self._auth_credentials} - def create_tenant(self, payload): + def create_tenant(self, payload: TenantCSPPayload): """ payload is an instance of TenantCSPPayload data class """ @@ -575,68 +573,41 @@ class MockCloudProvider(CloudProviderInterface): self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) - # return tenant id, tenant owner id and tenant owner object id from: - response = {"tenantId": "string", "userId": "string", "objectId": "string"} - return { - "tenant_id": response["tenantId"], - "user_id": response["userId"], - "user_object_id": response["objectId"], - "tenant_admin_username": "test", - "tenant_admin_password": "test", - } - def create_billing_profile(self, payload): - # call billing profile creation endpoint, specifying owner - # Payload: - """ - { - "displayName": "string", - "poNumber": "string", - "address": { - "firstName": "string", - "lastName": "string", - "companyName": "string", - "addressLine1": "string", - "addressLine2": "string", - "addressLine3": "string", - "city": "string", - "region": "string", - "country": "string", - "postalCode": "string" - }, - "invoiceEmailOptIn": true, - Note: These last 2 are also the body for adding/updating new TOs/clins - "enabledAzurePlans": [ - { - "skuId": "string" - } - ], - "clinBudget": { - "amount": 0, - "startDate": "2019-12-18T16:47:40.909Z", - "endDate": "2019-12-18T16:47:40.909Z", - "externalReferenceId": "string" - } - } - """ + return TenantCSPResult(**{ + "tenant_id": "", + "user_id": "", + "user_object_id": "", + "tenant_admin_username": "test", + "tenant_admin_password": "test" + }).dict() + + def create_billing_profile_creation(self, payload: BillingProfileCreationCSPPayload): # response will be mostly the same as the body, but we only really care about the id self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) - response = {"id": "string"} - # return {"billing_profile_id": response["id"]} - return { - "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB", - "name": "KQWI-W2SU-BG7-TGB", - "properties": { - "address": { - "addressLine1": "123 S Broad Street, Suite 2400", - "city": "Philadelphia", - "companyName": "Promptworks", - "country": "US", - "postalCode": "19109", - "region": "PA", + return BillingProfileCreationCSPResult(**dict( + billing_profile_verify_url = "https://zombo.com", + billing_profile_retry_after = 10 + )).dict() + + def create_billing_profile_verification(self, payload: BillingProfileVerificationCSPPayload): + self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) + self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) + self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) + return BillingProfileVerificationCSPResult(**{ + 'id': '/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB', + 'name': 'KQWI-W2SU-BG7-TGB', + 'properties': { + 'address': { + 'addressLine1': '123 S Broad Street, Suite 2400', + 'city': 'Philadelphia', + 'companyName': 'Promptworks', + 'country': 'US', + 'postalCode': '19109', + 'region': 'PA' }, "currency": "USD", "displayName": "Test Billing Profile", @@ -653,8 +624,8 @@ class MockCloudProvider(CloudProviderInterface): } ], }, - "type": "Microsoft.Billing/billingAccounts/billingProfiles", - } + 'type': 'Microsoft.Billing/billingAccounts/billingProfiles' + }).dict() def create_billing_profile_tenant_access(self, payload): self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) @@ -795,7 +766,7 @@ class AzureCloudProvider(CloudProviderInterface): ) return secret_client.set_secret(secret_key, secret_value) - def get_secret(secret_key) + def get_secret(secret_key): credential = self._get_client_secret_credential_obj() secret_client = self.secrets.SecretClient( vault_url=self.vault_url, @@ -902,7 +873,7 @@ class AzureCloudProvider(CloudProviderInterface): else: return self._error(result.json()) - def create_billing_profile(self, payload: BillingProfileCSPPayload): + def create_billing_profile_creation(self, payload: BillingProfileCreationCSPPayload): sp_token = self._get_sp_token(payload.creds) if sp_token is None: raise AuthenticationException( @@ -925,14 +896,14 @@ class AzureCloudProvider(CloudProviderInterface): if result.status_code == 202: # 202 has location/retry after headers - return self._ok(BillingProfileCreateCSPResult(**result.headers)) + return self._ok(BillingProfileCreationCSPResult(**result.headers)) elif result.status_code == 200: # NB: Swagger docs imply call can sometimes resolve immediately - return self._ok(BillingProfileCSPResult(**result.json())) + return self._ok(BillingProfileVerificationCSPResult(**result.json())) else: return self._error(result.json()) - def validate_billing_profile_created(self, payload: BillingProfileVerifyCSPPayload): + def create_billing_profile_verification(self, payload: BillingProfileVerificationCSPPayload): sp_token = self._get_sp_token(payload.creds) if sp_token is None: raise AuthenticationException( @@ -944,14 +915,14 @@ class AzureCloudProvider(CloudProviderInterface): } result = self.sdk.requests.get( - payload.billing_profile_validate_url, headers=auth_header + payload.billing_profile_verify_url, headers=auth_header ) if result.status_code == 202: # 202 has location/retry after headers - return self._ok(BillingProfileCreateCSPResult(**result.headers)) + return self._ok(BillingProfileCreationCSPResult(**result.headers)) elif result.status_code == 200: - return self._ok(BillingProfileCSPResult(**result.json())) + return self._ok(BillingProfileVerificationCSPResult(**result.json())) else: return self._error(result.json()) @@ -979,7 +950,7 @@ class AzureCloudProvider(CloudProviderInterface): else: return self._error(result.json()) - def enable_task_order_billing(self, payload: TaskOrderBillingCSPPayload): + def create_task_order_billing_creation(self, payload: TaskOrderBillingCreationCSPPayload): sp_token = self._get_sp_token(payload.creds) request_body = [ { @@ -1001,13 +972,13 @@ class AzureCloudProvider(CloudProviderInterface): if result.status_code == 202: # 202 has location/retry after headers - return self._ok(BillingProfileCreateCSPResult(**result.headers)) + return self._ok(TaskOrderBillingCreationCSPResult(**result.headers)) elif result.status_code == 200: - return self._ok(TaskOrderBillingCSPResult(**result.json())) + return self._ok(TaskOrderBillingVerificationCSPResult(**result.json())) else: return self._error(result.json()) - def validate_task_order_billing_enabled(self, payload: TaskOrderBillingCSPPayload): + def create_task_order_billing_verification(self, payload: TaskOrderBillingVerificationCSPPayload): sp_token = self._get_sp_token(payload.creds) if sp_token is None: raise AuthenticationException( @@ -1018,17 +989,17 @@ class AzureCloudProvider(CloudProviderInterface): "Authorization": f"Bearer {sp_token}", } - result = self.sdk.requests.get(payload.task_order_billing_validate_url, headers=auth_header) + result = self.sdk.requests.get(payload.task_order_billing_verify_url, headers=auth_header) if result.status_code == 202: # 202 has location/retry after headers - return self._ok(TaskOrderBillingCSPResult(**result.headers)) + return self._ok(TaskOrderBillingCreationCSPResult(**result.headers)) elif result.status_code == 200: - return self._ok(TaskOrderBillingCSPResult(**result.json())) + return self._ok(TaskOrderBillingVerificationCSPResult(**result.json())) else: return self._error(result.json()) - def create_billing_instruction(self, payload: ReportCLINCSPPayload): + def create_billing_instruction(self, payload: BillingInstructionCSPPayload): sp_token = self._get_sp_token(payload.creds) if sp_token is None: raise AuthenticationException( @@ -1052,7 +1023,7 @@ class AzureCloudProvider(CloudProviderInterface): result = self.sdk.requests.put(url, headers=auth_header, json=request_body) if result.status_code == 200: - return self._ok(ReportCLINCSPResult(**result.json())) + return self._ok(BillingInstructionCSPResult(**result.json())) else: return self._error(result.json()) diff --git a/atst/models/mixins/state_machines.py b/atst/models/mixins/state_machines.py index 6938c8a9..8d50eb15 100644 --- a/atst/models/mixins/state_machines.py +++ b/atst/models/mixins/state_machines.py @@ -9,9 +9,11 @@ class StageStates(Enum): class AzureStages(Enum): TENANT = "tenant" - BILLING_PROFILE = "billing profile" + BILLING_PROFILE_CREATION = "billing profile creation" + BILLING_PROFILE_VERIFICATION = "billing profile verification" BILLING_PROFILE_TENANT_ACCESS = "billing profile tenant access" - TASK_ORDER_BILLING = "task order billing" + TASK_ORDER_BILLING_CREATION = "task order billing creation" + TASK_ORDER_BILLING_VERIFICATION = "task order billing verification" BILLING_INSTRUCTION = "billing instruction" diff --git a/atst/models/portfolio_state_machine.py b/atst/models/portfolio_state_machine.py index b8f29f24..89dc073b 100644 --- a/atst/models/portfolio_state_machine.py +++ b/atst/models/portfolio_state_machine.py @@ -199,6 +199,8 @@ class PortfolioStateMachine( # self.store_creds(self.portfolio, new_creds) except PydanticValidationError as exc: + print("is_csp_data_valid: False") + print(cls) print(exc.json()) return False diff --git a/tests/domain/cloud/test_azure_csp.py b/tests/domain/cloud/test_azure_csp.py index 577582d2..b2099903 100644 --- a/tests/domain/cloud/test_azure_csp.py +++ b/tests/domain/cloud/test_azure_csp.py @@ -5,18 +5,20 @@ from uuid import uuid4 from atst.domain.csp.cloud import ( AzureCloudProvider, - BillingProfileCreateCSPResult, - BillingProfileCSPPayload, - BillingProfileCSPResult, + BillingProfileCreationCSPResult, + BillingProfileCreationCSPPayload, BillingProfileTenantAccessCSPPayload, BillingProfileTenantAccessCSPResult, - BillingProfileVerifyCSPPayload, - ReportCLINCSPPayload, - ReportCLINCSPResult, - TaskOrderBillingCSPPayload, + BillingProfileVerificationCSPPayload, + BillingProfileVerificationCSPResult, + BillingInstructionCSPPayload, + BillingInstructionCSPResult, + TaskOrderBillingCreationCSPPayload, + TaskOrderBillingCreationCSPResult, + TaskOrderBillingVerificationCSPPayload, + TaskOrderBillingVerificationCSPResult, TenantCSPPayload, TenantCSPResult, - VerifyTaskOrderBillingCSPPayload, ) from tests.mock_azure import mock_azure, AUTH_CREDENTIALS @@ -167,7 +169,7 @@ def test_create_tenant(mock_azure: AzureCloudProvider): assert body.tenant_id == "60ff9d34-82bf-4f21-b565-308ef0533435" -def test_create_billing_profile(mock_azure: AzureCloudProvider): +def test_create_billing_profile_creation(mock_azure: AzureCloudProvider): mock_azure.sdk.adal.AuthenticationContext.return_value.context.acquire_token_with_client_credentials.return_value = { "accessToken": "TOKEN" } @@ -179,7 +181,7 @@ def test_create_billing_profile(mock_azure: AzureCloudProvider): } mock_result.status_code = 202 mock_azure.sdk.requests.post.return_value = mock_result - payload = BillingProfileCSPPayload( + payload = BillingProfileCreationCSPPayload( **dict( address=dict( address_line_1="123 S Broad Street, Suite 2400", @@ -195,9 +197,9 @@ def test_create_billing_profile(mock_azure: AzureCloudProvider): billing_account_name=BILLING_ACCOUNT_NAME, ) ) - result = mock_azure.create_billing_profile(payload) - body: BillingProfileCreateCSPResult = result.get("body") - assert body.retry_after == 10 + result = mock_azure.create_billing_profile_creation(payload) + body: BillingProfileCreationCSPResult = result.get("body") + assert body.billing_profile_retry_after == 10 def test_validate_billing_profile_creation(mock_azure: AzureCloudProvider): @@ -238,19 +240,19 @@ def test_validate_billing_profile_creation(mock_azure: AzureCloudProvider): } mock_azure.sdk.requests.get.return_value = mock_result - payload = BillingProfileVerifyCSPPayload( + payload = BillingProfileVerificationCSPPayload( **dict( creds={ "username": "username", "password": "password", "tenant_id": "tenant_id", }, - billing_profile_validate_url="https://management.azure.com/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/operationResults/createBillingProfile_478d5706-71f9-4a8b-8d4e-2cbaca27a668?api-version=2019-10-01-preview", + billing_profile_verify_url="https://management.azure.com/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/operationResults/createBillingProfile_478d5706-71f9-4a8b-8d4e-2cbaca27a668?api-version=2019-10-01-preview", ) ) - result = mock_azure.validate_billing_profile_created(payload) - body: BillingProfileCSPResult = result.get("body") + result = mock_azure.create_billing_profile_verification(payload) + body: BillingProfileVerificationCSPResult = result.get("body") assert body.billing_profile_name == "KQWI-W2SU-BG7-TGB" assert ( body.billing_profile_properties.billing_profile_display_name @@ -303,7 +305,7 @@ def test_create_billing_profile_tenant_access(mock_azure: AzureCloudProvider): ) -def test_create_task_order_billing(mock_azure: AzureCloudProvider): +def test_create_task_order_billing_creation(mock_azure: AzureCloudProvider): mock_azure.sdk.adal.AuthenticationContext.return_value.context.acquire_token_with_client_credentials.return_value = { "accessToken": "TOKEN" } @@ -317,7 +319,7 @@ def test_create_task_order_billing(mock_azure: AzureCloudProvider): mock_azure.sdk.requests.patch.return_value = mock_result - payload = TaskOrderBillingCSPPayload( + payload = TaskOrderBillingCreationCSPPayload( **dict( creds={ "username": "username", @@ -329,15 +331,15 @@ def test_create_task_order_billing(mock_azure: AzureCloudProvider): ) ) - result = mock_azure.enable_task_order_billing(payload) - body: BillingProfileCreateCSPResult = result.get("body") + result = mock_azure.create_task_order_billing_creation(payload) + body: TaskOrderBillingCreationCSPResult = result.get("body") assert ( - body.billing_profile_validate_url + body.task_order_billing_verify_url == "https://management.azure.com/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/operationResults/patchBillingProfile_KQWI-W2SU-BG7-TGB:02715576-4118-466c-bca7-b1cd3169ff46?api-version=2019-10-01-preview" ) -def test_validate_task_order_billing_enabled(mock_azure): +def test_create_task_order_billing_verification(mock_azure): mock_azure.sdk.adal.AuthenticationContext.return_value.context.acquire_token_with_client_credentials.return_value = { "accessToken": "TOKEN" } @@ -381,19 +383,19 @@ def test_validate_task_order_billing_enabled(mock_azure): } mock_azure.sdk.requests.get.return_value = mock_result - payload = VerifyTaskOrderBillingCSPPayload( + payload = TaskOrderBillingVerificationCSPPayload( **dict( creds={ "username": "username", "password": "password", "tenant_id": "tenant_id", }, - task_order_billing_validate_url="https://management.azure.com/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/operationResults/createBillingProfile_478d5706-71f9-4a8b-8d4e-2cbaca27a668?api-version=2019-10-01-preview", + task_order_billing_verify_url="https://management.azure.com/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/operationResults/createBillingProfile_478d5706-71f9-4a8b-8d4e-2cbaca27a668?api-version=2019-10-01-preview", ) ) - result = mock_azure.validate_task_order_billing_enabled(payload) - body: BillingProfileEnabledCSPResult = result.get("body") + result = mock_azure.create_task_order_billing_verification(payload) + body: TaskOrderBillingVerificationCSPResult = result.get("body") assert body.billing_profile_name == "KQWI-W2SU-BG7-TGB" assert ( body.billing_profile_enabled_plan_details.enabled_azure_plans[0].get("skuId") @@ -420,7 +422,7 @@ def test_create_billing_instruction(mock_azure: AzureCloudProvider): mock_azure.sdk.requests.put.return_value = mock_result - payload = ReportCLINCSPPayload( + payload = BillingInstructionCSPPayload( **dict( creds={}, amount=1000.00, @@ -433,6 +435,5 @@ def test_create_billing_instruction(mock_azure: AzureCloudProvider): ) ) result = mock_azure.create_billing_instruction(payload) - body: ReportCLINCSPResult = result.get("body") + body: BillingInstructionCSPResult = result.get("body") assert body.reported_clin_name == "TO1:CLIN001" - diff --git a/tests/domain/test_portfolio_state_machine.py b/tests/domain/test_portfolio_state_machine.py index 9bd2c842..dcbc4bfd 100644 --- a/tests/domain/test_portfolio_state_machine.py +++ b/tests/domain/test_portfolio_state_machine.py @@ -119,17 +119,25 @@ def test_fsm_transition_start(portfolio): "billing_profile_display_name": "My Billing Profile", } - collected_data = dict(list(csp_data.items()) + list(portfolio_data.items())) + config = {"billing_account_name": "billing_account_name"} + + collected_data = dict( + list(csp_data.items()) + list(portfolio_data.items()) + list(config.items()) + ) sm.trigger_next_transition(creds=creds, csp_data=collected_data) assert sm.state == FSMStates.TENANT_CREATED assert portfolio.csp_data.get("tenant_id", None) is not None - + print(portfolio.csp_data.keys()) if portfolio.csp_data is not None: csp_data = portfolio.csp_data else: csp_data = {} - collected_data = dict(list(csp_data.items()) + list(portfolio_data.items())) + collected_data = dict( + list(csp_data.items()) + list(portfolio_data.items()) + list(config.items()) + ) sm.trigger_next_transition(creds=creds, csp_data=collected_data) assert sm.state == FSMStates.BILLING_PROFILE_CREATED + print(portfolio.csp_data.keys()) + diff --git a/tests/mock_azure.py b/tests/mock_azure.py index ecfafaac..23e9515c 100644 --- a/tests/mock_azure.py +++ b/tests/mock_azure.py @@ -8,6 +8,7 @@ AZURE_CONFIG = { "AZURE_SECRET_KEY": "MOCK", "AZURE_TENANT_ID": "MOCK", "AZURE_POLICY_LOCATION": "policies", + "AZURE_VAULT_URL": "http://vault", } AUTH_CREDENTIALS = { From 1b1a20cf52ece3ee70a5c6ea6873cf798cca7c42 Mon Sep 17 00:00:00 2001 From: tomdds Date: Wed, 22 Jan 2020 14:39:30 -0500 Subject: [PATCH 14/46] Restore implementations for policies and management group creation These were accidentally stripped out during a rebase. --- atst/domain/csp/cloud.py | 180 +++++++++++++++++++++------ tests/domain/cloud/test_azure_csp.py | 39 ++---- tests/mock_azure.py | 1 + 3 files changed, 153 insertions(+), 67 deletions(-) diff --git a/atst/domain/csp/cloud.py b/atst/domain/csp/cloud.py index afee36d9..c09731b2 100644 --- a/atst/domain/csp/cloud.py +++ b/atst/domain/csp/cloud.py @@ -5,9 +5,11 @@ from uuid import uuid4 from pydantic import BaseModel, validator from atst.models.user import User +from atst.models.application import Application from atst.models.environment import Environment from atst.models.environment_role import EnvironmentRole from atst.utils import snake_to_camel +from .policy import AzurePolicyManager class GeneralCSPException(Exception): @@ -718,11 +720,12 @@ SUBSCRIPTION_ID_REGEX = re.compile( # This needs to be a fully pathed role definition identifier, not just a UUID REMOTE_ROOT_ROLE_DEF_ID = "/providers/Microsoft.Authorization/roleDefinitions/00000000-0000-4000-8000-000000000000" - +AZURE_MANAGEMENT_API = "https://management.azure.com" class AzureSDKProvider(object): def __init__(self): - from azure.mgmt import subscription, authorization + from azure.mgmt import subscription, authorization, managementgroups + from azure.mgmt.resource import policy import azure.graphrbac as graphrbac import azure.common.credentials as credentials import azure.identity as identity @@ -733,6 +736,8 @@ class AzureSDKProvider(object): import requests self.subscription = subscription + self.policy = policy + self.managementgroups = managementgroups self.authorization = authorization self.adal = adal self.graphrbac = graphrbac @@ -758,6 +763,8 @@ class AzureCloudProvider(CloudProviderInterface): else: self.sdk = azure_sdk_provider + self.policy_manager = AzurePolicyManager(config["AZURE_POLICY_LOCATION"]) + def set_secret(secret_key, secret_value): credential = self._get_client_secret_credential_obj() secret_client = self.secrets.SecretClient( @@ -777,42 +784,23 @@ class AzureCloudProvider(CloudProviderInterface): def create_environment( self, auth_credentials: Dict, user: User, environment: Environment ): + # since this operation would only occur within a tenant, should we source the tenant + # via lookup from environment once we've created the portfolio csp data schema + # something like this: + # environment_tenant = environment.application.portfolio.csp_data.get('tenant_id', None) + # though we'd probably source the whole credentials for these calls from the portfolio csp + # data, as it would have to be where we store the creds for the at-at user within the portfolio tenant + # credentials = self._get_credential_obj(environment.application.portfolio.csp_data.get_creds()) credentials = self._get_credential_obj(self._root_creds) - sub_client = self.sdk.subscription.SubscriptionClient(credentials) - display_name = f"{environment.application.name}_{environment.name}_{environment.id}" # proposed format + management_group_id = "?" # management group id chained from environment + parent_id = "?" # from environment.application - billing_profile_id = "?" # something chained from environment? - sku_id = AZURE_SKU_ID - # we want to set AT-AT as an owner here - # we could potentially associate subscriptions with "management groups" per DOD component - body = self.sdk.subscription.models.ModernSubscriptionCreationParameters( - display_name, - billing_profile_id, - sku_id, - # owner= + management_group = self._create_management_group( + credentials, management_group_id, display_name, parent_id, ) - # These 2 seem like something that might be worthwhile to allow tiebacks to - # TOs filed for the environment - billing_account_name = "?" - invoice_section_name = "?" - # We may also want to create billing sections in the enrollment account - sub_creation_operation = sub_client.subscription_factory.create_subscription( - billing_account_name, invoice_section_name, body - ) - - # the resulting object from this process is a link to the new subscription - # not a subscription model, so we'll have to unpack the ID - new_sub = sub_creation_operation.result() - - subscription_id = self._extract_subscription_id(new_sub.subscription_link) - if subscription_id: - return subscription_id - else: - # troublesome error, subscription should exist at this point - # but we just don't have a valid ID - pass + return management_group def create_atat_admin_user( self, auth_credentials: Dict, csp_environment_id: str @@ -851,6 +839,125 @@ class AzureCloudProvider(CloudProviderInterface): "role_name": role_assignment_id, } + def _create_application(self, auth_credentials: Dict, application: Application): + management_group_name = str(uuid4()) # can be anything, not just uuid + display_name = application.name # Does this need to be unique? + credentials = self._get_credential_obj(auth_credentials) + parent_id = "?" # application.portfolio.csp_details.management_group_id + + return self._create_management_group( + credentials, management_group_name, display_name, parent_id, + ) + + def _create_management_group( + self, credentials, management_group_id, display_name, parent_id=None, + ): + mgmgt_group_client = self.sdk.managementgroups.ManagementGroupsAPI(credentials) + create_parent_grp_info = self.sdk.managementgroups.models.CreateParentGroupInfo( + id=parent_id + ) + create_mgmt_grp_details = self.sdk.managementgroups.models.CreateManagementGroupDetails( + parent=create_parent_grp_info + ) + mgmt_grp_create = self.sdk.managementgroups.models.CreateManagementGroupRequest( + name=management_group_id, + display_name=display_name, + details=create_mgmt_grp_details, + ) + create_request = mgmgt_group_client.management_groups.create_or_update( + management_group_id, mgmt_grp_create + ) + + # result is a synchronous wait, might need to do a poll instead to handle first mgmt group create + # since we were told it could take 10+ minutes to complete, unless this handles that polling internally + return create_request.result() + + def _create_subscription( + self, + credentials, + display_name, + billing_profile_id, + sku_id, + management_group_id, + billing_account_name, + invoice_section_name, + ): + sub_client = self.sdk.subscription.SubscriptionClient(credentials) + + billing_profile_id = "?" # where do we source this? + sku_id = AZURE_SKU_ID + # These 2 seem like something that might be worthwhile to allow tiebacks to + # TOs filed for the environment + billing_account_name = "?" # from TO? + invoice_section_name = "?" # from TO? + + body = self.sdk.subscription.models.ModernSubscriptionCreationParameters( + display_name=display_name, + billing_profile_id=billing_profile_id, + sku_id=sku_id, + management_group_id=management_group_id, + ) + + # We may also want to create billing sections in the enrollment account + sub_creation_operation = sub_client.subscription_factory.create_subscription( + billing_account_name, invoice_section_name, body + ) + + # the resulting object from this process is a link to the new subscription + # not a subscription model, so we'll have to unpack the ID + new_sub = sub_creation_operation.result() + + subscription_id = self._extract_subscription_id(new_sub.subscription_link) + if subscription_id: + return subscription_id + else: + # troublesome error, subscription should exist at this point + # but we just don't have a valid ID + pass + + def _create_policy_definition( + self, credentials, subscription_id, management_group_id, properties, + ): + """ + Requires credentials that have AZURE_MANAGEMENT_API + specified as the resource. The Service Principal + specified in the credentials must have the "Resource + Policy Contributor" role assigned with a scope at least + as high as the management group specified by + management_group_id. + + Arguments: + credentials -- ServicePrincipalCredentials + subscription_id -- str, ID of the subscription (just the UUID, not the path) + management_group_id -- str, ID of the management group (just the UUID, not the path) + properties -- dictionary, the "properties" section of a valid Azure policy definition document + + Returns: + azure.mgmt.resource.policy.[api version].models.PolicyDefinition: the PolicyDefinition object provided to Azure + + Raises: + TBD + """ + # TODO: which subscription would this be? + client = self.sdk.policy.PolicyClient(credentials, subscription_id) + + definition = client.policy_definitions.models.PolicyDefinition( + policy_type=properties.get("policyType"), + mode=properties.get("mode"), + display_name=properties.get("displayName"), + description=properties.get("description"), + policy_rule=properties.get("policyRule"), + parameters=properties.get("parameters"), + ) + + name = properties.get("displayName") + + return client.policy_definitions.create_or_update_at_management_group( + policy_definition_name=name, + parameters=definition, + management_group_id=management_group_id, + ) + def create_tenant(self, payload: TenantCSPPayload): sp_token = self._get_sp_token(payload.creds) if sp_token is None: @@ -1112,9 +1219,7 @@ class AzureCloudProvider(CloudProviderInterface): return sub_id_match.group(1) def _get_sp_token(self, creds): - home_tenant_id = creds.get( - "home_tenant_id" - ) + home_tenant_id = creds.get("home_tenant_id") client_id = creds.get("client_id") secret_key = creds.get("secret_key") @@ -1141,7 +1246,8 @@ class AzureCloudProvider(CloudProviderInterface): resource=resource, cloud_environment=self.sdk.cloud, ) - def _get_client_secret_credential_obj(): + + def _get_client_secret_credential_obj(self, creds): return self.sdk.identity.ClientSecretCredential( tenant_id=creds.get("tenant_id"), client_id =creds.get("client_id"), diff --git a/tests/domain/cloud/test_azure_csp.py b/tests/domain/cloud/test_azure_csp.py index b2099903..00830022 100644 --- a/tests/domain/cloud/test_azure_csp.py +++ b/tests/domain/cloud/test_azure_csp.py @@ -26,14 +26,13 @@ from tests.factories import EnvironmentFactory, ApplicationFactory creds = { - "home_tenant_id": "", - "client_id": "", - "secret_key": "", + "home_tenant_id": "tenant_id", + "client_id": "client_id", + "secret_key": "secret_key", } BILLING_ACCOUNT_NAME = "52865e4c-52e8-5a6c-da6b-c58f0814f06f:7ea5de9d-b8ce-4901-b1c5-d864320c7b03_2019-05-31" -@pytest.mark.skip("Skipping legacy azure integration tests") def test_create_subscription_succeeds(mock_azure: AzureCloudProvider): environment = EnvironmentFactory.create() @@ -74,7 +73,6 @@ def mock_management_group_create(mock_azure, spec_dict): ) -@pytest.mark.skip("Skipping legacy azure integration tests") def test_create_environment_succeeds(mock_azure: AzureCloudProvider): environment = EnvironmentFactory.create() @@ -87,7 +85,6 @@ def test_create_environment_succeeds(mock_azure: AzureCloudProvider): assert result.id == "Test Id" -@pytest.mark.skip("Skipping legacy azure integration tests") def test_create_application_succeeds(mock_azure: AzureCloudProvider): application = ApplicationFactory.create() @@ -98,7 +95,6 @@ def test_create_application_succeeds(mock_azure: AzureCloudProvider): assert result.id == "Test Id" -@pytest.mark.skip("Skipping legacy azure integration tests") def test_create_atat_admin_user_succeeds(mock_azure: AzureCloudProvider): environment_id = str(uuid4()) @@ -113,7 +109,6 @@ def test_create_atat_admin_user_succeeds(mock_azure: AzureCloudProvider): assert result.get("csp_user_id") == csp_user_id -@pytest.mark.skip("Skipping legacy azure integration tests") def test_create_policy_definition_succeeds(mock_azure: AzureCloudProvider): subscription_id = str(uuid4()) management_group_id = str(uuid4()) @@ -191,7 +186,7 @@ def test_create_billing_profile_creation(mock_azure: AzureCloudProvider): country="US", postal_code="19109", ), - creds={"username": "mock-cloud", "password": "shh"}, + creds=creds, tenant_id="60ff9d34-82bf-4f21-b565-308ef0533435", billing_profile_display_name="Test Billing Profile", billing_account_name=BILLING_ACCOUNT_NAME, @@ -242,11 +237,7 @@ def test_validate_billing_profile_creation(mock_azure: AzureCloudProvider): payload = BillingProfileVerificationCSPPayload( **dict( - creds={ - "username": "username", - "password": "password", - "tenant_id": "tenant_id", - }, + creds=creds, billing_profile_verify_url="https://management.azure.com/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/operationResults/createBillingProfile_478d5706-71f9-4a8b-8d4e-2cbaca27a668?api-version=2019-10-01-preview", ) ) @@ -285,11 +276,7 @@ def test_create_billing_profile_tenant_access(mock_azure: AzureCloudProvider): payload = BillingProfileTenantAccessCSPPayload( **dict( - creds={ - "username": "username", - "password": "password", - "tenant_id": "tenant_id", - }, + creds=creds, tenant_id="60ff9d34-82bf-4f21-b565-308ef0533435", user_object_id="0a5f4926-e3ee-4f47-a6e3-8b0a30a40e3d", billing_account_name="7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31", @@ -321,11 +308,7 @@ def test_create_task_order_billing_creation(mock_azure: AzureCloudProvider): payload = TaskOrderBillingCreationCSPPayload( **dict( - creds={ - "username": "username", - "password": "password", - "tenant_id": "tenant_id", - }, + creds=creds, billing_account_name="7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31", billing_profile_name="KQWI-W2SU-BG7-TGB", ) @@ -385,11 +368,7 @@ def test_create_task_order_billing_verification(mock_azure): payload = TaskOrderBillingVerificationCSPPayload( **dict( - creds={ - "username": "username", - "password": "password", - "tenant_id": "tenant_id", - }, + creds=creds, task_order_billing_verify_url="https://management.azure.com/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/operationResults/createBillingProfile_478d5706-71f9-4a8b-8d4e-2cbaca27a668?api-version=2019-10-01-preview", ) ) @@ -424,7 +403,7 @@ def test_create_billing_instruction(mock_azure: AzureCloudProvider): payload = BillingInstructionCSPPayload( **dict( - creds={}, + creds=creds, amount=1000.00, start_date="2020/1/1", end_date="2020/3/1", diff --git a/tests/mock_azure.py b/tests/mock_azure.py index 23e9515c..d9622b05 100644 --- a/tests/mock_azure.py +++ b/tests/mock_azure.py @@ -72,6 +72,7 @@ class MockAzureSDK(object): self.subscription = mock_subscription() self.authorization = mock_authorization() + self.policy = mock_policy() self.adal = mock_adal() self.managementgroups = mock_managementgroups() self.graphrbac = mock_graphrbac() From e5332897f16b0ea95db81b2d2937c12d61b31c5d Mon Sep 17 00:00:00 2001 From: tomdds Date: Wed, 22 Jan 2020 14:52:06 -0500 Subject: [PATCH 15/46] Fix formatting --- atst/domain/csp/cloud.py | 131 +++++++++++-------- tests/domain/test_portfolio_state_machine.py | 1 - 2 files changed, 78 insertions(+), 54 deletions(-) diff --git a/atst/domain/csp/cloud.py b/atst/domain/csp/cloud.py index c09731b2..9643dc7b 100644 --- a/atst/domain/csp/cloud.py +++ b/atst/domain/csp/cloud.py @@ -576,58 +576,68 @@ class MockCloudProvider(CloudProviderInterface): self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) - return TenantCSPResult(**{ - "tenant_id": "", - "user_id": "", - "user_object_id": "", - "tenant_admin_username": "test", - "tenant_admin_password": "test" - }).dict() + return TenantCSPResult( + **{ + "tenant_id": "", + "user_id": "", + "user_object_id": "", + "tenant_admin_username": "test", + "tenant_admin_password": "test", + } + ).dict() - def create_billing_profile_creation(self, payload: BillingProfileCreationCSPPayload): + def create_billing_profile_creation( + self, payload: BillingProfileCreationCSPPayload + ): # response will be mostly the same as the body, but we only really care about the id self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) - return BillingProfileCreationCSPResult(**dict( - billing_profile_verify_url = "https://zombo.com", - billing_profile_retry_after = 10 - )).dict() + return BillingProfileCreationCSPResult( + **dict( + billing_profile_verify_url="https://zombo.com", + billing_profile_retry_after=10, + ) + ).dict() - def create_billing_profile_verification(self, payload: BillingProfileVerificationCSPPayload): + def create_billing_profile_verification( + self, payload: BillingProfileVerificationCSPPayload + ): self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) - return BillingProfileVerificationCSPResult(**{ - 'id': '/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB', - 'name': 'KQWI-W2SU-BG7-TGB', - 'properties': { - 'address': { - 'addressLine1': '123 S Broad Street, Suite 2400', - 'city': 'Philadelphia', - 'companyName': 'Promptworks', - 'country': 'US', - 'postalCode': '19109', - 'region': 'PA' + return BillingProfileVerificationCSPResult( + **{ + "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB", + "name": "KQWI-W2SU-BG7-TGB", + "properties": { + "address": { + "addressLine1": "123 S Broad Street, Suite 2400", + "city": "Philadelphia", + "companyName": "Promptworks", + "country": "US", + "postalCode": "19109", + "region": "PA", + }, + "currency": "USD", + "displayName": "Test Billing Profile", + "enabledAzurePlans": [], + "hasReadAccess": True, + "invoiceDay": 5, + "invoiceEmailOptIn": False, + "invoiceSections": [ + { + "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB/invoiceSections/CHCO-BAAR-PJA-TGB", + "name": "CHCO-BAAR-PJA-TGB", + "properties": {"displayName": "Test Billing Profile"}, + "type": "Microsoft.Billing/billingAccounts/billingProfiles/invoiceSections", + } + ], }, - "currency": "USD", - "displayName": "Test Billing Profile", - "enabledAzurePlans": [], - "hasReadAccess": True, - "invoiceDay": 5, - "invoiceEmailOptIn": False, - "invoiceSections": [ - { - "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB/invoiceSections/CHCO-BAAR-PJA-TGB", - "name": "CHCO-BAAR-PJA-TGB", - "properties": {"displayName": "Test Billing Profile"}, - "type": "Microsoft.Billing/billingAccounts/billingProfiles/invoiceSections", - } - ], - }, - 'type': 'Microsoft.Billing/billingAccounts/billingProfiles' - }).dict() + "type": "Microsoft.Billing/billingAccounts/billingProfiles", + } + ).dict() def create_billing_profile_tenant_access(self, payload): self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) @@ -722,6 +732,7 @@ SUBSCRIPTION_ID_REGEX = re.compile( REMOTE_ROOT_ROLE_DEF_ID = "/providers/Microsoft.Authorization/roleDefinitions/00000000-0000-4000-8000-000000000000" AZURE_MANAGEMENT_API = "https://management.azure.com" + class AzureSDKProvider(object): def __init__(self): from azure.mgmt import subscription, authorization, managementgroups @@ -768,16 +779,14 @@ class AzureCloudProvider(CloudProviderInterface): def set_secret(secret_key, secret_value): credential = self._get_client_secret_credential_obj() secret_client = self.secrets.SecretClient( - vault_url=self.vault_url, - credential=credential, + vault_url=self.vault_url, credential=credential, ) return secret_client.set_secret(secret_key, secret_value) def get_secret(secret_key): credential = self._get_client_secret_credential_obj() secret_client = self.secrets.SecretClient( - vault_url=self.vault_url, - credential=credential, + vault_url=self.vault_url, credential=credential, ) return secret_client.get_secret(secret_key).value @@ -976,11 +985,19 @@ class AzureCloudProvider(CloudProviderInterface): ) if result.status_code == 200: - return self._ok(TenantCSPResult(**result.json(), tenant_admin_password=payload.password, tenant_admin_username=payload.user_id)) + return self._ok( + TenantCSPResult( + **result.json(), + tenant_admin_password=payload.password, + tenant_admin_username=payload.user_id, + ) + ) else: return self._error(result.json()) - def create_billing_profile_creation(self, payload: BillingProfileCreationCSPPayload): + def create_billing_profile_creation( + self, payload: BillingProfileCreationCSPPayload + ): sp_token = self._get_sp_token(payload.creds) if sp_token is None: raise AuthenticationException( @@ -1010,7 +1027,9 @@ class AzureCloudProvider(CloudProviderInterface): else: return self._error(result.json()) - def create_billing_profile_verification(self, payload: BillingProfileVerificationCSPPayload): + def create_billing_profile_verification( + self, payload: BillingProfileVerificationCSPPayload + ): sp_token = self._get_sp_token(payload.creds) if sp_token is None: raise AuthenticationException( @@ -1057,7 +1076,9 @@ class AzureCloudProvider(CloudProviderInterface): else: return self._error(result.json()) - def create_task_order_billing_creation(self, payload: TaskOrderBillingCreationCSPPayload): + def create_task_order_billing_creation( + self, payload: TaskOrderBillingCreationCSPPayload + ): sp_token = self._get_sp_token(payload.creds) request_body = [ { @@ -1085,7 +1106,9 @@ class AzureCloudProvider(CloudProviderInterface): else: return self._error(result.json()) - def create_task_order_billing_verification(self, payload: TaskOrderBillingVerificationCSPPayload): + def create_task_order_billing_verification( + self, payload: TaskOrderBillingVerificationCSPPayload + ): sp_token = self._get_sp_token(payload.creds) if sp_token is None: raise AuthenticationException( @@ -1096,7 +1119,9 @@ class AzureCloudProvider(CloudProviderInterface): "Authorization": f"Bearer {sp_token}", } - result = self.sdk.requests.get(payload.task_order_billing_verify_url, headers=auth_header) + result = self.sdk.requests.get( + payload.task_order_billing_verify_url, headers=auth_header + ) if result.status_code == 202: # 202 has location/retry after headers @@ -1250,8 +1275,8 @@ class AzureCloudProvider(CloudProviderInterface): def _get_client_secret_credential_obj(self, creds): return self.sdk.identity.ClientSecretCredential( tenant_id=creds.get("tenant_id"), - client_id =creds.get("client_id"), - client_secret = creds.get("secret_key"), + client_id=creds.get("client_id"), + client_secret=creds.get("secret_key"), ) def _make_tenant_admin_cred_obj(self, username, password): diff --git a/tests/domain/test_portfolio_state_machine.py b/tests/domain/test_portfolio_state_machine.py index dcbc4bfd..a3a3f5c0 100644 --- a/tests/domain/test_portfolio_state_machine.py +++ b/tests/domain/test_portfolio_state_machine.py @@ -140,4 +140,3 @@ def test_fsm_transition_start(portfolio): assert sm.state == FSMStates.BILLING_PROFILE_CREATED print(portfolio.csp_data.keys()) - From 00b10c484fbcb3886cd742c4984d3c4ef123e438 Mon Sep 17 00:00:00 2001 From: Philip Kalinsky Date: Wed, 22 Jan 2020 16:04:51 -0500 Subject: [PATCH 16/46] alembic migration to extend the state machine states enum --- alembic/env.py | 7 +++- ...19c44a8d5_state_machine_states_extended.py | 42 +++++++++++++++++++ 2 files changed, 47 insertions(+), 2 deletions(-) create mode 100644 alembic/versions/26319c44a8d5_state_machine_states_extended.py diff --git a/alembic/env.py b/alembic/env.py index 91b96364..825f5e19 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -51,7 +51,9 @@ def run_migrations_offline(): """ url = config.get_main_option("sqlalchemy.url") context.configure( - url=url, target_metadata=target_metadata, literal_binds=True) + url=url, target_metadata=target_metadata, literal_binds=True, + compare_type=True + ) with context.begin_transaction(): context.run_migrations() @@ -72,7 +74,8 @@ def run_migrations_online(): with connectable.connect() as connection: context.configure( connection=connection, - target_metadata=target_metadata + target_metadata=target_metadata, + compare_type=True ) with context.begin_transaction(): diff --git a/alembic/versions/26319c44a8d5_state_machine_states_extended.py b/alembic/versions/26319c44a8d5_state_machine_states_extended.py new file mode 100644 index 00000000..a183295c --- /dev/null +++ b/alembic/versions/26319c44a8d5_state_machine_states_extended.py @@ -0,0 +1,42 @@ +"""state machine states extended + +Revision ID: 26319c44a8d5 +Revises: 59973fa17ded +Create Date: 2020-01-22 15:54:03.186751 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = '26319c44a8d5' # pragma: allowlist secret +down_revision = '59973fa17ded' # pragma: allowlist secret +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('portfolio_state_machines', 'state', + existing_type=sa.VARCHAR(length=30), + type_=sa.Enum('UNSTARTED', 'STARTING', 'STARTED', 'COMPLETED', 'FAILED', 'TENANT_CREATED', 'TENANT_IN_PROGRESS', 'TENANT_FAILED', 'BILLING_PROFILE_CREATION_CREATED', 'BILLING_PROFILE_CREATION_IN_PROGRESS', 'BILLING_PROFILE_CREATION_FAILED', 'BILLING_PROFILE_VERIFICATION_CREATED', 'BILLING_PROFILE_VERIFICATION_IN_PROGRESS', 'BILLING_PROFILE_VERIFICATION_FAILED', 'BILLING_PROFILE_TENANT_ACCESS_CREATED', 'BILLING_PROFILE_TENANT_ACCESS_IN_PROGRESS', 'BILLING_PROFILE_TENANT_ACCESS_FAILED', 'TASK_ORDER_BILLING_CREATION_CREATED', 'TASK_ORDER_BILLING_CREATION_IN_PROGRESS', 'TASK_ORDER_BILLING_CREATION_FAILED', 'TASK_ORDER_BILLING_VERIFICATION_CREATED', 'TASK_ORDER_BILLING_VERIFICATION_IN_PROGRESS', 'TASK_ORDER_BILLING_VERIFICATION_FAILED', 'BILLING_INSTRUCTION_CREATED', 'BILLING_INSTRUCTION_IN_PROGRESS', 'BILLING_INSTRUCTION_FAILED', name='fsmstates', native_enum=False, create_constraint=False), + existing_nullable=False) + op.alter_column('portfolios', 'defense_component', + existing_type=postgresql.ARRAY(sa.VARCHAR()), + type_=sa.String(), + existing_nullable=False) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('portfolios', 'defense_component', + existing_type=sa.String(), + type_=postgresql.ARRAY(sa.VARCHAR()), + existing_nullable=False) + op.alter_column('portfolio_state_machines', 'state', + existing_type=sa.Enum('UNSTARTED', 'STARTING', 'STARTED', 'COMPLETED', 'FAILED', 'TENANT_CREATED', 'TENANT_IN_PROGRESS', 'TENANT_FAILED', 'BILLING_PROFILE_CREATION_CREATED', 'BILLING_PROFILE_CREATION_IN_PROGRESS', 'BILLING_PROFILE_CREATION_FAILED', 'BILLING_PROFILE_VERIFICATION_CREATED', 'BILLING_PROFILE_VERIFICATION_IN_PROGRESS', 'BILLING_PROFILE_VERIFICATION_FAILED', 'BILLING_PROFILE_TENANT_ACCESS_CREATED', 'BILLING_PROFILE_TENANT_ACCESS_IN_PROGRESS', 'BILLING_PROFILE_TENANT_ACCESS_FAILED', 'TASK_ORDER_BILLING_CREATION_CREATED', 'TASK_ORDER_BILLING_CREATION_IN_PROGRESS', 'TASK_ORDER_BILLING_CREATION_FAILED', 'TASK_ORDER_BILLING_VERIFICATION_CREATED', 'TASK_ORDER_BILLING_VERIFICATION_IN_PROGRESS', 'TASK_ORDER_BILLING_VERIFICATION_FAILED', 'BILLING_INSTRUCTION_CREATED', 'BILLING_INSTRUCTION_IN_PROGRESS', 'BILLING_INSTRUCTION_FAILED', name='fsmstates', native_enum=False), + type_=sa.VARCHAR(length=30), + existing_nullable=False) + # ### end Alembic commands ### From 9042a960bb2859a2a665ab61b7e0021f776df8e4 Mon Sep 17 00:00:00 2001 From: Rob Gil Date: Wed, 22 Jan 2020 19:35:19 -0500 Subject: [PATCH 17/46] Adds configurable service endpoints to subnets in the vpc module --- terraform/modules/vpc/main.tf | 2 ++ terraform/modules/vpc/variables.tf | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/terraform/modules/vpc/main.tf b/terraform/modules/vpc/main.tf index dbbe4bfa..a27f2139 100644 --- a/terraform/modules/vpc/main.tf +++ b/terraform/modules/vpc/main.tf @@ -39,6 +39,8 @@ resource "azurerm_subnet" "subnet" { lifecycle { ignore_changes = [route_table_id] } + + service_endpoints = split(",", var.service_endpoints[each.key]) #delegation { # name = "acctestdelegation" # diff --git a/terraform/modules/vpc/variables.tf b/terraform/modules/vpc/variables.tf index 9f331534..ac2dbac9 100644 --- a/terraform/modules/vpc/variables.tf +++ b/terraform/modules/vpc/variables.tf @@ -46,3 +46,9 @@ variable "gateway_subnet" { type = string description = "The Subnet CIDR that we'll use for the virtual_network_gateway 'GatewaySubnet'" } + +variable "service_endpoints" { + type = map + description = "A map of the service endpoints and its mapping to subnets" + +} From 01703b1488d6899ac149f92180f433e0f7e4e9ee Mon Sep 17 00:00:00 2001 From: Rob Gil Date: Wed, 22 Jan 2020 19:35:54 -0500 Subject: [PATCH 18/46] Configures storage buckets to be optionally exposed via service endpoints --- terraform/modules/bucket/main.tf | 6 ++++++ terraform/modules/bucket/variables.tf | 11 +++++++++++ terraform/providers/dev/buckets.tf | 4 ++++ terraform/providers/dev/variables.tf | 8 ++++++++ terraform/providers/dev/vpc.tf | 21 +++++++++++---------- 5 files changed, 40 insertions(+), 10 deletions(-) diff --git a/terraform/modules/bucket/main.tf b/terraform/modules/bucket/main.tf index 13231685..f8e7b9d7 100644 --- a/terraform/modules/bucket/main.tf +++ b/terraform/modules/bucket/main.tf @@ -9,6 +9,12 @@ resource "azurerm_storage_account" "bucket" { location = azurerm_resource_group.bucket.location account_tier = "Standard" account_replication_type = "LRS" + + network_rules { + default_action = var.policy + virtual_network_subnet_ids = var.subnet_ids + #ip_rules = ["66.220.238.246/30"] + } } resource "azurerm_storage_container" "bucket" { diff --git a/terraform/modules/bucket/variables.tf b/terraform/modules/bucket/variables.tf index 6278355e..7b2ae300 100644 --- a/terraform/modules/bucket/variables.tf +++ b/terraform/modules/bucket/variables.tf @@ -29,3 +29,14 @@ variable "service_name" { description = "Name of the service using this bucket" type = string } + +variable "subnet_ids" { + description = "List of subnet_ids that will have access to this service" + type = list +} + +variable "policy" { + description = "The default policy for the network access rules (Allow/Deny)" + default = "Deny" + type = string +} diff --git a/terraform/providers/dev/buckets.tf b/terraform/providers/dev/buckets.tf index d58987fc..d798214f 100644 --- a/terraform/providers/dev/buckets.tf +++ b/terraform/providers/dev/buckets.tf @@ -5,6 +5,8 @@ module "task_order_bucket" { name = var.name environment = var.environment region = var.region + policy = "Deny" + subnet_ids = [module.vpc.subnets] } module "tf_state" { @@ -14,4 +16,6 @@ module "tf_state" { name = var.name environment = var.environment region = var.region + policy = "Allow" + subnet_ids = [] } diff --git a/terraform/providers/dev/variables.tf b/terraform/providers/dev/variables.tf index 32ba5688..fc3afa30 100644 --- a/terraform/providers/dev/variables.tf +++ b/terraform/providers/dev/variables.tf @@ -36,6 +36,14 @@ variable "networks" { } } +variable "service_endpoints" { + type = map + default = { + public = "" + private = "Microsoft.Storage,Microsoft.KeyVault" + } +} + variable "gateway_subnet" { type = string default = "10.1.20.0/24" diff --git a/terraform/providers/dev/vpc.tf b/terraform/providers/dev/vpc.tf index b7fac8ae..44ecf35c 100644 --- a/terraform/providers/dev/vpc.tf +++ b/terraform/providers/dev/vpc.tf @@ -1,13 +1,14 @@ module "vpc" { - source = "../../modules/vpc/" - environment = var.environment - region = var.region - virtual_network = var.virtual_network - networks = var.networks - gateway_subnet = var.gateway_subnet - route_tables = var.route_tables - owner = var.owner - name = var.name - dns_servers = var.dns_servers + source = "../../modules/vpc/" + environment = var.environment + region = var.region + virtual_network = var.virtual_network + networks = var.networks + gateway_subnet = var.gateway_subnet + route_tables = var.route_tables + owner = var.owner + name = var.name + dns_servers = var.dns_servers + service_endpoints = var.service_endpoints } From 635ccb0fd349a60f495ad3f682cbfddd226097e2 Mon Sep 17 00:00:00 2001 From: Rob Gil Date: Wed, 22 Jan 2020 19:36:33 -0500 Subject: [PATCH 19/46] Fixes postgres character collation --- terraform/modules/postgres/main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/modules/postgres/main.tf b/terraform/modules/postgres/main.tf index 641c4102..c3252264 100644 --- a/terraform/modules/postgres/main.tf +++ b/terraform/modules/postgres/main.tf @@ -37,9 +37,9 @@ resource "azurerm_postgresql_virtual_network_rule" "sql" { } resource "azurerm_postgresql_database" "db" { - name = "${var.environment}-atat" + name = "${var.name}-${var.environment}-atat" resource_group_name = azurerm_resource_group.sql.name server_name = azurerm_postgresql_server.sql.name charset = "UTF8" - collation = "en_US.utf8" + collation = "en-US" } From d22357e6093bd175d23feeadca733cedebfdd9eb Mon Sep 17 00:00:00 2001 From: Rob Gil Date: Wed, 22 Jan 2020 19:37:04 -0500 Subject: [PATCH 20/46] Adds step to manually configure MFA in AD --- terraform/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/terraform/README.md b/terraform/README.md index 40460cb9..ec0fbdeb 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -281,3 +281,4 @@ secrets-tool secrets --keyvault https://ops-jedidev-keyvault.vault.azure.net/ cr `terraform apply` +*[Configure AD for MFA](https://docs.microsoft.com/en-us/azure/vpn-gateway/openvpn-azure-ad-mfa)* \ No newline at end of file From 48482785aca5b12e0f1eb68c1cf8c2a81dd79121 Mon Sep 17 00:00:00 2001 From: Rob Gil Date: Thu, 23 Jan 2020 10:02:31 -0500 Subject: [PATCH 21/46] Adds IP whitelisting to storage buckets --- terraform/modules/bucket/main.tf | 2 +- terraform/modules/bucket/variables.tf | 6 ++++++ terraform/providers/dev/buckets.tf | 12 ++++++++++-- terraform/providers/dev/variables.tf | 7 +++++++ 4 files changed, 24 insertions(+), 3 deletions(-) diff --git a/terraform/modules/bucket/main.tf b/terraform/modules/bucket/main.tf index f8e7b9d7..a92dc84f 100644 --- a/terraform/modules/bucket/main.tf +++ b/terraform/modules/bucket/main.tf @@ -13,7 +13,7 @@ resource "azurerm_storage_account" "bucket" { network_rules { default_action = var.policy virtual_network_subnet_ids = var.subnet_ids - #ip_rules = ["66.220.238.246/30"] + ip_rules = values(var.whitelist) } } diff --git a/terraform/modules/bucket/variables.tf b/terraform/modules/bucket/variables.tf index 7b2ae300..82367ed7 100644 --- a/terraform/modules/bucket/variables.tf +++ b/terraform/modules/bucket/variables.tf @@ -40,3 +40,9 @@ variable "policy" { default = "Deny" type = string } + +variable "whitelist" { + type = map + description = "A map of whitelisted IPs and CIDR ranges. For single IPs, Azure expects just the IP, NOT a /32." + default = {} +} diff --git a/terraform/providers/dev/buckets.tf b/terraform/providers/dev/buckets.tf index d798214f..ad2c7ab5 100644 --- a/terraform/providers/dev/buckets.tf +++ b/terraform/providers/dev/buckets.tf @@ -1,3 +1,5 @@ +# Task order bucket is required to be accessible publicly by the users. +# which is why the policy here is "Allow" module "task_order_bucket" { source = "../../modules/bucket" service_name = "jeditasksatat" @@ -5,10 +7,15 @@ module "task_order_bucket" { name = var.name environment = var.environment region = var.region - policy = "Deny" + policy = "Allow" subnet_ids = [module.vpc.subnets] + whitelist = var.admin_user_whitelist } +# TF State should be restricted to admins only, but IP protected +# This has to be public due to a chicken/egg issue of VPN not +# existing until TF is run. If this bucket is private, you would +# not be able to access it when running TF without being on a VPN. module "tf_state" { source = "../../modules/bucket" service_name = "jedidevtfstate" @@ -16,6 +23,7 @@ module "tf_state" { name = var.name environment = var.environment region = var.region - policy = "Allow" + policy = "Deny" subnet_ids = [] + whitelist = var.admin_user_whitelist } diff --git a/terraform/providers/dev/variables.tf b/terraform/providers/dev/variables.tf index fc3afa30..573b6cc9 100644 --- a/terraform/providers/dev/variables.tf +++ b/terraform/providers/dev/variables.tf @@ -87,3 +87,10 @@ variable "admin_users" { "Dan Corrigan" = "7e852ceb-eb0d-49b1-b71e-e9dcd1082ffc" } } + +variable "admin_user_whitelist" { + type = map + default = { + "Rob Gil" = "66.220.238.246" + } +} From 597ea32e426b7a879cd5969b7d2d03535efe8ae2 Mon Sep 17 00:00:00 2001 From: Philip Kalinsky Date: Thu, 23 Jan 2020 10:14:22 -0500 Subject: [PATCH 22/46] portfolio provision process. add logging, __repr___ method, fix broken tests --- atst/domain/csp/cloud.py | 14 ++++++-- atst/models/mixins/state_machines.py | 4 +++ atst/models/portfolio_state_machine.py | 29 ++++++++++------- tests/domain/test_portfolio_state_machine.py | 34 ++++++++++---------- 4 files changed, 50 insertions(+), 31 deletions(-) diff --git a/atst/domain/csp/cloud.py b/atst/domain/csp/cloud.py index 9643dc7b..5d9a8397 100644 --- a/atst/domain/csp/cloud.py +++ b/atst/domain/csp/cloud.py @@ -4,6 +4,8 @@ from uuid import uuid4 from pydantic import BaseModel, validator +from flask import current_app as app + from atst.models.user import User from atst.models.application import Application from atst.models.environment import Environment @@ -754,6 +756,7 @@ class AzureSDKProvider(object): self.graphrbac = graphrbac self.credentials = credentials self.identity = identity + self.exceptions = exceptions self.secrets = secrets self.requests = requests # may change to a JEDI cloud @@ -781,14 +784,21 @@ class AzureCloudProvider(CloudProviderInterface): secret_client = self.secrets.SecretClient( vault_url=self.vault_url, credential=credential, ) - return secret_client.set_secret(secret_key, secret_value) + try: + return secret_client.set_secret(secret_key, secret_value) + except self.exceptions.HttpResponseError as exc: + app.logger.error(f"Could not SET secret in Azure keyvault for key {secret_key}.", exc_info=1) + def get_secret(secret_key): credential = self._get_client_secret_credential_obj() secret_client = self.secrets.SecretClient( vault_url=self.vault_url, credential=credential, ) - return secret_client.get_secret(secret_key).value + try: + return secret_client.get_secret(secret_key).value + except self.exceptions.HttpResponseError as exc: + app.logger.error(f"Could not GET secret in Azure keyvault for key {secret_key}.", exc_info=1) def create_environment( self, auth_credentials: Dict, user: User, environment: Environment diff --git a/atst/models/mixins/state_machines.py b/atst/models/mixins/state_machines.py index 8d50eb15..29f7a877 100644 --- a/atst/models/mixins/state_machines.py +++ b/atst/models/mixins/state_machines.py @@ -1,5 +1,7 @@ from enum import Enum +from flask import current_app as app + class StageStates(Enum): CREATED = "created" @@ -107,10 +109,12 @@ class FSMMixin: fail_trigger = "fail" + stage if fail_trigger in self.machine.get_triggers(self.current_state.name): self.trigger(fail_trigger) + app.logger.info(f"calling fail trigger '{fail_trigger}' for '{self.__repr__()}'") def finish_stage(self, stage): finish_trigger = "finish_" + stage if finish_trigger in self.machine.get_triggers(self.current_state.name): + app.logger.info(f"calling finish trigger '{finish_trigger}' for '{self.__repr__()}'") self.trigger(finish_trigger) def prepare_init(self, event): diff --git a/atst/models/portfolio_state_machine.py b/atst/models/portfolio_state_machine.py index 89dc073b..04d09bd6 100644 --- a/atst/models/portfolio_state_machine.py +++ b/atst/models/portfolio_state_machine.py @@ -64,6 +64,10 @@ class PortfolioStateMachine( db.session.add(self) db.session.commit() + def __repr__(self): + return f" Date: Wed, 22 Jan 2020 16:47:11 -0500 Subject: [PATCH 23/46] Fix Enum Migration to properly deal with constraint changes Since we're using non-native enums for our model, alembic has some issues knowing what the previous "type" actually was, and not specifying it correctly causes a bad constraint. --- alembic/env.py | 17 +-- ...19c44a8d5_state_machine_states_extended.py | 127 +++++++++++++++--- 2 files changed, 118 insertions(+), 26 deletions(-) diff --git a/alembic/env.py b/alembic/env.py index 825f5e19..5ea815e3 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -29,11 +29,13 @@ parent_dir = Path(__file__).parent.parent sys.path.append(parent_dir) from atst.app import make_config + app_config = make_config() -config.set_main_option('sqlalchemy.url', app_config['DATABASE_URI']) +config.set_main_option("sqlalchemy.url", app_config["DATABASE_URI"]) from atst.database import db from atst.models import * + target_metadata = Base.metadata @@ -51,8 +53,7 @@ def run_migrations_offline(): """ url = config.get_main_option("sqlalchemy.url") context.configure( - url=url, target_metadata=target_metadata, literal_binds=True, - compare_type=True + url=url, target_metadata=target_metadata, literal_binds=True, compare_type=True ) with context.begin_transaction(): @@ -68,19 +69,19 @@ def run_migrations_online(): """ connectable = engine_from_config( config.get_section(config.config_ini_section), - prefix='sqlalchemy.', - poolclass=pool.NullPool) + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) with connectable.connect() as connection: context.configure( - connection=connection, - target_metadata=target_metadata, - compare_type=True + connection=connection, target_metadata=target_metadata, compare_type=True ) with context.begin_transaction(): context.run_migrations() + if context.is_offline_mode(): run_migrations_offline() else: diff --git a/alembic/versions/26319c44a8d5_state_machine_states_extended.py b/alembic/versions/26319c44a8d5_state_machine_states_extended.py index a183295c..41ca30b5 100644 --- a/alembic/versions/26319c44a8d5_state_machine_states_extended.py +++ b/alembic/versions/26319c44a8d5_state_machine_states_extended.py @@ -10,33 +10,124 @@ import sqlalchemy as sa from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. -revision = '26319c44a8d5' # pragma: allowlist secret -down_revision = '59973fa17ded' # pragma: allowlist secret +revision = "26319c44a8d5" # pragma: allowlist secret +down_revision = "59973fa17ded" # pragma: allowlist secret branch_labels = None depends_on = None def upgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.alter_column('portfolio_state_machines', 'state', - existing_type=sa.VARCHAR(length=30), - type_=sa.Enum('UNSTARTED', 'STARTING', 'STARTED', 'COMPLETED', 'FAILED', 'TENANT_CREATED', 'TENANT_IN_PROGRESS', 'TENANT_FAILED', 'BILLING_PROFILE_CREATION_CREATED', 'BILLING_PROFILE_CREATION_IN_PROGRESS', 'BILLING_PROFILE_CREATION_FAILED', 'BILLING_PROFILE_VERIFICATION_CREATED', 'BILLING_PROFILE_VERIFICATION_IN_PROGRESS', 'BILLING_PROFILE_VERIFICATION_FAILED', 'BILLING_PROFILE_TENANT_ACCESS_CREATED', 'BILLING_PROFILE_TENANT_ACCESS_IN_PROGRESS', 'BILLING_PROFILE_TENANT_ACCESS_FAILED', 'TASK_ORDER_BILLING_CREATION_CREATED', 'TASK_ORDER_BILLING_CREATION_IN_PROGRESS', 'TASK_ORDER_BILLING_CREATION_FAILED', 'TASK_ORDER_BILLING_VERIFICATION_CREATED', 'TASK_ORDER_BILLING_VERIFICATION_IN_PROGRESS', 'TASK_ORDER_BILLING_VERIFICATION_FAILED', 'BILLING_INSTRUCTION_CREATED', 'BILLING_INSTRUCTION_IN_PROGRESS', 'BILLING_INSTRUCTION_FAILED', name='fsmstates', native_enum=False, create_constraint=False), - existing_nullable=False) - op.alter_column('portfolios', 'defense_component', - existing_type=postgresql.ARRAY(sa.VARCHAR()), - type_=sa.String(), - existing_nullable=False) + op.alter_column( + "portfolio_state_machines", + "state", + existing_type=sa.Enum( + "UNSTARTED", + "STARTING", + "STARTED", + "COMPLETED", + "FAILED", + "TENANT_CREATED", + "TENANT_IN_PROGRESS", + "TENANT_FAILED", + "BILLING_PROFILE_CREATED", + "BILLING_PROFILE_IN_PROGRESS", + "BILLING_PROFILE_FAILED", + "ADMIN_SUBSCRIPTION_CREATED", + "ADMIN_SUBSCRIPTION_IN_PROGRESS", + "ADMIN_SUBSCRIPTION_FAILED", + name="fsmstates", + native_enum=False, + ), + type_=sa.Enum( + "UNSTARTED", + "STARTING", + "STARTED", + "COMPLETED", + "FAILED", + "TENANT_CREATED", + "TENANT_IN_PROGRESS", + "TENANT_FAILED", + "BILLING_PROFILE_CREATION_CREATED", + "BILLING_PROFILE_CREATION_IN_PROGRESS", + "BILLING_PROFILE_CREATION_FAILED", + "BILLING_PROFILE_VERIFICATION_CREATED", + "BILLING_PROFILE_VERIFICATION_IN_PROGRESS", + "BILLING_PROFILE_VERIFICATION_FAILED", + "BILLING_PROFILE_TENANT_ACCESS_CREATED", + "BILLING_PROFILE_TENANT_ACCESS_IN_PROGRESS", + "BILLING_PROFILE_TENANT_ACCESS_FAILED", + "TASK_ORDER_BILLING_CREATION_CREATED", + "TASK_ORDER_BILLING_CREATION_IN_PROGRESS", + "TASK_ORDER_BILLING_CREATION_FAILED", + "TASK_ORDER_BILLING_VERIFICATION_CREATED", + "TASK_ORDER_BILLING_VERIFICATION_IN_PROGRESS", + "TASK_ORDER_BILLING_VERIFICATION_FAILED", + "BILLING_INSTRUCTION_CREATED", + "BILLING_INSTRUCTION_IN_PROGRESS", + "BILLING_INSTRUCTION_FAILED", + name="fsmstates", + native_enum=False, + create_constraint=False, + ), + existing_nullable=False, + ) # ### end Alembic commands ### def downgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.alter_column('portfolios', 'defense_component', - existing_type=sa.String(), - type_=postgresql.ARRAY(sa.VARCHAR()), - existing_nullable=False) - op.alter_column('portfolio_state_machines', 'state', - existing_type=sa.Enum('UNSTARTED', 'STARTING', 'STARTED', 'COMPLETED', 'FAILED', 'TENANT_CREATED', 'TENANT_IN_PROGRESS', 'TENANT_FAILED', 'BILLING_PROFILE_CREATION_CREATED', 'BILLING_PROFILE_CREATION_IN_PROGRESS', 'BILLING_PROFILE_CREATION_FAILED', 'BILLING_PROFILE_VERIFICATION_CREATED', 'BILLING_PROFILE_VERIFICATION_IN_PROGRESS', 'BILLING_PROFILE_VERIFICATION_FAILED', 'BILLING_PROFILE_TENANT_ACCESS_CREATED', 'BILLING_PROFILE_TENANT_ACCESS_IN_PROGRESS', 'BILLING_PROFILE_TENANT_ACCESS_FAILED', 'TASK_ORDER_BILLING_CREATION_CREATED', 'TASK_ORDER_BILLING_CREATION_IN_PROGRESS', 'TASK_ORDER_BILLING_CREATION_FAILED', 'TASK_ORDER_BILLING_VERIFICATION_CREATED', 'TASK_ORDER_BILLING_VERIFICATION_IN_PROGRESS', 'TASK_ORDER_BILLING_VERIFICATION_FAILED', 'BILLING_INSTRUCTION_CREATED', 'BILLING_INSTRUCTION_IN_PROGRESS', 'BILLING_INSTRUCTION_FAILED', name='fsmstates', native_enum=False), - type_=sa.VARCHAR(length=30), - existing_nullable=False) + op.alter_column( + "portfolio_state_machines", + "state", + existing_type=sa.Enum( + "UNSTARTED", + "STARTING", + "STARTED", + "COMPLETED", + "FAILED", + "TENANT_CREATED", + "TENANT_IN_PROGRESS", + "TENANT_FAILED", + "BILLING_PROFILE_CREATION_CREATED", + "BILLING_PROFILE_CREATION_IN_PROGRESS", + "BILLING_PROFILE_CREATION_FAILED", + "BILLING_PROFILE_VERIFICATION_CREATED", + "BILLING_PROFILE_VERIFICATION_IN_PROGRESS", + "BILLING_PROFILE_VERIFICATION_FAILED", + "BILLING_PROFILE_TENANT_ACCESS_CREATED", + "BILLING_PROFILE_TENANT_ACCESS_IN_PROGRESS", + "BILLING_PROFILE_TENANT_ACCESS_FAILED", + "TASK_ORDER_BILLING_CREATION_CREATED", + "TASK_ORDER_BILLING_CREATION_IN_PROGRESS", + "TASK_ORDER_BILLING_CREATION_FAILED", + "TASK_ORDER_BILLING_VERIFICATION_CREATED", + "TASK_ORDER_BILLING_VERIFICATION_IN_PROGRESS", + "TASK_ORDER_BILLING_VERIFICATION_FAILED", + "BILLING_INSTRUCTION_CREATED", + "BILLING_INSTRUCTION_IN_PROGRESS", + "BILLING_INSTRUCTION_FAILED", + name="fsmstates", + native_enum=False, + ), + type_=sa.Enum( + "UNSTARTED", + "STARTING", + "STARTED", + "COMPLETED", + "FAILED", + "TENANT_CREATED", + "TENANT_IN_PROGRESS", + "TENANT_FAILED", + "BILLING_PROFILE_CREATED", + "BILLING_PROFILE_IN_PROGRESS", + "BILLING_PROFILE_FAILED", + "ADMIN_SUBSCRIPTION_CREATED", + "ADMIN_SUBSCRIPTION_IN_PROGRESS", + "ADMIN_SUBSCRIPTION_FAILED", + name="fsmstates", + native_enum=False, + ), + existing_nullable=False, + ) # ### end Alembic commands ### From c31d68a18c13a31e601779c66af366c39d37ce45 Mon Sep 17 00:00:00 2001 From: Rob Gil Date: Thu, 23 Jan 2020 10:50:16 -0500 Subject: [PATCH 24/46] Makes client vpn cidr range configurable --- terraform/modules/vpc/main.tf | 2 +- terraform/modules/vpc/variables.tf | 6 ++++++ terraform/providers/dev/variables.tf | 8 +++++++- terraform/providers/dev/vpc.tf | 1 + 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/terraform/modules/vpc/main.tf b/terraform/modules/vpc/main.tf index a27f2139..d0ea9a2a 100644 --- a/terraform/modules/vpc/main.tf +++ b/terraform/modules/vpc/main.tf @@ -110,7 +110,7 @@ resource "azurerm_virtual_network_gateway" "vnet_gateway" { } vpn_client_configuration { - address_space = ["172.16.1.0/24"] + address_space = var.vpn_client_cidr vpn_client_protocols = ["OpenVPN"] } } \ No newline at end of file diff --git a/terraform/modules/vpc/variables.tf b/terraform/modules/vpc/variables.tf index ac2dbac9..aae7ef45 100644 --- a/terraform/modules/vpc/variables.tf +++ b/terraform/modules/vpc/variables.tf @@ -52,3 +52,9 @@ variable "service_endpoints" { description = "A map of the service endpoints and its mapping to subnets" } + +variable "vpn_client_cidr" { + type = list + description = "The CIDR range used for clients on the VPN" + default = ["172.16.0.0/16"] +} diff --git a/terraform/providers/dev/variables.tf b/terraform/providers/dev/variables.tf index 573b6cc9..467f806c 100644 --- a/terraform/providers/dev/variables.tf +++ b/terraform/providers/dev/variables.tf @@ -91,6 +91,12 @@ variable "admin_users" { variable "admin_user_whitelist" { type = map default = { - "Rob Gil" = "66.220.238.246" + "Rob Gil" = "66.220.238.246" + "Dan Corrigan Work" = "108.16.207.173" } } + +variable "vpn_client_cidr" { + type = list + default = ["172.16.255.0/24"] +} diff --git a/terraform/providers/dev/vpc.tf b/terraform/providers/dev/vpc.tf index 44ecf35c..8d43a82f 100644 --- a/terraform/providers/dev/vpc.tf +++ b/terraform/providers/dev/vpc.tf @@ -10,5 +10,6 @@ module "vpc" { name = var.name dns_servers = var.dns_servers service_endpoints = var.service_endpoints + vpn_client_cidr = var.vpn_client_cidr } From dab6cdb7dca7dce086d5a33e9674481f47688a37 Mon Sep 17 00:00:00 2001 From: Rob Gil Date: Thu, 23 Jan 2020 11:02:12 -0500 Subject: [PATCH 25/46] Locks down keyvaults to subnets and administrator ip addresses --- terraform/modules/keyvault/main.tf | 7 +++++++ terraform/modules/keyvault/variables.tf | 17 +++++++++++++++++ terraform/providers/dev/keyvault.tf | 3 +++ terraform/providers/dev/secrets.tf | 3 +++ 4 files changed, 30 insertions(+) diff --git a/terraform/modules/keyvault/main.tf b/terraform/modules/keyvault/main.tf index ddfb8465..1df84367 100644 --- a/terraform/modules/keyvault/main.tf +++ b/terraform/modules/keyvault/main.tf @@ -13,6 +13,13 @@ resource "azurerm_key_vault" "keyvault" { sku_name = "premium" + network_acls { + default_action = var.policy + bypass = "AzureServices" + virtual_network_subnet_ids = var.subnet_ids + ip_rules = values(var.whitelist) + } + tags = { environment = var.environment owner = var.owner diff --git a/terraform/modules/keyvault/variables.tf b/terraform/modules/keyvault/variables.tf index d2484793..56e7cc13 100644 --- a/terraform/modules/keyvault/variables.tf +++ b/terraform/modules/keyvault/variables.tf @@ -32,3 +32,20 @@ variable "admin_principals" { type = map description = "A list of user principals who need access to manage the keyvault" } + +variable "subnet_ids" { + description = "List of subnet_ids that will have access to this service" + type = list +} + +variable "policy" { + description = "The default policy for the network access rules (Allow/Deny)" + default = "Deny" + type = string +} + +variable "whitelist" { + type = map + description = "A map of whitelisted IPs and CIDR ranges. For single IPs, Azure expects just the IP, NOT a /32." + default = {} +} \ No newline at end of file diff --git a/terraform/providers/dev/keyvault.tf b/terraform/providers/dev/keyvault.tf index 75f7b13d..4d35fa0f 100644 --- a/terraform/providers/dev/keyvault.tf +++ b/terraform/providers/dev/keyvault.tf @@ -7,5 +7,8 @@ module "keyvault" { tenant_id = var.tenant_id principal_id = "f9bcbe58-8b73-4957-aee2-133dc3e58063" admin_principals = var.admin_users + policy = "Deny" + subnet_ids = [module.vpc.subnets] + whitelist = var.admin_user_whitelist } diff --git a/terraform/providers/dev/secrets.tf b/terraform/providers/dev/secrets.tf index bccdcf50..7a67205e 100644 --- a/terraform/providers/dev/secrets.tf +++ b/terraform/providers/dev/secrets.tf @@ -7,4 +7,7 @@ module "operator_keyvault" { tenant_id = var.tenant_id principal_id = "" admin_principals = var.admin_users + policy = "Deny" + subnet_ids = [module.vpc.subnets] + whitelist = var.admin_user_whitelist } From 536eccdb909d4b0596e4e7ec356ba4466a0e9e10 Mon Sep 17 00:00:00 2001 From: Rob Gil Date: Thu, 23 Jan 2020 13:13:56 -0500 Subject: [PATCH 26/46] Container registry private networking and bucket cidr range fix --- terraform/modules/bucket/main.tf | 26 +++++++++--- terraform/modules/container_registry/main.tf | 33 +++++++++++++++ .../modules/container_registry/variables.tf | 17 ++++++++ terraform/providers/dev/container_registry.tf | 3 ++ terraform/providers/dev/k8s-test.tf-old | 41 +++++++++++++++++++ terraform/providers/dev/variables.tf | 4 +- 6 files changed, 117 insertions(+), 7 deletions(-) create mode 100644 terraform/providers/dev/k8s-test.tf-old diff --git a/terraform/modules/bucket/main.tf b/terraform/modules/bucket/main.tf index a92dc84f..2e43e304 100644 --- a/terraform/modules/bucket/main.tf +++ b/terraform/modules/bucket/main.tf @@ -1,3 +1,11 @@ +#locals { +# whitelist = [ +# for cidr in values(var.whitelist): { +# ip = cidrhost(cidr, 0) +# } +# ] +#} + resource "azurerm_resource_group" "bucket" { name = "${var.name}-${var.environment}-${var.service_name}" location = var.region @@ -9,12 +17,20 @@ resource "azurerm_storage_account" "bucket" { location = azurerm_resource_group.bucket.location account_tier = "Standard" account_replication_type = "LRS" +} - network_rules { - default_action = var.policy - virtual_network_subnet_ids = var.subnet_ids - ip_rules = values(var.whitelist) - } +resource "azurerm_storage_account_network_rules" "acls" { + resource_group_name = azurerm_resource_group.bucket.name + storage_account_name = azurerm_storage_account.bucket.name + + default_action = var.policy + # Azure Storage CIDR ACLs do not accept /32 CIDR ranges, so + # it must be stripped to just the IP (no CIDR) + ip_rules = [ + for cidr in values(var.whitelist) : cidrhost(cidr, 0) + ] + virtual_network_subnet_ids = var.subnet_ids + bypass = ["AzureServices"] } resource "azurerm_storage_container" "bucket" { diff --git a/terraform/modules/container_registry/main.tf b/terraform/modules/container_registry/main.tf index a22bacf0..6ee6022d 100644 --- a/terraform/modules/container_registry/main.tf +++ b/terraform/modules/container_registry/main.tf @@ -1,3 +1,7 @@ +locals { + whitelist = values(var.whitelist) +} + resource "azurerm_resource_group" "acr" { name = "${var.name}-${var.environment}-acr" location = var.region @@ -10,4 +14,33 @@ resource "azurerm_container_registry" "acr" { sku = var.sku admin_enabled = var.admin_enabled #georeplication_locations = [azurerm_resource_group.acr.location, var.backup_region] + network_rule_set { + default_action = var.policy + + ip_rule = [ + for cidr in values(var.whitelist) : { + action = "Allow" + ip_range = cidr + } + ] + # Dynamic rule should work, but doesn't - See https://github.com/hashicorp/terraform/issues/22340#issuecomment-518779733 + #dynamic "ip_rule" { + # for_each = values(var.whitelist) + # content { + # action = "Allow" + # ip_range = ip_rule.value + # } + #} + + virtual_network = [ + for subnet in var.subnet_ids : { + action = "Allow" + subnet_id = subnet.value + } + ] + #virtual_network { + # action = "Allow" + # subnet_id = var.subnet_ids + #} + } } \ No newline at end of file diff --git a/terraform/modules/container_registry/variables.tf b/terraform/modules/container_registry/variables.tf index 6fe16ad5..48fbb64a 100644 --- a/terraform/modules/container_registry/variables.tf +++ b/terraform/modules/container_registry/variables.tf @@ -35,3 +35,20 @@ variable "admin_enabled" { default = false } + +variable "subnet_ids" { + description = "List of subnet_ids that will have access to this service" + type = list +} + +variable "policy" { + description = "The default policy for the network access rules (Allow/Deny)" + default = "Deny" + type = string +} + +variable "whitelist" { + type = map + description = "A map of whitelisted IPs and CIDR ranges. For single IPs, Azure expects just the IP, NOT a /32." + default = {} +} diff --git a/terraform/providers/dev/container_registry.tf b/terraform/providers/dev/container_registry.tf index 0bbf0901..805ef3e8 100644 --- a/terraform/providers/dev/container_registry.tf +++ b/terraform/providers/dev/container_registry.tf @@ -5,4 +5,7 @@ module "container_registry" { environment = var.environment owner = var.owner backup_region = var.backup_region + policy = "Deny" + subnet_ids = [] + whitelist = var.admin_user_whitelist } diff --git a/terraform/providers/dev/k8s-test.tf-old b/terraform/providers/dev/k8s-test.tf-old new file mode 100644 index 00000000..c681d2db --- /dev/null +++ b/terraform/providers/dev/k8s-test.tf-old @@ -0,0 +1,41 @@ +resource "azurerm_resource_group" "k8s" { + name = "${var.name}-${var.environment}-k8s-test" + location = var.region +} + +resource "azurerm_kubernetes_cluster" "k8s" { + name = "${var.name}-${var.environment}-k8s-test" + location = azurerm_resource_group.k8s.location + resource_group_name = azurerm_resource_group.k8s.name + dns_prefix = var.k8s_dns_prefix + + service_principal { + client_id = "f05a4457-bd5e-4c63-98e1-89aab42645d0" + client_secret = "19b69e2c-9f55-4850-87cb-88c67a8dc811" + } + + default_node_pool { + name = "default" + vm_size = "Standard_D1_v2" + os_disk_size_gb = 30 + vnet_subnet_id = module.vpc.subnets + enable_node_public_ip = true # Nodes need a public IP for external resources. FIXME: Switch to NAT Gateway if its available in our subscription + enable_auto_scaling = true + max_count = 2 + min_count = 1 + } + + identity { + type = "SystemAssigned" + } + lifecycle { + ignore_changes = [ + default_node_pool.0.node_count + ] + } + + tags = { + environment = var.environment + owner = var.owner + } +} diff --git a/terraform/providers/dev/variables.tf b/terraform/providers/dev/variables.tf index 467f806c..99bd1b87 100644 --- a/terraform/providers/dev/variables.tf +++ b/terraform/providers/dev/variables.tf @@ -91,8 +91,8 @@ variable "admin_users" { variable "admin_user_whitelist" { type = map default = { - "Rob Gil" = "66.220.238.246" - "Dan Corrigan Work" = "108.16.207.173" + "Rob Gil" = "66.220.238.246/32" + "Dan Corrigan Work" = "108.16.207.173/32" } } From 38ce1ef2b269cb05d24ea8900f5c4e4361f49b06 Mon Sep 17 00:00:00 2001 From: Rob Gil Date: Thu, 23 Jan 2020 18:41:29 -0500 Subject: [PATCH 27/46] Adds list of users for access to storage and more service endpoints This sets up the rest of the service endpoints on the subnets. It also adds a variable map specifically to grant IP access to the storage buckets. This new variable map is necessary since the azure storage ip rules do not accept /32 CIDR ranges. The rest of the services do support cidr ranges. --- terraform/modules/bucket/main.tf | 6 +++--- terraform/providers/dev/buckets.tf | 4 ++-- terraform/providers/dev/variables.tf | 12 ++++++++++-- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/terraform/modules/bucket/main.tf b/terraform/modules/bucket/main.tf index 2e43e304..3b2463ed 100644 --- a/terraform/modules/bucket/main.tf +++ b/terraform/modules/bucket/main.tf @@ -24,10 +24,10 @@ resource "azurerm_storage_account_network_rules" "acls" { storage_account_name = azurerm_storage_account.bucket.name default_action = var.policy - # Azure Storage CIDR ACLs do not accept /32 CIDR ranges, so - # it must be stripped to just the IP (no CIDR) + + # Azure Storage CIDR ACLs do not accept /32 CIDR ranges. ip_rules = [ - for cidr in values(var.whitelist) : cidrhost(cidr, 0) + for cidr in values(var.whitelist) : cidr ] virtual_network_subnet_ids = var.subnet_ids bypass = ["AzureServices"] diff --git a/terraform/providers/dev/buckets.tf b/terraform/providers/dev/buckets.tf index ad2c7ab5..36510f3e 100644 --- a/terraform/providers/dev/buckets.tf +++ b/terraform/providers/dev/buckets.tf @@ -9,7 +9,7 @@ module "task_order_bucket" { region = var.region policy = "Allow" subnet_ids = [module.vpc.subnets] - whitelist = var.admin_user_whitelist + whitelist = var.storage_admin_whitelist } # TF State should be restricted to admins only, but IP protected @@ -25,5 +25,5 @@ module "tf_state" { region = var.region policy = "Deny" subnet_ids = [] - whitelist = var.admin_user_whitelist + whitelist = var.storage_admin_whitelist } diff --git a/terraform/providers/dev/variables.tf b/terraform/providers/dev/variables.tf index 99bd1b87..c4e0f338 100644 --- a/terraform/providers/dev/variables.tf +++ b/terraform/providers/dev/variables.tf @@ -39,8 +39,8 @@ variable "networks" { variable "service_endpoints" { type = map default = { - public = "" - private = "Microsoft.Storage,Microsoft.KeyVault" + public = "Microsoft.ContainerRegistry" # Not necessary but added to avoid infinite state loop + private = "Microsoft.Storage,Microsoft.KeyVault,Microsoft.ContainerRegistry,Microsoft.Sql" } } @@ -96,6 +96,14 @@ variable "admin_user_whitelist" { } } +variable "storage_admin_whitelist" { + type = map + default = { + "Rob Gil" = "66.220.238.246" + "Dan Corrigan Work" = "108.16.207.173" + } +} + variable "vpn_client_cidr" { type = list default = ["172.16.255.0/24"] From 0f5f5bd92604a9bdf0d3f8f18712753cceb93662 Mon Sep 17 00:00:00 2001 From: Rob Gil Date: Thu, 23 Jan 2020 19:16:00 -0500 Subject: [PATCH 28/46] Converts redis to use service_endpoints This is still a WIP. --- terraform/modules/postgres/variables.tf | 1 - terraform/modules/redis/main.tf | 1 + terraform/modules/redis/variables.tf | 10 +++++----- terraform/providers/dev/redis.tf | 3 +++ 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/terraform/modules/postgres/variables.tf b/terraform/modules/postgres/variables.tf index 2ee62685..f3366cdb 100644 --- a/terraform/modules/postgres/variables.tf +++ b/terraform/modules/postgres/variables.tf @@ -93,4 +93,3 @@ variable "ssl_enforcement" { description = "Enforce SSL (Enabled/Disable)" default = "Enabled" } - diff --git a/terraform/modules/redis/main.tf b/terraform/modules/redis/main.tf index 90a88a2b..b12bf92d 100644 --- a/terraform/modules/redis/main.tf +++ b/terraform/modules/redis/main.tf @@ -13,6 +13,7 @@ resource "azurerm_redis_cache" "redis" { sku_name = var.sku_name enable_non_ssl_port = var.enable_non_ssl_port minimum_tls_version = var.minimum_tls_version + subnet_id = var.subnet_id redis_configuration { enable_authentication = var.enable_authentication diff --git a/terraform/modules/redis/variables.tf b/terraform/modules/redis/variables.tf index dac8819b..06ddd36d 100644 --- a/terraform/modules/redis/variables.tf +++ b/terraform/modules/redis/variables.tf @@ -22,35 +22,30 @@ variable "capacity" { type = string default = 2 description = "The capacity of the redis cache" - } variable "family" { type = string default = "C" description = "The subscription family for redis" - } variable "sku_name" { type = string default = "Standard" description = "The sku to use" - } variable "enable_non_ssl_port" { type = bool default = false description = "Enable non TLS port (default: false)" - } variable "minimum_tls_version" { type = string default = "1.2" description = "Minimum TLS version to use" - } variable "enable_authentication" { @@ -58,3 +53,8 @@ variable "enable_authentication" { default = true description = "Enable or disable authentication (default: true)" } + +variable "subnet_id" { + type = string + description = "Subnet ID that the service_endpoint should reside" +} diff --git a/terraform/providers/dev/redis.tf b/terraform/providers/dev/redis.tf index bfe47a84..78cb4b2b 100644 --- a/terraform/providers/dev/redis.tf +++ b/terraform/providers/dev/redis.tf @@ -4,4 +4,7 @@ module "redis" { environment = var.environment region = var.region name = var.name + subnet_id = module.vpc.subnets + sku_name = "Premium" + family = "P" } From 9f0904c201954eb6c340215aa43f2579472e7337 Mon Sep 17 00:00:00 2001 From: Rob Gil Date: Thu, 23 Jan 2020 19:57:45 -0500 Subject: [PATCH 29/46] Adds dedicated redis subnet --- terraform/providers/dev/variables.tf | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/terraform/providers/dev/variables.tf b/terraform/providers/dev/variables.tf index c4e0f338..42890a31 100644 --- a/terraform/providers/dev/variables.tf +++ b/terraform/providers/dev/variables.tf @@ -32,7 +32,8 @@ variable "networks" { #format #name = "CIDR, route table, Security Group Name" public = "10.1.1.0/24,public" # LBs - private = "10.1.2.0/24,private" # k8s, postgres, redis, dns, ad + private = "10.1.2.0/24,private" # k8s, postgres, keyvault + redis = "10.1.3.0/24,private" # Redis } } @@ -41,6 +42,7 @@ variable "service_endpoints" { default = { public = "Microsoft.ContainerRegistry" # Not necessary but added to avoid infinite state loop private = "Microsoft.Storage,Microsoft.KeyVault,Microsoft.ContainerRegistry,Microsoft.Sql" + redis = "Microsoft.Storage,Microsoft.Sql" # FIXME: There is no Microsoft.Redis } } @@ -56,6 +58,7 @@ variable "route_tables" { default = { public = "Internet" private = "Internet" + redis = "VnetLocal" #private = "VnetLocal" } } From 3f5bbf2c5e5a970e73355e68e0d559648231e239 Mon Sep 17 00:00:00 2001 From: Rob Gil Date: Thu, 23 Jan 2020 19:58:06 -0500 Subject: [PATCH 30/46] Cleans out comments --- terraform/modules/container_registry/main.tf | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/terraform/modules/container_registry/main.tf b/terraform/modules/container_registry/main.tf index 6ee6022d..30b2b1cc 100644 --- a/terraform/modules/container_registry/main.tf +++ b/terraform/modules/container_registry/main.tf @@ -14,6 +14,7 @@ resource "azurerm_container_registry" "acr" { sku = var.sku admin_enabled = var.admin_enabled #georeplication_locations = [azurerm_resource_group.acr.location, var.backup_region] + network_rule_set { default_action = var.policy @@ -38,9 +39,5 @@ resource "azurerm_container_registry" "acr" { subnet_id = subnet.value } ] - #virtual_network { - # action = "Allow" - # subnet_id = var.subnet_ids - #} } } \ No newline at end of file From e0d59eb1662ffba4b417aa5374c6bf81dd89a70f Mon Sep 17 00:00:00 2001 From: Rob Gil Date: Thu, 23 Jan 2020 20:22:53 -0500 Subject: [PATCH 31/46] Finally fixes subnet list output This finally fixes the output coming from the vpc module so that it returns a full list of subnets. Now they can be referenced just like the redis module is using in this commit. --- terraform/modules/vpc/outputs.tf | 8 +++++++- terraform/providers/dev/redis.tf | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/terraform/modules/vpc/outputs.tf b/terraform/modules/vpc/outputs.tf index eedaab6c..baa32935 100644 --- a/terraform/modules/vpc/outputs.tf +++ b/terraform/modules/vpc/outputs.tf @@ -1,3 +1,9 @@ output "subnets" { - value = azurerm_subnet.subnet["private"].id #FIXME - output should be a map + value = azurerm_subnet.subnet["private"].id #FIXED: this is now legacy, use subnet_list } + +output "subnet_list" { + value = { + for k, id in azurerm_subnet.subnet : k => id + } +} \ No newline at end of file diff --git a/terraform/providers/dev/redis.tf b/terraform/providers/dev/redis.tf index 78cb4b2b..8c89dc92 100644 --- a/terraform/providers/dev/redis.tf +++ b/terraform/providers/dev/redis.tf @@ -4,7 +4,7 @@ module "redis" { environment = var.environment region = var.region name = var.name - subnet_id = module.vpc.subnets + subnet_id = module.vpc.subnet_list["redis"].id sku_name = "Premium" family = "P" } From daa07f8631bd62338b0e3fc999145eebb821a112 Mon Sep 17 00:00:00 2001 From: Rob Gil Date: Thu, 23 Jan 2020 20:26:27 -0500 Subject: [PATCH 32/46] Removes unnecessary locals in the bucket module --- terraform/modules/bucket/main.tf | 8 -------- 1 file changed, 8 deletions(-) diff --git a/terraform/modules/bucket/main.tf b/terraform/modules/bucket/main.tf index 3b2463ed..e2f91f58 100644 --- a/terraform/modules/bucket/main.tf +++ b/terraform/modules/bucket/main.tf @@ -1,11 +1,3 @@ -#locals { -# whitelist = [ -# for cidr in values(var.whitelist): { -# ip = cidrhost(cidr, 0) -# } -# ] -#} - resource "azurerm_resource_group" "bucket" { name = "${var.name}-${var.environment}-${var.service_name}" location = var.region From 7b2523254d59ecb9d2f8a10e5cfd72ca378e2708 Mon Sep 17 00:00:00 2001 From: Rob Gil Date: Fri, 24 Jan 2020 07:36:02 -0500 Subject: [PATCH 33/46] Adds Dans home ip --- terraform/providers/dev/variables.tf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/terraform/providers/dev/variables.tf b/terraform/providers/dev/variables.tf index 42890a31..b13c0d57 100644 --- a/terraform/providers/dev/variables.tf +++ b/terraform/providers/dev/variables.tf @@ -96,6 +96,7 @@ variable "admin_user_whitelist" { default = { "Rob Gil" = "66.220.238.246/32" "Dan Corrigan Work" = "108.16.207.173/32" + "Dan Corrigan Home" = "71.162.221.27/32" } } @@ -104,6 +105,7 @@ variable "storage_admin_whitelist" { default = { "Rob Gil" = "66.220.238.246" "Dan Corrigan Work" = "108.16.207.173" + "Dan Corrigan Home" = "71.162.221.27" } } From 76465e978acc55fea319392aa1d9c92c7e5a3707 Mon Sep 17 00:00:00 2001 From: Rob Gil Date: Fri, 24 Jan 2020 07:36:24 -0500 Subject: [PATCH 34/46] Remove k8s test tf --- terraform/providers/dev/k8s-test.tf-old | 41 ------------------------- 1 file changed, 41 deletions(-) delete mode 100644 terraform/providers/dev/k8s-test.tf-old diff --git a/terraform/providers/dev/k8s-test.tf-old b/terraform/providers/dev/k8s-test.tf-old deleted file mode 100644 index c681d2db..00000000 --- a/terraform/providers/dev/k8s-test.tf-old +++ /dev/null @@ -1,41 +0,0 @@ -resource "azurerm_resource_group" "k8s" { - name = "${var.name}-${var.environment}-k8s-test" - location = var.region -} - -resource "azurerm_kubernetes_cluster" "k8s" { - name = "${var.name}-${var.environment}-k8s-test" - location = azurerm_resource_group.k8s.location - resource_group_name = azurerm_resource_group.k8s.name - dns_prefix = var.k8s_dns_prefix - - service_principal { - client_id = "f05a4457-bd5e-4c63-98e1-89aab42645d0" - client_secret = "19b69e2c-9f55-4850-87cb-88c67a8dc811" - } - - default_node_pool { - name = "default" - vm_size = "Standard_D1_v2" - os_disk_size_gb = 30 - vnet_subnet_id = module.vpc.subnets - enable_node_public_ip = true # Nodes need a public IP for external resources. FIXME: Switch to NAT Gateway if its available in our subscription - enable_auto_scaling = true - max_count = 2 - min_count = 1 - } - - identity { - type = "SystemAssigned" - } - lifecycle { - ignore_changes = [ - default_node_pool.0.node_count - ] - } - - tags = { - environment = var.environment - owner = var.owner - } -} From ea040a914ea56fa3dca28d139c2d5f113fb7153f Mon Sep 17 00:00:00 2001 From: tomdds Date: Fri, 24 Jan 2020 11:01:53 -0500 Subject: [PATCH 35/46] Properly report initial clin information Includes fixed up state machine test as well as adds some missing dependencies --- Pipfile | 1 + Pipfile.lock | 82 +++++++++++---- atst/domain/csp/cloud.py | 101 +++++++++++++++---- atst/models/portfolio_state_machine.py | 21 +++- tests/domain/cloud/test_azure_csp.py | 10 +- tests/domain/test_portfolio_state_machine.py | 67 +++++++----- tests/mock_azure.py | 7 ++ 7 files changed, 208 insertions(+), 81 deletions(-) diff --git a/Pipfile b/Pipfile index 06bd2675..5d3a7920 100644 --- a/Pipfile +++ b/Pipfile @@ -36,6 +36,7 @@ transitions = "*" azure-mgmt-consumption = "*" adal = "*" azure-identity = "*" +azure-keyvault = "*" [dev-packages] bandit = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 8a030e11..53a28b17 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "3760f0b1df1156211d671afa2eb417b7bf980aa33d2f74d390e8eed6a3ce8c8b" + "sha256": "4dbb023bcb860eb6dc56e1c201c91f272e1e67ad03e5e5eeb3a7a7fdff350eed" }, "pipfile-spec": 6, "requires": { @@ -26,10 +26,10 @@ }, "alembic": { "hashes": [ - "sha256:3b0cb1948833e062f4048992fbc97ecfaaaac24aaa0d83a1202a99fb58af8c6d" + "sha256:d412982920653db6e5a44bfd13b1d0db5685cbaaccaf226195749c706e1e862a" ], "index": "pypi", - "version": "==1.3.2" + "version": "==1.3.3" }, "amqp": { "hashes": [ @@ -68,6 +68,28 @@ "index": "pypi", "version": "==1.2.0" }, + "azure-keyvault": { + "hashes": [ + "sha256:76f75cb83929f312a08616d426ad6f597f1beae180131cf445876fb88f2c8ef1", + "sha256:e85f5bd6cb4f10b3248b99bbf02e3acc6371d366846897027d4153f18025a2d7" + ], + "index": "pypi", + "version": "==4.0.0" + }, + "azure-keyvault-keys": { + "hashes": [ + "sha256:2983fa42e20a0e6bf6b87976716129c108e613e0292d34c5b0f0c8dc1d488e89", + "sha256:38c27322637a2c52620a8b96da1942ad6a8d22d09b5a01f6fa257f7a51e52ed0" + ], + "version": "==4.0.0" + }, + "azure-keyvault-secrets": { + "hashes": [ + "sha256:2eae9264a8f6f59277e1a9bfdbc8b0a15969ee5a80d8efe403d7744805b4a481", + "sha256:97a602406a833e8f117c540c66059c818f4321a35168dd17365fab1e4527d718" + ], + "version": "==4.0.0" + }, "azure-mgmt-authorization": { "hashes": [ "sha256:31e875a34ac2c5d6fefe77b4a8079a8b2bdbe9edb957e47e8b44222fb212d6a7", @@ -232,6 +254,14 @@ ], "version": "==2.8" }, + "dataclasses": { + "hashes": [ + "sha256:3459118f7ede7c8bea0fe795bff7c6c2ce287d01dd226202f7c9ebc0610a7836", + "sha256:494a6dcae3b8bcf80848eea2ef64c0cc5cd307ffc263e17cdf42f3e5420808e6" + ], + "markers": "python_version < '3.7'", + "version": "==0.7" + }, "flask": { "hashes": [ "sha256:13f9f196f330c7c2c5d7a5cf91af894110ca0215ac051b5844701f2bfd934d52", @@ -325,9 +355,9 @@ }, "mako": { "hashes": [ - "sha256:a36919599a9b7dc5d86a7a8988f23a9a3a3d083070023bab23d64f7f1d1e0a4b" + "sha256:2984a6733e1d472796ceef37ad48c26f4a984bb18119bb2dbc37a44d8f6e75a4" ], - "version": "==1.1.0" + "version": "==1.1.1" }, "markupsafe": { "hashes": [ @@ -584,10 +614,10 @@ }, "sqlalchemy": { "hashes": [ - "sha256:bfb8f464a5000b567ac1d350b9090cf081180ec1ab4aa87e7bca12dab25320ec" + "sha256:64a7b71846db6423807e96820993fa12a03b89127d278290ca25c0b11ed7b4fb" ], "index": "pypi", - "version": "==1.3.12" + "version": "==1.3.13" }, "sqlalchemy-json": { "hashes": [ @@ -620,10 +650,10 @@ }, "urllib3": { "hashes": [ - "sha256:a8a318824cc77d1fd4b2bec2ded92646630d7fe8619497b142c84a9e6f5a7293", - "sha256:f3c5fd51747d450d4dcf6f923c81f78f811aab8205fda64b0aba34a4e48b0745" + "sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc", + "sha256:87716c2d2a7121198ebcb7ce7cccf6ce5e9ba539041cfbaeecfb641dc0bf6acc" ], - "version": "==1.25.7" + "version": "==1.25.8" }, "vine": { "hashes": [ @@ -657,10 +687,10 @@ }, "zipp": { "hashes": [ - "sha256:57147f6b0403b59f33fd357f169f860e031303415aeb7d04ede4839d23905ab8", - "sha256:7ae5ccaca427bafa9760ac3cd8f8c244bfc259794b5b6bb9db4dda2241575d09" + "sha256:b338014b9bc7102ca69e0fb96ed07215a8954d2989bc5d83658494ab2ba634af", + "sha256:e013e7800f60ec4dde789ebf4e9f7a54236e4bbf5df2a1a4e20ce9e1d9609d67" ], - "version": "==2.0.0" + "version": "==2.0.1" } }, "develop": { @@ -671,6 +701,14 @@ ], "version": "==1.4.3" }, + "appnope": { + "hashes": [ + "sha256:5b26757dc6f79a3b7dc9fab95359328d5747fcb2409d331ea66d0272b90ab2a0", + "sha256:8b995ffe925347a2138d7ac0fe77155e4311a0ea6d6da4f5128fe4b3cbe5ed71" + ], + "markers": "sys_platform == 'darwin'", + "version": "==0.1.0" + }, "argh": { "hashes": [ "sha256:a9b3aaa1904eeb78e32394cd46c6f37ac0fb4af6dc488daa58971bdc7d7fcaf3", @@ -1062,11 +1100,11 @@ }, "pexpect": { "hashes": [ - "sha256:2094eefdfcf37a1fdbfb9aa090862c1a4878e5c7e0e7e7088bdb511c558e5cd1", - "sha256:9e2c1fd0e6ee3a49b28f95d4b33bc389c89b20af6a1255906e90ff1262ce62eb" + "sha256:0b48a55dcb3c05f3329815901ea4fc1537514d6ba867a152b581d69ae3710937", + "sha256:fc65a43959d153d0114afe13997d439c22823a27cefceb5ff35c2178c6784c0c" ], "markers": "sys_platform != 'win32'", - "version": "==4.7.0" + "version": "==4.8.0" }, "pickleshare": { "hashes": [ @@ -1325,10 +1363,10 @@ }, "urllib3": { "hashes": [ - "sha256:a8a318824cc77d1fd4b2bec2ded92646630d7fe8619497b142c84a9e6f5a7293", - "sha256:f3c5fd51747d450d4dcf6f923c81f78f811aab8205fda64b0aba34a4e48b0745" + "sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc", + "sha256:87716c2d2a7121198ebcb7ce7cccf6ce5e9ba539041cfbaeecfb641dc0bf6acc" ], - "version": "==1.25.7" + "version": "==1.25.8" }, "watchdog": { "hashes": [ @@ -1359,10 +1397,10 @@ }, "zipp": { "hashes": [ - "sha256:57147f6b0403b59f33fd357f169f860e031303415aeb7d04ede4839d23905ab8", - "sha256:7ae5ccaca427bafa9760ac3cd8f8c244bfc259794b5b6bb9db4dda2241575d09" + "sha256:b338014b9bc7102ca69e0fb96ed07215a8954d2989bc5d83658494ab2ba634af", + "sha256:e013e7800f60ec4dde789ebf4e9f7a54236e4bbf5df2a1a4e20ce9e1d9609d67" ], - "version": "==2.0.0" + "version": "==2.0.1" } } } diff --git a/atst/domain/csp/cloud.py b/atst/domain/csp/cloud.py index 5d9a8397..44bb8517 100644 --- a/atst/domain/csp/cloud.py +++ b/atst/domain/csp/cloud.py @@ -327,12 +327,12 @@ class TaskOrderBillingCreationCSPPayload(BaseCSPPayload): class TaskOrderBillingCreationCSPResult(AliasModel): task_order_billing_verify_url: str - retry_after: int + task_order_retry_after: int class Config: fields = { "task_order_billing_verify_url": "Location", - "retry_after": "Retry-After", + "task_order_retry_after": "Retry-After", } @@ -358,11 +358,11 @@ class TaskOrderBillingVerificationCSPResult(AliasModel): class BillingInstructionCSPPayload(BaseCSPPayload): - amount: float - start_date: str - end_date: str - clin_type: str - task_order_id: str + initial_clin_amount: float + initial_clin_start_date: str + initial_clin_end_date: str + initial_clin_type: str + initial_task_order_id: str billing_account_name: str billing_profile_name: str @@ -646,19 +646,76 @@ class MockCloudProvider(CloudProviderInterface): self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) - return { - "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB/billingRoleAssignments/40000000-aaaa-bbbb-cccc-100000000000_0a5f4926-e3ee-4f47-a6e3-8b0a30a40e3d", - "name": "40000000-aaaa-bbbb-cccc-100000000000_0a5f4926-e3ee-4f47-a6e3-8b0a30a40e3d", + return BillingProfileTenantAccessCSPResult( + **{ + "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB/billingRoleAssignments/40000000-aaaa-bbbb-cccc-100000000000_0a5f4926-e3ee-4f47-a6e3-8b0a30a40e3d", + "name": "40000000-aaaa-bbbb-cccc-100000000000_0a5f4926-e3ee-4f47-a6e3-8b0a30a40e3d", + "properties": { + "createdOn": "2020-01-14T14:39:26.3342192+00:00", + "createdByPrincipalId": "82e2b376-3297-4096-8743-ed65b3be0b03", + "principalId": "0a5f4926-e3ee-4f47-a6e3-8b0a30a40e3d", + "principalTenantId": "60ff9d34-82bf-4f21-b565-308ef0533435", + "roleDefinitionId": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB/billingRoleDefinitions/40000000-aaaa-bbbb-cccc-100000000000", + "scope": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB", + }, + "type": "Microsoft.Billing/billingRoleAssignments", + } + ).dict() + + def create_task_order_billing_creation(self, payload: TaskOrderBillingCreationCSPPayload): + self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) + self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) + self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) + + return TaskOrderBillingCreationCSPResult(**{"Location": "https://somelocation", "Retry-After": "10"}).dict() + + def create_task_order_billing_verification(self, payload: TaskOrderBillingVerificationCSPPayload): + self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) + self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) + self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) + + return TaskOrderBillingVerificationCSPResult(**{ + "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/XC36-GRNZ-BG7-TGB", + "name": "XC36-GRNZ-BG7-TGB", "properties": { - "createdOn": "2020-01-14T14:39:26.3342192+00:00", - "createdByPrincipalId": "82e2b376-3297-4096-8743-ed65b3be0b03", - "principalId": "0a5f4926-e3ee-4f47-a6e3-8b0a30a40e3d", - "principalTenantId": "60ff9d34-82bf-4f21-b565-308ef0533435", - "roleDefinitionId": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB/billingRoleDefinitions/40000000-aaaa-bbbb-cccc-100000000000", - "scope": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB", + "address": { + "addressLine1": "123 S Broad Street, Suite 2400", + "city": "Philadelphia", + "companyName": "Promptworks", + "country": "US", + "postalCode": "19109", + "region": "PA" + }, + "currency": "USD", + "displayName": "First Portfolio Billing Profile", + "enabledAzurePlans": [ + { + "productId": "DZH318Z0BPS6", + "skuId": "0001", + "skuDescription": "Microsoft Azure Plan" + } + ], + "hasReadAccess": True, + "invoiceDay": 5, + "invoiceEmailOptIn": False }, - "type": "Microsoft.Billing/billingRoleAssignments", - } + "type": "Microsoft.Billing/billingAccounts/billingProfiles" + }).dict() + + def create_billing_instruction(self, payload: BillingInstructionCSPPayload): + self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) + self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) + self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) + + return BillingInstructionCSPResult(**{ + "name": "TO1:CLIN001", + "properties": { + "amount": 1000.0, + "endDate": "2020-03-01T00:00:00+00:00", + "startDate": "2020-01-01T00:00:00+00:00" + }, + "type": "Microsoft.Billing/billingAccounts/billingProfiles/billingInstructions" + }).dict() def create_or_update_user(self, auth_credentials, user_info, csp_role_id): self._authorize(auth_credentials) @@ -1150,13 +1207,13 @@ class AzureCloudProvider(CloudProviderInterface): request_body = { "properties": { - "amount": payload.amount, - "startDate": payload.start_date, - "endDate": payload.end_date, + "amount": payload.initial_clin_amount, + "startDate": payload.initial_clin_start_date, + "endDate": payload.initial_clin_end_date, } } - url = f"https://management.azure.com/providers/Microsoft.Billing/billingAccounts/{payload.billing_account_name}/billingProfiles/{payload.billing_profile_name}/instructions/{payload.task_order_id}:CLIN00{payload.clin_type}?api-version=2019-10-01-preview" + url = f"https://management.azure.com/providers/Microsoft.Billing/billingAccounts/{payload.billing_account_name}/billingProfiles/{payload.billing_profile_name}/instructions/{payload.initial_task_order_id}:CLIN00{payload.initial_clin_type}?api-version=2019-10-01-preview" auth_header = { "Authorization": f"Bearer {sp_token}", diff --git a/atst/models/portfolio_state_machine.py b/atst/models/portfolio_state_machine.py index 04d09bd6..3f0e4d7d 100644 --- a/atst/models/portfolio_state_machine.py +++ b/atst/models/portfolio_state_machine.py @@ -67,7 +67,6 @@ class PortfolioStateMachine( def __repr__(self): return f" Date: Fri, 24 Jan 2020 11:03:38 -0500 Subject: [PATCH 36/46] Formatting fixes --- atst/domain/csp/cloud.py | 97 +++++++++++--------- atst/models/mixins/state_machines.py | 8 +- tests/domain/test_portfolio_state_machine.py | 1 - 3 files changed, 62 insertions(+), 44 deletions(-) diff --git a/atst/domain/csp/cloud.py b/atst/domain/csp/cloud.py index 44bb8517..2257ace1 100644 --- a/atst/domain/csp/cloud.py +++ b/atst/domain/csp/cloud.py @@ -662,60 +662,70 @@ class MockCloudProvider(CloudProviderInterface): } ).dict() - def create_task_order_billing_creation(self, payload: TaskOrderBillingCreationCSPPayload): + def create_task_order_billing_creation( + self, payload: TaskOrderBillingCreationCSPPayload + ): self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) - return TaskOrderBillingCreationCSPResult(**{"Location": "https://somelocation", "Retry-After": "10"}).dict() + return TaskOrderBillingCreationCSPResult( + **{"Location": "https://somelocation", "Retry-After": "10"} + ).dict() - def create_task_order_billing_verification(self, payload: TaskOrderBillingVerificationCSPPayload): + def create_task_order_billing_verification( + self, payload: TaskOrderBillingVerificationCSPPayload + ): self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) - return TaskOrderBillingVerificationCSPResult(**{ - "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/XC36-GRNZ-BG7-TGB", - "name": "XC36-GRNZ-BG7-TGB", - "properties": { - "address": { - "addressLine1": "123 S Broad Street, Suite 2400", - "city": "Philadelphia", - "companyName": "Promptworks", - "country": "US", - "postalCode": "19109", - "region": "PA" + return TaskOrderBillingVerificationCSPResult( + **{ + "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/XC36-GRNZ-BG7-TGB", + "name": "XC36-GRNZ-BG7-TGB", + "properties": { + "address": { + "addressLine1": "123 S Broad Street, Suite 2400", + "city": "Philadelphia", + "companyName": "Promptworks", + "country": "US", + "postalCode": "19109", + "region": "PA", + }, + "currency": "USD", + "displayName": "First Portfolio Billing Profile", + "enabledAzurePlans": [ + { + "productId": "DZH318Z0BPS6", + "skuId": "0001", + "skuDescription": "Microsoft Azure Plan", + } + ], + "hasReadAccess": True, + "invoiceDay": 5, + "invoiceEmailOptIn": False, }, - "currency": "USD", - "displayName": "First Portfolio Billing Profile", - "enabledAzurePlans": [ - { - "productId": "DZH318Z0BPS6", - "skuId": "0001", - "skuDescription": "Microsoft Azure Plan" - } - ], - "hasReadAccess": True, - "invoiceDay": 5, - "invoiceEmailOptIn": False - }, - "type": "Microsoft.Billing/billingAccounts/billingProfiles" - }).dict() + "type": "Microsoft.Billing/billingAccounts/billingProfiles", + } + ).dict() def create_billing_instruction(self, payload: BillingInstructionCSPPayload): self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) - return BillingInstructionCSPResult(**{ - "name": "TO1:CLIN001", - "properties": { - "amount": 1000.0, - "endDate": "2020-03-01T00:00:00+00:00", - "startDate": "2020-01-01T00:00:00+00:00" - }, - "type": "Microsoft.Billing/billingAccounts/billingProfiles/billingInstructions" - }).dict() + return BillingInstructionCSPResult( + **{ + "name": "TO1:CLIN001", + "properties": { + "amount": 1000.0, + "endDate": "2020-03-01T00:00:00+00:00", + "startDate": "2020-01-01T00:00:00+00:00", + }, + "type": "Microsoft.Billing/billingAccounts/billingProfiles/billingInstructions", + } + ).dict() def create_or_update_user(self, auth_credentials, user_info, csp_role_id): self._authorize(auth_credentials) @@ -844,8 +854,10 @@ class AzureCloudProvider(CloudProviderInterface): try: return secret_client.set_secret(secret_key, secret_value) except self.exceptions.HttpResponseError as exc: - app.logger.error(f"Could not SET secret in Azure keyvault for key {secret_key}.", exc_info=1) - + app.logger.error( + f"Could not SET secret in Azure keyvault for key {secret_key}.", + exc_info=1, + ) def get_secret(secret_key): credential = self._get_client_secret_credential_obj() @@ -855,7 +867,10 @@ class AzureCloudProvider(CloudProviderInterface): try: return secret_client.get_secret(secret_key).value except self.exceptions.HttpResponseError as exc: - app.logger.error(f"Could not GET secret in Azure keyvault for key {secret_key}.", exc_info=1) + app.logger.error( + f"Could not GET secret in Azure keyvault for key {secret_key}.", + exc_info=1, + ) def create_environment( self, auth_credentials: Dict, user: User, environment: Environment diff --git a/atst/models/mixins/state_machines.py b/atst/models/mixins/state_machines.py index 29f7a877..37682e5b 100644 --- a/atst/models/mixins/state_machines.py +++ b/atst/models/mixins/state_machines.py @@ -109,12 +109,16 @@ class FSMMixin: fail_trigger = "fail" + stage if fail_trigger in self.machine.get_triggers(self.current_state.name): self.trigger(fail_trigger) - app.logger.info(f"calling fail trigger '{fail_trigger}' for '{self.__repr__()}'") + app.logger.info( + f"calling fail trigger '{fail_trigger}' for '{self.__repr__()}'" + ) def finish_stage(self, stage): finish_trigger = "finish_" + stage if finish_trigger in self.machine.get_triggers(self.current_state.name): - app.logger.info(f"calling finish trigger '{finish_trigger}' for '{self.__repr__()}'") + app.logger.info( + f"calling finish trigger '{finish_trigger}' for '{self.__repr__()}'" + ) self.trigger(finish_trigger) def prepare_init(self, event): diff --git a/tests/domain/test_portfolio_state_machine.py b/tests/domain/test_portfolio_state_machine.py index 8ec9615d..82a6d086 100644 --- a/tests/domain/test_portfolio_state_machine.py +++ b/tests/domain/test_portfolio_state_machine.py @@ -152,4 +152,3 @@ def test_fsm_transition_start(portfolio: Portfolio): csp_data = portfolio.csp_data else: csp_data = {} - From e9d03ec68bc6cd19b05b0854926ea0b9020da7b0 Mon Sep 17 00:00:00 2001 From: tomdds Date: Fri, 24 Jan 2020 11:12:06 -0500 Subject: [PATCH 37/46] Fix some LGTM errors and start sketching in credential update functionality --- atst/domain/csp/cloud.py | 12 +++++++++--- atst/models/portfolio_state_machine.py | 11 ++++------- tests/domain/test_portfolio_state_machine.py | 5 ++--- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/atst/domain/csp/cloud.py b/atst/domain/csp/cloud.py index 2257ace1..b46b03b4 100644 --- a/atst/domain/csp/cloud.py +++ b/atst/domain/csp/cloud.py @@ -380,7 +380,7 @@ class CloudProviderInterface: def set_secret(self, secret_key: str, secret_value: str): raise NotImplementedError() - def get_secret(self, secret_key: str, secret_value: str): + def get_secret(self, secret_key: str): raise NotImplementedError() def root_creds(self) -> Dict: @@ -520,6 +520,12 @@ class MockCloudProvider(CloudProviderInterface): def root_creds(self): return self._auth_credentials + def set_secret(self, secret_key: str, secret_value: str): + pass + + def get_secret(self, secret_key: str): + return {} + def create_environment(self, auth_credentials, user, environment): self._authorize(auth_credentials) @@ -846,7 +852,7 @@ class AzureCloudProvider(CloudProviderInterface): self.policy_manager = AzurePolicyManager(config["AZURE_POLICY_LOCATION"]) - def set_secret(secret_key, secret_value): + def set_secret(self, secret_key, secret_value): credential = self._get_client_secret_credential_obj() secret_client = self.secrets.SecretClient( vault_url=self.vault_url, credential=credential, @@ -859,7 +865,7 @@ class AzureCloudProvider(CloudProviderInterface): exc_info=1, ) - def get_secret(secret_key): + def get_secret(self, secret_key): credential = self._get_client_secret_credential_obj() secret_client = self.secrets.SecretClient( vault_url=self.vault_url, credential=credential, diff --git a/atst/models/portfolio_state_machine.py b/atst/models/portfolio_state_machine.py index 3f0e4d7d..6e3015c4 100644 --- a/atst/models/portfolio_state_machine.py +++ b/atst/models/portfolio_state_machine.py @@ -1,5 +1,4 @@ from random import choice, choices -import re import string from sqlalchemy import Column, ForeignKey, Enum as SQLAEnum @@ -119,7 +118,6 @@ class PortfolioStateMachine( elif state_obj.is_CREATED: # the create trigger for the next stage should be in the available # triggers for the current state - triggers = self.machine.get_triggers(state_obj.name) create_trigger = next( filter( lambda trigger: trigger.startswith("create_"), @@ -205,11 +203,10 @@ class PortfolioStateMachine( dc = cls(**stage_data) if getattr(dc, "get_creds", None) is not None: new_creds = dc.get_creds() - print("creds to report") - print(new_creds) - # TODO: how/where to store these - # TODO: credential schema - # self.store_creds(self.portfolio, new_creds) + tenant_id = new_creds.get("tenant_id") + secret = self.csp.get_secret(tenant_id) + secret.update(new_creds) + self.csp.set_secret(tenant_id, secret) except PydanticValidationError as exc: app.logger.error( diff --git a/tests/domain/test_portfolio_state_machine.py b/tests/domain/test_portfolio_state_machine.py index 82a6d086..c5ca68cb 100644 --- a/tests/domain/test_portfolio_state_machine.py +++ b/tests/domain/test_portfolio_state_machine.py @@ -2,7 +2,6 @@ import pytest import re from tests.factories import ( - PortfolioFactory, PortfolioStateMachineFactory, TaskOrderFactory, CLINFactory, @@ -36,7 +35,7 @@ def test_state_machine_trigger_next_transition(portfolio): def test_state_machine_compose_state(portfolio): - sm = PortfolioStateMachineFactory.create(portfolio=portfolio) + PortfolioStateMachineFactory.create(portfolio=portfolio) assert ( compose_state(AzureStages.TENANT, StageStates.CREATED) == FSMStates.TENANT_CREATED @@ -44,7 +43,7 @@ def test_state_machine_compose_state(portfolio): def test_state_machine_valid_data_classes_for_stages(portfolio): - sm = PortfolioStateMachineFactory.create(portfolio=portfolio) + PortfolioStateMachineFactory.create(portfolio=portfolio) for stage in AzureStages: assert get_stage_csp_class(stage.name.lower(), "payload") is not None assert get_stage_csp_class(stage.name.lower(), "result") is not None From b9206ed7be509c3cf868022fc8f064e761cf8d08 Mon Sep 17 00:00:00 2001 From: tomdds Date: Fri, 24 Jan 2020 13:35:46 -0500 Subject: [PATCH 38/46] Some more LGTM fixes --- alembic/versions/26319c44a8d5_state_machine_states_extended.py | 1 - tests/domain/cloud/test_azure_csp.py | 1 - tests/domain/test_portfolio_state_machine.py | 2 +- 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/alembic/versions/26319c44a8d5_state_machine_states_extended.py b/alembic/versions/26319c44a8d5_state_machine_states_extended.py index 41ca30b5..720e50ee 100644 --- a/alembic/versions/26319c44a8d5_state_machine_states_extended.py +++ b/alembic/versions/26319c44a8d5_state_machine_states_extended.py @@ -7,7 +7,6 @@ Create Date: 2020-01-22 15:54:03.186751 """ from alembic import op import sqlalchemy as sa -from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "26319c44a8d5" # pragma: allowlist secret diff --git a/tests/domain/cloud/test_azure_csp.py b/tests/domain/cloud/test_azure_csp.py index f247a961..9e87e7a7 100644 --- a/tests/domain/cloud/test_azure_csp.py +++ b/tests/domain/cloud/test_azure_csp.py @@ -1,4 +1,3 @@ -import pytest from unittest.mock import Mock from uuid import uuid4 diff --git a/tests/domain/test_portfolio_state_machine.py b/tests/domain/test_portfolio_state_machine.py index c5ca68cb..0d37c9c7 100644 --- a/tests/domain/test_portfolio_state_machine.py +++ b/tests/domain/test_portfolio_state_machine.py @@ -3,7 +3,6 @@ import re from tests.factories import ( PortfolioStateMachineFactory, - TaskOrderFactory, CLINFactory, ) @@ -15,6 +14,7 @@ from atst.domain.csp import get_stage_csp_class @pytest.fixture(scope="function") def portfolio(): + # TODO: setup clin/to as active/funded/ready portfolio = CLINFactory.create().task_order.portfolio return portfolio From 7e0fda67b06621a95a4a42f4ecf9e2ee9b9c13d4 Mon Sep 17 00:00:00 2001 From: tomdds Date: Fri, 24 Jan 2020 13:52:41 -0500 Subject: [PATCH 39/46] Clean up unusued imports and variables --- atst/domain/csp/cloud.py | 10 +++++----- atst/models/portfolio_state_machine.py | 13 ------------- 2 files changed, 5 insertions(+), 18 deletions(-) diff --git a/atst/domain/csp/cloud.py b/atst/domain/csp/cloud.py index 104e3e0e..d22f9475 100644 --- a/atst/domain/csp/cloud.py +++ b/atst/domain/csp/cloud.py @@ -799,7 +799,7 @@ class MockCloudProvider(CloudProviderInterface): @property def _auth_credentials(self): - return {"username": "mock-cloud", "password": "shh"} + return {"username": "mock-cloud", "password": "shh"} # pragma: allowlist secret def _authorize(self, credentials): self._delay(1, 5) @@ -864,26 +864,26 @@ class AzureCloudProvider(CloudProviderInterface): self.policy_manager = AzurePolicyManager(config["AZURE_POLICY_LOCATION"]) def set_secret(self, secret_key, secret_value): - credential = self._get_client_secret_credential_obj() + credential = self._get_client_secret_credential_obj({}) secret_client = self.secrets.SecretClient( vault_url=self.vault_url, credential=credential, ) try: return secret_client.set_secret(secret_key, secret_value) - except self.exceptions.HttpResponseError as exc: + except self.exceptions.HttpResponseError: app.logger.error( f"Could not SET secret in Azure keyvault for key {secret_key}.", exc_info=1, ) def get_secret(self, secret_key): - credential = self._get_client_secret_credential_obj() + credential = self._get_client_secret_credential_obj({}) secret_client = self.secrets.SecretClient( vault_url=self.vault_url, credential=credential, ) try: return secret_client.get_secret(secret_key).value - except self.exceptions.HttpResponseError as exc: + except self.exceptions.HttpResponseError: app.logger.error( f"Could not GET secret in Azure keyvault for key {secret_key}.", exc_info=1, diff --git a/atst/models/portfolio_state_machine.py b/atst/models/portfolio_state_machine.py index 6e3015c4..a0cc77cd 100644 --- a/atst/models/portfolio_state_machine.py +++ b/atst/models/portfolio_state_machine.py @@ -1,6 +1,3 @@ -from random import choice, choices -import string - from sqlalchemy import Column, ForeignKey, Enum as SQLAEnum from sqlalchemy.orm import relationship, reconstructor from sqlalchemy.dialects.postgresql import UUID @@ -20,16 +17,6 @@ import atst.models.mixins as mixins from atst.models.mixins.state_machines import FSMStates, AzureStages, _build_transitions -def make_password(): - return choice(string.ascii_letters) + "".join( - choices(string.ascii_letters + string.digits + string.punctuation, k=15) - ) - - -def fetch_portfolio_creds(portfolio): - return dict(username="mock-cloud", password="shh") - - @add_state_features(Tags) class StateMachineWithTags(Machine): pass From 7857fffc1c5b625ca9a5c41a50ef8c4f57d5d690 Mon Sep 17 00:00:00 2001 From: leigh-mil Date: Fri, 24 Jan 2020 11:53:29 -0500 Subject: [PATCH 40/46] Add alert for when portfolio isn't funded, Update styling for env pending label --- static/icons/clock.svg | 1 + styles/elements/_labels.scss | 2 +- .../applications/fragments/environments.html | 37 +++++++++++-------- templates/components/label.html | 5 +++ translations.yaml | 1 + 5 files changed, 30 insertions(+), 16 deletions(-) create mode 100644 static/icons/clock.svg diff --git a/static/icons/clock.svg b/static/icons/clock.svg new file mode 100644 index 00000000..ef1d84a1 --- /dev/null +++ b/static/icons/clock.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/styles/elements/_labels.scss b/styles/elements/_labels.scss index 8c044f8d..1e431957 100644 --- a/styles/elements/_labels.scss +++ b/styles/elements/_labels.scss @@ -21,7 +21,7 @@ text-transform: uppercase; &--default { - background-color: $color-gray-dark; + background-color: $color-gray; } &--info { diff --git a/templates/applications/fragments/environments.html b/templates/applications/fragments/environments.html index fa4e5959..d0934268 100644 --- a/templates/applications/fragments/environments.html +++ b/templates/applications/fragments/environments.html @@ -1,3 +1,4 @@ +{% from "components/alert.html" import Alert %} {% from "components/icon.html" import Icon %} {% from "components/label.html" import Label %} {% from 'components/save_button.html' import SaveButton %} @@ -10,10 +11,13 @@ new_env_form) %}

{{ "portfolios.applications.settings.environments" | translate }}

+ {% if portfolio.num_task_orders == 0 -%} + {{ Alert(message="portfolios.applications.environments.funding_alert"|translate({'name': portfolio.name})) }} + {%- endif %} + {% if g.matchesPath("application-environments") -%} + {% include "fragments/flash.html" %} + {%- endif %}
- {% if g.matchesPath("application-environments") -%} - {% include "fragments/flash.html" %} - {%- endif %} {% if 0 == environments_obj | length -%}

@@ -30,14 +34,21 @@

  • - - - {{ env['name'] }} {{ Icon('link', classes='icon--medium icon--primary') }} - - + {% if not env["pending"] -%} + + + {{ env['name'] }} {{ Icon('link', classes='icon--medium icon--primary') }} + + + {% else -%} + + {{ env['name'] }} + + {{ Label(type="pending_creation", classes='label--below')}} + {%- endif %} {% if user_can(permissions.EDIT_ENVIRONMENT) -%} {{ ToggleButton( @@ -57,10 +68,6 @@ classes="environment-list__item__members" ) }} -
    - {% if env['pending'] -%} - {{ Label(type="changes_pending", classes='label--below')}} - {%- endif %}
    diff --git a/templates/components/label.html b/templates/components/label.html index 27f1c1b1..4d2679f1 100644 --- a/templates/components/label.html +++ b/templates/components/label.html @@ -9,6 +9,11 @@ "text": "changes pending", "color": "default", }, + "pending_creation": { + "icon": "clock", + "text": "pending creation", + "color": "default", + }, "ppoc": {"text": "primary point of contact"} } %} diff --git a/translations.yaml b/translations.yaml index d9615c4d..9f85ac63 100644 --- a/translations.yaml +++ b/translations.yaml @@ -417,6 +417,7 @@ portfolios: add_subscription: Add new subscription blank_slate: This Application has no environments disabled: ": Access Suspended" + funding_alert: "Application environments will not be created until the {name} portfolio is funded." environments_heading: Application Environments existing_application_title: "{application_name} Application Settings" member_count: "{count} Members" From f08d53d7a03bbcbbb37a084040fb4052ad9d0cef Mon Sep 17 00:00:00 2001 From: tomdds Date: Fri, 24 Jan 2020 15:42:23 -0500 Subject: [PATCH 41/46] Transition all Cloud Interface Methods to use Dataclasses --- atst/domain/csp/cloud.py | 18 ++--- atst/models/portfolio_state_machine.py | 83 ++++++++------------ tests/domain/test_portfolio_state_machine.py | 4 +- 3 files changed, 44 insertions(+), 61 deletions(-) diff --git a/atst/domain/csp/cloud.py b/atst/domain/csp/cloud.py index d22f9475..c7134431 100644 --- a/atst/domain/csp/cloud.py +++ b/atst/domain/csp/cloud.py @@ -529,8 +529,8 @@ class MockCloudProvider(CloudProviderInterface): def set_secret(self, secret_key: str, secret_value: str): pass - def get_secret(self, secret_key: str): - return {} + def get_secret(self, secret_key: str, default=dict()): + return default def create_environment(self, auth_credentials, user, environment): self._authorize(auth_credentials) @@ -598,7 +598,7 @@ class MockCloudProvider(CloudProviderInterface): "tenant_admin_username": "test", "tenant_admin_password": "test", } - ).dict() + ) def create_billing_profile_creation( self, payload: BillingProfileCreationCSPPayload @@ -613,7 +613,7 @@ class MockCloudProvider(CloudProviderInterface): billing_profile_verify_url="https://zombo.com", billing_profile_retry_after=10, ) - ).dict() + ) def create_billing_profile_verification( self, payload: BillingProfileVerificationCSPPayload @@ -651,7 +651,7 @@ class MockCloudProvider(CloudProviderInterface): }, "type": "Microsoft.Billing/billingAccounts/billingProfiles", } - ).dict() + ) def create_billing_profile_tenant_access(self, payload): self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) @@ -672,7 +672,7 @@ class MockCloudProvider(CloudProviderInterface): }, "type": "Microsoft.Billing/billingRoleAssignments", } - ).dict() + ) def create_task_order_billing_creation( self, payload: TaskOrderBillingCreationCSPPayload @@ -683,7 +683,7 @@ class MockCloudProvider(CloudProviderInterface): return TaskOrderBillingCreationCSPResult( **{"Location": "https://somelocation", "Retry-After": "10"} - ).dict() + ) def create_task_order_billing_verification( self, payload: TaskOrderBillingVerificationCSPPayload @@ -720,7 +720,7 @@ class MockCloudProvider(CloudProviderInterface): }, "type": "Microsoft.Billing/billingAccounts/billingProfiles", } - ).dict() + ) def create_billing_instruction(self, payload: BillingInstructionCSPPayload): self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) @@ -737,7 +737,7 @@ class MockCloudProvider(CloudProviderInterface): }, "type": "Microsoft.Billing/billingAccounts/billingProfiles/billingInstructions", } - ).dict() + ) def create_or_update_user(self, auth_credentials, user_info, csp_role_id): self._authorize(auth_credentials) diff --git a/atst/models/portfolio_state_machine.py b/atst/models/portfolio_state_machine.py index a0cc77cd..1390ceb1 100644 --- a/atst/models/portfolio_state_machine.py +++ b/atst/models/portfolio_state_machine.py @@ -144,70 +144,51 @@ class PortfolioStateMachine( else: self.csp = MockCSP(app).cloud - attempts_count = 5 - for attempt in range(attempts_count): - try: - func_name = f"create_{stage}" - response = getattr(self.csp, func_name)(payload_data) - except (ConnectionException, UnknownServerException) as exc: - app.logger.error( - f"CSP api call. Caught exception for {self.__repr__()}. Retry attempt {attempt}", - exc_info=1, - ) - continue - else: - break - else: - # failed all attempts - logger.info(f"CSP api call failed after {attempts_count} attempts.") + try: + func_name = f"create_{stage}" + response = getattr(self.csp, func_name)(payload_data) + if self.portfolio.csp_data is None: + self.portfolio.csp_data = {} + self.portfolio.csp_data.update(response.dict()) + db.session.add(self.portfolio) + db.session.commit() + + if getattr(response, "get_creds", None) is not None: + new_creds = response.get_creds() + # TODO: one way salted hash of tenant_id to use as kv key name? + tenant_id = new_creds.get("tenant_id") + secret = self.csp.get_secret(tenant_id, new_creds) + secret.update(new_creds) + self.csp.set_secret(tenant_id, secret) + except PydanticValidationError as exc: + app.logger.error( + f"Failed to cast response to valid result class {self.__repr__()}:", + exc_info=1, + ) + app.logger.info(exc.json()) + print(exc.json()) + app.logger.info(payload_data) + self.fail_stage(stage) + except (ConnectionException, UnknownServerException) as exc: + app.logger.error( + f"CSP api call. Caught exception for {self.__repr__()}.", exc_info=1, + ) self.fail_stage(stage) - - if self.portfolio.csp_data is None: - self.portfolio.csp_data = {} - self.portfolio.csp_data.update(response) - db.session.add(self.portfolio) - db.session.commit() - - # store any updated creds, if necessary self.finish_stage(stage) def is_csp_data_valid(self, event): - # check portfolio csp details json field for fields + """ + This function guards advancing states from *_IN_PROGRESS to *_COMPLETED. + """ if self.portfolio.csp_data is None or not isinstance( self.portfolio.csp_data, dict ): print("no csp data") return False - stage = self.current_state.name.split("_IN_PROGRESS")[0].lower() - stage_data = self.portfolio.csp_data - cls = get_stage_csp_class(stage, "result") - if not cls: - return False - - try: - dc = cls(**stage_data) - if getattr(dc, "get_creds", None) is not None: - new_creds = dc.get_creds() - tenant_id = new_creds.get("tenant_id") - secret = self.csp.get_secret(tenant_id) - secret.update(new_creds) - self.csp.set_secret(tenant_id, secret) - - except PydanticValidationError as exc: - app.logger.error( - f"Payload Validation Error in {self.__repr__()}:", exc_info=1 - ) - app.logger.info(exc.json()) - app.logger.info(payload) - - return False - return True - # print('failed condition', self.portfolio.csp_data) - @property def application_id(self): return None diff --git a/tests/domain/test_portfolio_state_machine.py b/tests/domain/test_portfolio_state_machine.py index 0d37c9c7..eeebf6e9 100644 --- a/tests/domain/test_portfolio_state_machine.py +++ b/tests/domain/test_portfolio_state_machine.py @@ -11,6 +11,8 @@ from atst.models.mixins.state_machines import AzureStages, StageStates, compose_ from atst.models.portfolio import Portfolio from atst.domain.csp import get_stage_csp_class +# TODO: Write failure case tests + @pytest.fixture(scope="function") def portfolio(): @@ -122,7 +124,7 @@ def test_fsm_transition_start(portfolio: Portfolio): "last_name": ppoc.last_name, "country_code": "US", "password_recovery_email_address": ppoc.email, - "address": { + "address": { # TODO: TBD if we're sourcing this from data or config "company_name": "", "address_line_1": "", "city": "", From d02d47615ef79f02843ab37c4a1b371242d2427d Mon Sep 17 00:00:00 2001 From: tomdds Date: Sun, 26 Jan 2020 13:01:37 -0500 Subject: [PATCH 42/46] First step in breaking out cloud.py Move cloud.py to a module init. Move policy with it. Update related unit tests. Also adds a patch to state machine test to prevent randomness in mock from failing test. --- atst/domain/csp/{cloud.py => cloud/__init__.py} | 5 +++-- atst/domain/csp/{ => cloud}/policy.py | 0 script/include/test_functions.inc.sh | 2 +- tests/domain/cloud/test_policy.py | 2 +- tests/domain/test_portfolio_state_machine.py | 11 +++++++---- 5 files changed, 12 insertions(+), 8 deletions(-) rename atst/domain/csp/{cloud.py => cloud/__init__.py} (99%) rename atst/domain/csp/{ => cloud}/policy.py (100%) diff --git a/atst/domain/csp/cloud.py b/atst/domain/csp/cloud/__init__.py similarity index 99% rename from atst/domain/csp/cloud.py rename to atst/domain/csp/cloud/__init__.py index c7134431..1597134f 100644 --- a/atst/domain/csp/cloud.py +++ b/atst/domain/csp/cloud/__init__.py @@ -1,4 +1,5 @@ import re +from secrets import token_urlsafe from typing import Dict, List, Optional from uuid import uuid4 @@ -177,7 +178,7 @@ class BaseCSPPayload(AliasModel): class TenantCSPPayload(BaseCSPPayload): user_id: str - password: str + password: Optional[str] domain_name: str first_name: str last_name: str @@ -1070,7 +1071,7 @@ class AzureCloudProvider(CloudProviderInterface): sp_token = self._get_sp_token(payload.creds) if sp_token is None: raise AuthenticationException("Could not resolve token for tenant creation") - + payload.password = token_urlsafe(16) create_tenant_body = payload.dict(by_alias=True) create_tenant_headers = { diff --git a/atst/domain/csp/policy.py b/atst/domain/csp/cloud/policy.py similarity index 100% rename from atst/domain/csp/policy.py rename to atst/domain/csp/cloud/policy.py diff --git a/script/include/test_functions.inc.sh b/script/include/test_functions.inc.sh index d6bf0b4d..bfd4d343 100644 --- a/script/include/test_functions.inc.sh +++ b/script/include/test_functions.inc.sh @@ -8,7 +8,7 @@ run_python_lint() { } run_python_typecheck() { - run_command "mypy --ignore-missing-imports --follow-imports=skip atst/domain/csp/cloud.py" + run_command "mypy --ignore-missing-imports --follow-imports=skip atst/domain/csp/cloud/__init__.py" return $? } diff --git a/tests/domain/cloud/test_policy.py b/tests/domain/cloud/test_policy.py index c0189262..18f0a7ab 100644 --- a/tests/domain/cloud/test_policy.py +++ b/tests/domain/cloud/test_policy.py @@ -1,4 +1,4 @@ -from atst.domain.csp.policy import AzurePolicyManager, AzurePolicy +from atst.domain.csp.cloud.policy import AzurePolicyManager, AzurePolicy def test_portfolio_definitions(): diff --git a/tests/domain/test_portfolio_state_machine.py b/tests/domain/test_portfolio_state_machine.py index eeebf6e9..97056ae7 100644 --- a/tests/domain/test_portfolio_state_machine.py +++ b/tests/domain/test_portfolio_state_machine.py @@ -1,5 +1,6 @@ import pytest import re +from unittest import mock from tests.factories import ( PortfolioStateMachineFactory, @@ -81,7 +82,10 @@ def test_state_machine_initialization(portfolio): assert ["reset", "fail", create_trigger] == started_triggers -def test_fsm_transition_start(portfolio: Portfolio): +@mock.patch("atst.domain.csp.cloud.MockCloudProvider") +def test_fsm_transition_start(mock_cloud_provider, portfolio: Portfolio): + mock_cloud_provider._authorize.return_value = None + mock_cloud_provider._maybe_raise.return_value = None sm: PortfolioStateMachine = PortfolioStateMachineFactory.create(portfolio=portfolio) assert sm.portfolio assert sm.state == FSMStates.UNSTARTED @@ -103,7 +107,7 @@ def test_fsm_transition_start(portfolio: Portfolio): ] # Should source all creds for portfolio? might be easier to manage than per-step specific ones - creds = {"username": "mock-cloud", "password": "shh"} + creds = {"username": "mock-cloud", "password": "shh"} # pragma: allowlist secret if portfolio.csp_data is not None: csp_data = portfolio.csp_data else: @@ -118,7 +122,7 @@ def test_fsm_transition_start(portfolio: Portfolio): portfolio_data = { "user_id": user_id, - "password": "jklfsdNCVD83nklds2#202", + "password": "jklfsdNCVD83nklds2#202", # pragma: allowlist secret "domain_name": domain_name, "first_name": ppoc.first_name, "last_name": ppoc.last_name, @@ -143,7 +147,6 @@ def test_fsm_transition_start(portfolio: Portfolio): config = {"billing_account_name": "billing_account_name"} for expected_state in expected_states: - print(expected_state) collected_data = dict( list(csp_data.items()) + list(portfolio_data.items()) + list(config.items()) ) From b28281d04e76dbdc0f62121ad44b5efbb634a71f Mon Sep 17 00:00:00 2001 From: tomdds Date: Sun, 26 Jan 2020 14:02:50 -0500 Subject: [PATCH 43/46] Break out cloud integration into smaller files. --- atst/domain/csp/cloud/__init__.py | 1414 +---------------- atst/domain/csp/cloud/azure_cloud_provider.py | 613 +++++++ .../csp/cloud/cloud_provider_interface.py | 126 ++ atst/domain/csp/cloud/exceptions.py | 131 ++ atst/domain/csp/cloud/mock_cloud_provider.py | 315 ++++ atst/domain/csp/cloud/models.py | 234 +++ 6 files changed, 1424 insertions(+), 1409 deletions(-) create mode 100644 atst/domain/csp/cloud/azure_cloud_provider.py create mode 100644 atst/domain/csp/cloud/cloud_provider_interface.py create mode 100644 atst/domain/csp/cloud/exceptions.py create mode 100644 atst/domain/csp/cloud/mock_cloud_provider.py create mode 100644 atst/domain/csp/cloud/models.py diff --git a/atst/domain/csp/cloud/__init__.py b/atst/domain/csp/cloud/__init__.py index 1597134f..fc754c7d 100644 --- a/atst/domain/csp/cloud/__init__.py +++ b/atst/domain/csp/cloud/__init__.py @@ -1,1409 +1,5 @@ -import re -from secrets import token_urlsafe -from typing import Dict, List, Optional -from uuid import uuid4 - -from pydantic import BaseModel, validator - -from flask import current_app as app - -from atst.models.user import User -from atst.models.application import Application -from atst.models.environment import Environment -from atst.models.environment_role import EnvironmentRole -from atst.utils import snake_to_camel -from .policy import AzurePolicyManager - - -class GeneralCSPException(Exception): - pass - - -class OperationInProgressException(GeneralCSPException): - """Throw this for instances when the CSP reports that the current entity is already - being operated on/created/deleted/etc - """ - - def __init__(self, operation_desc): - self.operation_desc = operation_desc - - @property - def message(self): - return "An operation for this entity is already in progress: {}".format( - self.operation_desc - ) - - -class AuthenticationException(GeneralCSPException): - """Throw this for instances when there is a problem with the auth credentials: - * Missing credentials - * Incorrect credentials - * Other credential problems - """ - - def __init__(self, auth_error): - self.auth_error = auth_error - - @property - def message(self): - return "An error occurred with authentication: {}".format(self.auth_error) - - -class AuthorizationException(GeneralCSPException): - """Throw this for instances when the current credentials are not authorized - for the current action. - """ - - def __init__(self, auth_error): - self.auth_error = auth_error - - @property - def message(self): - return "An error occurred with authorization: {}".format(self.auth_error) - - -class ConnectionException(GeneralCSPException): - """A general problem with the connection, timeouts or unresolved endpoints - """ - - def __init__(self, connection_error): - self.connection_error = connection_error - - @property - def message(self): - return "Could not connect to cloud provider: {}".format(self.connection_error) - - -class UnknownServerException(GeneralCSPException): - """An error occured on the CSP side (5xx) and we don't know why - """ - - def __init__(self, server_error): - self.server_error = server_error - - @property - def message(self): - return "A server error occured: {}".format(self.server_error) - - -class EnvironmentCreationException(GeneralCSPException): - """If there was an error in creating the environment - """ - - def __init__(self, env_identifier, reason): - self.env_identifier = env_identifier - self.reason = reason - - @property - def message(self): - return "The envionment {} couldn't be created: {}".format( - self.env_identifier, self.reason - ) - - -class UserProvisioningException(GeneralCSPException): - """Failed to provision a user - """ - - def __init__(self, env_identifier, user_identifier, reason): - self.env_identifier = env_identifier - self.user_identifier = user_identifier - self.reason = reason - - @property - def message(self): - return "Failed to create user {} for environment {}: {}".format( - self.user_identifier, self.env_identifier, self.reason - ) - - -class UserRemovalException(GeneralCSPException): - """Failed to remove a user - """ - - def __init__(self, user_csp_id, reason): - self.user_csp_id = user_csp_id - self.reason = reason - - @property - def message(self): - return "Failed to suspend or delete user {}: {}".format( - self.user_csp_id, self.reason - ) - - -class BaselineProvisionException(GeneralCSPException): - """If there's any issues standing up whatever is required - for an environment baseline - """ - - def __init__(self, env_identifier, reason): - self.env_identifier = env_identifier - self.reason = reason - - @property - def message(self): - return "Could not complete baseline provisioning for environment ({}): {}".format( - self.env_identifier, self.reason - ) - - -class AliasModel(BaseModel): - """ - This provides automatic camel <-> snake conversion for serializing to/from json - You can override the alias generation in subclasses by providing a Config that defines - a fields property with a dict mapping variables to their cast names, for cases like: - * some_url:someURL - * user_object_id:objectId - """ - - class Config: - alias_generator = snake_to_camel - allow_population_by_field_name = True - - -class BaseCSPPayload(AliasModel): - # {"username": "mock-cloud", "pass": "shh"} - creds: Dict - - def dict(self, *args, **kwargs): - exclude = {"creds"} - if "exclude" not in kwargs: - kwargs["exclude"] = exclude - else: - kwargs["exclude"].update(exclude) - - return super().dict(*args, **kwargs) - - -class TenantCSPPayload(BaseCSPPayload): - user_id: str - password: Optional[str] - domain_name: str - first_name: str - last_name: str - country_code: str - password_recovery_email_address: str - - -class TenantCSPResult(AliasModel): - user_id: str - tenant_id: str - user_object_id: str - - tenant_admin_username: Optional[str] - tenant_admin_password: Optional[str] - - class Config: - fields = { - "user_object_id": "objectId", - } - - def dict(self, *args, **kwargs): - exclude = {"tenant_admin_username", "tenant_admin_password"} - if "exclude" not in kwargs: - kwargs["exclude"] = exclude - else: - kwargs["exclude"].update(exclude) - - return super().dict(*args, **kwargs) - - def get_creds(self): - return { - "tenant_admin_username": self.tenant_admin_username, - "tenant_admin_password": self.tenant_admin_password, - "tenant_id": self.tenant_id, - } - - -class BillingProfileAddress(AliasModel): - company_name: str - address_line_1: str - city: str - region: str - country: str - postal_code: str - - -class BillingProfileCLINBudget(AliasModel): - clin_budget: Dict - """ - "clinBudget": { - "amount": 0, - "startDate": "2019-12-18T16:47:40.909Z", - "endDate": "2019-12-18T16:47:40.909Z", - "externalReferenceId": "string" - } - """ - - -class BillingProfileCreationCSPPayload(BaseCSPPayload): - tenant_id: str - billing_profile_display_name: str - billing_account_name: str - enabled_azure_plans: Optional[List[str]] - address: BillingProfileAddress - - @validator("enabled_azure_plans", pre=True, always=True) - def default_enabled_azure_plans(cls, v): - """ - Normally you'd implement this by setting the field with a value of: - dataclasses.field(default_factory=list) - but that prevents the object from being correctly pickled, so instead we need - to rely on a validator to ensure this has an empty value when not specified - """ - return v or [] - - class Config: - fields = {"billing_profile_display_name": "displayName"} - - -class BillingProfileCreationCSPResult(AliasModel): - billing_profile_verify_url: str - billing_profile_retry_after: int - - class Config: - fields = { - "billing_profile_verify_url": "Location", - "billing_profile_retry_after": "Retry-After", - } - - -class BillingProfileVerificationCSPPayload(BaseCSPPayload): - billing_profile_verify_url: str - - -class BillingInvoiceSection(AliasModel): - invoice_section_id: str - invoice_section_name: str - - class Config: - fields = {"invoice_section_id": "id", "invoice_section_name": "name"} - - -class BillingProfileProperties(AliasModel): - address: BillingProfileAddress - billing_profile_display_name: str - invoice_sections: List[BillingInvoiceSection] - - class Config: - fields = {"billing_profile_display_name": "displayName"} - - -class BillingProfileVerificationCSPResult(AliasModel): - billing_profile_id: str - billing_profile_name: str - billing_profile_properties: BillingProfileProperties - - class Config: - fields = { - "billing_profile_id": "id", - "billing_profile_name": "name", - "billing_profile_properties": "properties", - } - - -class BillingProfileTenantAccessCSPPayload(BaseCSPPayload): - tenant_id: str - user_object_id: str - billing_account_name: str - billing_profile_name: str - - -class BillingProfileTenantAccessCSPResult(AliasModel): - billing_role_assignment_id: str - billing_role_assignment_name: str - - class Config: - fields = { - "billing_role_assignment_id": "id", - "billing_role_assignment_name": "name", - } - - -class TaskOrderBillingCreationCSPPayload(BaseCSPPayload): - billing_account_name: str - billing_profile_name: str - - -class TaskOrderBillingCreationCSPResult(AliasModel): - task_order_billing_verify_url: str - task_order_retry_after: int - - class Config: - fields = { - "task_order_billing_verify_url": "Location", - "task_order_retry_after": "Retry-After", - } - - -class TaskOrderBillingVerificationCSPPayload(BaseCSPPayload): - task_order_billing_verify_url: str - - -class BillingProfileEnabledPlanDetails(AliasModel): - enabled_azure_plans: List[Dict] - - -class TaskOrderBillingVerificationCSPResult(AliasModel): - billing_profile_id: str - billing_profile_name: str - billing_profile_enabled_plan_details: BillingProfileEnabledPlanDetails - - class Config: - fields = { - "billing_profile_id": "id", - "billing_profile_name": "name", - "billing_profile_enabled_plan_details": "properties", - } - - -class BillingInstructionCSPPayload(BaseCSPPayload): - initial_clin_amount: float - initial_clin_start_date: str - initial_clin_end_date: str - initial_clin_type: str - initial_task_order_id: str - billing_account_name: str - billing_profile_name: str - - -class BillingInstructionCSPResult(AliasModel): - reported_clin_name: str - - class Config: - fields = { - "reported_clin_name": "name", - } - - -class CloudProviderInterface: - def set_secret(self, secret_key: str, secret_value: str): - raise NotImplementedError() - - def get_secret(self, secret_key: str): - raise NotImplementedError() - - def root_creds(self) -> Dict: - raise NotImplementedError() - - def create_environment( - self, auth_credentials: Dict, user: User, environment: Environment - ) -> str: - """Create a new environment in the CSP. - - Arguments: - auth_credentials -- Object containing CSP account credentials - user -- ATAT user authorizing the environment creation - environment -- ATAT Environment model - - Returns: - string: ID of created environment - - Raises: - AuthenticationException: Problem with the credentials - AuthorizationException: Credentials not authorized for current action(s) - ConnectionException: Issue with the CSP API connection - UnknownServerException: Unknown issue on the CSP side - EnvironmentExistsException: Environment already exists and has been created - """ - raise NotImplementedError() - - def create_atat_admin_user( - self, auth_credentials: Dict, csp_environment_id: str - ) -> Dict: - """Creates a new, programmatic user in the CSP. Grants this user full permissions to administer - the CSP. - - Arguments: - auth_credentials -- Object containing CSP account credentials - csp_environment_id -- ID of the CSP Environment the admin user should be created in - - Returns: - object: Object representing new remote admin user, including credentials - Something like: - { - "user_id": string, - "credentials": dict, # structure TBD based on csp - } - - Raises: - AuthenticationException: Problem with the credentials - AuthorizationException: Credentials not authorized for current action(s) - ConnectionException: Issue with the CSP API connection - UnknownServerException: Unknown issue on the CSP side - UserProvisioningException: Problem creating the root user - """ - raise NotImplementedError() - - def create_or_update_user( - self, auth_credentials: Dict, user_info: EnvironmentRole, csp_role_id: str - ) -> str: - """Creates a user or updates an existing user's role. - - Arguments: - auth_credentials -- Object containing CSP account credentials - user_info -- instance of EnvironmentRole containing user data - if it has a csp_user_id it will try to update that user - csp_role_id -- The id of the role the user should be given in the CSP - - Returns: - string: Returns the interal csp_user_id of the created/updated user account - - Raises: - AuthenticationException: Problem with the credentials - AuthorizationException: Credentials not authorized for current action(s) - ConnectionException: Issue with the CSP API connection - UnknownServerException: Unknown issue on the CSP side - UserProvisioningException: User couldn't be created or modified - """ - raise NotImplementedError() - - def disable_user(self, auth_credentials: Dict, csp_user_id: str) -> bool: - """Revoke all privileges for a user. Used to prevent user access while a full - delete is being processed. - - Arguments: - auth_credentials -- Object containing CSP account credentials - csp_user_id -- CSP internal user identifier - - Returns: - bool -- True on success - - Raises: - AuthenticationException: Problem with the credentials - AuthorizationException: Credentials not authorized for current action(s) - ConnectionException: Issue with the CSP API connection - UnknownServerException: Unknown issue on the CSP side - UserRemovalException: User couldn't be suspended - """ - raise NotImplementedError() - - def get_calculator_url(self) -> str: - """Returns the calculator url for the CSP. - This will likely be a static property elsewhere once a CSP is chosen. - """ - raise NotImplementedError() - - def get_environment_login_url(self, environment) -> str: - """Returns the login url for a given environment - This may move to be a computed property on the Environment domain object - """ - raise NotImplementedError() - - def create_subscription(self, environment): - """Returns True if a new subscription has been created or raises an - exception if an error occurs while creating a subscription. - """ - raise NotImplementedError() - - -class MockCloudProvider(CloudProviderInterface): - - # TODO: All of these constants - AUTHENTICATION_EXCEPTION = AuthenticationException("Authentication failure.") - AUTHORIZATION_EXCEPTION = AuthorizationException("Not authorized.") - NETWORK_EXCEPTION = ConnectionException("Network failure.") - SERVER_EXCEPTION = UnknownServerException("Not our fault.") - - SERVER_FAILURE_PCT = 1 - NETWORK_FAILURE_PCT = 7 - ENV_CREATE_FAILURE_PCT = 12 - ATAT_ADMIN_CREATE_FAILURE_PCT = 12 - UNAUTHORIZED_RATE = 2 - - def __init__( - self, config, with_delay=True, with_failure=True, with_authorization=True - ): - from time import sleep - import random - - self._with_delay = with_delay - self._with_failure = with_failure - self._with_authorization = with_authorization - self._sleep = sleep - self._random = random - - def root_creds(self): - return self._auth_credentials - - def set_secret(self, secret_key: str, secret_value: str): - pass - - def get_secret(self, secret_key: str, default=dict()): - return default - - def create_environment(self, auth_credentials, user, environment): - self._authorize(auth_credentials) - - self._delay(1, 5) - self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) - self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) - self._maybe_raise( - self.ENV_CREATE_FAILURE_PCT, - EnvironmentCreationException( - environment.id, "Could not create environment." - ), - ) - - csp_environment_id = self._id() - - self._delay(1, 5) - self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) - self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) - self._maybe_raise( - self.ATAT_ADMIN_CREATE_FAILURE_PCT, - BaselineProvisionException( - csp_environment_id, "Could not create environment baseline." - ), - ) - self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) - - return csp_environment_id - - def create_atat_admin_user(self, auth_credentials, csp_environment_id): - self._authorize(auth_credentials) - - self._delay(1, 5) - self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) - self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) - self._maybe_raise( - self.ATAT_ADMIN_CREATE_FAILURE_PCT, - UserProvisioningException( - csp_environment_id, "atat_admin", "Could not create admin user." - ), - ) - - self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) - - return {"id": self._id(), "credentials": self._auth_credentials} - - def create_tenant(self, payload: TenantCSPPayload): - """ - payload is an instance of TenantCSPPayload data class - """ - - self._authorize(payload.creds) - - self._delay(1, 5) - - self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) - self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) - self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) - - return TenantCSPResult( - **{ - "tenant_id": "", - "user_id": "", - "user_object_id": "", - "tenant_admin_username": "test", - "tenant_admin_password": "test", - } - ) - - def create_billing_profile_creation( - self, payload: BillingProfileCreationCSPPayload - ): - # response will be mostly the same as the body, but we only really care about the id - self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) - self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) - self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) - - return BillingProfileCreationCSPResult( - **dict( - billing_profile_verify_url="https://zombo.com", - billing_profile_retry_after=10, - ) - ) - - def create_billing_profile_verification( - self, payload: BillingProfileVerificationCSPPayload - ): - self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) - self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) - self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) - return BillingProfileVerificationCSPResult( - **{ - "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB", - "name": "KQWI-W2SU-BG7-TGB", - "properties": { - "address": { - "addressLine1": "123 S Broad Street, Suite 2400", - "city": "Philadelphia", - "companyName": "Promptworks", - "country": "US", - "postalCode": "19109", - "region": "PA", - }, - "currency": "USD", - "displayName": "Test Billing Profile", - "enabledAzurePlans": [], - "hasReadAccess": True, - "invoiceDay": 5, - "invoiceEmailOptIn": False, - "invoiceSections": [ - { - "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB/invoiceSections/CHCO-BAAR-PJA-TGB", - "name": "CHCO-BAAR-PJA-TGB", - "properties": {"displayName": "Test Billing Profile"}, - "type": "Microsoft.Billing/billingAccounts/billingProfiles/invoiceSections", - } - ], - }, - "type": "Microsoft.Billing/billingAccounts/billingProfiles", - } - ) - - def create_billing_profile_tenant_access(self, payload): - self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) - self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) - self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) - - return BillingProfileTenantAccessCSPResult( - **{ - "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB/billingRoleAssignments/40000000-aaaa-bbbb-cccc-100000000000_0a5f4926-e3ee-4f47-a6e3-8b0a30a40e3d", - "name": "40000000-aaaa-bbbb-cccc-100000000000_0a5f4926-e3ee-4f47-a6e3-8b0a30a40e3d", - "properties": { - "createdOn": "2020-01-14T14:39:26.3342192+00:00", - "createdByPrincipalId": "82e2b376-3297-4096-8743-ed65b3be0b03", - "principalId": "0a5f4926-e3ee-4f47-a6e3-8b0a30a40e3d", - "principalTenantId": "60ff9d34-82bf-4f21-b565-308ef0533435", - "roleDefinitionId": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB/billingRoleDefinitions/40000000-aaaa-bbbb-cccc-100000000000", - "scope": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB", - }, - "type": "Microsoft.Billing/billingRoleAssignments", - } - ) - - def create_task_order_billing_creation( - self, payload: TaskOrderBillingCreationCSPPayload - ): - self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) - self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) - self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) - - return TaskOrderBillingCreationCSPResult( - **{"Location": "https://somelocation", "Retry-After": "10"} - ) - - def create_task_order_billing_verification( - self, payload: TaskOrderBillingVerificationCSPPayload - ): - self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) - self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) - self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) - - return TaskOrderBillingVerificationCSPResult( - **{ - "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/XC36-GRNZ-BG7-TGB", - "name": "XC36-GRNZ-BG7-TGB", - "properties": { - "address": { - "addressLine1": "123 S Broad Street, Suite 2400", - "city": "Philadelphia", - "companyName": "Promptworks", - "country": "US", - "postalCode": "19109", - "region": "PA", - }, - "currency": "USD", - "displayName": "First Portfolio Billing Profile", - "enabledAzurePlans": [ - { - "productId": "DZH318Z0BPS6", - "skuId": "0001", - "skuDescription": "Microsoft Azure Plan", - } - ], - "hasReadAccess": True, - "invoiceDay": 5, - "invoiceEmailOptIn": False, - }, - "type": "Microsoft.Billing/billingAccounts/billingProfiles", - } - ) - - def create_billing_instruction(self, payload: BillingInstructionCSPPayload): - self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) - self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) - self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) - - return BillingInstructionCSPResult( - **{ - "name": "TO1:CLIN001", - "properties": { - "amount": 1000.0, - "endDate": "2020-03-01T00:00:00+00:00", - "startDate": "2020-01-01T00:00:00+00:00", - }, - "type": "Microsoft.Billing/billingAccounts/billingProfiles/billingInstructions", - } - ) - - def create_or_update_user(self, auth_credentials, user_info, csp_role_id): - self._authorize(auth_credentials) - - self._delay(1, 5) - self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) - self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) - self._maybe_raise( - self.ATAT_ADMIN_CREATE_FAILURE_PCT, - UserProvisioningException( - user_info.environment.id, - user_info.application_role.user_id, - "Could not create user.", - ), - ) - - self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) - return self._id() - - def disable_user(self, auth_credentials, csp_user_id): - self._authorize(auth_credentials) - self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) - self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) - - self._maybe_raise( - self.ATAT_ADMIN_CREATE_FAILURE_PCT, - UserRemovalException(csp_user_id, "Could not disable user."), - ) - - return self._maybe(12) - - def create_subscription(self, environment): - self._maybe_raise(self.UNAUTHORIZED_RATE, GeneralCSPException) - - return True - - def get_calculator_url(self): - return "https://www.rackspace.com/en-us/calculator" - - def get_environment_login_url(self, environment): - """Returns the login url for a given environment - """ - return "https://www.mycloud.com/my-env-login" - - def _id(self): - return uuid4().hex - - def _delay(self, min_secs, max_secs): - if self._with_delay: - duration = self._random.randrange(min_secs, max_secs) - self._sleep(duration) - - def _maybe(self, pct): - return not self._with_failure or self._random.randrange(0, 100) < pct - - def _maybe_raise(self, pct, exc): - if self._with_failure and self._maybe(pct): - raise exc - - @property - def _auth_credentials(self): - return {"username": "mock-cloud", "password": "shh"} # pragma: allowlist secret - - def _authorize(self, credentials): - self._delay(1, 5) - if self._with_authorization and credentials != self._auth_credentials: - raise self.AUTHENTICATION_EXCEPTION - - -AZURE_ENVIRONMENT = "AZURE_PUBLIC_CLOUD" # TBD -AZURE_SKU_ID = "?" # probably a static sku specific to ATAT/JEDI -SUBSCRIPTION_ID_REGEX = re.compile( - "subscriptions\/([0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})", - re.I, -) - -# This needs to be a fully pathed role definition identifier, not just a UUID -REMOTE_ROOT_ROLE_DEF_ID = "/providers/Microsoft.Authorization/roleDefinitions/00000000-0000-4000-8000-000000000000" -AZURE_MANAGEMENT_API = "https://management.azure.com" - - -class AzureSDKProvider(object): - def __init__(self): - from azure.mgmt import subscription, authorization, managementgroups - from azure.mgmt.resource import policy - import azure.graphrbac as graphrbac - import azure.common.credentials as credentials - import azure.identity as identity - from azure.keyvault import secrets - - from msrestazure.azure_cloud import AZURE_PUBLIC_CLOUD - import adal - import requests - - self.subscription = subscription - self.policy = policy - self.managementgroups = managementgroups - self.authorization = authorization - self.adal = adal - self.graphrbac = graphrbac - self.credentials = credentials - self.identity = identity - self.exceptions = exceptions - self.secrets = secrets - self.requests = requests - # may change to a JEDI cloud - self.cloud = AZURE_PUBLIC_CLOUD - - -class AzureCloudProvider(CloudProviderInterface): - def __init__(self, config, azure_sdk_provider=None): - self.config = config - - self.client_id = config["AZURE_CLIENT_ID"] - self.secret_key = config["AZURE_SECRET_KEY"] - self.tenant_id = config["AZURE_TENANT_ID"] - self.vault_url = config["AZURE_VAULT_URL"] - - if azure_sdk_provider is None: - self.sdk = AzureSDKProvider() - else: - self.sdk = azure_sdk_provider - - self.policy_manager = AzurePolicyManager(config["AZURE_POLICY_LOCATION"]) - - def set_secret(self, secret_key, secret_value): - credential = self._get_client_secret_credential_obj({}) - secret_client = self.secrets.SecretClient( - vault_url=self.vault_url, credential=credential, - ) - try: - return secret_client.set_secret(secret_key, secret_value) - except self.exceptions.HttpResponseError: - app.logger.error( - f"Could not SET secret in Azure keyvault for key {secret_key}.", - exc_info=1, - ) - - def get_secret(self, secret_key): - credential = self._get_client_secret_credential_obj({}) - secret_client = self.secrets.SecretClient( - vault_url=self.vault_url, credential=credential, - ) - try: - return secret_client.get_secret(secret_key).value - except self.exceptions.HttpResponseError: - app.logger.error( - f"Could not GET secret in Azure keyvault for key {secret_key}.", - exc_info=1, - ) - - def create_environment( - self, auth_credentials: Dict, user: User, environment: Environment - ): - # since this operation would only occur within a tenant, should we source the tenant - # via lookup from environment once we've created the portfolio csp data schema - # something like this: - # environment_tenant = environment.application.portfolio.csp_data.get('tenant_id', None) - # though we'd probably source the whole credentials for these calls from the portfolio csp - # data, as it would have to be where we store the creds for the at-at user within the portfolio tenant - # credentials = self._get_credential_obj(environment.application.portfolio.csp_data.get_creds()) - credentials = self._get_credential_obj(self._root_creds) - display_name = f"{environment.application.name}_{environment.name}_{environment.id}" # proposed format - management_group_id = "?" # management group id chained from environment - parent_id = "?" # from environment.application - - management_group = self._create_management_group( - credentials, management_group_id, display_name, parent_id, - ) - - return management_group - - def create_atat_admin_user( - self, auth_credentials: Dict, csp_environment_id: str - ) -> Dict: - root_creds = self._root_creds - credentials = self._get_credential_obj(root_creds) - - sub_client = self.sdk.subscription.SubscriptionClient(credentials) - subscription = sub_client.subscriptions.get(csp_environment_id) - - managment_principal = self._get_management_service_principal() - - auth_client = self.sdk.authorization.AuthorizationManagementClient( - credentials, - # TODO: Determine which subscription this needs to point at - # Once we're in a multi-sub environment - subscription.id, - ) - - # Create role assignment for - role_assignment_id = str(uuid4()) - role_assignment_create_params = auth_client.role_assignments.models.RoleAssignmentCreateParameters( - role_definition_id=REMOTE_ROOT_ROLE_DEF_ID, - principal_id=managment_principal.id, - ) - - auth_client.role_assignments.create( - scope=f"/subscriptions/{subscription.id}/", - role_assignment_name=role_assignment_id, - parameters=role_assignment_create_params, - ) - - return { - "csp_user_id": managment_principal.object_id, - "credentials": managment_principal.password_credentials, - "role_name": role_assignment_id, - } - - def _create_application(self, auth_credentials: Dict, application: Application): - management_group_name = str(uuid4()) # can be anything, not just uuid - display_name = application.name # Does this need to be unique? - credentials = self._get_credential_obj(auth_credentials) - parent_id = "?" # application.portfolio.csp_details.management_group_id - - return self._create_management_group( - credentials, management_group_name, display_name, parent_id, - ) - - def _create_management_group( - self, credentials, management_group_id, display_name, parent_id=None, - ): - mgmgt_group_client = self.sdk.managementgroups.ManagementGroupsAPI(credentials) - create_parent_grp_info = self.sdk.managementgroups.models.CreateParentGroupInfo( - id=parent_id - ) - create_mgmt_grp_details = self.sdk.managementgroups.models.CreateManagementGroupDetails( - parent=create_parent_grp_info - ) - mgmt_grp_create = self.sdk.managementgroups.models.CreateManagementGroupRequest( - name=management_group_id, - display_name=display_name, - details=create_mgmt_grp_details, - ) - create_request = mgmgt_group_client.management_groups.create_or_update( - management_group_id, mgmt_grp_create - ) - - # result is a synchronous wait, might need to do a poll instead to handle first mgmt group create - # since we were told it could take 10+ minutes to complete, unless this handles that polling internally - return create_request.result() - - def _create_subscription( - self, - credentials, - display_name, - billing_profile_id, - sku_id, - management_group_id, - billing_account_name, - invoice_section_name, - ): - sub_client = self.sdk.subscription.SubscriptionClient(credentials) - - billing_profile_id = "?" # where do we source this? - sku_id = AZURE_SKU_ID - # These 2 seem like something that might be worthwhile to allow tiebacks to - # TOs filed for the environment - billing_account_name = "?" # from TO? - invoice_section_name = "?" # from TO? - - body = self.sdk.subscription.models.ModernSubscriptionCreationParameters( - display_name=display_name, - billing_profile_id=billing_profile_id, - sku_id=sku_id, - management_group_id=management_group_id, - ) - - # We may also want to create billing sections in the enrollment account - sub_creation_operation = sub_client.subscription_factory.create_subscription( - billing_account_name, invoice_section_name, body - ) - - # the resulting object from this process is a link to the new subscription - # not a subscription model, so we'll have to unpack the ID - new_sub = sub_creation_operation.result() - - subscription_id = self._extract_subscription_id(new_sub.subscription_link) - if subscription_id: - return subscription_id - else: - # troublesome error, subscription should exist at this point - # but we just don't have a valid ID - pass - - def _create_policy_definition( - self, credentials, subscription_id, management_group_id, properties, - ): - """ - Requires credentials that have AZURE_MANAGEMENT_API - specified as the resource. The Service Principal - specified in the credentials must have the "Resource - Policy Contributor" role assigned with a scope at least - as high as the management group specified by - management_group_id. - - Arguments: - credentials -- ServicePrincipalCredentials - subscription_id -- str, ID of the subscription (just the UUID, not the path) - management_group_id -- str, ID of the management group (just the UUID, not the path) - properties -- dictionary, the "properties" section of a valid Azure policy definition document - - Returns: - azure.mgmt.resource.policy.[api version].models.PolicyDefinition: the PolicyDefinition object provided to Azure - - Raises: - TBD - """ - # TODO: which subscription would this be? - client = self.sdk.policy.PolicyClient(credentials, subscription_id) - - definition = client.policy_definitions.models.PolicyDefinition( - policy_type=properties.get("policyType"), - mode=properties.get("mode"), - display_name=properties.get("displayName"), - description=properties.get("description"), - policy_rule=properties.get("policyRule"), - parameters=properties.get("parameters"), - ) - - name = properties.get("displayName") - - return client.policy_definitions.create_or_update_at_management_group( - policy_definition_name=name, - parameters=definition, - management_group_id=management_group_id, - ) - - def create_tenant(self, payload: TenantCSPPayload): - sp_token = self._get_sp_token(payload.creds) - if sp_token is None: - raise AuthenticationException("Could not resolve token for tenant creation") - payload.password = token_urlsafe(16) - create_tenant_body = payload.dict(by_alias=True) - - create_tenant_headers = { - "Authorization": f"Bearer {sp_token}", - } - - result = self.sdk.requests.post( - "https://management.azure.com/providers/Microsoft.SignUp/createTenant?api-version=2020-01-01-preview", - json=create_tenant_body, - headers=create_tenant_headers, - ) - - if result.status_code == 200: - return self._ok( - TenantCSPResult( - **result.json(), - tenant_admin_password=payload.password, - tenant_admin_username=payload.user_id, - ) - ) - else: - return self._error(result.json()) - - def create_billing_profile_creation( - self, payload: BillingProfileCreationCSPPayload - ): - sp_token = self._get_sp_token(payload.creds) - if sp_token is None: - raise AuthenticationException( - "Could not resolve token for billing profile creation" - ) - - create_billing_account_body = payload.dict(by_alias=True) - - create_billing_account_headers = { - "Authorization": f"Bearer {sp_token}", - } - - billing_account_create_url = f"https://management.azure.com/providers/Microsoft.Billing/billingAccounts/{payload.billing_account_name}/billingProfiles?api-version=2019-10-01-preview" - - result = self.sdk.requests.post( - billing_account_create_url, - json=create_billing_account_body, - headers=create_billing_account_headers, - ) - - if result.status_code == 202: - # 202 has location/retry after headers - return self._ok(BillingProfileCreationCSPResult(**result.headers)) - elif result.status_code == 200: - # NB: Swagger docs imply call can sometimes resolve immediately - return self._ok(BillingProfileVerificationCSPResult(**result.json())) - else: - return self._error(result.json()) - - def create_billing_profile_verification( - self, payload: BillingProfileVerificationCSPPayload - ): - sp_token = self._get_sp_token(payload.creds) - if sp_token is None: - raise AuthenticationException( - "Could not resolve token for billing profile validation" - ) - - auth_header = { - "Authorization": f"Bearer {sp_token}", - } - - result = self.sdk.requests.get( - payload.billing_profile_verify_url, headers=auth_header - ) - - if result.status_code == 202: - # 202 has location/retry after headers - return self._ok(BillingProfileCreationCSPResult(**result.headers)) - elif result.status_code == 200: - return self._ok(BillingProfileVerificationCSPResult(**result.json())) - else: - return self._error(result.json()) - - def create_billing_profile_tenant_access( - self, payload: BillingProfileTenantAccessCSPPayload - ): - sp_token = self._get_sp_token(payload.creds) - request_body = { - "properties": { - "principalTenantId": payload.tenant_id, # from tenant creation - "principalId": payload.user_object_id, # from tenant creationn - "roleDefinitionId": f"/providers/Microsoft.Billing/billingAccounts/{payload.billing_account_name}/billingProfiles/{payload.billing_profile_name}/billingRoleDefinitions/40000000-aaaa-bbbb-cccc-100000000000", - } - } - - headers = { - "Authorization": f"Bearer {sp_token}", - } - - url = f"https://management.azure.com/providers/Microsoft.Billing/billingAccounts/{payload.billing_account_name}/billingProfiles/{payload.billing_profile_name}/createBillingRoleAssignment?api-version=2019-10-01-preview" - - result = self.sdk.requests.post(url, headers=headers, json=request_body) - if result.status_code == 201: - return self._ok(BillingProfileTenantAccessCSPResult(**result.json())) - else: - return self._error(result.json()) - - def create_task_order_billing_creation( - self, payload: TaskOrderBillingCreationCSPPayload - ): - sp_token = self._get_sp_token(payload.creds) - request_body = [ - { - "op": "replace", - "path": "/enabledAzurePlans", - "value": [{"skuId": "0001"}], - } - ] - - request_headers = { - "Authorization": f"Bearer {sp_token}", - } - - url = f"https://management.azure.com/providers/Microsoft.Billing/billingAccounts/{payload.billing_account_name}/billingProfiles/{payload.billing_profile_name}?api-version=2019-10-01-preview" - - result = self.sdk.requests.patch( - url, headers=request_headers, json=request_body - ) - - if result.status_code == 202: - # 202 has location/retry after headers - return self._ok(TaskOrderBillingCreationCSPResult(**result.headers)) - elif result.status_code == 200: - return self._ok(TaskOrderBillingVerificationCSPResult(**result.json())) - else: - return self._error(result.json()) - - def create_task_order_billing_verification( - self, payload: TaskOrderBillingVerificationCSPPayload - ): - sp_token = self._get_sp_token(payload.creds) - if sp_token is None: - raise AuthenticationException( - "Could not resolve token for task order billing validation" - ) - - auth_header = { - "Authorization": f"Bearer {sp_token}", - } - - result = self.sdk.requests.get( - payload.task_order_billing_verify_url, headers=auth_header - ) - - if result.status_code == 202: - # 202 has location/retry after headers - return self._ok(TaskOrderBillingCreationCSPResult(**result.headers)) - elif result.status_code == 200: - return self._ok(TaskOrderBillingVerificationCSPResult(**result.json())) - else: - return self._error(result.json()) - - def create_billing_instruction(self, payload: BillingInstructionCSPPayload): - sp_token = self._get_sp_token(payload.creds) - if sp_token is None: - raise AuthenticationException( - "Could not resolve token for task order billing validation" - ) - - request_body = { - "properties": { - "amount": payload.initial_clin_amount, - "startDate": payload.initial_clin_start_date, - "endDate": payload.initial_clin_end_date, - } - } - - url = f"https://management.azure.com/providers/Microsoft.Billing/billingAccounts/{payload.billing_account_name}/billingProfiles/{payload.billing_profile_name}/instructions/{payload.initial_task_order_id}:CLIN00{payload.initial_clin_type}?api-version=2019-10-01-preview" - - auth_header = { - "Authorization": f"Bearer {sp_token}", - } - - result = self.sdk.requests.put(url, headers=auth_header, json=request_body) - - if result.status_code == 200: - return self._ok(BillingInstructionCSPResult(**result.json())) - else: - return self._error(result.json()) - - def create_remote_admin(self, creds, tenant_details): - # create app/service principal within tenant, with name constructed from tenant details - # assign principal global admin - - # needs to call out to CLI with tenant owner username/password, prototyping for that underway - - # return identifier and creds to consumer for storage - response = {"clientId": "string", "secretKey": "string", "tenantId": "string"} - return self._ok( - { - "client_id": response["clientId"], - "secret_key": response["secret_key"], - "tenant_id": response["tenantId"], - } - ) - - def force_tenant_admin_pw_update(self, creds, tenant_owner_id): - # use creds to update to force password recovery? - # not sure what the endpoint/method for this is, yet - - return self._ok() - - def create_billing_alerts(self, TBD): - # TODO: Add azure-mgmt-consumption for Budget and Notification entities/operations - # TODO: Determine how to auth against that API using the SDK, doesn't seeem possible at the moment - # TODO: billing alerts are registered as Notifications on Budget objects, which have start/end dates - # TODO: determine what the keys in the Notifications dict are supposed to be - # we may need to rotate budget objects when new TOs/CLINs are reported? - - # we likely only want the budget ID, can be updated or replaced? - response = {"id": "id"} - return self._ok({"budget_id": response["id"]}) - - def _get_management_service_principal(self): - # we really should be using graph.microsoft.com, but i'm getting - # "expired token" errors for that - # graph_resource = "https://graph.microsoft.com" - graph_resource = "https://graph.windows.net" - graph_creds = self._get_credential_obj( - self._root_creds, resource=graph_resource - ) - # I needed to set permissions for the graph.windows.net API before I - # could get this to work. - - # how do we scope the graph client to the new subscription rather than - # the cloud0 subscription? tenant id seems to be separate from subscription id - graph_client = self.sdk.graphrbac.GraphRbacManagementClient( - graph_creds, self._root_creds.get("tenant_id") - ) - - # do we need to create a new application to manage each subscripition - # or should we manage access to each subscription from a single service - # principal with multiple role assignments? - app_display_name = "?" # name should reflect the subscription it exists - app_create_param = self.sdk.graphrbac.models.ApplicationCreateParameters( - display_name=app_display_name - ) - - # we need the appropriate perms here: - # https://docs.microsoft.com/en-us/graph/api/application-post-applications?view=graph-rest-beta&tabs=http - # https://docs.microsoft.com/en-us/graph/permissions-reference#microsoft-graph-permission-names - # set app perms in app registration portal - # https://docs.microsoft.com/en-us/graph/auth-v2-service#2-configure-permissions-for-microsoft-graph - app: self.sdk.graphrbac.models.Application = graph_client.applications.create( - app_create_param - ) - - # create a new service principle for the new application, which should be scoped - # to the new subscription - app_id = app.app_id - sp_create_params = self.sdk.graphrbac.models.ServicePrincipalCreateParameters( - app_id=app_id, account_enabled=True - ) - - service_principal = graph_client.service_principals.create(sp_create_params) - - return service_principal - - def _extract_subscription_id(self, subscription_url): - sub_id_match = SUBSCRIPTION_ID_REGEX.match(subscription_url) - - if sub_id_match: - return sub_id_match.group(1) - - def _get_sp_token(self, creds): - home_tenant_id = creds.get("home_tenant_id") - client_id = creds.get("client_id") - secret_key = creds.get("secret_key") - - # TODO: Make endpoints consts or configs - authentication_endpoint = "https://login.microsoftonline.com/" - resource = "https://management.azure.com/" - - context = self.sdk.adal.AuthenticationContext( - authentication_endpoint + home_tenant_id - ) - - # TODO: handle failure states here - token_response = context.acquire_token_with_client_credentials( - resource, client_id, secret_key - ) - - return token_response.get("accessToken", None) - - def _get_credential_obj(self, creds, resource=None): - return self.sdk.credentials.ServicePrincipalCredentials( - client_id=creds.get("client_id"), - secret=creds.get("secret_key"), - tenant=creds.get("tenant_id"), - resource=resource, - cloud_environment=self.sdk.cloud, - ) - - def _get_client_secret_credential_obj(self, creds): - return self.sdk.identity.ClientSecretCredential( - tenant_id=creds.get("tenant_id"), - client_id=creds.get("client_id"), - client_secret=creds.get("secret_key"), - ) - - def _make_tenant_admin_cred_obj(self, username, password): - return self.sdk.credentials.UserPassCredentials(username, password) - - def _ok(self, body=None): - return self._make_response("ok", body) - - def _error(self, body=None): - return self._make_response("error", body) - - def _make_response(self, status, body=dict()): - """Create body for responses from API - - Arguments: - status {string} -- "ok" or "error" - body {dict} -- dict containing details of response or error, if applicable - - Returns: - dict -- status of call with body containing details - """ - return {"status": status, "body": body} - - @property - def _root_creds(self): - return { - "client_id": self.client_id, - "secret_key": self.secret_key, - "tenant_id": self.tenant_id, - } +from .exceptions import * +from .models import * +from .cloud_provider_interface import CloudProviderInterface +from .mock_cloud_provider import MockCloudProvider +from .azure_cloud_provider import AzureCloudProvider diff --git a/atst/domain/csp/cloud/azure_cloud_provider.py b/atst/domain/csp/cloud/azure_cloud_provider.py new file mode 100644 index 00000000..67303fb7 --- /dev/null +++ b/atst/domain/csp/cloud/azure_cloud_provider.py @@ -0,0 +1,613 @@ +import re +from uuid import uuid4 +from secrets import token_urlsafe + +from .cloud_provider_interface import CloudProviderInterface +from .exceptions import * +from .models import * +from .policy import AzurePolicyManager + +from atst.models.user import User +from atst.models.environment import Environment +from atst.models.application import Application + + +AZURE_ENVIRONMENT = "AZURE_PUBLIC_CLOUD" # TBD +AZURE_SKU_ID = "?" # probably a static sku specific to ATAT/JEDI +SUBSCRIPTION_ID_REGEX = re.compile( + "subscriptions\/([0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})", + re.I, +) + +# This needs to be a fully pathed role definition identifier, not just a UUID +REMOTE_ROOT_ROLE_DEF_ID = "/providers/Microsoft.Authorization/roleDefinitions/00000000-0000-4000-8000-000000000000" +AZURE_MANAGEMENT_API = "https://management.azure.com" + + +class AzureSDKProvider(object): + def __init__(self): + from azure.mgmt import subscription, authorization, managementgroups + from azure.mgmt.resource import policy + import azure.graphrbac as graphrbac + import azure.common.credentials as credentials + import azure.identity as identity + from azure.keyvault import secrets + + from msrestazure.azure_cloud import AZURE_PUBLIC_CLOUD + import adal + import requests + + self.subscription = subscription + self.policy = policy + self.managementgroups = managementgroups + self.authorization = authorization + self.adal = adal + self.graphrbac = graphrbac + self.credentials = credentials + self.identity = identity + self.exceptions = exceptions + self.secrets = secrets + self.requests = requests + # may change to a JEDI cloud + self.cloud = AZURE_PUBLIC_CLOUD + + +class AzureCloudProvider(CloudProviderInterface): + def __init__(self, config, azure_sdk_provider=None): + self.config = config + + self.client_id = config["AZURE_CLIENT_ID"] + self.secret_key = config["AZURE_SECRET_KEY"] + self.tenant_id = config["AZURE_TENANT_ID"] + self.vault_url = config["AZURE_VAULT_URL"] + + if azure_sdk_provider is None: + self.sdk = AzureSDKProvider() + else: + self.sdk = azure_sdk_provider + + self.policy_manager = AzurePolicyManager(config["AZURE_POLICY_LOCATION"]) + + def set_secret(self, secret_key, secret_value): + credential = self._get_client_secret_credential_obj({}) + secret_client = self.secrets.SecretClient( + vault_url=self.vault_url, credential=credential, + ) + try: + return secret_client.set_secret(secret_key, secret_value) + except self.exceptions.HttpResponseError: + app.logger.error( + f"Could not SET secret in Azure keyvault for key {secret_key}.", + exc_info=1, + ) + + def get_secret(self, secret_key): + credential = self._get_client_secret_credential_obj({}) + secret_client = self.secrets.SecretClient( + vault_url=self.vault_url, credential=credential, + ) + try: + return secret_client.get_secret(secret_key).value + except self.exceptions.HttpResponseError: + app.logger.error( + f"Could not GET secret in Azure keyvault for key {secret_key}.", + exc_info=1, + ) + + def create_environment( + self, auth_credentials: Dict, user: User, environment: Environment + ): + # since this operation would only occur within a tenant, should we source the tenant + # via lookup from environment once we've created the portfolio csp data schema + # something like this: + # environment_tenant = environment.application.portfolio.csp_data.get('tenant_id', None) + # though we'd probably source the whole credentials for these calls from the portfolio csp + # data, as it would have to be where we store the creds for the at-at user within the portfolio tenant + # credentials = self._get_credential_obj(environment.application.portfolio.csp_data.get_creds()) + credentials = self._get_credential_obj(self._root_creds) + display_name = f"{environment.application.name}_{environment.name}_{environment.id}" # proposed format + management_group_id = "?" # management group id chained from environment + parent_id = "?" # from environment.application + + management_group = self._create_management_group( + credentials, management_group_id, display_name, parent_id, + ) + + return management_group + + def create_atat_admin_user( + self, auth_credentials: Dict, csp_environment_id: str + ) -> Dict: + root_creds = self._root_creds + credentials = self._get_credential_obj(root_creds) + + sub_client = self.sdk.subscription.SubscriptionClient(credentials) + subscription = sub_client.subscriptions.get(csp_environment_id) + + managment_principal = self._get_management_service_principal() + + auth_client = self.sdk.authorization.AuthorizationManagementClient( + credentials, + # TODO: Determine which subscription this needs to point at + # Once we're in a multi-sub environment + subscription.id, + ) + + # Create role assignment for + role_assignment_id = str(uuid4()) + role_assignment_create_params = auth_client.role_assignments.models.RoleAssignmentCreateParameters( + role_definition_id=REMOTE_ROOT_ROLE_DEF_ID, + principal_id=managment_principal.id, + ) + + auth_client.role_assignments.create( + scope=f"/subscriptions/{subscription.id}/", + role_assignment_name=role_assignment_id, + parameters=role_assignment_create_params, + ) + + return { + "csp_user_id": managment_principal.object_id, + "credentials": managment_principal.password_credentials, + "role_name": role_assignment_id, + } + + def _create_application(self, auth_credentials: Dict, application: Application): + management_group_name = str(uuid4()) # can be anything, not just uuid + display_name = application.name # Does this need to be unique? + credentials = self._get_credential_obj(auth_credentials) + parent_id = "?" # application.portfolio.csp_details.management_group_id + + return self._create_management_group( + credentials, management_group_name, display_name, parent_id, + ) + + def _create_management_group( + self, credentials, management_group_id, display_name, parent_id=None, + ): + mgmgt_group_client = self.sdk.managementgroups.ManagementGroupsAPI(credentials) + create_parent_grp_info = self.sdk.managementgroups.models.CreateParentGroupInfo( + id=parent_id + ) + create_mgmt_grp_details = self.sdk.managementgroups.models.CreateManagementGroupDetails( + parent=create_parent_grp_info + ) + mgmt_grp_create = self.sdk.managementgroups.models.CreateManagementGroupRequest( + name=management_group_id, + display_name=display_name, + details=create_mgmt_grp_details, + ) + create_request = mgmgt_group_client.management_groups.create_or_update( + management_group_id, mgmt_grp_create + ) + + # result is a synchronous wait, might need to do a poll instead to handle first mgmt group create + # since we were told it could take 10+ minutes to complete, unless this handles that polling internally + return create_request.result() + + def _create_subscription( + self, + credentials, + display_name, + billing_profile_id, + sku_id, + management_group_id, + billing_account_name, + invoice_section_name, + ): + sub_client = self.sdk.subscription.SubscriptionClient(credentials) + + billing_profile_id = "?" # where do we source this? + sku_id = AZURE_SKU_ID + # These 2 seem like something that might be worthwhile to allow tiebacks to + # TOs filed for the environment + billing_account_name = "?" # from TO? + invoice_section_name = "?" # from TO? + + body = self.sdk.subscription.models.ModernSubscriptionCreationParameters( + display_name=display_name, + billing_profile_id=billing_profile_id, + sku_id=sku_id, + management_group_id=management_group_id, + ) + + # We may also want to create billing sections in the enrollment account + sub_creation_operation = sub_client.subscription_factory.create_subscription( + billing_account_name, invoice_section_name, body + ) + + # the resulting object from this process is a link to the new subscription + # not a subscription model, so we'll have to unpack the ID + new_sub = sub_creation_operation.result() + + subscription_id = self._extract_subscription_id(new_sub.subscription_link) + if subscription_id: + return subscription_id + else: + # troublesome error, subscription should exist at this point + # but we just don't have a valid ID + pass + + def _create_policy_definition( + self, credentials, subscription_id, management_group_id, properties, + ): + """ + Requires credentials that have AZURE_MANAGEMENT_API + specified as the resource. The Service Principal + specified in the credentials must have the "Resource + Policy Contributor" role assigned with a scope at least + as high as the management group specified by + management_group_id. + + Arguments: + credentials -- ServicePrincipalCredentials + subscription_id -- str, ID of the subscription (just the UUID, not the path) + management_group_id -- str, ID of the management group (just the UUID, not the path) + properties -- dictionary, the "properties" section of a valid Azure policy definition document + + Returns: + azure.mgmt.resource.policy.[api version].models.PolicyDefinition: the PolicyDefinition object provided to Azure + + Raises: + TBD + """ + # TODO: which subscription would this be? + client = self.sdk.policy.PolicyClient(credentials, subscription_id) + + definition = client.policy_definitions.models.PolicyDefinition( + policy_type=properties.get("policyType"), + mode=properties.get("mode"), + display_name=properties.get("displayName"), + description=properties.get("description"), + policy_rule=properties.get("policyRule"), + parameters=properties.get("parameters"), + ) + + name = properties.get("displayName") + + return client.policy_definitions.create_or_update_at_management_group( + policy_definition_name=name, + parameters=definition, + management_group_id=management_group_id, + ) + + def create_tenant(self, payload: TenantCSPPayload): + sp_token = self._get_sp_token(payload.creds) + if sp_token is None: + raise AuthenticationException("Could not resolve token for tenant creation") + payload.password = token_urlsafe(16) + create_tenant_body = payload.dict(by_alias=True) + + create_tenant_headers = { + "Authorization": f"Bearer {sp_token}", + } + + result = self.sdk.requests.post( + "https://management.azure.com/providers/Microsoft.SignUp/createTenant?api-version=2020-01-01-preview", + json=create_tenant_body, + headers=create_tenant_headers, + ) + + if result.status_code == 200: + return self._ok( + TenantCSPResult( + **result.json(), + tenant_admin_password=payload.password, + tenant_admin_username=payload.user_id, + ) + ) + else: + return self._error(result.json()) + + def create_billing_profile_creation( + self, payload: BillingProfileCreationCSPPayload + ): + sp_token = self._get_sp_token(payload.creds) + if sp_token is None: + raise AuthenticationException( + "Could not resolve token for billing profile creation" + ) + + create_billing_account_body = payload.dict(by_alias=True) + + create_billing_account_headers = { + "Authorization": f"Bearer {sp_token}", + } + + billing_account_create_url = f"https://management.azure.com/providers/Microsoft.Billing/billingAccounts/{payload.billing_account_name}/billingProfiles?api-version=2019-10-01-preview" + + result = self.sdk.requests.post( + billing_account_create_url, + json=create_billing_account_body, + headers=create_billing_account_headers, + ) + + if result.status_code == 202: + # 202 has location/retry after headers + return self._ok(BillingProfileCreationCSPResult(**result.headers)) + elif result.status_code == 200: + # NB: Swagger docs imply call can sometimes resolve immediately + return self._ok(BillingProfileVerificationCSPResult(**result.json())) + else: + return self._error(result.json()) + + def create_billing_profile_verification( + self, payload: BillingProfileVerificationCSPPayload + ): + sp_token = self._get_sp_token(payload.creds) + if sp_token is None: + raise AuthenticationException( + "Could not resolve token for billing profile validation" + ) + + auth_header = { + "Authorization": f"Bearer {sp_token}", + } + + result = self.sdk.requests.get( + payload.billing_profile_verify_url, headers=auth_header + ) + + if result.status_code == 202: + # 202 has location/retry after headers + return self._ok(BillingProfileCreationCSPResult(**result.headers)) + elif result.status_code == 200: + return self._ok(BillingProfileVerificationCSPResult(**result.json())) + else: + return self._error(result.json()) + + def create_billing_profile_tenant_access( + self, payload: BillingProfileTenantAccessCSPPayload + ): + sp_token = self._get_sp_token(payload.creds) + request_body = { + "properties": { + "principalTenantId": payload.tenant_id, # from tenant creation + "principalId": payload.user_object_id, # from tenant creationn + "roleDefinitionId": f"/providers/Microsoft.Billing/billingAccounts/{payload.billing_account_name}/billingProfiles/{payload.billing_profile_name}/billingRoleDefinitions/40000000-aaaa-bbbb-cccc-100000000000", + } + } + + headers = { + "Authorization": f"Bearer {sp_token}", + } + + url = f"https://management.azure.com/providers/Microsoft.Billing/billingAccounts/{payload.billing_account_name}/billingProfiles/{payload.billing_profile_name}/createBillingRoleAssignment?api-version=2019-10-01-preview" + + result = self.sdk.requests.post(url, headers=headers, json=request_body) + if result.status_code == 201: + return self._ok(BillingProfileTenantAccessCSPResult(**result.json())) + else: + return self._error(result.json()) + + def create_task_order_billing_creation( + self, payload: TaskOrderBillingCreationCSPPayload + ): + sp_token = self._get_sp_token(payload.creds) + request_body = [ + { + "op": "replace", + "path": "/enabledAzurePlans", + "value": [{"skuId": "0001"}], + } + ] + + request_headers = { + "Authorization": f"Bearer {sp_token}", + } + + url = f"https://management.azure.com/providers/Microsoft.Billing/billingAccounts/{payload.billing_account_name}/billingProfiles/{payload.billing_profile_name}?api-version=2019-10-01-preview" + + result = self.sdk.requests.patch( + url, headers=request_headers, json=request_body + ) + + if result.status_code == 202: + # 202 has location/retry after headers + return self._ok(TaskOrderBillingCreationCSPResult(**result.headers)) + elif result.status_code == 200: + return self._ok(TaskOrderBillingVerificationCSPResult(**result.json())) + else: + return self._error(result.json()) + + def create_task_order_billing_verification( + self, payload: TaskOrderBillingVerificationCSPPayload + ): + sp_token = self._get_sp_token(payload.creds) + if sp_token is None: + raise AuthenticationException( + "Could not resolve token for task order billing validation" + ) + + auth_header = { + "Authorization": f"Bearer {sp_token}", + } + + result = self.sdk.requests.get( + payload.task_order_billing_verify_url, headers=auth_header + ) + + if result.status_code == 202: + # 202 has location/retry after headers + return self._ok(TaskOrderBillingCreationCSPResult(**result.headers)) + elif result.status_code == 200: + return self._ok(TaskOrderBillingVerificationCSPResult(**result.json())) + else: + return self._error(result.json()) + + def create_billing_instruction(self, payload: BillingInstructionCSPPayload): + sp_token = self._get_sp_token(payload.creds) + if sp_token is None: + raise AuthenticationException( + "Could not resolve token for task order billing validation" + ) + + request_body = { + "properties": { + "amount": payload.initial_clin_amount, + "startDate": payload.initial_clin_start_date, + "endDate": payload.initial_clin_end_date, + } + } + + url = f"https://management.azure.com/providers/Microsoft.Billing/billingAccounts/{payload.billing_account_name}/billingProfiles/{payload.billing_profile_name}/instructions/{payload.initial_task_order_id}:CLIN00{payload.initial_clin_type}?api-version=2019-10-01-preview" + + auth_header = { + "Authorization": f"Bearer {sp_token}", + } + + result = self.sdk.requests.put(url, headers=auth_header, json=request_body) + + if result.status_code == 200: + return self._ok(BillingInstructionCSPResult(**result.json())) + else: + return self._error(result.json()) + + def create_remote_admin(self, creds, tenant_details): + # create app/service principal within tenant, with name constructed from tenant details + # assign principal global admin + + # needs to call out to CLI with tenant owner username/password, prototyping for that underway + + # return identifier and creds to consumer for storage + response = {"clientId": "string", "secretKey": "string", "tenantId": "string"} + return self._ok( + { + "client_id": response["clientId"], + "secret_key": response["secret_key"], + "tenant_id": response["tenantId"], + } + ) + + def force_tenant_admin_pw_update(self, creds, tenant_owner_id): + # use creds to update to force password recovery? + # not sure what the endpoint/method for this is, yet + + return self._ok() + + def create_billing_alerts(self, TBD): + # TODO: Add azure-mgmt-consumption for Budget and Notification entities/operations + # TODO: Determine how to auth against that API using the SDK, doesn't seeem possible at the moment + # TODO: billing alerts are registered as Notifications on Budget objects, which have start/end dates + # TODO: determine what the keys in the Notifications dict are supposed to be + # we may need to rotate budget objects when new TOs/CLINs are reported? + + # we likely only want the budget ID, can be updated or replaced? + response = {"id": "id"} + return self._ok({"budget_id": response["id"]}) + + def _get_management_service_principal(self): + # we really should be using graph.microsoft.com, but i'm getting + # "expired token" errors for that + # graph_resource = "https://graph.microsoft.com" + graph_resource = "https://graph.windows.net" + graph_creds = self._get_credential_obj( + self._root_creds, resource=graph_resource + ) + # I needed to set permissions for the graph.windows.net API before I + # could get this to work. + + # how do we scope the graph client to the new subscription rather than + # the cloud0 subscription? tenant id seems to be separate from subscription id + graph_client = self.sdk.graphrbac.GraphRbacManagementClient( + graph_creds, self._root_creds.get("tenant_id") + ) + + # do we need to create a new application to manage each subscripition + # or should we manage access to each subscription from a single service + # principal with multiple role assignments? + app_display_name = "?" # name should reflect the subscription it exists + app_create_param = self.sdk.graphrbac.models.ApplicationCreateParameters( + display_name=app_display_name + ) + + # we need the appropriate perms here: + # https://docs.microsoft.com/en-us/graph/api/application-post-applications?view=graph-rest-beta&tabs=http + # https://docs.microsoft.com/en-us/graph/permissions-reference#microsoft-graph-permission-names + # set app perms in app registration portal + # https://docs.microsoft.com/en-us/graph/auth-v2-service#2-configure-permissions-for-microsoft-graph + app: self.sdk.graphrbac.models.Application = graph_client.applications.create( + app_create_param + ) + + # create a new service principle for the new application, which should be scoped + # to the new subscription + app_id = app.app_id + sp_create_params = self.sdk.graphrbac.models.ServicePrincipalCreateParameters( + app_id=app_id, account_enabled=True + ) + + service_principal = graph_client.service_principals.create(sp_create_params) + + return service_principal + + def _extract_subscription_id(self, subscription_url): + sub_id_match = SUBSCRIPTION_ID_REGEX.match(subscription_url) + + if sub_id_match: + return sub_id_match.group(1) + + def _get_sp_token(self, creds): + home_tenant_id = creds.get("home_tenant_id") + client_id = creds.get("client_id") + secret_key = creds.get("secret_key") + + # TODO: Make endpoints consts or configs + authentication_endpoint = "https://login.microsoftonline.com/" + resource = "https://management.azure.com/" + + context = self.sdk.adal.AuthenticationContext( + authentication_endpoint + home_tenant_id + ) + + # TODO: handle failure states here + token_response = context.acquire_token_with_client_credentials( + resource, client_id, secret_key + ) + + return token_response.get("accessToken", None) + + def _get_credential_obj(self, creds, resource=None): + return self.sdk.credentials.ServicePrincipalCredentials( + client_id=creds.get("client_id"), + secret=creds.get("secret_key"), + tenant=creds.get("tenant_id"), + resource=resource, + cloud_environment=self.sdk.cloud, + ) + + def _get_client_secret_credential_obj(self, creds): + return self.sdk.identity.ClientSecretCredential( + tenant_id=creds.get("tenant_id"), + client_id=creds.get("client_id"), + client_secret=creds.get("secret_key"), + ) + + def _make_tenant_admin_cred_obj(self, username, password): + return self.sdk.credentials.UserPassCredentials(username, password) + + def _ok(self, body=None): + return self._make_response("ok", body) + + def _error(self, body=None): + return self._make_response("error", body) + + def _make_response(self, status, body=dict()): + """Create body for responses from API + + Arguments: + status {string} -- "ok" or "error" + body {dict} -- dict containing details of response or error, if applicable + + Returns: + dict -- status of call with body containing details + """ + return {"status": status, "body": body} + + @property + def _root_creds(self): + return { + "client_id": self.client_id, + "secret_key": self.secret_key, + "tenant_id": self.tenant_id, + } diff --git a/atst/domain/csp/cloud/cloud_provider_interface.py b/atst/domain/csp/cloud/cloud_provider_interface.py new file mode 100644 index 00000000..7f975c07 --- /dev/null +++ b/atst/domain/csp/cloud/cloud_provider_interface.py @@ -0,0 +1,126 @@ +from typing import Dict + +from atst.models.user import User +from atst.models.environment import Environment +from atst.models.environment_role import EnvironmentRole + + +class CloudProviderInterface: + def set_secret(self, secret_key: str, secret_value: str): + raise NotImplementedError() + + def get_secret(self, secret_key: str): + raise NotImplementedError() + + def root_creds(self) -> Dict: + raise NotImplementedError() + + def create_environment( + self, auth_credentials: Dict, user: User, environment: Environment + ) -> str: + """Create a new environment in the CSP. + + Arguments: + auth_credentials -- Object containing CSP account credentials + user -- ATAT user authorizing the environment creation + environment -- ATAT Environment model + + Returns: + string: ID of created environment + + Raises: + AuthenticationException: Problem with the credentials + AuthorizationException: Credentials not authorized for current action(s) + ConnectionException: Issue with the CSP API connection + UnknownServerException: Unknown issue on the CSP side + EnvironmentExistsException: Environment already exists and has been created + """ + raise NotImplementedError() + + def create_atat_admin_user( + self, auth_credentials: Dict, csp_environment_id: str + ) -> Dict: + """Creates a new, programmatic user in the CSP. Grants this user full permissions to administer + the CSP. + + Arguments: + auth_credentials -- Object containing CSP account credentials + csp_environment_id -- ID of the CSP Environment the admin user should be created in + + Returns: + object: Object representing new remote admin user, including credentials + Something like: + { + "user_id": string, + "credentials": dict, # structure TBD based on csp + } + + Raises: + AuthenticationException: Problem with the credentials + AuthorizationException: Credentials not authorized for current action(s) + ConnectionException: Issue with the CSP API connection + UnknownServerException: Unknown issue on the CSP side + UserProvisioningException: Problem creating the root user + """ + raise NotImplementedError() + + def create_or_update_user( + self, auth_credentials: Dict, user_info: EnvironmentRole, csp_role_id: str + ) -> str: + """Creates a user or updates an existing user's role. + + Arguments: + auth_credentials -- Object containing CSP account credentials + user_info -- instance of EnvironmentRole containing user data + if it has a csp_user_id it will try to update that user + csp_role_id -- The id of the role the user should be given in the CSP + + Returns: + string: Returns the interal csp_user_id of the created/updated user account + + Raises: + AuthenticationException: Problem with the credentials + AuthorizationException: Credentials not authorized for current action(s) + ConnectionException: Issue with the CSP API connection + UnknownServerException: Unknown issue on the CSP side + UserProvisioningException: User couldn't be created or modified + """ + raise NotImplementedError() + + def disable_user(self, auth_credentials: Dict, csp_user_id: str) -> bool: + """Revoke all privileges for a user. Used to prevent user access while a full + delete is being processed. + + Arguments: + auth_credentials -- Object containing CSP account credentials + csp_user_id -- CSP internal user identifier + + Returns: + bool -- True on success + + Raises: + AuthenticationException: Problem with the credentials + AuthorizationException: Credentials not authorized for current action(s) + ConnectionException: Issue with the CSP API connection + UnknownServerException: Unknown issue on the CSP side + UserRemovalException: User couldn't be suspended + """ + raise NotImplementedError() + + def get_calculator_url(self) -> str: + """Returns the calculator url for the CSP. + This will likely be a static property elsewhere once a CSP is chosen. + """ + raise NotImplementedError() + + def get_environment_login_url(self, environment) -> str: + """Returns the login url for a given environment + This may move to be a computed property on the Environment domain object + """ + raise NotImplementedError() + + def create_subscription(self, environment): + """Returns True if a new subscription has been created or raises an + exception if an error occurs while creating a subscription. + """ + raise NotImplementedError() diff --git a/atst/domain/csp/cloud/exceptions.py b/atst/domain/csp/cloud/exceptions.py new file mode 100644 index 00000000..6ed47dff --- /dev/null +++ b/atst/domain/csp/cloud/exceptions.py @@ -0,0 +1,131 @@ +class GeneralCSPException(Exception): + pass + + +class OperationInProgressException(GeneralCSPException): + """Throw this for instances when the CSP reports that the current entity is already + being operated on/created/deleted/etc + """ + + def __init__(self, operation_desc): + self.operation_desc = operation_desc + + @property + def message(self): + return "An operation for this entity is already in progress: {}".format( + self.operation_desc + ) + + +class AuthenticationException(GeneralCSPException): + """Throw this for instances when there is a problem with the auth credentials: + * Missing credentials + * Incorrect credentials + * Other credential problems + """ + + def __init__(self, auth_error): + self.auth_error = auth_error + + @property + def message(self): + return "An error occurred with authentication: {}".format(self.auth_error) + + +class AuthorizationException(GeneralCSPException): + """Throw this for instances when the current credentials are not authorized + for the current action. + """ + + def __init__(self, auth_error): + self.auth_error = auth_error + + @property + def message(self): + return "An error occurred with authorization: {}".format(self.auth_error) + + +class ConnectionException(GeneralCSPException): + """A general problem with the connection, timeouts or unresolved endpoints + """ + + def __init__(self, connection_error): + self.connection_error = connection_error + + @property + def message(self): + return "Could not connect to cloud provider: {}".format(self.connection_error) + + +class UnknownServerException(GeneralCSPException): + """An error occured on the CSP side (5xx) and we don't know why + """ + + def __init__(self, server_error): + self.server_error = server_error + + @property + def message(self): + return "A server error occured: {}".format(self.server_error) + + +class EnvironmentCreationException(GeneralCSPException): + """If there was an error in creating the environment + """ + + def __init__(self, env_identifier, reason): + self.env_identifier = env_identifier + self.reason = reason + + @property + def message(self): + return "The envionment {} couldn't be created: {}".format( + self.env_identifier, self.reason + ) + + +class UserProvisioningException(GeneralCSPException): + """Failed to provision a user + """ + + def __init__(self, env_identifier, user_identifier, reason): + self.env_identifier = env_identifier + self.user_identifier = user_identifier + self.reason = reason + + @property + def message(self): + return "Failed to create user {} for environment {}: {}".format( + self.user_identifier, self.env_identifier, self.reason + ) + + +class UserRemovalException(GeneralCSPException): + """Failed to remove a user + """ + + def __init__(self, user_csp_id, reason): + self.user_csp_id = user_csp_id + self.reason = reason + + @property + def message(self): + return "Failed to suspend or delete user {}: {}".format( + self.user_csp_id, self.reason + ) + + +class BaselineProvisionException(GeneralCSPException): + """If there's any issues standing up whatever is required + for an environment baseline + """ + + def __init__(self, env_identifier, reason): + self.env_identifier = env_identifier + self.reason = reason + + @property + def message(self): + return "Could not complete baseline provisioning for environment ({}): {}".format( + self.env_identifier, self.reason + ) diff --git a/atst/domain/csp/cloud/mock_cloud_provider.py b/atst/domain/csp/cloud/mock_cloud_provider.py new file mode 100644 index 00000000..153c5a9d --- /dev/null +++ b/atst/domain/csp/cloud/mock_cloud_provider.py @@ -0,0 +1,315 @@ +from uuid import uuid4 + +from .cloud_provider_interface import CloudProviderInterface +from .exceptions import * +from .models import * + + +class MockCloudProvider(CloudProviderInterface): + + # TODO: All of these constants + AUTHENTICATION_EXCEPTION = AuthenticationException("Authentication failure.") + AUTHORIZATION_EXCEPTION = AuthorizationException("Not authorized.") + NETWORK_EXCEPTION = ConnectionException("Network failure.") + SERVER_EXCEPTION = UnknownServerException("Not our fault.") + + SERVER_FAILURE_PCT = 1 + NETWORK_FAILURE_PCT = 7 + ENV_CREATE_FAILURE_PCT = 12 + ATAT_ADMIN_CREATE_FAILURE_PCT = 12 + UNAUTHORIZED_RATE = 2 + + def __init__( + self, config, with_delay=True, with_failure=True, with_authorization=True + ): + from time import sleep + import random + + self._with_delay = with_delay + self._with_failure = with_failure + self._with_authorization = with_authorization + self._sleep = sleep + self._random = random + + def root_creds(self): + return self._auth_credentials + + def set_secret(self, secret_key: str, secret_value: str): + pass + + def get_secret(self, secret_key: str, default=dict()): + return default + + def create_environment(self, auth_credentials, user, environment): + self._authorize(auth_credentials) + + self._delay(1, 5) + self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) + self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) + self._maybe_raise( + self.ENV_CREATE_FAILURE_PCT, + EnvironmentCreationException( + environment.id, "Could not create environment." + ), + ) + + csp_environment_id = self._id() + + self._delay(1, 5) + self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) + self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) + self._maybe_raise( + self.ATAT_ADMIN_CREATE_FAILURE_PCT, + BaselineProvisionException( + csp_environment_id, "Could not create environment baseline." + ), + ) + self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) + + return csp_environment_id + + def create_atat_admin_user(self, auth_credentials, csp_environment_id): + self._authorize(auth_credentials) + + self._delay(1, 5) + self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) + self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) + self._maybe_raise( + self.ATAT_ADMIN_CREATE_FAILURE_PCT, + UserProvisioningException( + csp_environment_id, "atat_admin", "Could not create admin user." + ), + ) + + self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) + + return {"id": self._id(), "credentials": self._auth_credentials} + + def create_tenant(self, payload: TenantCSPPayload): + """ + payload is an instance of TenantCSPPayload data class + """ + + self._authorize(payload.creds) + + self._delay(1, 5) + + self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) + self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) + self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) + + return TenantCSPResult( + **{ + "tenant_id": "", + "user_id": "", + "user_object_id": "", + "tenant_admin_username": "test", + "tenant_admin_password": "test", + } + ) + + def create_billing_profile_creation( + self, payload: BillingProfileCreationCSPPayload + ): + # response will be mostly the same as the body, but we only really care about the id + self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) + self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) + self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) + + return BillingProfileCreationCSPResult( + **dict( + billing_profile_verify_url="https://zombo.com", + billing_profile_retry_after=10, + ) + ) + + def create_billing_profile_verification( + self, payload: BillingProfileVerificationCSPPayload + ): + self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) + self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) + self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) + return BillingProfileVerificationCSPResult( + **{ + "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB", + "name": "KQWI-W2SU-BG7-TGB", + "properties": { + "address": { + "addressLine1": "123 S Broad Street, Suite 2400", + "city": "Philadelphia", + "companyName": "Promptworks", + "country": "US", + "postalCode": "19109", + "region": "PA", + }, + "currency": "USD", + "displayName": "Test Billing Profile", + "enabledAzurePlans": [], + "hasReadAccess": True, + "invoiceDay": 5, + "invoiceEmailOptIn": False, + "invoiceSections": [ + { + "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB/invoiceSections/CHCO-BAAR-PJA-TGB", + "name": "CHCO-BAAR-PJA-TGB", + "properties": {"displayName": "Test Billing Profile"}, + "type": "Microsoft.Billing/billingAccounts/billingProfiles/invoiceSections", + } + ], + }, + "type": "Microsoft.Billing/billingAccounts/billingProfiles", + } + ) + + def create_billing_profile_tenant_access(self, payload): + self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) + self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) + self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) + + return BillingProfileTenantAccessCSPResult( + **{ + "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB/billingRoleAssignments/40000000-aaaa-bbbb-cccc-100000000000_0a5f4926-e3ee-4f47-a6e3-8b0a30a40e3d", + "name": "40000000-aaaa-bbbb-cccc-100000000000_0a5f4926-e3ee-4f47-a6e3-8b0a30a40e3d", + "properties": { + "createdOn": "2020-01-14T14:39:26.3342192+00:00", + "createdByPrincipalId": "82e2b376-3297-4096-8743-ed65b3be0b03", + "principalId": "0a5f4926-e3ee-4f47-a6e3-8b0a30a40e3d", + "principalTenantId": "60ff9d34-82bf-4f21-b565-308ef0533435", + "roleDefinitionId": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB/billingRoleDefinitions/40000000-aaaa-bbbb-cccc-100000000000", + "scope": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB", + }, + "type": "Microsoft.Billing/billingRoleAssignments", + } + ) + + def create_task_order_billing_creation( + self, payload: TaskOrderBillingCreationCSPPayload + ): + self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) + self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) + self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) + + return TaskOrderBillingCreationCSPResult( + **{"Location": "https://somelocation", "Retry-After": "10"} + ) + + def create_task_order_billing_verification( + self, payload: TaskOrderBillingVerificationCSPPayload + ): + self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) + self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) + self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) + + return TaskOrderBillingVerificationCSPResult( + **{ + "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/XC36-GRNZ-BG7-TGB", + "name": "XC36-GRNZ-BG7-TGB", + "properties": { + "address": { + "addressLine1": "123 S Broad Street, Suite 2400", + "city": "Philadelphia", + "companyName": "Promptworks", + "country": "US", + "postalCode": "19109", + "region": "PA", + }, + "currency": "USD", + "displayName": "First Portfolio Billing Profile", + "enabledAzurePlans": [ + { + "productId": "DZH318Z0BPS6", + "skuId": "0001", + "skuDescription": "Microsoft Azure Plan", + } + ], + "hasReadAccess": True, + "invoiceDay": 5, + "invoiceEmailOptIn": False, + }, + "type": "Microsoft.Billing/billingAccounts/billingProfiles", + } + ) + + def create_billing_instruction(self, payload: BillingInstructionCSPPayload): + self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) + self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) + self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) + + return BillingInstructionCSPResult( + **{ + "name": "TO1:CLIN001", + "properties": { + "amount": 1000.0, + "endDate": "2020-03-01T00:00:00+00:00", + "startDate": "2020-01-01T00:00:00+00:00", + }, + "type": "Microsoft.Billing/billingAccounts/billingProfiles/billingInstructions", + } + ) + + def create_or_update_user(self, auth_credentials, user_info, csp_role_id): + self._authorize(auth_credentials) + + self._delay(1, 5) + self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) + self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) + self._maybe_raise( + self.ATAT_ADMIN_CREATE_FAILURE_PCT, + UserProvisioningException( + user_info.environment.id, + user_info.application_role.user_id, + "Could not create user.", + ), + ) + + self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) + return self._id() + + def disable_user(self, auth_credentials, csp_user_id): + self._authorize(auth_credentials) + self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) + self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) + + self._maybe_raise( + self.ATAT_ADMIN_CREATE_FAILURE_PCT, + UserRemovalException(csp_user_id, "Could not disable user."), + ) + + return self._maybe(12) + + def create_subscription(self, environment): + self._maybe_raise(self.UNAUTHORIZED_RATE, GeneralCSPException) + + return True + + def get_calculator_url(self): + return "https://www.rackspace.com/en-us/calculator" + + def get_environment_login_url(self, environment): + """Returns the login url for a given environment + """ + return "https://www.mycloud.com/my-env-login" + + def _id(self): + return uuid4().hex + + def _delay(self, min_secs, max_secs): + if self._with_delay: + duration = self._random.randrange(min_secs, max_secs) + self._sleep(duration) + + def _maybe(self, pct): + return not self._with_failure or self._random.randrange(0, 100) < pct + + def _maybe_raise(self, pct, exc): + if self._with_failure and self._maybe(pct): + raise exc + + @property + def _auth_credentials(self): + return {"username": "mock-cloud", "password": "shh"} # pragma: allowlist secret + + def _authorize(self, credentials): + self._delay(1, 5) + if self._with_authorization and credentials != self._auth_credentials: + raise self.AUTHENTICATION_EXCEPTION diff --git a/atst/domain/csp/cloud/models.py b/atst/domain/csp/cloud/models.py new file mode 100644 index 00000000..c6bf0ede --- /dev/null +++ b/atst/domain/csp/cloud/models.py @@ -0,0 +1,234 @@ +from typing import Dict, List, Optional + +from pydantic import BaseModel, validator + +from atst.utils import snake_to_camel + + +class AliasModel(BaseModel): + """ + This provides automatic camel <-> snake conversion for serializing to/from json + You can override the alias generation in subclasses by providing a Config that defines + a fields property with a dict mapping variables to their cast names, for cases like: + * some_url:someURL + * user_object_id:objectId + """ + + class Config: + alias_generator = snake_to_camel + allow_population_by_field_name = True + + +class BaseCSPPayload(AliasModel): + # {"username": "mock-cloud", "pass": "shh"} + creds: Dict + + def dict(self, *args, **kwargs): + exclude = {"creds"} + if "exclude" not in kwargs: + kwargs["exclude"] = exclude + else: + kwargs["exclude"].update(exclude) + + return super().dict(*args, **kwargs) + + +class TenantCSPPayload(BaseCSPPayload): + user_id: str + password: Optional[str] + domain_name: str + first_name: str + last_name: str + country_code: str + password_recovery_email_address: str + + +class TenantCSPResult(AliasModel): + user_id: str + tenant_id: str + user_object_id: str + + tenant_admin_username: Optional[str] + tenant_admin_password: Optional[str] + + class Config: + fields = { + "user_object_id": "objectId", + } + + def dict(self, *args, **kwargs): + exclude = {"tenant_admin_username", "tenant_admin_password"} + if "exclude" not in kwargs: + kwargs["exclude"] = exclude + else: + kwargs["exclude"].update(exclude) + + return super().dict(*args, **kwargs) + + def get_creds(self): + return { + "tenant_admin_username": self.tenant_admin_username, + "tenant_admin_password": self.tenant_admin_password, + "tenant_id": self.tenant_id, + } + + +class BillingProfileAddress(AliasModel): + company_name: str + address_line_1: str + city: str + region: str + country: str + postal_code: str + + +class BillingProfileCLINBudget(AliasModel): + clin_budget: Dict + """ + "clinBudget": { + "amount": 0, + "startDate": "2019-12-18T16:47:40.909Z", + "endDate": "2019-12-18T16:47:40.909Z", + "externalReferenceId": "string" + } + """ + + +class BillingProfileCreationCSPPayload(BaseCSPPayload): + tenant_id: str + billing_profile_display_name: str + billing_account_name: str + enabled_azure_plans: Optional[List[str]] + address: BillingProfileAddress + + @validator("enabled_azure_plans", pre=True, always=True) + def default_enabled_azure_plans(cls, v): + """ + Normally you'd implement this by setting the field with a value of: + dataclasses.field(default_factory=list) + but that prevents the object from being correctly pickled, so instead we need + to rely on a validator to ensure this has an empty value when not specified + """ + return v or [] + + class Config: + fields = {"billing_profile_display_name": "displayName"} + + +class BillingProfileCreationCSPResult(AliasModel): + billing_profile_verify_url: str + billing_profile_retry_after: int + + class Config: + fields = { + "billing_profile_verify_url": "Location", + "billing_profile_retry_after": "Retry-After", + } + + +class BillingProfileVerificationCSPPayload(BaseCSPPayload): + billing_profile_verify_url: str + + +class BillingInvoiceSection(AliasModel): + invoice_section_id: str + invoice_section_name: str + + class Config: + fields = {"invoice_section_id": "id", "invoice_section_name": "name"} + + +class BillingProfileProperties(AliasModel): + address: BillingProfileAddress + billing_profile_display_name: str + invoice_sections: List[BillingInvoiceSection] + + class Config: + fields = {"billing_profile_display_name": "displayName"} + + +class BillingProfileVerificationCSPResult(AliasModel): + billing_profile_id: str + billing_profile_name: str + billing_profile_properties: BillingProfileProperties + + class Config: + fields = { + "billing_profile_id": "id", + "billing_profile_name": "name", + "billing_profile_properties": "properties", + } + + +class BillingProfileTenantAccessCSPPayload(BaseCSPPayload): + tenant_id: str + user_object_id: str + billing_account_name: str + billing_profile_name: str + + +class BillingProfileTenantAccessCSPResult(AliasModel): + billing_role_assignment_id: str + billing_role_assignment_name: str + + class Config: + fields = { + "billing_role_assignment_id": "id", + "billing_role_assignment_name": "name", + } + + +class TaskOrderBillingCreationCSPPayload(BaseCSPPayload): + billing_account_name: str + billing_profile_name: str + + +class TaskOrderBillingCreationCSPResult(AliasModel): + task_order_billing_verify_url: str + task_order_retry_after: int + + class Config: + fields = { + "task_order_billing_verify_url": "Location", + "task_order_retry_after": "Retry-After", + } + + +class TaskOrderBillingVerificationCSPPayload(BaseCSPPayload): + task_order_billing_verify_url: str + + +class BillingProfileEnabledPlanDetails(AliasModel): + enabled_azure_plans: List[Dict] + + +class TaskOrderBillingVerificationCSPResult(AliasModel): + billing_profile_id: str + billing_profile_name: str + billing_profile_enabled_plan_details: BillingProfileEnabledPlanDetails + + class Config: + fields = { + "billing_profile_id": "id", + "billing_profile_name": "name", + "billing_profile_enabled_plan_details": "properties", + } + + +class BillingInstructionCSPPayload(BaseCSPPayload): + initial_clin_amount: float + initial_clin_start_date: str + initial_clin_end_date: str + initial_clin_type: str + initial_task_order_id: str + billing_account_name: str + billing_profile_name: str + + +class BillingInstructionCSPResult(AliasModel): + reported_clin_name: str + + class Config: + fields = { + "reported_clin_name": "name", + } From 466a575229f68c2dd46b00f4d3c6293c2f2c4d72 Mon Sep 17 00:00:00 2001 From: tomdds Date: Sun, 26 Jan 2020 15:17:53 -0500 Subject: [PATCH 44/46] Move portfolio state machine helpers directly to model file to prevent import issues. Having `get_stage_csp_class` in the csp module meant that any file that interacted with that import path would throw an error in a REPL. This will allow importing of the Azure and Mock providers for interactive dev. --- atst/domain/csp/__init__.py | 19 ------------ .../csp/cloud/cloud_provider_interface.py | 1 + atst/models/portfolio_state_machine.py | 30 ++++++++++++++----- tests/domain/test_portfolio_state_machine.py | 2 +- 4 files changed, 25 insertions(+), 27 deletions(-) diff --git a/atst/domain/csp/__init__.py b/atst/domain/csp/__init__.py index f15ac1cd..d886f8a2 100644 --- a/atst/domain/csp/__init__.py +++ b/atst/domain/csp/__init__.py @@ -1,5 +1,3 @@ -import importlib - from .cloud import MockCloudProvider from .file_uploads import AzureUploader, MockUploader from .reports import MockReportingProvider @@ -31,20 +29,3 @@ def make_csp_provider(app, csp=None): app.csp = MockCSP(app, test_mode=True) else: app.csp = MockCSP(app) - - -def _stage_to_classname(stage): - return "".join(map(lambda word: word.capitalize(), stage.split("_"))) - - -def get_stage_csp_class(stage, class_type): - """ - given a stage name and class_type return the class - class_type is either 'payload' or 'result' - - """ - cls_name = f"{_stage_to_classname(stage)}CSP{class_type.capitalize()}" - try: - return getattr(importlib.import_module("atst.domain.csp.cloud"), cls_name) - except AttributeError: - print("could not import CSP Result class <%s>" % cls_name) diff --git a/atst/domain/csp/cloud/cloud_provider_interface.py b/atst/domain/csp/cloud/cloud_provider_interface.py index 7f975c07..087aada0 100644 --- a/atst/domain/csp/cloud/cloud_provider_interface.py +++ b/atst/domain/csp/cloud/cloud_provider_interface.py @@ -124,3 +124,4 @@ class CloudProviderInterface: exception if an error occurs while creating a subscription. """ raise NotImplementedError() + diff --git a/atst/models/portfolio_state_machine.py b/atst/models/portfolio_state_machine.py index 1390ceb1..cf42710b 100644 --- a/atst/models/portfolio_state_machine.py +++ b/atst/models/portfolio_state_machine.py @@ -1,3 +1,5 @@ +import importlib + from sqlalchemy import Column, ForeignKey, Enum as SQLAEnum from sqlalchemy.orm import relationship, reconstructor from sqlalchemy.dialects.postgresql import UUID @@ -8,8 +10,7 @@ from transitions.extensions.states import add_state_features, Tags from flask import current_app as app -from atst.domain.csp.cloud import ConnectionException, UnknownServerException -from atst.domain.csp import MockCSP, AzureCSP, get_stage_csp_class +from atst.domain.csp.cloud.exceptions import ConnectionException, UnknownServerException from atst.database import db from atst.models.types import Id from atst.models.base import Base @@ -17,6 +18,25 @@ import atst.models.mixins as mixins from atst.models.mixins.state_machines import FSMStates, AzureStages, _build_transitions +def _stage_to_classname(stage): + return "".join(map(lambda word: word.capitalize(), stage.split("_"))) + + +def get_stage_csp_class(stage, class_type): + """ + given a stage name and class_type return the class + class_type is either 'payload' or 'result' + + """ + cls_name = f"{_stage_to_classname(stage)}CSP{class_type.capitalize()}" + try: + return getattr( + importlib.import_module("atst.domain.csp.cloud.models"), cls_name + ) + except AttributeError: + print("could not import CSP Result class <%s>" % cls_name) + + @add_state_features(Tags) class StateMachineWithTags(Machine): pass @@ -138,11 +158,7 @@ class PortfolioStateMachine( self.fail_stage(stage) # TODO: Determine best place to do this, maybe @reconstructor - csp = event.kwargs.get("csp") - if csp is not None: - self.csp = AzureCSP(app).cloud - else: - self.csp = MockCSP(app).cloud + self.csp = app.csp.cloud try: func_name = f"create_{stage}" diff --git a/tests/domain/test_portfolio_state_machine.py b/tests/domain/test_portfolio_state_machine.py index 97056ae7..2e412653 100644 --- a/tests/domain/test_portfolio_state_machine.py +++ b/tests/domain/test_portfolio_state_machine.py @@ -10,7 +10,7 @@ from tests.factories import ( from atst.models import FSMStates, PortfolioStateMachine, TaskOrder from atst.models.mixins.state_machines import AzureStages, StageStates, compose_state from atst.models.portfolio import Portfolio -from atst.domain.csp import get_stage_csp_class +from atst.models.portfolio_state_machine import get_stage_csp_class # TODO: Write failure case tests From 5b0a523e921585070ac29c1a8ad997d05824c385 Mon Sep 17 00:00:00 2001 From: tomdds Date: Sun, 26 Jan 2020 15:52:55 -0500 Subject: [PATCH 45/46] Use more specific imports for cloud models and exceptions. --- atst/domain/csp/cloud/__init__.py | 4 +-- .../csp/cloud/cloud_provider_interface.py | 1 - atst/domain/csp/cloud/mock_cloud_provider.py | 31 +++++++++++++++++-- atst/jobs.py | 3 +- atst/routes/applications/settings.py | 2 +- tests/domain/cloud/test_azure_csp.py | 20 ++++++------ tests/routes/applications/test_settings.py | 2 +- 7 files changed, 43 insertions(+), 20 deletions(-) diff --git a/atst/domain/csp/cloud/__init__.py b/atst/domain/csp/cloud/__init__.py index fc754c7d..99128d9c 100644 --- a/atst/domain/csp/cloud/__init__.py +++ b/atst/domain/csp/cloud/__init__.py @@ -1,5 +1,3 @@ -from .exceptions import * -from .models import * +from .azure_cloud_provider import AzureCloudProvider from .cloud_provider_interface import CloudProviderInterface from .mock_cloud_provider import MockCloudProvider -from .azure_cloud_provider import AzureCloudProvider diff --git a/atst/domain/csp/cloud/cloud_provider_interface.py b/atst/domain/csp/cloud/cloud_provider_interface.py index 087aada0..7f975c07 100644 --- a/atst/domain/csp/cloud/cloud_provider_interface.py +++ b/atst/domain/csp/cloud/cloud_provider_interface.py @@ -124,4 +124,3 @@ class CloudProviderInterface: exception if an error occurs while creating a subscription. """ raise NotImplementedError() - diff --git a/atst/domain/csp/cloud/mock_cloud_provider.py b/atst/domain/csp/cloud/mock_cloud_provider.py index 153c5a9d..a6c338b5 100644 --- a/atst/domain/csp/cloud/mock_cloud_provider.py +++ b/atst/domain/csp/cloud/mock_cloud_provider.py @@ -1,8 +1,35 @@ from uuid import uuid4 +from atst.domain.csp.cloud.exceptions import ( + BaselineProvisionException, + EnvironmentCreationException, + GeneralCSPException, + UserProvisioningException, + UserRemovalException, +) +from atst.domain.csp.cloud.models import BillingProfileTenantAccessCSPResult + from .cloud_provider_interface import CloudProviderInterface -from .exceptions import * -from .models import * +from .exceptions import ( + AuthenticationException, + AuthorizationException, + ConnectionException, + UnknownServerException, +) +from .models import ( + BillingInstructionCSPPayload, + BillingInstructionCSPResult, + BillingProfileCreationCSPPayload, + BillingProfileCreationCSPResult, + BillingProfileVerificationCSPPayload, + BillingProfileVerificationCSPResult, + TaskOrderBillingCreationCSPPayload, + TaskOrderBillingCreationCSPResult, + TaskOrderBillingVerificationCSPPayload, + TaskOrderBillingVerificationCSPResult, + TenantCSPPayload, + TenantCSPResult, +) class MockCloudProvider(CloudProviderInterface): diff --git a/atst/jobs.py b/atst/jobs.py index ab52cf17..7172343b 100644 --- a/atst/jobs.py +++ b/atst/jobs.py @@ -9,7 +9,8 @@ from atst.models import ( EnvironmentRole, PortfolioJobFailure, ) -from atst.domain.csp.cloud import CloudProviderInterface, GeneralCSPException +from atst.domain.csp.cloud.exceptions import GeneralCSPException +from atst.domain.csp.cloud import CloudProviderInterface from atst.domain.environments import Environments from atst.domain.portfolios import Portfolios from atst.domain.environment_roles import EnvironmentRoles diff --git a/atst/routes/applications/settings.py b/atst/routes/applications/settings.py index b4e75fc1..443989db 100644 --- a/atst/routes/applications/settings.py +++ b/atst/routes/applications/settings.py @@ -13,7 +13,7 @@ from atst.domain.environments import Environments from atst.domain.applications import Applications from atst.domain.application_roles import ApplicationRoles from atst.domain.audit_log import AuditLog -from atst.domain.csp.cloud import GeneralCSPException +from atst.domain.csp.cloud.exceptions import GeneralCSPException from atst.domain.common import Paginator from atst.domain.environment_roles import EnvironmentRoles from atst.domain.invitations import ApplicationInvitations diff --git a/tests/domain/cloud/test_azure_csp.py b/tests/domain/cloud/test_azure_csp.py index 9e87e7a7..0648ec1e 100644 --- a/tests/domain/cloud/test_azure_csp.py +++ b/tests/domain/cloud/test_azure_csp.py @@ -1,17 +1,19 @@ from unittest.mock import Mock - from uuid import uuid4 -from atst.domain.csp.cloud import ( - AzureCloudProvider, - BillingProfileCreationCSPResult, +from tests.factories import ApplicationFactory, EnvironmentFactory +from tests.mock_azure import AUTH_CREDENTIALS, mock_azure + +from atst.domain.csp.cloud import AzureCloudProvider +from atst.domain.csp.cloud.models import ( + BillingInstructionCSPPayload, + BillingInstructionCSPResult, BillingProfileCreationCSPPayload, + BillingProfileCreationCSPResult, BillingProfileTenantAccessCSPPayload, BillingProfileTenantAccessCSPResult, BillingProfileVerificationCSPPayload, BillingProfileVerificationCSPResult, - BillingInstructionCSPPayload, - BillingInstructionCSPResult, TaskOrderBillingCreationCSPPayload, TaskOrderBillingCreationCSPResult, TaskOrderBillingVerificationCSPPayload, @@ -20,10 +22,6 @@ from atst.domain.csp.cloud import ( TenantCSPResult, ) -from tests.mock_azure import mock_azure, AUTH_CREDENTIALS -from tests.factories import EnvironmentFactory, ApplicationFactory - - creds = { "home_tenant_id": "tenant_id", "client_id": "client_id", @@ -150,7 +148,7 @@ def test_create_tenant(mock_azure: AzureCloudProvider): **dict( creds=creds, user_id="admin", - password="JediJan13$coot", + password="JediJan13$coot", # pragma: allowlist secret domain_name="jediccpospawnedtenant2", first_name="Tedry", last_name="Tenet", diff --git a/tests/routes/applications/test_settings.py b/tests/routes/applications/test_settings.py index 37c71878..8f2595f2 100644 --- a/tests/routes/applications/test_settings.py +++ b/tests/routes/applications/test_settings.py @@ -12,7 +12,7 @@ from atst.domain.application_roles import ApplicationRoles from atst.domain.environment_roles import EnvironmentRoles from atst.domain.invitations import ApplicationInvitations from atst.domain.common import Paginator -from atst.domain.csp.cloud import GeneralCSPException +from atst.domain.csp.cloud.exceptions import GeneralCSPException from atst.domain.permission_sets import PermissionSets from atst.models.application_role import Status as ApplicationRoleStatus from atst.models.environment_role import CSPRole, EnvironmentRole From bcd774ffe0a6885bcd181213dbed83da87d85bfa Mon Sep 17 00:00:00 2001 From: tomdds Date: Sun, 26 Jan 2020 16:11:53 -0500 Subject: [PATCH 46/46] Fix wildcard import in azure cloud provider. --- atst/domain/csp/cloud/azure_cloud_provider.py | 31 ++++++++++++++----- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/atst/domain/csp/cloud/azure_cloud_provider.py b/atst/domain/csp/cloud/azure_cloud_provider.py index 67303fb7..5a6a9253 100644 --- a/atst/domain/csp/cloud/azure_cloud_provider.py +++ b/atst/domain/csp/cloud/azure_cloud_provider.py @@ -1,17 +1,32 @@ import re -from uuid import uuid4 from secrets import token_urlsafe +from typing import Dict +from uuid import uuid4 + +from atst.models.application import Application +from atst.models.environment import Environment +from atst.models.user import User from .cloud_provider_interface import CloudProviderInterface -from .exceptions import * -from .models import * +from .exceptions import AuthenticationException +from .models import ( + BillingInstructionCSPPayload, + BillingInstructionCSPResult, + BillingProfileCreationCSPPayload, + BillingProfileCreationCSPResult, + BillingProfileTenantAccessCSPPayload, + BillingProfileTenantAccessCSPResult, + BillingProfileVerificationCSPPayload, + BillingProfileVerificationCSPResult, + TaskOrderBillingCreationCSPPayload, + TaskOrderBillingCreationCSPResult, + TaskOrderBillingVerificationCSPPayload, + TaskOrderBillingVerificationCSPResult, + TenantCSPPayload, + TenantCSPResult, +) from .policy import AzurePolicyManager -from atst.models.user import User -from atst.models.environment import Environment -from atst.models.application import Application - - AZURE_ENVIRONMENT = "AZURE_PUBLIC_CLOUD" # TBD AZURE_SKU_ID = "?" # probably a static sku specific to ATAT/JEDI SUBSCRIPTION_ID_REGEX = re.compile(