From ba47053a1c98f809967e2052e310a1adbd2338ce Mon Sep 17 00:00:00 2001 From: Philip Kalinsky Date: Wed, 8 Jan 2020 11:01:55 -0500 Subject: [PATCH 01/23] provision portfolio state machine --- atst/jobs.py | 1 - 1 file changed, 1 deletion(-) diff --git a/atst/jobs.py b/atst/jobs.py index f4611a9a..ab52cf17 100644 --- a/atst/jobs.py +++ b/atst/jobs.py @@ -12,7 +12,6 @@ from atst.models import ( from atst.domain.csp.cloud import CloudProviderInterface, GeneralCSPException from atst.domain.environments import Environments from atst.domain.portfolios import Portfolios - from atst.domain.environment_roles import EnvironmentRoles from atst.models.utils import claim_for_update from atst.utils.localization import translate From 2ac333e0b738d5fa5ebc94e988a53ab40327cfdb Mon Sep 17 00:00:00 2001 From: tomdds Date: Thu, 9 Jan 2020 17:29:34 -0500 Subject: [PATCH 02/23] Sample create tenant itegration This integration works with the happy path, we'll need to expand some fields and handle error states more coherently. --- Pipfile | 2 + Pipfile.lock | 11 +++- atst/domain/csp/cloud.py | 76 ++++++++++++++++++++++++---- tests/domain/cloud/test_azure_csp.py | 21 +++++++- tests/mock_azure.py | 14 +++++ 5 files changed, 111 insertions(+), 13 deletions(-) diff --git a/Pipfile b/Pipfile index 0de14fa2..f1d852b9 100644 --- a/Pipfile +++ b/Pipfile @@ -33,6 +33,8 @@ azure-mgmt-authorization = "*" azure-mgmt-managementgroups = "*" azure-mgmt-resource = "*" transitions = "*" +azure-mgmt-consumption = "*" +adal = "*" [dev-packages] bandit = "*" diff --git a/Pipfile.lock b/Pipfile.lock index ef6d0203..681fddb1 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "63b8f9d203f306a6f0ff20514b024909aa7e64917e1befcc9ea79931b5b4bd34" + "sha256": "a127b88e6c64842786f1868cb93bb1cdc828aa78040ea8ba4079bb3de0316dab" }, "pipfile-spec": 6, "requires": { @@ -21,6 +21,7 @@ "sha256:5a7f1e037c6290c6d7609cab33a9e5e988c2fbec5c51d1c4c649ee3faff37eaf", "sha256:fd17e5661f60634ddf96a569b95d34ccb8a98de60593d729c28bdcfe360eaad1" ], + "index": "pypi", "version": "==1.2.2" }, "alembic": { @@ -60,6 +61,14 @@ "index": "pypi", "version": "==0.60.0" }, + "azure-mgmt-consumption": { + "hashes": [ + "sha256:035d4b74ca7c47e2683bea17105fd9014c27060336fb6255324ac86b27f70f5b", + "sha256:af319ad6e3ec162a7578563f149e3cdd7d833a62ec80761cfd93caf79467610b" + ], + "index": "pypi", + "version": "==3.0.0" + }, "azure-mgmt-managementgroups": { "hashes": [ "sha256:3d5237947458dc94b4a392141174b1c1258d26611241ee104e9006d1d798f682", diff --git a/atst/domain/csp/cloud.py b/atst/domain/csp/cloud.py index 9f764da0..b9323585 100644 --- a/atst/domain/csp/cloud.py +++ b/atst/domain/csp/cloud.py @@ -156,12 +156,31 @@ class TenantCSPPayload(BaseCSPPayload): country_code: str password_recovery_email_address: str + class Config: + fields = { + "user_id": "userId", + "domain_name": "domainName", + "first_name": "firstName", + "last_name": "lastName", + "country_code": "countryCode", + "password_recovery_email_address": "passwordRecoveryEmailAddress", + } + allow_population_by_field_name = True + class TenantCSPResult(BaseModel): user_id: str tenant_id: str user_object_id: str + class Config: + allow_population_by_field_name = True + fields = { + "user_id": "userId", + "tenant_id": "tenantId", + "user_object_id": "objectId", + } + class BillingProfileAddress(BaseModel): address: Dict @@ -558,11 +577,15 @@ class AzureSDKProvider(object): import azure.graphrbac as graphrbac import azure.common.credentials as credentials from msrestazure.azure_cloud import AZURE_PUBLIC_CLOUD + import adal + import requests self.subscription = subscription self.authorization = authorization + self.adal = adal self.graphrbac = graphrbac self.credentials = credentials + self.requests = requests # may change to a JEDI cloud self.cloud = AZURE_PUBLIC_CLOUD @@ -657,20 +680,31 @@ class AzureCloudProvider(CloudProviderInterface): "role_name": role_assignment_id, } - def create_tenant(self, payload): - # auth as SP that is allowed to create tenant? (tenant creation sp creds) - # create tenant with owner details (populated from portfolio point of contact, pw is generated) + def create_tenant(self, payload: TenantCSPPayload): + sp_token = self._get_sp_token(payload.creds) + if sp_token is None: + raise AuthenticationException("Could not resolve token for tenant creation") - # return tenant id, tenant owner id and tenant owner object id from: - response = {"tenantId": "string", "userId": "string", "objectId": "string"} - return self._ok( - { - "tenant_id": response["tenantId"], - "user_id": response["userId"], - "user_object_id": response["objectId"], - } + create_tenant_body = payload.dict(by_alias=True) + + print(create_tenant_body) + + create_tenant_headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {sp_token}", + } + + result = self.sdk.requests.post( + "https://management.azure.com/providers/Microsoft.SignUp/createTenant?api-version=2020-01-01-preview", + json=create_tenant_body, + headers=create_tenant_headers, ) + if result.status_code == 200: + return self._ok(TenantCSPResult(**result.json())) + else: + return self._error(result.json()) + def create_billing_owner(self, creds, tenant_admin_details): # authenticate as tenant_admin # create billing owner identity @@ -838,6 +872,26 @@ class AzureCloudProvider(CloudProviderInterface): if sub_id_match: return sub_id_match.group(1) + def _get_sp_token(self, creds): + home_tenant_id = creds.get("home_tenant_id") + client_id = creds.get("client_id") + secret_key = creds.get("secret_key") + + # TODO: Make endpoints consts or configs + authentication_endpoint = "https://login.microsoftonline.com/" + resource = "https://management.azure.com/" + + context = self.sdk.adal.AuthenticationContext( + authentication_endpoint + home_tenant_id + ) + + # TODO: handle failure states here + token_response = context.acquire_token_with_client_credentials( + resource, client_id, secret_key + ) + + return token_response.get("accessToken", None) + def _get_credential_obj(self, creds, resource=None): return self.sdk.credentials.ServicePrincipalCredentials( diff --git a/tests/domain/cloud/test_azure_csp.py b/tests/domain/cloud/test_azure_csp.py index cc6beb5b..b6e0d380 100644 --- a/tests/domain/cloud/test_azure_csp.py +++ b/tests/domain/cloud/test_azure_csp.py @@ -3,7 +3,7 @@ from unittest.mock import Mock from uuid import uuid4 -from atst.domain.csp.cloud import AzureCloudProvider +from atst.domain.csp.cloud import AzureCloudProvider, TenantCSPResult from tests.mock_azure import mock_azure, AUTH_CREDENTIALS from tests.factories import EnvironmentFactory, ApplicationFactory @@ -121,3 +121,22 @@ def test_create_policy_definition_succeeds(mock_azure: AzureCloudProvider): policy_definition_name=properties.get("displayName"), parameters=mock_policy_definition, ) + + +def test_create_tenant(mock_azure: AzureCloudProvider): + mock_azure.sdk.adal.AuthenticationContext.return_value.context.acquire_token_with_client_credentials.return_value = { + "accessToken": "TOKEN" + } + + mock_result = Mock() + mock_result.json.return_value = { + "objectId": "0a5f4926-e3ee-4f47-a6e3-8b0a30a40e3d", + "tenantId": "60ff9d34-82bf-4f21-b565-308ef0533435", + "userId": "1153801116406515559", + } + mock_result.status_code = 200 + mock_azure.sdk.requests.post.return_value = mock_result + result = mock_azure.create_tenant(None, suffix=2) + print(result) + body: TenantCSPResult = result.get("body") + assert body.tenant_id == "60ff9d34-82bf-4f21-b565-308ef0533435" diff --git a/tests/mock_azure.py b/tests/mock_azure.py index 417e69fb..ecfafaac 100644 --- a/tests/mock_azure.py +++ b/tests/mock_azure.py @@ -53,16 +53,30 @@ def mock_policy(): return Mock(spec=policy) +def mock_adal(): + import adal + + return Mock(spec=adal) + + +def mock_requests(): + import requests + + return Mock(spec=requests) + + class MockAzureSDK(object): def __init__(self): from msrestazure.azure_cloud import AZURE_PUBLIC_CLOUD self.subscription = mock_subscription() self.authorization = mock_authorization() + self.adal = mock_adal() self.managementgroups = mock_managementgroups() self.graphrbac = mock_graphrbac() self.credentials = mock_credentials() self.policy = mock_policy() + self.requests = mock_requests() # may change to a JEDI cloud self.cloud = AZURE_PUBLIC_CLOUD From 7c22922d6d7489cde4d7611086b75da0b4ba20ce Mon Sep 17 00:00:00 2001 From: tomdds Date: Mon, 13 Jan 2020 16:40:17 -0500 Subject: [PATCH 03/23] Create new AliasModel for CSP datalcasses, ignore credentials when converting to dictionary.This will allow all of our dataclasses to convert automatically between python style snake_case and the camelCase that the Azure APIs use. This also allows us to default to that behavior while specifying aliases for any fields as necessary.Additionally, any dataclass including the creds schema will have those creds removed from their dict representation. This can help keep creds out of logs as well as making the dataclasses more consumable for API usage. --- atst/domain/csp/cloud.py | 63 +++++++++++++--------------- atst/utils/__init__.py | 5 +++ tests/domain/cloud/test_azure_csp.py | 20 ++++++++- 3 files changed, 52 insertions(+), 36 deletions(-) diff --git a/atst/domain/csp/cloud.py b/atst/domain/csp/cloud.py index b9323585..39f11d36 100644 --- a/atst/domain/csp/cloud.py +++ b/atst/domain/csp/cloud.py @@ -7,6 +7,7 @@ from pydantic import BaseModel from atst.models.user import User from atst.models.environment import Environment from atst.models.environment_role import EnvironmentRole +from atst.utils import snake_to_camel class GeneralCSPException(Exception): @@ -142,10 +143,33 @@ class BaselineProvisionException(GeneralCSPException): ) -class BaseCSPPayload(BaseModel): +class AliasModel(BaseModel): + """ + This provides automatic camel <-> snake conversion for serializing to/from json + You can override the alias generation in subclasses by providing a Config that defines + a fields property with a dict mapping variables to their cast names, for cases like: + * some_url:someURL + * user_object_id:objectId + """ + + class Config: + alias_generator = snake_to_camel + allow_population_by_field_name = True + + +class BaseCSPPayload(AliasModel): # {"username": "mock-cloud", "pass": "shh"} creds: Dict + def dict(self, *args, **kwargs): + exclude = {"creds"} + if "exclude" not in kwargs: + kwargs["exclude"] = exclude + else: + kwargs["exclude"].update(exclude) + + return super().dict(*args, **kwargs) + class TenantCSPPayload(BaseCSPPayload): user_id: str @@ -156,52 +180,23 @@ class TenantCSPPayload(BaseCSPPayload): country_code: str password_recovery_email_address: str - class Config: - fields = { - "user_id": "userId", - "domain_name": "domainName", - "first_name": "firstName", - "last_name": "lastName", - "country_code": "countryCode", - "password_recovery_email_address": "passwordRecoveryEmailAddress", - } - allow_population_by_field_name = True - -class TenantCSPResult(BaseModel): +class TenantCSPResult(AliasModel): user_id: str tenant_id: str user_object_id: str class Config: - allow_population_by_field_name = True fields = { - "user_id": "userId", - "tenant_id": "tenantId", "user_object_id": "objectId", } -class BillingProfileAddress(BaseModel): - address: Dict - """ - "address": { - "firstName": "string", - "lastName": "string", - "companyName": "string", - "addressLine1": "string", - "addressLine2": "string", - "addressLine3": "string", - "city": "string", - "region": "string", - "country": "string", - "postalCode": "string" - }, - """ +class BillingProfileAddress(AliasModel): -class BillingProfileCLINBudget(BaseModel): - clinBudget: Dict +class BillingProfileCLINBudget(AliasModel): + clin_budget: Dict """ "clinBudget": { "amount": 0, diff --git a/atst/utils/__init__.py b/atst/utils/__init__.py index d3f284cc..09c63dea 100644 --- a/atst/utils/__init__.py +++ b/atst/utils/__init__.py @@ -25,6 +25,11 @@ def camel_to_snake(camel_cased): return re.sub("([a-z0-9])([A-Z])", r"\1_\2", s1).lower() +def snake_to_camel(snake_cased): + parts = snake_cased.split("_") + return f"{parts[0]}{''.join([w.capitalize() for w in parts[1:]])}" + + def pick(keys, dct): _keys = set(keys) return {k: v for (k, v) in dct.items() if k in _keys} diff --git a/tests/domain/cloud/test_azure_csp.py b/tests/domain/cloud/test_azure_csp.py index b6e0d380..1a9b5c56 100644 --- a/tests/domain/cloud/test_azure_csp.py +++ b/tests/domain/cloud/test_azure_csp.py @@ -3,7 +3,11 @@ from unittest.mock import Mock from uuid import uuid4 -from atst.domain.csp.cloud import AzureCloudProvider, TenantCSPResult +from atst.domain.csp.cloud import ( + AzureCloudProvider, + TenantCSPResult, + TenantCSPPayload, +) from tests.mock_azure import mock_azure, AUTH_CREDENTIALS from tests.factories import EnvironmentFactory, ApplicationFactory @@ -136,7 +140,19 @@ def test_create_tenant(mock_azure: AzureCloudProvider): } mock_result.status_code = 200 mock_azure.sdk.requests.post.return_value = mock_result - result = mock_azure.create_tenant(None, suffix=2) + payload = TenantCSPPayload( + **dict( + creds={"username": "mock-cloud", "pass": "shh"}, + user_id="123", + password="123", + domain_name="123", + first_name="john", + last_name="doe", + country_code="US", + password_recovery_email_address="password@email.com", + ) + ) + result = mock_azure.create_tenant(payload) print(result) body: TenantCSPResult = result.get("body") assert body.tenant_id == "60ff9d34-82bf-4f21-b565-308ef0533435" From 161462f3cb9cae2479cecfc21648ed5b0e9fc46f Mon Sep 17 00:00:00 2001 From: tomdds Date: Mon, 13 Jan 2020 16:48:05 -0500 Subject: [PATCH 04/23] Sample create and validate billing profile integration Adds 2 methods to the azure csp interface to create and subsequently validate creation of the billing profile. --- atst/domain/csp/cloud.py | 186 +++++++++++++++------------ tests/domain/cloud/test_azure_csp.py | 97 ++++++++++++++ 2 files changed, 204 insertions(+), 79 deletions(-) diff --git a/atst/domain/csp/cloud.py b/atst/domain/csp/cloud.py index 39f11d36..86c436e5 100644 --- a/atst/domain/csp/cloud.py +++ b/atst/domain/csp/cloud.py @@ -1,8 +1,8 @@ import re -from typing import Dict +from typing import Dict, List, Optional from uuid import uuid4 -from pydantic import BaseModel +from pydantic import BaseModel, validator from atst.models.user import User from atst.models.environment import Environment @@ -193,6 +193,12 @@ class TenantCSPResult(AliasModel): class BillingProfileAddress(AliasModel): + company_name: str + address_line_1: str + city: str + region: str + country: str + postal_code: str class BillingProfileCLINBudget(AliasModel): @@ -207,44 +213,60 @@ class BillingProfileCLINBudget(AliasModel): """ -class BillingProfileCSPPayload( - BaseCSPPayload, BillingProfileAddress, BillingProfileCLINBudget -): - displayName: str - poNumber: str - invoiceEmailOptIn: str +class BillingProfileCSPPayload(BaseCSPPayload): + tenant_id: str + display_name: str + enabled_azure_plans: Optional[List[str]] + address: BillingProfileAddress - """ - { - "displayName": "string", - "poNumber": "string", - "address": { - "firstName": "string", - "lastName": "string", - "companyName": "string", - "addressLine1": "string", - "addressLine2": "string", - "addressLine3": "string", - "city": "string", - "region": "string", - "country": "string", - "postalCode": "string" - }, - "invoiceEmailOptIn": true, - Note: These last 2 are also the body for adding/updating new TOs/clins - "enabledAzurePlans": [ - { - "skuId": "string" - } - ], - "clinBudget": { - "amount": 0, - "startDate": "2019-12-18T16:47:40.909Z", - "endDate": "2019-12-18T16:47:40.909Z", - "externalReferenceId": "string" + @validator("enabled_azure_plans", pre=True, always=True) + def default_enabled_azure_plans(cls, v): + """ + Normally you'd implement this by setting the field with a value of: + dataclasses.field(default_factory=list) + but that prevents the object from being correctly pickled, so instead we need + to rely on a validator to ensure this has an empty value when not specified + """ + return v or [] + + +class BillingProfileCreateCSPResult(AliasModel): + location: str + retry_after: int + + class Config: + fields = {"location": "Location", "retry_after": "Retry-After"} + + +class BillingProfileVerifyCSPPayload(BaseCSPPayload): + location: str + + +class BillingInvoiceSection(AliasModel): + invoice_section_id: str + invoice_section_name: str + + class Config: + fields = {"invoice_section_id": "id", "invoice_section_name": "name"} + + +class BillingProfileProperties(AliasModel): + address: BillingProfileAddress + display_name: str + invoice_sections: List[BillingInvoiceSection] + + +class BillingProfileCSPResult(AliasModel): + billing_profile_id: str + billing_profile_name: str + billing_profile_properties: BillingProfileProperties + + class Config: + fields = { + "billing_profile_id": "id", + "billing_profile_name": "name", + "billing_profile_properties": "properties", } - } - """ class CloudProviderInterface: @@ -682,8 +704,6 @@ class AzureCloudProvider(CloudProviderInterface): create_tenant_body = payload.dict(by_alias=True) - print(create_tenant_body) - create_tenant_headers = { "Content-Type": "application/json", "Authorization": f"Bearer {sp_token}", @@ -700,6 +720,53 @@ class AzureCloudProvider(CloudProviderInterface): else: return self._error(result.json()) + def create_billing_profile(self, payload: BillingProfileCSPPayload): + sp_token = self._get_sp_token(payload.creds) + if sp_token is None: + raise AuthenticationException( + "Could not resolve token for billing profile creation" + ) + + create_billing_account_body = payload.dict(by_alias=True) + + create_billing_account_headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {sp_token}", + } + + # TODO: unsure if this is a static value or needs to be constructed/configurable + BILLING_ACCOUT_NAME = "7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31" + billing_account_create_url = f"https://management.azure.com/providers/Microsoft.Billing/billingAccounts/{BILLING_ACCOUT_NAME}/billingProfiles?api-version=2019-10-01-preview" + + result = self.sdk.requests.post( + billing_account_create_url, + json=create_billing_account_body, + headers=create_billing_account_headers, + ) + + if result.status_code == 202: + return self._ok(BillingProfileCreateCSPResult(**result.headers)) + else: + return self._error(result.json()) + + def validate_billing_profile_created(self, payload: BillingProfileVerifyCSPPayload): + sp_token = self._get_sp_token(payload.creds) + if sp_token is None: + raise AuthenticationException( + "Could not resolve token for billing profile validation" + ) + + auth_header = { + "Authorization": f"Bearer {sp_token}", + } + + result = self.sdk.requests.get(payload.location, headers=auth_header) + + if result.status_code == 200: + return self._ok(BillingProfileCSPResult(**result.json())) + else: + return self._error(result.json()) + def create_billing_owner(self, creds, tenant_admin_details): # authenticate as tenant_admin # create billing owner identity @@ -722,45 +789,6 @@ class AzureCloudProvider(CloudProviderInterface): return self.ok() - def create_billing_profile(self, creds, tenant_admin_details, billing_owner_id): - # call billing profile creation endpoint, specifying owner - # Payload: - """ - { - "displayName": "string", - "poNumber": "string", - "address": { - "firstName": "string", - "lastName": "string", - "companyName": "string", - "addressLine1": "string", - "addressLine2": "string", - "addressLine3": "string", - "city": "string", - "region": "string", - "country": "string", - "postalCode": "string" - }, - "invoiceEmailOptIn": true, - Note: These last 2 are also the body for adding/updating new TOs/clins - "enabledAzurePlans": [ - { - "skuId": "string" - } - ], - "clinBudget": { - "amount": 0, - "startDate": "2019-12-18T16:47:40.909Z", - "endDate": "2019-12-18T16:47:40.909Z", - "externalReferenceId": "string" - } - } - """ - - # response will be mostly the same as the body, but we only really care about the id - response = {"id": "string"} - return self._ok({"billing_profile_id": response["id"]}) - def report_clin(self, creds, clin_id, clin_amount, clin_start, clin_end, clin_to): # should consumer be responsible for reporting each clin or # should this take a list and manage the sequential reporting? diff --git a/tests/domain/cloud/test_azure_csp.py b/tests/domain/cloud/test_azure_csp.py index 1a9b5c56..6b9a8b12 100644 --- a/tests/domain/cloud/test_azure_csp.py +++ b/tests/domain/cloud/test_azure_csp.py @@ -7,6 +7,11 @@ from atst.domain.csp.cloud import ( AzureCloudProvider, TenantCSPResult, TenantCSPPayload, + BillingProfileCSPPayload, + BillingProfileAddress, + BillingProfileCreateCSPResult, + BillingProfileVerifyCSPPayload, + BillingProfileCSPResult, ) from tests.mock_azure import mock_azure, AUTH_CREDENTIALS @@ -156,3 +161,95 @@ def test_create_tenant(mock_azure: AzureCloudProvider): print(result) body: TenantCSPResult = result.get("body") assert body.tenant_id == "60ff9d34-82bf-4f21-b565-308ef0533435" + + +def test_create_billing_profile(mock_azure: AzureCloudProvider): + # mock_azure.sdk.adal.AuthenticationContext.return_value.context.acquire_token_with_client_credentials.return_value = { + # "accessToken": "TOKEN" + # } + + # mock_result = Mock() + # mock_result.headers = { + # "Location": "http://retry-url", + # "Retry-After": "10", + # } + # mock_result.status_code = 202 + # mock_azure.sdk.requests.post.return_value = mock_result + payload = BillingProfileCSPPayload( + **dict( + address=dict( + address_line_1="123 S Broad Street, Suite 2400", + company_name="Promptworks", + city="Philadelphia", + region="PA", + country="US", + postal_code="19109", + ), + creds={"username": "mock-cloud", "pass": "shh"}, + tenant_id="60ff9d34-82bf-4f21-b565-308ef0533435", + display_name="Test Billing Profile", + ) + ) + result = mock_azure.create_billing_profile(payload) + print(result) + body: BillingProfileCreateCSPResult = result.get("body") + assert body.retry_after == 10 + + +def test_validate_billing_profile_creation(mock_azure: AzureCloudProvider): + mock_azure.sdk.adal.AuthenticationContext.return_value.context.acquire_token_with_client_credentials.return_value = { + "accessToken": "TOKEN" + } + + mock_result = Mock() + mock_result.status_code = 200 + mock_result.json.return_value = { + "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/XC36-GRNZ-BG7-TGB", + "name": "XC36-GRNZ-BG7-TGB", + "properties": { + "address": { + "addressLine1": "123 S Broad Street, Suite 2400", + "city": "Philadelphia", + "companyName": "Promptworks", + "country": "US", + "postalCode": "19109", + "region": "PA", + }, + "currency": "USD", + "displayName": "First Portfolio Billing Profile", + "enabledAzurePlans": [], + "hasReadAccess": True, + "invoiceDay": 5, + "invoiceEmailOptIn": False, + "invoiceSections": [ + { + "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/XC36-GRNZ-BG7-TGB/invoiceSections/6HMZ-2HLO-PJA-TGB", + "name": "6HMZ-2HLO-PJA-TGB", + "properties": {"displayName": "First Portfolio Billing Profile"}, + "type": "Microsoft.Billing/billingAccounts/billingProfiles/invoiceSections", + } + ], + }, + "type": "Microsoft.Billing/billingAccounts/billingProfiles", + } + mock_azure.sdk.requests.get.return_value = mock_result + + payload = BillingProfileVerifyCSPPayload( + **dict( + creds={ + "username": "username", + "password": "password", + "tenant_id": "tenant_id", + }, + location="https://management.azure.com/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/operationResults/createBillingProfile_478d5706-71f9-4a8b-8d4e-2cbaca27a668?api-version=2019-10-01-preview", + ) + ) + + result = mock_azure.validate_billing_profile_created(payload) + body: BillingProfileCreateCSPResult = result.get("body") + assert body.billing_profile_name == "XC36-GRNZ-BG7-TGB" + assert ( + body.billing_profile_properties.display_name + == "First Portfolio Billing Profile" + ) + From 81f23ebc22cc405a70d1bf4c557558822253694a Mon Sep 17 00:00:00 2001 From: tomdds Date: Tue, 14 Jan 2020 15:04:13 -0500 Subject: [PATCH 05/23] Finish first passes at baseline tenant integration Add last of the integrations for setting up billing and reporting a CLIN. --- atst/domain/csp/cloud.py | 220 ++++++++++++++++++------ tests/domain/cloud/test_azure_csp.py | 241 +++++++++++++++++++++++---- 2 files changed, 385 insertions(+), 76 deletions(-) diff --git a/atst/domain/csp/cloud.py b/atst/domain/csp/cloud.py index 86c436e5..703dec53 100644 --- a/atst/domain/csp/cloud.py +++ b/atst/domain/csp/cloud.py @@ -231,15 +231,15 @@ class BillingProfileCSPPayload(BaseCSPPayload): class BillingProfileCreateCSPResult(AliasModel): - location: str + billing_profile_validate_url: str retry_after: int class Config: - fields = {"location": "Location", "retry_after": "Retry-After"} + fields = {"billing_profile_validate_url": "Location", "retry_after": "Retry-After"} class BillingProfileVerifyCSPPayload(BaseCSPPayload): - location: str + billing_profile_validate_url: str class BillingInvoiceSection(AliasModel): @@ -269,6 +269,70 @@ class BillingProfileCSPResult(AliasModel): } +class BillingRoleAssignmentCSPPayload(BaseCSPPayload): + tenant_id: str + user_object_id: str + billing_account_name: str + billing_profile_name: str + + +class BillingRoleAssignmentCSPResult(AliasModel): + billing_role_assignment_id: str + billing_role_assignment_name: str + + class Config: + fields = { + "billing_role_assignment_id": "id", + "billing_role_assignment_name": "name", + } + +class EnableTaskOrderBillingCSPPayload(BaseCSPPayload): + billing_account_name: str + billing_profile_name: str + +class EnableTaskOrderBillingCSPResult(AliasModel): + task_order_billing_validation_url: str + retry_after: int + + class Config: + fields = {"task_order_billing_validation_url": "Location", "retry_after": "Retry-After"} + +class VerifyTaskOrderBillingCSPPayload(BaseCSPPayload): + task_order_billing_validation_url: str + +class BillingProfileEnabledPlanDetails(AliasModel): + enabled_azure_plans: List[Dict] + + +class BillingProfileEnabledCSPResult(AliasModel): + billing_profile_id: str + billing_profile_name: str + billing_profile_enabled_plan_details: BillingProfileEnabledPlanDetails + + class Config: + fields = { + "billing_profile_id": "id", + "billing_profile_name": "name", + "billing_profile_enabled_plan_details": "properties", + } + +class ReportCLINCSPPayload(BaseCSPPayload): + amount: float + start_date: str + end_date: str + clin_type: str + task_order_id: str + billing_account_name: str + billing_profile_name: str + +class ReportCLINCSPResult(AliasModel): + reported_clin_name: str + + class Config: + fields = { + "reported_clin_name": "name", + } + class CloudProviderInterface: def root_creds(self) -> Dict: raise NotImplementedError() @@ -705,7 +769,6 @@ class AzureCloudProvider(CloudProviderInterface): create_tenant_body = payload.dict(by_alias=True) create_tenant_headers = { - "Content-Type": "application/json", "Authorization": f"Bearer {sp_token}", } @@ -730,7 +793,6 @@ class AzureCloudProvider(CloudProviderInterface): create_billing_account_body = payload.dict(by_alias=True) create_billing_account_headers = { - "Content-Type": "application/json", "Authorization": f"Bearer {sp_token}", } @@ -745,7 +807,11 @@ class AzureCloudProvider(CloudProviderInterface): ) if result.status_code == 202: + # 202 has location/retry after headers return self._ok(BillingProfileCreateCSPResult(**result.headers)) + elif result.status_code == 200: + # NB: Swagger docs imply call can sometimes resolve immediately + return self._ok(BillingProfileCSPResult(**result.json())) else: return self._error(result.json()) @@ -760,56 +826,116 @@ class AzureCloudProvider(CloudProviderInterface): "Authorization": f"Bearer {sp_token}", } - result = self.sdk.requests.get(payload.location, headers=auth_header) + result = self.sdk.requests.get(payload.billing_profile_validate_url, headers=auth_header) - if result.status_code == 200: + if result.status_code == 202: + # 202 has location/retry after headers + return self._ok(BillingProfileCreateCSPResult(**result.headers)) + elif result.status_code == 200: return self._ok(BillingProfileCSPResult(**result.json())) else: return self._error(result.json()) - def create_billing_owner(self, creds, tenant_admin_details): - # authenticate as tenant_admin - # create billing owner identity - - # TODO: Lookup response format - # Managed service identity? - response = {"id": "string"} - return self._ok({"billing_owner_id": response["id"]}) - - def assign_billing_owner(self, creds, billing_owner_id, tenant_id): - # TODO: Do we source role definition ID from config, api or self-defined? - # TODO: If from api, - """ - { - "principalId": "string", - "principalTenantId": "string", - "billingRoleDefinitionId": "string" - } - """ - - return self.ok() - - def report_clin(self, creds, clin_id, clin_amount, clin_start, clin_end, clin_to): - # should consumer be responsible for reporting each clin or - # should this take a list and manage the sequential reporting? - """ Payload - { - "enabledAzurePlans": [ - { - "skuId": "string" - } - ], - "clinBudget": { - "amount": 0, - "startDate": "2019-12-18T16:47:40.909Z", - "endDate": "2019-12-18T16:47:40.909Z", - "externalReferenceId": "string" + def grant_billing_profile_tenant_access(self, payload: BillingRoleAssignmentCSPPayload): + sp_token = self._get_sp_token(payload.creds) + request_body = { + "properties": { + "principalTenantId": payload.tenant_id, # from tenant creation + "principalId": payload.user_object_id, # from tenant creationn + "roleDefinitionId": f"/providers/Microsoft.Billing/billingAccounts/{payload.billing_account_name}/billingProfiles/{payload.billing_profile_name}/billingRoleDefinitions/40000000-aaaa-bbbb-cccc-100000000000", } } - """ - # we don't need any of the returned info for this - return self._ok() + headers = { + "Authorization": f"Bearer {sp_token}", + } + + url = f"https://management.azure.com/providers/Microsoft.Billing/billingAccounts/{payload.billing_account_name}/billingProfiles/{payload.billing_profile_name}/createBillingRoleAssignment?api-version=2019-10-01-preview" + + result = self.sdk.requests.post(url, headers=headers, json=request_body) + if result.status_code == 201: + return self._ok(BillingRoleAssignmentCSPResult(**result.json())) + else: + return self._error(result.json()) + + def enable_task_order_billing(self, payload: EnableTaskOrderBillingCSPPayload): + sp_token = self._get_sp_token(payload.creds) + request_body = [ + { + "op": "replace", + "path": "/enabledAzurePlans", + "value": [ + { + "skuId": "0001" + } + ] + } + ] + + request_headers = { + "Authorization": f"Bearer {sp_token}", + } + + url = f"https://management.azure.com/providers/Microsoft.Billing/billingAccounts/{payload.billing_account_name}/billingProfiles/{payload.billing_profile_name}?api-version=2019-10-01-preview" + + result = self.sdk.requests.patch(url, headers=request_headers, json=request_body) + + if result.status_code == 202: + # 202 has location/retry after headers + return self._ok(BillingProfileCreateCSPResult(**result.headers)) + elif result.status_code == 200: + return self._ok(BillingProfileEnabledCSPResult(**result.json())) + else: + return self._error(result.json()) + + def validate_task_order_billing_enabled(self, payload: VerifyTaskOrderBillingCSPPayload): + sp_token = self._get_sp_token(payload.creds) + if sp_token is None: + raise AuthenticationException( + "Could not resolve token for task order billing validation" + ) + + auth_header = { + "Authorization": f"Bearer {sp_token}", + } + + result = self.sdk.requests.get(payload.task_order_billing_validation_url, headers=auth_header) + + if result.status_code == 202: + # 202 has location/retry after headers + return self._ok(EnableTaskOrderBillingCSPResult(**result.headers)) + elif result.status_code == 200: + return self._ok(BillingProfileEnabledCSPResult(**result.json())) + else: + return self._error(result.json()) + + def report_clin(self, payload: ReportCLINCSPPayload): + sp_token = self._get_sp_token(payload.creds) + if sp_token is None: + raise AuthenticationException( + "Could not resolve token for task order billing validation" + ) + + request_body = { + "properties": { + "amount": payload.amount, + "startDate": payload.start_date, + "endDate": payload.end_date + } + } + + url = f"https://management.azure.com/providers/Microsoft.Billing/billingAccounts/{payload.billing_account_name}/billingProfiles/{payload.billing_profile_name}/instructions/{payload.task_order_id}:CLIN00{payload.clin_type}?api-version=2019-10-01-preview" + + auth_header = { + "Authorization": f"Bearer {sp_token}", + } + + result = self.sdk.requests.put(url, headers=auth_header, json=request_body) + + if result.status_code == 200: + return self._ok(ReportCLINCSPResult(**result.json())) + else: + return self._error(result.json()) def create_remote_admin(self, creds, tenant_details): # create app/service principal within tenant, with name constructed from tenant details diff --git a/tests/domain/cloud/test_azure_csp.py b/tests/domain/cloud/test_azure_csp.py index 6b9a8b12..9e9809c7 100644 --- a/tests/domain/cloud/test_azure_csp.py +++ b/tests/domain/cloud/test_azure_csp.py @@ -12,6 +12,13 @@ from atst.domain.csp.cloud import ( BillingProfileCreateCSPResult, BillingProfileVerifyCSPPayload, BillingProfileCSPResult, + BillingRoleAssignmentCSPPayload, + BillingRoleAssignmentCSPResult, + EnableTaskOrderBillingCSPPayload, + VerifyTaskOrderBillingCSPPayload, + BillingProfileEnabledCSPResult, + ReportCLINCSPPayload, + ReportCLINCSPResult, ) from tests.mock_azure import mock_azure, AUTH_CREDENTIALS @@ -25,7 +32,7 @@ from tests.factories import EnvironmentFactory, ApplicationFactory # -@pytest.mark.skip() +@pytest.mark.skip("Skipping legacy azure integration tests") def test_create_subscription_succeeds(mock_azure: AzureCloudProvider): environment = EnvironmentFactory.create() @@ -60,14 +67,13 @@ def test_create_subscription_succeeds(mock_azure: AzureCloudProvider): assert result == subscription_id -@pytest.mark.skip() def mock_management_group_create(mock_azure, spec_dict): mock_azure.sdk.managementgroups.ManagementGroupsAPI.return_value.management_groups.create_or_update.return_value.result.return_value = Mock( **spec_dict ) -@pytest.mark.skip() +@pytest.mark.skip("Skipping legacy azure integration tests") def test_create_environment_succeeds(mock_azure: AzureCloudProvider): environment = EnvironmentFactory.create() @@ -80,7 +86,7 @@ def test_create_environment_succeeds(mock_azure: AzureCloudProvider): assert result.id == "Test Id" -@pytest.mark.skip() +@pytest.mark.skip("Skipping legacy azure integration tests") def test_create_application_succeeds(mock_azure: AzureCloudProvider): application = ApplicationFactory.create() @@ -91,7 +97,7 @@ def test_create_application_succeeds(mock_azure: AzureCloudProvider): assert result.id == "Test Id" -@pytest.mark.skip() +@pytest.mark.skip("Skipping legacy azure integration tests") def test_create_atat_admin_user_succeeds(mock_azure: AzureCloudProvider): environment_id = str(uuid4()) @@ -106,7 +112,7 @@ def test_create_atat_admin_user_succeeds(mock_azure: AzureCloudProvider): assert result.get("csp_user_id") == csp_user_id -@pytest.mark.skip() +@pytest.mark.skip("Skipping legacy azure integration tests") def test_create_policy_definition_succeeds(mock_azure: AzureCloudProvider): subscription_id = str(uuid4()) management_group_id = str(uuid4()) @@ -148,33 +154,32 @@ def test_create_tenant(mock_azure: AzureCloudProvider): payload = TenantCSPPayload( **dict( creds={"username": "mock-cloud", "pass": "shh"}, - user_id="123", - password="123", - domain_name="123", - first_name="john", - last_name="doe", + user_id="admin", + password="JediJan13$coot", + domain_name="jediccpospawnedtenant2", + first_name="Tedry", + last_name="Tenet", country_code="US", - password_recovery_email_address="password@email.com", + password_recovery_email_address="thomas@promptworks.com", ) ) result = mock_azure.create_tenant(payload) - print(result) body: TenantCSPResult = result.get("body") assert body.tenant_id == "60ff9d34-82bf-4f21-b565-308ef0533435" def test_create_billing_profile(mock_azure: AzureCloudProvider): - # mock_azure.sdk.adal.AuthenticationContext.return_value.context.acquire_token_with_client_credentials.return_value = { - # "accessToken": "TOKEN" - # } + mock_azure.sdk.adal.AuthenticationContext.return_value.context.acquire_token_with_client_credentials.return_value = { + "accessToken": "TOKEN" + } - # mock_result = Mock() - # mock_result.headers = { - # "Location": "http://retry-url", - # "Retry-After": "10", - # } - # mock_result.status_code = 202 - # mock_azure.sdk.requests.post.return_value = mock_result + mock_result = Mock() + mock_result.headers = { + "Location": "http://retry-url", + "Retry-After": "10", + } + mock_result.status_code = 202 + mock_azure.sdk.requests.post.return_value = mock_result payload = BillingProfileCSPPayload( **dict( address=dict( @@ -191,7 +196,6 @@ def test_create_billing_profile(mock_azure: AzureCloudProvider): ) ) result = mock_azure.create_billing_profile(payload) - print(result) body: BillingProfileCreateCSPResult = result.get("body") assert body.retry_after == 10 @@ -204,8 +208,8 @@ def test_validate_billing_profile_creation(mock_azure: AzureCloudProvider): mock_result = Mock() mock_result.status_code = 200 mock_result.json.return_value = { - "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/XC36-GRNZ-BG7-TGB", - "name": "XC36-GRNZ-BG7-TGB", + "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB", + "name": "KQWI-W2SU-BG7-TGB", "properties": { "address": { "addressLine1": "123 S Broad Street, Suite 2400", @@ -223,7 +227,7 @@ def test_validate_billing_profile_creation(mock_azure: AzureCloudProvider): "invoiceEmailOptIn": False, "invoiceSections": [ { - "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/XC36-GRNZ-BG7-TGB/invoiceSections/6HMZ-2HLO-PJA-TGB", + "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB/invoiceSections/6HMZ-2HLO-PJA-TGB", "name": "6HMZ-2HLO-PJA-TGB", "properties": {"displayName": "First Portfolio Billing Profile"}, "type": "Microsoft.Billing/billingAccounts/billingProfiles/invoiceSections", @@ -241,15 +245,194 @@ def test_validate_billing_profile_creation(mock_azure: AzureCloudProvider): "password": "password", "tenant_id": "tenant_id", }, - location="https://management.azure.com/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/operationResults/createBillingProfile_478d5706-71f9-4a8b-8d4e-2cbaca27a668?api-version=2019-10-01-preview", + billing_profile_validate_url="https://management.azure.com/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/operationResults/createBillingProfile_478d5706-71f9-4a8b-8d4e-2cbaca27a668?api-version=2019-10-01-preview", ) ) result = mock_azure.validate_billing_profile_created(payload) body: BillingProfileCreateCSPResult = result.get("body") - assert body.billing_profile_name == "XC36-GRNZ-BG7-TGB" + assert body.billing_profile_name == "KQWI-W2SU-BG7-TGB" assert ( body.billing_profile_properties.display_name == "First Portfolio Billing Profile" ) + +def test_grant_billing_profile_tenant_access(mock_azure: AzureCloudProvider): + mock_azure.sdk.adal.AuthenticationContext.return_value.context.acquire_token_with_client_credentials.return_value = { + "accessToken": "TOKEN" + } + + mock_result = Mock() + mock_result.status_code = 201 + mock_result.json.return_value = { + "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB/billingRoleAssignments/40000000-aaaa-bbbb-cccc-100000000000_0a5f4926-e3ee-4f47-a6e3-8b0a30a40e3d", + "name": "40000000-aaaa-bbbb-cccc-100000000000_0a5f4926-e3ee-4f47-a6e3-8b0a30a40e3d", + "properties": { + "createdOn": "2020-01-14T14:39:26.3342192+00:00", + "createdByPrincipalId": "82e2b376-3297-4096-8743-ed65b3be0b03", + "principalId": "0a5f4926-e3ee-4f47-a6e3-8b0a30a40e3d", + "principalTenantId": "60ff9d34-82bf-4f21-b565-308ef0533435", + "roleDefinitionId": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB/billingRoleDefinitions/40000000-aaaa-bbbb-cccc-100000000000", + "scope": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB", + }, + "type": "Microsoft.Billing/billingRoleAssignments", + } + + mock_azure.sdk.requests.post.return_value = mock_result + + payload = BillingRoleAssignmentCSPPayload( + **dict( + creds={ + "username": "username", + "password": "password", + "tenant_id": "tenant_id", + }, + tenant_id="60ff9d34-82bf-4f21-b565-308ef0533435", + user_object_id="0a5f4926-e3ee-4f47-a6e3-8b0a30a40e3d", + billing_account_name="7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31", + billing_profile_name="KQWI-W2SU-BG7-TGB", + ) + ) + + result = mock_azure.grant_billing_profile_tenant_access(payload) + body: BillingRoleAssignmentCSPResult = result.get("body") + assert ( + body.billing_role_assignment_name + == "40000000-aaaa-bbbb-cccc-100000000000_0a5f4926-e3ee-4f47-a6e3-8b0a30a40e3d" + ) + + +def test_enable_task_order_billing(mock_azure: AzureCloudProvider): + mock_azure.sdk.adal.AuthenticationContext.return_value.context.acquire_token_with_client_credentials.return_value = { + "accessToken": "TOKEN" + } + + mock_result = Mock() + mock_result.status_code = 202 + mock_result.headers = { + "Location": "https://management.azure.com/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/operationResults/patchBillingProfile_KQWI-W2SU-BG7-TGB:02715576-4118-466c-bca7-b1cd3169ff46?api-version=2019-10-01-preview", + "Retry-After": "10", + } + + mock_azure.sdk.requests.patch.return_value = mock_result + + payload = EnableTaskOrderBillingCSPPayload( + **dict( + creds={ + "username": "username", + "password": "password", + "tenant_id": "tenant_id", + }, + billing_account_name="7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31", + billing_profile_name="KQWI-W2SU-BG7-TGB", + ) + ) + + result = mock_azure.enable_task_order_billing(payload) + body: BillingProfileCreateCSPResult = result.get("body") + assert ( + body.billing_profile_validate_url + == "https://management.azure.com/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/operationResults/patchBillingProfile_KQWI-W2SU-BG7-TGB:02715576-4118-466c-bca7-b1cd3169ff46?api-version=2019-10-01-preview" + ) + + +def test_validate_task_order_billing_enabled(mock_azure): + mock_azure.sdk.adal.AuthenticationContext.return_value.context.acquire_token_with_client_credentials.return_value = { + "accessToken": "TOKEN" + } + + mock_result = Mock() + mock_result.status_code = 200 + mock_result.json.return_value = { + "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB", + "name": "KQWI-W2SU-BG7-TGB", + "properties": { + "address": { + "addressLine1": "123 S Broad Street, Suite 2400", + "city": "Philadelphia", + "companyName": "Promptworks", + "country": "US", + "postalCode": "19109", + "region": "PA", + }, + "currency": "USD", + "displayName": "Test Billing Profile", + "enabledAzurePlans": [ + { + "productId": "DZH318Z0BPS6", + "skuId": "0001", + "skuDescription": "Microsoft Azure Plan", + } + ], + "hasReadAccess": True, + "invoiceDay": 5, + "invoiceEmailOptIn": False, + "invoiceSections": [ + { + "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB/invoiceSections/CHCO-BAAR-PJA-TGB", + "name": "CHCO-BAAR-PJA-TGB", + "properties": {"displayName": "Test Billing Profile"}, + "type": "Microsoft.Billing/billingAccounts/billingProfiles/invoiceSections", + } + ], + }, + "type": "Microsoft.Billing/billingAccounts/billingProfiles", + } + mock_azure.sdk.requests.get.return_value = mock_result + + payload = VerifyTaskOrderBillingCSPPayload( + **dict( + creds={ + "username": "username", + "password": "password", + "tenant_id": "tenant_id", + }, + task_order_billing_validation_url="https://management.azure.com/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/operationResults/createBillingProfile_478d5706-71f9-4a8b-8d4e-2cbaca27a668?api-version=2019-10-01-preview", + ) + ) + + result = mock_azure.validate_task_order_billing_enabled(payload) + body: BillingProfileEnabledCSPResult = result.get("body") + assert body.billing_profile_name == "KQWI-W2SU-BG7-TGB" + assert ( + body.billing_profile_enabled_plan_details.enabled_azure_plans[0].get("skuId") + == "0001" + ) + + +def test_report_clin(mock_azure: AzureCloudProvider): + mock_azure.sdk.adal.AuthenticationContext.return_value.context.acquire_token_with_client_credentials.return_value = { + "accessToken": "TOKEN" + } + + mock_result = Mock() + mock_result.status_code = 200 + mock_result.json.return_value = { + "name": "TO1:CLIN001", + "properties": { + "amount": 1000.0, + "endDate": "2020-03-01T00:00:00+00:00", + "startDate": "2020-01-01T00:00:00+00:00", + }, + "type": "Microsoft.Billing/billingAccounts/billingProfiles/billingInstructions", + } + + mock_azure.sdk.requests.put.return_value = mock_result + + payload = ReportCLINCSPPayload( + **dict( + creds={}, + amount=1000.00, + start_date="2020/1/1", + end_date="2020/3/1", + clin_type="1", + task_order_id="TO1", + billing_account_name="7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31", + billing_profile_name="KQWI-W2SU-BG7-TGB", + ) + ) + result = mock_azure.report_clin(payload) + body: ReportCLINCSPResult = result.get("body") + assert body.reported_clin_name == "TO1:CLIN001" + From 187ee0033e4b7277243d0fbac3c39597f4bcb943 Mon Sep 17 00:00:00 2001 From: Philip Kalinsky Date: Wed, 15 Jan 2020 16:07:50 -0500 Subject: [PATCH 06/23] state machine unit tests --- atst/models/mixins/state_machines.py | 37 +++++++----- atst/models/portfolio_state_machine.py | 12 ++-- tests/domain/test_portfolio_state_machine.py | 61 +++++++++++++++++++- 3 files changed, 85 insertions(+), 25 deletions(-) diff --git a/atst/models/mixins/state_machines.py b/atst/models/mixins/state_machines.py index bc35209d..b2eda399 100644 --- a/atst/models/mixins/state_machines.py +++ b/atst/models/mixins/state_machines.py @@ -10,7 +10,6 @@ class StageStates(Enum): class AzureStages(Enum): TENANT = "tenant" BILLING_PROFILE = "billing profile" - ADMIN_SUBSCRIPTION = "admin subscription" def _build_csp_states(csp_stages): @@ -31,14 +30,14 @@ def _build_csp_states(csp_stages): FSMStates = Enum("FSMStates", _build_csp_states(AzureStages)) +compose_state = lambda csp_stage, state: getattr( + FSMStates, "_".join([csp_stage.name, state.name]) +) + def _build_transitions(csp_stages): transitions = [] states = [] - compose_state = lambda csp_stage, state: getattr( - FSMStates, "_".join([csp_stage.name, state.name]) - ) - for stage_i, csp_stage in enumerate(csp_stages): for state in StageStates: states.append( @@ -99,6 +98,24 @@ class FSMMixin: {"trigger": "fail", "source": "*", "dest": FSMStates.FAILED,}, ] + def fail_stage(self, stage): + fail_trigger = "fail" + stage + if fail_trigger in self.machine.get_triggers(self.current_state.name): + self.trigger(fail_trigger) + + def finish_stage(self, stage): + finish_trigger = "finish_" + stage + if finish_trigger in self.machine.get_triggers(self.current_state.name): + self.trigger(finish_trigger) + + def _get_first_stage_create_trigger(self): + return list( + filter( + lambda trigger: trigger.startswith("create_"), + self.machine.get_triggers(FSMStates.STARTED.name), + ) + )[0] + def prepare_init(self, event): pass @@ -125,13 +142,3 @@ class FSMMixin: def after_reset(self, event): pass - - def fail_stage(self, stage): - fail_trigger = "fail" + stage - if fail_trigger in self.machine.get_triggers(self.current_state.name): - self.trigger(fail_trigger) - - def finish_stage(self, stage): - finish_trigger = "finish_" + stage - if finish_trigger in self.machine.get_triggers(self.current_state.name): - self.trigger(finish_trigger) diff --git a/atst/models/portfolio_state_machine.py b/atst/models/portfolio_state_machine.py index 3c934197..d7b6a36e 100644 --- a/atst/models/portfolio_state_machine.py +++ b/atst/models/portfolio_state_machine.py @@ -84,13 +84,11 @@ class PortfolioStateMachine( elif self.current_state == FSMStates.STARTED: # get the first trigger that starts with 'create_' - create_trigger = list( - filter( - lambda trigger: trigger.startswith("create_"), - self.machine.get_triggers(FSMStates.STARTED.name), - ) - )[0] - self.trigger(create_trigger) + create_trigger = self._get_first_stage_create_trigger() + if create_trigger: + self.trigger(create_trigger) + else: + self.fail_stage(stage) elif state_obj.is_IN_PROGRESS: pass diff --git a/tests/domain/test_portfolio_state_machine.py b/tests/domain/test_portfolio_state_machine.py index 0aa90867..d0a78fa0 100644 --- a/tests/domain/test_portfolio_state_machine.py +++ b/tests/domain/test_portfolio_state_machine.py @@ -6,6 +6,8 @@ from tests.factories import ( ) from atst.models import FSMStates +from atst.models.mixins.state_machines import AzureStages, StageStates, compose_state +from atst.domain.csp import get_stage_csp_class @pytest.fixture(scope="function") @@ -19,14 +21,67 @@ def test_fsm_creation(portfolio): assert sm.portfolio +def test_state_machine_trigger_next_transition(portfolio): + sm = PortfolioStateMachineFactory.create(portfolio=portfolio) + + sm.trigger_next_transition() + assert sm.current_state == FSMStates.STARTING + + sm.trigger_next_transition() + assert sm.current_state == FSMStates.STARTED + + +def test_state_machine_compose_state(portfolio): + sm = PortfolioStateMachineFactory.create(portfolio=portfolio) + assert ( + compose_state(AzureStages.TENANT, StageStates.CREATED) + == FSMStates.TENANT_CREATED + ) + + +def test_state_machine_first_stage_create_trigger(portfolio): + sm = PortfolioStateMachineFactory.create(portfolio=portfolio) + first_stage_create_trigger = sm._get_first_stage_create_trigger() + first_stage_name = list(AzureStages)[0].name.lower() + assert "create_" + first_stage_name == first_stage_create_trigger + + +def test_state_machine_valid_data_classes_for_stages(portfolio): + sm = PortfolioStateMachineFactory.create(portfolio=portfolio) + for stage in AzureStages: + assert get_stage_csp_class(stage.name.lower(), "payload") is not None + assert get_stage_csp_class(stage.name.lower(), "result") is not None + + +def test_state_machine_initialization(portfolio): + + sm = PortfolioStateMachineFactory.create(portfolio=portfolio) + for stage in AzureStages: + + # check that all stages have a 'create' and 'fail' triggers + stage_name = stage.name.lower() + for trigger_prefix in ["create", "fail"]: + assert hasattr(sm, trigger_prefix + "_" + stage_name) + + # check that machine + in_progress_triggers = sm.machine.get_triggers(stage.name + "_IN_PROGRESS") + assert [ + "reset", + "fail", + "finish_" + stage_name, + "fail_" + stage_name, + ] == in_progress_triggers + + started_triggers = sm.machine.get_triggers("STARTED") + first_stage_create_trigger = sm._get_first_stage_create_trigger() + assert ["reset", "fail", first_stage_create_trigger] == started_triggers + + def test_fsm_transition_start(portfolio): sm = PortfolioStateMachineFactory.create(portfolio=portfolio) assert sm.portfolio assert sm.state == FSMStates.UNSTARTED - # next_state does not create the trigger callbacks !!! - # sm.next_state() - sm.init() assert sm.state == FSMStates.STARTING From b1adaf771d30efbd03b53bfbc81b67b87e4a0724 Mon Sep 17 00:00:00 2001 From: tomdds Date: Thu, 16 Jan 2020 13:44:10 -0500 Subject: [PATCH 07/23] state machine integration wip --- atst/domain/csp/__init__.py | 6 +- atst/domain/csp/cloud.py | 119 ++++++++++++++++--- atst/models/mixins/state_machines.py | 3 + atst/models/portfolio_state_machine.py | 71 ++++++----- tests/domain/cloud/test_azure_csp.py | 14 +-- tests/domain/test_portfolio_state_machine.py | 50 +++++++- 6 files changed, 204 insertions(+), 59 deletions(-) diff --git a/atst/domain/csp/__init__.py b/atst/domain/csp/__init__.py index fc452935..f15ac1cd 100644 --- a/atst/domain/csp/__init__.py +++ b/atst/domain/csp/__init__.py @@ -34,9 +34,7 @@ def make_csp_provider(app, csp=None): def _stage_to_classname(stage): - return "".join( - map(lambda word: word.capitalize(), stage.replace("_", " ").split(" ")) - ) + return "".join(map(lambda word: word.capitalize(), stage.split("_"))) def get_stage_csp_class(stage, class_type): @@ -45,7 +43,7 @@ def get_stage_csp_class(stage, class_type): class_type is either 'payload' or 'result' """ - cls_name = "".join([_stage_to_classname(stage), "CSP", class_type.capitalize()]) + cls_name = f"{_stage_to_classname(stage)}CSP{class_type.capitalize()}" try: return getattr(importlib.import_module("atst.domain.csp.cloud"), cls_name) except AttributeError: diff --git a/atst/domain/csp/cloud.py b/atst/domain/csp/cloud.py index 703dec53..87492f19 100644 --- a/atst/domain/csp/cloud.py +++ b/atst/domain/csp/cloud.py @@ -186,11 +186,29 @@ class TenantCSPResult(AliasModel): tenant_id: str user_object_id: str + tenant_admin_username: str + tenant_admin_password: str + class Config: fields = { "user_object_id": "objectId", } + def dict(self, *args, **kwargs): + exclude = {"tenant_admin_username", "tenant_admin_password"} + if "exclude" not in kwargs: + kwargs["exclude"] = exclude + else: + kwargs["exclude"].update(exclude) + + return super().dict(*args, **kwargs) + + def get_creds(self): + return { + "tenant_admin_username": self.tenant_admin_username, + "tenant_admin_password": self.tenant_admin_password, + "tenant_id": self.tenant_id + } class BillingProfileAddress(AliasModel): company_name: str @@ -215,7 +233,7 @@ class BillingProfileCLINBudget(AliasModel): class BillingProfileCSPPayload(BaseCSPPayload): tenant_id: str - display_name: str + billing_profile_display_name: str enabled_azure_plans: Optional[List[str]] address: BillingProfileAddress @@ -229,6 +247,11 @@ class BillingProfileCSPPayload(BaseCSPPayload): """ return v or [] + class Config: + fields = { + "billing_profile_display_name": "displayName" + } + class BillingProfileCreateCSPResult(AliasModel): billing_profile_validate_url: str @@ -252,9 +275,14 @@ class BillingInvoiceSection(AliasModel): class BillingProfileProperties(AliasModel): address: BillingProfileAddress - display_name: str + billing_profile_display_name: str invoice_sections: List[BillingInvoiceSection] + class Config: + fields = { + "billing_profile_display_name": "displayName" + } + class BillingProfileCSPResult(AliasModel): billing_profile_id: str @@ -269,14 +297,14 @@ class BillingProfileCSPResult(AliasModel): } -class BillingRoleAssignmentCSPPayload(BaseCSPPayload): +class BillingProfileTenantAccessCSPPayload(BaseCSPPayload): tenant_id: str user_object_id: str billing_account_name: str billing_profile_name: str -class BillingRoleAssignmentCSPResult(AliasModel): +class BillingProfileTenantAccessCSPResult(AliasModel): billing_role_assignment_id: str billing_role_assignment_name: str @@ -286,7 +314,7 @@ class BillingRoleAssignmentCSPResult(AliasModel): "billing_role_assignment_name": "name", } -class EnableTaskOrderBillingCSPPayload(BaseCSPPayload): +class TaskOrderBillingCSPPayload(BaseCSPPayload): billing_account_name: str billing_profile_name: str @@ -297,14 +325,14 @@ class EnableTaskOrderBillingCSPResult(AliasModel): class Config: fields = {"task_order_billing_validation_url": "Location", "retry_after": "Retry-After"} -class VerifyTaskOrderBillingCSPPayload(BaseCSPPayload): +class TaskOrderBillingCSPResult(BaseCSPPayload): task_order_billing_validation_url: str class BillingProfileEnabledPlanDetails(AliasModel): enabled_azure_plans: List[Dict] -class BillingProfileEnabledCSPResult(AliasModel): +class TaskOrderBillingCSPResult(AliasModel): billing_profile_id: str billing_profile_name: str billing_profile_enabled_plan_details: BillingProfileEnabledPlanDetails @@ -534,9 +562,11 @@ class MockCloudProvider(CloudProviderInterface): "tenant_id": response["tenantId"], "user_id": response["userId"], "user_object_id": response["objectId"], + "tenant_admin_username": "test", + "tenant_admin_password": "test" } - def create_billing_profile(self, creds, tenant_admin_details, billing_owner_id): + def create_billing_profile(self, payload): # call billing profile creation endpoint, specifying owner # Payload: """ @@ -576,7 +606,55 @@ class MockCloudProvider(CloudProviderInterface): self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) response = {"id": "string"} - return {"billing_profile_id": response["id"]} + # return {"billing_profile_id": response["id"]} + return { + 'id': '/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB', + 'name': 'KQWI-W2SU-BG7-TGB', + 'properties': { + 'address': { + 'addressLine1': '123 S Broad Street, Suite 2400', + 'city': 'Philadelphia', + 'companyName': 'Promptworks', + 'country': 'US', + 'postalCode': '19109', + 'region': 'PA' + }, + 'currency': 'USD', + 'displayName': 'Test Billing Profile', + 'enabledAzurePlans': [], + 'hasReadAccess': True, + 'invoiceDay': 5, + 'invoiceEmailOptIn': False, + 'invoiceSections': [{ + 'id': '/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB/invoiceSections/CHCO-BAAR-PJA-TGB', + 'name': 'CHCO-BAAR-PJA-TGB', + 'properties': { + 'displayName': 'Test Billing Profile' + }, + 'type': 'Microsoft.Billing/billingAccounts/billingProfiles/invoiceSections' + }] + }, + 'type': 'Microsoft.Billing/billingAccounts/billingProfiles' + } + + def create_billing_profile_tenant_access(self, payload): + self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) + self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) + self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) + + return { + "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB/billingRoleAssignments/40000000-aaaa-bbbb-cccc-100000000000_0a5f4926-e3ee-4f47-a6e3-8b0a30a40e3d", + "name": "40000000-aaaa-bbbb-cccc-100000000000_0a5f4926-e3ee-4f47-a6e3-8b0a30a40e3d", + "properties": { + "createdOn": "2020-01-14T14:39:26.3342192+00:00", + "createdByPrincipalId": "82e2b376-3297-4096-8743-ed65b3be0b03", + "principalId": "0a5f4926-e3ee-4f47-a6e3-8b0a30a40e3d", + "principalTenantId": "60ff9d34-82bf-4f21-b565-308ef0533435", + "roleDefinitionId": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB/billingRoleDefinitions/40000000-aaaa-bbbb-cccc-100000000000", + "scope": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB" + }, + "type": "Microsoft.Billing/billingRoleAssignments" + } def create_or_update_user(self, auth_credentials, user_info, csp_role_id): self._authorize(auth_credentials) @@ -633,7 +711,7 @@ class MockCloudProvider(CloudProviderInterface): @property def _auth_credentials(self): - return {"username": "mock-cloud", "pass": "shh"} + return {"username": "mock-cloud", "password": "shh"} def _authorize(self, credentials): self._delay(1, 5) @@ -778,6 +856,9 @@ class AzureCloudProvider(CloudProviderInterface): headers=create_tenant_headers, ) + print('create tenant result') + print(result.json()) + if result.status_code == 200: return self._ok(TenantCSPResult(**result.json())) else: @@ -836,7 +917,7 @@ class AzureCloudProvider(CloudProviderInterface): else: return self._error(result.json()) - def grant_billing_profile_tenant_access(self, payload: BillingRoleAssignmentCSPPayload): + def create_billing_profile_tenant_access(self, payload: BillingProfileTenantAccessCSPPayload): sp_token = self._get_sp_token(payload.creds) request_body = { "properties": { @@ -854,11 +935,11 @@ class AzureCloudProvider(CloudProviderInterface): result = self.sdk.requests.post(url, headers=headers, json=request_body) if result.status_code == 201: - return self._ok(BillingRoleAssignmentCSPResult(**result.json())) + return self._ok(BillingProfileTenantAccessCSPResult(**result.json())) else: return self._error(result.json()) - def enable_task_order_billing(self, payload: EnableTaskOrderBillingCSPPayload): + def enable_task_order_billing(self, payload: TaskOrderBillingCSPPayload): sp_token = self._get_sp_token(payload.creds) request_body = [ { @@ -884,7 +965,7 @@ class AzureCloudProvider(CloudProviderInterface): # 202 has location/retry after headers return self._ok(BillingProfileCreateCSPResult(**result.headers)) elif result.status_code == 200: - return self._ok(BillingProfileEnabledCSPResult(**result.json())) + return self._ok(TaskOrderBillingCSPResult(**result.json())) else: return self._error(result.json()) @@ -903,13 +984,13 @@ class AzureCloudProvider(CloudProviderInterface): if result.status_code == 202: # 202 has location/retry after headers - return self._ok(EnableTaskOrderBillingCSPResult(**result.headers)) + return self._ok(TaskOrderBillingCSPResult(**result.headers)) elif result.status_code == 200: - return self._ok(BillingProfileEnabledCSPResult(**result.json())) + return self._ok(TaskOrderBillingCSPResult(**result.json())) else: return self._error(result.json()) - def report_clin(self, payload: ReportCLINCSPPayload): + def create_billing_instruction(self, payload: ReportCLINCSPPayload): sp_token = self._get_sp_token(payload.creds) if sp_token is None: raise AuthenticationException( @@ -1022,7 +1103,9 @@ class AzureCloudProvider(CloudProviderInterface): return sub_id_match.group(1) def _get_sp_token(self, creds): - home_tenant_id = creds.get("home_tenant_id") + home_tenant_id = creds.get( + "home_tenant_id" + ) client_id = creds.get("client_id") secret_key = creds.get("secret_key") diff --git a/atst/models/mixins/state_machines.py b/atst/models/mixins/state_machines.py index b2eda399..43044be0 100644 --- a/atst/models/mixins/state_machines.py +++ b/atst/models/mixins/state_machines.py @@ -10,6 +10,9 @@ class StageStates(Enum): class AzureStages(Enum): TENANT = "tenant" BILLING_PROFILE = "billing profile" + BILLING_PROFILE_TENANT_ACCESS = "billing profile tenant access" + TASK_ORDER_BILLING = "task order billing" + BILLING_INSTRUCTION = "billing instruction" def _build_csp_states(csp_stages): diff --git a/atst/models/portfolio_state_machine.py b/atst/models/portfolio_state_machine.py index d7b6a36e..aef03b22 100644 --- a/atst/models/portfolio_state_machine.py +++ b/atst/models/portfolio_state_machine.py @@ -1,3 +1,7 @@ +from random import choice, choices +import re +import string + from sqlalchemy import Column, ForeignKey, Enum as SQLAEnum from sqlalchemy.orm import relationship, reconstructor from sqlalchemy.dialects.postgresql import UUID @@ -17,6 +21,16 @@ import atst.models.mixins as mixins from atst.models.mixins.state_machines import FSMStates, AzureStages, _build_transitions +def make_password(): + return choice(string.ascii_letters) + "".join( + choices(string.ascii_letters + string.digits + string.punctuation, k=15) + ) + + +def fetch_portfolio_creds(portfolio): + return dict(username="mock-cloud", password="shh") + + @add_state_features(Tags) class StateMachineWithTags(Machine): pass @@ -73,57 +87,49 @@ class PortfolioStateMachine( return getattr(FSMStates, self.state) return self.state - def trigger_next_transition(self): + def trigger_next_transition(self, **kwargs): state_obj = self.machine.get_state(self.state) if state_obj.is_system: if self.current_state in (FSMStates.UNSTARTED, FSMStates.STARTING): # call the first trigger availabe for these two system states trigger_name = self.machine.get_triggers(self.current_state.name)[0] - self.trigger(trigger_name) + self.trigger(trigger_name, **kwargs) elif self.current_state == FSMStates.STARTED: # get the first trigger that starts with 'create_' create_trigger = self._get_first_stage_create_trigger() if create_trigger: - self.trigger(create_trigger) + self.trigger(create_trigger, **kwargs) else: self.fail_stage(stage) - elif state_obj.is_IN_PROGRESS: - pass - - # elif state_obj.is_TENANT: - # pass - # elif state_obj.is_BILLING_PROFILE: - # pass + elif state_obj.is_CREATED: + triggers = self.machine.get_triggers(state_obj.name) + self.trigger(triggers[-1], **kwargs) # @with_payload def after_in_progress_callback(self, event): stage = self.current_state.name.split("_IN_PROGRESS")[0].lower() - if stage == "tenant": - payload = dict( # nosec - creds={"username": "mock-cloud", "pass": "shh"}, - user_id="123", - password="123", - domain_name="123", - first_name="john", - last_name="doe", - country_code="US", - password_recovery_email_address="password@email.com", - ) - elif stage == "billing_profile": - payload = dict(creds={"username": "mock-cloud", "pass": "shh"},) + + # Accumulate payload w/ creds + payload = event.kwargs.get("csp_data") + payload["creds"] = event.kwargs.get("creds") payload_data_cls = get_stage_csp_class(stage, "payload") if not payload_data_cls: + print("could not resolve payload data class") self.fail_stage(stage) try: payload_data = payload_data_cls(**payload) except PydanticValidationError as exc: + print("Payload Validation Error:") print(exc.json()) + print("got") + print(payload) self.fail_stage(stage) + # TODO: Determine best place to do this, maybe @reconstructor csp = event.kwargs.get("csp") if csp is not None: self.csp = AzureCSP(app).cloud @@ -132,7 +138,8 @@ class PortfolioStateMachine( for attempt in range(5): try: - response = getattr(self.csp, "create_" + stage)(payload_data) + func_name = f"create_{stage}" + response = getattr(self.csp, func_name)(payload_data) except (ConnectionException, UnknownServerException) as exc: print("caught exception. retry", attempt) continue @@ -140,14 +147,17 @@ class PortfolioStateMachine( break else: # failed all attempts + print("failed") self.fail_stage(stage) if self.portfolio.csp_data is None: self.portfolio.csp_data = {} - self.portfolio.csp_data[stage + "_data"] = response + self.portfolio.csp_data.update(response) db.session.add(self.portfolio) db.session.commit() + # store any updated creds, if necessary + self.finish_stage(stage) def is_csp_data_valid(self, event): @@ -156,16 +166,23 @@ class PortfolioStateMachine( if self.portfolio.csp_data is None or not isinstance( self.portfolio.csp_data, dict ): + print("no csp data") return False stage = self.current_state.name.split("_IN_PROGRESS")[0].lower() - stage_data = self.portfolio.csp_data.get(stage + "_data") + stage_data = self.portfolio.csp_data cls = get_stage_csp_class(stage, "result") if not cls: return False try: - cls(**stage_data) + dc = cls(**stage_data) + if getattr(dc, "get_creds", None) is not None: + new_creds = dc.get_creds() + # TODO: how/where to store these + # TODO: credential schema + # self.store_creds(self.portfolio, new_creds) + except PydanticValidationError as exc: print(exc.json()) return False diff --git a/tests/domain/cloud/test_azure_csp.py b/tests/domain/cloud/test_azure_csp.py index 9e9809c7..bf228874 100644 --- a/tests/domain/cloud/test_azure_csp.py +++ b/tests/domain/cloud/test_azure_csp.py @@ -153,7 +153,7 @@ def test_create_tenant(mock_azure: AzureCloudProvider): mock_azure.sdk.requests.post.return_value = mock_result payload = TenantCSPPayload( **dict( - creds={"username": "mock-cloud", "pass": "shh"}, + creds={"username": "mock-cloud", "password": "shh"}, user_id="admin", password="JediJan13$coot", domain_name="jediccpospawnedtenant2", @@ -190,7 +190,7 @@ def test_create_billing_profile(mock_azure: AzureCloudProvider): country="US", postal_code="19109", ), - creds={"username": "mock-cloud", "pass": "shh"}, + creds={"username": "mock-cloud", "password": "shh"}, tenant_id="60ff9d34-82bf-4f21-b565-308ef0533435", display_name="Test Billing Profile", ) @@ -258,7 +258,7 @@ def test_validate_billing_profile_creation(mock_azure: AzureCloudProvider): ) -def test_grant_billing_profile_tenant_access(mock_azure: AzureCloudProvider): +def test_create_billing_profile_tenant_access(mock_azure: AzureCloudProvider): mock_azure.sdk.adal.AuthenticationContext.return_value.context.acquire_token_with_client_credentials.return_value = { "accessToken": "TOKEN" } @@ -295,7 +295,7 @@ def test_grant_billing_profile_tenant_access(mock_azure: AzureCloudProvider): ) ) - result = mock_azure.grant_billing_profile_tenant_access(payload) + result = mock_azure.create_billing_profile_tenant_access(payload) body: BillingRoleAssignmentCSPResult = result.get("body") assert ( body.billing_role_assignment_name @@ -303,7 +303,7 @@ def test_grant_billing_profile_tenant_access(mock_azure: AzureCloudProvider): ) -def test_enable_task_order_billing(mock_azure: AzureCloudProvider): +def test_create_task_order_billing(mock_azure: AzureCloudProvider): mock_azure.sdk.adal.AuthenticationContext.return_value.context.acquire_token_with_client_credentials.return_value = { "accessToken": "TOKEN" } @@ -401,7 +401,7 @@ def test_validate_task_order_billing_enabled(mock_azure): ) -def test_report_clin(mock_azure: AzureCloudProvider): +def test_create_billing_instruction(mock_azure: AzureCloudProvider): mock_azure.sdk.adal.AuthenticationContext.return_value.context.acquire_token_with_client_credentials.return_value = { "accessToken": "TOKEN" } @@ -432,7 +432,7 @@ def test_report_clin(mock_azure: AzureCloudProvider): billing_profile_name="KQWI-W2SU-BG7-TGB", ) ) - result = mock_azure.report_clin(payload) + result = mock_azure.create_billing_instruction(payload) body: ReportCLINCSPResult = result.get("body") assert body.reported_clin_name == "TO1:CLIN001" diff --git a/tests/domain/test_portfolio_state_machine.py b/tests/domain/test_portfolio_state_machine.py index d0a78fa0..9bd2c842 100644 --- a/tests/domain/test_portfolio_state_machine.py +++ b/tests/domain/test_portfolio_state_machine.py @@ -1,11 +1,12 @@ import pytest +import re from tests.factories import ( PortfolioFactory, PortfolioStateMachineFactory, ) -from atst.models import FSMStates +from atst.models import FSMStates, PortfolioStateMachine from atst.models.mixins.state_machines import AzureStages, StageStates, compose_state from atst.domain.csp import get_stage_csp_class @@ -78,7 +79,7 @@ def test_state_machine_initialization(portfolio): def test_fsm_transition_start(portfolio): - sm = PortfolioStateMachineFactory.create(portfolio=portfolio) + sm: PortfolioStateMachine = PortfolioStateMachineFactory.create(portfolio=portfolio) assert sm.portfolio assert sm.state == FSMStates.UNSTARTED @@ -87,5 +88,48 @@ def test_fsm_transition_start(portfolio): sm.start() assert sm.state == FSMStates.STARTED - sm.create_tenant(a=1, b=2) + + # Should source all creds for portfolio? might be easier to manage than per-step specific ones + creds = {"username": "mock-cloud", "password": "shh"} + if portfolio.csp_data is not None: + csp_data = portfolio.csp_data + else: + csp_data = {} + + ppoc = portfolio.owner + user_id = f"{ppoc.first_name[0]}{ppoc.last_name}".lower() + domain_name = re.sub("[^0-9a-zA-Z]+", "", portfolio.name).lower() + + portfolio_data = { + "user_id": user_id, + "password": "jklfsdNCVD83nklds2#202", + "domain_name": domain_name, + "first_name": ppoc.first_name, + "last_name": ppoc.last_name, + "country_code": "US", + "password_recovery_email_address": ppoc.email, + "address": { + "company_name": "", + "address_line_1": "", + "city": "", + "region": "", + "country": "", + "postal_code": "", + }, + "billing_profile_display_name": "My Billing Profile", + } + + collected_data = dict(list(csp_data.items()) + list(portfolio_data.items())) + sm.trigger_next_transition(creds=creds, csp_data=collected_data) + assert sm.state == FSMStates.TENANT_CREATED + assert portfolio.csp_data.get("tenant_id", None) is not None + + if portfolio.csp_data is not None: + csp_data = portfolio.csp_data + else: + csp_data = {} + collected_data = dict(list(csp_data.items()) + list(portfolio_data.items())) + sm.trigger_next_transition(creds=creds, csp_data=collected_data) + assert sm.state == FSMStates.BILLING_PROFILE_CREATED + From 743a91d658d101938788c5fb8356eebea2a4d9ed Mon Sep 17 00:00:00 2001 From: Philip Kalinsky Date: Thu, 16 Jan 2020 14:50:16 -0500 Subject: [PATCH 08/23] trigger_next_transition method will call the create trigger of the next stage for machines in CREATED state --- atst/models/portfolio_state_machine.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/atst/models/portfolio_state_machine.py b/atst/models/portfolio_state_machine.py index aef03b22..c5a5bb2d 100644 --- a/atst/models/portfolio_state_machine.py +++ b/atst/models/portfolio_state_machine.py @@ -105,8 +105,17 @@ class PortfolioStateMachine( self.fail_stage(stage) elif state_obj.is_CREATED: + # the create trigger for the next stage should be in the available + # triggers for the current state triggers = self.machine.get_triggers(state_obj.name) - self.trigger(triggers[-1], **kwargs) + create_trigger = list( + filter( + lambda trigger: trigger.startswith("create_"), + self.machine.get_triggers(self.state.name), + ) + )[0] + if create_trigger: + self.trigger(create_trigger, **kwargs) # @with_payload def after_in_progress_callback(self, event): From dfaea2d937270b954c0ac5509d9b46ffa5e4f91f Mon Sep 17 00:00:00 2001 From: Philip Kalinsky Date: Thu, 16 Jan 2020 15:29:14 -0500 Subject: [PATCH 09/23] trigger_next_transition method will call the create trigger of the next stage for machines in CREATED state --- atst/models/portfolio_state_machine.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/atst/models/portfolio_state_machine.py b/atst/models/portfolio_state_machine.py index c5a5bb2d..d3794a04 100644 --- a/atst/models/portfolio_state_machine.py +++ b/atst/models/portfolio_state_machine.py @@ -108,13 +108,17 @@ class PortfolioStateMachine( # the create trigger for the next stage should be in the available # triggers for the current state triggers = self.machine.get_triggers(state_obj.name) - create_trigger = list( - filter( - lambda trigger: trigger.startswith("create_"), - self.machine.get_triggers(self.state.name), - ) - )[0] - if create_trigger: + try: + create_trigger = list( + filter( + lambda trigger: trigger.startswith("create_"), + self.machine.get_triggers(self.state.name), + ) + )[0] + except IndexError: + # are we done ? + pass + else: self.trigger(create_trigger, **kwargs) # @with_payload From 67842748b8e4950c402682fac2fcd3ad3a5503d4 Mon Sep 17 00:00:00 2001 From: Philip Kalinsky Date: Thu, 16 Jan 2020 16:14:01 -0500 Subject: [PATCH 10/23] call next on the results of filter function to get the first value --- atst/models/mixins/state_machines.py | 8 -------- atst/models/portfolio_state_machine.py | 28 ++++++++++++++------------ 2 files changed, 15 insertions(+), 21 deletions(-) diff --git a/atst/models/mixins/state_machines.py b/atst/models/mixins/state_machines.py index 43044be0..6938c8a9 100644 --- a/atst/models/mixins/state_machines.py +++ b/atst/models/mixins/state_machines.py @@ -111,14 +111,6 @@ class FSMMixin: if finish_trigger in self.machine.get_triggers(self.current_state.name): self.trigger(finish_trigger) - def _get_first_stage_create_trigger(self): - return list( - filter( - lambda trigger: trigger.startswith("create_"), - self.machine.get_triggers(FSMStates.STARTED.name), - ) - )[0] - def prepare_init(self, event): pass diff --git a/atst/models/portfolio_state_machine.py b/atst/models/portfolio_state_machine.py index d3794a04..b8f29f24 100644 --- a/atst/models/portfolio_state_machine.py +++ b/atst/models/portfolio_state_machine.py @@ -98,7 +98,13 @@ class PortfolioStateMachine( elif self.current_state == FSMStates.STARTED: # get the first trigger that starts with 'create_' - create_trigger = self._get_first_stage_create_trigger() + create_trigger = next( + filter( + lambda trigger: trigger.startswith("create_"), + self.machine.get_triggers(FSMStates.STARTED.name), + ), + None, + ) if create_trigger: self.trigger(create_trigger, **kwargs) else: @@ -108,20 +114,16 @@ class PortfolioStateMachine( # the create trigger for the next stage should be in the available # triggers for the current state triggers = self.machine.get_triggers(state_obj.name) - try: - create_trigger = list( - filter( - lambda trigger: trigger.startswith("create_"), - self.machine.get_triggers(self.state.name), - ) - )[0] - except IndexError: - # are we done ? - pass - else: + create_trigger = next( + filter( + lambda trigger: trigger.startswith("create_"), + self.machine.get_triggers(self.state.name), + ), + None, + ) + if create_trigger is not None: self.trigger(create_trigger, **kwargs) - # @with_payload def after_in_progress_callback(self, event): stage = self.current_state.name.split("_IN_PROGRESS")[0].lower() From becc3630c2f7e7d3de2c466e9b22f89b483213dc Mon Sep 17 00:00:00 2001 From: Philip Kalinsky Date: Mon, 20 Jan 2020 14:17:01 -0500 Subject: [PATCH 11/23] azure integration. methods to authenticate and set/get value in keyvault --- Pipfile | 1 + Pipfile.lock | 73 +++++++++++++------ atst/domain/csp/cloud.py | 149 ++++++++++++++++++++++++++------------- 3 files changed, 152 insertions(+), 71 deletions(-) diff --git a/Pipfile b/Pipfile index f1d852b9..06bd2675 100644 --- a/Pipfile +++ b/Pipfile @@ -35,6 +35,7 @@ azure-mgmt-resource = "*" transitions = "*" azure-mgmt-consumption = "*" adal = "*" +azure-identity = "*" [dev-packages] bandit = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 681fddb1..8a030e11 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "a127b88e6c64842786f1868cb93bb1cdc828aa78040ea8ba4079bb3de0316dab" + "sha256": "3760f0b1df1156211d671afa2eb417b7bf980aa33d2f74d390e8eed6a3ce8c8b" }, "pipfile-spec": 6, "requires": { @@ -45,6 +45,13 @@ ], "version": "==1.1.24" }, + "azure-core": { + "hashes": [ + "sha256:b8ccbd901d085048e4e3e72627b066923c5bd3780e4c43cf9cf9948aee9bdf9e", + "sha256:e2cd99f0c0aef12c168d498cb5bc47a3a45c8ab08112183e3ec97e4dcb33ceb9" + ], + "version": "==1.2.1" + }, "azure-graphrbac": { "hashes": [ "sha256:53e98ae2ca7c19b349e9e9bb1b6a824aeae8dcfcbe17190d20fe69c0f185b2e2", @@ -53,6 +60,14 @@ "index": "pypi", "version": "==0.61.1" }, + "azure-identity": { + "hashes": [ + "sha256:4ce65058461c277991763ed3f121efc6b9eb9c2edefb62c414dfa85c814690d3", + "sha256:b32acd1cdb6202bfe10d9a0858dc463d8960295da70ae18097eb3b85ab12cb91" + ], + "index": "pypi", + "version": "==1.2.0" + }, "azure-mgmt-authorization": { "hashes": [ "sha256:31e875a34ac2c5d6fefe77b4a8079a8b2bdbe9edb957e47e8b44222fb212d6a7", @@ -354,6 +369,20 @@ ], "version": "==8.1.0" }, + "msal": { + "hashes": [ + "sha256:c944b833bf686dfbc973e9affdef94b77e616cb52ab397e76cde82e26b8a3373", + "sha256:ecbe3f5ac77facad16abf08eb9d8562af3bc7184be5d4d90c9ef4db5bde26340" + ], + "version": "==1.0.0" + }, + "msal-extensions": { + "hashes": [ + "sha256:59e171a9a4baacdbf001c66915efeaef372fb424421f1a4397115a3ddd6205dc", + "sha256:c5a32b8e1dce1c67733dcdf8aa8bebcff5ab123e779ef7bc14e416bd0da90037" + ], + "version": "==0.1.3" + }, "msrest": { "hashes": [ "sha256:56b8b5b4556fb2a92cac640df267d560889bdc9e2921187772d4691d97bc4e8d", @@ -388,6 +417,13 @@ "index": "pypi", "version": "==2.0.5" }, + "portalocker": { + "hashes": [ + "sha256:6f57aabb25ba176462dc7c63b86c42ad6a9b5bd3d679a9d776d0536bfb803d54", + "sha256:dac62e53e5670cb40d2ee4cdc785e6b829665932c3ee75307ad677cf5f7d2e9f" + ], + "version": "==1.5.2" + }, "psycopg2-binary": { "hashes": [ "sha256:040234f8a4a8dfd692662a8308d78f63f31a97e1c42d2480e5e6810c48966a29", @@ -453,6 +489,9 @@ "version": "==1.3" }, "pyjwt": { + "extras": [ + "crypto" + ], "hashes": [ "sha256:5c6eca3c2940464d106b99ba83b00c6add741c9becaec087fb7ccdefea71350e", "sha256:8d59a976fb773f3e6a39c85636357c4f0e242707394cadadd9814f5cbaa20e96" @@ -538,10 +577,10 @@ }, "six": { "hashes": [ - "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd", - "sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66" + "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a", + "sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c" ], - "version": "==1.13.0" + "version": "==1.14.0" }, "sqlalchemy": { "hashes": [ @@ -618,10 +657,10 @@ }, "zipp": { "hashes": [ - "sha256:8dda78f06bd1674bd8720df8a50bb47b6e1233c503a4eed8e7810686bde37656", - "sha256:d38fbe01bbf7a3593a32bc35a9c4453c32bc42b98c377f9bff7e9f8da157786c" + "sha256:57147f6b0403b59f33fd357f169f860e031303415aeb7d04ede4839d23905ab8", + "sha256:7ae5ccaca427bafa9760ac3cd8f8c244bfc259794b5b6bb9db4dda2241575d09" ], - "version": "==1.0.0" + "version": "==2.0.0" } }, "develop": { @@ -632,14 +671,6 @@ ], "version": "==1.4.3" }, - "appnope": { - "hashes": [ - "sha256:5b26757dc6f79a3b7dc9fab95359328d5747fcb2409d331ea66d0272b90ab2a0", - "sha256:8b995ffe925347a2138d7ac0fe77155e4311a0ea6d6da4f5128fe4b3cbe5ed71" - ], - "markers": "sys_platform == 'darwin'", - "version": "==0.1.0" - }, "argh": { "hashes": [ "sha256:a9b3aaa1904eeb78e32394cd46c6f37ac0fb4af6dc488daa58971bdc7d7fcaf3", @@ -1210,10 +1241,10 @@ }, "six": { "hashes": [ - "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd", - "sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66" + "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a", + "sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c" ], - "version": "==1.13.0" + "version": "==1.14.0" }, "smmap2": { "hashes": [ @@ -1328,10 +1359,10 @@ }, "zipp": { "hashes": [ - "sha256:8dda78f06bd1674bd8720df8a50bb47b6e1233c503a4eed8e7810686bde37656", - "sha256:d38fbe01bbf7a3593a32bc35a9c4453c32bc42b98c377f9bff7e9f8da157786c" + "sha256:57147f6b0403b59f33fd357f169f860e031303415aeb7d04ede4839d23905ab8", + "sha256:7ae5ccaca427bafa9760ac3cd8f8c244bfc259794b5b6bb9db4dda2241575d09" ], - "version": "==1.0.0" + "version": "==2.0.0" } } } diff --git a/atst/domain/csp/cloud.py b/atst/domain/csp/cloud.py index 87492f19..f14e468d 100644 --- a/atst/domain/csp/cloud.py +++ b/atst/domain/csp/cloud.py @@ -207,9 +207,10 @@ class TenantCSPResult(AliasModel): return { "tenant_admin_username": self.tenant_admin_username, "tenant_admin_password": self.tenant_admin_password, - "tenant_id": self.tenant_id + "tenant_id": self.tenant_id, } + class BillingProfileAddress(AliasModel): company_name: str address_line_1: str @@ -248,9 +249,7 @@ class BillingProfileCSPPayload(BaseCSPPayload): return v or [] class Config: - fields = { - "billing_profile_display_name": "displayName" - } + fields = {"billing_profile_display_name": "displayName"} class BillingProfileCreateCSPResult(AliasModel): @@ -258,7 +257,10 @@ class BillingProfileCreateCSPResult(AliasModel): retry_after: int class Config: - fields = {"billing_profile_validate_url": "Location", "retry_after": "Retry-After"} + fields = { + "billing_profile_validate_url": "Location", + "retry_after": "Retry-After", + } class BillingProfileVerifyCSPPayload(BaseCSPPayload): @@ -279,9 +281,7 @@ class BillingProfileProperties(AliasModel): invoice_sections: List[BillingInvoiceSection] class Config: - fields = { - "billing_profile_display_name": "displayName" - } + fields = {"billing_profile_display_name": "displayName"} class BillingProfileCSPResult(AliasModel): @@ -314,20 +314,27 @@ class BillingProfileTenantAccessCSPResult(AliasModel): "billing_role_assignment_name": "name", } + class TaskOrderBillingCSPPayload(BaseCSPPayload): billing_account_name: str billing_profile_name: str + class EnableTaskOrderBillingCSPResult(AliasModel): task_order_billing_validation_url: str retry_after: int class Config: - fields = {"task_order_billing_validation_url": "Location", "retry_after": "Retry-After"} + fields = { + "task_order_billing_validation_url": "Location", + "retry_after": "Retry-After", + } + class TaskOrderBillingCSPResult(BaseCSPPayload): task_order_billing_validation_url: str + class BillingProfileEnabledPlanDetails(AliasModel): enabled_azure_plans: List[Dict] @@ -344,6 +351,7 @@ class TaskOrderBillingCSPResult(AliasModel): "billing_profile_enabled_plan_details": "properties", } + class ReportCLINCSPPayload(BaseCSPPayload): amount: float start_date: str @@ -353,6 +361,7 @@ class ReportCLINCSPPayload(BaseCSPPayload): billing_account_name: str billing_profile_name: str + class ReportCLINCSPResult(AliasModel): reported_clin_name: str @@ -361,7 +370,16 @@ class ReportCLINCSPResult(AliasModel): "reported_clin_name": "name", } + class CloudProviderInterface: + + + def set_secret(secret_key: str, secret_value: str): + raise NotImplementedError() + + def get_secret(secret_key: str, secret_value: str): + raise NotImplementedError() + def root_creds(self) -> Dict: raise NotImplementedError() @@ -563,7 +581,7 @@ class MockCloudProvider(CloudProviderInterface): "user_id": response["userId"], "user_object_id": response["objectId"], "tenant_admin_username": "test", - "tenant_admin_password": "test" + "tenant_admin_password": "test", } def create_billing_profile(self, payload): @@ -608,33 +626,33 @@ class MockCloudProvider(CloudProviderInterface): response = {"id": "string"} # return {"billing_profile_id": response["id"]} return { - 'id': '/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB', - 'name': 'KQWI-W2SU-BG7-TGB', - 'properties': { - 'address': { - 'addressLine1': '123 S Broad Street, Suite 2400', - 'city': 'Philadelphia', - 'companyName': 'Promptworks', - 'country': 'US', - 'postalCode': '19109', - 'region': 'PA' + "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB", + "name": "KQWI-W2SU-BG7-TGB", + "properties": { + "address": { + "addressLine1": "123 S Broad Street, Suite 2400", + "city": "Philadelphia", + "companyName": "Promptworks", + "country": "US", + "postalCode": "19109", + "region": "PA", }, - 'currency': 'USD', - 'displayName': 'Test Billing Profile', - 'enabledAzurePlans': [], - 'hasReadAccess': True, - 'invoiceDay': 5, - 'invoiceEmailOptIn': False, - 'invoiceSections': [{ - 'id': '/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB/invoiceSections/CHCO-BAAR-PJA-TGB', - 'name': 'CHCO-BAAR-PJA-TGB', - 'properties': { - 'displayName': 'Test Billing Profile' - }, - 'type': 'Microsoft.Billing/billingAccounts/billingProfiles/invoiceSections' - }] + "currency": "USD", + "displayName": "Test Billing Profile", + "enabledAzurePlans": [], + "hasReadAccess": True, + "invoiceDay": 5, + "invoiceEmailOptIn": False, + "invoiceSections": [ + { + "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB/invoiceSections/CHCO-BAAR-PJA-TGB", + "name": "CHCO-BAAR-PJA-TGB", + "properties": {"displayName": "Test Billing Profile"}, + "type": "Microsoft.Billing/billingAccounts/billingProfiles/invoiceSections", + } + ], }, - 'type': 'Microsoft.Billing/billingAccounts/billingProfiles' + "type": "Microsoft.Billing/billingAccounts/billingProfiles", } def create_billing_profile_tenant_access(self, payload): @@ -651,9 +669,9 @@ class MockCloudProvider(CloudProviderInterface): "principalId": "0a5f4926-e3ee-4f47-a6e3-8b0a30a40e3d", "principalTenantId": "60ff9d34-82bf-4f21-b565-308ef0533435", "roleDefinitionId": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB/billingRoleDefinitions/40000000-aaaa-bbbb-cccc-100000000000", - "scope": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB" + "scope": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB", }, - "type": "Microsoft.Billing/billingRoleAssignments" + "type": "Microsoft.Billing/billingRoleAssignments", } def create_or_update_user(self, auth_credentials, user_info, csp_role_id): @@ -735,6 +753,9 @@ class AzureSDKProvider(object): from azure.mgmt import subscription, authorization import azure.graphrbac as graphrbac import azure.common.credentials as credentials + import azure.identity as identity + from azure.keyvault import secrets import secrets + from msrestazure.azure_cloud import AZURE_PUBLIC_CLOUD import adal import requests @@ -744,6 +765,8 @@ class AzureSDKProvider(object): self.adal = adal self.graphrbac = graphrbac self.credentials = credentials + self.identity = identity + self.secrets = secrets self.requests = requests # may change to a JEDI cloud self.cloud = AZURE_PUBLIC_CLOUD @@ -756,12 +779,29 @@ class AzureCloudProvider(CloudProviderInterface): self.client_id = config["AZURE_CLIENT_ID"] self.secret_key = config["AZURE_SECRET_KEY"] self.tenant_id = config["AZURE_TENANT_ID"] + self.vault_url = config["AZURE_VAULT_URL"] if azure_sdk_provider is None: self.sdk = AzureSDKProvider() else: self.sdk = azure_sdk_provider + def set_secret(secret_key, secret_value): + credential = self._get_client_secret_credential_obj() + secret_client = self.secrets.SecretClient( + vault_url=self.vault_url, + credential=credential, + ) + return secret_client.set_secret(secret_key, secret_value) + + def get_secret(secret_key) + credential = self._get_client_secret_credential_obj() + secret_client = self.secrets.SecretClient( + vault_url=self.vault_url, + credential=credential, + ) + return secret_client.get_secret(secret_key).value + def create_environment( self, auth_credentials: Dict, user: User, environment: Environment ): @@ -856,7 +896,7 @@ class AzureCloudProvider(CloudProviderInterface): headers=create_tenant_headers, ) - print('create tenant result') + print("create tenant result") print(result.json()) if result.status_code == 200: @@ -907,7 +947,9 @@ class AzureCloudProvider(CloudProviderInterface): "Authorization": f"Bearer {sp_token}", } - result = self.sdk.requests.get(payload.billing_profile_validate_url, headers=auth_header) + result = self.sdk.requests.get( + payload.billing_profile_validate_url, headers=auth_header + ) if result.status_code == 202: # 202 has location/retry after headers @@ -917,7 +959,9 @@ class AzureCloudProvider(CloudProviderInterface): else: return self._error(result.json()) - def create_billing_profile_tenant_access(self, payload: BillingProfileTenantAccessCSPPayload): + def create_billing_profile_tenant_access( + self, payload: BillingProfileTenantAccessCSPPayload + ): sp_token = self._get_sp_token(payload.creds) request_body = { "properties": { @@ -945,11 +989,7 @@ class AzureCloudProvider(CloudProviderInterface): { "op": "replace", "path": "/enabledAzurePlans", - "value": [ - { - "skuId": "0001" - } - ] + "value": [{"skuId": "0001"}], } ] @@ -959,7 +999,9 @@ class AzureCloudProvider(CloudProviderInterface): url = f"https://management.azure.com/providers/Microsoft.Billing/billingAccounts/{payload.billing_account_name}/billingProfiles/{payload.billing_profile_name}?api-version=2019-10-01-preview" - result = self.sdk.requests.patch(url, headers=request_headers, json=request_body) + result = self.sdk.requests.patch( + url, headers=request_headers, json=request_body + ) if result.status_code == 202: # 202 has location/retry after headers @@ -969,7 +1011,7 @@ class AzureCloudProvider(CloudProviderInterface): else: return self._error(result.json()) - def validate_task_order_billing_enabled(self, payload: VerifyTaskOrderBillingCSPPayload): + def validate_task_order_billing_enabled(self, payload: TaskOrderBillingCSPPayload): sp_token = self._get_sp_token(payload.creds) if sp_token is None: raise AuthenticationException( @@ -980,7 +1022,9 @@ class AzureCloudProvider(CloudProviderInterface): "Authorization": f"Bearer {sp_token}", } - result = self.sdk.requests.get(payload.task_order_billing_validation_url, headers=auth_header) + result = self.sdk.requests.get( + payload.task_order_billing_validation_url, headers=auth_header + ) if result.status_code == 202: # 202 has location/retry after headers @@ -1001,7 +1045,7 @@ class AzureCloudProvider(CloudProviderInterface): "properties": { "amount": payload.amount, "startDate": payload.start_date, - "endDate": payload.end_date + "endDate": payload.end_date, } } @@ -1125,7 +1169,6 @@ class AzureCloudProvider(CloudProviderInterface): return token_response.get("accessToken", None) def _get_credential_obj(self, creds, resource=None): - return self.sdk.credentials.ServicePrincipalCredentials( client_id=creds.get("client_id"), secret=creds.get("secret_key"), @@ -1133,6 +1176,12 @@ class AzureCloudProvider(CloudProviderInterface): resource=resource, cloud_environment=self.sdk.cloud, ) + def _get_client_secret_credential_obj(): + return self.sdk.identity.ClientSecretCredential( + tenant_id=creds.get("tenant_id"), + client_id =creds.get("client_id"), + client_secret = creds.get("secret_key"), + ) def _make_tenant_admin_cred_obj(self, username, password): return self.sdk.credentials.UserPassCredentials(username, password) From d646c3c00fc71b4922db74a480639fb4d08d3499 Mon Sep 17 00:00:00 2001 From: tomdds Date: Mon, 20 Jan 2020 12:51:02 -0500 Subject: [PATCH 12/23] Updates from Production Scripts Made a bunch of tweaks when using these tests to run production scripts for initial setup, this brings over a bunch of those changes --- atst/domain/csp/cloud.py | 22 +++++-------- tests/domain/cloud/test_azure_csp.py | 46 ++++++++++++++-------------- 2 files changed, 31 insertions(+), 37 deletions(-) diff --git a/atst/domain/csp/cloud.py b/atst/domain/csp/cloud.py index f14e468d..cd958781 100644 --- a/atst/domain/csp/cloud.py +++ b/atst/domain/csp/cloud.py @@ -235,6 +235,7 @@ class BillingProfileCLINBudget(AliasModel): class BillingProfileCSPPayload(BaseCSPPayload): tenant_id: str billing_profile_display_name: str + billing_account_name: str enabled_azure_plans: Optional[List[str]] address: BillingProfileAddress @@ -321,7 +322,7 @@ class TaskOrderBillingCSPPayload(BaseCSPPayload): class EnableTaskOrderBillingCSPResult(AliasModel): - task_order_billing_validation_url: str + task_order_billing_validate_url: str retry_after: int class Config: @@ -331,8 +332,8 @@ class EnableTaskOrderBillingCSPResult(AliasModel): } -class TaskOrderBillingCSPResult(BaseCSPPayload): - task_order_billing_validation_url: str +class VerifyTaskOrderBillingCSPPayload(BaseCSPPayload): + task_order_billing_validate_url: str class BillingProfileEnabledPlanDetails(AliasModel): @@ -754,7 +755,7 @@ class AzureSDKProvider(object): import azure.graphrbac as graphrbac import azure.common.credentials as credentials import azure.identity as identity - from azure.keyvault import secrets import secrets + from azure.keyvault import secrets from msrestazure.azure_cloud import AZURE_PUBLIC_CLOUD import adal @@ -896,11 +897,8 @@ class AzureCloudProvider(CloudProviderInterface): headers=create_tenant_headers, ) - print("create tenant result") - print(result.json()) - if result.status_code == 200: - return self._ok(TenantCSPResult(**result.json())) + return self._ok(TenantCSPResult(**result.json(), tenant_admin_password=payload.password, tenant_admin_username=payload.user_id)) else: return self._error(result.json()) @@ -917,9 +915,7 @@ class AzureCloudProvider(CloudProviderInterface): "Authorization": f"Bearer {sp_token}", } - # TODO: unsure if this is a static value or needs to be constructed/configurable - BILLING_ACCOUT_NAME = "7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31" - billing_account_create_url = f"https://management.azure.com/providers/Microsoft.Billing/billingAccounts/{BILLING_ACCOUT_NAME}/billingProfiles?api-version=2019-10-01-preview" + billing_account_create_url = f"https://management.azure.com/providers/Microsoft.Billing/billingAccounts/{payload.billing_account_name}/billingProfiles?api-version=2019-10-01-preview" result = self.sdk.requests.post( billing_account_create_url, @@ -1022,9 +1018,7 @@ class AzureCloudProvider(CloudProviderInterface): "Authorization": f"Bearer {sp_token}", } - result = self.sdk.requests.get( - payload.task_order_billing_validation_url, headers=auth_header - ) + result = self.sdk.requests.get(payload.task_order_billing_validate_url, headers=auth_header) if result.status_code == 202: # 202 has location/retry after headers diff --git a/tests/domain/cloud/test_azure_csp.py b/tests/domain/cloud/test_azure_csp.py index bf228874..577582d2 100644 --- a/tests/domain/cloud/test_azure_csp.py +++ b/tests/domain/cloud/test_azure_csp.py @@ -5,31 +5,30 @@ from uuid import uuid4 from atst.domain.csp.cloud import ( AzureCloudProvider, - TenantCSPResult, - TenantCSPPayload, - BillingProfileCSPPayload, - BillingProfileAddress, BillingProfileCreateCSPResult, - BillingProfileVerifyCSPPayload, + BillingProfileCSPPayload, BillingProfileCSPResult, - BillingRoleAssignmentCSPPayload, - BillingRoleAssignmentCSPResult, - EnableTaskOrderBillingCSPPayload, - VerifyTaskOrderBillingCSPPayload, - BillingProfileEnabledCSPResult, + BillingProfileTenantAccessCSPPayload, + BillingProfileTenantAccessCSPResult, + BillingProfileVerifyCSPPayload, ReportCLINCSPPayload, ReportCLINCSPResult, + TaskOrderBillingCSPPayload, + TenantCSPPayload, + TenantCSPResult, + VerifyTaskOrderBillingCSPPayload, ) from tests.mock_azure import mock_azure, AUTH_CREDENTIALS from tests.factories import EnvironmentFactory, ApplicationFactory -# TODO: Directly test create subscription, provide all args √ -# TODO: Test create environment (create management group with parent) -# TODO: Test create application (create manageemnt group with parent) -# Create reusable mock for mocking the management group calls for multiple services -# +creds = { + "home_tenant_id": "", + "client_id": "", + "secret_key": "", +} +BILLING_ACCOUNT_NAME = "52865e4c-52e8-5a6c-da6b-c58f0814f06f:7ea5de9d-b8ce-4901-b1c5-d864320c7b03_2019-05-31" @pytest.mark.skip("Skipping legacy azure integration tests") @@ -153,7 +152,7 @@ def test_create_tenant(mock_azure: AzureCloudProvider): mock_azure.sdk.requests.post.return_value = mock_result payload = TenantCSPPayload( **dict( - creds={"username": "mock-cloud", "password": "shh"}, + creds=creds, user_id="admin", password="JediJan13$coot", domain_name="jediccpospawnedtenant2", @@ -192,7 +191,8 @@ def test_create_billing_profile(mock_azure: AzureCloudProvider): ), creds={"username": "mock-cloud", "password": "shh"}, tenant_id="60ff9d34-82bf-4f21-b565-308ef0533435", - display_name="Test Billing Profile", + billing_profile_display_name="Test Billing Profile", + billing_account_name=BILLING_ACCOUNT_NAME, ) ) result = mock_azure.create_billing_profile(payload) @@ -250,10 +250,10 @@ def test_validate_billing_profile_creation(mock_azure: AzureCloudProvider): ) result = mock_azure.validate_billing_profile_created(payload) - body: BillingProfileCreateCSPResult = result.get("body") + body: BillingProfileCSPResult = result.get("body") assert body.billing_profile_name == "KQWI-W2SU-BG7-TGB" assert ( - body.billing_profile_properties.display_name + body.billing_profile_properties.billing_profile_display_name == "First Portfolio Billing Profile" ) @@ -281,7 +281,7 @@ def test_create_billing_profile_tenant_access(mock_azure: AzureCloudProvider): mock_azure.sdk.requests.post.return_value = mock_result - payload = BillingRoleAssignmentCSPPayload( + payload = BillingProfileTenantAccessCSPPayload( **dict( creds={ "username": "username", @@ -296,7 +296,7 @@ def test_create_billing_profile_tenant_access(mock_azure: AzureCloudProvider): ) result = mock_azure.create_billing_profile_tenant_access(payload) - body: BillingRoleAssignmentCSPResult = result.get("body") + body: BillingProfileTenantAccessCSPResult = result.get("body") assert ( body.billing_role_assignment_name == "40000000-aaaa-bbbb-cccc-100000000000_0a5f4926-e3ee-4f47-a6e3-8b0a30a40e3d" @@ -317,7 +317,7 @@ def test_create_task_order_billing(mock_azure: AzureCloudProvider): mock_azure.sdk.requests.patch.return_value = mock_result - payload = EnableTaskOrderBillingCSPPayload( + payload = TaskOrderBillingCSPPayload( **dict( creds={ "username": "username", @@ -388,7 +388,7 @@ def test_validate_task_order_billing_enabled(mock_azure): "password": "password", "tenant_id": "tenant_id", }, - task_order_billing_validation_url="https://management.azure.com/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/operationResults/createBillingProfile_478d5706-71f9-4a8b-8d4e-2cbaca27a668?api-version=2019-10-01-preview", + task_order_billing_validate_url="https://management.azure.com/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/operationResults/createBillingProfile_478d5706-71f9-4a8b-8d4e-2cbaca27a668?api-version=2019-10-01-preview", ) ) From f5e4b603cb9b8b286f954fea667760f7c174a04e Mon Sep 17 00:00:00 2001 From: tomdds Date: Wed, 22 Jan 2020 09:46:03 -0500 Subject: [PATCH 13/23] Bring naming conventions for methods and classes related to CSP provisioning in line with state machine --- atst/domain/csp/cloud.py | 171 ++++++++----------- atst/models/mixins/state_machines.py | 6 +- atst/models/portfolio_state_machine.py | 2 + tests/domain/cloud/test_azure_csp.py | 61 +++---- tests/domain/test_portfolio_state_machine.py | 14 +- tests/mock_azure.py | 1 + 6 files changed, 120 insertions(+), 135 deletions(-) diff --git a/atst/domain/csp/cloud.py b/atst/domain/csp/cloud.py index cd958781..afee36d9 100644 --- a/atst/domain/csp/cloud.py +++ b/atst/domain/csp/cloud.py @@ -186,8 +186,8 @@ class TenantCSPResult(AliasModel): tenant_id: str user_object_id: str - tenant_admin_username: str - tenant_admin_password: str + tenant_admin_username: Optional[str] + tenant_admin_password: Optional[str] class Config: fields = { @@ -232,7 +232,7 @@ class BillingProfileCLINBudget(AliasModel): """ -class BillingProfileCSPPayload(BaseCSPPayload): +class BillingProfileCreationCSPPayload(BaseCSPPayload): tenant_id: str billing_profile_display_name: str billing_account_name: str @@ -253,19 +253,19 @@ class BillingProfileCSPPayload(BaseCSPPayload): fields = {"billing_profile_display_name": "displayName"} -class BillingProfileCreateCSPResult(AliasModel): - billing_profile_validate_url: str - retry_after: int +class BillingProfileCreationCSPResult(AliasModel): + billing_profile_verify_url: str + billing_profile_retry_after: int class Config: fields = { - "billing_profile_validate_url": "Location", - "retry_after": "Retry-After", + "billing_profile_verify_url": "Location", + "billing_profile_retry_after": "Retry-After", } -class BillingProfileVerifyCSPPayload(BaseCSPPayload): - billing_profile_validate_url: str +class BillingProfileVerificationCSPPayload(BaseCSPPayload): + billing_profile_verify_url: str class BillingInvoiceSection(AliasModel): @@ -285,7 +285,7 @@ class BillingProfileProperties(AliasModel): fields = {"billing_profile_display_name": "displayName"} -class BillingProfileCSPResult(AliasModel): +class BillingProfileVerificationCSPResult(AliasModel): billing_profile_id: str billing_profile_name: str billing_profile_properties: BillingProfileProperties @@ -316,31 +316,31 @@ class BillingProfileTenantAccessCSPResult(AliasModel): } -class TaskOrderBillingCSPPayload(BaseCSPPayload): +class TaskOrderBillingCreationCSPPayload(BaseCSPPayload): billing_account_name: str billing_profile_name: str -class EnableTaskOrderBillingCSPResult(AliasModel): - task_order_billing_validate_url: str +class TaskOrderBillingCreationCSPResult(AliasModel): + task_order_billing_verify_url: str retry_after: int class Config: fields = { - "task_order_billing_validation_url": "Location", + "task_order_billing_verify_url": "Location", "retry_after": "Retry-After", } -class VerifyTaskOrderBillingCSPPayload(BaseCSPPayload): - task_order_billing_validate_url: str +class TaskOrderBillingVerificationCSPPayload(BaseCSPPayload): + task_order_billing_verify_url: str class BillingProfileEnabledPlanDetails(AliasModel): enabled_azure_plans: List[Dict] -class TaskOrderBillingCSPResult(AliasModel): +class TaskOrderBillingVerificationCSPResult(AliasModel): billing_profile_id: str billing_profile_name: str billing_profile_enabled_plan_details: BillingProfileEnabledPlanDetails @@ -353,7 +353,7 @@ class TaskOrderBillingCSPResult(AliasModel): } -class ReportCLINCSPPayload(BaseCSPPayload): +class BillingInstructionCSPPayload(BaseCSPPayload): amount: float start_date: str end_date: str @@ -363,7 +363,7 @@ class ReportCLINCSPPayload(BaseCSPPayload): billing_profile_name: str -class ReportCLINCSPResult(AliasModel): +class BillingInstructionCSPResult(AliasModel): reported_clin_name: str class Config: @@ -373,12 +373,10 @@ class ReportCLINCSPResult(AliasModel): class CloudProviderInterface: - - - def set_secret(secret_key: str, secret_value: str): + def set_secret(self, secret_key: str, secret_value: str): raise NotImplementedError() - def get_secret(secret_key: str, secret_value: str): + def get_secret(self, secret_key: str, secret_value: str): raise NotImplementedError() def root_creds(self) -> Dict: @@ -563,7 +561,7 @@ class MockCloudProvider(CloudProviderInterface): return {"id": self._id(), "credentials": self._auth_credentials} - def create_tenant(self, payload): + def create_tenant(self, payload: TenantCSPPayload): """ payload is an instance of TenantCSPPayload data class """ @@ -575,68 +573,41 @@ class MockCloudProvider(CloudProviderInterface): self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) - # return tenant id, tenant owner id and tenant owner object id from: - response = {"tenantId": "string", "userId": "string", "objectId": "string"} - return { - "tenant_id": response["tenantId"], - "user_id": response["userId"], - "user_object_id": response["objectId"], - "tenant_admin_username": "test", - "tenant_admin_password": "test", - } - def create_billing_profile(self, payload): - # call billing profile creation endpoint, specifying owner - # Payload: - """ - { - "displayName": "string", - "poNumber": "string", - "address": { - "firstName": "string", - "lastName": "string", - "companyName": "string", - "addressLine1": "string", - "addressLine2": "string", - "addressLine3": "string", - "city": "string", - "region": "string", - "country": "string", - "postalCode": "string" - }, - "invoiceEmailOptIn": true, - Note: These last 2 are also the body for adding/updating new TOs/clins - "enabledAzurePlans": [ - { - "skuId": "string" - } - ], - "clinBudget": { - "amount": 0, - "startDate": "2019-12-18T16:47:40.909Z", - "endDate": "2019-12-18T16:47:40.909Z", - "externalReferenceId": "string" - } - } - """ + return TenantCSPResult(**{ + "tenant_id": "", + "user_id": "", + "user_object_id": "", + "tenant_admin_username": "test", + "tenant_admin_password": "test" + }).dict() + + def create_billing_profile_creation(self, payload: BillingProfileCreationCSPPayload): # response will be mostly the same as the body, but we only really care about the id self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) - response = {"id": "string"} - # return {"billing_profile_id": response["id"]} - return { - "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB", - "name": "KQWI-W2SU-BG7-TGB", - "properties": { - "address": { - "addressLine1": "123 S Broad Street, Suite 2400", - "city": "Philadelphia", - "companyName": "Promptworks", - "country": "US", - "postalCode": "19109", - "region": "PA", + return BillingProfileCreationCSPResult(**dict( + billing_profile_verify_url = "https://zombo.com", + billing_profile_retry_after = 10 + )).dict() + + def create_billing_profile_verification(self, payload: BillingProfileVerificationCSPPayload): + self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) + self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) + self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) + return BillingProfileVerificationCSPResult(**{ + 'id': '/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB', + 'name': 'KQWI-W2SU-BG7-TGB', + 'properties': { + 'address': { + 'addressLine1': '123 S Broad Street, Suite 2400', + 'city': 'Philadelphia', + 'companyName': 'Promptworks', + 'country': 'US', + 'postalCode': '19109', + 'region': 'PA' }, "currency": "USD", "displayName": "Test Billing Profile", @@ -653,8 +624,8 @@ class MockCloudProvider(CloudProviderInterface): } ], }, - "type": "Microsoft.Billing/billingAccounts/billingProfiles", - } + 'type': 'Microsoft.Billing/billingAccounts/billingProfiles' + }).dict() def create_billing_profile_tenant_access(self, payload): self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) @@ -795,7 +766,7 @@ class AzureCloudProvider(CloudProviderInterface): ) return secret_client.set_secret(secret_key, secret_value) - def get_secret(secret_key) + def get_secret(secret_key): credential = self._get_client_secret_credential_obj() secret_client = self.secrets.SecretClient( vault_url=self.vault_url, @@ -902,7 +873,7 @@ class AzureCloudProvider(CloudProviderInterface): else: return self._error(result.json()) - def create_billing_profile(self, payload: BillingProfileCSPPayload): + def create_billing_profile_creation(self, payload: BillingProfileCreationCSPPayload): sp_token = self._get_sp_token(payload.creds) if sp_token is None: raise AuthenticationException( @@ -925,14 +896,14 @@ class AzureCloudProvider(CloudProviderInterface): if result.status_code == 202: # 202 has location/retry after headers - return self._ok(BillingProfileCreateCSPResult(**result.headers)) + return self._ok(BillingProfileCreationCSPResult(**result.headers)) elif result.status_code == 200: # NB: Swagger docs imply call can sometimes resolve immediately - return self._ok(BillingProfileCSPResult(**result.json())) + return self._ok(BillingProfileVerificationCSPResult(**result.json())) else: return self._error(result.json()) - def validate_billing_profile_created(self, payload: BillingProfileVerifyCSPPayload): + def create_billing_profile_verification(self, payload: BillingProfileVerificationCSPPayload): sp_token = self._get_sp_token(payload.creds) if sp_token is None: raise AuthenticationException( @@ -944,14 +915,14 @@ class AzureCloudProvider(CloudProviderInterface): } result = self.sdk.requests.get( - payload.billing_profile_validate_url, headers=auth_header + payload.billing_profile_verify_url, headers=auth_header ) if result.status_code == 202: # 202 has location/retry after headers - return self._ok(BillingProfileCreateCSPResult(**result.headers)) + return self._ok(BillingProfileCreationCSPResult(**result.headers)) elif result.status_code == 200: - return self._ok(BillingProfileCSPResult(**result.json())) + return self._ok(BillingProfileVerificationCSPResult(**result.json())) else: return self._error(result.json()) @@ -979,7 +950,7 @@ class AzureCloudProvider(CloudProviderInterface): else: return self._error(result.json()) - def enable_task_order_billing(self, payload: TaskOrderBillingCSPPayload): + def create_task_order_billing_creation(self, payload: TaskOrderBillingCreationCSPPayload): sp_token = self._get_sp_token(payload.creds) request_body = [ { @@ -1001,13 +972,13 @@ class AzureCloudProvider(CloudProviderInterface): if result.status_code == 202: # 202 has location/retry after headers - return self._ok(BillingProfileCreateCSPResult(**result.headers)) + return self._ok(TaskOrderBillingCreationCSPResult(**result.headers)) elif result.status_code == 200: - return self._ok(TaskOrderBillingCSPResult(**result.json())) + return self._ok(TaskOrderBillingVerificationCSPResult(**result.json())) else: return self._error(result.json()) - def validate_task_order_billing_enabled(self, payload: TaskOrderBillingCSPPayload): + def create_task_order_billing_verification(self, payload: TaskOrderBillingVerificationCSPPayload): sp_token = self._get_sp_token(payload.creds) if sp_token is None: raise AuthenticationException( @@ -1018,17 +989,17 @@ class AzureCloudProvider(CloudProviderInterface): "Authorization": f"Bearer {sp_token}", } - result = self.sdk.requests.get(payload.task_order_billing_validate_url, headers=auth_header) + result = self.sdk.requests.get(payload.task_order_billing_verify_url, headers=auth_header) if result.status_code == 202: # 202 has location/retry after headers - return self._ok(TaskOrderBillingCSPResult(**result.headers)) + return self._ok(TaskOrderBillingCreationCSPResult(**result.headers)) elif result.status_code == 200: - return self._ok(TaskOrderBillingCSPResult(**result.json())) + return self._ok(TaskOrderBillingVerificationCSPResult(**result.json())) else: return self._error(result.json()) - def create_billing_instruction(self, payload: ReportCLINCSPPayload): + def create_billing_instruction(self, payload: BillingInstructionCSPPayload): sp_token = self._get_sp_token(payload.creds) if sp_token is None: raise AuthenticationException( @@ -1052,7 +1023,7 @@ class AzureCloudProvider(CloudProviderInterface): result = self.sdk.requests.put(url, headers=auth_header, json=request_body) if result.status_code == 200: - return self._ok(ReportCLINCSPResult(**result.json())) + return self._ok(BillingInstructionCSPResult(**result.json())) else: return self._error(result.json()) diff --git a/atst/models/mixins/state_machines.py b/atst/models/mixins/state_machines.py index 6938c8a9..8d50eb15 100644 --- a/atst/models/mixins/state_machines.py +++ b/atst/models/mixins/state_machines.py @@ -9,9 +9,11 @@ class StageStates(Enum): class AzureStages(Enum): TENANT = "tenant" - BILLING_PROFILE = "billing profile" + BILLING_PROFILE_CREATION = "billing profile creation" + BILLING_PROFILE_VERIFICATION = "billing profile verification" BILLING_PROFILE_TENANT_ACCESS = "billing profile tenant access" - TASK_ORDER_BILLING = "task order billing" + TASK_ORDER_BILLING_CREATION = "task order billing creation" + TASK_ORDER_BILLING_VERIFICATION = "task order billing verification" BILLING_INSTRUCTION = "billing instruction" diff --git a/atst/models/portfolio_state_machine.py b/atst/models/portfolio_state_machine.py index b8f29f24..89dc073b 100644 --- a/atst/models/portfolio_state_machine.py +++ b/atst/models/portfolio_state_machine.py @@ -199,6 +199,8 @@ class PortfolioStateMachine( # self.store_creds(self.portfolio, new_creds) except PydanticValidationError as exc: + print("is_csp_data_valid: False") + print(cls) print(exc.json()) return False diff --git a/tests/domain/cloud/test_azure_csp.py b/tests/domain/cloud/test_azure_csp.py index 577582d2..b2099903 100644 --- a/tests/domain/cloud/test_azure_csp.py +++ b/tests/domain/cloud/test_azure_csp.py @@ -5,18 +5,20 @@ from uuid import uuid4 from atst.domain.csp.cloud import ( AzureCloudProvider, - BillingProfileCreateCSPResult, - BillingProfileCSPPayload, - BillingProfileCSPResult, + BillingProfileCreationCSPResult, + BillingProfileCreationCSPPayload, BillingProfileTenantAccessCSPPayload, BillingProfileTenantAccessCSPResult, - BillingProfileVerifyCSPPayload, - ReportCLINCSPPayload, - ReportCLINCSPResult, - TaskOrderBillingCSPPayload, + BillingProfileVerificationCSPPayload, + BillingProfileVerificationCSPResult, + BillingInstructionCSPPayload, + BillingInstructionCSPResult, + TaskOrderBillingCreationCSPPayload, + TaskOrderBillingCreationCSPResult, + TaskOrderBillingVerificationCSPPayload, + TaskOrderBillingVerificationCSPResult, TenantCSPPayload, TenantCSPResult, - VerifyTaskOrderBillingCSPPayload, ) from tests.mock_azure import mock_azure, AUTH_CREDENTIALS @@ -167,7 +169,7 @@ def test_create_tenant(mock_azure: AzureCloudProvider): assert body.tenant_id == "60ff9d34-82bf-4f21-b565-308ef0533435" -def test_create_billing_profile(mock_azure: AzureCloudProvider): +def test_create_billing_profile_creation(mock_azure: AzureCloudProvider): mock_azure.sdk.adal.AuthenticationContext.return_value.context.acquire_token_with_client_credentials.return_value = { "accessToken": "TOKEN" } @@ -179,7 +181,7 @@ def test_create_billing_profile(mock_azure: AzureCloudProvider): } mock_result.status_code = 202 mock_azure.sdk.requests.post.return_value = mock_result - payload = BillingProfileCSPPayload( + payload = BillingProfileCreationCSPPayload( **dict( address=dict( address_line_1="123 S Broad Street, Suite 2400", @@ -195,9 +197,9 @@ def test_create_billing_profile(mock_azure: AzureCloudProvider): billing_account_name=BILLING_ACCOUNT_NAME, ) ) - result = mock_azure.create_billing_profile(payload) - body: BillingProfileCreateCSPResult = result.get("body") - assert body.retry_after == 10 + result = mock_azure.create_billing_profile_creation(payload) + body: BillingProfileCreationCSPResult = result.get("body") + assert body.billing_profile_retry_after == 10 def test_validate_billing_profile_creation(mock_azure: AzureCloudProvider): @@ -238,19 +240,19 @@ def test_validate_billing_profile_creation(mock_azure: AzureCloudProvider): } mock_azure.sdk.requests.get.return_value = mock_result - payload = BillingProfileVerifyCSPPayload( + payload = BillingProfileVerificationCSPPayload( **dict( creds={ "username": "username", "password": "password", "tenant_id": "tenant_id", }, - billing_profile_validate_url="https://management.azure.com/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/operationResults/createBillingProfile_478d5706-71f9-4a8b-8d4e-2cbaca27a668?api-version=2019-10-01-preview", + billing_profile_verify_url="https://management.azure.com/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/operationResults/createBillingProfile_478d5706-71f9-4a8b-8d4e-2cbaca27a668?api-version=2019-10-01-preview", ) ) - result = mock_azure.validate_billing_profile_created(payload) - body: BillingProfileCSPResult = result.get("body") + result = mock_azure.create_billing_profile_verification(payload) + body: BillingProfileVerificationCSPResult = result.get("body") assert body.billing_profile_name == "KQWI-W2SU-BG7-TGB" assert ( body.billing_profile_properties.billing_profile_display_name @@ -303,7 +305,7 @@ def test_create_billing_profile_tenant_access(mock_azure: AzureCloudProvider): ) -def test_create_task_order_billing(mock_azure: AzureCloudProvider): +def test_create_task_order_billing_creation(mock_azure: AzureCloudProvider): mock_azure.sdk.adal.AuthenticationContext.return_value.context.acquire_token_with_client_credentials.return_value = { "accessToken": "TOKEN" } @@ -317,7 +319,7 @@ def test_create_task_order_billing(mock_azure: AzureCloudProvider): mock_azure.sdk.requests.patch.return_value = mock_result - payload = TaskOrderBillingCSPPayload( + payload = TaskOrderBillingCreationCSPPayload( **dict( creds={ "username": "username", @@ -329,15 +331,15 @@ def test_create_task_order_billing(mock_azure: AzureCloudProvider): ) ) - result = mock_azure.enable_task_order_billing(payload) - body: BillingProfileCreateCSPResult = result.get("body") + result = mock_azure.create_task_order_billing_creation(payload) + body: TaskOrderBillingCreationCSPResult = result.get("body") assert ( - body.billing_profile_validate_url + body.task_order_billing_verify_url == "https://management.azure.com/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/operationResults/patchBillingProfile_KQWI-W2SU-BG7-TGB:02715576-4118-466c-bca7-b1cd3169ff46?api-version=2019-10-01-preview" ) -def test_validate_task_order_billing_enabled(mock_azure): +def test_create_task_order_billing_verification(mock_azure): mock_azure.sdk.adal.AuthenticationContext.return_value.context.acquire_token_with_client_credentials.return_value = { "accessToken": "TOKEN" } @@ -381,19 +383,19 @@ def test_validate_task_order_billing_enabled(mock_azure): } mock_azure.sdk.requests.get.return_value = mock_result - payload = VerifyTaskOrderBillingCSPPayload( + payload = TaskOrderBillingVerificationCSPPayload( **dict( creds={ "username": "username", "password": "password", "tenant_id": "tenant_id", }, - task_order_billing_validate_url="https://management.azure.com/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/operationResults/createBillingProfile_478d5706-71f9-4a8b-8d4e-2cbaca27a668?api-version=2019-10-01-preview", + task_order_billing_verify_url="https://management.azure.com/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/operationResults/createBillingProfile_478d5706-71f9-4a8b-8d4e-2cbaca27a668?api-version=2019-10-01-preview", ) ) - result = mock_azure.validate_task_order_billing_enabled(payload) - body: BillingProfileEnabledCSPResult = result.get("body") + result = mock_azure.create_task_order_billing_verification(payload) + body: TaskOrderBillingVerificationCSPResult = result.get("body") assert body.billing_profile_name == "KQWI-W2SU-BG7-TGB" assert ( body.billing_profile_enabled_plan_details.enabled_azure_plans[0].get("skuId") @@ -420,7 +422,7 @@ def test_create_billing_instruction(mock_azure: AzureCloudProvider): mock_azure.sdk.requests.put.return_value = mock_result - payload = ReportCLINCSPPayload( + payload = BillingInstructionCSPPayload( **dict( creds={}, amount=1000.00, @@ -433,6 +435,5 @@ def test_create_billing_instruction(mock_azure: AzureCloudProvider): ) ) result = mock_azure.create_billing_instruction(payload) - body: ReportCLINCSPResult = result.get("body") + body: BillingInstructionCSPResult = result.get("body") assert body.reported_clin_name == "TO1:CLIN001" - diff --git a/tests/domain/test_portfolio_state_machine.py b/tests/domain/test_portfolio_state_machine.py index 9bd2c842..dcbc4bfd 100644 --- a/tests/domain/test_portfolio_state_machine.py +++ b/tests/domain/test_portfolio_state_machine.py @@ -119,17 +119,25 @@ def test_fsm_transition_start(portfolio): "billing_profile_display_name": "My Billing Profile", } - collected_data = dict(list(csp_data.items()) + list(portfolio_data.items())) + config = {"billing_account_name": "billing_account_name"} + + collected_data = dict( + list(csp_data.items()) + list(portfolio_data.items()) + list(config.items()) + ) sm.trigger_next_transition(creds=creds, csp_data=collected_data) assert sm.state == FSMStates.TENANT_CREATED assert portfolio.csp_data.get("tenant_id", None) is not None - + print(portfolio.csp_data.keys()) if portfolio.csp_data is not None: csp_data = portfolio.csp_data else: csp_data = {} - collected_data = dict(list(csp_data.items()) + list(portfolio_data.items())) + collected_data = dict( + list(csp_data.items()) + list(portfolio_data.items()) + list(config.items()) + ) sm.trigger_next_transition(creds=creds, csp_data=collected_data) assert sm.state == FSMStates.BILLING_PROFILE_CREATED + print(portfolio.csp_data.keys()) + diff --git a/tests/mock_azure.py b/tests/mock_azure.py index ecfafaac..23e9515c 100644 --- a/tests/mock_azure.py +++ b/tests/mock_azure.py @@ -8,6 +8,7 @@ AZURE_CONFIG = { "AZURE_SECRET_KEY": "MOCK", "AZURE_TENANT_ID": "MOCK", "AZURE_POLICY_LOCATION": "policies", + "AZURE_VAULT_URL": "http://vault", } AUTH_CREDENTIALS = { From 1b1a20cf52ece3ee70a5c6ea6873cf798cca7c42 Mon Sep 17 00:00:00 2001 From: tomdds Date: Wed, 22 Jan 2020 14:39:30 -0500 Subject: [PATCH 14/23] Restore implementations for policies and management group creation These were accidentally stripped out during a rebase. --- atst/domain/csp/cloud.py | 180 +++++++++++++++++++++------ tests/domain/cloud/test_azure_csp.py | 39 ++---- tests/mock_azure.py | 1 + 3 files changed, 153 insertions(+), 67 deletions(-) diff --git a/atst/domain/csp/cloud.py b/atst/domain/csp/cloud.py index afee36d9..c09731b2 100644 --- a/atst/domain/csp/cloud.py +++ b/atst/domain/csp/cloud.py @@ -5,9 +5,11 @@ from uuid import uuid4 from pydantic import BaseModel, validator from atst.models.user import User +from atst.models.application import Application from atst.models.environment import Environment from atst.models.environment_role import EnvironmentRole from atst.utils import snake_to_camel +from .policy import AzurePolicyManager class GeneralCSPException(Exception): @@ -718,11 +720,12 @@ SUBSCRIPTION_ID_REGEX = re.compile( # This needs to be a fully pathed role definition identifier, not just a UUID REMOTE_ROOT_ROLE_DEF_ID = "/providers/Microsoft.Authorization/roleDefinitions/00000000-0000-4000-8000-000000000000" - +AZURE_MANAGEMENT_API = "https://management.azure.com" class AzureSDKProvider(object): def __init__(self): - from azure.mgmt import subscription, authorization + from azure.mgmt import subscription, authorization, managementgroups + from azure.mgmt.resource import policy import azure.graphrbac as graphrbac import azure.common.credentials as credentials import azure.identity as identity @@ -733,6 +736,8 @@ class AzureSDKProvider(object): import requests self.subscription = subscription + self.policy = policy + self.managementgroups = managementgroups self.authorization = authorization self.adal = adal self.graphrbac = graphrbac @@ -758,6 +763,8 @@ class AzureCloudProvider(CloudProviderInterface): else: self.sdk = azure_sdk_provider + self.policy_manager = AzurePolicyManager(config["AZURE_POLICY_LOCATION"]) + def set_secret(secret_key, secret_value): credential = self._get_client_secret_credential_obj() secret_client = self.secrets.SecretClient( @@ -777,42 +784,23 @@ class AzureCloudProvider(CloudProviderInterface): def create_environment( self, auth_credentials: Dict, user: User, environment: Environment ): + # since this operation would only occur within a tenant, should we source the tenant + # via lookup from environment once we've created the portfolio csp data schema + # something like this: + # environment_tenant = environment.application.portfolio.csp_data.get('tenant_id', None) + # though we'd probably source the whole credentials for these calls from the portfolio csp + # data, as it would have to be where we store the creds for the at-at user within the portfolio tenant + # credentials = self._get_credential_obj(environment.application.portfolio.csp_data.get_creds()) credentials = self._get_credential_obj(self._root_creds) - sub_client = self.sdk.subscription.SubscriptionClient(credentials) - display_name = f"{environment.application.name}_{environment.name}_{environment.id}" # proposed format + management_group_id = "?" # management group id chained from environment + parent_id = "?" # from environment.application - billing_profile_id = "?" # something chained from environment? - sku_id = AZURE_SKU_ID - # we want to set AT-AT as an owner here - # we could potentially associate subscriptions with "management groups" per DOD component - body = self.sdk.subscription.models.ModernSubscriptionCreationParameters( - display_name, - billing_profile_id, - sku_id, - # owner= + management_group = self._create_management_group( + credentials, management_group_id, display_name, parent_id, ) - # These 2 seem like something that might be worthwhile to allow tiebacks to - # TOs filed for the environment - billing_account_name = "?" - invoice_section_name = "?" - # We may also want to create billing sections in the enrollment account - sub_creation_operation = sub_client.subscription_factory.create_subscription( - billing_account_name, invoice_section_name, body - ) - - # the resulting object from this process is a link to the new subscription - # not a subscription model, so we'll have to unpack the ID - new_sub = sub_creation_operation.result() - - subscription_id = self._extract_subscription_id(new_sub.subscription_link) - if subscription_id: - return subscription_id - else: - # troublesome error, subscription should exist at this point - # but we just don't have a valid ID - pass + return management_group def create_atat_admin_user( self, auth_credentials: Dict, csp_environment_id: str @@ -851,6 +839,125 @@ class AzureCloudProvider(CloudProviderInterface): "role_name": role_assignment_id, } + def _create_application(self, auth_credentials: Dict, application: Application): + management_group_name = str(uuid4()) # can be anything, not just uuid + display_name = application.name # Does this need to be unique? + credentials = self._get_credential_obj(auth_credentials) + parent_id = "?" # application.portfolio.csp_details.management_group_id + + return self._create_management_group( + credentials, management_group_name, display_name, parent_id, + ) + + def _create_management_group( + self, credentials, management_group_id, display_name, parent_id=None, + ): + mgmgt_group_client = self.sdk.managementgroups.ManagementGroupsAPI(credentials) + create_parent_grp_info = self.sdk.managementgroups.models.CreateParentGroupInfo( + id=parent_id + ) + create_mgmt_grp_details = self.sdk.managementgroups.models.CreateManagementGroupDetails( + parent=create_parent_grp_info + ) + mgmt_grp_create = self.sdk.managementgroups.models.CreateManagementGroupRequest( + name=management_group_id, + display_name=display_name, + details=create_mgmt_grp_details, + ) + create_request = mgmgt_group_client.management_groups.create_or_update( + management_group_id, mgmt_grp_create + ) + + # result is a synchronous wait, might need to do a poll instead to handle first mgmt group create + # since we were told it could take 10+ minutes to complete, unless this handles that polling internally + return create_request.result() + + def _create_subscription( + self, + credentials, + display_name, + billing_profile_id, + sku_id, + management_group_id, + billing_account_name, + invoice_section_name, + ): + sub_client = self.sdk.subscription.SubscriptionClient(credentials) + + billing_profile_id = "?" # where do we source this? + sku_id = AZURE_SKU_ID + # These 2 seem like something that might be worthwhile to allow tiebacks to + # TOs filed for the environment + billing_account_name = "?" # from TO? + invoice_section_name = "?" # from TO? + + body = self.sdk.subscription.models.ModernSubscriptionCreationParameters( + display_name=display_name, + billing_profile_id=billing_profile_id, + sku_id=sku_id, + management_group_id=management_group_id, + ) + + # We may also want to create billing sections in the enrollment account + sub_creation_operation = sub_client.subscription_factory.create_subscription( + billing_account_name, invoice_section_name, body + ) + + # the resulting object from this process is a link to the new subscription + # not a subscription model, so we'll have to unpack the ID + new_sub = sub_creation_operation.result() + + subscription_id = self._extract_subscription_id(new_sub.subscription_link) + if subscription_id: + return subscription_id + else: + # troublesome error, subscription should exist at this point + # but we just don't have a valid ID + pass + + def _create_policy_definition( + self, credentials, subscription_id, management_group_id, properties, + ): + """ + Requires credentials that have AZURE_MANAGEMENT_API + specified as the resource. The Service Principal + specified in the credentials must have the "Resource + Policy Contributor" role assigned with a scope at least + as high as the management group specified by + management_group_id. + + Arguments: + credentials -- ServicePrincipalCredentials + subscription_id -- str, ID of the subscription (just the UUID, not the path) + management_group_id -- str, ID of the management group (just the UUID, not the path) + properties -- dictionary, the "properties" section of a valid Azure policy definition document + + Returns: + azure.mgmt.resource.policy.[api version].models.PolicyDefinition: the PolicyDefinition object provided to Azure + + Raises: + TBD + """ + # TODO: which subscription would this be? + client = self.sdk.policy.PolicyClient(credentials, subscription_id) + + definition = client.policy_definitions.models.PolicyDefinition( + policy_type=properties.get("policyType"), + mode=properties.get("mode"), + display_name=properties.get("displayName"), + description=properties.get("description"), + policy_rule=properties.get("policyRule"), + parameters=properties.get("parameters"), + ) + + name = properties.get("displayName") + + return client.policy_definitions.create_or_update_at_management_group( + policy_definition_name=name, + parameters=definition, + management_group_id=management_group_id, + ) + def create_tenant(self, payload: TenantCSPPayload): sp_token = self._get_sp_token(payload.creds) if sp_token is None: @@ -1112,9 +1219,7 @@ class AzureCloudProvider(CloudProviderInterface): return sub_id_match.group(1) def _get_sp_token(self, creds): - home_tenant_id = creds.get( - "home_tenant_id" - ) + home_tenant_id = creds.get("home_tenant_id") client_id = creds.get("client_id") secret_key = creds.get("secret_key") @@ -1141,7 +1246,8 @@ class AzureCloudProvider(CloudProviderInterface): resource=resource, cloud_environment=self.sdk.cloud, ) - def _get_client_secret_credential_obj(): + + def _get_client_secret_credential_obj(self, creds): return self.sdk.identity.ClientSecretCredential( tenant_id=creds.get("tenant_id"), client_id =creds.get("client_id"), diff --git a/tests/domain/cloud/test_azure_csp.py b/tests/domain/cloud/test_azure_csp.py index b2099903..00830022 100644 --- a/tests/domain/cloud/test_azure_csp.py +++ b/tests/domain/cloud/test_azure_csp.py @@ -26,14 +26,13 @@ from tests.factories import EnvironmentFactory, ApplicationFactory creds = { - "home_tenant_id": "", - "client_id": "", - "secret_key": "", + "home_tenant_id": "tenant_id", + "client_id": "client_id", + "secret_key": "secret_key", } BILLING_ACCOUNT_NAME = "52865e4c-52e8-5a6c-da6b-c58f0814f06f:7ea5de9d-b8ce-4901-b1c5-d864320c7b03_2019-05-31" -@pytest.mark.skip("Skipping legacy azure integration tests") def test_create_subscription_succeeds(mock_azure: AzureCloudProvider): environment = EnvironmentFactory.create() @@ -74,7 +73,6 @@ def mock_management_group_create(mock_azure, spec_dict): ) -@pytest.mark.skip("Skipping legacy azure integration tests") def test_create_environment_succeeds(mock_azure: AzureCloudProvider): environment = EnvironmentFactory.create() @@ -87,7 +85,6 @@ def test_create_environment_succeeds(mock_azure: AzureCloudProvider): assert result.id == "Test Id" -@pytest.mark.skip("Skipping legacy azure integration tests") def test_create_application_succeeds(mock_azure: AzureCloudProvider): application = ApplicationFactory.create() @@ -98,7 +95,6 @@ def test_create_application_succeeds(mock_azure: AzureCloudProvider): assert result.id == "Test Id" -@pytest.mark.skip("Skipping legacy azure integration tests") def test_create_atat_admin_user_succeeds(mock_azure: AzureCloudProvider): environment_id = str(uuid4()) @@ -113,7 +109,6 @@ def test_create_atat_admin_user_succeeds(mock_azure: AzureCloudProvider): assert result.get("csp_user_id") == csp_user_id -@pytest.mark.skip("Skipping legacy azure integration tests") def test_create_policy_definition_succeeds(mock_azure: AzureCloudProvider): subscription_id = str(uuid4()) management_group_id = str(uuid4()) @@ -191,7 +186,7 @@ def test_create_billing_profile_creation(mock_azure: AzureCloudProvider): country="US", postal_code="19109", ), - creds={"username": "mock-cloud", "password": "shh"}, + creds=creds, tenant_id="60ff9d34-82bf-4f21-b565-308ef0533435", billing_profile_display_name="Test Billing Profile", billing_account_name=BILLING_ACCOUNT_NAME, @@ -242,11 +237,7 @@ def test_validate_billing_profile_creation(mock_azure: AzureCloudProvider): payload = BillingProfileVerificationCSPPayload( **dict( - creds={ - "username": "username", - "password": "password", - "tenant_id": "tenant_id", - }, + creds=creds, billing_profile_verify_url="https://management.azure.com/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/operationResults/createBillingProfile_478d5706-71f9-4a8b-8d4e-2cbaca27a668?api-version=2019-10-01-preview", ) ) @@ -285,11 +276,7 @@ def test_create_billing_profile_tenant_access(mock_azure: AzureCloudProvider): payload = BillingProfileTenantAccessCSPPayload( **dict( - creds={ - "username": "username", - "password": "password", - "tenant_id": "tenant_id", - }, + creds=creds, tenant_id="60ff9d34-82bf-4f21-b565-308ef0533435", user_object_id="0a5f4926-e3ee-4f47-a6e3-8b0a30a40e3d", billing_account_name="7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31", @@ -321,11 +308,7 @@ def test_create_task_order_billing_creation(mock_azure: AzureCloudProvider): payload = TaskOrderBillingCreationCSPPayload( **dict( - creds={ - "username": "username", - "password": "password", - "tenant_id": "tenant_id", - }, + creds=creds, billing_account_name="7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31", billing_profile_name="KQWI-W2SU-BG7-TGB", ) @@ -385,11 +368,7 @@ def test_create_task_order_billing_verification(mock_azure): payload = TaskOrderBillingVerificationCSPPayload( **dict( - creds={ - "username": "username", - "password": "password", - "tenant_id": "tenant_id", - }, + creds=creds, task_order_billing_verify_url="https://management.azure.com/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/operationResults/createBillingProfile_478d5706-71f9-4a8b-8d4e-2cbaca27a668?api-version=2019-10-01-preview", ) ) @@ -424,7 +403,7 @@ def test_create_billing_instruction(mock_azure: AzureCloudProvider): payload = BillingInstructionCSPPayload( **dict( - creds={}, + creds=creds, amount=1000.00, start_date="2020/1/1", end_date="2020/3/1", diff --git a/tests/mock_azure.py b/tests/mock_azure.py index 23e9515c..d9622b05 100644 --- a/tests/mock_azure.py +++ b/tests/mock_azure.py @@ -72,6 +72,7 @@ class MockAzureSDK(object): self.subscription = mock_subscription() self.authorization = mock_authorization() + self.policy = mock_policy() self.adal = mock_adal() self.managementgroups = mock_managementgroups() self.graphrbac = mock_graphrbac() From e5332897f16b0ea95db81b2d2937c12d61b31c5d Mon Sep 17 00:00:00 2001 From: tomdds Date: Wed, 22 Jan 2020 14:52:06 -0500 Subject: [PATCH 15/23] Fix formatting --- atst/domain/csp/cloud.py | 131 +++++++++++-------- tests/domain/test_portfolio_state_machine.py | 1 - 2 files changed, 78 insertions(+), 54 deletions(-) diff --git a/atst/domain/csp/cloud.py b/atst/domain/csp/cloud.py index c09731b2..9643dc7b 100644 --- a/atst/domain/csp/cloud.py +++ b/atst/domain/csp/cloud.py @@ -576,58 +576,68 @@ class MockCloudProvider(CloudProviderInterface): self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) - return TenantCSPResult(**{ - "tenant_id": "", - "user_id": "", - "user_object_id": "", - "tenant_admin_username": "test", - "tenant_admin_password": "test" - }).dict() + return TenantCSPResult( + **{ + "tenant_id": "", + "user_id": "", + "user_object_id": "", + "tenant_admin_username": "test", + "tenant_admin_password": "test", + } + ).dict() - def create_billing_profile_creation(self, payload: BillingProfileCreationCSPPayload): + def create_billing_profile_creation( + self, payload: BillingProfileCreationCSPPayload + ): # response will be mostly the same as the body, but we only really care about the id self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) - return BillingProfileCreationCSPResult(**dict( - billing_profile_verify_url = "https://zombo.com", - billing_profile_retry_after = 10 - )).dict() + return BillingProfileCreationCSPResult( + **dict( + billing_profile_verify_url="https://zombo.com", + billing_profile_retry_after=10, + ) + ).dict() - def create_billing_profile_verification(self, payload: BillingProfileVerificationCSPPayload): + def create_billing_profile_verification( + self, payload: BillingProfileVerificationCSPPayload + ): self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) - return BillingProfileVerificationCSPResult(**{ - 'id': '/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB', - 'name': 'KQWI-W2SU-BG7-TGB', - 'properties': { - 'address': { - 'addressLine1': '123 S Broad Street, Suite 2400', - 'city': 'Philadelphia', - 'companyName': 'Promptworks', - 'country': 'US', - 'postalCode': '19109', - 'region': 'PA' + return BillingProfileVerificationCSPResult( + **{ + "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB", + "name": "KQWI-W2SU-BG7-TGB", + "properties": { + "address": { + "addressLine1": "123 S Broad Street, Suite 2400", + "city": "Philadelphia", + "companyName": "Promptworks", + "country": "US", + "postalCode": "19109", + "region": "PA", + }, + "currency": "USD", + "displayName": "Test Billing Profile", + "enabledAzurePlans": [], + "hasReadAccess": True, + "invoiceDay": 5, + "invoiceEmailOptIn": False, + "invoiceSections": [ + { + "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB/invoiceSections/CHCO-BAAR-PJA-TGB", + "name": "CHCO-BAAR-PJA-TGB", + "properties": {"displayName": "Test Billing Profile"}, + "type": "Microsoft.Billing/billingAccounts/billingProfiles/invoiceSections", + } + ], }, - "currency": "USD", - "displayName": "Test Billing Profile", - "enabledAzurePlans": [], - "hasReadAccess": True, - "invoiceDay": 5, - "invoiceEmailOptIn": False, - "invoiceSections": [ - { - "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB/invoiceSections/CHCO-BAAR-PJA-TGB", - "name": "CHCO-BAAR-PJA-TGB", - "properties": {"displayName": "Test Billing Profile"}, - "type": "Microsoft.Billing/billingAccounts/billingProfiles/invoiceSections", - } - ], - }, - 'type': 'Microsoft.Billing/billingAccounts/billingProfiles' - }).dict() + "type": "Microsoft.Billing/billingAccounts/billingProfiles", + } + ).dict() def create_billing_profile_tenant_access(self, payload): self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) @@ -722,6 +732,7 @@ SUBSCRIPTION_ID_REGEX = re.compile( REMOTE_ROOT_ROLE_DEF_ID = "/providers/Microsoft.Authorization/roleDefinitions/00000000-0000-4000-8000-000000000000" AZURE_MANAGEMENT_API = "https://management.azure.com" + class AzureSDKProvider(object): def __init__(self): from azure.mgmt import subscription, authorization, managementgroups @@ -768,16 +779,14 @@ class AzureCloudProvider(CloudProviderInterface): def set_secret(secret_key, secret_value): credential = self._get_client_secret_credential_obj() secret_client = self.secrets.SecretClient( - vault_url=self.vault_url, - credential=credential, + vault_url=self.vault_url, credential=credential, ) return secret_client.set_secret(secret_key, secret_value) def get_secret(secret_key): credential = self._get_client_secret_credential_obj() secret_client = self.secrets.SecretClient( - vault_url=self.vault_url, - credential=credential, + vault_url=self.vault_url, credential=credential, ) return secret_client.get_secret(secret_key).value @@ -976,11 +985,19 @@ class AzureCloudProvider(CloudProviderInterface): ) if result.status_code == 200: - return self._ok(TenantCSPResult(**result.json(), tenant_admin_password=payload.password, tenant_admin_username=payload.user_id)) + return self._ok( + TenantCSPResult( + **result.json(), + tenant_admin_password=payload.password, + tenant_admin_username=payload.user_id, + ) + ) else: return self._error(result.json()) - def create_billing_profile_creation(self, payload: BillingProfileCreationCSPPayload): + def create_billing_profile_creation( + self, payload: BillingProfileCreationCSPPayload + ): sp_token = self._get_sp_token(payload.creds) if sp_token is None: raise AuthenticationException( @@ -1010,7 +1027,9 @@ class AzureCloudProvider(CloudProviderInterface): else: return self._error(result.json()) - def create_billing_profile_verification(self, payload: BillingProfileVerificationCSPPayload): + def create_billing_profile_verification( + self, payload: BillingProfileVerificationCSPPayload + ): sp_token = self._get_sp_token(payload.creds) if sp_token is None: raise AuthenticationException( @@ -1057,7 +1076,9 @@ class AzureCloudProvider(CloudProviderInterface): else: return self._error(result.json()) - def create_task_order_billing_creation(self, payload: TaskOrderBillingCreationCSPPayload): + def create_task_order_billing_creation( + self, payload: TaskOrderBillingCreationCSPPayload + ): sp_token = self._get_sp_token(payload.creds) request_body = [ { @@ -1085,7 +1106,9 @@ class AzureCloudProvider(CloudProviderInterface): else: return self._error(result.json()) - def create_task_order_billing_verification(self, payload: TaskOrderBillingVerificationCSPPayload): + def create_task_order_billing_verification( + self, payload: TaskOrderBillingVerificationCSPPayload + ): sp_token = self._get_sp_token(payload.creds) if sp_token is None: raise AuthenticationException( @@ -1096,7 +1119,9 @@ class AzureCloudProvider(CloudProviderInterface): "Authorization": f"Bearer {sp_token}", } - result = self.sdk.requests.get(payload.task_order_billing_verify_url, headers=auth_header) + result = self.sdk.requests.get( + payload.task_order_billing_verify_url, headers=auth_header + ) if result.status_code == 202: # 202 has location/retry after headers @@ -1250,8 +1275,8 @@ class AzureCloudProvider(CloudProviderInterface): def _get_client_secret_credential_obj(self, creds): return self.sdk.identity.ClientSecretCredential( tenant_id=creds.get("tenant_id"), - client_id =creds.get("client_id"), - client_secret = creds.get("secret_key"), + client_id=creds.get("client_id"), + client_secret=creds.get("secret_key"), ) def _make_tenant_admin_cred_obj(self, username, password): diff --git a/tests/domain/test_portfolio_state_machine.py b/tests/domain/test_portfolio_state_machine.py index dcbc4bfd..a3a3f5c0 100644 --- a/tests/domain/test_portfolio_state_machine.py +++ b/tests/domain/test_portfolio_state_machine.py @@ -140,4 +140,3 @@ def test_fsm_transition_start(portfolio): assert sm.state == FSMStates.BILLING_PROFILE_CREATED print(portfolio.csp_data.keys()) - From 00b10c484fbcb3886cd742c4984d3c4ef123e438 Mon Sep 17 00:00:00 2001 From: Philip Kalinsky Date: Wed, 22 Jan 2020 16:04:51 -0500 Subject: [PATCH 16/23] alembic migration to extend the state machine states enum --- alembic/env.py | 7 +++- ...19c44a8d5_state_machine_states_extended.py | 42 +++++++++++++++++++ 2 files changed, 47 insertions(+), 2 deletions(-) create mode 100644 alembic/versions/26319c44a8d5_state_machine_states_extended.py diff --git a/alembic/env.py b/alembic/env.py index 91b96364..825f5e19 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -51,7 +51,9 @@ def run_migrations_offline(): """ url = config.get_main_option("sqlalchemy.url") context.configure( - url=url, target_metadata=target_metadata, literal_binds=True) + url=url, target_metadata=target_metadata, literal_binds=True, + compare_type=True + ) with context.begin_transaction(): context.run_migrations() @@ -72,7 +74,8 @@ def run_migrations_online(): with connectable.connect() as connection: context.configure( connection=connection, - target_metadata=target_metadata + target_metadata=target_metadata, + compare_type=True ) with context.begin_transaction(): diff --git a/alembic/versions/26319c44a8d5_state_machine_states_extended.py b/alembic/versions/26319c44a8d5_state_machine_states_extended.py new file mode 100644 index 00000000..a183295c --- /dev/null +++ b/alembic/versions/26319c44a8d5_state_machine_states_extended.py @@ -0,0 +1,42 @@ +"""state machine states extended + +Revision ID: 26319c44a8d5 +Revises: 59973fa17ded +Create Date: 2020-01-22 15:54:03.186751 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = '26319c44a8d5' # pragma: allowlist secret +down_revision = '59973fa17ded' # pragma: allowlist secret +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('portfolio_state_machines', 'state', + existing_type=sa.VARCHAR(length=30), + type_=sa.Enum('UNSTARTED', 'STARTING', 'STARTED', 'COMPLETED', 'FAILED', 'TENANT_CREATED', 'TENANT_IN_PROGRESS', 'TENANT_FAILED', 'BILLING_PROFILE_CREATION_CREATED', 'BILLING_PROFILE_CREATION_IN_PROGRESS', 'BILLING_PROFILE_CREATION_FAILED', 'BILLING_PROFILE_VERIFICATION_CREATED', 'BILLING_PROFILE_VERIFICATION_IN_PROGRESS', 'BILLING_PROFILE_VERIFICATION_FAILED', 'BILLING_PROFILE_TENANT_ACCESS_CREATED', 'BILLING_PROFILE_TENANT_ACCESS_IN_PROGRESS', 'BILLING_PROFILE_TENANT_ACCESS_FAILED', 'TASK_ORDER_BILLING_CREATION_CREATED', 'TASK_ORDER_BILLING_CREATION_IN_PROGRESS', 'TASK_ORDER_BILLING_CREATION_FAILED', 'TASK_ORDER_BILLING_VERIFICATION_CREATED', 'TASK_ORDER_BILLING_VERIFICATION_IN_PROGRESS', 'TASK_ORDER_BILLING_VERIFICATION_FAILED', 'BILLING_INSTRUCTION_CREATED', 'BILLING_INSTRUCTION_IN_PROGRESS', 'BILLING_INSTRUCTION_FAILED', name='fsmstates', native_enum=False, create_constraint=False), + existing_nullable=False) + op.alter_column('portfolios', 'defense_component', + existing_type=postgresql.ARRAY(sa.VARCHAR()), + type_=sa.String(), + existing_nullable=False) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('portfolios', 'defense_component', + existing_type=sa.String(), + type_=postgresql.ARRAY(sa.VARCHAR()), + existing_nullable=False) + op.alter_column('portfolio_state_machines', 'state', + existing_type=sa.Enum('UNSTARTED', 'STARTING', 'STARTED', 'COMPLETED', 'FAILED', 'TENANT_CREATED', 'TENANT_IN_PROGRESS', 'TENANT_FAILED', 'BILLING_PROFILE_CREATION_CREATED', 'BILLING_PROFILE_CREATION_IN_PROGRESS', 'BILLING_PROFILE_CREATION_FAILED', 'BILLING_PROFILE_VERIFICATION_CREATED', 'BILLING_PROFILE_VERIFICATION_IN_PROGRESS', 'BILLING_PROFILE_VERIFICATION_FAILED', 'BILLING_PROFILE_TENANT_ACCESS_CREATED', 'BILLING_PROFILE_TENANT_ACCESS_IN_PROGRESS', 'BILLING_PROFILE_TENANT_ACCESS_FAILED', 'TASK_ORDER_BILLING_CREATION_CREATED', 'TASK_ORDER_BILLING_CREATION_IN_PROGRESS', 'TASK_ORDER_BILLING_CREATION_FAILED', 'TASK_ORDER_BILLING_VERIFICATION_CREATED', 'TASK_ORDER_BILLING_VERIFICATION_IN_PROGRESS', 'TASK_ORDER_BILLING_VERIFICATION_FAILED', 'BILLING_INSTRUCTION_CREATED', 'BILLING_INSTRUCTION_IN_PROGRESS', 'BILLING_INSTRUCTION_FAILED', name='fsmstates', native_enum=False), + type_=sa.VARCHAR(length=30), + existing_nullable=False) + # ### end Alembic commands ### From 597ea32e426b7a879cd5969b7d2d03535efe8ae2 Mon Sep 17 00:00:00 2001 From: Philip Kalinsky Date: Thu, 23 Jan 2020 10:14:22 -0500 Subject: [PATCH 17/23] portfolio provision process. add logging, __repr___ method, fix broken tests --- atst/domain/csp/cloud.py | 14 ++++++-- atst/models/mixins/state_machines.py | 4 +++ atst/models/portfolio_state_machine.py | 29 ++++++++++------- tests/domain/test_portfolio_state_machine.py | 34 ++++++++++---------- 4 files changed, 50 insertions(+), 31 deletions(-) diff --git a/atst/domain/csp/cloud.py b/atst/domain/csp/cloud.py index 9643dc7b..5d9a8397 100644 --- a/atst/domain/csp/cloud.py +++ b/atst/domain/csp/cloud.py @@ -4,6 +4,8 @@ from uuid import uuid4 from pydantic import BaseModel, validator +from flask import current_app as app + from atst.models.user import User from atst.models.application import Application from atst.models.environment import Environment @@ -754,6 +756,7 @@ class AzureSDKProvider(object): self.graphrbac = graphrbac self.credentials = credentials self.identity = identity + self.exceptions = exceptions self.secrets = secrets self.requests = requests # may change to a JEDI cloud @@ -781,14 +784,21 @@ class AzureCloudProvider(CloudProviderInterface): secret_client = self.secrets.SecretClient( vault_url=self.vault_url, credential=credential, ) - return secret_client.set_secret(secret_key, secret_value) + try: + return secret_client.set_secret(secret_key, secret_value) + except self.exceptions.HttpResponseError as exc: + app.logger.error(f"Could not SET secret in Azure keyvault for key {secret_key}.", exc_info=1) + def get_secret(secret_key): credential = self._get_client_secret_credential_obj() secret_client = self.secrets.SecretClient( vault_url=self.vault_url, credential=credential, ) - return secret_client.get_secret(secret_key).value + try: + return secret_client.get_secret(secret_key).value + except self.exceptions.HttpResponseError as exc: + app.logger.error(f"Could not GET secret in Azure keyvault for key {secret_key}.", exc_info=1) def create_environment( self, auth_credentials: Dict, user: User, environment: Environment diff --git a/atst/models/mixins/state_machines.py b/atst/models/mixins/state_machines.py index 8d50eb15..29f7a877 100644 --- a/atst/models/mixins/state_machines.py +++ b/atst/models/mixins/state_machines.py @@ -1,5 +1,7 @@ from enum import Enum +from flask import current_app as app + class StageStates(Enum): CREATED = "created" @@ -107,10 +109,12 @@ class FSMMixin: fail_trigger = "fail" + stage if fail_trigger in self.machine.get_triggers(self.current_state.name): self.trigger(fail_trigger) + app.logger.info(f"calling fail trigger '{fail_trigger}' for '{self.__repr__()}'") def finish_stage(self, stage): finish_trigger = "finish_" + stage if finish_trigger in self.machine.get_triggers(self.current_state.name): + app.logger.info(f"calling finish trigger '{finish_trigger}' for '{self.__repr__()}'") self.trigger(finish_trigger) def prepare_init(self, event): diff --git a/atst/models/portfolio_state_machine.py b/atst/models/portfolio_state_machine.py index 89dc073b..04d09bd6 100644 --- a/atst/models/portfolio_state_machine.py +++ b/atst/models/portfolio_state_machine.py @@ -64,6 +64,10 @@ class PortfolioStateMachine( db.session.add(self) db.session.commit() + def __repr__(self): + return f" Date: Wed, 22 Jan 2020 16:47:11 -0500 Subject: [PATCH 18/23] Fix Enum Migration to properly deal with constraint changes Since we're using non-native enums for our model, alembic has some issues knowing what the previous "type" actually was, and not specifying it correctly causes a bad constraint. --- alembic/env.py | 17 +-- ...19c44a8d5_state_machine_states_extended.py | 127 +++++++++++++++--- 2 files changed, 118 insertions(+), 26 deletions(-) diff --git a/alembic/env.py b/alembic/env.py index 825f5e19..5ea815e3 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -29,11 +29,13 @@ parent_dir = Path(__file__).parent.parent sys.path.append(parent_dir) from atst.app import make_config + app_config = make_config() -config.set_main_option('sqlalchemy.url', app_config['DATABASE_URI']) +config.set_main_option("sqlalchemy.url", app_config["DATABASE_URI"]) from atst.database import db from atst.models import * + target_metadata = Base.metadata @@ -51,8 +53,7 @@ def run_migrations_offline(): """ url = config.get_main_option("sqlalchemy.url") context.configure( - url=url, target_metadata=target_metadata, literal_binds=True, - compare_type=True + url=url, target_metadata=target_metadata, literal_binds=True, compare_type=True ) with context.begin_transaction(): @@ -68,19 +69,19 @@ def run_migrations_online(): """ connectable = engine_from_config( config.get_section(config.config_ini_section), - prefix='sqlalchemy.', - poolclass=pool.NullPool) + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) with connectable.connect() as connection: context.configure( - connection=connection, - target_metadata=target_metadata, - compare_type=True + connection=connection, target_metadata=target_metadata, compare_type=True ) with context.begin_transaction(): context.run_migrations() + if context.is_offline_mode(): run_migrations_offline() else: diff --git a/alembic/versions/26319c44a8d5_state_machine_states_extended.py b/alembic/versions/26319c44a8d5_state_machine_states_extended.py index a183295c..41ca30b5 100644 --- a/alembic/versions/26319c44a8d5_state_machine_states_extended.py +++ b/alembic/versions/26319c44a8d5_state_machine_states_extended.py @@ -10,33 +10,124 @@ import sqlalchemy as sa from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. -revision = '26319c44a8d5' # pragma: allowlist secret -down_revision = '59973fa17ded' # pragma: allowlist secret +revision = "26319c44a8d5" # pragma: allowlist secret +down_revision = "59973fa17ded" # pragma: allowlist secret branch_labels = None depends_on = None def upgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.alter_column('portfolio_state_machines', 'state', - existing_type=sa.VARCHAR(length=30), - type_=sa.Enum('UNSTARTED', 'STARTING', 'STARTED', 'COMPLETED', 'FAILED', 'TENANT_CREATED', 'TENANT_IN_PROGRESS', 'TENANT_FAILED', 'BILLING_PROFILE_CREATION_CREATED', 'BILLING_PROFILE_CREATION_IN_PROGRESS', 'BILLING_PROFILE_CREATION_FAILED', 'BILLING_PROFILE_VERIFICATION_CREATED', 'BILLING_PROFILE_VERIFICATION_IN_PROGRESS', 'BILLING_PROFILE_VERIFICATION_FAILED', 'BILLING_PROFILE_TENANT_ACCESS_CREATED', 'BILLING_PROFILE_TENANT_ACCESS_IN_PROGRESS', 'BILLING_PROFILE_TENANT_ACCESS_FAILED', 'TASK_ORDER_BILLING_CREATION_CREATED', 'TASK_ORDER_BILLING_CREATION_IN_PROGRESS', 'TASK_ORDER_BILLING_CREATION_FAILED', 'TASK_ORDER_BILLING_VERIFICATION_CREATED', 'TASK_ORDER_BILLING_VERIFICATION_IN_PROGRESS', 'TASK_ORDER_BILLING_VERIFICATION_FAILED', 'BILLING_INSTRUCTION_CREATED', 'BILLING_INSTRUCTION_IN_PROGRESS', 'BILLING_INSTRUCTION_FAILED', name='fsmstates', native_enum=False, create_constraint=False), - existing_nullable=False) - op.alter_column('portfolios', 'defense_component', - existing_type=postgresql.ARRAY(sa.VARCHAR()), - type_=sa.String(), - existing_nullable=False) + op.alter_column( + "portfolio_state_machines", + "state", + existing_type=sa.Enum( + "UNSTARTED", + "STARTING", + "STARTED", + "COMPLETED", + "FAILED", + "TENANT_CREATED", + "TENANT_IN_PROGRESS", + "TENANT_FAILED", + "BILLING_PROFILE_CREATED", + "BILLING_PROFILE_IN_PROGRESS", + "BILLING_PROFILE_FAILED", + "ADMIN_SUBSCRIPTION_CREATED", + "ADMIN_SUBSCRIPTION_IN_PROGRESS", + "ADMIN_SUBSCRIPTION_FAILED", + name="fsmstates", + native_enum=False, + ), + type_=sa.Enum( + "UNSTARTED", + "STARTING", + "STARTED", + "COMPLETED", + "FAILED", + "TENANT_CREATED", + "TENANT_IN_PROGRESS", + "TENANT_FAILED", + "BILLING_PROFILE_CREATION_CREATED", + "BILLING_PROFILE_CREATION_IN_PROGRESS", + "BILLING_PROFILE_CREATION_FAILED", + "BILLING_PROFILE_VERIFICATION_CREATED", + "BILLING_PROFILE_VERIFICATION_IN_PROGRESS", + "BILLING_PROFILE_VERIFICATION_FAILED", + "BILLING_PROFILE_TENANT_ACCESS_CREATED", + "BILLING_PROFILE_TENANT_ACCESS_IN_PROGRESS", + "BILLING_PROFILE_TENANT_ACCESS_FAILED", + "TASK_ORDER_BILLING_CREATION_CREATED", + "TASK_ORDER_BILLING_CREATION_IN_PROGRESS", + "TASK_ORDER_BILLING_CREATION_FAILED", + "TASK_ORDER_BILLING_VERIFICATION_CREATED", + "TASK_ORDER_BILLING_VERIFICATION_IN_PROGRESS", + "TASK_ORDER_BILLING_VERIFICATION_FAILED", + "BILLING_INSTRUCTION_CREATED", + "BILLING_INSTRUCTION_IN_PROGRESS", + "BILLING_INSTRUCTION_FAILED", + name="fsmstates", + native_enum=False, + create_constraint=False, + ), + existing_nullable=False, + ) # ### end Alembic commands ### def downgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.alter_column('portfolios', 'defense_component', - existing_type=sa.String(), - type_=postgresql.ARRAY(sa.VARCHAR()), - existing_nullable=False) - op.alter_column('portfolio_state_machines', 'state', - existing_type=sa.Enum('UNSTARTED', 'STARTING', 'STARTED', 'COMPLETED', 'FAILED', 'TENANT_CREATED', 'TENANT_IN_PROGRESS', 'TENANT_FAILED', 'BILLING_PROFILE_CREATION_CREATED', 'BILLING_PROFILE_CREATION_IN_PROGRESS', 'BILLING_PROFILE_CREATION_FAILED', 'BILLING_PROFILE_VERIFICATION_CREATED', 'BILLING_PROFILE_VERIFICATION_IN_PROGRESS', 'BILLING_PROFILE_VERIFICATION_FAILED', 'BILLING_PROFILE_TENANT_ACCESS_CREATED', 'BILLING_PROFILE_TENANT_ACCESS_IN_PROGRESS', 'BILLING_PROFILE_TENANT_ACCESS_FAILED', 'TASK_ORDER_BILLING_CREATION_CREATED', 'TASK_ORDER_BILLING_CREATION_IN_PROGRESS', 'TASK_ORDER_BILLING_CREATION_FAILED', 'TASK_ORDER_BILLING_VERIFICATION_CREATED', 'TASK_ORDER_BILLING_VERIFICATION_IN_PROGRESS', 'TASK_ORDER_BILLING_VERIFICATION_FAILED', 'BILLING_INSTRUCTION_CREATED', 'BILLING_INSTRUCTION_IN_PROGRESS', 'BILLING_INSTRUCTION_FAILED', name='fsmstates', native_enum=False), - type_=sa.VARCHAR(length=30), - existing_nullable=False) + op.alter_column( + "portfolio_state_machines", + "state", + existing_type=sa.Enum( + "UNSTARTED", + "STARTING", + "STARTED", + "COMPLETED", + "FAILED", + "TENANT_CREATED", + "TENANT_IN_PROGRESS", + "TENANT_FAILED", + "BILLING_PROFILE_CREATION_CREATED", + "BILLING_PROFILE_CREATION_IN_PROGRESS", + "BILLING_PROFILE_CREATION_FAILED", + "BILLING_PROFILE_VERIFICATION_CREATED", + "BILLING_PROFILE_VERIFICATION_IN_PROGRESS", + "BILLING_PROFILE_VERIFICATION_FAILED", + "BILLING_PROFILE_TENANT_ACCESS_CREATED", + "BILLING_PROFILE_TENANT_ACCESS_IN_PROGRESS", + "BILLING_PROFILE_TENANT_ACCESS_FAILED", + "TASK_ORDER_BILLING_CREATION_CREATED", + "TASK_ORDER_BILLING_CREATION_IN_PROGRESS", + "TASK_ORDER_BILLING_CREATION_FAILED", + "TASK_ORDER_BILLING_VERIFICATION_CREATED", + "TASK_ORDER_BILLING_VERIFICATION_IN_PROGRESS", + "TASK_ORDER_BILLING_VERIFICATION_FAILED", + "BILLING_INSTRUCTION_CREATED", + "BILLING_INSTRUCTION_IN_PROGRESS", + "BILLING_INSTRUCTION_FAILED", + name="fsmstates", + native_enum=False, + ), + type_=sa.Enum( + "UNSTARTED", + "STARTING", + "STARTED", + "COMPLETED", + "FAILED", + "TENANT_CREATED", + "TENANT_IN_PROGRESS", + "TENANT_FAILED", + "BILLING_PROFILE_CREATED", + "BILLING_PROFILE_IN_PROGRESS", + "BILLING_PROFILE_FAILED", + "ADMIN_SUBSCRIPTION_CREATED", + "ADMIN_SUBSCRIPTION_IN_PROGRESS", + "ADMIN_SUBSCRIPTION_FAILED", + name="fsmstates", + native_enum=False, + ), + existing_nullable=False, + ) # ### end Alembic commands ### From ea040a914ea56fa3dca28d139c2d5f113fb7153f Mon Sep 17 00:00:00 2001 From: tomdds Date: Fri, 24 Jan 2020 11:01:53 -0500 Subject: [PATCH 19/23] Properly report initial clin information Includes fixed up state machine test as well as adds some missing dependencies --- Pipfile | 1 + Pipfile.lock | 82 +++++++++++---- atst/domain/csp/cloud.py | 101 +++++++++++++++---- atst/models/portfolio_state_machine.py | 21 +++- tests/domain/cloud/test_azure_csp.py | 10 +- tests/domain/test_portfolio_state_machine.py | 67 +++++++----- tests/mock_azure.py | 7 ++ 7 files changed, 208 insertions(+), 81 deletions(-) diff --git a/Pipfile b/Pipfile index 06bd2675..5d3a7920 100644 --- a/Pipfile +++ b/Pipfile @@ -36,6 +36,7 @@ transitions = "*" azure-mgmt-consumption = "*" adal = "*" azure-identity = "*" +azure-keyvault = "*" [dev-packages] bandit = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 8a030e11..53a28b17 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "3760f0b1df1156211d671afa2eb417b7bf980aa33d2f74d390e8eed6a3ce8c8b" + "sha256": "4dbb023bcb860eb6dc56e1c201c91f272e1e67ad03e5e5eeb3a7a7fdff350eed" }, "pipfile-spec": 6, "requires": { @@ -26,10 +26,10 @@ }, "alembic": { "hashes": [ - "sha256:3b0cb1948833e062f4048992fbc97ecfaaaac24aaa0d83a1202a99fb58af8c6d" + "sha256:d412982920653db6e5a44bfd13b1d0db5685cbaaccaf226195749c706e1e862a" ], "index": "pypi", - "version": "==1.3.2" + "version": "==1.3.3" }, "amqp": { "hashes": [ @@ -68,6 +68,28 @@ "index": "pypi", "version": "==1.2.0" }, + "azure-keyvault": { + "hashes": [ + "sha256:76f75cb83929f312a08616d426ad6f597f1beae180131cf445876fb88f2c8ef1", + "sha256:e85f5bd6cb4f10b3248b99bbf02e3acc6371d366846897027d4153f18025a2d7" + ], + "index": "pypi", + "version": "==4.0.0" + }, + "azure-keyvault-keys": { + "hashes": [ + "sha256:2983fa42e20a0e6bf6b87976716129c108e613e0292d34c5b0f0c8dc1d488e89", + "sha256:38c27322637a2c52620a8b96da1942ad6a8d22d09b5a01f6fa257f7a51e52ed0" + ], + "version": "==4.0.0" + }, + "azure-keyvault-secrets": { + "hashes": [ + "sha256:2eae9264a8f6f59277e1a9bfdbc8b0a15969ee5a80d8efe403d7744805b4a481", + "sha256:97a602406a833e8f117c540c66059c818f4321a35168dd17365fab1e4527d718" + ], + "version": "==4.0.0" + }, "azure-mgmt-authorization": { "hashes": [ "sha256:31e875a34ac2c5d6fefe77b4a8079a8b2bdbe9edb957e47e8b44222fb212d6a7", @@ -232,6 +254,14 @@ ], "version": "==2.8" }, + "dataclasses": { + "hashes": [ + "sha256:3459118f7ede7c8bea0fe795bff7c6c2ce287d01dd226202f7c9ebc0610a7836", + "sha256:494a6dcae3b8bcf80848eea2ef64c0cc5cd307ffc263e17cdf42f3e5420808e6" + ], + "markers": "python_version < '3.7'", + "version": "==0.7" + }, "flask": { "hashes": [ "sha256:13f9f196f330c7c2c5d7a5cf91af894110ca0215ac051b5844701f2bfd934d52", @@ -325,9 +355,9 @@ }, "mako": { "hashes": [ - "sha256:a36919599a9b7dc5d86a7a8988f23a9a3a3d083070023bab23d64f7f1d1e0a4b" + "sha256:2984a6733e1d472796ceef37ad48c26f4a984bb18119bb2dbc37a44d8f6e75a4" ], - "version": "==1.1.0" + "version": "==1.1.1" }, "markupsafe": { "hashes": [ @@ -584,10 +614,10 @@ }, "sqlalchemy": { "hashes": [ - "sha256:bfb8f464a5000b567ac1d350b9090cf081180ec1ab4aa87e7bca12dab25320ec" + "sha256:64a7b71846db6423807e96820993fa12a03b89127d278290ca25c0b11ed7b4fb" ], "index": "pypi", - "version": "==1.3.12" + "version": "==1.3.13" }, "sqlalchemy-json": { "hashes": [ @@ -620,10 +650,10 @@ }, "urllib3": { "hashes": [ - "sha256:a8a318824cc77d1fd4b2bec2ded92646630d7fe8619497b142c84a9e6f5a7293", - "sha256:f3c5fd51747d450d4dcf6f923c81f78f811aab8205fda64b0aba34a4e48b0745" + "sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc", + "sha256:87716c2d2a7121198ebcb7ce7cccf6ce5e9ba539041cfbaeecfb641dc0bf6acc" ], - "version": "==1.25.7" + "version": "==1.25.8" }, "vine": { "hashes": [ @@ -657,10 +687,10 @@ }, "zipp": { "hashes": [ - "sha256:57147f6b0403b59f33fd357f169f860e031303415aeb7d04ede4839d23905ab8", - "sha256:7ae5ccaca427bafa9760ac3cd8f8c244bfc259794b5b6bb9db4dda2241575d09" + "sha256:b338014b9bc7102ca69e0fb96ed07215a8954d2989bc5d83658494ab2ba634af", + "sha256:e013e7800f60ec4dde789ebf4e9f7a54236e4bbf5df2a1a4e20ce9e1d9609d67" ], - "version": "==2.0.0" + "version": "==2.0.1" } }, "develop": { @@ -671,6 +701,14 @@ ], "version": "==1.4.3" }, + "appnope": { + "hashes": [ + "sha256:5b26757dc6f79a3b7dc9fab95359328d5747fcb2409d331ea66d0272b90ab2a0", + "sha256:8b995ffe925347a2138d7ac0fe77155e4311a0ea6d6da4f5128fe4b3cbe5ed71" + ], + "markers": "sys_platform == 'darwin'", + "version": "==0.1.0" + }, "argh": { "hashes": [ "sha256:a9b3aaa1904eeb78e32394cd46c6f37ac0fb4af6dc488daa58971bdc7d7fcaf3", @@ -1062,11 +1100,11 @@ }, "pexpect": { "hashes": [ - "sha256:2094eefdfcf37a1fdbfb9aa090862c1a4878e5c7e0e7e7088bdb511c558e5cd1", - "sha256:9e2c1fd0e6ee3a49b28f95d4b33bc389c89b20af6a1255906e90ff1262ce62eb" + "sha256:0b48a55dcb3c05f3329815901ea4fc1537514d6ba867a152b581d69ae3710937", + "sha256:fc65a43959d153d0114afe13997d439c22823a27cefceb5ff35c2178c6784c0c" ], "markers": "sys_platform != 'win32'", - "version": "==4.7.0" + "version": "==4.8.0" }, "pickleshare": { "hashes": [ @@ -1325,10 +1363,10 @@ }, "urllib3": { "hashes": [ - "sha256:a8a318824cc77d1fd4b2bec2ded92646630d7fe8619497b142c84a9e6f5a7293", - "sha256:f3c5fd51747d450d4dcf6f923c81f78f811aab8205fda64b0aba34a4e48b0745" + "sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc", + "sha256:87716c2d2a7121198ebcb7ce7cccf6ce5e9ba539041cfbaeecfb641dc0bf6acc" ], - "version": "==1.25.7" + "version": "==1.25.8" }, "watchdog": { "hashes": [ @@ -1359,10 +1397,10 @@ }, "zipp": { "hashes": [ - "sha256:57147f6b0403b59f33fd357f169f860e031303415aeb7d04ede4839d23905ab8", - "sha256:7ae5ccaca427bafa9760ac3cd8f8c244bfc259794b5b6bb9db4dda2241575d09" + "sha256:b338014b9bc7102ca69e0fb96ed07215a8954d2989bc5d83658494ab2ba634af", + "sha256:e013e7800f60ec4dde789ebf4e9f7a54236e4bbf5df2a1a4e20ce9e1d9609d67" ], - "version": "==2.0.0" + "version": "==2.0.1" } } } diff --git a/atst/domain/csp/cloud.py b/atst/domain/csp/cloud.py index 5d9a8397..44bb8517 100644 --- a/atst/domain/csp/cloud.py +++ b/atst/domain/csp/cloud.py @@ -327,12 +327,12 @@ class TaskOrderBillingCreationCSPPayload(BaseCSPPayload): class TaskOrderBillingCreationCSPResult(AliasModel): task_order_billing_verify_url: str - retry_after: int + task_order_retry_after: int class Config: fields = { "task_order_billing_verify_url": "Location", - "retry_after": "Retry-After", + "task_order_retry_after": "Retry-After", } @@ -358,11 +358,11 @@ class TaskOrderBillingVerificationCSPResult(AliasModel): class BillingInstructionCSPPayload(BaseCSPPayload): - amount: float - start_date: str - end_date: str - clin_type: str - task_order_id: str + initial_clin_amount: float + initial_clin_start_date: str + initial_clin_end_date: str + initial_clin_type: str + initial_task_order_id: str billing_account_name: str billing_profile_name: str @@ -646,19 +646,76 @@ class MockCloudProvider(CloudProviderInterface): self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) - return { - "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB/billingRoleAssignments/40000000-aaaa-bbbb-cccc-100000000000_0a5f4926-e3ee-4f47-a6e3-8b0a30a40e3d", - "name": "40000000-aaaa-bbbb-cccc-100000000000_0a5f4926-e3ee-4f47-a6e3-8b0a30a40e3d", + return BillingProfileTenantAccessCSPResult( + **{ + "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB/billingRoleAssignments/40000000-aaaa-bbbb-cccc-100000000000_0a5f4926-e3ee-4f47-a6e3-8b0a30a40e3d", + "name": "40000000-aaaa-bbbb-cccc-100000000000_0a5f4926-e3ee-4f47-a6e3-8b0a30a40e3d", + "properties": { + "createdOn": "2020-01-14T14:39:26.3342192+00:00", + "createdByPrincipalId": "82e2b376-3297-4096-8743-ed65b3be0b03", + "principalId": "0a5f4926-e3ee-4f47-a6e3-8b0a30a40e3d", + "principalTenantId": "60ff9d34-82bf-4f21-b565-308ef0533435", + "roleDefinitionId": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB/billingRoleDefinitions/40000000-aaaa-bbbb-cccc-100000000000", + "scope": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB", + }, + "type": "Microsoft.Billing/billingRoleAssignments", + } + ).dict() + + def create_task_order_billing_creation(self, payload: TaskOrderBillingCreationCSPPayload): + self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) + self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) + self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) + + return TaskOrderBillingCreationCSPResult(**{"Location": "https://somelocation", "Retry-After": "10"}).dict() + + def create_task_order_billing_verification(self, payload: TaskOrderBillingVerificationCSPPayload): + self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) + self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) + self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) + + return TaskOrderBillingVerificationCSPResult(**{ + "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/XC36-GRNZ-BG7-TGB", + "name": "XC36-GRNZ-BG7-TGB", "properties": { - "createdOn": "2020-01-14T14:39:26.3342192+00:00", - "createdByPrincipalId": "82e2b376-3297-4096-8743-ed65b3be0b03", - "principalId": "0a5f4926-e3ee-4f47-a6e3-8b0a30a40e3d", - "principalTenantId": "60ff9d34-82bf-4f21-b565-308ef0533435", - "roleDefinitionId": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB/billingRoleDefinitions/40000000-aaaa-bbbb-cccc-100000000000", - "scope": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/KQWI-W2SU-BG7-TGB", + "address": { + "addressLine1": "123 S Broad Street, Suite 2400", + "city": "Philadelphia", + "companyName": "Promptworks", + "country": "US", + "postalCode": "19109", + "region": "PA" + }, + "currency": "USD", + "displayName": "First Portfolio Billing Profile", + "enabledAzurePlans": [ + { + "productId": "DZH318Z0BPS6", + "skuId": "0001", + "skuDescription": "Microsoft Azure Plan" + } + ], + "hasReadAccess": True, + "invoiceDay": 5, + "invoiceEmailOptIn": False }, - "type": "Microsoft.Billing/billingRoleAssignments", - } + "type": "Microsoft.Billing/billingAccounts/billingProfiles" + }).dict() + + def create_billing_instruction(self, payload: BillingInstructionCSPPayload): + self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) + self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) + self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) + + return BillingInstructionCSPResult(**{ + "name": "TO1:CLIN001", + "properties": { + "amount": 1000.0, + "endDate": "2020-03-01T00:00:00+00:00", + "startDate": "2020-01-01T00:00:00+00:00" + }, + "type": "Microsoft.Billing/billingAccounts/billingProfiles/billingInstructions" + }).dict() def create_or_update_user(self, auth_credentials, user_info, csp_role_id): self._authorize(auth_credentials) @@ -1150,13 +1207,13 @@ class AzureCloudProvider(CloudProviderInterface): request_body = { "properties": { - "amount": payload.amount, - "startDate": payload.start_date, - "endDate": payload.end_date, + "amount": payload.initial_clin_amount, + "startDate": payload.initial_clin_start_date, + "endDate": payload.initial_clin_end_date, } } - url = f"https://management.azure.com/providers/Microsoft.Billing/billingAccounts/{payload.billing_account_name}/billingProfiles/{payload.billing_profile_name}/instructions/{payload.task_order_id}:CLIN00{payload.clin_type}?api-version=2019-10-01-preview" + url = f"https://management.azure.com/providers/Microsoft.Billing/billingAccounts/{payload.billing_account_name}/billingProfiles/{payload.billing_profile_name}/instructions/{payload.initial_task_order_id}:CLIN00{payload.initial_clin_type}?api-version=2019-10-01-preview" auth_header = { "Authorization": f"Bearer {sp_token}", diff --git a/atst/models/portfolio_state_machine.py b/atst/models/portfolio_state_machine.py index 04d09bd6..3f0e4d7d 100644 --- a/atst/models/portfolio_state_machine.py +++ b/atst/models/portfolio_state_machine.py @@ -67,7 +67,6 @@ class PortfolioStateMachine( def __repr__(self): return f" Date: Fri, 24 Jan 2020 11:03:38 -0500 Subject: [PATCH 20/23] Formatting fixes --- atst/domain/csp/cloud.py | 97 +++++++++++--------- atst/models/mixins/state_machines.py | 8 +- tests/domain/test_portfolio_state_machine.py | 1 - 3 files changed, 62 insertions(+), 44 deletions(-) diff --git a/atst/domain/csp/cloud.py b/atst/domain/csp/cloud.py index 44bb8517..2257ace1 100644 --- a/atst/domain/csp/cloud.py +++ b/atst/domain/csp/cloud.py @@ -662,60 +662,70 @@ class MockCloudProvider(CloudProviderInterface): } ).dict() - def create_task_order_billing_creation(self, payload: TaskOrderBillingCreationCSPPayload): + def create_task_order_billing_creation( + self, payload: TaskOrderBillingCreationCSPPayload + ): self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) - return TaskOrderBillingCreationCSPResult(**{"Location": "https://somelocation", "Retry-After": "10"}).dict() + return TaskOrderBillingCreationCSPResult( + **{"Location": "https://somelocation", "Retry-After": "10"} + ).dict() - def create_task_order_billing_verification(self, payload: TaskOrderBillingVerificationCSPPayload): + def create_task_order_billing_verification( + self, payload: TaskOrderBillingVerificationCSPPayload + ): self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) - return TaskOrderBillingVerificationCSPResult(**{ - "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/XC36-GRNZ-BG7-TGB", - "name": "XC36-GRNZ-BG7-TGB", - "properties": { - "address": { - "addressLine1": "123 S Broad Street, Suite 2400", - "city": "Philadelphia", - "companyName": "Promptworks", - "country": "US", - "postalCode": "19109", - "region": "PA" + return TaskOrderBillingVerificationCSPResult( + **{ + "id": "/providers/Microsoft.Billing/billingAccounts/7c89b735-b22b-55c0-ab5a-c624843e8bf6:de4416ce-acc6-44b1-8122-c87c4e903c91_2019-05-31/billingProfiles/XC36-GRNZ-BG7-TGB", + "name": "XC36-GRNZ-BG7-TGB", + "properties": { + "address": { + "addressLine1": "123 S Broad Street, Suite 2400", + "city": "Philadelphia", + "companyName": "Promptworks", + "country": "US", + "postalCode": "19109", + "region": "PA", + }, + "currency": "USD", + "displayName": "First Portfolio Billing Profile", + "enabledAzurePlans": [ + { + "productId": "DZH318Z0BPS6", + "skuId": "0001", + "skuDescription": "Microsoft Azure Plan", + } + ], + "hasReadAccess": True, + "invoiceDay": 5, + "invoiceEmailOptIn": False, }, - "currency": "USD", - "displayName": "First Portfolio Billing Profile", - "enabledAzurePlans": [ - { - "productId": "DZH318Z0BPS6", - "skuId": "0001", - "skuDescription": "Microsoft Azure Plan" - } - ], - "hasReadAccess": True, - "invoiceDay": 5, - "invoiceEmailOptIn": False - }, - "type": "Microsoft.Billing/billingAccounts/billingProfiles" - }).dict() + "type": "Microsoft.Billing/billingAccounts/billingProfiles", + } + ).dict() def create_billing_instruction(self, payload: BillingInstructionCSPPayload): self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) - return BillingInstructionCSPResult(**{ - "name": "TO1:CLIN001", - "properties": { - "amount": 1000.0, - "endDate": "2020-03-01T00:00:00+00:00", - "startDate": "2020-01-01T00:00:00+00:00" - }, - "type": "Microsoft.Billing/billingAccounts/billingProfiles/billingInstructions" - }).dict() + return BillingInstructionCSPResult( + **{ + "name": "TO1:CLIN001", + "properties": { + "amount": 1000.0, + "endDate": "2020-03-01T00:00:00+00:00", + "startDate": "2020-01-01T00:00:00+00:00", + }, + "type": "Microsoft.Billing/billingAccounts/billingProfiles/billingInstructions", + } + ).dict() def create_or_update_user(self, auth_credentials, user_info, csp_role_id): self._authorize(auth_credentials) @@ -844,8 +854,10 @@ class AzureCloudProvider(CloudProviderInterface): try: return secret_client.set_secret(secret_key, secret_value) except self.exceptions.HttpResponseError as exc: - app.logger.error(f"Could not SET secret in Azure keyvault for key {secret_key}.", exc_info=1) - + app.logger.error( + f"Could not SET secret in Azure keyvault for key {secret_key}.", + exc_info=1, + ) def get_secret(secret_key): credential = self._get_client_secret_credential_obj() @@ -855,7 +867,10 @@ class AzureCloudProvider(CloudProviderInterface): try: return secret_client.get_secret(secret_key).value except self.exceptions.HttpResponseError as exc: - app.logger.error(f"Could not GET secret in Azure keyvault for key {secret_key}.", exc_info=1) + app.logger.error( + f"Could not GET secret in Azure keyvault for key {secret_key}.", + exc_info=1, + ) def create_environment( self, auth_credentials: Dict, user: User, environment: Environment diff --git a/atst/models/mixins/state_machines.py b/atst/models/mixins/state_machines.py index 29f7a877..37682e5b 100644 --- a/atst/models/mixins/state_machines.py +++ b/atst/models/mixins/state_machines.py @@ -109,12 +109,16 @@ class FSMMixin: fail_trigger = "fail" + stage if fail_trigger in self.machine.get_triggers(self.current_state.name): self.trigger(fail_trigger) - app.logger.info(f"calling fail trigger '{fail_trigger}' for '{self.__repr__()}'") + app.logger.info( + f"calling fail trigger '{fail_trigger}' for '{self.__repr__()}'" + ) def finish_stage(self, stage): finish_trigger = "finish_" + stage if finish_trigger in self.machine.get_triggers(self.current_state.name): - app.logger.info(f"calling finish trigger '{finish_trigger}' for '{self.__repr__()}'") + app.logger.info( + f"calling finish trigger '{finish_trigger}' for '{self.__repr__()}'" + ) self.trigger(finish_trigger) def prepare_init(self, event): diff --git a/tests/domain/test_portfolio_state_machine.py b/tests/domain/test_portfolio_state_machine.py index 8ec9615d..82a6d086 100644 --- a/tests/domain/test_portfolio_state_machine.py +++ b/tests/domain/test_portfolio_state_machine.py @@ -152,4 +152,3 @@ def test_fsm_transition_start(portfolio: Portfolio): csp_data = portfolio.csp_data else: csp_data = {} - From e9d03ec68bc6cd19b05b0854926ea0b9020da7b0 Mon Sep 17 00:00:00 2001 From: tomdds Date: Fri, 24 Jan 2020 11:12:06 -0500 Subject: [PATCH 21/23] Fix some LGTM errors and start sketching in credential update functionality --- atst/domain/csp/cloud.py | 12 +++++++++--- atst/models/portfolio_state_machine.py | 11 ++++------- tests/domain/test_portfolio_state_machine.py | 5 ++--- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/atst/domain/csp/cloud.py b/atst/domain/csp/cloud.py index 2257ace1..b46b03b4 100644 --- a/atst/domain/csp/cloud.py +++ b/atst/domain/csp/cloud.py @@ -380,7 +380,7 @@ class CloudProviderInterface: def set_secret(self, secret_key: str, secret_value: str): raise NotImplementedError() - def get_secret(self, secret_key: str, secret_value: str): + def get_secret(self, secret_key: str): raise NotImplementedError() def root_creds(self) -> Dict: @@ -520,6 +520,12 @@ class MockCloudProvider(CloudProviderInterface): def root_creds(self): return self._auth_credentials + def set_secret(self, secret_key: str, secret_value: str): + pass + + def get_secret(self, secret_key: str): + return {} + def create_environment(self, auth_credentials, user, environment): self._authorize(auth_credentials) @@ -846,7 +852,7 @@ class AzureCloudProvider(CloudProviderInterface): self.policy_manager = AzurePolicyManager(config["AZURE_POLICY_LOCATION"]) - def set_secret(secret_key, secret_value): + def set_secret(self, secret_key, secret_value): credential = self._get_client_secret_credential_obj() secret_client = self.secrets.SecretClient( vault_url=self.vault_url, credential=credential, @@ -859,7 +865,7 @@ class AzureCloudProvider(CloudProviderInterface): exc_info=1, ) - def get_secret(secret_key): + def get_secret(self, secret_key): credential = self._get_client_secret_credential_obj() secret_client = self.secrets.SecretClient( vault_url=self.vault_url, credential=credential, diff --git a/atst/models/portfolio_state_machine.py b/atst/models/portfolio_state_machine.py index 3f0e4d7d..6e3015c4 100644 --- a/atst/models/portfolio_state_machine.py +++ b/atst/models/portfolio_state_machine.py @@ -1,5 +1,4 @@ from random import choice, choices -import re import string from sqlalchemy import Column, ForeignKey, Enum as SQLAEnum @@ -119,7 +118,6 @@ class PortfolioStateMachine( elif state_obj.is_CREATED: # the create trigger for the next stage should be in the available # triggers for the current state - triggers = self.machine.get_triggers(state_obj.name) create_trigger = next( filter( lambda trigger: trigger.startswith("create_"), @@ -205,11 +203,10 @@ class PortfolioStateMachine( dc = cls(**stage_data) if getattr(dc, "get_creds", None) is not None: new_creds = dc.get_creds() - print("creds to report") - print(new_creds) - # TODO: how/where to store these - # TODO: credential schema - # self.store_creds(self.portfolio, new_creds) + tenant_id = new_creds.get("tenant_id") + secret = self.csp.get_secret(tenant_id) + secret.update(new_creds) + self.csp.set_secret(tenant_id, secret) except PydanticValidationError as exc: app.logger.error( diff --git a/tests/domain/test_portfolio_state_machine.py b/tests/domain/test_portfolio_state_machine.py index 82a6d086..c5ca68cb 100644 --- a/tests/domain/test_portfolio_state_machine.py +++ b/tests/domain/test_portfolio_state_machine.py @@ -2,7 +2,6 @@ import pytest import re from tests.factories import ( - PortfolioFactory, PortfolioStateMachineFactory, TaskOrderFactory, CLINFactory, @@ -36,7 +35,7 @@ def test_state_machine_trigger_next_transition(portfolio): def test_state_machine_compose_state(portfolio): - sm = PortfolioStateMachineFactory.create(portfolio=portfolio) + PortfolioStateMachineFactory.create(portfolio=portfolio) assert ( compose_state(AzureStages.TENANT, StageStates.CREATED) == FSMStates.TENANT_CREATED @@ -44,7 +43,7 @@ def test_state_machine_compose_state(portfolio): def test_state_machine_valid_data_classes_for_stages(portfolio): - sm = PortfolioStateMachineFactory.create(portfolio=portfolio) + PortfolioStateMachineFactory.create(portfolio=portfolio) for stage in AzureStages: assert get_stage_csp_class(stage.name.lower(), "payload") is not None assert get_stage_csp_class(stage.name.lower(), "result") is not None From b9206ed7be509c3cf868022fc8f064e761cf8d08 Mon Sep 17 00:00:00 2001 From: tomdds Date: Fri, 24 Jan 2020 13:35:46 -0500 Subject: [PATCH 22/23] Some more LGTM fixes --- alembic/versions/26319c44a8d5_state_machine_states_extended.py | 1 - tests/domain/cloud/test_azure_csp.py | 1 - tests/domain/test_portfolio_state_machine.py | 2 +- 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/alembic/versions/26319c44a8d5_state_machine_states_extended.py b/alembic/versions/26319c44a8d5_state_machine_states_extended.py index 41ca30b5..720e50ee 100644 --- a/alembic/versions/26319c44a8d5_state_machine_states_extended.py +++ b/alembic/versions/26319c44a8d5_state_machine_states_extended.py @@ -7,7 +7,6 @@ Create Date: 2020-01-22 15:54:03.186751 """ from alembic import op import sqlalchemy as sa -from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "26319c44a8d5" # pragma: allowlist secret diff --git a/tests/domain/cloud/test_azure_csp.py b/tests/domain/cloud/test_azure_csp.py index f247a961..9e87e7a7 100644 --- a/tests/domain/cloud/test_azure_csp.py +++ b/tests/domain/cloud/test_azure_csp.py @@ -1,4 +1,3 @@ -import pytest from unittest.mock import Mock from uuid import uuid4 diff --git a/tests/domain/test_portfolio_state_machine.py b/tests/domain/test_portfolio_state_machine.py index c5ca68cb..0d37c9c7 100644 --- a/tests/domain/test_portfolio_state_machine.py +++ b/tests/domain/test_portfolio_state_machine.py @@ -3,7 +3,6 @@ import re from tests.factories import ( PortfolioStateMachineFactory, - TaskOrderFactory, CLINFactory, ) @@ -15,6 +14,7 @@ from atst.domain.csp import get_stage_csp_class @pytest.fixture(scope="function") def portfolio(): + # TODO: setup clin/to as active/funded/ready portfolio = CLINFactory.create().task_order.portfolio return portfolio From 7e0fda67b06621a95a4a42f4ecf9e2ee9b9c13d4 Mon Sep 17 00:00:00 2001 From: tomdds Date: Fri, 24 Jan 2020 13:52:41 -0500 Subject: [PATCH 23/23] Clean up unusued imports and variables --- atst/domain/csp/cloud.py | 10 +++++----- atst/models/portfolio_state_machine.py | 13 ------------- 2 files changed, 5 insertions(+), 18 deletions(-) diff --git a/atst/domain/csp/cloud.py b/atst/domain/csp/cloud.py index 104e3e0e..d22f9475 100644 --- a/atst/domain/csp/cloud.py +++ b/atst/domain/csp/cloud.py @@ -799,7 +799,7 @@ class MockCloudProvider(CloudProviderInterface): @property def _auth_credentials(self): - return {"username": "mock-cloud", "password": "shh"} + return {"username": "mock-cloud", "password": "shh"} # pragma: allowlist secret def _authorize(self, credentials): self._delay(1, 5) @@ -864,26 +864,26 @@ class AzureCloudProvider(CloudProviderInterface): self.policy_manager = AzurePolicyManager(config["AZURE_POLICY_LOCATION"]) def set_secret(self, secret_key, secret_value): - credential = self._get_client_secret_credential_obj() + credential = self._get_client_secret_credential_obj({}) secret_client = self.secrets.SecretClient( vault_url=self.vault_url, credential=credential, ) try: return secret_client.set_secret(secret_key, secret_value) - except self.exceptions.HttpResponseError as exc: + except self.exceptions.HttpResponseError: app.logger.error( f"Could not SET secret in Azure keyvault for key {secret_key}.", exc_info=1, ) def get_secret(self, secret_key): - credential = self._get_client_secret_credential_obj() + credential = self._get_client_secret_credential_obj({}) secret_client = self.secrets.SecretClient( vault_url=self.vault_url, credential=credential, ) try: return secret_client.get_secret(secret_key).value - except self.exceptions.HttpResponseError as exc: + except self.exceptions.HttpResponseError: app.logger.error( f"Could not GET secret in Azure keyvault for key {secret_key}.", exc_info=1, diff --git a/atst/models/portfolio_state_machine.py b/atst/models/portfolio_state_machine.py index 6e3015c4..a0cc77cd 100644 --- a/atst/models/portfolio_state_machine.py +++ b/atst/models/portfolio_state_machine.py @@ -1,6 +1,3 @@ -from random import choice, choices -import string - from sqlalchemy import Column, ForeignKey, Enum as SQLAEnum from sqlalchemy.orm import relationship, reconstructor from sqlalchemy.dialects.postgresql import UUID @@ -20,16 +17,6 @@ import atst.models.mixins as mixins from atst.models.mixins.state_machines import FSMStates, AzureStages, _build_transitions -def make_password(): - return choice(string.ascii_letters) + "".join( - choices(string.ascii_letters + string.digits + string.punctuation, k=15) - ) - - -def fetch_portfolio_creds(portfolio): - return dict(username="mock-cloud", password="shh") - - @add_state_features(Tags) class StateMachineWithTags(Machine): pass