From 5cd20c650a3041f25795456356647a75a33cd39a Mon Sep 17 00:00:00 2001 From: tomdds Date: Tue, 24 Sep 2019 17:27:22 -0400 Subject: [PATCH 01/10] Draft of Azure create_environment --- Pipfile | 1 + Pipfile.lock | 64 +++++++++++++++++--- atst/domain/csp/cloud.py | 123 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 180 insertions(+), 8 deletions(-) diff --git a/Pipfile b/Pipfile index 68c97392..8b0383fb 100644 --- a/Pipfile +++ b/Pipfile @@ -25,6 +25,7 @@ PyYAML = "*" azure-storage = "*" azure-storage-common = "*" celery = "*" +azure-mgmt-subscription = "*" [dev-packages] bandit = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 3cf4e198..aafa0dbb 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,21 +1,26 @@ { "_meta": { "hash": { - "sha256": "2f366c5a5f62ba5a451ca024759cc8cbf481a15e5198d73311fa9cf194d7f547" + "sha256": "eca6e06b6a34bdba3872c595957d840da21f3e58cb41f3ec02b7c3a424a5e4c4" }, "pipfile-spec": 6, "requires": { "python_version": "3.7.3" }, - "sources": [ - { - "name": "pypi", - "url": "https://pypi.org/simple", - "verify_ssl": true - } - ] + "sources": [{ + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + }] }, "default": { + "adal": { + "hashes": [ + "sha256:5a7f1e037c6290c6d7609cab33a9e5e988c2fbec5c51d1c4c649ee3faff37eaf", + "sha256:fd17e5661f60634ddf96a569b95d34ccb8a98de60593d729c28bdcfe360eaad1" + ], + "version": "==1.2.2" + }, "alembic": { "hashes": [ "sha256:9f907d7e8b286a1cfb22db9084f9ce4fde7ad7956bb496dc7c952e10ac90e36a" @@ -45,6 +50,14 @@ ], "version": "==1.1.23" }, + "azure-mgmt-subscription": { + "hashes": [ + "sha256:504b4c42ba859070c3c50637ec07ca36aca600e613fcccaa398db22822fe21f1", + "sha256:850f86de5078f61f3a9bd81cb2b938ea25ce9a266c9fe22c7fca94a2b5968c16" + ], + "index": "pypi", + "version": "==0.5.0" + }, "azure-nspkg": { "hashes": [ "sha256:1d0bbb2157cf57b1bef6c8c8e5b41133957364456c43b0a43599890023cca0a8", @@ -221,6 +234,13 @@ ], "version": "==0.23" }, + "isodate": { + "hashes": [ + "sha256:2e364a3d5759479cdb2d37cce6b9376ea504db2ff90252a2e5b7cc89cc9ff2d8", + "sha256:aa4d33c06640f5352aca96e4b81afd8ab3b47337cc12089822d6f322ac772c81" + ], + "version": "==0.6.0" + }, "itsdangerous": { "hashes": [ "sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19", @@ -296,6 +316,27 @@ ], "version": "==7.2.0" }, + "msrest": { + "hashes": [ + "sha256:56b8b5b4556fb2a92cac640df267d560889bdc9e2921187772d4691d97bc4e8d", + "sha256:f5153bfe60ee757725816aedaa0772cbfe0bddb52cd2d6db4cb8b4c3c6c6f928" + ], + "version": "==0.6.10" + }, + "msrestazure": { + "hashes": [ + "sha256:63db9f646fffc9244b332090e679d1e5f283ac491ee0cc321f5116f9450deb4a", + "sha256:fecb6a72a3eb5483e4deff38210d26ae42d3f6d488a7a275bd2423a1a014b22c" + ], + "version": "==0.6.2" + }, + "oauthlib": { + "hashes": [ + "sha256:bee41cc35fcca6e988463cacc3bcb8a96224f470ca547e697b604cc697b2f889", + "sha256:df884cd6cbe20e32633f1db1072e9356f53638e4361bef4e8b03c9127c9328ea" + ], + "version": "==3.1.0" + }, "pendulum": { "hashes": [ "sha256:1cde6e3c6310fb882c98f373795f807cb2bd6af01f34d2857e6e283b5ee91e09", @@ -350,6 +391,13 @@ ], "version": "==2.19" }, + "pyjwt": { + "hashes": [ + "sha256:5c6eca3c2940464d106b99ba83b00c6add741c9becaec087fb7ccdefea71350e", + "sha256:8d59a976fb773f3e6a39c85636357c4f0e242707394cadadd9814f5cbaa20e96" + ], + "version": "==1.7.1" + }, "pyopenssl": { "hashes": [ "sha256:aeca66338f6de19d1aa46ed634c3b9ae519a64b458f8468aec688e7e3c20f200", diff --git a/atst/domain/csp/cloud.py b/atst/domain/csp/cloud.py index 7bcbf44f..58dac63a 100644 --- a/atst/domain/csp/cloud.py +++ b/atst/domain/csp/cloud.py @@ -383,3 +383,126 @@ class MockCloudProvider(CloudProviderInterface): self._delay(1, 5) if self._with_authorization and credentials != self._auth_credentials: raise self.AUTHENTICATION_EXCEPTION + + +AZURE_ENVIRONMENT = "AZURE_PUBLIC_CLOUD" # TBD +AZURE_SKU_ID = "?" # probably a static sku specific to ATAT/JEDI +SUBSCRIPTION_ID_REGEX = re.compile( + "subscriptions\/([0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})", + re.I, +) + +class AzureCloudProvider(CloudProviderInterface): + def __init__(self, config): + self.config = config + + self.client_id = config["AZURE_CLIENT_ID"] + self.secret_key = config["AZURE_SECRET_KEY"] + self.tenant_id = config["AZURE_TENANT_ID"] + + import azure.mgmt as mgmt + import azure.graphrbac as graphrbac + import azure.common.credentials as credentials + + self.azure_mgmt = mgmt + self.azure_graph = graphrbac + self.azure_credentials = credentials + + def root_creds(self): + return { + "client_id": self.client_id, + "secret_key": self.secret_key, + "tenant_id": self.tenant_id, + } + + def create_environment( + self, auth_credentials: Dict, user: User, environment: Environment + ): + credentials = self._get_credential_obj(self.root_creds()) + sub_client = self.azure_mgmt.subscription.SubscriptionClient(credentials) + + display_name = ( + f"{environment.application.name}_{environment.name}_{environment.id}" + ) # proposed format + + billing_profile_id = "?" # something chained from environment? + sku_id = AZURE_SKU_ID + body = self.azure_mgmt.subscription.models.ModernSubscriptionCreationParameters( + display_name, billing_profile_id, sku_id + ) + + # These 2 seem like something that might be worthwhile to allow tiebacks to + # TOs filed for the environment + billing_account_name = "?" + invoice_section_name = "?" + sub_creation_operation = sub_client.subscription_factory.create_subscription( + billing_account_name, invoice_section_name, body + ) + + # the resulting object from this process is a link to the new subscription + # not a subscription model, so we'll have to unpack the ID + new_sub = sub_creation_operation.result() + + subscription_id = self._extract_subscription_id(new_sub.subscription_link) + if subscription_id: + return subscription_id + else: + # troublesome error, subscription should exist at this point + # but we just don't have a valid ID + pass + + def create_atat_admin_user( + self, auth_credentials: Dict, csp_environment_id: str + ) -> Dict: + root_creds = self.root_creds() + credentials = self._get_credential_obj(root_creds) + + self.azure_mgmt. + + sub_client = self.azure_mgmt.subscription.SubscriptionClient(credentials) + subscription: self.azure_mgmt.subscription.models.Subscription = sub_client.subscriptions.get( + csp_environment_id + ) + + # how do we scope the graph client to the new subscription rather than + # the cloud0 subscription? tenant id seems to be separate from subscription id + graph_client = self.azure_graph.GraphRbacManagementClient( + credentials, root_creds.get("tenant_id") + app_create_param = self.azure_graph.models.ApplicationCreateParameters( + display_name=app_display_name + ) + app: self.azure_graph.models.Application = graph_client.applications.create( + app_create_param + ) + + self.azure_graph.models. + + # create a new service principle for the new application, which should be scoped + # to the new subscription + app_id = app.app_id + sp_create_params = self.azure_graph.models.ServicePrincipalCreateParameters( + app_id=app_id, account_enabled=True + ) + + service_principal = graph_client.service_principals.create(sp_create_params) + + return { + "csp_user_id": service_principal.object_id, + "credentials": service_principal.password_credentials, + } + + def _extract_subscription_id(self, subscription_url): + sub_id_match = SUBSCRIPTION_ID_REGEX.match(subscription_url) + + if sub_id_match: + return sub_id_match.group(1) + + def _get_credential_obj(self, creds, resource="https://graph.windows.net"): + return self.azure_credentials.ServicePrincipalCredentials( + client_id=creds.get("client_id"), + secret=creds.get("secret_key"), + tenant=creds.get("tenant_id"), + resource=resource, + cloud_environment=AZURE_ENVIRONMENT, + ) + From 15ff4a01f1d3e373954938fdcb043eecabc86488 Mon Sep 17 00:00:00 2001 From: tomdds Date: Fri, 27 Sep 2019 14:41:12 -0400 Subject: [PATCH 02/10] Add Azure Graph API --- Pipfile | 1 + Pipfile.lock | 10 +++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/Pipfile b/Pipfile index 8b0383fb..08f4ee13 100644 --- a/Pipfile +++ b/Pipfile @@ -26,6 +26,7 @@ azure-storage = "*" azure-storage-common = "*" celery = "*" azure-mgmt-subscription = "*" +azure-graphrbac = "*" [dev-packages] bandit = "*" diff --git a/Pipfile.lock b/Pipfile.lock index aafa0dbb..56ffb5c1 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "eca6e06b6a34bdba3872c595957d840da21f3e58cb41f3ec02b7c3a424a5e4c4" + "sha256": "a736295b3c9c21285eb6c83c331fa7f40651c67c8baf70377db7329b7f74e7c6" }, "pipfile-spec": 6, "requires": { @@ -50,6 +50,14 @@ ], "version": "==1.1.23" }, + "azure-graphrbac": { + "hashes": [ + "sha256:53e98ae2ca7c19b349e9e9bb1b6a824aeae8dcfcbe17190d20fe69c0f185b2e2", + "sha256:7b4e0f05676acc912f2b33c71c328d9fb2e4dc8e70ebadc9d3de8ab08bf0b175" + ], + "index": "pypi", + "version": "==0.61.1" + }, "azure-mgmt-subscription": { "hashes": [ "sha256:504b4c42ba859070c3c50637ec07ca36aca600e613fcccaa398db22822fe21f1", From 608f988b71e54c3668a86f54520c7d97c9926ad0 Mon Sep 17 00:00:00 2001 From: tomdds Date: Fri, 27 Sep 2019 14:41:42 -0400 Subject: [PATCH 03/10] First pass at process of adding admin to azure --- atst/domain/csp/cloud.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/atst/domain/csp/cloud.py b/atst/domain/csp/cloud.py index 58dac63a..420518d1 100644 --- a/atst/domain/csp/cloud.py +++ b/atst/domain/csp/cloud.py @@ -457,8 +457,6 @@ class AzureCloudProvider(CloudProviderInterface): root_creds = self.root_creds() credentials = self._get_credential_obj(root_creds) - self.azure_mgmt. - sub_client = self.azure_mgmt.subscription.SubscriptionClient(credentials) subscription: self.azure_mgmt.subscription.models.Subscription = sub_client.subscriptions.get( csp_environment_id @@ -468,6 +466,10 @@ class AzureCloudProvider(CloudProviderInterface): # the cloud0 subscription? tenant id seems to be separate from subscription id graph_client = self.azure_graph.GraphRbacManagementClient( credentials, root_creds.get("tenant_id") + ) + + # assuming the graph_client is scoped to the new subscription, create an application + app_display_name = "?" app_create_param = self.azure_graph.models.ApplicationCreateParameters( display_name=app_display_name ) @@ -475,8 +477,6 @@ class AzureCloudProvider(CloudProviderInterface): app_create_param ) - self.azure_graph.models. - # create a new service principle for the new application, which should be scoped # to the new subscription app_id = app.app_id From 06dc193c2877b891b17d3295d1070a069a510d42 Mon Sep 17 00:00:00 2001 From: dandds Date: Wed, 2 Oct 2019 14:27:13 -0400 Subject: [PATCH 04/10] WIP: can add new app/SP --- Pipfile | 1 + atst/domain/csp/cloud.py | 50 ++++++++++++++++++++++++++++++++-------- 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/Pipfile b/Pipfile index 08f4ee13..5ad90c9c 100644 --- a/Pipfile +++ b/Pipfile @@ -27,6 +27,7 @@ azure-storage-common = "*" celery = "*" azure-mgmt-subscription = "*" azure-graphrbac = "*" +mrestazure = "*" [dev-packages] bandit = "*" diff --git a/atst/domain/csp/cloud.py b/atst/domain/csp/cloud.py index 420518d1..b5244e97 100644 --- a/atst/domain/csp/cloud.py +++ b/atst/domain/csp/cloud.py @@ -1,4 +1,5 @@ from typing import Dict +import re from uuid import uuid4 from atst.models.user import User @@ -400,13 +401,16 @@ class AzureCloudProvider(CloudProviderInterface): self.secret_key = config["AZURE_SECRET_KEY"] self.tenant_id = config["AZURE_TENANT_ID"] - import azure.mgmt as mgmt + from azure.mgmt import subscription import azure.graphrbac as graphrbac import azure.common.credentials as credentials + from msrestazure.azure_cloud import AZURE_PUBLIC_CLOUD - self.azure_mgmt = mgmt + self.azure_subscription = subscription self.azure_graph = graphrbac self.azure_credentials = credentials + # may change to a JEDI cloud + self.azure_cloud = AZURE_PUBLIC_CLOUD def root_creds(self): return { @@ -427,14 +431,20 @@ class AzureCloudProvider(CloudProviderInterface): billing_profile_id = "?" # something chained from environment? sku_id = AZURE_SKU_ID + # we want to set AT-AT as an owner here + # we could potentially associate subscriptions with "management groups" per DOD component body = self.azure_mgmt.subscription.models.ModernSubscriptionCreationParameters( - display_name, billing_profile_id, sku_id + display_name, + billing_profile_id, + sku_id, + # owner= ) # These 2 seem like something that might be worthwhile to allow tiebacks to # TOs filed for the environment billing_account_name = "?" invoice_section_name = "?" + # We may also want to create billing sections in the enrollment account sub_creation_operation = sub_client.subscription_factory.create_subscription( billing_account_name, invoice_section_name, body ) @@ -455,17 +465,34 @@ class AzureCloudProvider(CloudProviderInterface): self, auth_credentials: Dict, csp_environment_id: str ) -> Dict: root_creds = self.root_creds() - credentials = self._get_credential_obj(root_creds) + credentials = self._get_credential_obj( + root_creds, resource="https://management.azure.com" + ) - sub_client = self.azure_mgmt.subscription.SubscriptionClient(credentials) - subscription: self.azure_mgmt.subscription.models.Subscription = sub_client.subscriptions.get( + sub_client = self.azure_subscription.SubscriptionClient(credentials) + subscription: self.azure_subscription.models.Subscription = sub_client.subscriptions.get( csp_environment_id ) + from azure.common.credentials import ServicePrincipalCredentials + + graph_creds = ServicePrincipalCredentials( + client_id=self.client_id, + secret=self.secret_key, + tenant=self.tenant_id, + cloud_environment=self.azure_cloud, + # we really should be using graph.microsoft.com, but i'm getting + # "expired token" errors for that + # resource = "https://graph.microsoft.com" + resource="https://graph.windows.net", + ) + # I needed to set permissions for the graph.windows.net API before I + # could get this to work. + # how do we scope the graph client to the new subscription rather than # the cloud0 subscription? tenant id seems to be separate from subscription id graph_client = self.azure_graph.GraphRbacManagementClient( - credentials, root_creds.get("tenant_id") + graph_creds, root_creds.get("tenant_id") ) # assuming the graph_client is scoped to the new subscription, create an application @@ -473,6 +500,12 @@ class AzureCloudProvider(CloudProviderInterface): app_create_param = self.azure_graph.models.ApplicationCreateParameters( display_name=app_display_name ) + + # we need the appropriate perms here: + # https://docs.microsoft.com/en-us/graph/api/application-post-applications?view=graph-rest-beta&tabs=http + # https://docs.microsoft.com/en-us/graph/permissions-reference#microsoft-graph-permission-names + # set app perms in app registration portal + # https://docs.microsoft.com/en-us/graph/auth-v2-service#2-configure-permissions-for-microsoft-graph app: self.azure_graph.models.Application = graph_client.applications.create( app_create_param ) @@ -503,6 +536,5 @@ class AzureCloudProvider(CloudProviderInterface): secret=creds.get("secret_key"), tenant=creds.get("tenant_id"), resource=resource, - cloud_environment=AZURE_ENVIRONMENT, + cloud_environment=self.azure_cloud, ) - From 41633417d8a2c836566bd3a0858fca77416f7ca1 Mon Sep 17 00:00:00 2001 From: tomdds Date: Fri, 25 Oct 2019 13:52:47 -0400 Subject: [PATCH 05/10] Add Azure Libraries --- Pipfile | 4 +++- Pipfile.lock | 20 +++++++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/Pipfile b/Pipfile index 5ad90c9c..6456afb1 100644 --- a/Pipfile +++ b/Pipfile @@ -27,7 +27,8 @@ azure-storage-common = "*" celery = "*" azure-mgmt-subscription = "*" azure-graphrbac = "*" -mrestazure = "*" +msrestazure = "*" +azure-mgmt-authorization = "*" [dev-packages] bandit = "*" @@ -48,6 +49,7 @@ pytest-mock = "*" detect-secrets = "*" beautifulsoup4 = "*" mypy = "*" +rope = "*" [requires] python_version = "3.7.3" diff --git a/Pipfile.lock b/Pipfile.lock index 56ffb5c1..45241281 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "a736295b3c9c21285eb6c83c331fa7f40651c67c8baf70377db7329b7f74e7c6" + "sha256": "3e1d71e50dafe3cd2aec48615fba53e12c35761459a6c6cf7e4a3df30502f4ee" }, "pipfile-spec": 6, "requires": { @@ -58,6 +58,14 @@ "index": "pypi", "version": "==0.61.1" }, + "azure-mgmt-authorization": { + "hashes": [ + "sha256:31e875a34ac2c5d6fefe77b4a8079a8b2bdbe9edb957e47e8b44222fb212d6a7", + "sha256:9d64295cf4210ec14e98fb024a6b4d79d68ef50cdb3804f0b53f8567e52d847f" + ], + "index": "pypi", + "version": "==0.60.0" + }, "azure-mgmt-subscription": { "hashes": [ "sha256:504b4c42ba859070c3c50637ec07ca36aca600e613fcccaa398db22822fe21f1", @@ -336,6 +344,7 @@ "sha256:63db9f646fffc9244b332090e679d1e5f283ac491ee0cc321f5116f9450deb4a", "sha256:fecb6a72a3eb5483e4deff38210d26ae42d3f6d488a7a275bd2423a1a014b22c" ], + "index": "pypi", "version": "==0.6.2" }, "oauthlib": { @@ -1074,6 +1083,15 @@ "index": "pypi", "version": "==2.22.0" }, + "rope": { + "hashes": [ + "sha256:6b728fdc3e98a83446c27a91fc5d56808a004f8beab7a31ab1d7224cecc7d969", + "sha256:c5c5a6a87f7b1a2095fb311135e2a3d1f194f5ecb96900fdd0a9100881f48aaf", + "sha256:f0dcf719b63200d492b85535ebe5ea9b29e0d0b8aebeb87fe03fc1a65924fdaf" + ], + "index": "pypi", + "version": "==0.14.0" + }, "selenium": { "hashes": [ "sha256:49d9b3bc156b795b94b06ae4e65dcd9c454b8e5f29624f2a117f02e3020ce411", From 1a92cd35d12304c858ce75498dfae717ea8817c6 Mon Sep 17 00:00:00 2001 From: tomdds Date: Mon, 28 Oct 2019 11:18:44 -0400 Subject: [PATCH 06/10] Extract service principal resolution to private method Also made root creds a property --- atst/domain/csp/cloud.py | 91 ++++++++++++++++++++++++++-------------- 1 file changed, 59 insertions(+), 32 deletions(-) diff --git a/atst/domain/csp/cloud.py b/atst/domain/csp/cloud.py index b5244e97..1593914a 100644 --- a/atst/domain/csp/cloud.py +++ b/atst/domain/csp/cloud.py @@ -386,13 +386,17 @@ class MockCloudProvider(CloudProviderInterface): raise self.AUTHENTICATION_EXCEPTION -AZURE_ENVIRONMENT = "AZURE_PUBLIC_CLOUD" # TBD +AZURE_ENVIRONMENT = "AZURE_PUBLIC_CLOUD" # TBD AZURE_SKU_ID = "?" # probably a static sku specific to ATAT/JEDI SUBSCRIPTION_ID_REGEX = re.compile( "subscriptions\/([0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})", re.I, ) +# This needs to be a fully pathed role definition identifier, not just a UUID +REMOTE_ROOT_ROLE_DEF_ID = "/providers/Microsoft.Authorization/roleDefinitions/00000000-0000-4000-8000-000000000000" + + class AzureCloudProvider(CloudProviderInterface): def __init__(self, config): self.config = config @@ -401,28 +405,22 @@ class AzureCloudProvider(CloudProviderInterface): self.secret_key = config["AZURE_SECRET_KEY"] self.tenant_id = config["AZURE_TENANT_ID"] - from azure.mgmt import subscription + from azure.mgmt import subscription, authorization import azure.graphrbac as graphrbac import azure.common.credentials as credentials from msrestazure.azure_cloud import AZURE_PUBLIC_CLOUD self.azure_subscription = subscription + self.azure_authorization = authorization self.azure_graph = graphrbac self.azure_credentials = credentials # may change to a JEDI cloud self.azure_cloud = AZURE_PUBLIC_CLOUD - def root_creds(self): - return { - "client_id": self.client_id, - "secret_key": self.secret_key, - "tenant_id": self.tenant_id, - } - def create_environment( self, auth_credentials: Dict, user: User, environment: Environment ): - credentials = self._get_credential_obj(self.root_creds()) + credentials = self._get_credential_obj(self._root_creds) sub_client = self.azure_mgmt.subscription.SubscriptionClient(credentials) display_name = ( @@ -464,27 +462,49 @@ class AzureCloudProvider(CloudProviderInterface): def create_atat_admin_user( self, auth_credentials: Dict, csp_environment_id: str ) -> Dict: - root_creds = self.root_creds() - credentials = self._get_credential_obj( - root_creds, resource="https://management.azure.com" - ) + root_creds = self._root_creds + credentials = self._get_credential_obj(root_creds) sub_client = self.azure_subscription.SubscriptionClient(credentials) subscription: self.azure_subscription.models.Subscription = sub_client.subscriptions.get( csp_environment_id ) - from azure.common.credentials import ServicePrincipalCredentials + managment_principal = self._get_management_service_principal() - graph_creds = ServicePrincipalCredentials( - client_id=self.client_id, - secret=self.secret_key, - tenant=self.tenant_id, - cloud_environment=self.azure_cloud, - # we really should be using graph.microsoft.com, but i'm getting - # "expired token" errors for that - # resource = "https://graph.microsoft.com" - resource="https://graph.windows.net", + auth_client = self.azure_authorization.AuthorizationManagementClient( + credentials, + # TODO: Determine which subscription this needs to point at + # Once we're in a multi-sub environment + subscription.id, + ) + + # Create role assignment for + role_assignment_id = uuid.uuid4() + role_assignment_create_params = auth_client.role_assignments.models.RoleAssignmentCreateParameters( + role_definition_id=REMOTE_ROOT_ROLE_DEF_ID, + principal_id=managment_principal.id, + ) + + self.azure_authorization.models.RoleAssignment = auth_client.role_assignments.create( + scope=f"/subscriptions/{subscription.id}/", + role_assignment_name=role_assignment_id, + parameters=role_assignment_create_params, + ) + + return { + "csp_user_id": service_principal.object_id, + "credentials": service_principal.password_credentials, + "role_name": role_assignment_id, + } + + def _get_management_service_principal(self): + # we really should be using graph.microsoft.com, but i'm getting + # "expired token" errors for that + # graph_resource = "https://graph.microsoft.com" + graph_resource = "https://graph.windows.net" + graph_creds = self._get_credential_obj( + self._root_creds, resource=graph_resource ) # I needed to set permissions for the graph.windows.net API before I # could get this to work. @@ -492,11 +512,13 @@ class AzureCloudProvider(CloudProviderInterface): # how do we scope the graph client to the new subscription rather than # the cloud0 subscription? tenant id seems to be separate from subscription id graph_client = self.azure_graph.GraphRbacManagementClient( - graph_creds, root_creds.get("tenant_id") + graph_creds, self._root_creds.get("tenant_id") ) - # assuming the graph_client is scoped to the new subscription, create an application - app_display_name = "?" + # do we need to create a new application to manage each subscripition + # or should we manage access to each subscription from a single service + # principal with multiple role assignments? + app_display_name = "?" # name should reflect the subscription it exists app_create_param = self.azure_graph.models.ApplicationCreateParameters( display_name=app_display_name ) @@ -519,10 +541,7 @@ class AzureCloudProvider(CloudProviderInterface): service_principal = graph_client.service_principals.create(sp_create_params) - return { - "csp_user_id": service_principal.object_id, - "credentials": service_principal.password_credentials, - } + return service_principal def _extract_subscription_id(self, subscription_url): sub_id_match = SUBSCRIPTION_ID_REGEX.match(subscription_url) @@ -530,7 +549,7 @@ class AzureCloudProvider(CloudProviderInterface): if sub_id_match: return sub_id_match.group(1) - def _get_credential_obj(self, creds, resource="https://graph.windows.net"): + def _get_credential_obj(self, creds, resource=None): return self.azure_credentials.ServicePrincipalCredentials( client_id=creds.get("client_id"), secret=creds.get("secret_key"), @@ -538,3 +557,11 @@ class AzureCloudProvider(CloudProviderInterface): resource=resource, cloud_environment=self.azure_cloud, ) + + @property + def _root_creds(self): + return { + "client_id": self.client_id, + "secret_key": self.secret_key, + "tenant_id": self.tenant_id, + } From 99e306e6025571f9d050c9400eaf48c419c627be Mon Sep 17 00:00:00 2001 From: tomdds Date: Mon, 28 Oct 2019 15:05:44 -0400 Subject: [PATCH 07/10] First pass at mocking and testing azure integration --- atst/domain/csp/cloud.py | 56 ++++++++++++++----------- tests/domain/cloud/test_azure_csp.py | 16 ++++++++ tests/mock_azure.py | 61 ++++++++++++++++++++++++++++ 3 files changed, 109 insertions(+), 24 deletions(-) create mode 100644 tests/domain/cloud/test_azure_csp.py create mode 100644 tests/mock_azure.py diff --git a/atst/domain/csp/cloud.py b/atst/domain/csp/cloud.py index 1593914a..d4ea39d1 100644 --- a/atst/domain/csp/cloud.py +++ b/atst/domain/csp/cloud.py @@ -397,31 +397,39 @@ SUBSCRIPTION_ID_REGEX = re.compile( REMOTE_ROOT_ROLE_DEF_ID = "/providers/Microsoft.Authorization/roleDefinitions/00000000-0000-4000-8000-000000000000" +class AzureSDKProvider(object): + def __init__(self): + from azure.mgmt import subscription, authorization + import azure.graphrbac as graphrbac + import azure.common.credentials as credentials + from msrestazure.azure_cloud import AZURE_PUBLIC_CLOUD + + self.subscription = subscription + self.authorization = authorization + self.graphrbac = graphrbac + self.credentials = credentials + # may change to a JEDI cloud + self.cloud = AZURE_PUBLIC_CLOUD + + class AzureCloudProvider(CloudProviderInterface): - def __init__(self, config): + def __init__(self, config, azure_sdk_provider=None): self.config = config self.client_id = config["AZURE_CLIENT_ID"] self.secret_key = config["AZURE_SECRET_KEY"] self.tenant_id = config["AZURE_TENANT_ID"] - from azure.mgmt import subscription, authorization - import azure.graphrbac as graphrbac - import azure.common.credentials as credentials - from msrestazure.azure_cloud import AZURE_PUBLIC_CLOUD - - self.azure_subscription = subscription - self.azure_authorization = authorization - self.azure_graph = graphrbac - self.azure_credentials = credentials - # may change to a JEDI cloud - self.azure_cloud = AZURE_PUBLIC_CLOUD + if azure_sdk_provider is None: + self.sdk = AzureSDKProvider() + else: + self.sdk = azure_sdk_provider def create_environment( self, auth_credentials: Dict, user: User, environment: Environment ): credentials = self._get_credential_obj(self._root_creds) - sub_client = self.azure_mgmt.subscription.SubscriptionClient(credentials) + sub_client = self.sdk.subscription.SubscriptionClient(credentials) display_name = ( f"{environment.application.name}_{environment.name}_{environment.id}" @@ -431,7 +439,7 @@ class AzureCloudProvider(CloudProviderInterface): sku_id = AZURE_SKU_ID # we want to set AT-AT as an owner here # we could potentially associate subscriptions with "management groups" per DOD component - body = self.azure_mgmt.subscription.models.ModernSubscriptionCreationParameters( + body = self.sdk.subscription.models.ModernSubscriptionCreationParameters( display_name, billing_profile_id, sku_id, @@ -465,14 +473,14 @@ class AzureCloudProvider(CloudProviderInterface): root_creds = self._root_creds credentials = self._get_credential_obj(root_creds) - sub_client = self.azure_subscription.SubscriptionClient(credentials) - subscription: self.azure_subscription.models.Subscription = sub_client.subscriptions.get( + sub_client = self.sdk.subscription.SubscriptionClient(credentials) + subscription: self.sdk.subscription.models.Subscription = sub_client.subscriptions.get( csp_environment_id ) managment_principal = self._get_management_service_principal() - auth_client = self.azure_authorization.AuthorizationManagementClient( + auth_client = self.sdk.authorization.AuthorizationManagementClient( credentials, # TODO: Determine which subscription this needs to point at # Once we're in a multi-sub environment @@ -486,7 +494,7 @@ class AzureCloudProvider(CloudProviderInterface): principal_id=managment_principal.id, ) - self.azure_authorization.models.RoleAssignment = auth_client.role_assignments.create( + self.sdk.authorization.models.RoleAssignment = auth_client.role_assignments.create( scope=f"/subscriptions/{subscription.id}/", role_assignment_name=role_assignment_id, parameters=role_assignment_create_params, @@ -511,7 +519,7 @@ class AzureCloudProvider(CloudProviderInterface): # how do we scope the graph client to the new subscription rather than # the cloud0 subscription? tenant id seems to be separate from subscription id - graph_client = self.azure_graph.GraphRbacManagementClient( + graph_client = self.sdk.graphrbac.GraphRbacManagementClient( graph_creds, self._root_creds.get("tenant_id") ) @@ -519,7 +527,7 @@ class AzureCloudProvider(CloudProviderInterface): # or should we manage access to each subscription from a single service # principal with multiple role assignments? app_display_name = "?" # name should reflect the subscription it exists - app_create_param = self.azure_graph.models.ApplicationCreateParameters( + app_create_param = self.sdk.graphrbac.models.ApplicationCreateParameters( display_name=app_display_name ) @@ -528,14 +536,14 @@ class AzureCloudProvider(CloudProviderInterface): # https://docs.microsoft.com/en-us/graph/permissions-reference#microsoft-graph-permission-names # set app perms in app registration portal # https://docs.microsoft.com/en-us/graph/auth-v2-service#2-configure-permissions-for-microsoft-graph - app: self.azure_graph.models.Application = graph_client.applications.create( + app: self.sdk.graphrbac.models.Application = graph_client.applications.create( app_create_param ) # create a new service principle for the new application, which should be scoped # to the new subscription app_id = app.app_id - sp_create_params = self.azure_graph.models.ServicePrincipalCreateParameters( + sp_create_params = self.sdk.graphrbac.models.ServicePrincipalCreateParameters( app_id=app_id, account_enabled=True ) @@ -550,12 +558,12 @@ class AzureCloudProvider(CloudProviderInterface): return sub_id_match.group(1) def _get_credential_obj(self, creds, resource=None): - return self.azure_credentials.ServicePrincipalCredentials( + return self.sdk.credentials.ServicePrincipalCredentials( client_id=creds.get("client_id"), secret=creds.get("secret_key"), tenant=creds.get("tenant_id"), resource=resource, - cloud_environment=self.azure_cloud, + cloud_environment=self.sdk.cloud, ) @property diff --git a/tests/domain/cloud/test_azure_csp.py b/tests/domain/cloud/test_azure_csp.py new file mode 100644 index 00000000..95d6e013 --- /dev/null +++ b/tests/domain/cloud/test_azure_csp.py @@ -0,0 +1,16 @@ +import pytest +from unittest.mock import Mock + +from atst.domain.csp.cloud import EnvironmentCreationException, AzureCloudProvider +from atst.jobs import ( + do_create_environment, + do_create_atat_admin_user, + do_create_environment_baseline, +) + +from tests.mock_azure import mock_azure, AUTH_CREDENTIALS +from tests.factories import EnvironmentFactory + + +def test_create_environment_succeeds(mock_azure: AzureCloudProvider): + print(mock_azure._get_credential_obj(mock_azure._root_creds)) diff --git a/tests/mock_azure.py b/tests/mock_azure.py new file mode 100644 index 00000000..a94aad62 --- /dev/null +++ b/tests/mock_azure.py @@ -0,0 +1,61 @@ +import pytest +from unittest.mock import Mock + +from atst.domain.csp.cloud import AzureCloudProvider + +AZURE_CONFIG = { + "AZURE_CLIENT_ID": "MOCK", + "AZURE_SECRET_KEY": "MOCK", + "AZURE_TENANT_ID": "MOCK", +} + +AUTH_CREDENTIALS = { + "CLIENT_ID": AZURE_CONFIG["AZURE_CLIENT_ID"], + "SECRET_KEY": AZURE_CONFIG["AZURE_SECRET_KEY"], + "TENANT_ID": AZURE_CONFIG["AZURE_TENANT_ID"], +} + + +def mock_subscription(): + from azure.mgmt import subscription + + sub_mock = Mock(spec=subscription) + + return sub_mock + + +def mock_authorization(): + from azure.mgmt import authorization + + return Mock(spec=authorization) + + +def mock_graphrbac(): + import azure.graphrbac as graphrbac + + return Mock(spec=graphrbac) + + +def mock_credentials(): + import azure.common.credentials as credentials + + cred_mock = Mock(spec=credentials) + return cred_mock + + +class MockAzureSDK(object): + def __init__(self): + from msrestazure.azure_cloud import AZURE_PUBLIC_CLOUD + + self.subscription = mock_subscription() + self.authorization = mock_authorization() + self.graphrbac = mock_graphrbac() + self.credentials = mock_credentials() + # may change to a JEDI cloud + self.cloud = AZURE_PUBLIC_CLOUD + + +@pytest.fixture(scope="function") +def mock_azure(): + return AzureCloudProvider(AZURE_CONFIG, azure_sdk_provider=MockAzureSDK()) + From 63ea7db390df0f7e940cfc7ee1cce0199e9af182 Mon Sep 17 00:00:00 2001 From: tomdds Date: Tue, 29 Oct 2019 16:15:02 -0400 Subject: [PATCH 08/10] Rudimentary tests to validate mocking --- atst/domain/csp/cloud.py | 8 ++++---- tests/domain/cloud/test_azure_csp.py | 30 +++++++++++++++++++++++++++- tests/mock_azure.py | 7 ++----- 3 files changed, 35 insertions(+), 10 deletions(-) diff --git a/atst/domain/csp/cloud.py b/atst/domain/csp/cloud.py index d4ea39d1..35e7b2bf 100644 --- a/atst/domain/csp/cloud.py +++ b/atst/domain/csp/cloud.py @@ -488,21 +488,21 @@ class AzureCloudProvider(CloudProviderInterface): ) # Create role assignment for - role_assignment_id = uuid.uuid4() + role_assignment_id = str(uuid4()) role_assignment_create_params = auth_client.role_assignments.models.RoleAssignmentCreateParameters( role_definition_id=REMOTE_ROOT_ROLE_DEF_ID, principal_id=managment_principal.id, ) - self.sdk.authorization.models.RoleAssignment = auth_client.role_assignments.create( + auth_client.role_assignments.create( scope=f"/subscriptions/{subscription.id}/", role_assignment_name=role_assignment_id, parameters=role_assignment_create_params, ) return { - "csp_user_id": service_principal.object_id, - "credentials": service_principal.password_credentials, + "csp_user_id": managment_principal.object_id, + "credentials": managment_principal.password_credentials, "role_name": role_assignment_id, } diff --git a/tests/domain/cloud/test_azure_csp.py b/tests/domain/cloud/test_azure_csp.py index 95d6e013..67ca30f9 100644 --- a/tests/domain/cloud/test_azure_csp.py +++ b/tests/domain/cloud/test_azure_csp.py @@ -1,6 +1,8 @@ import pytest from unittest.mock import Mock +from uuid import uuid4 + from atst.domain.csp.cloud import EnvironmentCreationException, AzureCloudProvider from atst.jobs import ( do_create_environment, @@ -13,4 +15,30 @@ from tests.factories import EnvironmentFactory def test_create_environment_succeeds(mock_azure: AzureCloudProvider): - print(mock_azure._get_credential_obj(mock_azure._root_creds)) + environment = EnvironmentFactory.create() + + subscription_id = str(uuid4()) + + mock_azure.sdk.subscription.SubscriptionClient.return_value.subscription_factory.create_subscription.return_value.result.return_value.subscription_link = ( + f"subscriptions/{subscription_id}" + ) + + result = mock_azure.create_environment( + AUTH_CREDENTIALS, environment.creator, environment + ) + + assert result == subscription_id + + +def test_create_atat_admin_user_succeeds(mock_azure: AzureCloudProvider): + environment_id = str(uuid4()) + + csp_user_id = str(uuid4) + + mock_azure.sdk.graphrbac.GraphRbacManagementClient.return_value.service_principals.create.return_value.object_id = ( + csp_user_id + ) + + result = mock_azure.create_atat_admin_user(AUTH_CREDENTIALS, environment_id) + + assert result.get("csp_user_id") == csp_user_id diff --git a/tests/mock_azure.py b/tests/mock_azure.py index a94aad62..d838b506 100644 --- a/tests/mock_azure.py +++ b/tests/mock_azure.py @@ -19,9 +19,7 @@ AUTH_CREDENTIALS = { def mock_subscription(): from azure.mgmt import subscription - sub_mock = Mock(spec=subscription) - - return sub_mock + return Mock(spec=subscription) def mock_authorization(): @@ -39,8 +37,7 @@ def mock_graphrbac(): def mock_credentials(): import azure.common.credentials as credentials - cred_mock = Mock(spec=credentials) - return cred_mock + return Mock(spec=credentials) class MockAzureSDK(object): From 3e7a720ffbbe293cc1f9bbad799e0495b58087e5 Mon Sep 17 00:00:00 2001 From: tomdds Date: Tue, 29 Oct 2019 16:28:28 -0400 Subject: [PATCH 09/10] Post-rebase fixes --- Pipfile.lock | 55 ++++++++++++++++++++++------ atst/domain/csp/cloud.py | 4 +- atst/models/permissions.py | 4 +- tests/domain/cloud/test_azure_csp.py | 6 +-- 4 files changed, 47 insertions(+), 22 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index 45241281..815fc74d 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,17 +1,19 @@ { "_meta": { "hash": { - "sha256": "3e1d71e50dafe3cd2aec48615fba53e12c35761459a6c6cf7e4a3df30502f4ee" + "sha256": "6d2ab855267daac877ae7464de9dba5b62b7d89288992f87d8fc6ff0c0d2520f" }, "pipfile-spec": 6, "requires": { "python_version": "3.7.3" }, - "sources": [{ - "name": "pypi", - "url": "https://pypi.org/simple", - "verify_ssl": true - }] + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] }, "default": { "adal": { @@ -487,6 +489,13 @@ "index": "pypi", "version": "==2.22.0" }, + "requests-oauthlib": { + "hashes": [ + "sha256:bd6533330e8748e94bf0b214775fed487d309b8b8fe823dc45641ebcd9a32f57", + "sha256:d3ed0c8f2e3bbc6b344fa63d6f933745ab394469da38db16bdddb461c7e25140" + ], + "version": "==1.2.0" + }, "six": { "hashes": [ "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", @@ -623,11 +632,11 @@ }, "black": { "hashes": [ - "sha256:09a9dcb7c46ed496a9850b76e4e825d6049ecd38b611f1224857a79bd985a8cf", - "sha256:68950ffd4d9169716bcb8719a56c07a2f4485354fec061cdd5910aa07369731c" + "sha256:1b30e59be925fafc1ee4565e5e08abef6b03fe455102883820fe5ee2e4734e0b", + "sha256:c2edb73a08e9e0e6f65a0e6af18b059b8b1cdd5bef997d7a0b181df93dc81539" ], "index": "pypi", - "version": "==19.3b0" + "version": "==19.10b0" }, "blinker": { "hashes": [ @@ -703,11 +712,11 @@ }, "detect-secrets": { "hashes": [ - "sha256:d6b22e93fa5ccdf11391f87d18c45cba64e11463fdb367e2314cdbeba6963ec0", - "sha256:e2189cd21619fc95a3ee7ec7adfb61adf66e2e4e78d518318a6025ca0f62b364" + "sha256:2f1dfe05d1f1bcd46205a46a4117c82a49b2c837771efc625e967c0a710b25a9", + "sha256:8e3cc5bb21fee76e71fee4d57c65b2928b4badb7ae45a5839e70f625057f1a21" ], "index": "pypi", - "version": "==0.12.7" + "version": "==0.13.0" }, "docopt": { "hashes": [ @@ -931,6 +940,12 @@ ], "version": "==0.5.1" }, + "pathspec": { + "hashes": [ + "sha256:e285ccc8b0785beadd4c18e5708b12bb8fcf529a1e61215b3feff1d1e559ea5c" + ], + "version": "==0.6.0" + }, "pathtools": { "hashes": [ "sha256:7c35c5421a39bb82e58018febd90e3b6e5db34c5443aaaf742b3f33d4655f1c0" @@ -1075,6 +1090,22 @@ "index": "pypi", "version": "==5.1.2" }, + "regex": { + "hashes": [ + "sha256:1e9f9bc44ca195baf0040b1938e6801d2f3409661c15fe57f8164c678cfc663f", + "sha256:587b62d48ca359d2d4f02d486f1f0aa9a20fbaf23a9d4198c4bed72ab2f6c849", + "sha256:835ccdcdc612821edf132c20aef3eaaecfb884c9454fdc480d5887562594ac61", + "sha256:93f6c9da57e704e128d90736430c5c59dd733327882b371b0cae8833106c2a21", + "sha256:a46f27d267665016acb3ec8c6046ec5eae8cf80befe85ba47f43c6f5ec636dcd", + "sha256:c5c8999b3a341b21ac2c6ec704cfcccbc50f1fedd61b6a8ee915ca7fd4b0a557", + "sha256:d4d1829cf97632673aa49f378b0a2c3925acd795148c5ace8ef854217abbee89", + "sha256:d96479257e8e4d1d7800adb26bf9c5ca5bab1648a1eddcac84d107b73dc68327", + "sha256:f20f4912daf443220436759858f96fefbfc6c6ba9e67835fd6e4e9b73582791a", + "sha256:f2b37b5b2c2a9d56d9e88efef200ec09c36c7f323f9d58d0b985a90923df386d", + "sha256:fe765b809a1f7ce642c2edeee351e7ebd84391640031ba4b60af8d91a9045890" + ], + "version": "==2019.8.19" + }, "requests": { "hashes": [ "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4", diff --git a/atst/domain/csp/cloud.py b/atst/domain/csp/cloud.py index 35e7b2bf..9e921757 100644 --- a/atst/domain/csp/cloud.py +++ b/atst/domain/csp/cloud.py @@ -431,9 +431,7 @@ class AzureCloudProvider(CloudProviderInterface): credentials = self._get_credential_obj(self._root_creds) sub_client = self.sdk.subscription.SubscriptionClient(credentials) - display_name = ( - f"{environment.application.name}_{environment.name}_{environment.id}" - ) # proposed format + display_name = f"{environment.application.name}_{environment.name}_{environment.id}" # proposed format billing_profile_id = "?" # something chained from environment? sku_id = AZURE_SKU_ID diff --git a/atst/models/permissions.py b/atst/models/permissions.py index f9e73046..f68bfc12 100644 --- a/atst/models/permissions.py +++ b/atst/models/permissions.py @@ -30,8 +30,8 @@ class Permissions(object): CREATE_TASK_ORDER = "create_task_order" # create a new TO VIEW_TASK_ORDER_DETAILS = "view_task_order_details" # individual TO page EDIT_TASK_ORDER_DETAILS = ( - "edit_task_order_details" - ) # edit TO that has not been finalized + "edit_task_order_details" # edit TO that has not been finalized + ) # reporting VIEW_PORTFOLIO_REPORTS = "view_portfolio_reports" diff --git a/tests/domain/cloud/test_azure_csp.py b/tests/domain/cloud/test_azure_csp.py index 67ca30f9..661cd2f2 100644 --- a/tests/domain/cloud/test_azure_csp.py +++ b/tests/domain/cloud/test_azure_csp.py @@ -4,11 +4,7 @@ from unittest.mock import Mock from uuid import uuid4 from atst.domain.csp.cloud import EnvironmentCreationException, AzureCloudProvider -from atst.jobs import ( - do_create_environment, - do_create_atat_admin_user, - do_create_environment_baseline, -) +from atst.jobs import do_create_environment, do_create_atat_admin_user from tests.mock_azure import mock_azure, AUTH_CREDENTIALS from tests.factories import EnvironmentFactory From d0746a3bf640a20973566b4b9dc6172d69f6f3ee Mon Sep 17 00:00:00 2001 From: tomdds Date: Wed, 30 Oct 2019 13:55:14 -0400 Subject: [PATCH 10/10] Cleanup imports and formatting in azure testing code --- tests/domain/cloud/test_azure_csp.py | 4 +--- tests/mock_azure.py | 1 - 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/domain/cloud/test_azure_csp.py b/tests/domain/cloud/test_azure_csp.py index 661cd2f2..19ad63c8 100644 --- a/tests/domain/cloud/test_azure_csp.py +++ b/tests/domain/cloud/test_azure_csp.py @@ -1,10 +1,8 @@ import pytest -from unittest.mock import Mock from uuid import uuid4 -from atst.domain.csp.cloud import EnvironmentCreationException, AzureCloudProvider -from atst.jobs import do_create_environment, do_create_atat_admin_user +from atst.domain.csp.cloud import AzureCloudProvider from tests.mock_azure import mock_azure, AUTH_CREDENTIALS from tests.factories import EnvironmentFactory diff --git a/tests/mock_azure.py b/tests/mock_azure.py index d838b506..34d0aa53 100644 --- a/tests/mock_azure.py +++ b/tests/mock_azure.py @@ -55,4 +55,3 @@ class MockAzureSDK(object): @pytest.fixture(scope="function") def mock_azure(): return AzureCloudProvider(AZURE_CONFIG, azure_sdk_provider=MockAzureSDK()) -