Merge pull request #1144 from dod-ccpo/azure-integration

Azure Integration
This commit is contained in:
tomdds 2019-10-31 13:07:11 -04:00 committed by GitHub
commit 52de1471e0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 402 additions and 9 deletions

View File

@ -25,6 +25,10 @@ PyYAML = "*"
azure-storage = "*"
azure-storage-common = "*"
celery = "*"
azure-mgmt-subscription = "*"
azure-graphrbac = "*"
msrestazure = "*"
azure-mgmt-authorization = "*"
[dev-packages]
bandit = "*"
@ -45,6 +49,7 @@ pytest-mock = "*"
detect-secrets = "*"
beautifulsoup4 = "*"
mypy = "*"
rope = "*"
[requires]
python_version = "3.7.3"

119
Pipfile.lock generated
View File

@ -1,7 +1,7 @@
{
"_meta": {
"hash": {
"sha256": "2f366c5a5f62ba5a451ca024759cc8cbf481a15e5198d73311fa9cf194d7f547"
"sha256": "6d2ab855267daac877ae7464de9dba5b62b7d89288992f87d8fc6ff0c0d2520f"
},
"pipfile-spec": 6,
"requires": {
@ -16,6 +16,13 @@
]
},
"default": {
"adal": {
"hashes": [
"sha256:5a7f1e037c6290c6d7609cab33a9e5e988c2fbec5c51d1c4c649ee3faff37eaf",
"sha256:fd17e5661f60634ddf96a569b95d34ccb8a98de60593d729c28bdcfe360eaad1"
],
"version": "==1.2.2"
},
"alembic": {
"hashes": [
"sha256:9f907d7e8b286a1cfb22db9084f9ce4fde7ad7956bb496dc7c952e10ac90e36a"
@ -45,6 +52,30 @@
],
"version": "==1.1.23"
},
"azure-graphrbac": {
"hashes": [
"sha256:53e98ae2ca7c19b349e9e9bb1b6a824aeae8dcfcbe17190d20fe69c0f185b2e2",
"sha256:7b4e0f05676acc912f2b33c71c328d9fb2e4dc8e70ebadc9d3de8ab08bf0b175"
],
"index": "pypi",
"version": "==0.61.1"
},
"azure-mgmt-authorization": {
"hashes": [
"sha256:31e875a34ac2c5d6fefe77b4a8079a8b2bdbe9edb957e47e8b44222fb212d6a7",
"sha256:9d64295cf4210ec14e98fb024a6b4d79d68ef50cdb3804f0b53f8567e52d847f"
],
"index": "pypi",
"version": "==0.60.0"
},
"azure-mgmt-subscription": {
"hashes": [
"sha256:504b4c42ba859070c3c50637ec07ca36aca600e613fcccaa398db22822fe21f1",
"sha256:850f86de5078f61f3a9bd81cb2b938ea25ce9a266c9fe22c7fca94a2b5968c16"
],
"index": "pypi",
"version": "==0.5.0"
},
"azure-nspkg": {
"hashes": [
"sha256:1d0bbb2157cf57b1bef6c8c8e5b41133957364456c43b0a43599890023cca0a8",
@ -221,6 +252,13 @@
],
"version": "==0.23"
},
"isodate": {
"hashes": [
"sha256:2e364a3d5759479cdb2d37cce6b9376ea504db2ff90252a2e5b7cc89cc9ff2d8",
"sha256:aa4d33c06640f5352aca96e4b81afd8ab3b47337cc12089822d6f322ac772c81"
],
"version": "==0.6.0"
},
"itsdangerous": {
"hashes": [
"sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19",
@ -296,6 +334,28 @@
],
"version": "==7.2.0"
},
"msrest": {
"hashes": [
"sha256:56b8b5b4556fb2a92cac640df267d560889bdc9e2921187772d4691d97bc4e8d",
"sha256:f5153bfe60ee757725816aedaa0772cbfe0bddb52cd2d6db4cb8b4c3c6c6f928"
],
"version": "==0.6.10"
},
"msrestazure": {
"hashes": [
"sha256:63db9f646fffc9244b332090e679d1e5f283ac491ee0cc321f5116f9450deb4a",
"sha256:fecb6a72a3eb5483e4deff38210d26ae42d3f6d488a7a275bd2423a1a014b22c"
],
"index": "pypi",
"version": "==0.6.2"
},
"oauthlib": {
"hashes": [
"sha256:bee41cc35fcca6e988463cacc3bcb8a96224f470ca547e697b604cc697b2f889",
"sha256:df884cd6cbe20e32633f1db1072e9356f53638e4361bef4e8b03c9127c9328ea"
],
"version": "==3.1.0"
},
"pendulum": {
"hashes": [
"sha256:1cde6e3c6310fb882c98f373795f807cb2bd6af01f34d2857e6e283b5ee91e09",
@ -350,6 +410,13 @@
],
"version": "==2.19"
},
"pyjwt": {
"hashes": [
"sha256:5c6eca3c2940464d106b99ba83b00c6add741c9becaec087fb7ccdefea71350e",
"sha256:8d59a976fb773f3e6a39c85636357c4f0e242707394cadadd9814f5cbaa20e96"
],
"version": "==1.7.1"
},
"pyopenssl": {
"hashes": [
"sha256:aeca66338f6de19d1aa46ed634c3b9ae519a64b458f8468aec688e7e3c20f200",
@ -422,6 +489,13 @@
"index": "pypi",
"version": "==2.22.0"
},
"requests-oauthlib": {
"hashes": [
"sha256:bd6533330e8748e94bf0b214775fed487d309b8b8fe823dc45641ebcd9a32f57",
"sha256:d3ed0c8f2e3bbc6b344fa63d6f933745ab394469da38db16bdddb461c7e25140"
],
"version": "==1.2.0"
},
"six": {
"hashes": [
"sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c",
@ -558,11 +632,11 @@
},
"black": {
"hashes": [
"sha256:09a9dcb7c46ed496a9850b76e4e825d6049ecd38b611f1224857a79bd985a8cf",
"sha256:68950ffd4d9169716bcb8719a56c07a2f4485354fec061cdd5910aa07369731c"
"sha256:1b30e59be925fafc1ee4565e5e08abef6b03fe455102883820fe5ee2e4734e0b",
"sha256:c2edb73a08e9e0e6f65a0e6af18b059b8b1cdd5bef997d7a0b181df93dc81539"
],
"index": "pypi",
"version": "==19.3b0"
"version": "==19.10b0"
},
"blinker": {
"hashes": [
@ -638,11 +712,11 @@
},
"detect-secrets": {
"hashes": [
"sha256:d6b22e93fa5ccdf11391f87d18c45cba64e11463fdb367e2314cdbeba6963ec0",
"sha256:e2189cd21619fc95a3ee7ec7adfb61adf66e2e4e78d518318a6025ca0f62b364"
"sha256:2f1dfe05d1f1bcd46205a46a4117c82a49b2c837771efc625e967c0a710b25a9",
"sha256:8e3cc5bb21fee76e71fee4d57c65b2928b4badb7ae45a5839e70f625057f1a21"
],
"index": "pypi",
"version": "==0.12.7"
"version": "==0.13.0"
},
"docopt": {
"hashes": [
@ -866,6 +940,12 @@
],
"version": "==0.5.1"
},
"pathspec": {
"hashes": [
"sha256:e285ccc8b0785beadd4c18e5708b12bb8fcf529a1e61215b3feff1d1e559ea5c"
],
"version": "==0.6.0"
},
"pathtools": {
"hashes": [
"sha256:7c35c5421a39bb82e58018febd90e3b6e5db34c5443aaaf742b3f33d4655f1c0"
@ -1010,6 +1090,22 @@
"index": "pypi",
"version": "==5.1.2"
},
"regex": {
"hashes": [
"sha256:1e9f9bc44ca195baf0040b1938e6801d2f3409661c15fe57f8164c678cfc663f",
"sha256:587b62d48ca359d2d4f02d486f1f0aa9a20fbaf23a9d4198c4bed72ab2f6c849",
"sha256:835ccdcdc612821edf132c20aef3eaaecfb884c9454fdc480d5887562594ac61",
"sha256:93f6c9da57e704e128d90736430c5c59dd733327882b371b0cae8833106c2a21",
"sha256:a46f27d267665016acb3ec8c6046ec5eae8cf80befe85ba47f43c6f5ec636dcd",
"sha256:c5c8999b3a341b21ac2c6ec704cfcccbc50f1fedd61b6a8ee915ca7fd4b0a557",
"sha256:d4d1829cf97632673aa49f378b0a2c3925acd795148c5ace8ef854217abbee89",
"sha256:d96479257e8e4d1d7800adb26bf9c5ca5bab1648a1eddcac84d107b73dc68327",
"sha256:f20f4912daf443220436759858f96fefbfc6c6ba9e67835fd6e4e9b73582791a",
"sha256:f2b37b5b2c2a9d56d9e88efef200ec09c36c7f323f9d58d0b985a90923df386d",
"sha256:fe765b809a1f7ce642c2edeee351e7ebd84391640031ba4b60af8d91a9045890"
],
"version": "==2019.8.19"
},
"requests": {
"hashes": [
"sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4",
@ -1018,6 +1114,15 @@
"index": "pypi",
"version": "==2.22.0"
},
"rope": {
"hashes": [
"sha256:6b728fdc3e98a83446c27a91fc5d56808a004f8beab7a31ab1d7224cecc7d969",
"sha256:c5c5a6a87f7b1a2095fb311135e2a3d1f194f5ecb96900fdd0a9100881f48aaf",
"sha256:f0dcf719b63200d492b85535ebe5ea9b29e0d0b8aebeb87fe03fc1a65924fdaf"
],
"index": "pypi",
"version": "==0.14.0"
},
"selenium": {
"hashes": [
"sha256:49d9b3bc156b795b94b06ae4e65dcd9c454b8e5f29624f2a117f02e3020ce411",

View File

@ -1,4 +1,5 @@
from typing import Dict
import re
from uuid import uuid4
from atst.models.user import User
@ -383,3 +384,190 @@ class MockCloudProvider(CloudProviderInterface):
self._delay(1, 5)
if self._with_authorization and credentials != self._auth_credentials:
raise self.AUTHENTICATION_EXCEPTION
AZURE_ENVIRONMENT = "AZURE_PUBLIC_CLOUD" # TBD
AZURE_SKU_ID = "?" # probably a static sku specific to ATAT/JEDI
SUBSCRIPTION_ID_REGEX = re.compile(
"subscriptions\/([0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})",
re.I,
)
# This needs to be a fully pathed role definition identifier, not just a UUID
REMOTE_ROOT_ROLE_DEF_ID = "/providers/Microsoft.Authorization/roleDefinitions/00000000-0000-4000-8000-000000000000"
class AzureSDKProvider(object):
def __init__(self):
from azure.mgmt import subscription, authorization
import azure.graphrbac as graphrbac
import azure.common.credentials as credentials
from msrestazure.azure_cloud import AZURE_PUBLIC_CLOUD
self.subscription = subscription
self.authorization = authorization
self.graphrbac = graphrbac
self.credentials = credentials
# may change to a JEDI cloud
self.cloud = AZURE_PUBLIC_CLOUD
class AzureCloudProvider(CloudProviderInterface):
def __init__(self, config, azure_sdk_provider=None):
self.config = config
self.client_id = config["AZURE_CLIENT_ID"]
self.secret_key = config["AZURE_SECRET_KEY"]
self.tenant_id = config["AZURE_TENANT_ID"]
if azure_sdk_provider is None:
self.sdk = AzureSDKProvider()
else:
self.sdk = azure_sdk_provider
def create_environment(
self, auth_credentials: Dict, user: User, environment: Environment
):
credentials = self._get_credential_obj(self._root_creds)
sub_client = self.sdk.subscription.SubscriptionClient(credentials)
display_name = f"{environment.application.name}_{environment.name}_{environment.id}" # proposed format
billing_profile_id = "?" # something chained from environment?
sku_id = AZURE_SKU_ID
# we want to set AT-AT as an owner here
# we could potentially associate subscriptions with "management groups" per DOD component
body = self.sdk.subscription.models.ModernSubscriptionCreationParameters(
display_name,
billing_profile_id,
sku_id,
# owner=<AdPrincipal: for AT-AT user>
)
# These 2 seem like something that might be worthwhile to allow tiebacks to
# TOs filed for the environment
billing_account_name = "?"
invoice_section_name = "?"
# We may also want to create billing sections in the enrollment account
sub_creation_operation = sub_client.subscription_factory.create_subscription(
billing_account_name, invoice_section_name, body
)
# the resulting object from this process is a link to the new subscription
# not a subscription model, so we'll have to unpack the ID
new_sub = sub_creation_operation.result()
subscription_id = self._extract_subscription_id(new_sub.subscription_link)
if subscription_id:
return subscription_id
else:
# troublesome error, subscription should exist at this point
# but we just don't have a valid ID
pass
def create_atat_admin_user(
self, auth_credentials: Dict, csp_environment_id: str
) -> Dict:
root_creds = self._root_creds
credentials = self._get_credential_obj(root_creds)
sub_client = self.sdk.subscription.SubscriptionClient(credentials)
subscription: self.sdk.subscription.models.Subscription = sub_client.subscriptions.get(
csp_environment_id
)
managment_principal = self._get_management_service_principal()
auth_client = self.sdk.authorization.AuthorizationManagementClient(
credentials,
# TODO: Determine which subscription this needs to point at
# Once we're in a multi-sub environment
subscription.id,
)
# Create role assignment for
role_assignment_id = str(uuid4())
role_assignment_create_params = auth_client.role_assignments.models.RoleAssignmentCreateParameters(
role_definition_id=REMOTE_ROOT_ROLE_DEF_ID,
principal_id=managment_principal.id,
)
auth_client.role_assignments.create(
scope=f"/subscriptions/{subscription.id}/",
role_assignment_name=role_assignment_id,
parameters=role_assignment_create_params,
)
return {
"csp_user_id": managment_principal.object_id,
"credentials": managment_principal.password_credentials,
"role_name": role_assignment_id,
}
def _get_management_service_principal(self):
# we really should be using graph.microsoft.com, but i'm getting
# "expired token" errors for that
# graph_resource = "https://graph.microsoft.com"
graph_resource = "https://graph.windows.net"
graph_creds = self._get_credential_obj(
self._root_creds, resource=graph_resource
)
# I needed to set permissions for the graph.windows.net API before I
# could get this to work.
# how do we scope the graph client to the new subscription rather than
# the cloud0 subscription? tenant id seems to be separate from subscription id
graph_client = self.sdk.graphrbac.GraphRbacManagementClient(
graph_creds, self._root_creds.get("tenant_id")
)
# do we need to create a new application to manage each subscripition
# or should we manage access to each subscription from a single service
# principal with multiple role assignments?
app_display_name = "?" # name should reflect the subscription it exists
app_create_param = self.sdk.graphrbac.models.ApplicationCreateParameters(
display_name=app_display_name
)
# we need the appropriate perms here:
# https://docs.microsoft.com/en-us/graph/api/application-post-applications?view=graph-rest-beta&tabs=http
# https://docs.microsoft.com/en-us/graph/permissions-reference#microsoft-graph-permission-names
# set app perms in app registration portal
# https://docs.microsoft.com/en-us/graph/auth-v2-service#2-configure-permissions-for-microsoft-graph
app: self.sdk.graphrbac.models.Application = graph_client.applications.create(
app_create_param
)
# create a new service principle for the new application, which should be scoped
# to the new subscription
app_id = app.app_id
sp_create_params = self.sdk.graphrbac.models.ServicePrincipalCreateParameters(
app_id=app_id, account_enabled=True
)
service_principal = graph_client.service_principals.create(sp_create_params)
return service_principal
def _extract_subscription_id(self, subscription_url):
sub_id_match = SUBSCRIPTION_ID_REGEX.match(subscription_url)
if sub_id_match:
return sub_id_match.group(1)
def _get_credential_obj(self, creds, resource=None):
return self.sdk.credentials.ServicePrincipalCredentials(
client_id=creds.get("client_id"),
secret=creds.get("secret_key"),
tenant=creds.get("tenant_id"),
resource=resource,
cloud_environment=self.sdk.cloud,
)
@property
def _root_creds(self):
return {
"client_id": self.client_id,
"secret_key": self.secret_key,
"tenant_id": self.tenant_id,
}

View File

@ -30,8 +30,8 @@ class Permissions(object):
CREATE_TASK_ORDER = "create_task_order" # create a new TO
VIEW_TASK_ORDER_DETAILS = "view_task_order_details" # individual TO page
EDIT_TASK_ORDER_DETAILS = (
"edit_task_order_details"
) # edit TO that has not been finalized
"edit_task_order_details" # edit TO that has not been finalized
)
# reporting
VIEW_PORTFOLIO_REPORTS = "view_portfolio_reports"

View File

@ -0,0 +1,38 @@
import pytest
from uuid import uuid4
from atst.domain.csp.cloud import AzureCloudProvider
from tests.mock_azure import mock_azure, AUTH_CREDENTIALS
from tests.factories import EnvironmentFactory
def test_create_environment_succeeds(mock_azure: AzureCloudProvider):
environment = EnvironmentFactory.create()
subscription_id = str(uuid4())
mock_azure.sdk.subscription.SubscriptionClient.return_value.subscription_factory.create_subscription.return_value.result.return_value.subscription_link = (
f"subscriptions/{subscription_id}"
)
result = mock_azure.create_environment(
AUTH_CREDENTIALS, environment.creator, environment
)
assert result == subscription_id
def test_create_atat_admin_user_succeeds(mock_azure: AzureCloudProvider):
environment_id = str(uuid4())
csp_user_id = str(uuid4)
mock_azure.sdk.graphrbac.GraphRbacManagementClient.return_value.service_principals.create.return_value.object_id = (
csp_user_id
)
result = mock_azure.create_atat_admin_user(AUTH_CREDENTIALS, environment_id)
assert result.get("csp_user_id") == csp_user_id

57
tests/mock_azure.py Normal file
View File

@ -0,0 +1,57 @@
import pytest
from unittest.mock import Mock
from atst.domain.csp.cloud import AzureCloudProvider
AZURE_CONFIG = {
"AZURE_CLIENT_ID": "MOCK",
"AZURE_SECRET_KEY": "MOCK",
"AZURE_TENANT_ID": "MOCK",
}
AUTH_CREDENTIALS = {
"CLIENT_ID": AZURE_CONFIG["AZURE_CLIENT_ID"],
"SECRET_KEY": AZURE_CONFIG["AZURE_SECRET_KEY"],
"TENANT_ID": AZURE_CONFIG["AZURE_TENANT_ID"],
}
def mock_subscription():
from azure.mgmt import subscription
return Mock(spec=subscription)
def mock_authorization():
from azure.mgmt import authorization
return Mock(spec=authorization)
def mock_graphrbac():
import azure.graphrbac as graphrbac
return Mock(spec=graphrbac)
def mock_credentials():
import azure.common.credentials as credentials
return Mock(spec=credentials)
class MockAzureSDK(object):
def __init__(self):
from msrestazure.azure_cloud import AZURE_PUBLIC_CLOUD
self.subscription = mock_subscription()
self.authorization = mock_authorization()
self.graphrbac = mock_graphrbac()
self.credentials = mock_credentials()
# may change to a JEDI cloud
self.cloud = AZURE_PUBLIC_CLOUD
@pytest.fixture(scope="function")
def mock_azure():
return AzureCloudProvider(AZURE_CONFIG, azure_sdk_provider=MockAzureSDK())