From 184b58d5d25b882e59cf7a168565c03230357fed Mon Sep 17 00:00:00 2001 From: richard-dds Date: Mon, 28 Oct 2019 13:46:12 -0400 Subject: [PATCH] Remove AWSCloudProvider --- atst/domain/csp/__init__.py | 13 +- atst/domain/csp/cloud.py | 276 ----------------------------- atst/domain/csp/file_uploads.py | 62 ------- tests/conftest.py | 3 +- tests/domain/cloud/test_aws_csp.py | 80 --------- tests/domain/test_environments.py | 37 ---- tests/mock_boto3.py | 171 ------------------ tests/test_jobs.py | 11 -- 8 files changed, 3 insertions(+), 650 deletions(-) delete mode 100644 tests/domain/cloud/test_aws_csp.py delete mode 100644 tests/mock_boto3.py diff --git a/atst/domain/csp/__init__.py b/atst/domain/csp/__init__.py index 8c460ea1..e8fc5236 100644 --- a/atst/domain/csp/__init__.py +++ b/atst/domain/csp/__init__.py @@ -1,5 +1,5 @@ from .cloud import MockCloudProvider -from .file_uploads import AwsUploader, AzureUploader, MockUploader +from .file_uploads import AzureUploader, MockUploader from .reports import MockReportingProvider @@ -19,17 +19,8 @@ class AzureCSP: self.reports = MockReportingProvider() -class AwsCSP: - def __init__(self, app): - self.cloud = MockCloudProvider(app.config) - self.files = AwsUploader(app.config) - self.reports = MockReportingProvider() - - def make_csp_provider(app, csp=None): - if csp == "aws": - app.csp = AwsCSP(app) - elif csp == "azure": + if csp == "azure": app.csp = AzureCSP(app) elif csp == "mock-test": app.csp = MockCSP(app, test_mode=True) diff --git a/atst/domain/csp/cloud.py b/atst/domain/csp/cloud.py index 4c1df6a5..9726a14f 100644 --- a/atst/domain/csp/cloud.py +++ b/atst/domain/csp/cloud.py @@ -1,15 +1,10 @@ from typing import Dict from uuid import uuid4 -import json -from jinja2 import Template -from atst.models.environment_role import CSPRole from atst.models.user import User from atst.models.environment import Environment from atst.models.environment_role import EnvironmentRole -from botocore.waiter import WaiterModel, create_waiter_with_client, WaiterError - class GeneralCSPException(Exception): pass @@ -417,274 +412,3 @@ class MockCloudProvider(CloudProviderInterface): self._delay(1, 5) if credentials != self._auth_credentials: raise self.AUTHENTICATION_EXCEPTION - - -class AWSCloudProvider(CloudProviderInterface): - # These are standins that will be replaced with "real" policies once we know what they are. - BASELINE_POLICIES = [ - { - "name": "BillingReadOnly", - "path": "/atat/billing-read-only/", - "document": { - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "VisualEditor0", - "Effect": "Allow", - "Action": [ - "aws-portal:ViewPaymentMethods", - "aws-portal:ViewAccount", - "aws-portal:ViewBilling", - "aws-portal:ViewUsage", - ], - "Resource": "*", - } - ], - }, - "description": "View billing information.", - } - ] - MAX_CREATE_ACCOUNT_ATTEMPTS = 10 - - # Placeholder permission boundary for root user - PERMISSION_BOUNDARY_ARN = "arn:aws:iam::aws:policy/AlexaForBusinessDeviceSetup" - - def __init__(self, config, boto3=None): - self.config = config - - self.access_key_id = config["AWS_ACCESS_KEY_ID"] - self.secret_key = config["AWS_SECRET_KEY"] - self.region_name = config["AWS_REGION_NAME"] - - # TODO: Discuss these values. - self.role_access_org_name = "OrganizationAccountAccessRole" - self.root_account_username = "atat" - self.root_account_policy_name = "OrganizationAccountAccessRole" - - if boto3: - self.boto3 = boto3 - else: - import boto3 - - self.boto3 = boto3 - - def root_creds(self): - return {"AccessKeyId": self.access_key_id, "SecretAccessKey": self.secret_key} - - def create_environment( - self, auth_credentials: Dict, user: User, environment: Environment - ): - - org_client = self._get_client("organizations") - - # Create an account. Requires organizations:CreateAccount permission - account_request = org_client.create_account( - Email=user.email, AccountName=uuid4().hex, IamUserAccessToBilling="ALLOW" - ) - - # Configuration for our CreateAccount Waiter. - # A waiter is a boto3 helper which can be configured to poll a given status - # endpoint until it succeeds or fails. boto3 has many built in waiters, but none - # for the organizations service so we're building our own here. - waiter_config = { - "version": 2, - "waiters": { - "AccountCreated": { - "operation": "DescribeCreateAccountStatus", - "delay": 20, - "maxAttempts": self.MAX_CREATE_ACCOUNT_ATTEMPTS, - "acceptors": [ - { - "matcher": "path", - "expected": "SUCCEEDED", - "argument": "CreateAccountStatus.State", - "state": "success", - }, - { - "matcher": "path", - "expected": "IN_PROGRESS", - "argument": "CreateAccountStatus.State", - "state": "retry", - }, - { - "matcher": "path", - "expected": "FAILED", - "argument": "CreateAccountStatus.State", - "state": "failure", - }, - ], - } - }, - } - waiter_model = WaiterModel(waiter_config) - account_waiter = create_waiter_with_client( - "AccountCreated", waiter_model, org_client - ) - - try: - # Poll until the CreateAccount request either succeeds or fails. - account_waiter.wait( - CreateAccountRequestId=account_request["CreateAccountStatus"]["Id"] - ) - except WaiterError: - # TODO: Possible failure reasons: - # 'ACCOUNT_LIMIT_EXCEEDED'|'EMAIL_ALREADY_EXISTS'|'INVALID_ADDRESS'|'INVALID_EMAIL'|'CONCURRENT_ACCOUNT_MODIFICATION'|'INTERNAL_FAILURE' - raise EnvironmentCreationException( - environment.id, "Failed to create account." - ) - - # We need to re-fetch this since the Waiter throws away the success response for some reason. - created_account_status = org_client.describe_create_account_status( - CreateAccountRequestId=account_request["CreateAccountStatus"]["Id"] - ) - account_id = created_account_status["CreateAccountStatus"]["AccountId"] - - return account_id - - def create_atat_admin_user( - self, auth_credentials: Dict, csp_environment_id: str - ) -> Dict: - """ - Create an IAM user within a given account. - """ - - # Create a policy which allows user to assume a role within the account. - iam_client = self._get_client("iam") - iam_client.put_user_policy( - UserName=self.root_account_username, - PolicyName=f"assume-role-{self.root_account_policy_name}-{csp_environment_id}", - PolicyDocument=self._inline_org_management_policy(csp_environment_id), - ) - - role_arn = ( - f"arn:aws:iam::{csp_environment_id}:role/{self.root_account_policy_name}" - ) - sts_client = self._get_client("sts", credentials=auth_credentials) - assumed_role_object = sts_client.assume_role( - RoleArn=role_arn, RoleSessionName="AssumeRoleSession1" - ) - - # From the response that contains the assumed role, get the temporary - # credentials that can be used to make subsequent API calls - credentials = assumed_role_object["Credentials"] - - # Use the temporary credentials that AssumeRole returns to make a new connection to IAM - iam_client = self._get_client("iam", credentials=credentials) - - # Create the user with a PermissionBoundary - try: - user = iam_client.create_user( - UserName=self.root_account_username, - PermissionsBoundary=self.PERMISSION_BOUNDARY_ARN, - Tags=[{"Key": "foo", "Value": "bar"}], - )["User"] - except iam_client.exceptions.EntityAlreadyExistsException as _exc: - # TODO: Find user, iterate through existing access keys and revoke them. - user = iam_client.get_user(UserName=self.root_account_username)["User"] - - access_key = iam_client.create_access_key(UserName=self.root_account_username)[ - "AccessKey" - ] - credentials = { - "AccessKeyId": access_key["AccessKeyId"], - "SecretAccessKey": access_key["SecretAccessKey"], - } - - # TODO: Create real policies in account. - - return { - "id": user["UserId"], - "username": user["UserName"], - "resource_id": user["Arn"], - "credentials": credentials, - } - - def create_environment_baseline( - self, auth_credentials: Dict, csp_environment_id: str - ) -> Dict: - """Provision the necessary baseline entities (such as roles) in the given environment - - Arguments: - auth_credentials -- Object containing CSP account credentials - csp_environment_id -- ID of the CSP Environment to provision roles against. - - Returns: - dict: Returns dict that associates the resource identities with their ATAT representations. - Raises: - AuthenticationException: Problem with the credentials - AuthorizationException: Credentials not authorized for current action(s) - ConnectionException: Issue with the CSP API connection - UnknownServerException: Unknown issue on the CSP side - BaselineProvisionException: Specific issue occurred with some aspect of baseline setup - """ - - client = self._get_client("iam", credentials=auth_credentials) - created_policies = [] - - for policy in self.BASELINE_POLICIES: - try: - response = client.create_policy( - PolicyName=policy["name"], - Path=policy["path"], - PolicyDocument=json.dumps(policy["document"]), - Description=policy["description"], - ) - created_policies.append({policy["name"]: response["Policy"]["Arn"]}) - except client.exceptions.EntityAlreadyExistsException: - # Policy already exists. We can determine its ARN based on the account id and policy path / name. - policy_arn = f"arn:aws:iam:{csp_environment_id}:policy{policy['path']}{policy['name']}" - created_policies.append({policy["name"]: policy_arn}) - - return {"policies": created_policies} - - def _get_client(self, service: str, credentials=None): - """ - A helper for creating a client of a given AWS service. - - If `credentials` aren't provided, the configured root credentials will be used. - - `credentials` format: - { - "AccessKeyId": "access-key-id", - "SecretAccessKey": "secret-access-key", - "SessionToken": "session-token" # optional - } - """ - - credentials = credentials or {} - credential_kwargs = { - "aws_access_key_id": credentials.get("AccessKeyId", self.access_key_id), - "aws_secret_access_key": credentials.get( - "SecretAccessKey", self.secret_key - ), - } - if "SessionToken" in credentials: - credential_kwargs["aws_session_token"] = credentials["SessionToken"] - - return self.boto3.client( - service, region_name=self.region_name, **credential_kwargs - ) - - def _inline_org_management_policy(self, account_id: str) -> Dict: - policy_template = Template( - """ - { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "sts:AssumeRole" - ], - "Resource": [ - "arn:aws:iam::{{ account_id }}:role/{{ role_name }}" - ] - } - ] - } - """ - ) - rendered = policy_template.render( - account_id=account_id, role_name=self.root_account_policy_name - ) - return json.loads(rendered) diff --git a/atst/domain/csp/file_uploads.py b/atst/domain/csp/file_uploads.py index a437cdbe..7916797d 100644 --- a/atst/domain/csp/file_uploads.py +++ b/atst/domain/csp/file_uploads.py @@ -75,65 +75,3 @@ class AzureUploader(Uploader): return bbs.make_blob_url( self.container_name, object_name, protocol="https", sas_token=sas_token ) - - -class AwsUploader(Uploader): - def __init__(self, config): - self.access_key_id = config["AWS_ACCESS_KEY_ID"] - self.secret_key = config["AWS_SECRET_KEY"] - self.region_name = config["AWS_REGION_NAME"] - self.bucket_name = config["AWS_BUCKET_NAME"] - self.timeout_secs = config["PERMANENT_SESSION_LIFETIME"] - - import boto3 - - self.boto3 = boto3 - - def get_token(self): - """ - Generates an AWS presigned post for pre-authorizing a file upload. - - Returns a tuple in the following format: (token_dict, object_name), where - - token_dict contains several fields that will be passed directly into the - form before being sent to S3 - - object_name is a string - """ - s3_client = self.boto3.client( - "s3", - aws_access_key_id=self.access_key_id, - aws_secret_access_key=self.secret_key, - config=self.boto3.session.Config( - signature_version="s3v4", region_name=self.region_name - ), - ) - object_name = self.object_name() - presigned_post = s3_client.generate_presigned_post( - self.bucket_name, - object_name, - ExpiresIn=self.timeout_secs, - Conditions=[ - ("eq", "$Content-Type", "application/pdf"), - ("starts-with", "$x-amz-meta-filename", ""), - ], - Fields={"Content-Type": "application/pdf", "x-amz-meta-filename": ""}, - ) - return (presigned_post, object_name) - - def generate_download_link(self, object_name, filename): - s3_client = self.boto3.client( - "s3", - aws_access_key_id=self.access_key_id, - aws_secret_access_key=self.secret_key, - config=self.boto3.session.Config( - signature_version="s3v4", region_name=self.region_name - ), - ) - return s3_client.generate_presigned_url( - "get_object", - Params={ - "Bucket": self.bucket_name, - "Key": object_name, - "ResponseContentDisposition": f"attachment; filename={filename}", - }, - ExpiresIn=self.timeout_secs, - ) diff --git a/tests/conftest.py b/tests/conftest.py index 0a75a48e..dce493ae 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,7 +4,6 @@ import alembic.config import alembic.command from logging.config import dictConfig from werkzeug.datastructures import FileStorage -from tempfile import TemporaryDirectory from collections import OrderedDict from atst.app import make_app, make_config @@ -41,7 +40,7 @@ def app(request): @pytest.fixture(autouse=True) def skip_audit_log(request): """ - Conditionally skip tests marked with 'audit_log' based on the + Conditionally skip tests marked with 'audit_log' based on the USE_AUDIT_LOG config value. """ config = make_config() diff --git a/tests/domain/cloud/test_aws_csp.py b/tests/domain/cloud/test_aws_csp.py deleted file mode 100644 index 2d17abbd..00000000 --- a/tests/domain/cloud/test_aws_csp.py +++ /dev/null @@ -1,80 +0,0 @@ -import pytest - -from atst.domain.csp.cloud import EnvironmentCreationException -from atst.jobs import do_create_environment, do_create_atat_admin_user - -# pylint: disable=unused-import -from tests.mock_boto3 import mock_aws, mock_boto3, AUTH_CREDENTIALS -from tests.factories import EnvironmentFactory - - -def test_create_environment_succeeds(mock_aws): - environment = EnvironmentFactory.create() - account_id = mock_aws.create_environment( - AUTH_CREDENTIALS, environment.creator, environment - ) - assert "account-id" == account_id - - -@pytest.mark.mock_boto3({"organizations.describe_create_account.failure": True}) -def test_create_environment_raises_x_when_account_creation_fails(mock_aws): - environment = EnvironmentFactory.create() - with pytest.raises(EnvironmentCreationException): - mock_aws.create_environment(AUTH_CREDENTIALS, environment.creator, environment) - - -def test_create_atat_admin_user_succeeds(mock_aws): - root_user_info = mock_aws.create_atat_admin_user( - AUTH_CREDENTIALS, "csp_environment_id" - ) - assert { - "id": "user-id", - "username": "user-name", - "resource_id": "user-arn", - "credentials": { - "AccessKeyId": "access-key-id", - "SecretAccessKey": "secret-access-key", - }, - } == root_user_info - - -@pytest.mark.mock_boto3({"iam.create_user.already_exists": True}) -def test_create_atat_admin_when_user_already_exists(mock_aws): - root_user_info = mock_aws.create_atat_admin_user( - AUTH_CREDENTIALS, "csp_environment_id" - ) - assert { - "id": "user-id", - "username": "user-name", - "resource_id": "user-arn", - "credentials": { - "AccessKeyId": "access-key-id", - "SecretAccessKey": "secret-access-key", - }, - } == root_user_info - - iam_client = mock_aws.boto3.client("iam") - iam_client.get_user.assert_any_call(UserName="atat") - - -def test_aws_provision_environment(mock_aws, session): - environment = EnvironmentFactory.create() - - do_create_environment(mock_aws, environment_id=environment.id) - do_create_atat_admin_user(mock_aws, environment_id=environment.id) - - session.refresh(environment) - - assert "account-id" == environment.cloud_id - assert { - "id": "user-id", - "username": "user-name", - "credentials": { - "AccessKeyId": "access-key-id", - "SecretAccessKey": "secret-access-key", - }, - "resource_id": "user-arn", - } == environment.root_user_info - assert { - "policies": [{"BillingReadOnly": "policy-arn"}] - } == environment.baseline_info diff --git a/tests/domain/test_environments.py b/tests/domain/test_environments.py index 2a2e3bb1..4264154c 100644 --- a/tests/domain/test_environments.py +++ b/tests/domain/test_environments.py @@ -175,40 +175,3 @@ class TestGetEnvironmentsPendingAtatUserCreation(EnvQueryTest): assert ( len(Environments.get_environments_pending_atat_user_creation(self.NOW)) == 0 ) - - -class TestGetEnvironmentsPendingBaselineCreation(EnvQueryTest): - def test_with_provisioned_environment(self): - self.create_portfolio_with_clins( - [(self.YESTERDAY, self.TOMORROW)], - { - "cloud_id": uuid4().hex, - "root_user_info": {"foo": "bar"}, - "baseline_info": {"foo": "bar"}, - }, - ) - assert ( - len(Environments.get_environments_pending_baseline_creation(self.NOW)) == 0 - ) - - def test_with_unprovisioned_environment(self): - self.create_portfolio_with_clins( - [(self.YESTERDAY, self.TOMORROW)], - { - "cloud_id": uuid4().hex, - "root_user_info": {"foo": "bar"}, - "baseline_info": None, - }, - ) - assert ( - len(Environments.get_environments_pending_baseline_creation(self.NOW)) == 1 - ) - - def test_with_unprovisioned_expired_clins_environment(self): - self.create_portfolio_with_clins( - [(self.YESTERDAY, self.YESTERDAY)], - {"cloud_id": uuid4().hex, "root_user_info": {"foo": "bar"}}, - ) - assert ( - len(Environments.get_environments_pending_baseline_creation(self.NOW)) == 0 - ) diff --git a/tests/mock_boto3.py b/tests/mock_boto3.py deleted file mode 100644 index dbab9fe6..00000000 --- a/tests/mock_boto3.py +++ /dev/null @@ -1,171 +0,0 @@ -import pytest -from unittest.mock import Mock - -from atst.domain.csp.cloud import AWSCloudProvider - - -AWS_CONFIG = { - "AWS_ACCESS_KEY_ID": "", - "AWS_SECRET_KEY": "", - "AWS_REGION_NAME": "us-fake-1", -} -AUTH_CREDENTIALS = { - "aws_access_key_id": AWS_CONFIG["AWS_ACCESS_KEY_ID"], - "aws_secret_access_key": AWS_CONFIG["AWS_SECRET_KEY"], -} - - -def mock_boto_organizations(_config=None, **kwargs): - describe_create_account_status = ( - "SUCCEEDED" - if _config.get("organizations.describe_create_account.failure", False) == False - else "FAILED" - ) - - import boto3 - - mock = Mock(wraps=boto3.client("organizations", **kwargs)) - - # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/organizations.html#Organizations.Client.create_account - mock.create_account = Mock( - return_value={ - "CreateAccountStatus": { - "Id": "create-account-status-id", - "AccountName": "account-name", - "AccountId": "account-id", - "State": "SUCCEEDED", - } - } - ) - - # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/organizations.html#Organizations.Client.describe_create_account_status - mock.describe_create_account_status = Mock( - return_value={ - "CreateAccountStatus": { - "Id": "create-account-status-id", - "AccountName": "account-name", - "AccountId": "account-id", - "State": describe_create_account_status, - } - } - ) - return mock - - -def mock_boto_iam(_config=None, **kwargs): - user_already_exists = _config.get("iam.create_user.already_exists", False) - policy_already_exists = _config.get("iam.create_policy.already_exists", False) - - def _raise_entity_already_exists(**kwargs): - raise real_iam_client.exceptions.EntityAlreadyExistsException( - {"Error": {}}, "operation-name" - ) - - import boto3 - - real_iam_client = boto3.client("iam", **kwargs) - mock = Mock(wraps=real_iam_client) - mock.exceptions.EntityAlreadyExistsException = ( - real_iam_client.exceptions.EntityAlreadyExistsException - ) - - mock.put_user_policy = Mock(return_value={"ResponseMetadata": {}}) - - if user_already_exists: - mock.create_user = Mock(side_effect=_raise_entity_already_exists) - else: - mock.create_user = Mock( - return_value={ - "User": { - "UserId": "user-id", - "Arn": "user-arn", - "UserName": "user-name", - } - } - ) - - mock.get_user = Mock( - return_value={ - "User": {"UserId": "user-id", "Arn": "user-arn", "UserName": "user-name"} - } - ) - - mock.create_access_key = Mock( - return_value={ - "AccessKey": { - "AccessKeyId": "access-key-id", - "SecretAccessKey": "secret-access-key", - } - } - ) - - if policy_already_exists: - mock.create_policy = Mock(side_effect=_raise_entity_already_exists) - else: - mock.create_policy = Mock(return_value={"Policy": {"Arn": "policy-arn"}}) - - return mock - - -def mock_boto_sts(_config=None, **kwargs): - import boto3 - - mock = Mock(wraps=boto3.client("sts", **kwargs)) - mock.assume_role = Mock( - return_value={ - "Credentials": { - "AccessKeyId": "access-key-id", - "SecretAccessKey": "secret-access-key", - "SessionToken": "session-token", - } - } - ) - - return mock - - -class MockBoto3: - CLIENTS = { - "organizations": mock_boto_organizations, - "iam": mock_boto_iam, - "sts": mock_boto_sts, - } - - def __init__(self, config=None): - self.config = config or {} - self.client_instances = {} - - def client(self, client_name, **kwargs): - """ - Return a new mock client for the given `client_name`, either by - retrieving it from the `client_instances` cache or by instantiating - it for the first time. - - Params should be the same ones you'd pass to `boto3.client`. - """ - - if client_name in self.client_instances: - return self.client_instances[client_name] - - try: - client_fn = self.CLIENTS[client_name] - client_instance = client_fn(**kwargs, _config=self.config) - self.client_instances[client_name] = client_instance - return client_instance - except KeyError: - raise ValueError(f"MockBoto3: {client_name} client is not yet implemented.") - - -@pytest.fixture(scope="function") -def mock_boto3(request): - marks = request.node.get_closest_marker("mock_boto3") - if marks: - mock_config = marks.args[0] if len(marks.args) else {} - else: - mock_config = {} - return MockBoto3(mock_config) - - -@pytest.fixture(scope="function") -def mock_aws(mock_boto3): - return AWSCloudProvider(AWS_CONFIG, boto3=mock_boto3) diff --git a/tests/test_jobs.py b/tests/test_jobs.py index d06eca33..bd702cb1 100644 --- a/tests/test_jobs.py +++ b/tests/test_jobs.py @@ -96,17 +96,6 @@ def test_create_atat_admin_user(csp, session): assert environment.root_user_info -def test_create_environment_baseline(csp, session, app): - environment = EnvironmentFactory.create( - root_user_info={"credentials": csp.root_creds()} - ) - do_create_environment_baseline(csp, environment.id) - session.refresh(environment) - - assert environment.baseline_info - assert len(app.mailer.messages) > 0 - - def test_dispatch_create_environment(session, monkeypatch): # Given that I have a portfolio with an active CLIN and two environments, # one of which is deleted