pk/atst
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove AWSCloudProvider

Browse Source
This commit is contained in:
richard-dds 2019-10-28 13:46:12 -04:00
parent 6ea17bb4f8
commit 184b58d5d2
8 changed files with 3 additions and 650 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
atst/domain/csp/__init__.py
View File

@ -1,5 +1,5 @@
from .cloud import MockCloudProvider
from .file_uploads import AwsUploader, AzureUploader, MockUploader
from .file_uploads import AzureUploader, MockUploader
from .reports import MockReportingProvider
@ -19,17 +19,8 @@ class AzureCSP:
self.reports = MockReportingProvider()
class AwsCSP:
def __init__(self, app):
self.cloud = MockCloudProvider(app.config)
self.files = AwsUploader(app.config)
self.reports = MockReportingProvider()
def make_csp_provider(app, csp=None):
if csp == "aws":
app.csp = AwsCSP(app)
elif csp == "azure":
if csp == "azure":
app.csp = AzureCSP(app)
elif csp == "mock-test":
app.csp = MockCSP(app, test_mode=True)

276
atst/domain/csp/cloud.py
View File

@ -1,15 +1,10 @@
from typing import Dict
from uuid import uuid4
import json
from jinja2 import Template
from atst.models.environment_role import CSPRole
from atst.models.user import User
from atst.models.environment import Environment
from atst.models.environment_role import EnvironmentRole
from botocore.waiter import WaiterModel, create_waiter_with_client, WaiterError
class GeneralCSPException(Exception):
pass
@ -417,274 +412,3 @@ class MockCloudProvider(CloudProviderInterface):
self._delay(1, 5)
if credentials != self._auth_credentials:
raise self.AUTHENTICATION_EXCEPTION
class AWSCloudProvider(CloudProviderInterface):
# These are standins that will be replaced with "real" policies once we know what they are.
BASELINE_POLICIES = [
{
"name": "BillingReadOnly",
"path": "/atat/billing-read-only/",
"document": {
"Version": "2012-10-17",
"Statement": [
{
"Sid": "VisualEditor0",
"Effect": "Allow",
"Action": [
"aws-portal:ViewPaymentMethods",
"aws-portal:ViewAccount",
"aws-portal:ViewBilling",
"aws-portal:ViewUsage",
],
"Resource": "*",
}
],
},
"description": "View billing information.",
}
]
MAX_CREATE_ACCOUNT_ATTEMPTS = 10
# Placeholder permission boundary for root user
PERMISSION_BOUNDARY_ARN = "arn:aws:iam::aws:policy/AlexaForBusinessDeviceSetup"
def __init__(self, config, boto3=None):
self.config = config
self.access_key_id = config["AWS_ACCESS_KEY_ID"]
self.secret_key = config["AWS_SECRET_KEY"]
self.region_name = config["AWS_REGION_NAME"]
# TODO: Discuss these values.
self.role_access_org_name = "OrganizationAccountAccessRole"
self.root_account_username = "atat"
self.root_account_policy_name = "OrganizationAccountAccessRole"
if boto3:
self.boto3 = boto3
else:
import boto3
self.boto3 = boto3
def root_creds(self):
return {"AccessKeyId": self.access_key_id, "SecretAccessKey": self.secret_key}
def create_environment(
self, auth_credentials: Dict, user: User, environment: Environment
):
org_client = self._get_client("organizations")
# Create an account. Requires organizations:CreateAccount permission
account_request = org_client.create_account(
Email=user.email, AccountName=uuid4().hex, IamUserAccessToBilling="ALLOW"
)
# Configuration for our CreateAccount Waiter.
# A waiter is a boto3 helper which can be configured to poll a given status
# endpoint until it succeeds or fails. boto3 has many built in waiters, but none
# for the organizations service so we're building our own here.
waiter_config = {
"version": 2,
"waiters": {
"AccountCreated": {
"operation": "DescribeCreateAccountStatus",
"delay": 20,
"maxAttempts": self.MAX_CREATE_ACCOUNT_ATTEMPTS,
"acceptors": [
{
"matcher": "path",
"expected": "SUCCEEDED",
"argument": "CreateAccountStatus.State",
"state": "success",
},
{
"matcher": "path",
"expected": "IN_PROGRESS",
"argument": "CreateAccountStatus.State",
"state": "retry",
},
{
"matcher": "path",
"expected": "FAILED",
"argument": "CreateAccountStatus.State",
"state": "failure",
},
],
}
},
}
waiter_model = WaiterModel(waiter_config)
account_waiter = create_waiter_with_client(
"AccountCreated", waiter_model, org_client
)
try:
# Poll until the CreateAccount request either succeeds or fails.
account_waiter.wait(
CreateAccountRequestId=account_request["CreateAccountStatus"]["Id"]
)
except WaiterError:
# TODO: Possible failure reasons:
# 'ACCOUNT_LIMIT_EXCEEDED'|'EMAIL_ALREADY_EXISTS'|'INVALID_ADDRESS'|'INVALID_EMAIL'|'CONCURRENT_ACCOUNT_MODIFICATION'|'INTERNAL_FAILURE'
raise EnvironmentCreationException(
environment.id, "Failed to create account."
)
# We need to re-fetch this since the Waiter throws away the success response for some reason.
created_account_status = org_client.describe_create_account_status(
CreateAccountRequestId=account_request["CreateAccountStatus"]["Id"]
)
account_id = created_account_status["CreateAccountStatus"]["AccountId"]
return account_id
def create_atat_admin_user(
self, auth_credentials: Dict, csp_environment_id: str
) -> Dict:
"""
Create an IAM user within a given account.
"""
# Create a policy which allows user to assume a role within the account.
iam_client = self._get_client("iam")
iam_client.put_user_policy(
UserName=self.root_account_username,
PolicyName=f"assume-role-{self.root_account_policy_name}-{csp_environment_id}",
PolicyDocument=self._inline_org_management_policy(csp_environment_id),
)
role_arn = (
f"arn:aws:iam::{csp_environment_id}:role/{self.root_account_policy_name}"
)
sts_client = self._get_client("sts", credentials=auth_credentials)
assumed_role_object = sts_client.assume_role(
RoleArn=role_arn, RoleSessionName="AssumeRoleSession1"
)
# From the response that contains the assumed role, get the temporary
# credentials that can be used to make subsequent API calls
credentials = assumed_role_object["Credentials"]
# Use the temporary credentials that AssumeRole returns to make a new connection to IAM
iam_client = self._get_client("iam", credentials=credentials)
# Create the user with a PermissionBoundary
try:
user = iam_client.create_user(
UserName=self.root_account_username,
PermissionsBoundary=self.PERMISSION_BOUNDARY_ARN,
Tags=[{"Key": "foo", "Value": "bar"}],
)["User"]
except iam_client.exceptions.EntityAlreadyExistsException as _exc:
# TODO: Find user, iterate through existing access keys and revoke them.
user = iam_client.get_user(UserName=self.root_account_username)["User"]
access_key = iam_client.create_access_key(UserName=self.root_account_username)[
"AccessKey"
]
credentials = {
"AccessKeyId": access_key["AccessKeyId"],
"SecretAccessKey": access_key["SecretAccessKey"],
}
# TODO: Create real policies in account.
return {
"id": user["UserId"],
"username": user["UserName"],
"resource_id": user["Arn"],
"credentials": credentials,
}
def create_environment_baseline(
self, auth_credentials: Dict, csp_environment_id: str
) -> Dict:
"""Provision the necessary baseline entities (such as roles) in the given environment
Arguments:
auth_credentials -- Object containing CSP account credentials
csp_environment_id -- ID of the CSP Environment to provision roles against.
Returns:
dict: Returns dict that associates the resource identities with their ATAT representations.
Raises:
AuthenticationException: Problem with the credentials
AuthorizationException: Credentials not authorized for current action(s)
ConnectionException: Issue with the CSP API connection
UnknownServerException: Unknown issue on the CSP side
BaselineProvisionException: Specific issue occurred with some aspect of baseline setup
"""
client = self._get_client("iam", credentials=auth_credentials)
created_policies = []
for policy in self.BASELINE_POLICIES:
try:
response = client.create_policy(
PolicyName=policy["name"],
Path=policy["path"],
PolicyDocument=json.dumps(policy["document"]),
Description=policy["description"],
)
created_policies.append({policy["name"]: response["Policy"]["Arn"]})
except client.exceptions.EntityAlreadyExistsException:
# Policy already exists. We can determine its ARN based on the account id and policy path / name.
policy_arn = f"arn:aws:iam:{csp_environment_id}:policy{policy['path']}{policy['name']}"
created_policies.append({policy["name"]: policy_arn})
return {"policies": created_policies}
def _get_client(self, service: str, credentials=None):
"""
A helper for creating a client of a given AWS service.
If `credentials` aren't provided, the configured root credentials will be used.
`credentials` format:
{
"AccessKeyId": "access-key-id",
"SecretAccessKey": "secret-access-key",
"SessionToken": "session-token" # optional
}
"""
credentials = credentials or {}
credential_kwargs = {
"aws_access_key_id": credentials.get("AccessKeyId", self.access_key_id),
"aws_secret_access_key": credentials.get(
"SecretAccessKey", self.secret_key
),
}
if "SessionToken" in credentials:
credential_kwargs["aws_session_token"] = credentials["SessionToken"]
return self.boto3.client(
service, region_name=self.region_name, **credential_kwargs
)
def _inline_org_management_policy(self, account_id: str) -> Dict:
policy_template = Template(
"""
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"sts:AssumeRole"
],
"Resource": [
"arn:aws:iam::{{ account_id }}:role/{{ role_name }}"
]
}
]
}
"""
)
rendered = policy_template.render(
account_id=account_id, role_name=self.root_account_policy_name
)
return json.loads(rendered)

62
atst/domain/csp/file_uploads.py
View File

@ -75,65 +75,3 @@ class AzureUploader(Uploader):
return bbs.make_blob_url(
self.container_name, object_name, protocol="https", sas_token=sas_token
)
class AwsUploader(Uploader):
def __init__(self, config):
self.access_key_id = config["AWS_ACCESS_KEY_ID"]
self.secret_key = config["AWS_SECRET_KEY"]
self.region_name = config["AWS_REGION_NAME"]
self.bucket_name = config["AWS_BUCKET_NAME"]
self.timeout_secs = config["PERMANENT_SESSION_LIFETIME"]
import boto3
self.boto3 = boto3
def get_token(self):
"""
Generates an AWS presigned post for pre-authorizing a file upload.
Returns a tuple in the following format: (token_dict, object_name), where
- token_dict contains several fields that will be passed directly into the
form before being sent to S3
- object_name is a string
"""
s3_client = self.boto3.client(
"s3",
aws_access_key_id=self.access_key_id,
aws_secret_access_key=self.secret_key,
config=self.boto3.session.Config(
signature_version="s3v4", region_name=self.region_name
),
)
object_name = self.object_name()
presigned_post = s3_client.generate_presigned_post(
self.bucket_name,
object_name,
ExpiresIn=self.timeout_secs,
Conditions=[
("eq", "$Content-Type", "application/pdf"),
("starts-with", "$x-amz-meta-filename", ""),
],
Fields={"Content-Type": "application/pdf", "x-amz-meta-filename": ""},
)
return (presigned_post, object_name)
def generate_download_link(self, object_name, filename):
s3_client = self.boto3.client(
"s3",
aws_access_key_id=self.access_key_id,
aws_secret_access_key=self.secret_key,
config=self.boto3.session.Config(
signature_version="s3v4", region_name=self.region_name
),
)
return s3_client.generate_presigned_url(
"get_object",
Params={
"Bucket": self.bucket_name,
"Key": object_name,
"ResponseContentDisposition": f"attachment; filename={filename}",
},
ExpiresIn=self.timeout_secs,
)

3
tests/conftest.py
View File

@ -4,7 +4,6 @@ import alembic.config
import alembic.command
from logging.config import dictConfig
from werkzeug.datastructures import FileStorage
from tempfile import TemporaryDirectory
from collections import OrderedDict
from atst.app import make_app, make_config
@ -41,7 +40,7 @@ def app(request):
@pytest.fixture(autouse=True)
def skip_audit_log(request):
"""
Conditionally skip tests marked with 'audit_log' based on the
Conditionally skip tests marked with 'audit_log' based on the
USE_AUDIT_LOG config value.
"""
config = make_config()

80
tests/domain/cloud/test_aws_csp.py
View File

@ -1,80 +0,0 @@
import pytest
from atst.domain.csp.cloud import EnvironmentCreationException
from atst.jobs import do_create_environment, do_create_atat_admin_user
# pylint: disable=unused-import
from tests.mock_boto3 import mock_aws, mock_boto3, AUTH_CREDENTIALS
from tests.factories import EnvironmentFactory
def test_create_environment_succeeds(mock_aws):
environment = EnvironmentFactory.create()
account_id = mock_aws.create_environment(
AUTH_CREDENTIALS, environment.creator, environment
)
assert "account-id" == account_id
@pytest.mark.mock_boto3({"organizations.describe_create_account.failure": True})
def test_create_environment_raises_x_when_account_creation_fails(mock_aws):
environment = EnvironmentFactory.create()
with pytest.raises(EnvironmentCreationException):
mock_aws.create_environment(AUTH_CREDENTIALS, environment.creator, environment)
def test_create_atat_admin_user_succeeds(mock_aws):
root_user_info = mock_aws.create_atat_admin_user(
AUTH_CREDENTIALS, "csp_environment_id"
)
assert {
"id": "user-id",
"username": "user-name",
"resource_id": "user-arn",
"credentials": {
"AccessKeyId": "access-key-id",
"SecretAccessKey": "secret-access-key",
},
} == root_user_info
@pytest.mark.mock_boto3({"iam.create_user.already_exists": True})
def test_create_atat_admin_when_user_already_exists(mock_aws):
root_user_info = mock_aws.create_atat_admin_user(
AUTH_CREDENTIALS, "csp_environment_id"
)
assert {
"id": "user-id",
"username": "user-name",
"resource_id": "user-arn",
"credentials": {
"AccessKeyId": "access-key-id",
"SecretAccessKey": "secret-access-key",
},
} == root_user_info
iam_client = mock_aws.boto3.client("iam")
iam_client.get_user.assert_any_call(UserName="atat")
def test_aws_provision_environment(mock_aws, session):
environment = EnvironmentFactory.create()
do_create_environment(mock_aws, environment_id=environment.id)
do_create_atat_admin_user(mock_aws, environment_id=environment.id)
session.refresh(environment)
assert "account-id" == environment.cloud_id
assert {
"id": "user-id",
"username": "user-name",
"credentials": {
"AccessKeyId": "access-key-id",
"SecretAccessKey": "secret-access-key",
},
"resource_id": "user-arn",
} == environment.root_user_info
assert {
"policies": [{"BillingReadOnly": "policy-arn"}]
} == environment.baseline_info

37
tests/domain/test_environments.py
View File

@ -175,40 +175,3 @@ class TestGetEnvironmentsPendingAtatUserCreation(EnvQueryTest):
assert (
len(Environments.get_environments_pending_atat_user_creation(self.NOW)) == 0
)
class TestGetEnvironmentsPendingBaselineCreation(EnvQueryTest):
def test_with_provisioned_environment(self):
self.create_portfolio_with_clins(
[(self.YESTERDAY, self.TOMORROW)],
{
"cloud_id": uuid4().hex,
"root_user_info": {"foo": "bar"},
"baseline_info": {"foo": "bar"},
},
)
assert (
len(Environments.get_environments_pending_baseline_creation(self.NOW)) == 0
)
def test_with_unprovisioned_environment(self):
self.create_portfolio_with_clins(
[(self.YESTERDAY, self.TOMORROW)],
{
"cloud_id": uuid4().hex,
"root_user_info": {"foo": "bar"},
"baseline_info": None,
},
)
assert (
len(Environments.get_environments_pending_baseline_creation(self.NOW)) == 1
)
def test_with_unprovisioned_expired_clins_environment(self):
self.create_portfolio_with_clins(
[(self.YESTERDAY, self.YESTERDAY)],
{"cloud_id": uuid4().hex, "root_user_info": {"foo": "bar"}},
)
assert (
len(Environments.get_environments_pending_baseline_creation(self.NOW)) == 0
)

171
tests/mock_boto3.py
View File

@ -1,171 +0,0 @@
import pytest
from unittest.mock import Mock
from atst.domain.csp.cloud import AWSCloudProvider
AWS_CONFIG = {
"AWS_ACCESS_KEY_ID": "",
"AWS_SECRET_KEY": "",
"AWS_REGION_NAME": "us-fake-1",
}
AUTH_CREDENTIALS = {
"aws_access_key_id": AWS_CONFIG["AWS_ACCESS_KEY_ID"],
"aws_secret_access_key": AWS_CONFIG["AWS_SECRET_KEY"],
}
def mock_boto_organizations(_config=None, **kwargs):
describe_create_account_status = (
"SUCCEEDED"
if _config.get("organizations.describe_create_account.failure", False) == False
else "FAILED"
)
import boto3
mock = Mock(wraps=boto3.client("organizations", **kwargs))
# https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/organizations.html#Organizations.Client.create_account
mock.create_account = Mock(
return_value={
"CreateAccountStatus": {
"Id": "create-account-status-id",
"AccountName": "account-name",
"AccountId": "account-id",
"State": "SUCCEEDED",
}
}
)
# https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/organizations.html#Organizations.Client.describe_create_account_status
mock.describe_create_account_status = Mock(
return_value={
"CreateAccountStatus": {
"Id": "create-account-status-id",
"AccountName": "account-name",
"AccountId": "account-id",
"State": describe_create_account_status,
}
}
)
return mock
def mock_boto_iam(_config=None, **kwargs):
user_already_exists = _config.get("iam.create_user.already_exists", False)
policy_already_exists = _config.get("iam.create_policy.already_exists", False)
def _raise_entity_already_exists(**kwargs):
raise real_iam_client.exceptions.EntityAlreadyExistsException(
{"Error": {}}, "operation-name"
)
import boto3
real_iam_client = boto3.client("iam", **kwargs)
mock = Mock(wraps=real_iam_client)
mock.exceptions.EntityAlreadyExistsException = (
real_iam_client.exceptions.EntityAlreadyExistsException
)
mock.put_user_policy = Mock(return_value={"ResponseMetadata": {}})
if user_already_exists:
mock.create_user = Mock(side_effect=_raise_entity_already_exists)
else:
mock.create_user = Mock(
return_value={
"User": {
"UserId": "user-id",
"Arn": "user-arn",
"UserName": "user-name",
}
}
)
mock.get_user = Mock(
return_value={
"User": {"UserId": "user-id", "Arn": "user-arn", "UserName": "user-name"}
}
)
mock.create_access_key = Mock(
return_value={
"AccessKey": {
"AccessKeyId": "access-key-id",
"SecretAccessKey": "secret-access-key",
}
}
)
if policy_already_exists:
mock.create_policy = Mock(side_effect=_raise_entity_already_exists)
else:
mock.create_policy = Mock(return_value={"Policy": {"Arn": "policy-arn"}})
return mock
def mock_boto_sts(_config=None, **kwargs):
import boto3
mock = Mock(wraps=boto3.client("sts", **kwargs))
mock.assume_role = Mock(
return_value={
"Credentials": {
"AccessKeyId": "access-key-id",
"SecretAccessKey": "secret-access-key",
"SessionToken": "session-token",
}
}
)
return mock
class MockBoto3:
CLIENTS = {
"organizations": mock_boto_organizations,
"iam": mock_boto_iam,
"sts": mock_boto_sts,
}
def __init__(self, config=None):
self.config = config or {}
self.client_instances = {}
def client(self, client_name, **kwargs):
"""
Return a new mock client for the given `client_name`, either by
retrieving it from the `client_instances` cache or by instantiating
it for the first time.
Params should be the same ones you'd pass to `boto3.client`.
"""
if client_name in self.client_instances:
return self.client_instances[client_name]
try:
client_fn = self.CLIENTS[client_name]
client_instance = client_fn(**kwargs, _config=self.config)
self.client_instances[client_name] = client_instance
return client_instance
except KeyError:
raise ValueError(f"MockBoto3: {client_name} client is not yet implemented.")
@pytest.fixture(scope="function")
def mock_boto3(request):
marks = request.node.get_closest_marker("mock_boto3")
if marks:
mock_config = marks.args[0] if len(marks.args) else {}
else:
mock_config = {}
return MockBoto3(mock_config)
@pytest.fixture(scope="function")
def mock_aws(mock_boto3):
return AWSCloudProvider(AWS_CONFIG, boto3=mock_boto3)

11
tests/test_jobs.py
View File

@ -96,17 +96,6 @@ def test_create_atat_admin_user(csp, session):
assert environment.root_user_info
def test_create_environment_baseline(csp, session, app):
environment = EnvironmentFactory.create(
root_user_info={"credentials": csp.root_creds()}
)
do_create_environment_baseline(csp, environment.id)
session.refresh(environment)
assert environment.baseline_info
assert len(app.mailer.messages) > 0
def test_dispatch_create_environment(session, monkeypatch):
# Given that I have a portfolio with an active CLIN and two environments,
# one of which is deleted