diff --git a/.secrets.baseline b/.secrets.baseline index bb61cf1a..9da7d65d 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": "^.secrets.baseline$|^.*pgsslrootcert.yml$", "lines": null }, - "generated_at": "2019-10-24T18:44:10Z", + "generated_at": "2019-10-28T18:00:40Z", "plugins_used": [ { "base64_limit": 4.5, @@ -26,7 +26,7 @@ "results": { "Pipfile.lock": [ { - "hashed_secret": "b84ecc29e7eb4488e736295d049db2f6503ee5f8", + "hashed_secret": "526c14b5cd73155d72784fec39907f9b7d5ddcdd", "is_secret": false, "is_verified": false, "line_number": 4, diff --git a/Pipfile b/Pipfile index baa18d8e..68c97392 100644 --- a/Pipfile +++ b/Pipfile @@ -24,7 +24,6 @@ werkzeug = "*" PyYAML = "*" azure-storage = "*" azure-storage-common = "*" -boto3 = "*" celery = "*" [dev-packages] diff --git a/Pipfile.lock b/Pipfile.lock index dd6f5171..3cf4e198 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "1861915e6f65ef28eb96a7d908c9ab1ca374909ee691f538d6004be0b0ccd0eb" + "sha256": "2f366c5a5f62ba5a451ca024759cc8cbf481a15e5198d73311fa9cf194d7f547" }, "pipfile-spec": 6, "requires": { @@ -76,21 +76,6 @@ ], "version": "==3.6.1.0" }, - "boto3": { - "hashes": [ - "sha256:2904bfb928116fea3a83247de6c3687eb9bf942d764e361f5574d5ac11be2ad3", - "sha256:77806f23320554b5d3175f4ef864e4ca6eb04d97a95ad6d2b3e3ef7736472c35" - ], - "index": "pypi", - "version": "==1.10.1" - }, - "botocore": { - "hashes": [ - "sha256:05d42876001fe6513742edcdb550f0aeabff2b123678a6b661cfeb6c26066b8e", - "sha256:acceec0b79df7e5f8b15eab3033f6a7c7f69d0bc27aa01d65aa5ba4d90742787" - ], - "version": "==1.13.1" - }, "celery": { "hashes": [ "sha256:821d11967f0f3f8fe24bd61ecfc7b6acbb5a926b719f1e8c4d5ff7bc09e18d81", @@ -129,10 +114,12 @@ "sha256:7cfcfda59ef1f95b9f729c56fe8a4041899f96b72685d36ef16a3440a0f85da8", "sha256:819f8d5197c2684524637f940445c06e003c4a541f9983fd30d6deaa2a5487d8", "sha256:825ecffd9574557590e3225560a8a9d751f6ffe4a49e3c40918c9969b93395fa", + "sha256:8a2bcae2258d00fcfc96a9bde4a6177bc4274fe033f79311c5dd3d3148c26518", "sha256:9009e917d8f5ef780c2626e29b6bc126f4cb2a4d43ca67aa2b40f2a5d6385e78", "sha256:9c77564a51d4d914ed5af096cd9843d90c45b784b511723bd46a8a9d09cf16fc", "sha256:a19089fa74ed19c4fe96502a291cfdb89223a9705b1d73b3005df4256976142e", "sha256:a40ed527bffa2b7ebe07acc5a3f782da072e262ca994b4f2085100b5a444bbb2", + "sha256:b8f09f21544b9899defb09afbdaeb200e6a87a2b8e604892940044cf94444644", "sha256:bb75ba21d5716abc41af16eac1145ab2e471deedde1f22c6f99bd9f995504df0", "sha256:e22a00c0c81ffcecaf07c2bfb3672fa372c50e2bd1024ffee0da191c1b27fc71", "sha256:e55b5a746fb77f10c83e8af081979351722f6ea48facea79d470b3731c7b2891", @@ -181,14 +168,6 @@ ], "version": "==2.8" }, - "docutils": { - "hashes": [ - "sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0", - "sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827", - "sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99" - ], - "version": "==0.15.2" - }, "flask": { "hashes": [ "sha256:13f9f196f330c7c2c5d7a5cf91af894110ca0215ac051b5844701f2bfd934d52", @@ -256,13 +235,6 @@ ], "version": "==2.10.3" }, - "jmespath": { - "hashes": [ - "sha256:3720a4b1bd659dd2eecad0666459b9788813e032b83e7ba58578e48254e0a0e6", - "sha256:bde2aef6f44302dfb30320115b17d030798de8c4110e28d5cf6cf91a7a31074c" - ], - "version": "==0.9.4" - }, "kombu": { "hashes": [ "sha256:31edb84947996fdda065b6560c128d5673bb913ff34aa19e7b84755217a24deb", @@ -391,7 +363,6 @@ "sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb", "sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e" ], - "markers": "python_version >= '2.7'", "version": "==2.8.0" }, "python-editor": { @@ -451,13 +422,6 @@ "index": "pypi", "version": "==2.22.0" }, - "s3transfer": { - "hashes": [ - "sha256:6efc926738a3cd576c2a79725fed9afde92378aa5c6a957e3af010cb019fac9d", - "sha256:b780f2411b824cb541dbcd2c713d0cb61c7d1bcadae204cdddda2b35cef493ba" - ], - "version": "==0.2.1" - }, "six": { "hashes": [ "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", @@ -485,7 +449,6 @@ "sha256:3de946ffbed6e6746608990594d08faac602528ac7015ac28d33cee6a45b7398", "sha256:9a107b99a5393caf59c7aa3c1249c16e6879447533d0887f4336dde834c7be86" ], - "markers": "python_version >= '3.4'", "version": "==1.25.6" }, "vine": { @@ -668,10 +631,10 @@ }, "decorator": { "hashes": [ - "sha256:86156361c50488b84a3f148056ea716ca587df2f0de1d34750d35c21312725de", - "sha256:f069f3a01830ca754ba5258fde2278454a0b5b79e0d7f5c13b3b97e57d4acff6" + "sha256:54c38050039232e1db4ad7375cfce6748d7b41c29e95a081c8a6d2c30364a2ce", + "sha256:5d19b92a3c8f7f101c8dd86afd86b0f061a8ce4540ab8cd401fa2542756bce6d" ], - "version": "==4.4.0" + "version": "==4.4.1" }, "detect-secrets": { "hashes": [ @@ -755,11 +718,11 @@ }, "ipython": { "hashes": [ - "sha256:c4ab005921641e40a68e405e286e7a1fcc464497e14d81b6914b4fd95e5dee9b", - "sha256:dd76831f065f17bddd7eaa5c781f5ea32de5ef217592cf019e34043b56895aa1" + "sha256:dfd303b270b7b5232b3d08bd30ec6fd685d8a58cabd54055e3d69d8f029f7280", + "sha256:ed7ebe1cba899c1c3ccad6f7f1c2d2369464cc77dba8eebc65e2043e19cda995" ], "index": "pypi", - "version": "==7.8.0" + "version": "==7.9.0" }, "ipython-genutils": { "hashes": [ @@ -798,26 +761,29 @@ }, "lazy-object-proxy": { "hashes": [ - "sha256:02b260c8deb80db09325b99edf62ae344ce9bc64d68b7a634410b8e9a568edbf", - "sha256:18f9c401083a4ba6e162355873f906315332ea7035803d0fd8166051e3d402e3", - "sha256:1f2c6209a8917c525c1e2b55a716135ca4658a3042b5122d4e3413a4030c26ce", - "sha256:2f06d97f0ca0f414f6b707c974aaf8829c2292c1c497642f63824119d770226f", - "sha256:616c94f8176808f4018b39f9638080ed86f96b55370b5a9463b2ee5c926f6c5f", - "sha256:63b91e30ef47ef68a30f0c3c278fbfe9822319c15f34b7538a829515b84ca2a0", - "sha256:77b454f03860b844f758c5d5c6e5f18d27de899a3db367f4af06bec2e6013a8e", - "sha256:83fe27ba321e4cfac466178606147d3c0aa18e8087507caec78ed5a966a64905", - "sha256:84742532d39f72df959d237912344d8a1764c2d03fe58beba96a87bfa11a76d8", - "sha256:874ebf3caaf55a020aeb08acead813baf5a305927a71ce88c9377970fe7ad3c2", - "sha256:9f5caf2c7436d44f3cec97c2fa7791f8a675170badbfa86e1992ca1b84c37009", - "sha256:a0c8758d01fcdfe7ae8e4b4017b13552efa7f1197dd7358dc9da0576f9d0328a", - "sha256:a4def978d9d28cda2d960c279318d46b327632686d82b4917516c36d4c274512", - "sha256:ad4f4be843dace866af5fc142509e9b9817ca0c59342fdb176ab6ad552c927f5", - "sha256:ae33dd198f772f714420c5ab698ff05ff900150486c648d29951e9c70694338e", - "sha256:b4a2b782b8a8c5522ad35c93e04d60e2ba7f7dcb9271ec8e8c3e08239be6c7b4", - "sha256:c462eb33f6abca3b34cdedbe84d761f31a60b814e173b98ede3c81bb48967c4f", - "sha256:fd135b8d35dfdcdb984828c84d695937e58cc5f49e1c854eb311c4d6aa03f4f1" + "sha256:0c4b206227a8097f05c4dbdd323c50edf81f15db3b8dc064d08c62d37e1a504d", + "sha256:194d092e6f246b906e8f70884e620e459fc54db3259e60cf69a4d66c3fda3449", + "sha256:1be7e4c9f96948003609aa6c974ae59830a6baecc5376c25c92d7d697e684c08", + "sha256:4677f594e474c91da97f489fea5b7daa17b5517190899cf213697e48d3902f5a", + "sha256:48dab84ebd4831077b150572aec802f303117c8cc5c871e182447281ebf3ac50", + "sha256:5541cada25cd173702dbd99f8e22434105456314462326f06dba3e180f203dfd", + "sha256:59f79fef100b09564bc2df42ea2d8d21a64fdcda64979c0fa3db7bdaabaf6239", + "sha256:8d859b89baf8ef7f8bc6b00aa20316483d67f0b1cbf422f5b4dc56701c8f2ffb", + "sha256:9254f4358b9b541e3441b007a0ea0764b9d056afdeafc1a5569eee1cc6c1b9ea", + "sha256:9651375199045a358eb6741df3e02a651e0330be090b3bc79f6d0de31a80ec3e", + "sha256:97bb5884f6f1cdce0099f86b907aa41c970c3c672ac8b9c8352789e103cf3156", + "sha256:9b15f3f4c0f35727d3a0fba4b770b3c4ebbb1fa907dbcc046a1d2799f3edd142", + "sha256:a2238e9d1bb71a56cd710611a1614d1194dc10a175c1e08d75e1a7bcc250d442", + "sha256:a6ae12d08c0bf9909ce12385803a543bfe99b95fe01e752536a60af2b7797c62", + "sha256:ca0a928a3ddbc5725be2dd1cf895ec0a254798915fb3a36af0964a0a4149e3db", + "sha256:cb2c7c57005a6804ab66f106ceb8482da55f5314b7fcb06551db1edae4ad1531", + "sha256:d74bb8693bf9cf75ac3b47a54d716bbb1a92648d5f781fc799347cfc95952383", + "sha256:d945239a5639b3ff35b70a88c5f2f491913eb94871780ebfabb2568bd58afc5a", + "sha256:eba7011090323c1dadf18b3b689845fd96a61ba0a1dfbd7f24b921398affc357", + "sha256:efa1909120ce98bbb3777e8b6f92237f5d5c8ea6758efea36a473e1d38f7d3e4", + "sha256:f3900e8a5de27447acbf900b4750b0ddfd7ec1ea7fbaf11dfa911141bc522af0" ], - "version": "==1.4.2" + "version": "==1.4.3" }, "markupsafe": { "hashes": [ @@ -1023,7 +989,6 @@ "sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb", "sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e" ], - "markers": "python_version >= '2.7'", "version": "==2.8.0" }, "pyyaml": { @@ -1138,18 +1103,17 @@ }, "typing-extensions": { "hashes": [ - "sha256:2ed632b30bb54fc3941c382decfd0ee4148f5c591651c9272473fea2c6397d95", - "sha256:b1edbbf0652660e32ae780ac9433f4231e7339c7f9a8057d0f042fcbcea49b87", - "sha256:d8179012ec2c620d3791ca6fe2bf7979d979acdbef1fca0bc56b37411db682ed" + "sha256:091ecc894d5e908ac75209f10d5b4f118fbdb2eb1ede6a63544054bb1edb41f2", + "sha256:910f4656f54de5993ad9304959ce9bb903f90aadc7c67a0bef07e678014e892d", + "sha256:cf8b63fedea4d89bab840ecbb93e75578af28f76f66c35889bd7065f5af88575" ], - "version": "==3.7.4" + "version": "==3.7.4.1" }, "urllib3": { "hashes": [ "sha256:3de946ffbed6e6746608990594d08faac602528ac7015ac28d33cee6a45b7398", "sha256:9a107b99a5393caf59c7aa3c1249c16e6879447533d0887f4336dde834c7be86" ], - "markers": "python_version >= '3.4'", "version": "==1.25.6" }, "watchdog": { diff --git a/atst/domain/csp/__init__.py b/atst/domain/csp/__init__.py index 8c460ea1..e8fc5236 100644 --- a/atst/domain/csp/__init__.py +++ b/atst/domain/csp/__init__.py @@ -1,5 +1,5 @@ from .cloud import MockCloudProvider -from .file_uploads import AwsUploader, AzureUploader, MockUploader +from .file_uploads import AzureUploader, MockUploader from .reports import MockReportingProvider @@ -19,17 +19,8 @@ class AzureCSP: self.reports = MockReportingProvider() -class AwsCSP: - def __init__(self, app): - self.cloud = MockCloudProvider(app.config) - self.files = AwsUploader(app.config) - self.reports = MockReportingProvider() - - def make_csp_provider(app, csp=None): - if csp == "aws": - app.csp = AwsCSP(app) - elif csp == "azure": + if csp == "azure": app.csp = AzureCSP(app) elif csp == "mock-test": app.csp = MockCSP(app, test_mode=True) diff --git a/atst/domain/csp/cloud.py b/atst/domain/csp/cloud.py index 83383166..9726a14f 100644 --- a/atst/domain/csp/cloud.py +++ b/atst/domain/csp/cloud.py @@ -1,15 +1,10 @@ from typing import Dict from uuid import uuid4 -import json -from jinja2 import Template -from atst.models.environment_role import CSPRole from atst.models.user import User from atst.models.environment import Environment from atst.models.environment_role import EnvironmentRole -from botocore.waiter import WaiterModel, create_waiter_with_client, WaiterError - class GeneralCSPException(Exception): pass @@ -197,26 +192,6 @@ class CloudProviderInterface: """ raise NotImplementedError() - def create_environment_baseline( - self, auth_credentials: Dict, csp_environment_id: str - ) -> Dict: - """Provision the necessary baseline entities (such as roles) in the given environment - - Arguments: - auth_credentials -- Object containing CSP account credentials - csp_environment_id -- ID of the CSP Environment to provision roles against. - - Returns: - dict: Returns dict that associates the resource identities with their ATAT representations. - Raises: - AuthenticationException: Problem with the credentials - AuthorizationException: Credentials not authorized for current action(s) - ConnectionException: Issue with the CSP API connection - UnknownServerException: Unknown issue on the CSP side - BaselineProvisionException: Specific issue occurred with some aspect of baseline setup - """ - raise NotImplementedError() - def create_or_update_user( self, auth_credentials: Dict, user_info: EnvironmentRole, csp_role_id: str ) -> str: @@ -330,9 +305,21 @@ class MockCloudProvider(CloudProviderInterface): environment.id, "Could not create environment." ), ) + + csp_environment_id = self._id() + + self._delay(1, 5) + self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) + self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) + self._maybe_raise( + self.ATAT_ADMIN_CREATE_FAILURE_PCT, + BaselineProvisionException( + csp_environment_id, "Could not create environment baseline." + ), + ) self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) - return self._id() + return csp_environment_id def create_atat_admin_user(self, auth_credentials, csp_environment_id): self._authorize(auth_credentials) @@ -351,27 +338,6 @@ class MockCloudProvider(CloudProviderInterface): return {"id": self._id(), "credentials": self._auth_credentials} - def create_environment_baseline(self, auth_credentials, csp_environment_id): - self._authorize(auth_credentials) - - self._delay(1, 5) - self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) - self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) - self._maybe_raise( - self.ATAT_ADMIN_CREATE_FAILURE_PCT, - BaselineProvisionException( - csp_environment_id, "Could not create environment baseline." - ), - ) - - self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) - return { - CSPRole.BASIC_ACCESS.value: self._id(), - CSPRole.NETWORK_ADMIN.value: self._id(), - CSPRole.BUSINESS_READ.value: self._id(), - CSPRole.TECHNICAL_READ.value: self._id(), - } - def create_or_update_user(self, auth_credentials, user_info, csp_role_id): self._authorize(auth_credentials) @@ -446,274 +412,3 @@ class MockCloudProvider(CloudProviderInterface): self._delay(1, 5) if credentials != self._auth_credentials: raise self.AUTHENTICATION_EXCEPTION - - -class AWSCloudProvider(CloudProviderInterface): - # These are standins that will be replaced with "real" policies once we know what they are. - BASELINE_POLICIES = [ - { - "name": "BillingReadOnly", - "path": "/atat/billing-read-only/", - "document": { - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "VisualEditor0", - "Effect": "Allow", - "Action": [ - "aws-portal:ViewPaymentMethods", - "aws-portal:ViewAccount", - "aws-portal:ViewBilling", - "aws-portal:ViewUsage", - ], - "Resource": "*", - } - ], - }, - "description": "View billing information.", - } - ] - MAX_CREATE_ACCOUNT_ATTEMPTS = 10 - - # Placeholder permission boundary for root user - PERMISSION_BOUNDARY_ARN = "arn:aws:iam::aws:policy/AlexaForBusinessDeviceSetup" - - def __init__(self, config, boto3=None): - self.config = config - - self.access_key_id = config["AWS_ACCESS_KEY_ID"] - self.secret_key = config["AWS_SECRET_KEY"] - self.region_name = config["AWS_REGION_NAME"] - - # TODO: Discuss these values. - self.role_access_org_name = "OrganizationAccountAccessRole" - self.root_account_username = "atat" - self.root_account_policy_name = "OrganizationAccountAccessRole" - - if boto3: - self.boto3 = boto3 - else: - import boto3 - - self.boto3 = boto3 - - def root_creds(self): - return {"AccessKeyId": self.access_key_id, "SecretAccessKey": self.secret_key} - - def create_environment( - self, auth_credentials: Dict, user: User, environment: Environment - ): - - org_client = self._get_client("organizations") - - # Create an account. Requires organizations:CreateAccount permission - account_request = org_client.create_account( - Email=user.email, AccountName=uuid4().hex, IamUserAccessToBilling="ALLOW" - ) - - # Configuration for our CreateAccount Waiter. - # A waiter is a boto3 helper which can be configured to poll a given status - # endpoint until it succeeds or fails. boto3 has many built in waiters, but none - # for the organizations service so we're building our own here. - waiter_config = { - "version": 2, - "waiters": { - "AccountCreated": { - "operation": "DescribeCreateAccountStatus", - "delay": 20, - "maxAttempts": self.MAX_CREATE_ACCOUNT_ATTEMPTS, - "acceptors": [ - { - "matcher": "path", - "expected": "SUCCEEDED", - "argument": "CreateAccountStatus.State", - "state": "success", - }, - { - "matcher": "path", - "expected": "IN_PROGRESS", - "argument": "CreateAccountStatus.State", - "state": "retry", - }, - { - "matcher": "path", - "expected": "FAILED", - "argument": "CreateAccountStatus.State", - "state": "failure", - }, - ], - } - }, - } - waiter_model = WaiterModel(waiter_config) - account_waiter = create_waiter_with_client( - "AccountCreated", waiter_model, org_client - ) - - try: - # Poll until the CreateAccount request either succeeds or fails. - account_waiter.wait( - CreateAccountRequestId=account_request["CreateAccountStatus"]["Id"] - ) - except WaiterError: - # TODO: Possible failure reasons: - # 'ACCOUNT_LIMIT_EXCEEDED'|'EMAIL_ALREADY_EXISTS'|'INVALID_ADDRESS'|'INVALID_EMAIL'|'CONCURRENT_ACCOUNT_MODIFICATION'|'INTERNAL_FAILURE' - raise EnvironmentCreationException( - environment.id, "Failed to create account." - ) - - # We need to re-fetch this since the Waiter throws away the success response for some reason. - created_account_status = org_client.describe_create_account_status( - CreateAccountRequestId=account_request["CreateAccountStatus"]["Id"] - ) - account_id = created_account_status["CreateAccountStatus"]["AccountId"] - - return account_id - - def create_atat_admin_user( - self, auth_credentials: Dict, csp_environment_id: str - ) -> Dict: - """ - Create an IAM user within a given account. - """ - - # Create a policy which allows user to assume a role within the account. - iam_client = self._get_client("iam") - iam_client.put_user_policy( - UserName=self.root_account_username, - PolicyName=f"assume-role-{self.root_account_policy_name}-{csp_environment_id}", - PolicyDocument=self._inline_org_management_policy(csp_environment_id), - ) - - role_arn = ( - f"arn:aws:iam::{csp_environment_id}:role/{self.root_account_policy_name}" - ) - sts_client = self._get_client("sts", credentials=auth_credentials) - assumed_role_object = sts_client.assume_role( - RoleArn=role_arn, RoleSessionName="AssumeRoleSession1" - ) - - # From the response that contains the assumed role, get the temporary - # credentials that can be used to make subsequent API calls - credentials = assumed_role_object["Credentials"] - - # Use the temporary credentials that AssumeRole returns to make a new connection to IAM - iam_client = self._get_client("iam", credentials=credentials) - - # Create the user with a PermissionBoundary - try: - user = iam_client.create_user( - UserName=self.root_account_username, - PermissionsBoundary=self.PERMISSION_BOUNDARY_ARN, - Tags=[{"Key": "foo", "Value": "bar"}], - )["User"] - except iam_client.exceptions.EntityAlreadyExistsException as _exc: - # TODO: Find user, iterate through existing access keys and revoke them. - user = iam_client.get_user(UserName=self.root_account_username)["User"] - - access_key = iam_client.create_access_key(UserName=self.root_account_username)[ - "AccessKey" - ] - credentials = { - "AccessKeyId": access_key["AccessKeyId"], - "SecretAccessKey": access_key["SecretAccessKey"], - } - - # TODO: Create real policies in account. - - return { - "id": user["UserId"], - "username": user["UserName"], - "resource_id": user["Arn"], - "credentials": credentials, - } - - def create_environment_baseline( - self, auth_credentials: Dict, csp_environment_id: str - ) -> Dict: - """Provision the necessary baseline entities (such as roles) in the given environment - - Arguments: - auth_credentials -- Object containing CSP account credentials - csp_environment_id -- ID of the CSP Environment to provision roles against. - - Returns: - dict: Returns dict that associates the resource identities with their ATAT representations. - Raises: - AuthenticationException: Problem with the credentials - AuthorizationException: Credentials not authorized for current action(s) - ConnectionException: Issue with the CSP API connection - UnknownServerException: Unknown issue on the CSP side - BaselineProvisionException: Specific issue occurred with some aspect of baseline setup - """ - - client = self._get_client("iam", credentials=auth_credentials) - created_policies = [] - - for policy in self.BASELINE_POLICIES: - try: - response = client.create_policy( - PolicyName=policy["name"], - Path=policy["path"], - PolicyDocument=json.dumps(policy["document"]), - Description=policy["description"], - ) - created_policies.append({policy["name"]: response["Policy"]["Arn"]}) - except client.exceptions.EntityAlreadyExistsException: - # Policy already exists. We can determine its ARN based on the account id and policy path / name. - policy_arn = f"arn:aws:iam:{csp_environment_id}:policy{policy['path']}{policy['name']}" - created_policies.append({policy["name"]: policy_arn}) - - return {"policies": created_policies} - - def _get_client(self, service: str, credentials=None): - """ - A helper for creating a client of a given AWS service. - - If `credentials` aren't provided, the configured root credentials will be used. - - `credentials` format: - { - "AccessKeyId": "access-key-id", - "SecretAccessKey": "secret-access-key", - "SessionToken": "session-token" # optional - } - """ - - credentials = credentials or {} - credential_kwargs = { - "aws_access_key_id": credentials.get("AccessKeyId", self.access_key_id), - "aws_secret_access_key": credentials.get( - "SecretAccessKey", self.secret_key - ), - } - if "SessionToken" in credentials: - credential_kwargs["aws_session_token"] = credentials["SessionToken"] - - return self.boto3.client( - service, region_name=self.region_name, **credential_kwargs - ) - - def _inline_org_management_policy(self, account_id: str) -> Dict: - policy_template = Template( - """ - { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "sts:AssumeRole" - ], - "Resource": [ - "arn:aws:iam::{{ account_id }}:role/{{ role_name }}" - ] - } - ] - } - """ - ) - rendered = policy_template.render( - account_id=account_id, role_name=self.root_account_policy_name - ) - return json.loads(rendered) diff --git a/atst/domain/csp/file_uploads.py b/atst/domain/csp/file_uploads.py index a437cdbe..7916797d 100644 --- a/atst/domain/csp/file_uploads.py +++ b/atst/domain/csp/file_uploads.py @@ -75,65 +75,3 @@ class AzureUploader(Uploader): return bbs.make_blob_url( self.container_name, object_name, protocol="https", sas_token=sas_token ) - - -class AwsUploader(Uploader): - def __init__(self, config): - self.access_key_id = config["AWS_ACCESS_KEY_ID"] - self.secret_key = config["AWS_SECRET_KEY"] - self.region_name = config["AWS_REGION_NAME"] - self.bucket_name = config["AWS_BUCKET_NAME"] - self.timeout_secs = config["PERMANENT_SESSION_LIFETIME"] - - import boto3 - - self.boto3 = boto3 - - def get_token(self): - """ - Generates an AWS presigned post for pre-authorizing a file upload. - - Returns a tuple in the following format: (token_dict, object_name), where - - token_dict contains several fields that will be passed directly into the - form before being sent to S3 - - object_name is a string - """ - s3_client = self.boto3.client( - "s3", - aws_access_key_id=self.access_key_id, - aws_secret_access_key=self.secret_key, - config=self.boto3.session.Config( - signature_version="s3v4", region_name=self.region_name - ), - ) - object_name = self.object_name() - presigned_post = s3_client.generate_presigned_post( - self.bucket_name, - object_name, - ExpiresIn=self.timeout_secs, - Conditions=[ - ("eq", "$Content-Type", "application/pdf"), - ("starts-with", "$x-amz-meta-filename", ""), - ], - Fields={"Content-Type": "application/pdf", "x-amz-meta-filename": ""}, - ) - return (presigned_post, object_name) - - def generate_download_link(self, object_name, filename): - s3_client = self.boto3.client( - "s3", - aws_access_key_id=self.access_key_id, - aws_secret_access_key=self.secret_key, - config=self.boto3.session.Config( - signature_version="s3v4", region_name=self.region_name - ), - ) - return s3_client.generate_presigned_url( - "get_object", - Params={ - "Bucket": self.bucket_name, - "Key": object_name, - "ResponseContentDisposition": f"attachment; filename={filename}", - }, - ExpiresIn=self.timeout_secs, - ) diff --git a/atst/domain/environments.py b/atst/domain/environments.py index ceebd783..1526f6dc 100644 --- a/atst/domain/environments.py +++ b/atst/domain/environments.py @@ -134,17 +134,3 @@ class Environments(object): .filter(Environment.root_user_info == None) ).all() return [id_ for id_, in results] - - @classmethod - def get_environments_pending_baseline_creation(cls, now) -> List[UUID]: - """ - Any environment with an active CLIN that has a `cloud_id` and `root_user_info` - but no `baseline_info`. - """ - results = ( - cls.base_provision_query(now) - .filter(Environment.cloud_id != None) - .filter(Environment.root_user_info != None) - .filter(Environment.baseline_info == None) - ).all() - return [id_ for id_, in results] diff --git a/atst/jobs.py b/atst/jobs.py index 379e6b14..4ee67f51 100644 --- a/atst/jobs.py +++ b/atst/jobs.py @@ -77,6 +77,13 @@ def do_create_environment(csp: CloudProviderInterface, environment_id=None): db.session.add(environment) db.session.commit() + body = render_email( + "emails/application/environment_ready.txt", {"environment": environment} + ) + app.mailer.send( + [environment.creator.email], translate("email.environment_ready"), body + ) + def do_create_atat_admin_user(csp: CloudProviderInterface, environment_id=None): environment = Environments.get(environment_id) @@ -96,27 +103,6 @@ def render_email(template_path, context): return app.jinja_env.get_template(template_path).render(context) -def do_create_environment_baseline(csp: CloudProviderInterface, environment_id=None): - environment = Environments.get(environment_id) - - with claim_for_update(environment) as environment: - # ASAP switch to use remote root user for provisioning - atat_remote_root_creds = environment.root_user_info["credentials"] - - baseline_info = csp.create_environment_baseline( - atat_remote_root_creds, environment.cloud_id - ) - environment.baseline_info = baseline_info - body = render_email( - "emails/application/environment_ready.txt", {"environment": environment} - ) - app.mailer.send( - [environment.creator.email], translate("email.environment_ready"), body - ) - db.session.add(environment) - db.session.commit() - - def do_provision_user(csp: CloudProviderInterface, environment_role_id=None): environment_role = EnvironmentRoles.get_by_id(environment_role_id) @@ -151,16 +137,6 @@ def create_atat_admin_user(self, environment_id=None): ) -@celery.task(bind=True, base=RecordEnvironmentFailure) -def create_environment_baseline(self, environment_id=None): - do_work( - do_create_environment_baseline, - self, - app.csp.cloud, - environment_id=environment_id, - ) - - @celery.task(bind=True) def provision_user(self, environment_role_id=None): do_work( @@ -184,14 +160,6 @@ def dispatch_create_atat_admin_user(self): create_atat_admin_user.delay(environment_id=environment_id) -@celery.task(bind=True) -def dispatch_create_environment_baseline(self): - for environment_id in Environments.get_environments_pending_baseline_creation( - pendulum.now() - ): - create_environment_baseline.delay(environment_id=environment_id) - - @celery.task(bind=True) def dispatch_provision_user(self): for ( diff --git a/tests/conftest.py b/tests/conftest.py index 0a75a48e..dce493ae 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,7 +4,6 @@ import alembic.config import alembic.command from logging.config import dictConfig from werkzeug.datastructures import FileStorage -from tempfile import TemporaryDirectory from collections import OrderedDict from atst.app import make_app, make_config @@ -41,7 +40,7 @@ def app(request): @pytest.fixture(autouse=True) def skip_audit_log(request): """ - Conditionally skip tests marked with 'audit_log' based on the + Conditionally skip tests marked with 'audit_log' based on the USE_AUDIT_LOG config value. """ config = make_config() diff --git a/tests/domain/cloud/test_aws_csp.py b/tests/domain/cloud/test_aws_csp.py deleted file mode 100644 index ef1c3184..00000000 --- a/tests/domain/cloud/test_aws_csp.py +++ /dev/null @@ -1,100 +0,0 @@ -import pytest - -from atst.domain.csp.cloud import EnvironmentCreationException -from atst.jobs import ( - do_create_environment, - do_create_atat_admin_user, - do_create_environment_baseline, -) - -# pylint: disable=unused-import -from tests.mock_boto3 import mock_aws, mock_boto3, AUTH_CREDENTIALS -from tests.factories import EnvironmentFactory - - -def test_create_environment_succeeds(mock_aws): - environment = EnvironmentFactory.create() - account_id = mock_aws.create_environment( - AUTH_CREDENTIALS, environment.creator, environment - ) - assert "account-id" == account_id - - -@pytest.mark.mock_boto3({"organizations.describe_create_account.failure": True}) -def test_create_environment_raises_x_when_account_creation_fails(mock_aws): - environment = EnvironmentFactory.create() - with pytest.raises(EnvironmentCreationException): - mock_aws.create_environment(AUTH_CREDENTIALS, environment.creator, environment) - - -def test_create_atat_admin_user_succeeds(mock_aws): - root_user_info = mock_aws.create_atat_admin_user( - AUTH_CREDENTIALS, "csp_environment_id" - ) - assert { - "id": "user-id", - "username": "user-name", - "resource_id": "user-arn", - "credentials": { - "AccessKeyId": "access-key-id", - "SecretAccessKey": "secret-access-key", - }, - } == root_user_info - - -@pytest.mark.mock_boto3({"iam.create_user.already_exists": True}) -def test_create_atat_admin_when_user_already_exists(mock_aws): - root_user_info = mock_aws.create_atat_admin_user( - AUTH_CREDENTIALS, "csp_environment_id" - ) - assert { - "id": "user-id", - "username": "user-name", - "resource_id": "user-arn", - "credentials": { - "AccessKeyId": "access-key-id", - "SecretAccessKey": "secret-access-key", - }, - } == root_user_info - - iam_client = mock_aws.boto3.client("iam") - iam_client.get_user.assert_any_call(UserName="atat") - - -def test_create_environment_baseline_succeeds(mock_aws): - baseline_info = mock_aws.create_environment_baseline( - AUTH_CREDENTIALS, "csp_environment_id" - ) - assert {"policies": [{"BillingReadOnly": "policy-arn"}]} == baseline_info - - -@pytest.mark.mock_boto3({"iam.create_policy.already_exists": True}) -def test_create_environment_baseline_when_policy_already_exists(mock_aws): - baseline_info = mock_aws.create_environment_baseline( - AUTH_CREDENTIALS, "csp_environment_id" - ) - assert "policies" in baseline_info - - -def test_aws_provision_environment(mock_aws, session): - environment = EnvironmentFactory.create() - - do_create_environment(mock_aws, environment_id=environment.id) - do_create_atat_admin_user(mock_aws, environment_id=environment.id) - do_create_environment_baseline(mock_aws, environment_id=environment.id) - - session.refresh(environment) - - assert "account-id" == environment.cloud_id - assert { - "id": "user-id", - "username": "user-name", - "credentials": { - "AccessKeyId": "access-key-id", - "SecretAccessKey": "secret-access-key", - }, - "resource_id": "user-arn", - } == environment.root_user_info - assert { - "policies": [{"BillingReadOnly": "policy-arn"}] - } == environment.baseline_info diff --git a/tests/domain/test_environments.py b/tests/domain/test_environments.py index 2a2e3bb1..4264154c 100644 --- a/tests/domain/test_environments.py +++ b/tests/domain/test_environments.py @@ -175,40 +175,3 @@ class TestGetEnvironmentsPendingAtatUserCreation(EnvQueryTest): assert ( len(Environments.get_environments_pending_atat_user_creation(self.NOW)) == 0 ) - - -class TestGetEnvironmentsPendingBaselineCreation(EnvQueryTest): - def test_with_provisioned_environment(self): - self.create_portfolio_with_clins( - [(self.YESTERDAY, self.TOMORROW)], - { - "cloud_id": uuid4().hex, - "root_user_info": {"foo": "bar"}, - "baseline_info": {"foo": "bar"}, - }, - ) - assert ( - len(Environments.get_environments_pending_baseline_creation(self.NOW)) == 0 - ) - - def test_with_unprovisioned_environment(self): - self.create_portfolio_with_clins( - [(self.YESTERDAY, self.TOMORROW)], - { - "cloud_id": uuid4().hex, - "root_user_info": {"foo": "bar"}, - "baseline_info": None, - }, - ) - assert ( - len(Environments.get_environments_pending_baseline_creation(self.NOW)) == 1 - ) - - def test_with_unprovisioned_expired_clins_environment(self): - self.create_portfolio_with_clins( - [(self.YESTERDAY, self.YESTERDAY)], - {"cloud_id": uuid4().hex, "root_user_info": {"foo": "bar"}}, - ) - assert ( - len(Environments.get_environments_pending_baseline_creation(self.NOW)) == 0 - ) diff --git a/tests/mock_boto3.py b/tests/mock_boto3.py deleted file mode 100644 index dbab9fe6..00000000 --- a/tests/mock_boto3.py +++ /dev/null @@ -1,171 +0,0 @@ -import pytest -from unittest.mock import Mock - -from atst.domain.csp.cloud import AWSCloudProvider - - -AWS_CONFIG = { - "AWS_ACCESS_KEY_ID": "", - "AWS_SECRET_KEY": "", - "AWS_REGION_NAME": "us-fake-1", -} -AUTH_CREDENTIALS = { - "aws_access_key_id": AWS_CONFIG["AWS_ACCESS_KEY_ID"], - "aws_secret_access_key": AWS_CONFIG["AWS_SECRET_KEY"], -} - - -def mock_boto_organizations(_config=None, **kwargs): - describe_create_account_status = ( - "SUCCEEDED" - if _config.get("organizations.describe_create_account.failure", False) == False - else "FAILED" - ) - - import boto3 - - mock = Mock(wraps=boto3.client("organizations", **kwargs)) - - # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/organizations.html#Organizations.Client.create_account - mock.create_account = Mock( - return_value={ - "CreateAccountStatus": { - "Id": "create-account-status-id", - "AccountName": "account-name", - "AccountId": "account-id", - "State": "SUCCEEDED", - } - } - ) - - # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/organizations.html#Organizations.Client.describe_create_account_status - mock.describe_create_account_status = Mock( - return_value={ - "CreateAccountStatus": { - "Id": "create-account-status-id", - "AccountName": "account-name", - "AccountId": "account-id", - "State": describe_create_account_status, - } - } - ) - return mock - - -def mock_boto_iam(_config=None, **kwargs): - user_already_exists = _config.get("iam.create_user.already_exists", False) - policy_already_exists = _config.get("iam.create_policy.already_exists", False) - - def _raise_entity_already_exists(**kwargs): - raise real_iam_client.exceptions.EntityAlreadyExistsException( - {"Error": {}}, "operation-name" - ) - - import boto3 - - real_iam_client = boto3.client("iam", **kwargs) - mock = Mock(wraps=real_iam_client) - mock.exceptions.EntityAlreadyExistsException = ( - real_iam_client.exceptions.EntityAlreadyExistsException - ) - - mock.put_user_policy = Mock(return_value={"ResponseMetadata": {}}) - - if user_already_exists: - mock.create_user = Mock(side_effect=_raise_entity_already_exists) - else: - mock.create_user = Mock( - return_value={ - "User": { - "UserId": "user-id", - "Arn": "user-arn", - "UserName": "user-name", - } - } - ) - - mock.get_user = Mock( - return_value={ - "User": {"UserId": "user-id", "Arn": "user-arn", "UserName": "user-name"} - } - ) - - mock.create_access_key = Mock( - return_value={ - "AccessKey": { - "AccessKeyId": "access-key-id", - "SecretAccessKey": "secret-access-key", - } - } - ) - - if policy_already_exists: - mock.create_policy = Mock(side_effect=_raise_entity_already_exists) - else: - mock.create_policy = Mock(return_value={"Policy": {"Arn": "policy-arn"}}) - - return mock - - -def mock_boto_sts(_config=None, **kwargs): - import boto3 - - mock = Mock(wraps=boto3.client("sts", **kwargs)) - mock.assume_role = Mock( - return_value={ - "Credentials": { - "AccessKeyId": "access-key-id", - "SecretAccessKey": "secret-access-key", - "SessionToken": "session-token", - } - } - ) - - return mock - - -class MockBoto3: - CLIENTS = { - "organizations": mock_boto_organizations, - "iam": mock_boto_iam, - "sts": mock_boto_sts, - } - - def __init__(self, config=None): - self.config = config or {} - self.client_instances = {} - - def client(self, client_name, **kwargs): - """ - Return a new mock client for the given `client_name`, either by - retrieving it from the `client_instances` cache or by instantiating - it for the first time. - - Params should be the same ones you'd pass to `boto3.client`. - """ - - if client_name in self.client_instances: - return self.client_instances[client_name] - - try: - client_fn = self.CLIENTS[client_name] - client_instance = client_fn(**kwargs, _config=self.config) - self.client_instances[client_name] = client_instance - return client_instance - except KeyError: - raise ValueError(f"MockBoto3: {client_name} client is not yet implemented.") - - -@pytest.fixture(scope="function") -def mock_boto3(request): - marks = request.node.get_closest_marker("mock_boto3") - if marks: - mock_config = marks.args[0] if len(marks.args) else {} - else: - mock_config = {} - return MockBoto3(mock_config) - - -@pytest.fixture(scope="function") -def mock_aws(mock_boto3): - return AWSCloudProvider(AWS_CONFIG, boto3=mock_boto3) diff --git a/tests/test_jobs.py b/tests/test_jobs.py index c21728af..bd702cb1 100644 --- a/tests/test_jobs.py +++ b/tests/test_jobs.py @@ -10,10 +10,8 @@ from atst.jobs import ( RecordEnvironmentRoleFailure, do_create_environment, do_create_atat_admin_user, - do_create_environment_baseline, dispatch_create_environment, dispatch_create_atat_admin_user, - dispatch_create_environment_baseline, create_environment, dispatch_provision_user, do_provision_user, @@ -98,17 +96,6 @@ def test_create_atat_admin_user(csp, session): assert environment.root_user_info -def test_create_environment_baseline(csp, session, app): - environment = EnvironmentFactory.create( - root_user_info={"credentials": csp.root_creds()} - ) - do_create_environment_baseline(csp, environment.id) - session.refresh(environment) - - assert environment.baseline_info - assert len(app.mailer.messages) > 0 - - def test_dispatch_create_environment(session, monkeypatch): # Given that I have a portfolio with an active CLIN and two environments, # one of which is deleted @@ -166,39 +153,6 @@ def test_dispatch_create_atat_admin_user(session, monkeypatch): mock.delay.assert_called_once_with(environment_id=environment.id) -def test_dispatch_create_environment_baseline(session, monkeypatch): - portfolio = PortfolioFactory.create( - applications=[ - { - "environments": [ - { - "cloud_id": uuid4().hex, - "root_user_info": {}, - "baseline_info": None, - } - ] - } - ], - task_orders=[ - { - "create_clins": [ - { - "start_date": pendulum.now().subtract(days=1), - "end_date": pendulum.now().add(days=1), - } - ] - } - ], - ) - mock = Mock() - monkeypatch.setattr("atst.jobs.create_environment_baseline", mock) - environment = portfolio.applications[0].environments[0] - - dispatch_create_environment_baseline.run() - - mock.delay.assert_called_once_with(environment_id=environment.id) - - def test_create_environment_no_dupes(session, celery_app, celery_worker): portfolio = PortfolioFactory.create( applications=[