pk/atst
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

resolve conflict with staging

Browse Source
This commit is contained in:
Philip Kalinsky 2020-02-06 10:19:47 -05:00
commit 7559875d64
21 changed files with 333 additions and 392 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
alembic/versions/0039308c6351_remove_root_user_info_from_environment.py Normal file
View File

@ -0,0 +1,28 @@
"""Remove root_user_info from Environment
Revision ID: 0039308c6351
Revises: 17da2a475429
Create Date: 2020-02-04 14:37:06.814645
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision = '0039308c6351' # pragma: allowlist secret
down_revision = '17da2a475429' # pragma: allowlist secret
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column('environments', 'root_user_info')
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('environments', sa.Column('root_user_info', postgresql.JSONB(astext_type=sa.Text()), autoincrement=False, nullable=True))
# ### end Alembic commands ###

109
atst/domain/csp/cloud/azure_cloud_provider.py
View File

@ -6,12 +6,12 @@ from uuid import uuid4
from atst.utils import sha256_hex from atst.utils import sha256_hex
from .cloud_provider_interface import CloudProviderInterface from .cloud_provider_interface import CloudProviderInterface
from .exceptions import AuthenticationException, UserProvisioningException from .exceptions import (
AuthenticationException,
SecretException,
UserProvisioningException,
)
from .models import ( from .models import (
SubscriptionCreationCSPPayload,
SubscriptionCreationCSPResult,
SubscriptionVerificationCSPPayload,
SuscriptionVerificationCSPResult,
AdminRoleDefinitionCSPPayload, AdminRoleDefinitionCSPPayload,
AdminRoleDefinitionCSPResult, AdminRoleDefinitionCSPResult,
ApplicationCSPPayload, ApplicationCSPPayload,
@ -29,15 +29,20 @@ from .models import (
InitialMgmtGroupCSPResponse, InitialMgmtGroupCSPResponse,
InitialMgmtGroupVerificationCSPPayload, InitialMgmtGroupVerificationCSPPayload,
InitialMgmtGroupVerificationCSPResponse, InitialMgmtGroupVerificationCSPResponse,
EnvironmentCSPPayload,
EnvironmentCSPResult,
KeyVaultCredentials, KeyVaultCredentials,
ManagementGroupCSPResponse, PrincipalAdminRoleCSPPayload,
PrincipalAdminRoleCSPResult,
ProductPurchaseCSPPayload, ProductPurchaseCSPPayload,
ProductPurchaseCSPResult, ProductPurchaseCSPResult,
ProductPurchaseVerificationCSPPayload, ProductPurchaseVerificationCSPPayload,
ProductPurchaseVerificationCSPResult, ProductPurchaseVerificationCSPResult,
PrincipalAdminRoleCSPPayload,
PrincipalAdminRoleCSPResult,
ReportingCSPPayload, ReportingCSPPayload,
SubscriptionCreationCSPPayload,
SubscriptionCreationCSPResult,
SubscriptionVerificationCSPPayload,
SuscriptionVerificationCSPResult,
TaskOrderBillingCreationCSPPayload, TaskOrderBillingCreationCSPPayload,
TaskOrderBillingCreationCSPResult, TaskOrderBillingCreationCSPResult,
TaskOrderBillingVerificationCSPPayload, TaskOrderBillingVerificationCSPPayload,
@ -59,7 +64,6 @@ from .models import (
) )
from .policy import AzurePolicyManager from .policy import AzurePolicyManager
# This needs to be a fully pathed role definition identifier, not just a UUID # This needs to be a fully pathed role definition identifier, not just a UUID
# TODO: Extract these from sdk msrestazure.azure_cloud import AZURE_PUBLIC_CLOUD # TODO: Extract these from sdk msrestazure.azure_cloud import AZURE_PUBLIC_CLOUD
AZURE_SKU_ID = "0001" # probably a static sku specific to ATAT/JEDI AZURE_SKU_ID = "0001" # probably a static sku specific to ATAT/JEDI
@ -122,11 +126,15 @@ class AzureCloudProvider(CloudProviderInterface):
) )
try: try:
return secret_client.set_secret(secret_key, secret_value) return secret_client.set_secret(secret_key, secret_value)
except self.exceptions.HttpResponseError: except self.sdk.exceptions.HttpResponseError as exc:
app.logger.error( app.logger.error(
f"Could not SET secret in Azure keyvault for key {secret_key}.", f"Could not SET secret in Azure keyvault for key {secret_key}.",
exc_info=1, exc_info=1,
) )
raise SecretException(
f"Could not SET secret in Azure keyvault for key {secret_key}.",
exc.message,
)
def get_secret(self, secret_key): def get_secret(self, secret_key):
credential = self._get_client_secret_credential_obj() credential = self._get_client_secret_credential_obj()
@ -135,67 +143,35 @@ class AzureCloudProvider(CloudProviderInterface):
) )
try: try:
return secret_client.get_secret(secret_key).value return secret_client.get_secret(secret_key).value
except self.exceptions.HttpResponseError: except self.sdk.exceptions.HttpResponseError:
app.logger.error( app.logger.error(
f"Could not GET secret in Azure keyvault for key {secret_key}.", f"Could not GET secret in Azure keyvault for key {secret_key}.",
exc_info=1, exc_info=1,
) )
raise SecretException(
def create_environment(self, auth_credentials: Dict, user, environment): f"Could not GET secret in Azure keyvault for key {secret_key}.",
# since this operation would only occur within a tenant, should we source the tenant exc.message,
# via lookup from environment once we've created the portfolio csp data schema
# something like this:
# environment_tenant = environment.application.portfolio.csp_data.get('tenant_id', None)
# though we'd probably source the whole credentials for these calls from the portfolio csp
# data, as it would have to be where we store the creds for the at-at user within the portfolio tenant
# credentials = self._get_credential_obj(environment.application.portfolio.csp_data.get_creds())
credentials = self._get_credential_obj(self._root_creds)
display_name = f"{environment.application.name}_{environment.name}_{environment.id}" # proposed format
management_group_id = "?" # management group id chained from environment
parent_id = "?" # from environment.application
management_group = self._create_management_group(
credentials, management_group_id, display_name, parent_id,
) )
return ManagementGroupCSPResponse(**management_group) def create_environment(self, payload: EnvironmentCSPPayload):
creds = self._source_creds(payload.tenant_id)
credentials = self._get_credential_obj(
{
"client_id": creds.tenant_sp_client_id,
"secret_key": creds.tenant_sp_key,
"tenant_id": creds.tenant_id,
},
resource=self.sdk.cloud.endpoints.resource_manager,
)
def create_atat_admin_user( response = self._create_management_group(
self, auth_credentials: Dict, csp_environment_id: str
) -> Dict:
root_creds = self._root_creds
credentials = self._get_credential_obj(root_creds)
sub_client = self.sdk.subscription.SubscriptionClient(credentials)
subscription = sub_client.subscriptions.get(csp_environment_id)
managment_principal = self._get_management_service_principal()
auth_client = self.sdk.authorization.AuthorizationManagementClient(
credentials, credentials,
# TODO: Determine which subscription this needs to point at payload.management_group_name,
# Once we're in a multi-sub environment payload.display_name,
subscription.id, payload.parent_id,
) )
# Create role assignment for return EnvironmentCSPResult(**response)
role_assignment_id = str(uuid4())
role_assignment_create_params = auth_client.role_assignments.models.RoleAssignmentCreateParameters(
role_definition_id=REMOTE_ROOT_ROLE_DEF_ID,
principal_id=managment_principal.id,
)
auth_client.role_assignments.create(
scope=f"/subscriptions/{subscription.id}/",
role_assignment_name=role_assignment_id,
parameters=role_assignment_create_params,
)
return {
"csp_user_id": managment_principal.object_id,
"credentials": managment_principal.password_credentials,
"role_name": role_assignment_id,
}
def create_application(self, payload: ApplicationCSPPayload): def create_application(self, payload: ApplicationCSPPayload):
creds = self._source_creds(payload.tenant_id) creds = self._source_creds(payload.tenant_id)
@ -841,17 +817,6 @@ class AzureCloudProvider(CloudProviderInterface):
return self._ok() return self._ok()
def create_billing_alerts(self, TBD):
# TODO: Add azure-mgmt-consumption for Budget and Notification entities/operations
# TODO: Determine how to auth against that API using the SDK, doesn't seeem possible at the moment
# TODO: billing alerts are registered as Notifications on Budget objects, which have start/end dates
# TODO: determine what the keys in the Notifications dict are supposed to be
# we may need to rotate budget objects when new TOs/CLINs are reported?
# we likely only want the budget ID, can be updated or replaced?
response = {"id": "id"}
return self._ok({"budget_id": response["id"]})
def _get_management_service_principal(self): def _get_management_service_principal(self):
# we really should be using graph.microsoft.com, but i'm getting # we really should be using graph.microsoft.com, but i'm getting
# "expired token" errors for that # "expired token" errors for that

29
atst/domain/csp/cloud/cloud_provider_interface.py
View File

@ -11,7 +11,7 @@ class CloudProviderInterface:
def root_creds(self) -> Dict: def root_creds(self) -> Dict:
raise NotImplementedError() raise NotImplementedError()
def create_environment(self, auth_credentials: Dict, user, environment) -> str: def create_environment(self, payload):
"""Create a new environment in the CSP. """Create a new environment in the CSP.
Arguments: Arguments:
@ -31,33 +31,6 @@ class CloudProviderInterface:
""" """
raise NotImplementedError() raise NotImplementedError()
def create_atat_admin_user(
self, auth_credentials: Dict, csp_environment_id: str
) -> Dict:
"""Creates a new, programmatic user in the CSP. Grants this user full permissions to administer
the CSP.
Arguments:
auth_credentials -- Object containing CSP account credentials
csp_environment_id -- ID of the CSP Environment the admin user should be created in
Returns:
object: Object representing new remote admin user, including credentials
Something like:
{
"user_id": string,
"credentials": dict, # structure TBD based on csp
}
Raises:
AuthenticationException: Problem with the credentials
AuthorizationException: Credentials not authorized for current action(s)
ConnectionException: Issue with the CSP API connection
UnknownServerException: Unknown issue on the CSP side
UserProvisioningException: Problem creating the root user
"""
raise NotImplementedError()
def create_or_update_user( def create_or_update_user(
self, auth_credentials: Dict, user_info, csp_role_id: str self, auth_credentials: Dict, user_info, csp_role_id: str
) -> str: ) -> str:

14
atst/domain/csp/cloud/exceptions.py
View File

@ -118,3 +118,17 @@ class BaselineProvisionException(GeneralCSPException):
return "Could not complete baseline provisioning for environment ({}): {}".format( return "Could not complete baseline provisioning for environment ({}): {}".format(
self.env_identifier, self.reason self.env_identifier, self.reason
) )
class SecretException(GeneralCSPException):
"""A problem occurred with setting or getting secrets"""
def __init__(self, tenant_id, reason):
self.tenant_id = tenant_id
self.reason = reason
@property
def message(self):
return "Could not get or set secret for ({}): {}".format(
self.tenant_id, self.reason
)

56
atst/domain/csp/cloud/mock_cloud_provider.py
View File

@ -4,9 +4,7 @@ from .cloud_provider_interface import CloudProviderInterface
from .exceptions import ( from .exceptions import (
AuthenticationException, AuthenticationException,
AuthorizationException, AuthorizationException,
BaselineProvisionException,
ConnectionException, ConnectionException,
EnvironmentCreationException,
GeneralCSPException, GeneralCSPException,
UnknownServerException, UnknownServerException,
UserProvisioningException, UserProvisioningException,
@ -42,6 +40,8 @@ from .models import (
SubscriptionCreationCSPResult, SubscriptionCreationCSPResult,
SubscriptionVerificationCSPPayload, SubscriptionVerificationCSPPayload,
SuscriptionVerificationCSPResult, SuscriptionVerificationCSPResult,
EnvironmentCSPPayload,
EnvironmentCSPResult,
TaskOrderBillingCreationCSPPayload, TaskOrderBillingCreationCSPPayload,
TaskOrderBillingCreationCSPResult, TaskOrderBillingCreationCSPResult,
TaskOrderBillingVerificationCSPPayload, TaskOrderBillingVerificationCSPPayload,
@ -98,34 +98,6 @@ class MockCloudProvider(CloudProviderInterface):
def get_secret(self, secret_key: str, default=dict()): def get_secret(self, secret_key: str, default=dict()):
return default return default
def create_environment(self, auth_credentials, user, environment):
self._authorize(auth_credentials)
self._delay(1, 5)
self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION)
self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION)
self._maybe_raise(
self.ENV_CREATE_FAILURE_PCT,
EnvironmentCreationException(
environment.id, "Could not create environment."
),
)
csp_environment_id = self._id()
self._delay(1, 5)
self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION)
self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION)
self._maybe_raise(
self.ATAT_ADMIN_CREATE_FAILURE_PCT,
BaselineProvisionException(
csp_environment_id, "Could not create environment baseline."
),
)
self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION)
return csp_environment_id
def create_subscription(self, payload: SubscriptionCreationCSPPayload): def create_subscription(self, payload: SubscriptionCreationCSPPayload):
return self.create_subscription_creation(payload) return self.create_subscription_creation(payload)
@ -149,23 +121,6 @@ class MockCloudProvider(CloudProviderInterface):
subscription_id="subscriptions/60fbbb72-0516-4253-ab18-c92432ba3230" subscription_id="subscriptions/60fbbb72-0516-4253-ab18-c92432ba3230"
) )
def create_atat_admin_user(self, auth_credentials, csp_environment_id):
self._authorize(auth_credentials)
self._delay(1, 5)
self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION)
self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION)
self._maybe_raise(
self.ATAT_ADMIN_CREATE_FAILURE_PCT,
UserProvisioningException(
csp_environment_id, "atat_admin", "Could not create admin user."
),
)
self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION)
return {"id": self._id(), "credentials": self._auth_credentials}
def create_tenant(self, payload: TenantCSPPayload): def create_tenant(self, payload: TenantCSPPayload):
""" """
payload is an instance of TenantCSPPayload data class payload is an instance of TenantCSPPayload data class
@ -507,6 +462,13 @@ class MockCloudProvider(CloudProviderInterface):
id=f"{AZURE_MGMNT_PATH}{payload.management_group_name}" id=f"{AZURE_MGMNT_PATH}{payload.management_group_name}"
) )
def create_environment(self, payload: EnvironmentCSPPayload):
self._maybe_raise(self.UNAUTHORIZED_RATE, GeneralCSPException)
return EnvironmentCSPResult(
id=f"{AZURE_MGMNT_PATH}{payload.management_group_name}"
)
def create_user(self, payload: UserCSPPayload): def create_user(self, payload: UserCSPPayload):
self._maybe_raise(self.UNAUTHORIZED_RATE, GeneralCSPException) self._maybe_raise(self.UNAUTHORIZED_RATE, GeneralCSPException)

8
atst/domain/csp/cloud/models.py
View File

@ -383,6 +383,14 @@ class InitialMgmtGroupVerificationCSPResponse(ManagementGroupGetCSPResponse):
pass pass
class EnvironmentCSPPayload(ManagementGroupCSPPayload):
pass
class EnvironmentCSPResult(ManagementGroupCSPResponse):
pass
class KeyVaultCredentials(BaseModel): class KeyVaultCredentials(BaseModel):
root_sp_client_id: Optional[str] root_sp_client_id: Optional[str]
root_sp_key: Optional[str] root_sp_key: Optional[str]

8
atst/domain/environment_roles.py
View File

@ -106,9 +106,11 @@ class EnvironmentRoles(object):
def disable(cls, environment_role_id): def disable(cls, environment_role_id):
environment_role = EnvironmentRoles.get_by_id(environment_role_id) environment_role = EnvironmentRoles.get_by_id(environment_role_id)
if environment_role.csp_user_id and not environment_role.environment.is_pending: if environment_role.csp_user_id and not environment_role.environment.cloud_id:
credentials = environment_role.environment.csp_credentials tenant_id = environment_role.environment.application.portfolio.csp_data.get(
app.csp.cloud.disable_user(credentials, environment_role.csp_user_id) "tenant_id"
)
app.csp.cloud.disable_user(tenant_id, environment_role.csp_user_id)
environment_role.status = EnvironmentRole.Status.DISABLED environment_role.status = EnvironmentRole.Status.DISABLED
db.session.add(environment_role) db.session.add(environment_role)

17
atst/domain/environments.py
View File

@ -124,18 +124,9 @@ class Environments(object):
Any environment with an active CLIN that doesn't yet have a `cloud_id`. Any environment with an active CLIN that doesn't yet have a `cloud_id`.
""" """
results = ( results = (
cls.base_provision_query(now).filter(Environment.cloud_id == None).all() cls.base_provision_query(now)
.filter(Application.cloud_id != None)
.filter(Environment.cloud_id.is_(None))
.all()
) )
return [id_ for id_, in results] return [id_ for id_, in results]
@classmethod
def get_environments_pending_atat_user_creation(cls, now) -> List[UUID]:
"""
Any environment with an active CLIN that has a cloud_id but no `root_user_info`.
"""
results = (
cls.base_provision_query(now)
.filter(Environment.cloud_id != None)
.filter(Environment.root_user_info == None)
).all()
return [id_ for id_, in results]

76
atst/jobs.py
View File

@ -1,18 +1,21 @@
from flask import current_app as app
import pendulum import pendulum
from flask import current_app as app
from atst.database import db from atst.database import db
from atst.queue import celery from atst.domain.application_roles import ApplicationRoles
from atst.models import JobFailure
from atst.domain.csp.cloud.exceptions import GeneralCSPException
from atst.domain.csp.cloud import CloudProviderInterface
from atst.domain.applications import Applications from atst.domain.applications import Applications
from atst.domain.csp.cloud import CloudProviderInterface
from atst.domain.csp.cloud.exceptions import GeneralCSPException
from atst.domain.csp.cloud.models import (
ApplicationCSPPayload,
EnvironmentCSPPayload,
UserCSPPayload,
)
from atst.domain.environments import Environments from atst.domain.environments import Environments
from atst.domain.portfolios import Portfolios from atst.domain.portfolios import Portfolios
from atst.domain.application_roles import ApplicationRoles from atst.models import JobFailure
from atst.models.utils import claim_for_update, claim_many_for_update from atst.models.utils import claim_for_update, claim_many_for_update
from atst.utils.localization import translate from atst.queue import celery
from atst.domain.csp.cloud.models import ApplicationCSPPayload, UserCSPPayload
class RecordFailure(celery.Task): class RecordFailure(celery.Task):
@ -109,45 +112,17 @@ def do_create_environment(csp: CloudProviderInterface, environment_id=None):
with claim_for_update(environment) as environment: with claim_for_update(environment) as environment:
if environment.cloud_id is not None: if environment.cloud_id is not None:
# TODO: Return value for this?
return return
user = environment.creator csp_details = environment.application.portfolio.csp_data
parent_id = environment.application.cloud_id
# we'll need to do some checking in this job for cases where it's retrying tenant_id = csp_details.get("tenant_id")
# when a failure occured after some successful steps payload = EnvironmentCSPPayload(
# (e.g. if environment.cloud_id is not None, then we can skip first step) tenant_id=tenant_id, display_name=environment.name, parent_id=parent_id
# credentials either from a given user or pulled from config?
# if using global creds, do we need to log what user authorized action?
atat_root_creds = csp.root_creds()
# user is needed because baseline root account in the environment will
# be assigned to the requesting user, open question how to handle duplicate
# email addresses across new environments
csp_environment_id = csp.create_environment(atat_root_creds, user, environment)
environment.cloud_id = csp_environment_id
db.session.add(environment)
db.session.commit()
body = render_email(
"emails/application/environment_ready.txt", {"environment": environment}
)
app.mailer.send(
[environment.creator.email], translate("email.environment_ready"), body
) )
env_result = csp.create_environment(payload)
def do_create_atat_admin_user(csp: CloudProviderInterface, environment_id=None): environment.cloud_id = env_result.id
environment = Environments.get(environment_id)
with claim_for_update(environment) as environment:
atat_root_creds = csp.root_creds()
atat_remote_root_user = csp.create_atat_admin_user(
atat_root_creds, environment.cloud_id
)
environment.root_user_info = atat_remote_root_user
db.session.add(environment) db.session.add(environment)
db.session.commit() db.session.commit()
@ -191,13 +166,6 @@ def create_environment(self, environment_id=None):
do_work(do_create_environment, self, app.csp.cloud, environment_id=environment_id) do_work(do_create_environment, self, app.csp.cloud, environment_id=environment_id)
@celery.task(bind=True, base=RecordFailure)
def create_atat_admin_user(self, environment_id=None):
do_work(
do_create_atat_admin_user, self, app.csp.cloud, environment_id=environment_id
)
@celery.task(bind=True) @celery.task(bind=True)
def dispatch_provision_portfolio(self): def dispatch_provision_portfolio(self):
""" """
@ -225,11 +193,3 @@ def dispatch_create_environment(self):
pendulum.now() pendulum.now()
): ):
create_environment.delay(environment_id=environment_id) create_environment.delay(environment_id=environment_id)
@celery.task(bind=True)
def dispatch_create_atat_admin_user(self):
for environment_id in Environments.get_environments_pending_atat_user_creation(
pendulum.now()
):
create_atat_admin_user.delay(environment_id=environment_id)

28
atst/models/environment.py
View File

@ -1,11 +1,9 @@
from sqlalchemy import Column, ForeignKey, String, UniqueConstraint from sqlalchemy import Column, ForeignKey, String, UniqueConstraint
from sqlalchemy.orm import relationship from sqlalchemy.orm import relationship
from sqlalchemy.dialects.postgresql import JSONB
from enum import Enum
from atst.models.base import Base
import atst.models.mixins as mixins import atst.models.mixins as mixins
import atst.models.types as types import atst.models.types as types
from atst.models.base import Base
class Environment( class Environment(
@ -30,7 +28,6 @@ class Environment(
creator = relationship("User") creator = relationship("User")
cloud_id = Column(String) cloud_id = Column(String)
root_user_info = Column(JSONB(none_as_null=True))
roles = relationship( roles = relationship(
"EnvironmentRole", "EnvironmentRole",
@ -44,10 +41,6 @@ class Environment(
), ),
) )
class ProvisioningStatus(Enum):
PENDING = "pending"
COMPLETED = "completed"
@property @property
def users(self): def users(self):
return {r.application_role.user for r in self.roles} return {r.application_role.user for r in self.roles}
@ -68,17 +61,6 @@ class Environment(
def portfolio_id(self): def portfolio_id(self):
return self.application.portfolio_id return self.application.portfolio_id
@property
def provisioning_status(self) -> ProvisioningStatus:
if self.cloud_id is None or self.root_user_info is None:
return self.ProvisioningStatus.PENDING
else:
return self.ProvisioningStatus.COMPLETED
@property
def is_pending(self):
return self.provisioning_status == self.ProvisioningStatus.PENDING
def __repr__(self): def __repr__(self):
return "<Environment(name='{}', num_users='{}', application='{}', portfolio='{}', id='{}')>".format( return "<Environment(name='{}', num_users='{}', application='{}', portfolio='{}', id='{}')>".format(
self.name, self.name,
@ -91,11 +73,3 @@ class Environment(
@property @property
def history(self): def history(self):
return self.get_changes() return self.get_changes()
@property
def csp_credentials(self):
return (
self.root_user_info.get("credentials")
if self.root_user_info is not None
else None
)

3
atst/models/portfolio_state_machine.py
View File

@ -180,7 +180,8 @@ class PortfolioStateMachine(
# TODO: catch and handle general CSP exception here # TODO: catch and handle general CSP exception here
except (ConnectionException, UnknownServerException) as exc: except (ConnectionException, UnknownServerException) as exc:
app.logger.error( app.logger.error(
f"CSP api call. Caught exception for {self.__repr__()}.", exc_info=1, f"CSP api call. Caught exception for {self.__repr__()}.",
exc_info=1,
) )
# TODO: Ensure that failing the stage does not preclude a Celery retry # TODO: Ensure that failing the stage does not preclude a Celery retry
self.fail_stage(stage) self.fail_stage(stage)

4
atst/queue.py
View File

@ -19,10 +19,6 @@ def update_celery(celery, app):
"task": "atst.jobs.dispatch_create_environment", "task": "atst.jobs.dispatch_create_environment",
"schedule": 60, "schedule": 60,
}, },
"beat-dispatch_create_atat_admin_user": {
"task": "atst.jobs.dispatch_create_atat_admin_user",
"schedule": 60,
},
"beat-dispatch_create_user": { "beat-dispatch_create_user": {
"task": "atst.jobs.dispatch_create_user", "task": "atst.jobs.dispatch_create_user",
"schedule": 60, "schedule": 60,

1
atst/routes/applications/settings.py
View File

@ -39,7 +39,6 @@ def get_environments_obj_for_app(application):
{ {
"id": env.id, "id": env.id,
"name": env.name, "name": env.name,
"pending": env.is_pending,
"edit_form": EditEnvironmentForm(obj=env), "edit_form": EditEnvironmentForm(obj=env),
"member_count": len(env.roles), "member_count": len(env.roles),
"members": sorted( "members": sorted(

188
tests/domain/cloud/test_azure_csp.py
View File

@ -2,11 +2,11 @@ import json
from unittest.mock import Mock, patch from unittest.mock import Mock, patch
from uuid import uuid4 from uuid import uuid4
import pendulum
import pydantic
import pytest import pytest
from tests.factories import ApplicationFactory, EnvironmentFactory from tests.factories import ApplicationFactory, EnvironmentFactory
from tests.mock_azure import AUTH_CREDENTIALS, mock_azure from tests.mock_azure import AUTH_CREDENTIALS, mock_azure
import pendulum
import pydantic
from atst.domain.csp.cloud import AzureCloudProvider from atst.domain.csp.cloud import AzureCloudProvider
from atst.domain.csp.cloud.models import ( from atst.domain.csp.cloud.models import (
@ -27,6 +27,10 @@ from atst.domain.csp.cloud.models import (
InitialMgmtGroupVerificationCSPPayload, InitialMgmtGroupVerificationCSPPayload,
InitialMgmtGroupVerificationCSPResponse, InitialMgmtGroupVerificationCSPResponse,
CostManagementQueryCSPResult, CostManagementQueryCSPResult,
EnvironmentCSPPayload,
EnvironmentCSPResult,
PrincipalAdminRoleCSPPayload,
PrincipalAdminRoleCSPResult,
ProductPurchaseCSPPayload, ProductPurchaseCSPPayload,
ProductPurchaseCSPResult, ProductPurchaseCSPResult,
ProductPurchaseVerificationCSPPayload, ProductPurchaseVerificationCSPPayload,
@ -52,6 +56,7 @@ from atst.domain.csp.cloud.models import (
TenantPrincipalCSPResult, TenantPrincipalCSPResult,
TenantPrincipalOwnershipCSPPayload, TenantPrincipalOwnershipCSPPayload,
TenantPrincipalOwnershipCSPResult, TenantPrincipalOwnershipCSPResult,
UserCSPPayload,
) )
BILLING_ACCOUNT_NAME = "52865e4c-52e8-5a6c-da6b-c58f0814f06f:7ea5de9d-b8ce-4901-b1c5-d864320c7b03_2019-05-31" BILLING_ACCOUNT_NAME = "52865e4c-52e8-5a6c-da6b-c58f0814f06f:7ea5de9d-b8ce-4901-b1c5-d864320c7b03_2019-05-31"
@ -71,12 +76,14 @@ def mock_management_group_get(mock_azure, spec_dict):
def test_create_environment_succeeds(mock_azure: AzureCloudProvider): def test_create_environment_succeeds(mock_azure: AzureCloudProvider):
environment = EnvironmentFactory.create() environment = EnvironmentFactory.create()
mock_management_group_create(mock_azure, {"id": "Test Id"}) mock_management_group_create(mock_azure, {"id": "Test Id"})
result = mock_azure.create_environment( mock_azure = mock_get_secret(mock_azure)
AUTH_CREDENTIALS, environment.creator, environment
payload = EnvironmentCSPPayload(
tenant_id="1234", display_name=environment.name, parent_id=str(uuid4())
) )
result = mock_azure.create_environment(payload)
assert result.id == "Test Id" assert result.id == "Test Id"
@ -146,20 +153,6 @@ def test_create_initial_mgmt_group_verification_succeeds(
# assert result.name == management_group_name # assert result.name == management_group_name
def test_create_atat_admin_user_succeeds(mock_azure: AzureCloudProvider):
environment_id = str(uuid4())
csp_user_id = str(uuid4)
mock_azure.sdk.graphrbac.GraphRbacManagementClient.return_value.service_principals.create.return_value.object_id = (
csp_user_id
)
result = mock_azure.create_atat_admin_user(AUTH_CREDENTIALS, environment_id)
assert result.get("csp_user_id") == csp_user_id
def test_create_policy_definition_succeeds(mock_azure: AzureCloudProvider): def test_create_policy_definition_succeeds(mock_azure: AzureCloudProvider):
subscription_id = str(uuid4()) subscription_id = str(uuid4())
management_group_id = str(uuid4()) management_group_id = str(uuid4())
@ -211,6 +204,27 @@ def test_create_tenant(mock_azure: AzureCloudProvider):
assert body.tenant_id == "60ff9d34-82bf-4f21-b565-308ef0533435" assert body.tenant_id == "60ff9d34-82bf-4f21-b565-308ef0533435"
def test_create_tenant_fails(mock_azure: AzureCloudProvider):
mock_result = Mock()
mock_result.json.return_value = {"error": "body"}
mock_result.status_code = 403
mock_azure.sdk.requests.post.return_value = mock_result
payload = TenantCSPPayload(
**dict(
user_id="admin",
password="JediJan13$coot", # pragma: allowlist secret
domain_name="jediccpospawnedtenant2",
first_name="Tedry",
last_name="Tenet",
country_code="US",
password_recovery_email_address="thomas@promptworks.com",
)
)
mock_azure = mock_get_secret(mock_azure)
result = mock_azure.create_tenant(payload)
assert result.get("status") == "error"
def test_create_billing_profile_creation(mock_azure: AzureCloudProvider): def test_create_billing_profile_creation(mock_azure: AzureCloudProvider):
mock_azure.sdk.adal.AuthenticationContext.return_value.context.acquire_token_with_client_credentials.return_value = { mock_azure.sdk.adal.AuthenticationContext.return_value.context.acquire_token_with_client_credentials.return_value = {
"accessToken": "TOKEN" "accessToken": "TOKEN"
@ -622,10 +636,10 @@ def test_create_tenant_principal_credential(mock_azure: AzureCloudProvider):
def test_create_admin_role_definition(mock_azure: AzureCloudProvider): def test_create_admin_role_definition(mock_azure: AzureCloudProvider):
with patch.object( with patch.object(
AzureCloudProvider, AzureCloudProvider,
"_get_elevated_management_token", "_get_tenant_admin_token",
wraps=mock_azure._get_elevated_management_token, wraps=mock_azure._get_tenant_admin_token,
) as get_elevated_management_token: ) as get_tenant_admin_token:
get_elevated_management_token.return_value = "my fake token" get_tenant_admin_token.return_value = "my fake token"
mock_result = Mock() mock_result = Mock()
mock_result.ok = True mock_result.ok = True
@ -706,6 +720,35 @@ def test_create_tenant_principal_ownership(mock_azure: AzureCloudProvider):
assert result.principal_owner_assignment_id == "id" assert result.principal_owner_assignment_id == "id"
def test_create_principal_admin_role(mock_azure: AzureCloudProvider):
with patch.object(
AzureCloudProvider,
"_get_tenant_admin_token",
wraps=mock_azure._get_tenant_admin_token,
) as get_tenant_admin_token:
get_tenant_admin_token.return_value = "my fake token"
mock_result = Mock()
mock_result.ok = True
mock_result.json.return_value = {"id": "id"}
mock_azure.sdk.requests.post.return_value = mock_result
payload = PrincipalAdminRoleCSPPayload(
**{
"tenant_id": uuid4().hex,
"principal_id": "6d2d2d6c-a6d6-41e1-8bb1-73d11475f8f4",
"admin_role_def_id": uuid4().hex,
}
)
result: PrincipalAdminRoleCSPResult = mock_azure.create_principal_admin_role(
payload
)
assert result.principal_assignment_id == "id"
def test_create_subscription_creation(mock_azure: AzureCloudProvider): def test_create_subscription_creation(mock_azure: AzureCloudProvider):
with patch.object( with patch.object(
AzureCloudProvider, AzureCloudProvider,
@ -841,3 +884,102 @@ def test_get_reporting_data_malformed_payload(mock_azure: AzureCloudProvider):
from_date="foo", to_date="bar", **malformed_payload, from_date="foo", to_date="bar", **malformed_payload,
) )
) )
def test_get_secret(mock_azure: AzureCloudProvider):
with patch.object(
AzureCloudProvider,
"_get_client_secret_credential_obj",
wraps=mock_azure._get_client_secret_credential_obj,
) as _get_client_secret_credential_obj:
_get_client_secret_credential_obj.return_value = {}
mock_azure.sdk.secrets.SecretClient.return_value.get_secret.return_value.value = (
"my secret"
)
assert mock_azure.get_secret("secret key") == "my secret"
def test_set_secret(mock_azure: AzureCloudProvider):
with patch.object(
AzureCloudProvider,
"_get_client_secret_credential_obj",
wraps=mock_azure._get_client_secret_credential_obj,
) as _get_client_secret_credential_obj:
_get_client_secret_credential_obj.return_value = {}
mock_azure.sdk.secrets.SecretClient.return_value.set_secret.return_value = (
"my secret"
)
assert mock_azure.set_secret("secret key", "secret_value") == "my secret"
def test_create_active_directory_user(mock_azure: AzureCloudProvider):
mock_result = Mock()
mock_result.ok = True
mock_result.json.return_value = {"id": "id"}
mock_azure.sdk.requests.post.return_value = mock_result
payload = UserCSPPayload(
tenant_id=uuid4().hex,
display_name="Test Testerson",
tenant_host_name="testtenant",
email="test@testerson.test",
password="asdfghjkl", # pragma: allowlist secret
)
result = mock_azure._create_active_directory_user("token", payload)
assert result.id == "id"
def test_update_active_directory_user_email(mock_azure: AzureCloudProvider):
mock_result = Mock()
mock_result.ok = True
mock_azure.sdk.requests.patch.return_value = mock_result
payload = UserCSPPayload(
tenant_id=uuid4().hex,
display_name="Test Testerson",
tenant_host_name="testtenant",
email="test@testerson.test",
password="asdfghjkl", # pragma: allowlist secret
)
result = mock_azure._update_active_directory_user_email(
"token", uuid4().hex, payload
)
assert result
def test_create_user(mock_azure: AzureCloudProvider):
with patch.object(
AzureCloudProvider,
"_get_tenant_principal_token",
wraps=mock_azure._get_tenant_principal_token,
) as _get_tenant_principal_token:
_get_tenant_principal_token.return_value = "token"
mock_result_create = Mock()
mock_result_create.ok = True
mock_result_create.json.return_value = {"id": "id"}
mock_azure.sdk.requests.post.return_value = mock_result_create
mock_result_update = Mock()
mock_result_update.ok = True
mock_azure.sdk.requests.patch.return_value = mock_result_update
payload = UserCSPPayload(
tenant_id=uuid4().hex,
display_name="Test Testerson",
tenant_host_name="testtenant",
email="test@testerson.test",
password="asdfghjkl", # pragma: allowlist secret
)
result = mock_azure.create_user(payload)
assert result.id == "id"

26
tests/domain/cloud/test_mock_csp.py
View File

@ -1,6 +1,7 @@
import pytest import pytest
from atst.domain.csp import MockCloudProvider from atst.domain.csp import MockCloudProvider
from atst.domain.csp.cloud.models import EnvironmentCSPPayload, EnvironmentCSPResult
from tests.factories import EnvironmentFactory, EnvironmentRoleFactory, UserFactory from tests.factories import EnvironmentFactory, EnvironmentRoleFactory, UserFactory
@ -14,20 +15,17 @@ def mock_csp():
def test_create_environment(mock_csp: MockCloudProvider): def test_create_environment(mock_csp: MockCloudProvider):
environment = EnvironmentFactory.create() environment = EnvironmentFactory.create()
user = UserFactory.create() environment.application.cloud_id = "parent_id"
environment_id = mock_csp.create_environment(CREDENTIALS, user, environment) environment.application.portfolio.csp_data = {"tenant_id": "fake"}
assert isinstance(environment_id, str) payload = EnvironmentCSPPayload(
**dict(
tenant_id=environment.application.portfolio.csp_data.get("tenant_id"),
def test_create_admin_user(mock_csp: MockCloudProvider): display_name=environment.name,
admin_user = mock_csp.create_atat_admin_user(CREDENTIALS, "env_id") parent_id=environment.application.cloud_id,
assert isinstance(admin_user["id"], str) )
assert isinstance(admin_user["credentials"], dict) )
result = mock_csp.create_environment(payload)
assert isinstance(result, EnvironmentCSPResult)
def test_create_environment_baseline(mock_csp: MockCloudProvider):
baseline = mock_csp.create_atat_admin_user(CREDENTIALS, "env_id")
assert isinstance(baseline, dict)
def test_create_or_update_user(mock_csp: MockCloudProvider): def test_create_or_update_user(mock_csp: MockCloudProvider):

10
tests/domain/test_environment_roles.py
View File

@ -93,27 +93,25 @@ def test_disable_completed(application_role, environment):
def test_disable_checks_env_provisioning_status(session): def test_disable_checks_env_provisioning_status(session):
environment = EnvironmentFactory.create() environment = EnvironmentFactory.create()
assert environment.is_pending assert not environment.cloud_id
env_role1 = EnvironmentRoleFactory.create(environment=environment) env_role1 = EnvironmentRoleFactory.create(environment=environment)
env_role1 = EnvironmentRoles.disable(env_role1.id) env_role1 = EnvironmentRoles.disable(env_role1.id)
assert env_role1.disabled assert env_role1.disabled
environment.cloud_id = "cloud-id" environment.cloud_id = "cloud-id"
environment.root_user_info = {"credentials": "credentials"}
session.add(environment) session.add(environment)
session.commit() session.commit()
session.refresh(environment) session.refresh(environment)
assert not environment.is_pending assert environment.cloud_id
env_role2 = EnvironmentRoleFactory.create(environment=environment) env_role2 = EnvironmentRoleFactory.create(environment=environment)
env_role2 = EnvironmentRoles.disable(env_role2.id) env_role2 = EnvironmentRoles.disable(env_role2.id)
assert env_role2.disabled assert env_role2.disabled
def test_disable_checks_env_role_provisioning_status(): def test_disable_checks_env_role_provisioning_status():
environment = EnvironmentFactory.create( environment = EnvironmentFactory.create(cloud_id="cloud-id")
cloud_id="cloud-id", root_user_info={"credentials": "credentials"} environment.application.portfolio.csp_data = {"tenant_id": uuid4().hex}
)
env_role1 = EnvironmentRoleFactory.create(environment=environment) env_role1 = EnvironmentRoleFactory.create(environment=environment)
assert not env_role1.csp_user_id assert not env_role1.csp_user_id
env_role1 = EnvironmentRoles.disable(env_role1.id) env_role1 = EnvironmentRoles.disable(env_role1.id)

39
tests/domain/test_environments.py
View File

@ -134,37 +134,16 @@ class TestGetEnvironmentsPendingCreate(EnvQueryTest):
self.create_portfolio_with_clins([(self.TOMORROW, self.TOMORROW)]) self.create_portfolio_with_clins([(self.TOMORROW, self.TOMORROW)])
assert len(Environments.get_environments_pending_creation(self.NOW)) == 0 assert len(Environments.get_environments_pending_creation(self.NOW)) == 0
def test_with_already_provisioned_app(self, session):
self.create_portfolio_with_clins(
[(self.YESTERDAY, self.TOMORROW)], app_data={"cloud_id": uuid4().hex}
)
assert len(Environments.get_environments_pending_creation(self.NOW)) == 1
def test_with_already_provisioned_env(self, session): def test_with_already_provisioned_env(self, session):
self.create_portfolio_with_clins( self.create_portfolio_with_clins(
[(self.YESTERDAY, self.TOMORROW)], env_data={"cloud_id": uuid4().hex} [(self.YESTERDAY, self.TOMORROW)],
env_data={"cloud_id": uuid4().hex},
app_data={"cloud_id": uuid4().hex},
) )
assert len(Environments.get_environments_pending_creation(self.NOW)) == 0 assert len(Environments.get_environments_pending_creation(self.NOW)) == 0
class TestGetEnvironmentsPendingAtatUserCreation(EnvQueryTest):
def test_with_provisioned_environment(self):
self.create_portfolio_with_clins(
[(self.YESTERDAY, self.TOMORROW)],
{"cloud_id": uuid4().hex, "root_user_info": {}},
)
assert (
len(Environments.get_environments_pending_atat_user_creation(self.NOW)) == 0
)
def test_with_unprovisioned_environment(self):
self.create_portfolio_with_clins(
[(self.YESTERDAY, self.TOMORROW)],
{"cloud_id": uuid4().hex, "root_user_info": None},
)
assert (
len(Environments.get_environments_pending_atat_user_creation(self.NOW)) == 1
)
def test_with_unprovisioned_expired_clins_environment(self):
self.create_portfolio_with_clins(
[(self.YESTERDAY, self.YESTERDAY)],
{"cloud_id": uuid4().hex, "root_user_info": None},
)
assert (
len(Environments.get_environments_pending_atat_user_creation(self.NOW)) == 0
)

22
tests/models/test_environments.py
View File

@ -51,28 +51,6 @@ def test_audit_event_for_environment_deletion(session):
assert after assert after
@pytest.mark.parametrize(
"env_data,expected_status",
[
[
{"cloud_id": None, "root_user_info": None},
Environment.ProvisioningStatus.PENDING,
],
[
{"cloud_id": 1, "root_user_info": None},
Environment.ProvisioningStatus.PENDING,
],
[
{"cloud_id": 1, "root_user_info": {}},
Environment.ProvisioningStatus.COMPLETED,
],
],
)
def test_environment_provisioning_status(env_data, expected_status):
environment = EnvironmentFactory.create(**env_data)
assert environment.provisioning_status == expected_status
def test_environment_roles_do_not_include_deleted(): def test_environment_roles_do_not_include_deleted():
member_list = [ member_list = [
{"role_name": CSPRole.ADMIN}, {"role_name": CSPRole.ADMIN},

45
tests/test_jobs.py
View File

@ -12,14 +12,12 @@ from atst.jobs import (
dispatch_create_environment, dispatch_create_environment,
dispatch_create_application, dispatch_create_application,
dispatch_create_user, dispatch_create_user,
dispatch_create_atat_admin_user,
dispatch_provision_portfolio, dispatch_provision_portfolio,
create_environment, create_environment,
do_create_user, do_create_user,
do_provision_portfolio, do_provision_portfolio,
do_create_environment, do_create_environment,
do_create_application, do_create_application,
do_create_atat_admin_user,
) )
from tests.factories import ( from tests.factories import (
EnvironmentFactory, EnvironmentFactory,
@ -94,6 +92,10 @@ tomorrow = now.add(days=1)
def test_create_environment_job(session, csp): def test_create_environment_job(session, csp):
environment = EnvironmentFactory.create() environment = EnvironmentFactory.create()
environment.application.cloud_id = "parentId"
environment.application.portfolio.csp_data = {"tenant_id": "fake"}
session.add(environment)
session.commit()
do_create_environment(csp, environment.id) do_create_environment(csp, environment.id)
session.refresh(environment) session.refresh(environment)
@ -149,19 +151,11 @@ def test_create_user_job(session, csp):
assert app_role.cloud_id assert app_role.cloud_id
def test_create_atat_admin_user(csp, session):
environment = EnvironmentFactory.create(cloud_id="something")
do_create_atat_admin_user(csp, environment.id)
session.refresh(environment)
assert environment.root_user_info
def test_dispatch_create_environment(session, monkeypatch): def test_dispatch_create_environment(session, monkeypatch):
# Given that I have a portfolio with an active CLIN and two environments, # Given that I have a portfolio with an active CLIN and two environments,
# one of which is deleted # one of which is deleted
portfolio = PortfolioFactory.create( portfolio = PortfolioFactory.create(
applications=[{"environments": [{}, {}]}], applications=[{"environments": [{}, {}], "cloud_id": uuid4().hex}],
task_orders=[ task_orders=[
{ {
"create_clins": [ "create_clins": [
@ -227,36 +221,9 @@ def test_dispatch_create_user(monkeypatch):
mock.delay.assert_called_once_with(application_role_ids=[app_role.id]) mock.delay.assert_called_once_with(application_role_ids=[app_role.id])
def test_dispatch_create_atat_admin_user(session, monkeypatch):
portfolio = PortfolioFactory.create(
applications=[
{"environments": [{"cloud_id": uuid4().hex, "root_user_info": None}]}
],
task_orders=[
{
"create_clins": [
{
"start_date": pendulum.now().subtract(days=1),
"end_date": pendulum.now().add(days=1),
}
]
}
],
)
mock = Mock()
monkeypatch.setattr("atst.jobs.create_atat_admin_user", mock)
environment = portfolio.applications[0].environments[0]
dispatch_create_atat_admin_user.run()
mock.delay.assert_called_once_with(environment_id=environment.id)
def test_create_environment_no_dupes(session, celery_app, celery_worker): def test_create_environment_no_dupes(session, celery_app, celery_worker):
portfolio = PortfolioFactory.create( portfolio = PortfolioFactory.create(
applications=[ applications=[{"environments": [{"cloud_id": uuid4().hex}]}],
{"environments": [{"cloud_id": uuid4().hex, "root_user_info": {}}]}
],
task_orders=[ task_orders=[
{ {
"create_clins": [ "create_clins": [

8
tests/utils.py
View File

@ -81,9 +81,14 @@ class EnvQueryTest:
return self.NOW.add(days=1) return self.NOW.add(days=1)
def create_portfolio_with_clins( def create_portfolio_with_clins(
self, start_and_end_dates, env_data=None, state_machine_status=None self,
start_and_end_dates,
env_data=None,
app_data=None,
state_machine_status=None,
): ):
env_data = env_data or {} env_data = env_data or {}
app_data = app_data or {}
return factories.PortfolioFactory.create( return factories.PortfolioFactory.create(
state=state_machine_status, state=state_machine_status,
applications=[ applications=[
@ -91,6 +96,7 @@ class EnvQueryTest:
"name": "Mos Eisley", "name": "Mos Eisley",
"description": "Where Han shot first", "description": "Where Han shot first",
"environments": [{"name": "thebar", **env_data}], "environments": [{"name": "thebar", **env_data}],
**app_data,
} }
], ],
task_orders=[ task_orders=[