Merge branch 'staging' into to-builder-previous-button

This commit is contained in:
leigh-mil
2020-01-29 15:09:27 -05:00
committed by GitHub
114 changed files with 2845 additions and 859 deletions

View File

@@ -1,5 +1,9 @@
from . import BaseDomainClass
from flask import g
from sqlalchemy import func, or_
from typing import List
from uuid import UUID
from . import BaseDomainClass
from atst.database import db
from atst.domain.application_roles import ApplicationRoles
from atst.domain.environments import Environments
@@ -10,7 +14,10 @@ from atst.models import (
ApplicationRole,
ApplicationRoleStatus,
EnvironmentRole,
Portfolio,
PortfolioStateMachine,
)
from atst.models.mixins.state_machines import FSMStates
from atst.utils import first_or_none, commit_or_raise_already_exists_error
@@ -118,3 +125,21 @@ class Applications(BaseDomainClass):
db.session.commit()
return invitation
@classmethod
def get_applications_pending_creation(cls) -> List[UUID]:
results = (
db.session.query(Application.id)
.join(Portfolio)
.join(PortfolioStateMachine)
.filter(PortfolioStateMachine.state == FSMStates.COMPLETED)
.filter(Application.deleted == False)
.filter(Application.cloud_id.is_(None))
.filter(
or_(
Application.claimed_until.is_(None),
Application.claimed_until <= func.now(),
)
)
).all()
return [id_ for id_, in results]

View File

@@ -1,15 +1,14 @@
import json
import re
from secrets import token_urlsafe
from typing import Dict
from uuid import uuid4
from atst.models.application import Application
from atst.models.environment import Environment
from atst.models.user import User
from .cloud_provider_interface import CloudProviderInterface
from .exceptions import AuthenticationException
from .models import (
ApplicationCSPPayload,
ApplicationCSPResult,
BillingInstructionCSPPayload,
BillingInstructionCSPResult,
BillingProfileCreationCSPPayload,
@@ -18,6 +17,8 @@ from .models import (
BillingProfileTenantAccessCSPResult,
BillingProfileVerificationCSPPayload,
BillingProfileVerificationCSPResult,
KeyVaultCredentials,
ManagementGroupCSPResponse,
TaskOrderBillingCreationCSPPayload,
TaskOrderBillingCreationCSPResult,
TaskOrderBillingVerificationCSPPayload,
@@ -26,6 +27,7 @@ from .models import (
TenantCSPResult,
)
from .policy import AzurePolicyManager
from atst.utils import sha256_hex
AZURE_ENVIRONMENT = "AZURE_PUBLIC_CLOUD" # TBD
AZURE_SKU_ID = "?" # probably a static sku specific to ATAT/JEDI
@@ -47,6 +49,7 @@ class AzureSDKProvider(object):
import azure.common.credentials as credentials
import azure.identity as identity
from azure.keyvault import secrets
from azure.core import exceptions
from msrestazure.azure_cloud import AZURE_PUBLIC_CLOUD
import adal
@@ -85,7 +88,7 @@ class AzureCloudProvider(CloudProviderInterface):
def set_secret(self, secret_key, secret_value):
credential = self._get_client_secret_credential_obj({})
secret_client = self.secrets.SecretClient(
secret_client = self.sdk.secrets.SecretClient(
vault_url=self.vault_url, credential=credential,
)
try:
@@ -98,7 +101,7 @@ class AzureCloudProvider(CloudProviderInterface):
def get_secret(self, secret_key):
credential = self._get_client_secret_credential_obj({})
secret_client = self.secrets.SecretClient(
secret_client = self.sdk.secrets.SecretClient(
vault_url=self.vault_url, credential=credential,
)
try:
@@ -109,9 +112,7 @@ class AzureCloudProvider(CloudProviderInterface):
exc_info=1,
)
def create_environment(
self, auth_credentials: Dict, user: User, environment: Environment
):
def create_environment(self, auth_credentials: Dict, user, environment):
# since this operation would only occur within a tenant, should we source the tenant
# via lookup from environment once we've created the portfolio csp data schema
# something like this:
@@ -128,7 +129,7 @@ class AzureCloudProvider(CloudProviderInterface):
credentials, management_group_id, display_name, parent_id,
)
return management_group
return ManagementGroupCSPResponse(**management_group)
def create_atat_admin_user(
self, auth_credentials: Dict, csp_environment_id: str
@@ -167,16 +168,26 @@ class AzureCloudProvider(CloudProviderInterface):
"role_name": role_assignment_id,
}
def _create_application(self, auth_credentials: Dict, application: Application):
management_group_name = str(uuid4()) # can be anything, not just uuid
display_name = application.name # Does this need to be unique?
credentials = self._get_credential_obj(auth_credentials)
parent_id = "?" # application.portfolio.csp_details.management_group_id
return self._create_management_group(
credentials, management_group_name, display_name, parent_id,
def create_application(self, payload: ApplicationCSPPayload):
creds = self._source_creds(payload.tenant_id)
credentials = self._get_credential_obj(
{
"client_id": creds.root_sp_client_id,
"secret_key": creds.root_sp_key,
"tenant_id": creds.root_tenant_id,
},
resource=AZURE_MANAGEMENT_API,
)
response = self._create_management_group(
credentials,
payload.management_group_name,
payload.display_name,
payload.parent_id,
)
return ApplicationCSPResult(**response)
def _create_management_group(
self, credentials, management_group_id, display_name, parent_id=None,
):
@@ -198,6 +209,9 @@ class AzureCloudProvider(CloudProviderInterface):
# result is a synchronous wait, might need to do a poll instead to handle first mgmt group create
# since we were told it could take 10+ minutes to complete, unless this handles that polling internally
# TODO: what to do is status is not 'Succeeded' on the
# response object? Will it always raise its own error
# instead?
return create_request.result()
def _create_subscription(
@@ -290,6 +304,7 @@ class AzureCloudProvider(CloudProviderInterface):
sp_token = self._get_sp_token(payload.creds)
if sp_token is None:
raise AuthenticationException("Could not resolve token for tenant creation")
payload.password = token_urlsafe(16)
create_tenant_body = payload.dict(by_alias=True)
@@ -626,3 +641,24 @@ class AzureCloudProvider(CloudProviderInterface):
"secret_key": self.secret_key,
"tenant_id": self.tenant_id,
}
def _source_creds(self, tenant_id=None) -> KeyVaultCredentials:
if tenant_id:
return self._source_tenant_creds(tenant_id)
else:
return KeyVaultCredentials(
root_tenant_id=self._root_creds.get("tenant_id"),
root_sp_client_id=self._root_creds.get("client_id"),
root_sp_key=self._root_creds.get("secret_key"),
)
def update_tenant_creds(self, tenant_id, secret):
hashed = sha256_hex(tenant_id)
self.set_secret(hashed, json.dumps(secret))
return secret
def _source_tenant_creds(self, tenant_id):
hashed = sha256_hex(tenant_id)
raw_creds = self.get_secret(hashed)
return KeyVaultCredentials(**json.loads(raw_creds))

View File

@@ -1,9 +1,5 @@
from typing import Dict
from atst.models.user import User
from atst.models.environment import Environment
from atst.models.environment_role import EnvironmentRole
class CloudProviderInterface:
def set_secret(self, secret_key: str, secret_value: str):
@@ -15,9 +11,7 @@ class CloudProviderInterface:
def root_creds(self) -> Dict:
raise NotImplementedError()
def create_environment(
self, auth_credentials: Dict, user: User, environment: Environment
) -> str:
def create_environment(self, auth_credentials: Dict, user, environment) -> str:
"""Create a new environment in the CSP.
Arguments:
@@ -65,7 +59,7 @@ class CloudProviderInterface:
raise NotImplementedError()
def create_or_update_user(
self, auth_credentials: Dict, user_info: EnvironmentRole, csp_role_id: str
self, auth_credentials: Dict, user_info, csp_role_id: str
) -> str:
"""Creates a user or updates an existing user's role.

View File

@@ -17,6 +17,9 @@ from .exceptions import (
UnknownServerException,
)
from .models import (
AZURE_MGMNT_PATH,
ApplicationCSPPayload,
ApplicationCSPResult,
BillingInstructionCSPPayload,
BillingInstructionCSPResult,
BillingProfileCreationCSPPayload,
@@ -340,3 +343,16 @@ class MockCloudProvider(CloudProviderInterface):
self._delay(1, 5)
if self._with_authorization and credentials != self._auth_credentials:
raise self.AUTHENTICATION_EXCEPTION
def create_application(self, payload: ApplicationCSPPayload):
self._maybe_raise(self.UNAUTHORIZED_RATE, GeneralCSPException)
return ApplicationCSPResult(
id=f"{AZURE_MGMNT_PATH}{payload.management_group_name}"
)
def get_credentials(self, scope="portfolio", tenant_id=None):
return self.root_creds()
def update_tenant_creds(self, tenant_id, secret):
return secret

View File

@@ -1,6 +1,8 @@
from typing import Dict, List, Optional
import re
from uuid import uuid4
from pydantic import BaseModel, validator
from pydantic import BaseModel, validator, root_validator
from atst.utils import snake_to_camel
@@ -232,3 +234,110 @@ class BillingInstructionCSPResult(AliasModel):
fields = {
"reported_clin_name": "name",
}
AZURE_MGMNT_PATH = "/providers/Microsoft.Management/managementGroups/"
MANAGEMENT_GROUP_NAME_REGEX = "^[a-zA-Z0-9\-_\(\)\.]+$"
class ManagementGroupCSPPayload(AliasModel):
"""
:param: management_group_name: Just pass a UUID for this.
:param: display_name: This can contain any character and
spaces, but should be 90 characters or fewer long.
:param: parent_id: This should be the fully qualified Azure ID,
i.e. /providers/Microsoft.Management/managementGroups/[management group ID]
"""
tenant_id: str
management_group_name: Optional[str]
display_name: str
parent_id: str
@validator("management_group_name", pre=True, always=True)
def supply_management_group_name_default(cls, name):
if name:
if re.match(MANAGEMENT_GROUP_NAME_REGEX, name) is None:
raise ValueError(
f"Management group name must match {MANAGEMENT_GROUP_NAME_REGEX}"
)
return name[0:90]
else:
return str(uuid4())
@validator("display_name", pre=True, always=True)
def enforce_display_name_length(cls, name):
return name[0:90]
@validator("parent_id", pre=True, always=True)
def enforce_parent_id_pattern(cls, id_):
if AZURE_MGMNT_PATH not in id_:
return f"{AZURE_MGMNT_PATH}{id_}"
else:
return id_
class ManagementGroupCSPResponse(AliasModel):
id: str
class ApplicationCSPPayload(ManagementGroupCSPPayload):
pass
class ApplicationCSPResult(ManagementGroupCSPResponse):
pass
class KeyVaultCredentials(BaseModel):
root_sp_client_id: Optional[str]
root_sp_key: Optional[str]
root_tenant_id: Optional[str]
tenant_id: Optional[str]
tenant_admin_username: Optional[str]
tenant_admin_password: Optional[str]
tenant_sp_client_id: Optional[str]
tenant_sp_key: Optional[str]
@root_validator(pre=True)
def enforce_admin_creds(cls, values):
tenant_id = values.get("tenant_id")
username = values.get("tenant_admin_username")
password = values.get("tenant_admin_password")
if any([username, password]) and not all([tenant_id, username, password]):
raise ValueError(
"tenant_id, tenant_admin_username, and tenant_admin_password must all be set if any one is"
)
return values
@root_validator(pre=True)
def enforce_sp_creds(cls, values):
tenant_id = values.get("tenant_id")
client_id = values.get("tenant_sp_client_id")
key = values.get("tenant_sp_key")
if any([client_id, key]) and not all([tenant_id, client_id, key]):
raise ValueError(
"tenant_id, tenant_sp_client_id, and tenant_sp_key must all be set if any one is"
)
return values
@root_validator(pre=True)
def enforce_root_creds(cls, values):
sp_creds = [
values.get("root_tenant_id"),
values.get("root_sp_client_id"),
values.get("root_sp_key"),
]
if any(sp_creds) and not all(sp_creds):
raise ValueError(
"root_tenant_id, root_sp_client_id, and root_sp_key must all be set if any one is"
)
return values

View File

@@ -93,10 +93,13 @@ class Users(object):
return user
@classmethod
def give_ccpo_perms(cls, user):
def give_ccpo_perms(cls, user, commit=True):
user.permission_sets = PermissionSets.get_all()
db.session.add(user)
db.session.commit()
if commit:
db.session.commit()
return user
@classmethod

View File

@@ -10,11 +10,13 @@ from wtforms.fields.html5 import DateField
from wtforms.validators import (
Required,
Length,
Optional,
NumberRange,
ValidationError,
)
from flask_wtf import FlaskForm
import numbers
from atst.forms.validators import Number, AlphaNumeric
from .data import JEDI_CLIN_TYPES
@@ -60,6 +62,14 @@ def validate_date_in_range(form, field):
)
def remove_dashes(value):
return value.replace("-", "") if value else None
def coerce_upper(value):
return value.upper() if value else None
class CLINForm(FlaskForm):
jedi_clin_type = SelectField(
translate("task_orders.form.clin_type_label"),
@@ -149,8 +159,8 @@ class AttachmentForm(BaseForm):
class TaskOrderForm(BaseForm):
number = StringField(
label=translate("forms.task_order.number_description"),
filters=[remove_empty_string],
validators=[Number(), Length(max=13)],
filters=[remove_empty_string, remove_dashes, coerce_upper],
validators=[AlphaNumeric(), Length(min=13, max=17), Optional()],
)
pdf = FormField(
AttachmentForm,

View File

@@ -3,47 +3,38 @@ import pendulum
from atst.database import db
from atst.queue import celery
from atst.models import (
EnvironmentJobFailure,
EnvironmentRoleJobFailure,
EnvironmentRole,
PortfolioJobFailure,
)
from atst.models import EnvironmentRole, JobFailure
from atst.domain.csp.cloud.exceptions import GeneralCSPException
from atst.domain.csp.cloud import CloudProviderInterface
from atst.domain.applications import Applications
from atst.domain.environments import Environments
from atst.domain.portfolios import Portfolios
from atst.domain.environment_roles import EnvironmentRoles
from atst.models.utils import claim_for_update
from atst.utils.localization import translate
from atst.domain.csp.cloud.models import ApplicationCSPPayload
class RecordPortfolioFailure(celery.Task):
class RecordFailure(celery.Task):
_ENTITIES = [
"portfolio_id",
"application_id",
"environment_id",
"environment_role_id",
]
def _derive_entity_info(self, kwargs):
matches = [e for e in self._ENTITIES if e in kwargs.keys()]
if matches:
match = matches[0]
return {"entity": match.replace("_id", ""), "entity_id": kwargs[match]}
else:
return None
def on_failure(self, exc, task_id, args, kwargs, einfo):
if "portfolio_id" in kwargs:
failure = PortfolioJobFailure(
portfolio_id=kwargs["portfolio_id"], task_id=task_id
)
db.session.add(failure)
db.session.commit()
class RecordEnvironmentFailure(celery.Task):
def on_failure(self, exc, task_id, args, kwargs, einfo):
if "environment_id" in kwargs:
failure = EnvironmentJobFailure(
environment_id=kwargs["environment_id"], task_id=task_id
)
db.session.add(failure)
db.session.commit()
class RecordEnvironmentRoleFailure(celery.Task):
def on_failure(self, exc, task_id, args, kwargs, einfo):
if "environment_role_id" in kwargs:
failure = EnvironmentRoleJobFailure(
environment_role_id=kwargs["environment_role_id"], task_id=task_id
)
info = self._derive_entity_info(kwargs)
if info:
failure = JobFailure(**info, task_id=task_id)
db.session.add(failure)
db.session.commit()
@@ -63,6 +54,27 @@ def send_notification_mail(recipients, subject, body):
app.mailer.send(recipients, subject, body)
def do_create_application(csp: CloudProviderInterface, application_id=None):
application = Applications.get(application_id)
with claim_for_update(application) as application:
if application.cloud_id:
return
csp_details = application.portfolio.csp_data
parent_id = csp_details.get("root_management_group_id")
tenant_id = csp_details.get("tenant_id")
payload = ApplicationCSPPayload(
tenant_id=tenant_id, display_name=application.name, parent_id=parent_id
)
app_result = csp.create_application(payload)
application.cloud_id = app_result.id
db.session.add(application)
db.session.commit()
def do_create_environment(csp: CloudProviderInterface, environment_id=None):
environment = Environments.get(environment_id)
@@ -144,17 +156,22 @@ def do_provision_portfolio(csp: CloudProviderInterface, portfolio_id=None):
fsm.trigger_next_transition()
@celery.task(bind=True, base=RecordPortfolioFailure)
@celery.task(bind=True, base=RecordFailure)
def provision_portfolio(self, portfolio_id=None):
do_work(do_provision_portfolio, self, app.csp.cloud, portfolio_id=portfolio_id)
@celery.task(bind=True, base=RecordEnvironmentFailure)
@celery.task(bind=True, base=RecordFailure)
def create_application(self, application_id=None):
do_work(do_create_application, self, app.csp.cloud, application_id=application_id)
@celery.task(bind=True, base=RecordFailure)
def create_environment(self, environment_id=None):
do_work(do_create_environment, self, app.csp.cloud, environment_id=environment_id)
@celery.task(bind=True, base=RecordEnvironmentFailure)
@celery.task(bind=True, base=RecordFailure)
def create_atat_admin_user(self, environment_id=None):
do_work(
do_create_atat_admin_user, self, app.csp.cloud, environment_id=environment_id
@@ -177,6 +194,12 @@ def dispatch_provision_portfolio(self):
provision_portfolio.delay(portfolio_id=portfolio_id)
@celery.task(bind=True)
def dispatch_create_application(self):
for application_id in Applications.get_applications_pending_creation():
create_application.delay(application_id=application_id)
@celery.task(bind=True)
def dispatch_create_environment(self):
for environment_id in Environments.get_environments_pending_creation(

View File

@@ -7,11 +7,7 @@ from .audit_event import AuditEvent
from .clin import CLIN, JEDICLINType
from .environment import Environment
from .environment_role import EnvironmentRole, CSPRole
from .job_failure import (
EnvironmentJobFailure,
EnvironmentRoleJobFailure,
PortfolioJobFailure,
)
from .job_failure import JobFailure
from .notification_recipient import NotificationRecipient
from .permissions import Permissions
from .permission_set import PermissionSet

View File

@@ -1,4 +1,4 @@
from sqlalchemy import and_, Column, ForeignKey, String, UniqueConstraint
from sqlalchemy import and_, Column, ForeignKey, String, UniqueConstraint, TIMESTAMP
from sqlalchemy.orm import relationship, synonym
from atst.models.base import Base
@@ -40,6 +40,9 @@ class Application(
),
)
cloud_id = Column(String)
claimed_until = Column(TIMESTAMP(timezone=True))
@property
def users(self):
return set(role.user for role in self.members)

View File

@@ -30,8 +30,6 @@ class Environment(
claimed_until = Column(TIMESTAMP(timezone=True))
job_failures = relationship("EnvironmentJobFailure")
roles = relationship(
"EnvironmentRole",
back_populates="environment",

View File

@@ -32,8 +32,6 @@ class EnvironmentRole(
)
application_role = relationship("ApplicationRole")
job_failures = relationship("EnvironmentRoleJobFailure")
csp_user_id = Column(String())
claimed_until = Column(TIMESTAMP(timezone=True))

View File

@@ -1,22 +1,21 @@
from sqlalchemy import Column, ForeignKey
from celery.result import AsyncResult
from sqlalchemy import Column, String, Integer
from atst.models.base import Base
import atst.models.mixins as mixins
class EnvironmentJobFailure(Base, mixins.JobFailureMixin):
__tablename__ = "environment_job_failures"
class JobFailure(Base, mixins.TimestampsMixin):
__tablename__ = "job_failures"
environment_id = Column(ForeignKey("environments.id"), nullable=False)
id = Column(Integer(), primary_key=True)
task_id = Column(String(), nullable=False)
entity = Column(String(), nullable=False)
entity_id = Column(String(), nullable=False)
@property
def task(self):
if not hasattr(self, "_task"):
self._task = AsyncResult(self.task_id)
class EnvironmentRoleJobFailure(Base, mixins.JobFailureMixin):
__tablename__ = "environment_role_job_failures"
environment_role_id = Column(ForeignKey("environment_roles.id"), nullable=False)
class PortfolioJobFailure(Base, mixins.JobFailureMixin):
__tablename__ = "portfolio_job_failures"
portfolio_id = Column(ForeignKey("portfolios.id"), nullable=False)
return self._task

View File

@@ -3,5 +3,4 @@ from .auditable import AuditableMixin
from .permissions import PermissionsMixin
from .deletable import DeletableMixin
from .invites import InvitesMixin
from .job_failure import JobFailureMixin
from .state_machines import FSMMixin

View File

@@ -1,14 +0,0 @@
from celery.result import AsyncResult
from sqlalchemy import Column, String, Integer
class JobFailureMixin(object):
id = Column(Integer(), primary_key=True)
task_id = Column(String(), nullable=False)
@property
def task(self):
if not hasattr(self, "_task"):
self._task = AsyncResult(self.task_id)
return self._task

View File

@@ -175,7 +175,7 @@ class PortfolioStateMachine(
tenant_id = new_creds.get("tenant_id")
secret = self.csp.get_secret(tenant_id, new_creds)
secret.update(new_creds)
self.csp.set_secret(tenant_id, secret)
self.csp.update_tenant_creds(tenant_id, secret)
except PydanticValidationError as exc:
app.logger.error(
f"Failed to cast response to valid result class {self.__repr__()}:",

View File

@@ -11,6 +11,10 @@ def update_celery(celery, app):
"task": "atst.jobs.dispatch_provision_portfolio",
"schedule": 60,
},
"beat-dispatch_create_application": {
"task": "atst.jobs.dispatch_create_application",
"schedule": 60,
},
"beat-dispatch_create_environment": {
"task": "atst.jobs.dispatch_create_environment",
"schedule": 60,

View File

@@ -1,3 +1,4 @@
import hashlib
import re
from sqlalchemy.exc import IntegrityError
@@ -41,3 +42,8 @@ def commit_or_raise_already_exists_error(message):
except IntegrityError:
db.session.rollback()
raise AlreadyExistsError(message)
def sha256_hex(string):
hsh = hashlib.sha256(string.encode())
return hsh.digest().hex()