diff --git a/Pipfile b/Pipfile index 4030b90d..810da811 100644 --- a/Pipfile +++ b/Pipfile @@ -21,7 +21,7 @@ flask-wtf = "*" pyopenssl = "*" requests = "*" lockfile = "*" -werkzeug = "*" +werkzeug = "==0.16.1" PyYAML = "*" azure-storage = "*" azure-storage-common = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 8b3379bb..391f766d 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "faa5dab7bc6d13d39c0ef80f015f34a7fce2d66bec273ff38b7bd9ba232a3502" + "sha256": "44296f145fcb42cff5fadf14a706ec9598f4436ccbdf05e1d69fcd8316c89e8d" }, "pipfile-spec": 6, "requires": { @@ -164,10 +164,10 @@ }, "billiard": { "hashes": [ - "sha256:01afcb4e7c4fd6480940cfbd4d9edc19d7a7509d6ada533984d0d0f49901ec82", - "sha256:b8809c74f648dfe69b973c8e660bcec00603758c9db8ba89d7719f88d5f01f26" + "sha256:26fd494dc3251f8ce1f5559744f18aeed427fdaf29a75d7baae26752a5d3816f", + "sha256:f4e09366653aa3cb3ae8ed16423f9ba1665ff426f087bcdbbed86bf3664fe02c" ], - "version": "==3.6.1.0" + "version": "==3.6.2.0" }, "celery": { "hashes": [ @@ -582,11 +582,11 @@ }, "redis": { "hashes": [ - "sha256:7595976eb0b4e1fc3ad5478f1fd44215a814ee184a7820de92726f559bdff9cd", - "sha256:e933bdb504c69cbd5bdf4e2bb819a99644a36731cef4c59aa637cebfd5ddd4f9" + "sha256:0dcfb335921b88a850d461dc255ff4708294943322bd55de6cfd68972490ca1f", + "sha256:b205cffd05ebfd0a468db74f0eedbff8df1a7bfc47521516ade4692991bb0833" ], "index": "pypi", - "version": "==3.4.0" + "version": "==3.4.1" }, "requests": { "hashes": [ @@ -908,11 +908,11 @@ }, "ipython": { "hashes": [ - "sha256:0f4bcf18293fb666df8511feec0403bdb7e061a5842ea6e88a3177b0ceb34ead", - "sha256:387686dd7fc9caf29d2fddcf3116c4b07a11d9025701d220c589a430b0171d8a" + "sha256:d9459e7237e2e5858738ff9c3e26504b79899b58a6d49e574d352493d80684c6", + "sha256:f6689108b1734501d3b59c84427259fd5ac5141afe2e846cfa8598eb811886c9" ], "index": "pypi", - "version": "==7.11.1" + "version": "==7.12.0" }, "ipython-genutils": { "hashes": [ diff --git a/alembic/versions/17da2a475429_add_application_role_cloud_id.py b/alembic/versions/17da2a475429_add_application_role_cloud_id.py new file mode 100644 index 00000000..fffaaad8 --- /dev/null +++ b/alembic/versions/17da2a475429_add_application_role_cloud_id.py @@ -0,0 +1,29 @@ +"""add application_role.cloud_id + +Revision ID: 17da2a475429 +Revises: 50979d8ef680 +Create Date: 2020-02-01 10:43:03.073539 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '17da2a475429' # pragma: allowlist secret +down_revision = '50979d8ef680' # pragma: allowlist secret +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('application_roles', sa.Column('cloud_id', sa.String(), nullable=True)) + op.add_column('application_roles', sa.Column('claimed_until', sa.TIMESTAMP(timezone=True), nullable=True)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('application_roles', 'cloud_id') + op.drop_column('application_roles', 'claimed_until') + # ### end Alembic commands ### diff --git a/atst/domain/application_roles.py b/atst/domain/application_roles.py index 826f7f6c..1919b30f 100644 --- a/atst/domain/application_roles.py +++ b/atst/domain/application_roles.py @@ -1,8 +1,12 @@ +from itertools import groupby +from typing import List +from uuid import UUID + from sqlalchemy.orm.exc import NoResultFound from atst.database import db from atst.domain.environment_roles import EnvironmentRoles -from atst.models import ApplicationRole, ApplicationRoleStatus +from atst.models import Application, ApplicationRole, ApplicationRoleStatus, Portfolio from .permission_sets import PermissionSets from .exceptions import NotFoundError @@ -61,6 +65,15 @@ class ApplicationRoles(object): except NoResultFound: raise NotFoundError("application_role") + @classmethod + def get_many(cls, ids): + return ( + db.session.query(ApplicationRole) + .filter(ApplicationRole.id.in_(ids)) + .filter(ApplicationRole.status != ApplicationRoleStatus.DISABLED) + .all() + ) + @classmethod def update_permission_sets(cls, application_role, new_perm_sets_names): application_role.permission_sets = ApplicationRoles._permission_sets_for_names( @@ -92,3 +105,29 @@ class ApplicationRoles(object): db.session.add(application_role) db.session.commit() + + @classmethod + def get_pending_creation(cls) -> List[List[UUID]]: + """ + Returns a list of lists of ApplicationRole IDs. The IDs + should be grouped by user and portfolio. + """ + results = ( + db.session.query(ApplicationRole.id, ApplicationRole.user_id, Portfolio.id) + .join(Application, Application.id == ApplicationRole.application_id) + .join(Portfolio, Portfolio.id == Application.portfolio_id) + .filter(Application.cloud_id.isnot(None)) + .filter(ApplicationRole.deleted == False) + .filter(ApplicationRole.cloud_id.is_(None)) + .filter(ApplicationRole.user_id.isnot(None)) + .filter(ApplicationRole.status == ApplicationRoleStatus.ACTIVE) + ).all() + + groups = [] + keyfunc = lambda pair: (pair[1], pair[2]) + sorted_results = sorted(results, key=keyfunc) + for _, g in groupby(sorted_results, keyfunc): + group = [pair[0] for pair in list(g)] + groups.append(group) + + return groups diff --git a/atst/domain/csp/cloud/azure_cloud_provider.py b/atst/domain/csp/cloud/azure_cloud_provider.py index 0daa804c..5f97e161 100644 --- a/atst/domain/csp/cloud/azure_cloud_provider.py +++ b/atst/domain/csp/cloud/azure_cloud_provider.py @@ -6,7 +6,7 @@ from uuid import uuid4 from atst.utils import sha256_hex from .cloud_provider_interface import CloudProviderInterface -from .exceptions import AuthenticationException +from .exceptions import AuthenticationException, UserProvisioningException from .models import ( SubscriptionCreationCSPPayload, SubscriptionCreationCSPResult, @@ -24,6 +24,7 @@ from .models import ( BillingProfileTenantAccessCSPResult, BillingProfileVerificationCSPPayload, BillingProfileVerificationCSPResult, + CostManagementQueryCSPResult, KeyVaultCredentials, ManagementGroupCSPPayload, ManagementGroupCSPResponse, @@ -35,6 +36,7 @@ from .models import ( ProductPurchaseVerificationCSPResult, PrincipalAdminRoleCSPPayload, PrincipalAdminRoleCSPResult, + ReportingCSPPayload, TaskOrderBillingCreationCSPPayload, TaskOrderBillingCreationCSPResult, TaskOrderBillingVerificationCSPPayload, @@ -51,6 +53,8 @@ from .models import ( TenantPrincipalCSPResult, TenantPrincipalOwnershipCSPPayload, TenantPrincipalOwnershipCSPResult, + UserCSPPayload, + UserCSPResult, ) from .policy import AzurePolicyManager @@ -196,9 +200,9 @@ class AzureCloudProvider(CloudProviderInterface): creds = self._source_creds(payload.tenant_id) credentials = self._get_credential_obj( { - "client_id": creds.root_sp_client_id, - "secret_key": creds.root_sp_key, - "tenant_id": creds.root_tenant_id, + "client_id": creds.tenant_sp_client_id, + "secret_key": creds.tenant_sp_key, + "tenant_id": creds.tenant_id, }, resource=self.sdk.cloud.endpoints.resource_manager, ) @@ -352,7 +356,9 @@ class AzureCloudProvider(CloudProviderInterface): tenant_admin_password=payload.password, ), ) - return self._ok(TenantCSPResult(**result_dict)) + return self._ok( + TenantCSPResult(domain_name=payload.domain_name, **result_dict) + ) else: return self._error(result.json()) @@ -892,6 +898,80 @@ class AzureCloudProvider(CloudProviderInterface): return service_principal + def create_user(self, payload: UserCSPPayload) -> UserCSPResult: + """Create a user in an Azure Active Directory instance. + Unlike most of the methods on this interface, this requires + two API calls: one POST to create the user and one PATCH to + set the alternate email address. The email address cannot + be set on the first API call. The email address is + necessary so that users can do Self-Service Password + Recovery. + + Arguments: + payload {UserCSPPayload} -- a payload object with the + data necessary for both calls + + Returns: + UserCSPResult -- a result object containing the AAD ID. + """ + graph_token = self._get_tenant_principal_token( + payload.tenant_id, resource=self.graph_resource + ) + if graph_token is None: + raise AuthenticationException( + "Could not resolve graph token for tenant admin" + ) + + result = self._create_active_directory_user(graph_token, payload) + self._update_active_directory_user_email(graph_token, result.id, payload) + + return result + + def _create_active_directory_user(self, graph_token, payload: UserCSPPayload): + request_body = { + "accountEnabled": True, + "displayName": payload.display_name, + "mailNickname": payload.mail_nickname, + "userPrincipalName": payload.user_principal_name, + "passwordProfile": { + "forceChangePasswordNextSignIn": True, + "password": payload.password, + }, + } + + auth_header = { + "Authorization": f"Bearer {graph_token}", + } + + url = f"{self.graph_resource}v1.0/users" + + response = self.sdk.requests.post(url, headers=auth_header, json=request_body) + + if response.ok: + return UserCSPResult(**response.json()) + else: + raise UserProvisioningException(f"Failed to create user: {response.json()}") + + def _update_active_directory_user_email( + self, graph_token, user_id, payload: UserCSPPayload + ): + request_body = {"otherMails": [payload.email]} + + auth_header = { + "Authorization": f"Bearer {graph_token}", + } + + url = f"{self.graph_resource}v1.0/users/{user_id}" + + response = self.sdk.requests.patch(url, headers=auth_header, json=request_body) + + if response.ok: + return True + else: + raise UserProvisioningException( + f"Failed update user email: {response.json()}" + ) + def _extract_subscription_id(self, subscription_url): sub_id_match = SUBSCRIPTION_ID_REGEX.match(subscription_url) @@ -913,14 +993,15 @@ class AzureCloudProvider(CloudProviderInterface): creds.root_tenant_id, creds.root_sp_client_id, creds.root_sp_key ) - def _get_sp_token(self, tenant_id, client_id, secret_key): + def _get_sp_token(self, tenant_id, client_id, secret_key, resource=None): context = self.sdk.adal.AuthenticationContext( f"{self.sdk.cloud.endpoints.active_directory}/{tenant_id}" ) + resource = resource or self.sdk.cloud.endpoints.resource_manager # TODO: handle failure states here token_response = context.acquire_token_with_client_credentials( - self.sdk.cloud.endpoints.resource_manager, client_id, secret_key + resource, client_id, secret_key ) return token_response.get("accessToken", None) @@ -981,10 +1062,13 @@ class AzureCloudProvider(CloudProviderInterface): "tenant_id": self.tenant_id, } - def _get_tenant_principal_token(self, tenant_id): + def _get_tenant_principal_token(self, tenant_id, resource=None): creds = self._source_creds(tenant_id) return self._get_sp_token( - creds.tenant_id, creds.tenant_sp_client_id, creds.tenant_sp_key + creds.tenant_id, + creds.tenant_sp_client_id, + creds.tenant_sp_key, + resource=resource, ) def _get_elevated_management_token(self, tenant_id): @@ -1030,3 +1114,41 @@ class AzureCloudProvider(CloudProviderInterface): hashed = sha256_hex(tenant_id) raw_creds = self.get_secret(hashed) return KeyVaultCredentials(**json.loads(raw_creds)) + + def get_reporting_data(self, payload: ReportingCSPPayload): + """ + Queries the Cost Management API for an invoice section's raw reporting data + + We query at the invoiceSection scope. The full scope path is passed in + with the payload at the `invoice_section_id` key. + """ + creds = self._source_tenant_creds(payload.tenant_id) + token = self._get_sp_token( + payload.tenant_id, creds.tenant_sp_client_id, creds.tenant_sp_key + ) + + if not token: + raise AuthenticationException("Could not retrieve tenant access token") + + headers = {"Authorization": f"Bearer {token}"} + + request_body = { + "type": "Usage", + "timeframe": "Custom", + "timePeriod": {"from": payload.from_date, "to": payload.to_date,}, + "dataset": { + "granularity": "Daily", + "aggregation": {"totalCost": {"name": "PreTaxCost", "function": "Sum"}}, + "grouping": [{"type": "Dimension", "name": "InvoiceId"}], + }, + } + cost_mgmt_url = ( + f"/providers/Microsoft.CostManagement/query?api-version=2019-11-01" + ) + result = self.sdk.requests.post( + f"{self.sdk.cloud.endpoints.resource_manager}{payload.invoice_section_id}{cost_mgmt_url}", + json=request_body, + headers=headers, + ) + if result.ok: + return CostManagementQueryCSPResult(**result.json()) diff --git a/atst/domain/csp/cloud/exceptions.py b/atst/domain/csp/cloud/exceptions.py index 6ed47dff..49b05fb4 100644 --- a/atst/domain/csp/cloud/exceptions.py +++ b/atst/domain/csp/cloud/exceptions.py @@ -88,17 +88,6 @@ class UserProvisioningException(GeneralCSPException): """Failed to provision a user """ - def __init__(self, env_identifier, user_identifier, reason): - self.env_identifier = env_identifier - self.user_identifier = user_identifier - self.reason = reason - - @property - def message(self): - return "Failed to create user {} for environment {}: {}".format( - self.user_identifier, self.env_identifier, self.reason - ) - class UserRemovalException(GeneralCSPException): """Failed to remove a user diff --git a/atst/domain/csp/cloud/mock_cloud_provider.py b/atst/domain/csp/cloud/mock_cloud_provider.py index 0a573937..98e474bf 100644 --- a/atst/domain/csp/cloud/mock_cloud_provider.py +++ b/atst/domain/csp/cloud/mock_cloud_provider.py @@ -29,12 +29,15 @@ from .models import ( ManagementGroupCSPResponse, ManagementGroupGetCSPPayload, ManagementGroupGetCSPResponse, + CostManagementQueryCSPResult, + CostManagementQueryProperties, ProductPurchaseCSPPayload, ProductPurchaseCSPResult, ProductPurchaseVerificationCSPPayload, ProductPurchaseVerificationCSPResult, PrincipalAdminRoleCSPPayload, PrincipalAdminRoleCSPResult, + ReportingCSPPayload, SubscriptionCreationCSPPayload, SubscriptionCreationCSPResult, SubscriptionVerificationCSPPayload, @@ -55,6 +58,8 @@ from .models import ( TenantPrincipalCSPResult, TenantPrincipalOwnershipCSPPayload, TenantPrincipalOwnershipCSPResult, + UserCSPPayload, + UserCSPResult, ) @@ -179,6 +184,7 @@ class MockCloudProvider(CloudProviderInterface): "tenant_id": "", "user_id": "", "user_object_id": "", + "domain_name": "", "tenant_admin_username": "test", "tenant_admin_password": "test", } @@ -501,8 +507,35 @@ class MockCloudProvider(CloudProviderInterface): id=f"{AZURE_MGMNT_PATH}{payload.management_group_name}" ) + def create_user(self, payload: UserCSPPayload): + self._maybe_raise(self.UNAUTHORIZED_RATE, GeneralCSPException) + + return UserCSPResult(id=str(uuid4())) + def get_credentials(self, scope="portfolio", tenant_id=None): return self.root_creds() def update_tenant_creds(self, tenant_id, secret): return secret + + def get_reporting_data(self, payload: ReportingCSPPayload): + self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) + self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) + self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) + object_id = str(uuid4()) + + properties = CostManagementQueryProperties( + **dict( + columns=[ + {"name": "PreTaxCost", "type": "Number"}, + {"name": "UsageDate", "type": "Number"}, + {"name": "InvoiceId", "type": "String"}, + {"name": "Currency", "type": "String"}, + ], + rows=[], + ) + ) + + return CostManagementQueryCSPResult( + **dict(name=object_id, properties=properties,) + ) diff --git a/atst/domain/csp/cloud/models.py b/atst/domain/csp/cloud/models.py index bbf72815..ad095bf7 100644 --- a/atst/domain/csp/cloud/models.py +++ b/atst/domain/csp/cloud/models.py @@ -1,6 +1,7 @@ +from secrets import token_urlsafe from typing import Dict, List, Optional -import re from uuid import uuid4 +import re from pydantic import BaseModel, validator, root_validator @@ -39,6 +40,7 @@ class TenantCSPResult(AliasModel): user_id: str tenant_id: str user_object_id: str + domain_name: str tenant_admin_username: Optional[str] tenant_admin_password: Optional[str] @@ -484,3 +486,57 @@ class ProductPurchaseVerificationCSPPayload(BaseCSPPayload): class ProductPurchaseVerificationCSPResult(AliasModel): premium_purchase_date: str + + +class UserCSPPayload(BaseCSPPayload): + display_name: str + tenant_host_name: str + email: str + password: Optional[str] + + @property + def user_principal_name(self): + return f"{self.mail_nickname}@{self.tenant_host_name}.onmicrosoft.com" + + @property + def mail_nickname(self): + return self.display_name.replace(" ", ".").lower() + + @validator("password", pre=True, always=True) + def supply_password_default(cls, password): + return password or token_urlsafe(16) + + +class UserCSPResult(AliasModel): + id: str + + +class QueryColumn(AliasModel): + name: str + type: str + + +class CostManagementQueryProperties(AliasModel): + columns: List[QueryColumn] + rows: List[Optional[list]] + + +class CostManagementQueryCSPResult(AliasModel): + name: str + properties: CostManagementQueryProperties + + +class ReportingCSPPayload(BaseCSPPayload): + invoice_section_id: str + from_date: str + to_date: str + + @root_validator(pre=True) + def extract_invoice_section(cls, values): + try: + values["invoice_section_id"] = values["billing_profile_properties"][ + "invoice_sections" + ][0]["invoice_section_id"] + return values + except (KeyError, IndexError): + raise ValueError("Invoice section ID not present in payload") diff --git a/atst/domain/csp/files.py b/atst/domain/csp/files.py index 837cecbb..aade1775 100644 --- a/atst/domain/csp/files.py +++ b/atst/domain/csp/files.py @@ -79,7 +79,7 @@ class AzureFileService(FileService): sas_token = bbs.generate_blob_shared_access_signature( self.container_name, object_name, - permission=self.BlobPermissions.READ, + permission=self.BlobSasPermissions(read=True), expiry=datetime.utcnow() + self.timeout, content_disposition=f"attachment; filename={filename}", protocol="https", diff --git a/atst/domain/portfolios/portfolios.py b/atst/domain/portfolios/portfolios.py index 1254ac71..b8663730 100644 --- a/atst/domain/portfolios/portfolios.py +++ b/atst/domain/portfolios/portfolios.py @@ -15,6 +15,8 @@ from atst.models import ( Permissions, PortfolioRole, PortfolioRoleStatus, + TaskOrder, + CLIN, ) from .query import PortfoliosQuery, PortfolioStateMachinesQuery @@ -144,7 +146,7 @@ class Portfolios(object): return db.session.query(Portfolio.id) @classmethod - def get_portfolios_pending_provisioning(cls) -> List[UUID]: + def get_portfolios_pending_provisioning(cls, now) -> List[UUID]: """ Any portfolio with a corresponding State Machine that is either: not started yet, @@ -153,22 +155,18 @@ class Portfolios(object): """ results = ( - cls.base_provision_query() + db.session.query(Portfolio.id) .join(PortfolioStateMachine) + .join(TaskOrder) + .join(CLIN) + .filter(Portfolio.deleted == False) + .filter(CLIN.start_date <= now) + .filter(CLIN.end_date > now) .filter( or_( PortfolioStateMachine.state == FSMStates.UNSTARTED, - PortfolioStateMachine.state == FSMStates.FAILED, - PortfolioStateMachine.state == FSMStates.TENANT_FAILED, + PortfolioStateMachine.state.like("%CREATED"), ) ) ) return [id_ for id_, in results] - - # db.session.query(PortfolioStateMachine).\ - # filter( - # or_( - # PortfolioStateMachine.state==FSMStates.UNSTARTED, - # PortfolioStateMachine.state==FSMStates.UNSTARTED, - # ) - # ).all() diff --git a/atst/jobs.py b/atst/jobs.py index 14256336..ee820066 100644 --- a/atst/jobs.py +++ b/atst/jobs.py @@ -3,16 +3,16 @@ import pendulum from atst.database import db from atst.queue import celery -from atst.models import EnvironmentRole, JobFailure +from atst.models import JobFailure from atst.domain.csp.cloud.exceptions import GeneralCSPException from atst.domain.csp.cloud import CloudProviderInterface from atst.domain.applications import Applications from atst.domain.environments import Environments from atst.domain.portfolios import Portfolios -from atst.domain.environment_roles import EnvironmentRoles -from atst.models.utils import claim_for_update +from atst.domain.application_roles import ApplicationRoles +from atst.models.utils import claim_for_update, claim_many_for_update from atst.utils.localization import translate -from atst.domain.csp.cloud.models import ApplicationCSPPayload +from atst.domain.csp.cloud.models import ApplicationCSPPayload, UserCSPPayload class RecordFailure(celery.Task): @@ -75,6 +75,34 @@ def do_create_application(csp: CloudProviderInterface, application_id=None): db.session.commit() +def do_create_user(csp: CloudProviderInterface, application_role_ids=None): + if not application_role_ids: + return + + app_roles = ApplicationRoles.get_many(application_role_ids) + + with claim_many_for_update(app_roles) as app_roles: + + if any([ar.cloud_id for ar in app_roles]): + return + + csp_details = app_roles[0].application.portfolio.csp_data + user = app_roles[0].user + + payload = UserCSPPayload( + tenant_id=csp_details.get("tenant_id"), + tenant_host_name=csp_details.get("domain_name"), + display_name=user.full_name, + email=user.email, + ) + result = csp.create_user(payload) + for app_role in app_roles: + app_role.cloud_id = result.id + db.session.add(app_role) + + db.session.commit() + + def do_create_environment(csp: CloudProviderInterface, environment_id=None): environment = Environments.get(environment_id) @@ -128,21 +156,6 @@ def render_email(template_path, context): return app.jinja_env.get_template(template_path).render(context) -def do_provision_user(csp: CloudProviderInterface, environment_role_id=None): - environment_role = EnvironmentRoles.get_by_id(environment_role_id) - - with claim_for_update(environment_role) as environment_role: - credentials = environment_role.environment.csp_credentials - - csp_user_id = csp.create_or_update_user( - credentials, environment_role, environment_role.role - ) - environment_role.csp_user_id = csp_user_id - environment_role.status = EnvironmentRole.Status.COMPLETED - db.session.add(environment_role) - db.session.commit() - - def do_work(fn, task, csp, **kwargs): try: fn(csp, **kwargs) @@ -166,6 +179,13 @@ def create_application(self, application_id=None): do_work(do_create_application, self, app.csp.cloud, application_id=application_id) +@celery.task(bind=True, base=RecordFailure) +def create_user(self, application_role_ids=None): + do_work( + do_create_user, self, app.csp.cloud, application_role_ids=application_role_ids + ) + + @celery.task(bind=True, base=RecordFailure) def create_environment(self, environment_id=None): do_work(do_create_environment, self, app.csp.cloud, environment_id=environment_id) @@ -178,19 +198,12 @@ def create_atat_admin_user(self, environment_id=None): ) -@celery.task(bind=True) -def provision_user(self, environment_role_id=None): - do_work( - do_provision_user, self, app.csp.cloud, environment_role_id=environment_role_id - ) - - @celery.task(bind=True) def dispatch_provision_portfolio(self): """ Iterate over portfolios with a corresponding State Machine that have not completed. """ - for portfolio_id in Portfolios.get_portfolios_pending_provisioning(): + for portfolio_id in Portfolios.get_portfolios_pending_provisioning(pendulum.now()): provision_portfolio.delay(portfolio_id=portfolio_id) @@ -200,6 +213,12 @@ def dispatch_create_application(self): create_application.delay(application_id=application_id) +@celery.task(bind=True) +def dispatch_create_user(self): + for application_role_ids in ApplicationRoles.get_pending_creation(): + create_user.delay(application_role_ids=application_role_ids) + + @celery.task(bind=True) def dispatch_create_environment(self): for environment_id in Environments.get_environments_pending_creation( @@ -214,11 +233,3 @@ def dispatch_create_atat_admin_user(self): pendulum.now() ): create_atat_admin_user.delay(environment_id=environment_id) - - -@celery.task(bind=True) -def dispatch_provision_user(self): - for ( - environment_role_id - ) in EnvironmentRoles.get_environment_roles_pending_creation(): - provision_user.delay(environment_role_id=environment_role_id) diff --git a/atst/models/application.py b/atst/models/application.py index 1af9e39f..c8e8caf0 100644 --- a/atst/models/application.py +++ b/atst/models/application.py @@ -1,4 +1,4 @@ -from sqlalchemy import and_, Column, ForeignKey, String, UniqueConstraint, TIMESTAMP +from sqlalchemy import and_, Column, ForeignKey, String, UniqueConstraint from sqlalchemy.orm import relationship, synonym from atst.models.base import Base @@ -9,7 +9,11 @@ from atst.models.types import Id class Application( - Base, mixins.TimestampsMixin, mixins.AuditableMixin, mixins.DeletableMixin + Base, + mixins.TimestampsMixin, + mixins.AuditableMixin, + mixins.DeletableMixin, + mixins.ClaimableMixin, ): __tablename__ = "applications" @@ -41,7 +45,6 @@ class Application( ) cloud_id = Column(String) - claimed_until = Column(TIMESTAMP(timezone=True)) @property def users(self): diff --git a/atst/models/application_role.py b/atst/models/application_role.py index d65ceac7..68ed8512 100644 --- a/atst/models/application_role.py +++ b/atst/models/application_role.py @@ -1,5 +1,5 @@ from enum import Enum -from sqlalchemy import Index, ForeignKey, Column, Enum as SQLAEnum, Table +from sqlalchemy import Index, ForeignKey, Column, Enum as SQLAEnum, Table, String from sqlalchemy.dialects.postgresql import UUID from sqlalchemy.orm import relationship from sqlalchemy.event import listen @@ -33,6 +33,7 @@ class ApplicationRole( mixins.AuditableMixin, mixins.PermissionsMixin, mixins.DeletableMixin, + mixins.ClaimableMixin, ): __tablename__ = "application_roles" @@ -59,6 +60,8 @@ class ApplicationRole( primaryjoin="and_(EnvironmentRole.application_role_id == ApplicationRole.id, EnvironmentRole.deleted == False)", ) + cloud_id = Column(String) + @property def latest_invitation(self): if self.invitations: diff --git a/atst/models/environment.py b/atst/models/environment.py index a0713c63..b9d16fe0 100644 --- a/atst/models/environment.py +++ b/atst/models/environment.py @@ -1,4 +1,4 @@ -from sqlalchemy import Column, ForeignKey, String, TIMESTAMP, UniqueConstraint +from sqlalchemy import Column, ForeignKey, String, UniqueConstraint from sqlalchemy.orm import relationship from sqlalchemy.dialects.postgresql import JSONB from enum import Enum @@ -9,7 +9,11 @@ import atst.models.types as types class Environment( - Base, mixins.TimestampsMixin, mixins.AuditableMixin, mixins.DeletableMixin + Base, + mixins.TimestampsMixin, + mixins.AuditableMixin, + mixins.DeletableMixin, + mixins.ClaimableMixin, ): __tablename__ = "environments" @@ -28,8 +32,6 @@ class Environment( cloud_id = Column(String) root_user_info = Column(JSONB(none_as_null=True)) - claimed_until = Column(TIMESTAMP(timezone=True)) - roles = relationship( "EnvironmentRole", back_populates="environment", diff --git a/atst/models/environment_role.py b/atst/models/environment_role.py index 24aaeb7e..56fe78d4 100644 --- a/atst/models/environment_role.py +++ b/atst/models/environment_role.py @@ -1,5 +1,5 @@ from enum import Enum -from sqlalchemy import Index, ForeignKey, Column, String, TIMESTAMP, Enum as SQLAEnum +from sqlalchemy import Index, ForeignKey, Column, String, Enum as SQLAEnum from sqlalchemy.dialects.postgresql import UUID from sqlalchemy.orm import relationship @@ -15,7 +15,11 @@ class CSPRole(Enum): class EnvironmentRole( - Base, mixins.TimestampsMixin, mixins.AuditableMixin, mixins.DeletableMixin + Base, + mixins.TimestampsMixin, + mixins.AuditableMixin, + mixins.DeletableMixin, + mixins.ClaimableMixin, ): __tablename__ = "environment_roles" @@ -33,7 +37,6 @@ class EnvironmentRole( application_role = relationship("ApplicationRole") csp_user_id = Column(String()) - claimed_until = Column(TIMESTAMP(timezone=True)) class Status(Enum): PENDING = "pending" diff --git a/atst/models/mixins/__init__.py b/atst/models/mixins/__init__.py index e95b2516..36a5c43a 100644 --- a/atst/models/mixins/__init__.py +++ b/atst/models/mixins/__init__.py @@ -4,3 +4,4 @@ from .permissions import PermissionsMixin from .deletable import DeletableMixin from .invites import InvitesMixin from .state_machines import FSMMixin +from .claimable import ClaimableMixin diff --git a/atst/models/mixins/claimable.py b/atst/models/mixins/claimable.py new file mode 100644 index 00000000..afa33a59 --- /dev/null +++ b/atst/models/mixins/claimable.py @@ -0,0 +1,5 @@ +from sqlalchemy import Column, TIMESTAMP + + +class ClaimableMixin(object): + claimed_until = Column(TIMESTAMP(timezone=True)) diff --git a/atst/models/portfolio_state_machine.py b/atst/models/portfolio_state_machine.py index 14e9c01d..f5c1a461 100644 --- a/atst/models/portfolio_state_machine.py +++ b/atst/models/portfolio_state_machine.py @@ -175,11 +175,14 @@ class PortfolioStateMachine( app.logger.info(exc.json()) print(exc.json()) app.logger.info(payload_data) + # TODO: Ensure that failing the stage does not preclude a Celery retry self.fail_stage(stage) + # TODO: catch and handle general CSP exception here except (ConnectionException, UnknownServerException) as exc: app.logger.error( f"CSP api call. Caught exception for {self.__repr__()}.", exc_info=1, ) + # TODO: Ensure that failing the stage does not preclude a Celery retry self.fail_stage(stage) self.finish_stage(stage) diff --git a/atst/models/utils.py b/atst/models/utils.py index 6059d33e..7fba3206 100644 --- a/atst/models/utils.py +++ b/atst/models/utils.py @@ -1,3 +1,5 @@ +from typing import List + from sqlalchemy import func, sql, Interval, and_, or_ from contextlib import contextmanager @@ -28,7 +30,7 @@ def claim_for_update(resource, minutes=30): .filter( and_( Model.id == resource.id, - or_(Model.claimed_until == None, Model.claimed_until <= func.now()), + or_(Model.claimed_until.is_(None), Model.claimed_until <= func.now()), ) ) .update({"claimed_until": claim_until}, synchronize_session="fetch") @@ -48,3 +50,51 @@ def claim_for_update(resource, minutes=30): Model.claimed_until != None ).update({"claimed_until": None}, synchronize_session="fetch") db.session.commit() + + +@contextmanager +def claim_many_for_update(resources: List, minutes=30): + """ + Claim a mutually exclusive expiring hold on a group of resources. + Uses the database as a central source of time in case the server clocks have drifted. + + Args: + resources: A list of SQLAlchemy model instances with a `claimed_until` attribute. + minutes: The maximum amount of time, in minutes, to hold the claim. + """ + Model = resources[0].__class__ + + claim_until = func.now() + func.cast( + sql.functions.concat(minutes, " MINUTES"), Interval + ) + + ids = tuple(r.id for r in resources) + + # Optimistically query for and update the resources in question. If they're + # already claimed, `rows_updated` will be 0 and we can give up. + rows_updated = ( + db.session.query(Model) + .filter( + and_( + Model.id.in_(ids), + or_(Model.claimed_until.is_(None), Model.claimed_until <= func.now()), + ) + ) + .update({"claimed_until": claim_until}, synchronize_session="fetch") + ) + if rows_updated < 1: + # TODO: Generalize this exception class so it can take multiple resources + raise ClaimFailedException(resources[0]) + + # Fetch the claimed resources + claimed = db.session.query(Model).filter(Model.id.in_(ids)).all() + + try: + # Give the resource to the caller. + yield claimed + finally: + # Release the claim. + db.session.query(Model).filter(Model.id.in_(ids)).filter( + Model.claimed_until != None + ).update({"claimed_until": None}, synchronize_session="fetch") + db.session.commit() diff --git a/atst/queue.py b/atst/queue.py index 70718150..3f97f88c 100644 --- a/atst/queue.py +++ b/atst/queue.py @@ -23,8 +23,8 @@ def update_celery(celery, app): "task": "atst.jobs.dispatch_create_atat_admin_user", "schedule": 60, }, - "beat-dispatch_provision_user": { - "task": "atst.jobs.dispatch_provision_user", + "beat-dispatch_create_user": { + "task": "atst.jobs.dispatch_create_user", "schedule": 60, }, } diff --git a/atst/routes/applications/settings.py b/atst/routes/applications/settings.py index 8b744b04..8807c7f1 100644 --- a/atst/routes/applications/settings.py +++ b/atst/routes/applications/settings.py @@ -467,9 +467,10 @@ def revoke_invite(application_id, application_role_id): if invite.is_pending: ApplicationInvitations.revoke(invite.token) flash( - "application_invite_revoked", + "invite_revoked", + resource="Application", user_name=app_role.user_name, - application_name=g.application.name, + resource_name=g.application.name, ) else: flash( diff --git a/atst/routes/portfolios/invitations.py b/atst/routes/portfolios/invitations.py index 9ec56aa3..80375e75 100644 --- a/atst/routes/portfolios/invitations.py +++ b/atst/routes/portfolios/invitations.py @@ -37,8 +37,14 @@ def accept_invitation(portfolio_token): ) @user_can(Permissions.EDIT_PORTFOLIO_USERS, message="revoke invitation") def revoke_invitation(portfolio_id, portfolio_token): - PortfolioInvitations.revoke(portfolio_token) + invite = PortfolioInvitations.revoke(portfolio_token) + flash( + "invite_revoked", + resource="Portfolio", + user_name=invite.user_name, + resource_name=g.portfolio.name, + ) return redirect( url_for( "portfolios.admin", diff --git a/atst/utils/flash.py b/atst/utils/flash.py index ea85f1ef..b7ca0cb9 100644 --- a/atst/utils/flash.py +++ b/atst/utils/flash.py @@ -33,11 +33,6 @@ MESSAGES = { "message": "flash.application_invite.resent.message", "category": "success", }, - "application_invite_revoked": { - "title": "flash.application_invite.revoked.title", - "message": "flash.application_invite.revoked.message", - "category": "success", - }, "application_member_removed": { "title": "flash.application_member.removed.title", "message": "flash.application_member.removed.message", @@ -103,6 +98,11 @@ MESSAGES = { "message": None, "category": "warning", }, + "invite_revoked": { + "title": "flash.invite_revoked.title", + "message": "flash.invite_revoked.message", + "category": "success", + }, "logged_out": { "title": "flash.logged_out.title", "message": "flash.logged_out.message", diff --git a/js/components/sidenav_toggler.js b/js/components/sidenav_toggler.js index faba4c3b..11717849 100644 --- a/js/components/sidenav_toggler.js +++ b/js/components/sidenav_toggler.js @@ -1,30 +1,21 @@ +import ExpandSidenavMixin from '../mixins/expand_sidenav' import ToggleMixin from '../mixins/toggle' -const cookieName = 'expandSidenav' - export default { name: 'sidenav-toggler', - mixins: [ToggleMixin], + mixins: [ExpandSidenavMixin, ToggleMixin], - props: { - defaultVisible: { - type: Boolean, - default: function() { - if (document.cookie.match(cookieName)) { - return !!document.cookie.match(cookieName + ' *= *true') - } else { - return true - } - }, - }, + mounted: function() { + this.$parent.$emit('sidenavToggle', this.isVisible) }, methods: { toggle: function(e) { e.preventDefault() this.isVisible = !this.isVisible - document.cookie = cookieName + '=' + this.isVisible + '; path=/' + document.cookie = this.cookieName + '=' + this.isVisible + '; path=/' + this.$parent.$emit('sidenavToggle', this.isVisible) }, }, } diff --git a/js/components/upload_input.js b/js/components/upload_input.js index ae155c48..9856405b 100644 --- a/js/components/upload_input.js +++ b/js/components/upload_input.js @@ -2,6 +2,13 @@ import { buildUploader } from '../lib/upload' import { emitFieldChange } from '../lib/emitters' import inputValidations from '../lib/input_validations' +function uploadResponseOkay(response) { + // check BlobUploadCommonResponse: https://docs.microsoft.com/en-us/javascript/api/@azure/storage-blob/blobuploadcommonresponse?view=azure-node-latest + // The upload operation is a PUT that should return a 201 + // https://docs.microsoft.com/en-us/rest/api/storageservices/put-blob#status-code + return response._response.status === 201 +} + export default { name: 'uploadinput', @@ -21,7 +28,7 @@ export default { type: String, }, sizeLimit: { - type: String, + type: Number, }, }, @@ -34,7 +41,7 @@ export default { sizeError: false, filenameError: false, downloadLink: '', - fileSizeLimit: parseInt(this.sizeLimit), + fileSizeLimit: this.sizeLimit, } }, @@ -63,7 +70,7 @@ export default { const uploader = await this.getUploader() const response = await uploader.upload(file) - if (response.ok) { + if (uploadResponseOkay(response)) { this.attachment = e.target.value this.$refs.attachmentFilename.value = file.name this.$refs.attachmentObjectName.value = response.objectName @@ -73,7 +80,7 @@ export default { this.downloadLink = await this.getDownloadLink( file.name, - response.objectName + uploader.objectName ) } else { emitFieldChange(this) diff --git a/js/index.js b/js/index.js index fb5cdd6e..6495268b 100644 --- a/js/index.js +++ b/js/index.js @@ -32,12 +32,14 @@ import ToForm from './components/forms/to_form' import ClinFields from './components/clin_fields' import PopDateRange from './components/pop_date_range' import ToggleMenu from './components/toggle_menu' +import ExpandSidenav from './mixins/expand_sidenav' Vue.config.productionTip = false Vue.use(VTooltip) Vue.mixin(Modal) +Vue.mixin(ExpandSidenav) const app = new Vue({ el: '#app-root', @@ -67,6 +69,12 @@ const app = new Vue({ ToggleMenu, }, + data: function() { + return { + sidenavExpanded: this.defaultVisible, + } + }, + mounted: function() { this.$on('modalOpen', data => { if (data['isOpen']) { @@ -105,6 +113,10 @@ const app = new Vue({ } }) }) + + this.$on('sidenavToggle', data => { + this.sidenavExpanded = data + }) }, delimiters: ['!{', '}'], diff --git a/js/lib/upload.js b/js/lib/upload.js index ccbf9011..3f3df246 100644 --- a/js/lib/upload.js +++ b/js/lib/upload.js @@ -1,4 +1,4 @@ -import Azure from 'azure-storage' +import { BlobServiceClient } from '@azure/storage-blob' import 'whatwg-fetch' class AzureUploader { @@ -10,46 +10,22 @@ class AzureUploader { } async upload(file) { - const blobService = Azure.createBlobServiceWithSas( - `https://${this.accountName}.blob.core.windows.net`, - this.sasToken + const blobServiceClient = new BlobServiceClient( + `https://${this.accountName}.blob.core.windows.net?${this.sasToken}` ) - const fileReader = new FileReader() + const containerClient = blobServiceClient.getContainerClient( + this.containerName + ) + const blobClient = containerClient.getBlockBlobClient(this.objectName) const options = { - contentSettings: { - contentType: 'application/pdf', + blobHTTPHeaders: { + blobContentType: 'application/pdf', }, metadata: { filename: file.name, }, } - - return new Promise((resolve, reject) => { - fileReader.addEventListener('load', f => { - blobService.createBlockBlobFromText( - this.containerName, - `${this.objectName}`, - f.target.result, - options, - (err, result) => { - if (err) { - resolve({ ok: false }) - } else { - resolve({ ok: true, objectName: this.objectName }) - } - } - ) - }) - fileReader.readAsText(file) - }) - } - - downloadUrl(objectName) { - const blobService = Azure.createBlobServiceWithSas( - `https://${this.accountName}.blob.core.windows.net`, - this.sasToken - ) - return blobService.getUrl(this.containerName, objectName, this.sasToken) + return blobClient.uploadBrowserData(file, options) } } @@ -60,7 +36,8 @@ export class MockUploader { } async upload(file, objectName) { - return Promise.resolve({ ok: true, objectName: this.objectName }) + // mock BlobUploadCommonResponse structure: https://docs.microsoft.com/en-us/javascript/api/@azure/storage-blob/blobuploadcommonresponse?view=azure-node-latest + return Promise.resolve({ _response: { status: 201 } }) } } diff --git a/js/mixins/expand_sidenav.js b/js/mixins/expand_sidenav.js new file mode 100644 index 00000000..7553b7d4 --- /dev/null +++ b/js/mixins/expand_sidenav.js @@ -0,0 +1,15 @@ +export default { + props: { + cookieName: 'expandSidenav', + defaultVisible: { + type: Boolean, + default: function() { + if (document.cookie.match(this.cookieName)) { + return !!document.cookie.match(this.cookieName + ' *= *true') + } else { + return true + } + }, + }, + }, +} diff --git a/package.json b/package.json index 8521623c..f7aee1d1 100644 --- a/package.json +++ b/package.json @@ -14,9 +14,9 @@ "author": "", "license": "MIT", "dependencies": { + "@azure/storage-blob": "^12.0.2", "ally.js": "^1.4.1", "autoprefixer": "^9.1.3", - "azure-storage": "^2.10.3", "babel-polyfill": "^6.26.0", "date-fns": "^1.29.0", "ramda": "^0.25.0", diff --git a/styles/atat.scss b/styles/atat.scss index 72c7af40..8eb73473 100644 --- a/styles/atat.scss +++ b/styles/atat.scss @@ -47,3 +47,4 @@ @import "sections/application_edit"; @import "sections/reports"; @import "sections/task_order"; +@import "sections/ccpo"; diff --git a/styles/components/_alerts.scss b/styles/components/_alerts.scss index be326807..eb62a756 100644 --- a/styles/components/_alerts.scss +++ b/styles/components/_alerts.scss @@ -11,6 +11,7 @@ .usa-alert { padding-bottom: 2.4rem; + max-width: $max-panel-width; } @mixin alert { @@ -97,38 +98,3 @@ } } } - -.alert { - @include alert; - @include alert-level("info"); - - &.alert--success { - @include alert-level("success"); - - .alert__actions { - .icon-link { - @include icon-link-color($color-green, $color-white); - } - } - } - - &.alert--warning { - @include alert-level("warning"); - - .alert__actions { - .icon-link { - @include icon-link-color($color-gold-dark, $color-white); - } - } - } - - &.alert--error { - @include alert-level("error"); - - .alert__actions { - .icon-link { - @include icon-link-color($color-red, $color-white); - } - } - } -} diff --git a/styles/components/_error_page.scss b/styles/components/_error_page.scss index 19ae7531..683b527f 100644 --- a/styles/components/_error_page.scss +++ b/styles/components/_error_page.scss @@ -1,8 +1,13 @@ .error-page { - max-width: 475px; - margin: auto; + max-width: $max-page-width; .panel { + box-shadow: none; + background-color: unset; + border: none; + max-width: 475px; + margin: auto; + &__heading { text-align: center; padding: $gap 0; @@ -15,17 +20,6 @@ margin-bottom: $gap; } } - - &__body { - padding: $gap * 2; - margin: 0; - - hr { - width: 80%; - margin: auto; - margin-bottom: $gap * 3; - } - } } .icon { diff --git a/styles/components/_topbar.scss b/styles/components/_topbar.scss index 6d84f426..feca57b6 100644 --- a/styles/components/_topbar.scss +++ b/styles/components/_topbar.scss @@ -12,7 +12,7 @@ flex-direction: row; align-items: stretch; justify-content: space-between; - max-width: 1190px; + max-width: $max-page-width; a { color: $color-white; diff --git a/styles/core/_variables.scss b/styles/core/_variables.scss index 12657ca4..372fa868 100644 --- a/styles/core/_variables.scss +++ b/styles/core/_variables.scss @@ -19,6 +19,7 @@ $sidenav-collapsed-width: 10rem; $max-panel-width: 90rem; $home-pg-icon-width: 6rem; $large-spacing: 4rem; +$max-page-width: $max-panel-width + $sidenav-expanded-width + $large-spacing; /* * USWDS Variables diff --git a/styles/elements/_action_group.scss b/styles/elements/_action_group.scss index fe375f67..c2d11049 100644 --- a/styles/elements/_action_group.scss +++ b/styles/elements/_action_group.scss @@ -32,22 +32,35 @@ } .action-group-footer { - @extend .action-group; - - &:last-child { - margin-bottom: 0; - } - margin-top: 0; - margin-bottom: 0; padding-top: $gap; padding-bottom: $gap; - + padding-right: $gap * 4; position: fixed; bottom: $footer-height; + left: 0; background: white; - right: 0; - padding-right: $gap * 4; border-top: 1px solid $color-gray-lighter; - width: 100%; z-index: 1; + width: 100%; + + &.action-group-footer--expand-offset { + padding-left: $sidenav-expanded-width; + } + + &.action-group-footer--collapse-offset { + padding-left: $sidenav-collapsed-width; + } + + .action-group-footer--container { + @extend .action-group; + + margin-top: 0; + margin-bottom: 0; + margin-left: $large-spacing; + max-width: $max-panel-width; + + &:last-child { + margin-bottom: 0; + } + } } diff --git a/styles/sections/_ccpo.scss b/styles/sections/_ccpo.scss new file mode 100644 index 00000000..4033ac2e --- /dev/null +++ b/styles/sections/_ccpo.scss @@ -0,0 +1,3 @@ +.ccpo-panel-container { + max-width: $max-panel-width; +} diff --git a/templates/applications/new/step_1.html b/templates/applications/new/step_1.html index d06ddee0..3841bf96 100644 --- a/templates/applications/new/step_1.html +++ b/templates/applications/new/step_1.html @@ -39,14 +39,18 @@ -
+ diff --git a/templates/applications/new/step_2.html b/templates/applications/new/step_2.html index 462c0f46..2cd5cf98 100644 --- a/templates/applications/new/step_2.html +++ b/templates/applications/new/step_2.html @@ -58,20 +58,24 @@ {{ Icon("plus") }} + + + + - - diff --git a/templates/applications/new/step_3.html b/templates/applications/new/step_3.html index d88a704c..35af10fa 100644 --- a/templates/applications/new/step_3.html +++ b/templates/applications/new/step_3.html @@ -25,16 +25,20 @@ action_update="applications.update_new_application_step_3") }} - + {% endblock %} diff --git a/templates/ccpo/add_user.html b/templates/ccpo/add_user.html index e8544a2b..eccd27a9 100644 --- a/templates/ccpo/add_user.html +++ b/templates/ccpo/add_user.html @@ -4,21 +4,23 @@ {% from "components/text_input.html" import TextInput %} {% block content %} -