diff --git a/.secrets.baseline b/.secrets.baseline index a233e4cf..4e393738 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": "^.secrets.baseline$|^.*pgsslrootcert.yml$", "lines": null }, - "generated_at": "2020-01-27T19:24:43Z", + "generated_at": "2020-02-10T21:40:38Z", "plugins_used": [ { "base64_limit": 4.5, @@ -82,7 +82,7 @@ "hashed_secret": "afc848c316af1a89d49826c5ae9d00ed769415f3", "is_secret": false, "is_verified": false, - "line_number": 32, + "line_number": 43, "type": "Secret Keyword" } ], diff --git a/Dockerfile b/Dockerfile index 6f29d300..6dd7629a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -101,5 +101,7 @@ RUN mkdir /var/run/uwsgi && \ chown -R atst:atat /var/run/uwsgi && \ chown -R atst:atat "${APP_DIR}" +RUN update-ca-certificates + # Run as the unprivileged APP user USER atst diff --git a/alembic/versions/418b52c1cedf_change_to_environment_roles_cloud_id.py b/alembic/versions/418b52c1cedf_change_to_environment_roles_cloud_id.py new file mode 100644 index 00000000..93f11712 --- /dev/null +++ b/alembic/versions/418b52c1cedf_change_to_environment_roles_cloud_id.py @@ -0,0 +1,30 @@ +"""change to environment_roles.cloud_Id + +Revision ID: 418b52c1cedf +Revises: 542bd3215dec +Create Date: 2020-02-05 13:40:37.870183 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '418b52c1cedf' # pragma: allowlist secret +down_revision = '542bd3215dec' # pragma: allowlist secret +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('environment_roles', sa.Column('cloud_id', sa.String(), nullable=True)) + op.drop_column('environment_roles', 'csp_user_id') + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('environment_roles', sa.Column('csp_user_id', sa.VARCHAR(), autoincrement=False, nullable=True)) + op.drop_column('environment_roles', 'cloud_id') + # ### end Alembic commands ### diff --git a/alembic/versions/542bd3215dec_state_machine_stage_added.py b/alembic/versions/542bd3215dec_state_machine_stage_added.py new file mode 100644 index 00000000..8a3410af --- /dev/null +++ b/alembic/versions/542bd3215dec_state_machine_stage_added.py @@ -0,0 +1,264 @@ +"""state machine stage added. + +Revision ID: 542bd3215dec +Revises: 567bfb019a87 +Create Date: 2020-02-06 12:01:58.077840 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "542bd3215dec" # pragma: allowlist secret +down_revision = "567bfb019a87" # pragma: allowlist secret +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column( + "portfolio_state_machines", + "state", + type_=sa.Enum( + "UNSTARTED", + "STARTING", + "STARTED", + "COMPLETED", + "FAILED", + "TENANT_CREATED", + "TENANT_IN_PROGRESS", + "TENANT_FAILED", + "BILLING_PROFILE_CREATION_CREATED", + "BILLING_PROFILE_CREATION_IN_PROGRESS", + "BILLING_PROFILE_CREATION_FAILED", + "BILLING_PROFILE_VERIFICATION_CREATED", + "BILLING_PROFILE_VERIFICATION_IN_PROGRESS", + "BILLING_PROFILE_VERIFICATION_FAILED", + "BILLING_PROFILE_TENANT_ACCESS_CREATED", + "BILLING_PROFILE_TENANT_ACCESS_IN_PROGRESS", + "BILLING_PROFILE_TENANT_ACCESS_FAILED", + "TASK_ORDER_BILLING_CREATION_CREATED", + "TASK_ORDER_BILLING_CREATION_IN_PROGRESS", + "TASK_ORDER_BILLING_CREATION_FAILED", + "TASK_ORDER_BILLING_VERIFICATION_CREATED", + "TASK_ORDER_BILLING_VERIFICATION_IN_PROGRESS", + "TASK_ORDER_BILLING_VERIFICATION_FAILED", + "BILLING_INSTRUCTION_CREATED", + "BILLING_INSTRUCTION_IN_PROGRESS", + "BILLING_INSTRUCTION_FAILED", + "PRODUCT_PURCHASE_CREATED", + "PRODUCT_PURCHASE_IN_PROGRESS", + "PRODUCT_PURCHASE_FAILED", + "PRODUCT_PURCHASE_VERIFICATION_CREATED", + "PRODUCT_PURCHASE_VERIFICATION_IN_PROGRESS", + "PRODUCT_PURCHASE_VERIFICATION_FAILED", + "TENANT_PRINCIPAL_APP_CREATED", + "TENANT_PRINCIPAL_APP_IN_PROGRESS", + "TENANT_PRINCIPAL_APP_FAILED", + "TENANT_PRINCIPAL_CREATED", + "TENANT_PRINCIPAL_IN_PROGRESS", + "TENANT_PRINCIPAL_FAILED", + "TENANT_PRINCIPAL_CREDENTIAL_CREATED", + "TENANT_PRINCIPAL_CREDENTIAL_IN_PROGRESS", + "TENANT_PRINCIPAL_CREDENTIAL_FAILED", + "ADMIN_ROLE_DEFINITION_CREATED", + "ADMIN_ROLE_DEFINITION_IN_PROGRESS", + "ADMIN_ROLE_DEFINITION_FAILED", + "PRINCIPAL_ADMIN_ROLE_CREATED", + "PRINCIPAL_ADMIN_ROLE_IN_PROGRESS", + "PRINCIPAL_ADMIN_ROLE_FAILED", + "INITIAL_MGMT_GROUP_CREATED", + "INITIAL_MGMT_GROUP_IN_PROGRESS", + "INITIAL_MGMT_GROUP_FAILED", + "INITIAL_MGMT_GROUP_VERIFICATION_CREATED", + "INITIAL_MGMT_GROUP_VERIFICATION_IN_PROGRESS", + "INITIAL_MGMT_GROUP_VERIFICATION_FAILED", + "TENANT_ADMIN_OWNERSHIP_CREATED", + "TENANT_ADMIN_OWNERSHIP_IN_PROGRESS", + "TENANT_ADMIN_OWNERSHIP_FAILED", + "TENANT_PRINCIPAL_OWNERSHIP_CREATED", + "TENANT_PRINCIPAL_OWNERSHIP_IN_PROGRESS", + "TENANT_PRINCIPAL_OWNERSHIP_FAILED", + name="fsmstates", + native_enum=False, + ), + existing_type=sa.Enum( + "UNSTARTED", + "STARTING", + "STARTED", + "COMPLETED", + "FAILED", + "TENANT_CREATED", + "TENANT_IN_PROGRESS", + "TENANT_FAILED", + "BILLING_PROFILE_CREATION_CREATED", + "BILLING_PROFILE_CREATION_IN_PROGRESS", + "BILLING_PROFILE_CREATION_FAILED", + "BILLING_PROFILE_VERIFICATION_CREATED", + "BILLING_PROFILE_VERIFICATION_IN_PROGRESS", + "BILLING_PROFILE_VERIFICATION_FAILED", + "BILLING_PROFILE_TENANT_ACCESS_CREATED", + "BILLING_PROFILE_TENANT_ACCESS_IN_PROGRESS", + "BILLING_PROFILE_TENANT_ACCESS_FAILED", + "TASK_ORDER_BILLING_CREATION_CREATED", + "TASK_ORDER_BILLING_CREATION_IN_PROGRESS", + "TASK_ORDER_BILLING_CREATION_FAILED", + "TASK_ORDER_BILLING_VERIFICATION_CREATED", + "TASK_ORDER_BILLING_VERIFICATION_IN_PROGRESS", + "TASK_ORDER_BILLING_VERIFICATION_FAILED", + "BILLING_INSTRUCTION_CREATED", + "BILLING_INSTRUCTION_IN_PROGRESS", + "BILLING_INSTRUCTION_FAILED", + "TENANT_PRINCIPAL_APP_CREATED", + "TENANT_PRINCIPAL_APP_IN_PROGRESS", + "TENANT_PRINCIPAL_APP_FAILED", + "TENANT_PRINCIPAL_CREATED", + "TENANT_PRINCIPAL_IN_PROGRESS", + "TENANT_PRINCIPAL_FAILED", + "TENANT_PRINCIPAL_CREDENTIAL_CREATED", + "TENANT_PRINCIPAL_CREDENTIAL_IN_PROGRESS", + "TENANT_PRINCIPAL_CREDENTIAL_FAILED", + "ADMIN_ROLE_DEFINITION_CREATED", + "ADMIN_ROLE_DEFINITION_IN_PROGRESS", + "ADMIN_ROLE_DEFINITION_FAILED", + "PRINCIPAL_ADMIN_ROLE_CREATED", + "PRINCIPAL_ADMIN_ROLE_IN_PROGRESS", + "PRINCIPAL_ADMIN_ROLE_FAILED", + "TENANT_ADMIN_OWNERSHIP_CREATED", + "TENANT_ADMIN_OWNERSHIP_IN_PROGRESS", + "TENANT_ADMIN_OWNERSHIP_FAILED", + "TENANT_PRINCIPAL_OWNERSHIP_CREATED", + "TENANT_PRINCIPAL_OWNERSHIP_IN_PROGRESS", + "TENANT_PRINCIPAL_OWNERSHIP_FAILED", + name="fsmstates", + native_enum=False, + ), + existing_nullable=False, + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column( + "portfolio_state_machines", + "state", + type_=sa.Enum( + "UNSTARTED", + "STARTING", + "STARTED", + "COMPLETED", + "FAILED", + "TENANT_CREATED", + "TENANT_IN_PROGRESS", + "TENANT_FAILED", + "BILLING_PROFILE_CREATION_CREATED", + "BILLING_PROFILE_CREATION_IN_PROGRESS", + "BILLING_PROFILE_CREATION_FAILED", + "BILLING_PROFILE_VERIFICATION_CREATED", + "BILLING_PROFILE_VERIFICATION_IN_PROGRESS", + "BILLING_PROFILE_VERIFICATION_FAILED", + "BILLING_PROFILE_TENANT_ACCESS_CREATED", + "BILLING_PROFILE_TENANT_ACCESS_IN_PROGRESS", + "BILLING_PROFILE_TENANT_ACCESS_FAILED", + "TASK_ORDER_BILLING_CREATION_CREATED", + "TASK_ORDER_BILLING_CREATION_IN_PROGRESS", + "TASK_ORDER_BILLING_CREATION_FAILED", + "TASK_ORDER_BILLING_VERIFICATION_CREATED", + "TASK_ORDER_BILLING_VERIFICATION_IN_PROGRESS", + "TASK_ORDER_BILLING_VERIFICATION_FAILED", + "BILLING_INSTRUCTION_CREATED", + "BILLING_INSTRUCTION_IN_PROGRESS", + "BILLING_INSTRUCTION_FAILED", + "TENANT_PRINCIPAL_APP_CREATED", + "TENANT_PRINCIPAL_APP_IN_PROGRESS", + "TENANT_PRINCIPAL_APP_FAILED", + "TENANT_PRINCIPAL_CREATED", + "TENANT_PRINCIPAL_IN_PROGRESS", + "TENANT_PRINCIPAL_FAILED", + "TENANT_PRINCIPAL_CREDENTIAL_CREATED", + "TENANT_PRINCIPAL_CREDENTIAL_IN_PROGRESS", + "TENANT_PRINCIPAL_CREDENTIAL_FAILED", + "ADMIN_ROLE_DEFINITION_CREATED", + "ADMIN_ROLE_DEFINITION_IN_PROGRESS", + "ADMIN_ROLE_DEFINITION_FAILED", + "PRINCIPAL_ADMIN_ROLE_CREATED", + "PRINCIPAL_ADMIN_ROLE_IN_PROGRESS", + "PRINCIPAL_ADMIN_ROLE_FAILED", + "TENANT_ADMIN_OWNERSHIP_CREATED", + "TENANT_ADMIN_OWNERSHIP_IN_PROGRESS", + "TENANT_ADMIN_OWNERSHIP_FAILED", + "TENANT_PRINCIPAL_OWNERSHIP_CREATED", + "TENANT_PRINCIPAL_OWNERSHIP_IN_PROGRESS", + "TENANT_PRINCIPAL_OWNERSHIP_FAILED", + name="fsmstates", + native_enum=False, + ), + existing_type=sa.Enum( + "UNSTARTED", + "STARTING", + "STARTED", + "COMPLETED", + "FAILED", + "TENANT_CREATED", + "TENANT_IN_PROGRESS", + "TENANT_FAILED", + "BILLING_PROFILE_CREATION_CREATED", + "BILLING_PROFILE_CREATION_IN_PROGRESS", + "BILLING_PROFILE_CREATION_FAILED", + "BILLING_PROFILE_VERIFICATION_CREATED", + "BILLING_PROFILE_VERIFICATION_IN_PROGRESS", + "BILLING_PROFILE_VERIFICATION_FAILED", + "BILLING_PROFILE_TENANT_ACCESS_CREATED", + "BILLING_PROFILE_TENANT_ACCESS_IN_PROGRESS", + "BILLING_PROFILE_TENANT_ACCESS_FAILED", + "TASK_ORDER_BILLING_CREATION_CREATED", + "TASK_ORDER_BILLING_CREATION_IN_PROGRESS", + "TASK_ORDER_BILLING_CREATION_FAILED", + "TASK_ORDER_BILLING_VERIFICATION_CREATED", + "TASK_ORDER_BILLING_VERIFICATION_IN_PROGRESS", + "TASK_ORDER_BILLING_VERIFICATION_FAILED", + "BILLING_INSTRUCTION_CREATED", + "BILLING_INSTRUCTION_IN_PROGRESS", + "BILLING_INSTRUCTION_FAILED", + "PRODUCT_PURCHASE_CREATED", + "PRODUCT_PURCHASE_IN_PROGRESS", + "PRODUCT_PURCHASE_FAILED", + "PRODUCT_PURCHASE_VERIFICATION_CREATED", + "PRODUCT_PURCHASE_VERIFICATION_IN_PROGRESS", + "PRODUCT_PURCHASE_VERIFICATION_FAILED", + "TENANT_PRINCIPAL_APP_CREATED", + "TENANT_PRINCIPAL_APP_IN_PROGRESS", + "TENANT_PRINCIPAL_APP_FAILED", + "TENANT_PRINCIPAL_CREATED", + "TENANT_PRINCIPAL_IN_PROGRESS", + "TENANT_PRINCIPAL_FAILED", + "TENANT_PRINCIPAL_CREDENTIAL_CREATED", + "TENANT_PRINCIPAL_CREDENTIAL_IN_PROGRESS", + "TENANT_PRINCIPAL_CREDENTIAL_FAILED", + "ADMIN_ROLE_DEFINITION_CREATED", + "ADMIN_ROLE_DEFINITION_IN_PROGRESS", + "ADMIN_ROLE_DEFINITION_FAILED", + "PRINCIPAL_ADMIN_ROLE_CREATED", + "PRINCIPAL_ADMIN_ROLE_IN_PROGRESS", + "PRINCIPAL_ADMIN_ROLE_FAILED", + "INITIAL_MGMT_GROUP_CREATED", + "INITIAL_MGMT_GROUP_IN_PROGRESS", + "INITIAL_MGMT_GROUP_FAILED", + "INITIAL_MGMT_GROUP_VERIFICATION_CREATED", + "INITIAL_MGMT_GROUP_VERIFICATION_IN_PROGRESS", + "INITIAL_MGMT_GROUP_VERIFICATION_FAILED", + "TENANT_ADMIN_OWNERSHIP_CREATED", + "TENANT_ADMIN_OWNERSHIP_IN_PROGRESS", + "TENANT_ADMIN_OWNERSHIP_FAILED", + "TENANT_PRINCIPAL_OWNERSHIP_CREATED", + "TENANT_PRINCIPAL_OWNERSHIP_IN_PROGRESS", + "TENANT_PRINCIPAL_OWNERSHIP_FAILED", + name="fsmstates", + native_enum=False, + ), + existing_nullable=False, + ) + diff --git a/alembic/versions/567bfb019a87_add_last_sent_column_to_clins_and_pdf_.py b/alembic/versions/567bfb019a87_add_last_sent_column_to_clins_and_pdf_.py new file mode 100644 index 00000000..e56997ee --- /dev/null +++ b/alembic/versions/567bfb019a87_add_last_sent_column_to_clins_and_pdf_.py @@ -0,0 +1,29 @@ +"""add last_sent column to clins and pdf_last_sent to task_orders + +Revision ID: 567bfb019a87 +Revises: 0039308c6351 +Create Date: 2020-01-31 14:06:21.926019 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '567bfb019a87' # pragma: allowlist secret +down_revision = '0039308c6351' # pragma: allowlist secret +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('clins', sa.Column('last_sent_at', sa.DateTime(), nullable=True)) + op.add_column('task_orders', sa.Column('pdf_last_sent_at', sa.DateTime(), nullable=True)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('task_orders', 'pdf_last_sent_at') + op.drop_column('clins', 'last_sent_at') + # ### end Alembic commands ### diff --git a/atst/app.py b/atst/app.py index 05578827..04aed44d 100644 --- a/atst/app.py +++ b/atst/app.py @@ -157,7 +157,6 @@ def map_config(config): **config["default"], "USE_AUDIT_LOG": config["default"].getboolean("USE_AUDIT_LOG"), "ENV": config["default"]["ENVIRONMENT"], - "BROKER_URL": config["default"]["REDIS_URI"], "DEBUG": config["default"].getboolean("DEBUG"), "DEBUG_MAILER": config["default"].getboolean("DEBUG_MAILER"), "SQLALCHEMY_ECHO": config["default"].getboolean("SQLALCHEMY_ECHO"), @@ -233,13 +232,34 @@ def make_config(direct_config=None): config.set("default", "DATABASE_URI", database_uri) # Assemble REDIS_URI value + redis_use_tls = config["default"].getboolean("REDIS_TLS") redis_uri = "redis{}://{}:{}@{}".format( # pragma: allowlist secret - ("s" if config["default"].getboolean("REDIS_TLS") else ""), + ("s" if redis_use_tls else ""), (config.get("default", "REDIS_USER") or ""), (config.get("default", "REDIS_PASSWORD") or ""), config.get("default", "REDIS_HOST"), ) + celery_uri = redis_uri + if redis_use_tls: + tls_mode = config.get("default", "REDIS_SSLMODE") + tls_mode_str = tls_mode.lower() if tls_mode else "none" + redis_uri = f"{redis_uri}/?ssl_cert_reqs={tls_mode_str}" + + # TODO: Kombu, one of Celery's dependencies, still requires + # that ssl_cert_reqs be passed as the string version of an + # option on the ssl module. We can clean this up and use + # the REDIS_URI for both when this PR to Kombu is released: + # https://github.com/celery/kombu/pull/1139 + kombu_modes = { + "none": "CERT_NONE", + "required": "CERT_REQUIRED", + "optional": "CERT_OPTIONAL", + } + celery_tls_mode_str = kombu_modes[tls_mode_str] + celery_uri = f"{celery_uri}/?ssl_cert_reqs={celery_tls_mode_str}" + config.set("default", "REDIS_URI", redis_uri) + config.set("default", "BROKER_URL", celery_uri) return map_config(config) diff --git a/atst/domain/csp/cloud/azure_cloud_provider.py b/atst/domain/csp/cloud/azure_cloud_provider.py index 7ab53806..15c1fc3e 100644 --- a/atst/domain/csp/cloud/azure_cloud_provider.py +++ b/atst/domain/csp/cloud/azure_cloud_provider.py @@ -1,6 +1,5 @@ import json from secrets import token_urlsafe -from typing import Any, Dict from uuid import uuid4 import pydantic from flask import current_app as app @@ -29,6 +28,10 @@ from .models import ( BillingProfileVerificationCSPPayload, BillingProfileVerificationCSPResult, CostManagementQueryCSPResult, + InitialMgmtGroupCSPPayload, + InitialMgmtGroupCSPResult, + InitialMgmtGroupVerificationCSPPayload, + InitialMgmtGroupVerificationCSPResult, EnvironmentCSPPayload, EnvironmentCSPResult, KeyVaultCredentials, @@ -61,6 +64,8 @@ from .models import ( TenantPrincipalOwnershipCSPResult, UserCSPPayload, UserCSPResult, + UserRoleCSPPayload, + UserRoleCSPResult, ) from .policy import AzurePolicyManager @@ -107,10 +112,14 @@ class AzureCloudProvider(CloudProviderInterface): self.secret_key = config["AZURE_SECRET_KEY"] self.tenant_id = config["AZURE_TENANT_ID"] self.vault_url = config["AZURE_VAULT_URL"] - self.ps_client_id = config["POWERSHELL_CLIENT_ID"] - self.owner_role_def_id = config["AZURE_OWNER_ROLE_DEF_ID"] + self.ps_client_id = config["AZURE_POWERSHELL_CLIENT_ID"] self.graph_resource = config["AZURE_GRAPH_RESOURCE"] self.default_aadp_qty = config["AZURE_AADP_QTY"] + self.roles = { + "owner": config["AZURE_ROLE_DEF_ID_OWNER"], + "contributor": config["AZURE_ROLE_DEF_ID_CONTRIBUTOR"], + "billing": config["AZURE_ROLE_DEF_ID_BILLING_READER"], + } if azure_sdk_provider is None: self.sdk = AzureSDKProvider() @@ -193,6 +202,38 @@ class AzureCloudProvider(CloudProviderInterface): return ApplicationCSPResult(**response) + def create_initial_mgmt_group(self, payload: InitialMgmtGroupCSPPayload): + creds = self._source_creds(payload.tenant_id) + credentials = self._get_credential_obj( + { + "client_id": creds.root_sp_client_id, + "secret_key": creds.root_sp_key, + "tenant_id": creds.root_tenant_id, + }, + resource=self.sdk.cloud.endpoints.resource_manager, + ) + response = self._create_management_group( + credentials, payload.management_group_name, payload.display_name, + ) + + return InitialMgmtGroupCSPResult(**response) + + def create_initial_mgmt_group_verification( + self, payload: InitialMgmtGroupVerificationCSPPayload + ): + creds = self._source_creds(payload.tenant_id) + credentials = self._get_credential_obj( + { + "client_id": creds.root_sp_client_id, + "secret_key": creds.root_sp_key, + "tenant_id": creds.root_tenant_id, + }, + resource=self.sdk.cloud.endpoints.resource_manager, + ) + + response = self._get_management_group(credentials, payload.tenant_id,) + return InitialMgmtGroupVerificationCSPResult(**response.result()) + def _create_management_group( self, credentials, management_group_id, display_name, parent_id=None, ): @@ -219,6 +260,11 @@ class AzureCloudProvider(CloudProviderInterface): # instead? return create_request.result() + def _get_management_group(self, credentials, management_group_id): + mgmgt_group_client = self.sdk.managementgroups.ManagementGroupsAPI(credentials) + response = mgmgt_group_client.management_groups.get(management_group_id) + return response + def _create_policy_definition( self, credentials, subscription_id, management_group_id, properties, ): @@ -720,7 +766,7 @@ class AzureCloudProvider(CloudProviderInterface): def create_tenant_admin_ownership(self, payload: TenantAdminOwnershipCSPPayload): mgmt_token = self._get_elevated_management_token(payload.tenant_id) - role_definition_id = f"/providers/Microsoft.Management/managementGroups/{payload.tenant_id}/providers/Microsoft.Authorization/roleDefinitions/{self.owner_role_def_id}" + role_definition_id = f"/providers/Microsoft.Management/managementGroups/{payload.tenant_id}/providers/Microsoft.Authorization/roleDefinitions/{self.roles['owner']}" request_body = { "properties": { @@ -764,7 +810,7 @@ class AzureCloudProvider(CloudProviderInterface): mgmt_token = self._get_elevated_management_token(payload.tenant_id) # NOTE: the tenant_id is also the id of the root management group, once it is created - role_definition_id = f"/providers/Microsoft.Management/managementGroups/{payload.tenant_id}/providers/Microsoft.Authorization/roleDefinitions/{self.owner_role_def_id}" + role_definition_id = f"/providers/Microsoft.Management/managementGroups/{payload.tenant_id}/providers/Microsoft.Authorization/roleDefinitions/{self.roles['owner']}" request_body = { "properties": { @@ -1173,6 +1219,40 @@ class AzureCloudProvider(CloudProviderInterface): except self.sdk.requests.exceptions.HTTPError: raise UnknownServerException("azure application error creating tenant") + def create_user_role(self, payload: UserRoleCSPPayload): + graph_token = self._get_tenant_principal_token(payload.tenant_id) + if graph_token is None: + raise AuthenticationException( + "Could not resolve graph token for tenant admin" + ) + + role_guid = self.roles[payload.role] + role_definition_id = f"/providers/Microsoft.Management/managementGroups/{payload.management_group_id}/providers/Microsoft.Authorization/roleDefinitions/{role_guid}" + + request_body = { + "properties": { + "roleDefinitionId": role_definition_id, + "principalId": payload.user_object_id, + } + } + + auth_header = { + "Authorization": f"Bearer {graph_token}", + } + + assignment_guid = str(uuid4()) + + url = f"{self.sdk.cloud.endpoints.resource_manager}/providers/Microsoft.Management/managementGroups/{payload.management_group_id}/providers/Microsoft.Authorization/roleAssignments/{assignment_guid}?api-version=2015-07-01" + + response = self.sdk.requests.put(url, headers=auth_header, json=request_body) + + if response.ok: + return UserRoleCSPResult(**response.json()) + else: + raise UserProvisioningException( + f"Failed to create user role assignment: {response.json()}" + ) + def _extract_subscription_id(self, subscription_url): sub_id_match = SUBSCRIPTION_ID_REGEX.match(subscription_url) @@ -1300,12 +1380,10 @@ class AzureCloudProvider(CloudProviderInterface): def update_tenant_creds(self, tenant_id, secret: KeyVaultCredentials): hashed = sha256_hex(tenant_id) - new_secrets = secret.dict() curr_secrets = self._source_tenant_creds(tenant_id) - updated_secrets: Dict[str, Any] = {**curr_secrets.dict(), **new_secrets} - us = KeyVaultCredentials(**updated_secrets) - self.set_secret(hashed, json.dumps(us.dict())) - return us + updated_secrets = curr_secrets.merge_credentials(secret) + self.set_secret(hashed, json.dumps(updated_secrets.dict())) + return updated_secrets def _source_tenant_creds(self, tenant_id) -> KeyVaultCredentials: hashed = sha256_hex(tenant_id) @@ -1334,7 +1412,7 @@ class AzureCloudProvider(CloudProviderInterface): "timeframe": "Custom", "timePeriod": {"from": payload.from_date, "to": payload.to_date,}, "dataset": { - "granularity": "Daily", + "granularity": "Monthly", "aggregation": {"totalCost": {"name": "PreTaxCost", "function": "Sum"}}, "grouping": [{"type": "Dimension", "name": "InvoiceId"}], }, diff --git a/atst/domain/csp/cloud/cloud_provider_interface.py b/atst/domain/csp/cloud/cloud_provider_interface.py index 88b55f96..250ac6ef 100644 --- a/atst/domain/csp/cloud/cloud_provider_interface.py +++ b/atst/domain/csp/cloud/cloud_provider_interface.py @@ -1,7 +1,7 @@ from typing import Dict -class CloudProviderInterface: +class CloudProviderInterface: # pragma: no cover def set_secret(self, secret_key: str, secret_value: str): raise NotImplementedError() diff --git a/atst/domain/csp/cloud/mock_cloud_provider.py b/atst/domain/csp/cloud/mock_cloud_provider.py index 2d646145..c2844d40 100644 --- a/atst/domain/csp/cloud/mock_cloud_provider.py +++ b/atst/domain/csp/cloud/mock_cloud_provider.py @@ -1,4 +1,5 @@ from uuid import uuid4 +import pendulum from .cloud_provider_interface import CloudProviderInterface from .exceptions import ( @@ -23,6 +24,10 @@ from .models import ( BillingProfileTenantAccessCSPResult, BillingProfileVerificationCSPPayload, BillingProfileVerificationCSPResult, + InitialMgmtGroupCSPPayload, + InitialMgmtGroupCSPResult, + InitialMgmtGroupVerificationCSPPayload, + InitialMgmtGroupVerificationCSPResult, CostManagementQueryCSPResult, CostManagementQueryProperties, ProductPurchaseCSPPayload, @@ -276,6 +281,29 @@ class MockCloudProvider(CloudProviderInterface): } ) + def create_initial_mgmt_group(self, payload: InitialMgmtGroupCSPPayload): + self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) + self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) + self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) + + return InitialMgmtGroupCSPResult( + id=f"{AZURE_MGMNT_PATH}{payload.management_group_name}", + ) + + def create_initial_mgmt_group_verification( + self, payload: InitialMgmtGroupVerificationCSPPayload + ): + self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) + self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) + self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) + + return InitialMgmtGroupVerificationCSPResult( + **dict( + id="Test Id" + # id=f"{AZURE_MGMNT_PATH}{payload.management_group_name}" + ) + ) + def create_product_purchase(self, payload: ProductPurchaseCSPPayload): self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) @@ -455,15 +483,26 @@ class MockCloudProvider(CloudProviderInterface): self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) object_id = str(uuid4()) + start_of_month = pendulum.today(tz="utc").start_of("month").replace(tzinfo=None) + this_month = start_of_month.to_atom_string() + last_month = start_of_month.subtract(months=1).to_atom_string() + two_months_ago = start_of_month.subtract(months=2).to_atom_string() + properties = CostManagementQueryProperties( **dict( columns=[ {"name": "PreTaxCost", "type": "Number"}, - {"name": "UsageDate", "type": "Number"}, + {"name": "BillingMonth", "type": "Datetime"}, {"name": "InvoiceId", "type": "String"}, {"name": "Currency", "type": "String"}, ], - rows=[], + rows=[ + [1.0, two_months_ago, "", "USD"], + [500.0, two_months_ago, "e05009w9sf", "USD"], + [50.0, last_month, "", "USD"], + [1000.0, last_month, "e0500a4qhw", "USD"], + [500.0, this_month, "", "USD"], + ], ) ) diff --git a/atst/domain/csp/cloud/models.py b/atst/domain/csp/cloud/models.py index 358f7934..74702309 100644 --- a/atst/domain/csp/cloud/models.py +++ b/atst/domain/csp/cloud/models.py @@ -1,3 +1,4 @@ +from enum import Enum from secrets import token_urlsafe from typing import Dict, List, Optional from uuid import uuid4 @@ -320,7 +321,7 @@ class ManagementGroupCSPPayload(AliasModel): tenant_id: str management_group_name: Optional[str] display_name: str - parent_id: str + parent_id: Optional[str] @validator("management_group_name", pre=True, always=True) def supply_management_group_name_default(cls, name): @@ -340,16 +341,25 @@ class ManagementGroupCSPPayload(AliasModel): @validator("parent_id", pre=True, always=True) def enforce_parent_id_pattern(cls, id_): - if AZURE_MGMNT_PATH not in id_: - return f"{AZURE_MGMNT_PATH}{id_}" - else: - return id_ + if id_: + if AZURE_MGMNT_PATH not in id_: + return f"{AZURE_MGMNT_PATH}{id_}" + else: + return id_ class ManagementGroupCSPResponse(AliasModel): id: str +class ManagementGroupGetCSPPayload(BaseCSPPayload): + management_group_name: str + + +class ManagementGroupGetCSPResponse(AliasModel): + id: str + + class ApplicationCSPPayload(ManagementGroupCSPPayload): pass @@ -358,6 +368,22 @@ class ApplicationCSPResult(ManagementGroupCSPResponse): pass +class InitialMgmtGroupCSPPayload(ManagementGroupCSPPayload): + pass + + +class InitialMgmtGroupCSPResult(ManagementGroupCSPResponse): + pass + + +class InitialMgmtGroupVerificationCSPPayload(ManagementGroupGetCSPPayload): + pass + + +class InitialMgmtGroupVerificationCSPResult(ManagementGroupGetCSPResponse): + pass + + class EnvironmentCSPPayload(ManagementGroupCSPPayload): pass @@ -417,6 +443,15 @@ class KeyVaultCredentials(BaseModel): return values + def merge_credentials( + self, new_creds: "KeyVaultCredentials" + ) -> "KeyVaultCredentials": + updated_creds = {k: v for k, v in new_creds.dict().items() if v} + old_creds = self.dict() + old_creds.update(updated_creds) + + return KeyVaultCredentials(**old_creds) + class SubscriptionCreationCSPPayload(BaseCSPPayload): display_name: str @@ -509,6 +544,21 @@ class UserCSPResult(AliasModel): id: str +class UserRoleCSPPayload(BaseCSPPayload): + class Roles(str, Enum): + owner = "owner" + contributor = "contributor" + billing = "billing" + + management_group_id: str + role: Roles + user_object_id: str + + +class UserRoleCSPResult(AliasModel): + id: str + + class QueryColumn(AliasModel): name: str type: str diff --git a/atst/domain/csp/files.py b/atst/domain/csp/files.py index aade1775..0f3e05a0 100644 --- a/atst/domain/csp/files.py +++ b/atst/domain/csp/files.py @@ -43,10 +43,12 @@ class AzureFileService(FileService): from azure.storage.common import CloudStorageAccount from azure.storage.blob import BlobSasPermissions + from azure.storage.blob.models import BlobPermissions from azure.storage.blob.blockblobservice import BlockBlobService self.CloudStorageAccount = CloudStorageAccount self.BlobSasPermissions = BlobSasPermissions + self.BlobPermissions = BlobPermissions self.BlockBlobService = BlockBlobService def get_token(self): @@ -72,20 +74,22 @@ class AzureFileService(FileService): return ({"token": sas_token}, object_name) def generate_download_link(self, object_name, filename): - account = self.CloudStorageAccount( + block_blob_service = self.BlockBlobService( account_name=self.account_name, account_key=self.storage_key ) - bbs = account.create_block_blob_service() - sas_token = bbs.generate_blob_shared_access_signature( - self.container_name, - object_name, - permission=self.BlobSasPermissions(read=True), + sas_token = block_blob_service.generate_blob_shared_access_signature( + container_name=self.container_name, + blob_name=object_name, + permission=self.BlobPermissions(read=True), expiry=datetime.utcnow() + self.timeout, content_disposition=f"attachment; filename={filename}", protocol="https", ) - return bbs.make_blob_url( - self.container_name, object_name, protocol="https", sas_token=sas_token + return block_blob_service.make_blob_url( + container_name=self.container_name, + blob_name=object_name, + protocol="https", + sas_token=sas_token, ) def download_task_order(self, object_name): diff --git a/atst/domain/csp/reports.py b/atst/domain/csp/reports.py index 3f9ccbf8..700947f7 100644 --- a/atst/domain/csp/reports.py +++ b/atst/domain/csp/reports.py @@ -1,6 +1,6 @@ -from collections import defaultdict import json from decimal import Decimal +import pendulum def load_fixture_data(): @@ -11,128 +11,25 @@ def load_fixture_data(): class MockReportingProvider: FIXTURE_SPEND_DATA = load_fixture_data() - @classmethod - def get_portfolio_monthly_spending(cls, portfolio): - """ - returns an array of application and environment spending for the - portfolio. Applications and their nested environments are sorted in - alphabetical order by name. - [ - { - name - this_month - last_month - total - environments [ - { - name - this_month - last_month - total - } - ] - } - ] - """ - fixture_apps = cls.FIXTURE_SPEND_DATA.get(portfolio.name, {}).get( - "applications", [] - ) +def prepare_azure_reporting_data(rows: list): + """ + Returns a dict representing invoiced and estimated funds for a portfolio given + a list of rows from CostManagementQueryCSPResult.properties.rows + { + invoiced: Decimal, + estimated: Decimal + } + """ - for application in portfolio.applications: - if application.name not in [app["name"] for app in fixture_apps]: - fixture_apps.append({"name": application.name, "environments": []}) + estimated = [] + while rows: + if pendulum.parse(rows[-1][1]) >= pendulum.now(tz="utc").start_of("month"): + estimated.append(rows.pop()) + else: + break - return sorted( - [ - cls._get_application_monthly_totals(portfolio, fixture_app) - for fixture_app in fixture_apps - if fixture_app["name"] - in [application.name for application in portfolio.applications] - ], - key=lambda app: app["name"], - ) - - @classmethod - def _get_environment_monthly_totals(cls, environment): - """ - returns a dictionary that represents spending totals for an environment e.g. - { - name - this_month - last_month - total - } - """ - return { - "name": environment["name"], - "this_month": sum(environment["spending"]["this_month"].values()), - "last_month": sum(environment["spending"]["last_month"].values()), - "total": sum(environment["spending"]["total"].values()), - } - - @classmethod - def _get_application_monthly_totals(cls, portfolio, fixture_app): - """ - returns a dictionary that represents spending totals for an application - and its environments e.g. - { - name - this_month - last_month - total - environments: [ - { - name - this_month - last_month - total - } - ] - } - """ - application_envs = [ - env - for env in portfolio.all_environments - if env.application.name == fixture_app["name"] - ] - - environments = [ - cls._get_environment_monthly_totals(env) - for env in fixture_app["environments"] - if env["name"] in [e.name for e in application_envs] - ] - - for env in application_envs: - if env.name not in [env["name"] for env in environments]: - environments.append({"name": env.name}) - - return { - "name": fixture_app["name"], - "this_month": sum(env.get("this_month", 0) for env in environments), - "last_month": sum(env.get("last_month", 0) for env in environments), - "total": sum(env.get("total", 0) for env in environments), - "environments": sorted(environments, key=lambda env: env["name"]), - } - - @classmethod - def get_spending_by_JEDI_clin(cls, portfolio): - """ - returns an dictionary of spending per JEDI CLIN for a portfolio - { - jedi_clin: { - invoiced - estimated - }, - } - """ - if portfolio.name in cls.FIXTURE_SPEND_DATA: - CLIN_spend_dict = defaultdict(lambda: defaultdict(Decimal)) - for application in cls.FIXTURE_SPEND_DATA[portfolio.name]["applications"]: - for environment in application["environments"]: - for clin, spend in environment["spending"]["this_month"].items(): - CLIN_spend_dict[clin]["estimated"] += Decimal(spend) - for clin, spend in environment["spending"]["total"].items(): - CLIN_spend_dict[clin]["invoiced"] += Decimal(spend) - return CLIN_spend_dict - return {} + return dict( + invoiced=Decimal(sum([row[0] for row in rows])), + estimated=Decimal(sum([row[0] for row in estimated])), + ) diff --git a/atst/domain/environment_roles.py b/atst/domain/environment_roles.py index f0b600c6..cc942f2d 100644 --- a/atst/domain/environment_roles.py +++ b/atst/domain/environment_roles.py @@ -90,14 +90,18 @@ class EnvironmentRoles(object): ) @classmethod - def get_environment_roles_pending_creation(cls) -> List[UUID]: + def get_pending_creation(cls) -> List[UUID]: results = ( db.session.query(EnvironmentRole.id) .join(Environment) .join(ApplicationRole) .filter(Environment.deleted == False) - .filter(EnvironmentRole.status == EnvironmentRole.Status.PENDING) - .filter(ApplicationRole.status == ApplicationRoleStatus.ACTIVE) + .filter(EnvironmentRole.deleted == False) + .filter(ApplicationRole.deleted == False) + .filter(ApplicationRole.cloud_id != None) + .filter(ApplicationRole.status != ApplicationRoleStatus.DISABLED) + .filter(EnvironmentRole.status != EnvironmentRole.Status.DISABLED) + .filter(EnvironmentRole.cloud_id.is_(None)) .all() ) return [id_ for id_, in results] @@ -106,7 +110,7 @@ class EnvironmentRoles(object): def disable(cls, environment_role_id): environment_role = EnvironmentRoles.get_by_id(environment_role_id) - if environment_role.csp_user_id and not environment_role.environment.cloud_id: + if environment_role.cloud_id and not environment_role.environment.cloud_id: tenant_id = environment_role.environment.application.portfolio.csp_data.get( "tenant_id" ) diff --git a/atst/domain/reports.py b/atst/domain/reports.py index 99b229e3..fc619649 100644 --- a/atst/domain/reports.py +++ b/atst/domain/reports.py @@ -1,12 +1,13 @@ from flask import current_app -from itertools import groupby +from atst.domain.csp.cloud.models import ( + ReportingCSPPayload, + CostManagementQueryCSPResult, +) +from atst.domain.csp.reports import prepare_azure_reporting_data +import pendulum class Reports: - @classmethod - def monthly_spending(cls, portfolio): - return current_app.csp.reports.get_portfolio_monthly_spending(portfolio) - @classmethod def expired_task_orders(cls, portfolio): return [ @@ -14,31 +15,19 @@ class Reports: ] @classmethod - def obligated_funds_by_JEDI_clin(cls, portfolio): - clin_spending = current_app.csp.reports.get_spending_by_JEDI_clin(portfolio) - active_clins = portfolio.active_clins - for jedi_clin, clins in groupby( - active_clins, key=lambda clin: clin.jedi_clin_type - ): - if not clin_spending.get(jedi_clin.name): - clin_spending[jedi_clin.name] = {} - clin_spending[jedi_clin.name]["obligated"] = sum( - clin.obligated_amount for clin in clins - ) + def get_portfolio_spending(cls, portfolio): + # TODO: Extend this function to make from_date and to_date configurable + from_date = pendulum.now().subtract(years=1).add(days=1).format("YYYY-MM-DD") + to_date = pendulum.now().format("YYYY-MM-DD") + rows = [] - output = [] - for clin in clin_spending.keys(): - invoiced = clin_spending[clin].get("invoiced", 0) - estimated = clin_spending[clin].get("estimated", 0) - obligated = clin_spending[clin].get("obligated", 0) - remaining = obligated - (invoiced + estimated) - output.append( - { - "name": clin, - "invoiced": invoiced, - "estimated": estimated, - "obligated": obligated, - "remaining": remaining, - } + if portfolio.csp_data: + payload = ReportingCSPPayload( + from_date=from_date, to_date=to_date, **portfolio.csp_data ) - return output + response: CostManagementQueryCSPResult = current_app.csp.cloud.get_reporting_data( + payload + ) + rows = response.properties.rows + + return prepare_azure_reporting_data(rows) diff --git a/atst/domain/task_orders.py b/atst/domain/task_orders.py index 9ecf41e9..499bccb0 100644 --- a/atst/domain/task_orders.py +++ b/atst/domain/task_orders.py @@ -1,4 +1,5 @@ -import datetime +from datetime import datetime +from sqlalchemy import or_ from atst.database import db from atst.models.clin import CLIN @@ -40,7 +41,7 @@ class TaskOrders(BaseDomainClass): @classmethod def sign(cls, task_order, signer_dod_id): task_order.signer_dod_id = signer_dod_id - task_order.signed_at = datetime.datetime.now() + task_order.signed_at = datetime.now() db.session.add(task_order) db.session.commit() @@ -76,3 +77,17 @@ class TaskOrders(BaseDomainClass): task_order = TaskOrders.get(task_order_id) db.session.delete(task_order) db.session.commit() + + @classmethod + def get_for_send_task_order_files(cls): + return ( + db.session.query(TaskOrder) + .join(CLIN) + .filter( + or_( + TaskOrder.pdf_last_sent_at < CLIN.last_sent_at, + TaskOrder.pdf_last_sent_at.is_(None), + ) + ) + .all() + ) diff --git a/atst/filters.py b/atst/filters.py index 3508f1e9..84191017 100644 --- a/atst/filters.py +++ b/atst/filters.py @@ -5,7 +5,7 @@ from flask import render_template from jinja2 import contextfilter from jinja2.exceptions import TemplateNotFound from urllib.parse import urlparse, urlunparse, parse_qs, urlencode -from decimal import DivisionByZero as DivisionByZeroException +from decimal import DivisionByZero as DivisionByZeroException, InvalidOperation def iconSvg(name): @@ -43,7 +43,7 @@ def obligatedFundingGraphWidth(values): numerator, denominator = values try: return (numerator / denominator) * 100 - except DivisionByZeroException: + except (DivisionByZeroException, InvalidOperation): return 0 diff --git a/atst/jobs.py b/atst/jobs.py index f7ac2df9..0d462f0f 100644 --- a/atst/jobs.py +++ b/atst/jobs.py @@ -1,5 +1,7 @@ import pendulum from flask import current_app as app +from smtplib import SMTPException +from azure.core.exceptions import AzureError from atst.database import db from atst.domain.application_roles import ApplicationRoles @@ -10,12 +12,16 @@ from atst.domain.csp.cloud.models import ( ApplicationCSPPayload, EnvironmentCSPPayload, UserCSPPayload, + UserRoleCSPPayload, ) from atst.domain.environments import Environments +from atst.domain.environment_roles import EnvironmentRoles from atst.domain.portfolios import Portfolios -from atst.models import JobFailure +from atst.models import CSPRole, JobFailure +from atst.domain.task_orders import TaskOrders from atst.models.utils import claim_for_update, claim_many_for_update from atst.queue import celery +from atst.utils.localization import translate class RecordFailure(celery.Task): @@ -43,8 +49,8 @@ class RecordFailure(celery.Task): @celery.task(ignore_result=True) -def send_mail(recipients, subject, body): - app.mailer.send(recipients, subject, body) +def send_mail(recipients, subject, body, attachments=[]): + app.mailer.send(recipients, subject, body, attachments) @celery.task(ignore_result=True) @@ -120,13 +126,46 @@ def do_create_environment(csp: CloudProviderInterface, environment_id=None): payload = EnvironmentCSPPayload( tenant_id=tenant_id, display_name=environment.name, parent_id=parent_id ) - env_result = csp.create_environment(payload) environment.cloud_id = env_result.id db.session.add(environment) db.session.commit() +def do_create_environment_role(csp: CloudProviderInterface, environment_role_id=None): + env_role = EnvironmentRoles.get_by_id(environment_role_id) + + with claim_for_update(env_role) as env_role: + + if env_role.cloud_id is not None: + return + + env = env_role.environment + csp_details = env.application.portfolio.csp_data + app_role = env_role.application_role + + role = None + if env_role.role == CSPRole.ADMIN: + role = UserRoleCSPPayload.Roles.owner + elif env_role.role == CSPRole.BILLING_READ: + role = UserRoleCSPPayload.Roles.billing + elif env_role.role == CSPRole.CONTRIBUTOR: + role = UserRoleCSPPayload.Roles.contributor + + payload = UserRoleCSPPayload( + tenant_id=csp_details.get("tenant_id"), + management_group_id=env.cloud_id, + user_object_id=app_role.cloud_id, + role=role, + ) + result = csp.create_user_role(payload) + + env_role.cloud_id = result.id + db.session.add(env_role) + db.session.commit() + # TODO: should send notification email to the user, maybe with their portal login name + + def render_email(template_path, context): return app.jinja_env.get_template(template_path).render(context) @@ -141,7 +180,7 @@ def do_work(fn, task, csp, **kwargs): def do_provision_portfolio(csp: CloudProviderInterface, portfolio_id=None): portfolio = Portfolios.get_for_update(portfolio_id) fsm = Portfolios.get_or_create_state_machine(portfolio) - fsm.trigger_next_transition() + fsm.trigger_next_transition(csp_data=portfolio.to_dictionary()) @celery.task(bind=True, base=RecordFailure) @@ -161,6 +200,16 @@ def create_user(self, application_role_ids=None): ) +@celery.task(bind=True, base=RecordFailure) +def create_environment_role(self, environment_role_id=None): + do_work( + do_create_environment_role, + self, + app.csp.cloud, + environment_role_id=environment_role_id, + ) + + @celery.task(bind=True, base=RecordFailure) def create_environment(self, environment_id=None): do_work(do_create_environment, self, app.csp.cloud, environment_id=environment_id) @@ -187,9 +236,51 @@ def dispatch_create_user(self): create_user.delay(application_role_ids=application_role_ids) +@celery.task(bind=True) +def dispatch_create_environment_role(self): + for environment_role_id in EnvironmentRoles.get_pending_creation(): + create_environment_role.delay(environment_role_id=environment_role_id) + + @celery.task(bind=True) def dispatch_create_environment(self): for environment_id in Environments.get_environments_pending_creation( pendulum.now() ): create_environment.delay(environment_id=environment_id) + + +@celery.task(bind=True) +def dispatch_create_atat_admin_user(self): + for environment_id in Environments.get_environments_pending_atat_user_creation( + pendulum.now() + ): + create_atat_admin_user.delay(environment_id=environment_id) + + +@celery.task(bind=True) +def dispatch_send_task_order_files(self): + task_orders = TaskOrders.get_for_send_task_order_files() + recipients = [app.config.get("MICROSOFT_TASK_ORDER_EMAIL_ADDRESS")] + + for task_order in task_orders: + subject = translate( + "email.task_order_sent.subject", {"to_number": task_order.number} + ) + body = translate("email.task_order_sent.body", {"to_number": task_order.number}) + + try: + file = app.csp.files.download_task_order(task_order.pdf.object_name) + file["maintype"] = "application" + file["subtype"] = "pdf" + send_mail( + recipients=recipients, subject=subject, body=body, attachments=[file] + ) + except (AzureError, SMTPException) as err: + app.logger.exception(err) + continue + + task_order.pdf_last_sent_at = pendulum.now() + db.session.add(task_order) + + db.session.commit() diff --git a/atst/models/clin.py b/atst/models/clin.py index 2811bd6a..13a63cee 100644 --- a/atst/models/clin.py +++ b/atst/models/clin.py @@ -1,5 +1,13 @@ from enum import Enum -from sqlalchemy import Column, Date, Enum as SQLAEnum, ForeignKey, Numeric, String +from sqlalchemy import ( + Column, + Date, + DateTime, + Enum as SQLAEnum, + ForeignKey, + Numeric, + String, +) from sqlalchemy.orm import relationship from datetime import date @@ -29,6 +37,7 @@ class CLIN(Base, mixins.TimestampsMixin): total_amount = Column(Numeric(scale=2), nullable=False) obligated_amount = Column(Numeric(scale=2), nullable=False) jedi_clin_type = Column(SQLAEnum(JEDICLINType, native_enum=False), nullable=False) + last_sent_at = Column(DateTime) # # NOTE: For now obligated CLINS are CLIN 1 + CLIN 3 diff --git a/atst/models/environment.py b/atst/models/environment.py index cd202fd0..6eb0c02d 100644 --- a/atst/models/environment.py +++ b/atst/models/environment.py @@ -61,6 +61,10 @@ class Environment( def portfolio_id(self): return self.application.portfolio_id + @property + def is_pending(self): + return self.cloud_id is None + def __repr__(self): return "".format( self.name, diff --git a/atst/models/environment_role.py b/atst/models/environment_role.py index 56fe78d4..871f39a1 100644 --- a/atst/models/environment_role.py +++ b/atst/models/environment_role.py @@ -36,7 +36,7 @@ class EnvironmentRole( ) application_role = relationship("ApplicationRole") - csp_user_id = Column(String()) + cloud_id = Column(String()) class Status(Enum): PENDING = "pending" diff --git a/atst/models/mixins/state_machines.py b/atst/models/mixins/state_machines.py index f4b6fae5..e741db76 100644 --- a/atst/models/mixins/state_machines.py +++ b/atst/models/mixins/state_machines.py @@ -24,6 +24,8 @@ class AzureStages(Enum): TENANT_PRINCIPAL_CREDENTIAL = "tenant principal credential" ADMIN_ROLE_DEFINITION = "admin role definition" PRINCIPAL_ADMIN_ROLE = "tenant principal admin" + INITIAL_MGMT_GROUP = "initial management group" + INITIAL_MGMT_GROUP_VERIFICATION = "initial management group verification" TENANT_ADMIN_OWNERSHIP = "tenant admin ownership" TENANT_PRINCIPAL_OWNERSHIP = "tenant principial ownership" diff --git a/atst/models/portfolio.py b/atst/models/portfolio.py index 5a8f0f1e..c006dd37 100644 --- a/atst/models/portfolio.py +++ b/atst/models/portfolio.py @@ -1,11 +1,16 @@ +import re +from string import ascii_lowercase, digits +from random import choices +from itertools import chain + from sqlalchemy import Column, String from sqlalchemy.orm import relationship from sqlalchemy.types import ARRAY -from itertools import chain from atst.models.base import Base import atst.models.types as types import atst.models.mixins as mixins +from atst.models.task_order import TaskOrder from atst.models.portfolio_role import PortfolioRole, Status as PortfolioRoleStatus from atst.domain.permission_sets import PermissionSets from atst.utils import first_or_none @@ -89,6 +94,22 @@ class Portfolio( def active_task_orders(self): return [task_order for task_order in self.task_orders if task_order.is_active] + @property + def total_obligated_funds(self): + return sum( + (task_order.total_obligated_funds for task_order in self.active_task_orders) + ) + + @property + def upcoming_obligated_funds(self): + return sum( + ( + task_order.total_obligated_funds + for task_order in self.task_orders + if task_order.is_upcoming + ) + ) + @property def funding_duration(self): """ @@ -153,6 +174,51 @@ class Portfolio( def application_id(self): return None + def to_dictionary(self): + ppoc = self.owner + user_id = f"{ppoc.first_name[0]}{ppoc.last_name}".lower() + + domain_name = re.sub("[^0-9a-zA-Z]+", "", self.name).lower() + "".join( + choices(ascii_lowercase + digits, k=4) + ) + portfolio_data = { + "user_id": user_id, + "password": "", + "domain_name": domain_name, + "first_name": ppoc.first_name, + "last_name": ppoc.last_name, + "country_code": "US", + "password_recovery_email_address": ppoc.email, + "address": { # TODO: TBD if we're sourcing this from data or config + "company_name": "", + "address_line_1": "", + "city": "", + "region": "", + "country": "", + "postal_code": "", + }, + "billing_profile_display_name": "ATAT Billing Profile", + } + + try: + initial_task_order: TaskOrder = self.task_orders[0] + initial_clin = initial_task_order.sorted_clins[0] + portfolio_data.update( + { + "initial_clin_amount": initial_clin.obligated_amount, + "initial_clin_start_date": initial_clin.start_date.strftime( + "%Y/%m/%d" + ), + "initial_clin_end_date": initial_clin.end_date.strftime("%Y/%m/%d"), + "initial_clin_type": initial_clin.number, + "initial_task_order_id": initial_task_order.number, + } + ) + except IndexError: + pass + + return portfolio_data + def __repr__(self): return "".format( self.name, self.user_count, self.id diff --git a/atst/models/portfolio_state_machine.py b/atst/models/portfolio_state_machine.py index 39fe727a..16ee5851 100644 --- a/atst/models/portfolio_state_machine.py +++ b/atst/models/portfolio_state_machine.py @@ -119,6 +119,8 @@ class PortfolioStateMachine( def trigger_next_transition(self, **kwargs): state_obj = self.machine.get_state(self.state) + kwargs["csp_data"] = kwargs.get("csp_data", {}) + if state_obj.is_system: if self.current_state in (FSMStates.UNSTARTED, FSMStates.STARTING): # call the first trigger availabe for these two system states diff --git a/atst/models/task_order.py b/atst/models/task_order.py index 789a7e3f..c6dda237 100644 --- a/atst/models/task_order.py +++ b/atst/models/task_order.py @@ -25,7 +25,6 @@ SORT_ORDERING = [ Status.DRAFT, Status.UPCOMING, Status.EXPIRED, - Status.UNSIGNED, ] @@ -39,6 +38,7 @@ class TaskOrder(Base, mixins.TimestampsMixin): pdf_attachment_id = Column(ForeignKey("attachments.id")) _pdf = relationship("Attachment", foreign_keys=[pdf_attachment_id]) + pdf_last_sent_at = Column(DateTime) number = Column(String, unique=True,) # Task Order Number signer_dod_id = Column(String) signed_at = Column(DateTime) @@ -87,6 +87,10 @@ class TaskOrder(Base, mixins.TimestampsMixin): def is_expired(self): return self.status == Status.EXPIRED + @property + def is_upcoming(self): + return self.status == Status.UPCOMING + @property def clins_are_completed(self): return all([len(self.clins), (clin.is_completed for clin in self.clins)]) @@ -147,7 +151,10 @@ class TaskOrder(Base, mixins.TimestampsMixin): @property def display_status(self): - return self.status.value + if self.status == Status.UNSIGNED: + return Status.DRAFT.value + else: + return self.status.value @property def portfolio_name(self): diff --git a/atst/queue.py b/atst/queue.py index 0ea910fb..10bcb350 100644 --- a/atst/queue.py +++ b/atst/queue.py @@ -23,6 +23,10 @@ def update_celery(celery, app): "task": "atst.jobs.dispatch_create_user", "schedule": 60, }, + "beat-dispatch_create_environment_role": { + "task": "atst.jobs.dispatch_create_environment_role", + "schedule": 60, + }, } class ContextTask(celery.Task): diff --git a/atst/routes/applications/settings.py b/atst/routes/applications/settings.py index ad8a6540..8807c7f1 100644 --- a/atst/routes/applications/settings.py +++ b/atst/routes/applications/settings.py @@ -39,6 +39,7 @@ def get_environments_obj_for_app(application): { "id": env.id, "name": env.name, + "pending": env.is_pending, "edit_form": EditEnvironmentForm(obj=env), "member_count": len(env.roles), "members": sorted( diff --git a/atst/routes/portfolios/index.py b/atst/routes/portfolios/index.py index f9e7d5cf..44cac768 100644 --- a/atst/routes/portfolios/index.py +++ b/atst/routes/portfolios/index.py @@ -34,25 +34,27 @@ def create_portfolio(): @user_can(Permissions.VIEW_PORTFOLIO_REPORTS, message="view portfolio reports") def reports(portfolio_id): portfolio = Portfolios.get(g.current_user, portfolio_id) + spending = Reports.get_portfolio_spending(portfolio) + obligated = portfolio.total_obligated_funds + remaining = obligated - (spending["invoiced"] + spending["estimated"]) - current_obligated_funds = Reports.obligated_funds_by_JEDI_clin(portfolio) + current_obligated_funds = { + **spending, + "obligated": obligated, + "remaining": remaining, + } - if any(map(lambda clin: clin["remaining"] < 0, current_obligated_funds)): + if current_obligated_funds["remaining"] < 0: flash("insufficient_funds") - # wrapped in str() because the sum of obligated funds returns a Decimal object - total_portfolio_value = str( - sum( - task_order.total_obligated_funds - for task_order in portfolio.active_task_orders - ) - ) return render_template( "portfolios/reports/index.html", portfolio=portfolio, - total_portfolio_value=total_portfolio_value, + # wrapped in str() because this sum returns a Decimal object + total_portfolio_value=str( + portfolio.total_obligated_funds + portfolio.upcoming_obligated_funds + ), current_obligated_funds=current_obligated_funds, expired_task_orders=Reports.expired_task_orders(portfolio), - monthly_spending=Reports.monthly_spending(portfolio), retrieved=datetime.now(), # mocked datetime of reporting data retrival ) diff --git a/atst/utils/context_processors.py b/atst/utils/context_processors.py index 7d39b367..5bb4771d 100644 --- a/atst/utils/context_processors.py +++ b/atst/utils/context_processors.py @@ -19,6 +19,9 @@ from atst.models import ( def get_resources_from_context(view_args): query = None + if view_args is None: + view_args = {} + if "portfolio_token" in view_args: query = ( db.session.query(Portfolio) diff --git a/config/base.ini b/config/base.ini index 1f4c732a..727172d8 100644 --- a/config/base.ini +++ b/config/base.ini @@ -1,9 +1,19 @@ [default] ASSETS_URL +AZURE_AADP_QTY=5 AZURE_ACCOUNT_NAME -AZURE_STORAGE_KEY -AZURE_TO_BUCKET_NAME +AZURE_CLIENT_ID +AZURE_GRAPH_RESOURCE="https://graph.microsoft.com/" AZURE_POLICY_LOCATION=policies +AZURE_POWERSHELL_CLIENT_ID +AZURE_ROLE_DEF_ID_BILLING_READER="fa23ad8b-c56e-40d8-ac0c-ce449e1d2c64" +AZURE_ROLE_DEF_ID_CONTRIBUTOR="b24988ac-6180-42a0-ab88-20f7382dd24c" +AZURE_ROLE_DEF_ID_OWNER="8e3af657-a8ff-443c-a75c-2fe8c4bcb635" +AZURE_SECRET_KEY +AZURE_STORAGE_KEY +AZURE_TENANT_ID +AZURE_TO_BUCKET_NAME +AZURE_VAULT_URL BLOB_STORAGE_URL=http://localhost:8000/ CAC_URL = http://localhost:8000/login-redirect CA_CHAIN = ssl/server-certs/ca-chain.pem @@ -38,14 +48,15 @@ PGUSER = postgres PORT=8000 REDIS_HOST=localhost:6379 REDIS_PASSWORD +REDIS_SSLMODE REDIS_TLS=False REDIS_USER SECRET_KEY = change_me_into_something_secret SERVER_NAME -SESSION_COOKIE_NAME=atat SESSION_COOKIE_DOMAIN -SESSION_KEY_PREFIX=session: +SESSION_COOKIE_NAME=atat SESSION_COOKIE_SECURE=false +SESSION_KEY_PREFIX=session: SESSION_TYPE = redis SESSION_USE_SIGNER = True SQLALCHEMY_ECHO = False diff --git a/deploy/azure/kustomization.yaml b/deploy/azure/kustomization.yaml index d0162394..b46021b0 100644 --- a/deploy/azure/kustomization.yaml +++ b/deploy/azure/kustomization.yaml @@ -10,6 +10,5 @@ resources: - volume-claim.yml - nginx-client-ca-bundle.yml - acme-challenges.yml - - aadpodidentity.yml - nginx-snippets.yml - autoscaling.yml diff --git a/deploy/overlays/cloudzero-dev/envvars.yml b/deploy/overlays/cloudzero-dev/envvars.yml index 179811ed..ded47f7b 100644 --- a/deploy/overlays/cloudzero-dev/envvars.yml +++ b/deploy/overlays/cloudzero-dev/envvars.yml @@ -4,19 +4,30 @@ kind: ConfigMap metadata: name: atst-worker-envvars data: + AZURE_ACCOUNT_NAME: jeditasksatat CELERY_DEFAULT_QUEUE: celery-staging - SERVER_NAME: staging.atat.code.mil FLASK_ENV: staging + PGDATABASE: cloudzero_jedidev_atat + PGHOST: 191.238.6.43 + PGUSER: atat@cloudzero-jedidev-sql + PGSSLMODE: require + REDIS_HOST: 10.1.3.34:6380 + SERVER_NAME: dev.atat.cloud.mil --- apiVersion: v1 kind: ConfigMap metadata: name: atst-envvars data: - ASSETS_URL: https://atat-cdn-staging.azureedge.net/ - CDN_ORIGIN: https://staging.atat.code.mil + ASSETS_URL: "" + AZURE_ACCOUNT_NAME: jeditasksatat + CAC_URL: https://auth-dev.atat.cloud.mil + CDN_ORIGIN: https://dev.atat.cloud.mil CELERY_DEFAULT_QUEUE: celery-staging FLASK_ENV: staging - STATIC_URL: https://atat-cdn-staging.azureedge.net/static/ - PGHOST: cloudzero-dev-sql.postgres.database.azure.com - REDIS_HOST: cloudzero-dev-redis.redis.cache.windows.net:6380 + PGDATABASE: cloudzero_jedidev_atat + PGHOST: 191.238.6.43 + PGUSER: atat@cloudzero-jedidev-sql + PGSSLMODE: require + REDIS_HOST: 10.1.3.34:6380 + SESSION_COOKIE_DOMAIN: atat.cloud.mil diff --git a/deploy/overlays/cloudzero-dev/flex_vol.yml b/deploy/overlays/cloudzero-dev/flex_vol.yml index a3c65df7..990d21e5 100644 --- a/deploy/overlays/cloudzero-dev/flex_vol.yml +++ b/deploy/overlays/cloudzero-dev/flex_vol.yml @@ -9,23 +9,19 @@ spec: - name: nginx-secret flexVolume: options: - keyvaultname: "cloudzero-dev-keyvault" - # keyvaultobjectnames: "dhparam4096;cert;cert" - keyvaultobjectnames: "foo" - keyvaultobjectaliases: "FOO" - keyvaultobjecttypes: "secret" - usevmmanagedidentity: "true" usepodidentity: "false" + usevmmanagedidentity: "true" + vmmanagedidentityclientid: $VMSS_CLIENT_ID + keyvaultname: "cz-jedidev-keyvault" + keyvaultobjectnames: "dhparam4096;ATATCERT;ATATCERT" - name: flask-secret flexVolume: options: - keyvaultname: "cloudzero-dev-keyvault" - # keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY" - keyvaultobjectnames: "master-PGPASSWORD" - keyvaultobjectaliases: "PGPASSWORD" - keyvaultobjecttypes: "secret" - usevmmanagedidentity: "true" usepodidentity: "false" + usevmmanagedidentity: "true" + vmmanagedidentityclientid: $VMSS_CLIENT_ID + keyvaultname: "cz-jedidev-keyvault" + keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY" --- apiVersion: extensions/v1beta1 kind: Deployment @@ -38,10 +34,11 @@ spec: - name: flask-secret flexVolume: options: - keyvaultname: "cloudzero-dev-keyvault" - keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY" - usevmmanagedidentity: "true" usepodidentity: "false" + usevmmanagedidentity: "true" + vmmanagedidentityclientid: $VMSS_CLIENT_ID + keyvaultname: "cz-jedidev-keyvault" + keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY" --- apiVersion: extensions/v1beta1 kind: Deployment @@ -54,10 +51,11 @@ spec: - name: flask-secret flexVolume: options: - keyvaultname: "cloudzero-dev-keyvault" - keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY" - usevmmanagedidentity: "true" usepodidentity: "false" + usevmmanagedidentity: "true" + vmmanagedidentityclientid: $VMSS_CLIENT_ID + keyvaultname: "cz-jedidev-keyvault" + keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY" --- apiVersion: batch/v1beta1 kind: CronJob @@ -72,7 +70,8 @@ spec: - name: flask-secret flexVolume: options: - keyvaultname: "cloudzero-dev-keyvault" - keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY" - usevmmanagedidentity: "true" usepodidentity: "false" + usevmmanagedidentity: "true" + vmmanagedidentityclientid: $VMSS_CLIENT_ID + keyvaultname: "cz-jedidev-keyvault" + keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY" diff --git a/deploy/overlays/cloudzero-dev/kustomization.yaml b/deploy/overlays/cloudzero-dev/kustomization.yaml index 24705531..65262fbe 100644 --- a/deploy/overlays/cloudzero-dev/kustomization.yaml +++ b/deploy/overlays/cloudzero-dev/kustomization.yaml @@ -1,9 +1,8 @@ -namespace: staging +namespace: cloudzero-dev bases: - ../../azure/ resources: - namespace.yml - - reset-cron-job.yml patchesStrategicMerge: - ports.yml - envvars.yml diff --git a/deploy/overlays/cloudzero-dev/namespace.yml b/deploy/overlays/cloudzero-dev/namespace.yml index ee38adfb..242c3a2f 100644 --- a/deploy/overlays/cloudzero-dev/namespace.yml +++ b/deploy/overlays/cloudzero-dev/namespace.yml @@ -1,4 +1,4 @@ apiVersion: v1 kind: Namespace metadata: - name: staging + name: cloudzero-dev diff --git a/deploy/overlays/cloudzero-dev/ports.yml b/deploy/overlays/cloudzero-dev/ports.yml index 8dbbd0f1..5225cf3c 100644 --- a/deploy/overlays/cloudzero-dev/ports.yml +++ b/deploy/overlays/cloudzero-dev/ports.yml @@ -5,7 +5,7 @@ metadata: name: atst-main annotations: service.beta.kubernetes.io/azure-load-balancer-internal: "true" - service.beta.kubernetes.io/azure-load-balancer-internal-subnet: "cloudzero-dev-public" + service.beta.kubernetes.io/azure-load-balancer-internal-subnet: "cloudzero-jedidev-public" spec: loadBalancerIP: "" ports: @@ -22,7 +22,7 @@ metadata: name: atst-auth annotations: service.beta.kubernetes.io/azure-load-balancer-internal: "true" - service.beta.kubernetes.io/azure-load-balancer-internal-subnet: "cloudzero-dev-public" + service.beta.kubernetes.io/azure-load-balancer-internal-subnet: "cloudzero-jedidev-public" spec: loadBalancerIP: "" ports: diff --git a/deploy/overlays/cloudzero-dev/reset-cron-job.yml b/deploy/overlays/cloudzero-dev/reset-cron-job.yml deleted file mode 100644 index b4792e5d..00000000 --- a/deploy/overlays/cloudzero-dev/reset-cron-job.yml +++ /dev/null @@ -1,46 +0,0 @@ -apiVersion: batch/v1beta1 -kind: CronJob -metadata: - name: reset-db - namespace: atat -spec: - schedule: "0 4 * * *" - concurrencyPolicy: Replace - successfulJobsHistoryLimit: 1 - jobTemplate: - spec: - template: - metadata: - labels: - app: atst - role: reset-db - aadpodidbinding: atat-kv-id-binding - spec: - restartPolicy: OnFailure - containers: - - name: reset - image: $CONTAINER_IMAGE - command: [ - "/bin/sh", "-c" - ] - args: [ - "/opt/atat/atst/.venv/bin/python", - "/opt/atat/atst/script/reset_database.py" - ] - envFrom: - - configMapRef: - name: atst-worker-envvars - volumeMounts: - - name: flask-secret - mountPath: "/config" - volumes: - - name: flask-secret - flexVolume: - driver: "azure/kv" - options: - usepodidentity: "true" - keyvaultname: "atat-vault-test" - keyvaultobjectnames: "staging-AZURE-STORAGE-KEY;staging-MAIL-PASSWORD;staging-PGPASSWORD;staging-REDIS-PASSWORD;staging-SECRET-KEY" - keyvaultobjectaliases: "AZURE_STORAGE_KEY;MAIL_PASSWORD;PGPASSWORD;REDIS_PASSWORD;SECRET_KEY" - keyvaultobjecttypes: "secret;secret;secret;secret;key" - tenantid: $TENANT_ID diff --git a/deploy/overlays/migration-cloudzero-dev/kustomization.yaml b/deploy/overlays/migration-cloudzero-dev/kustomization.yaml new file mode 100644 index 00000000..b12c0b88 --- /dev/null +++ b/deploy/overlays/migration-cloudzero-dev/kustomization.yaml @@ -0,0 +1,5 @@ +namespace: cloudzero-dev +bases: + - ../../shared/ +patchesStrategicMerge: + - migration.yaml diff --git a/deploy/overlays/migration-cloudzero-dev/migration.yaml b/deploy/overlays/migration-cloudzero-dev/migration.yaml new file mode 100644 index 00000000..53a39dcc --- /dev/null +++ b/deploy/overlays/migration-cloudzero-dev/migration.yaml @@ -0,0 +1,16 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: migration +spec: + template: + spec: + volumes: + - name: flask-secret + flexVolume: + options: + usepodidentity: "false" + usevmmanagedidentity: "true" + vmmanagedidentityclientid: $VMSS_CLIENT_ID + keyvaultname: "cz-jedidev-keyvault" + keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY" diff --git a/deploy/shared/kustomization.yaml b/deploy/shared/kustomization.yaml new file mode 100644 index 00000000..38dddc7e --- /dev/null +++ b/deploy/shared/kustomization.yaml @@ -0,0 +1,3 @@ +namespace: atat +resources: + - migration.yaml diff --git a/js/components/forms/multi_step_modal_form.js b/js/components/forms/multi_step_modal_form.js index 0dfcaba0..5cc5f4df 100644 --- a/js/components/forms/multi_step_modal_form.js +++ b/js/components/forms/multi_step_modal_form.js @@ -34,8 +34,10 @@ export default { methods: { next: function() { + this.submitted = true if (this.validateFields()) { this.step += 1 + this.submitted = false } }, previous: function() { diff --git a/js/components/sidenav_toggler.js b/js/components/sidenav_toggler.js index 11717849..d545b3ed 100644 --- a/js/components/sidenav_toggler.js +++ b/js/components/sidenav_toggler.js @@ -1,5 +1,6 @@ import ExpandSidenavMixin from '../mixins/expand_sidenav' import ToggleMixin from '../mixins/toggle' +import { sidenavCookieName } from '../lib/constants' export default { name: 'sidenav-toggler', @@ -14,7 +15,7 @@ export default { toggle: function(e) { e.preventDefault() this.isVisible = !this.isVisible - document.cookie = this.cookieName + '=' + this.isVisible + '; path=/' + document.cookie = sidenavCookieName + '=' + this.isVisible + '; path=/' this.$parent.$emit('sidenavToggle', this.isVisible) }, }, diff --git a/js/components/toggle_menu.js b/js/components/toggle_menu.js index e17a201a..6c23ce06 100644 --- a/js/components/toggle_menu.js +++ b/js/components/toggle_menu.js @@ -5,6 +5,13 @@ export default { mixins: [ToggleMixin], + props: { + defaultVisible: { + type: Boolean, + default: false, + }, + }, + methods: { toggle: function(e) { if (this.$el.contains(e.target)) { diff --git a/js/components/upload_input.js b/js/components/upload_input.js index 9856405b..4f9f06fc 100644 --- a/js/components/upload_input.js +++ b/js/components/upload_input.js @@ -17,7 +17,7 @@ export default { filename: { type: String, }, - objectName: { + initialObjectName: { type: String, }, initialErrors: { @@ -42,6 +42,7 @@ export default { filenameError: false, downloadLink: '', fileSizeLimit: this.sizeLimit, + objectName: this.initialObjectName, } }, @@ -72,6 +73,7 @@ export default { const response = await uploader.upload(file) if (uploadResponseOkay(response)) { this.attachment = e.target.value + this.objectName = uploader.objectName this.$refs.attachmentFilename.value = file.name this.$refs.attachmentObjectName.value = response.objectName this.$refs.attachmentInput.disabled = true diff --git a/js/lib/constants.js b/js/lib/constants.js new file mode 100644 index 00000000..b4de4fcf --- /dev/null +++ b/js/lib/constants.js @@ -0,0 +1 @@ +export const sidenavCookieName = 'expandSidenav' diff --git a/js/lib/input_validations.js b/js/lib/input_validations.js index 9f113aa6..19b3d350 100644 --- a/js/lib/input_validations.js +++ b/js/lib/input_validations.js @@ -9,6 +9,12 @@ export default { unmask: [], validationError: 'Please enter a response', }, + applicationName: { + mask: false, + match: /^[A-Za-z0-9\-_,'".\s]{4,100}$$/, + unmask: [], + validationError: 'Application names can be between 4-100 characters', + }, clinNumber: { mask: false, match: /^\d{4}$/, @@ -42,14 +48,14 @@ export default { }, defaultStringField: { mask: false, - match: /^[A-Za-z0-9\-_ \.]{1,100}$/, + match: /^[A-Za-z0-9\-_,'".\s]{1,1000}$/, unmask: [], validationError: 'Please enter a response of no more than 100 alphanumeric characters', }, defaultTextAreaField: { mask: false, - match: /^[A-Za-z0-9\-_ \.]{1,1000}$/, + match: /^[A-Za-z0-9\-_,'".\s]{1,1000}$/, unmask: [], validationError: 'Please enter a response of no more than 1000 alphanumeric characters', @@ -94,7 +100,7 @@ export default { }, portfolioName: { mask: false, - match: /^.{4,100}$/, + match: /^[A-Za-z0-9\-_,'".\s]{4,100}$$/, unmask: [], validationError: 'Portfolio names can be between 4-100 characters', }, diff --git a/js/mixins/expand_sidenav.js b/js/mixins/expand_sidenav.js index 7553b7d4..2af9c87a 100644 --- a/js/mixins/expand_sidenav.js +++ b/js/mixins/expand_sidenav.js @@ -1,11 +1,12 @@ +import { sidenavCookieName } from '../lib/constants' + export default { props: { - cookieName: 'expandSidenav', defaultVisible: { type: Boolean, default: function() { - if (document.cookie.match(this.cookieName)) { - return !!document.cookie.match(this.cookieName + ' *= *true') + if (document.cookie.match(sidenavCookieName)) { + return !!document.cookie.match(sidenavCookieName + ' *= *true') } else { return true } diff --git a/js/mixins/form.js b/js/mixins/form.js index 45aa6261..070466de 100644 --- a/js/mixins/form.js +++ b/js/mixins/form.js @@ -15,6 +15,7 @@ export default { return { changed: this.hasChanges, valid: false, + submitted: false, } }, @@ -36,15 +37,16 @@ export default { handleSubmit: function(event) { if (!this.valid) { event.preventDefault() + this.submitted = true } }, }, computed: { canSave: function() { - if (this.changed && this.valid) { + if (this.changed && this.valid && !this.submitted) { return true - } else if (this.enableSave && this.valid) { + } else if (this.enableSave && this.valid && !this.submitted) { return true } else { return false diff --git a/script/database_setup.py b/script/database_setup.py index 7784be05..e4964516 100644 --- a/script/database_setup.py +++ b/script/database_setup.py @@ -16,16 +16,14 @@ from reset_database import reset_database def database_setup(username, password, dbname, ccpo_users): + print("Applying schema and seeding roles and permissions.") + reset_database() + print( f"Creating Postgres user role for '{username}' and granting all privileges to database '{dbname}'." ) - try: - _create_database_user(username, password, dbname) - except sqlalchemy.exc.ProgrammingError as err: - print(f"Postgres user role '{username}' already exists.") + _create_database_user(username, password, dbname) - print("Applying schema and seeding roles and permissions.") - reset_database() print("Creating initial set of CCPO users.") _add_ccpo_users(ccpo_users) @@ -47,6 +45,22 @@ def _create_database_user(username, password, dbname): f"ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL PRIVILEGES ON FUNCTIONS TO {username}; \n" ) + try: + # TODO: make this more configurable + engine.execute(f"GRANT {username} TO azure_pg_admin;") + except sqlalchemy.exc.ProgrammingError as err: + print(f"Cannot grant new role {username} to azure_pg_admin") + + for table in meta.tables: + engine.execute(f"ALTER TABLE {table} OWNER TO {username};\n") + + sequence_results = engine.execute( + "SELECT c.relname FROM pg_class c WHERE c.relkind = 'S';" + ).fetchall() + sequences = [p[0] for p in sequence_results] + for sequence in sequences: + engine.execute(f"ALTER SEQUENCE {sequence} OWNER TO {username};\n") + trans.commit() diff --git a/script/k8s_config b/script/k8s_config index b489c942..36dc5ef7 100755 --- a/script/k8s_config +++ b/script/k8s_config @@ -13,6 +13,7 @@ SETTINGS=( AUTH_DOMAIN KV_MI_ID KV_MI_CLIENT_ID + VMSS_CLIENT_ID TENANT_ID ) diff --git a/script/seed_sample.py b/script/seed_sample.py index 72c16c6c..d1cf1c9e 100644 --- a/script/seed_sample.py +++ b/script/seed_sample.py @@ -195,7 +195,7 @@ def add_task_orders_to_portfolio(portfolio): task_order=unsigned_to, start_date=(today - five_days), end_date=today ), CLINFactory.build( - task_order=upcoming_to, start_date=future, end_date=(today + five_days) + task_order=upcoming_to, start_date=(today + five_days), end_date=future ), CLINFactory.build( task_order=expired_to, start_date=(today - five_days), end_date=yesterday diff --git a/styles/components/_portfolio_layout.scss b/styles/components/_portfolio_layout.scss index 4a45cc5f..06fecf63 100644 --- a/styles/components/_portfolio_layout.scss +++ b/styles/components/_portfolio_layout.scss @@ -513,10 +513,6 @@ } &__header { margin: 0; - &-icon { - margin: 0; - padding: 0; - } } &__value { font-size: $lead-font-size; diff --git a/styles/core/_util.scss b/styles/core/_util.scss index 0790a121..7fd8e152 100644 --- a/styles/core/_util.scss +++ b/styles/core/_util.scss @@ -98,3 +98,7 @@ hr { .usa-section { padding: 0; } + +.form { + margin-bottom: $action-footer-height + $large-spacing; +} diff --git a/styles/core/_variables.scss b/styles/core/_variables.scss index 372fa868..09b838f3 100644 --- a/styles/core/_variables.scss +++ b/styles/core/_variables.scss @@ -20,6 +20,7 @@ $max-panel-width: 90rem; $home-pg-icon-width: 6rem; $large-spacing: 4rem; $max-page-width: $max-panel-width + $sidenav-expanded-width + $large-spacing; +$action-footer-height: 6rem; /* * USWDS Variables diff --git a/styles/elements/_action_group.scss b/styles/elements/_action_group.scss index c2d11049..8d7dadef 100644 --- a/styles/elements/_action_group.scss +++ b/styles/elements/_action_group.scss @@ -42,6 +42,7 @@ border-top: 1px solid $color-gray-lighter; z-index: 1; width: 100%; + height: $action-footer-height; &.action-group-footer--expand-offset { padding-left: $sidenav-expanded-width; diff --git a/styles/elements/_inputs.scss b/styles/elements/_inputs.scss index 195d0a2b..9e74ff50 100644 --- a/styles/elements/_inputs.scss +++ b/styles/elements/_inputs.scss @@ -228,6 +228,7 @@ &--validation { &--anything, + &--applicationName, &--portfolioName, &--requiredField, &--defaultStringField, diff --git a/styles/elements/_tooltip.scss b/styles/elements/_tooltip.scss index 46f1146b..b86506eb 100644 --- a/styles/elements/_tooltip.scss +++ b/styles/elements/_tooltip.scss @@ -95,4 +95,9 @@ .icon { @include icon-size(16); } + + &--tight { + margin: 0; + padding: 0; + } } diff --git a/styles/sections/_task_order.scss b/styles/sections/_task_order.scss index 05b90595..79f391e0 100644 --- a/styles/sections/_task_order.scss +++ b/styles/sections/_task_order.scss @@ -1,6 +1,5 @@ .task-order { margin-top: $gap * 4; - margin-bottom: $footer-height; width: 900px; &__amount { diff --git a/templates/applications/fragments/environments.html b/templates/applications/fragments/environments.html index d0934268..5b4be1de 100644 --- a/templates/applications/fragments/environments.html +++ b/templates/applications/fragments/environments.html @@ -47,7 +47,7 @@ {{ env['name'] }} - {{ Label(type="pending_creation", classes='label--below')}} + {{ Label(type="pending_creation")}} {%- endif %} {% if user_can(permissions.EDIT_ENVIRONMENT) -%} {{ diff --git a/templates/applications/fragments/member_form_fields.html b/templates/applications/fragments/member_form_fields.html index dd91dd5d..cb9a5c31 100644 --- a/templates/applications/fragments/member_form_fields.html +++ b/templates/applications/fragments/member_form_fields.html @@ -1,7 +1,8 @@ {% from "components/alert.html" import Alert %} {% from "components/checkbox_input.html" import CheckboxInput %} -{% from "components/text_input.html" import TextInput %} {% from "components/phone_input.html" import PhoneInput %} +{% from "components/text_input.html" import TextInput %} +{% from "components/tooltip.html" import Tooltip %} {% macro EnvRoleInput(sub_form, member_role_id=None) %} {% set role = sub_form.role.data if not sub_form.disabled.data else "Access Suspended" %} @@ -121,6 +122,6 @@ {{ TextInput(member_form.email, validation='email', optional=False) }} {{ PhoneInput(member_form.phone_number, member_form.phone_ext)}} {{ TextInput(member_form.dod_id, validation='dodId', optional=False) }} - {{ "forms.new_member.dod_help" | translate }} + {{ "forms.new_member.dod_help" | translate }} {{ Tooltip("forms.new_member.dod_text"|translate, title="", classes="icon-tooltip--tight") }} {% endmacro %} diff --git a/templates/applications/fragments/members.html b/templates/applications/fragments/members.html index 5cae077f..f60fcba8 100644 --- a/templates/applications/fragments/members.html +++ b/templates/applications/fragments/members.html @@ -14,7 +14,7 @@ action_new, action_update) %} -

+

{{ 'portfolios.applications.settings.team_members' | translate }}

@@ -22,7 +22,7 @@ {% include "fragments/flash.html" %} {% endif %} -
+
{% if not application.members %}

diff --git a/templates/applications/new/step_1.html b/templates/applications/new/step_1.html index 3841bf96..39656257 100644 --- a/templates/applications/new/step_1.html +++ b/templates/applications/new/step_1.html @@ -22,11 +22,11 @@ {% include "fragments/flash.html" %} -

+ {{ form.csrf_token }}
- {{ TextInput(form.name, validation="name", optional=False) }} + {{ TextInput(form.name, validation="applicationName", optional=False) }} {{ ('portfolios.applications.new.step_1_form_help_text.name' | translate | safe) }}
diff --git a/templates/applications/new/step_2.html b/templates/applications/new/step_2.html index 2cd5cf98..fe07b44d 100644 --- a/templates/applications/new/step_2.html +++ b/templates/applications/new/step_2.html @@ -21,7 +21,7 @@

- +
{{ 'portfolios.applications.environments_heading' | translate }}
diff --git a/templates/applications/settings.html b/templates/applications/settings.html index 2c641e27..1ec7be37 100644 --- a/templates/applications/settings.html +++ b/templates/applications/settings.html @@ -22,7 +22,7 @@ {{ application_form.csrf_token }} - {{ TextInput(application_form.name, validation="name", optional=False) }} + {{ TextInput(application_form.name, validation="applicationName", optional=False) }} {{ TextInput(application_form.description, validation="defaultTextAreaField", paragraph=True, optional=True, showOptional=False) }}
{{ SaveButton(text='common.save_changes'|translate) }} diff --git a/templates/components/accordion.html b/templates/components/accordion.html index 9ec92099..cdf72901 100644 --- a/templates/components/accordion.html +++ b/templates/components/accordion.html @@ -6,8 +6,12 @@ heading_tag="h2", heading_classes="", content_tag="div", - content_classes="") %} - + content_classes="", + default_visible=False) %} + <{{wrapper_tag}} class="{{ wrapper_classes }}"> <{{heading_tag}} class="accordion__button {{ heading_classes }}"> diff --git a/templates/components/upload_input.html b/templates/components/upload_input.html index 4f4f307f..bd4cd73c 100644 --- a/templates/components/upload_input.html +++ b/templates/components/upload_input.html @@ -5,7 +5,7 @@ inline-template {% if not field.errors %} v-bind:filename='{{ field.filename.data | tojson }}' - v-bind:object-name='{{ field.object_name.data | tojson }}' + v-bind:initial-object-name='{{ field.object_name.data | tojson }}' {% else %} v-bind:initial-errors='true' {% endif %} @@ -46,7 +46,7 @@ v-bind:value="attachment" type="file"> - +