diff --git a/.secrets.baseline b/.secrets.baseline index a233e4cf..4e393738 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": "^.secrets.baseline$|^.*pgsslrootcert.yml$", "lines": null }, - "generated_at": "2020-01-27T19:24:43Z", + "generated_at": "2020-02-10T21:40:38Z", "plugins_used": [ { "base64_limit": 4.5, @@ -82,7 +82,7 @@ "hashed_secret": "afc848c316af1a89d49826c5ae9d00ed769415f3", "is_secret": false, "is_verified": false, - "line_number": 32, + "line_number": 43, "type": "Secret Keyword" } ], diff --git a/Dockerfile b/Dockerfile index 6f29d300..6dd7629a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -101,5 +101,7 @@ RUN mkdir /var/run/uwsgi && \ chown -R atst:atat /var/run/uwsgi && \ chown -R atst:atat "${APP_DIR}" +RUN update-ca-certificates + # Run as the unprivileged APP user USER atst diff --git a/alembic/versions/418b52c1cedf_change_to_environment_roles_cloud_id.py b/alembic/versions/418b52c1cedf_change_to_environment_roles_cloud_id.py new file mode 100644 index 00000000..93f11712 --- /dev/null +++ b/alembic/versions/418b52c1cedf_change_to_environment_roles_cloud_id.py @@ -0,0 +1,30 @@ +"""change to environment_roles.cloud_Id + +Revision ID: 418b52c1cedf +Revises: 542bd3215dec +Create Date: 2020-02-05 13:40:37.870183 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '418b52c1cedf' # pragma: allowlist secret +down_revision = '542bd3215dec' # pragma: allowlist secret +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('environment_roles', sa.Column('cloud_id', sa.String(), nullable=True)) + op.drop_column('environment_roles', 'csp_user_id') + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('environment_roles', sa.Column('csp_user_id', sa.VARCHAR(), autoincrement=False, nullable=True)) + op.drop_column('environment_roles', 'cloud_id') + # ### end Alembic commands ### diff --git a/alembic/versions/542bd3215dec_state_machine_stage_added.py b/alembic/versions/542bd3215dec_state_machine_stage_added.py new file mode 100644 index 00000000..8a3410af --- /dev/null +++ b/alembic/versions/542bd3215dec_state_machine_stage_added.py @@ -0,0 +1,264 @@ +"""state machine stage added. + +Revision ID: 542bd3215dec +Revises: 567bfb019a87 +Create Date: 2020-02-06 12:01:58.077840 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "542bd3215dec" # pragma: allowlist secret +down_revision = "567bfb019a87" # pragma: allowlist secret +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column( + "portfolio_state_machines", + "state", + type_=sa.Enum( + "UNSTARTED", + "STARTING", + "STARTED", + "COMPLETED", + "FAILED", + "TENANT_CREATED", + "TENANT_IN_PROGRESS", + "TENANT_FAILED", + "BILLING_PROFILE_CREATION_CREATED", + "BILLING_PROFILE_CREATION_IN_PROGRESS", + "BILLING_PROFILE_CREATION_FAILED", + "BILLING_PROFILE_VERIFICATION_CREATED", + "BILLING_PROFILE_VERIFICATION_IN_PROGRESS", + "BILLING_PROFILE_VERIFICATION_FAILED", + "BILLING_PROFILE_TENANT_ACCESS_CREATED", + "BILLING_PROFILE_TENANT_ACCESS_IN_PROGRESS", + "BILLING_PROFILE_TENANT_ACCESS_FAILED", + "TASK_ORDER_BILLING_CREATION_CREATED", + "TASK_ORDER_BILLING_CREATION_IN_PROGRESS", + "TASK_ORDER_BILLING_CREATION_FAILED", + "TASK_ORDER_BILLING_VERIFICATION_CREATED", + "TASK_ORDER_BILLING_VERIFICATION_IN_PROGRESS", + "TASK_ORDER_BILLING_VERIFICATION_FAILED", + "BILLING_INSTRUCTION_CREATED", + "BILLING_INSTRUCTION_IN_PROGRESS", + "BILLING_INSTRUCTION_FAILED", + "PRODUCT_PURCHASE_CREATED", + "PRODUCT_PURCHASE_IN_PROGRESS", + "PRODUCT_PURCHASE_FAILED", + "PRODUCT_PURCHASE_VERIFICATION_CREATED", + "PRODUCT_PURCHASE_VERIFICATION_IN_PROGRESS", + "PRODUCT_PURCHASE_VERIFICATION_FAILED", + "TENANT_PRINCIPAL_APP_CREATED", + "TENANT_PRINCIPAL_APP_IN_PROGRESS", + "TENANT_PRINCIPAL_APP_FAILED", + "TENANT_PRINCIPAL_CREATED", + "TENANT_PRINCIPAL_IN_PROGRESS", + "TENANT_PRINCIPAL_FAILED", + "TENANT_PRINCIPAL_CREDENTIAL_CREATED", + "TENANT_PRINCIPAL_CREDENTIAL_IN_PROGRESS", + "TENANT_PRINCIPAL_CREDENTIAL_FAILED", + "ADMIN_ROLE_DEFINITION_CREATED", + "ADMIN_ROLE_DEFINITION_IN_PROGRESS", + "ADMIN_ROLE_DEFINITION_FAILED", + "PRINCIPAL_ADMIN_ROLE_CREATED", + "PRINCIPAL_ADMIN_ROLE_IN_PROGRESS", + "PRINCIPAL_ADMIN_ROLE_FAILED", + "INITIAL_MGMT_GROUP_CREATED", + "INITIAL_MGMT_GROUP_IN_PROGRESS", + "INITIAL_MGMT_GROUP_FAILED", + "INITIAL_MGMT_GROUP_VERIFICATION_CREATED", + "INITIAL_MGMT_GROUP_VERIFICATION_IN_PROGRESS", + "INITIAL_MGMT_GROUP_VERIFICATION_FAILED", + "TENANT_ADMIN_OWNERSHIP_CREATED", + "TENANT_ADMIN_OWNERSHIP_IN_PROGRESS", + "TENANT_ADMIN_OWNERSHIP_FAILED", + "TENANT_PRINCIPAL_OWNERSHIP_CREATED", + "TENANT_PRINCIPAL_OWNERSHIP_IN_PROGRESS", + "TENANT_PRINCIPAL_OWNERSHIP_FAILED", + name="fsmstates", + native_enum=False, + ), + existing_type=sa.Enum( + "UNSTARTED", + "STARTING", + "STARTED", + "COMPLETED", + "FAILED", + "TENANT_CREATED", + "TENANT_IN_PROGRESS", + "TENANT_FAILED", + "BILLING_PROFILE_CREATION_CREATED", + "BILLING_PROFILE_CREATION_IN_PROGRESS", + "BILLING_PROFILE_CREATION_FAILED", + "BILLING_PROFILE_VERIFICATION_CREATED", + "BILLING_PROFILE_VERIFICATION_IN_PROGRESS", + "BILLING_PROFILE_VERIFICATION_FAILED", + "BILLING_PROFILE_TENANT_ACCESS_CREATED", + "BILLING_PROFILE_TENANT_ACCESS_IN_PROGRESS", + "BILLING_PROFILE_TENANT_ACCESS_FAILED", + "TASK_ORDER_BILLING_CREATION_CREATED", + "TASK_ORDER_BILLING_CREATION_IN_PROGRESS", + "TASK_ORDER_BILLING_CREATION_FAILED", + "TASK_ORDER_BILLING_VERIFICATION_CREATED", + "TASK_ORDER_BILLING_VERIFICATION_IN_PROGRESS", + "TASK_ORDER_BILLING_VERIFICATION_FAILED", + "BILLING_INSTRUCTION_CREATED", + "BILLING_INSTRUCTION_IN_PROGRESS", + "BILLING_INSTRUCTION_FAILED", + "TENANT_PRINCIPAL_APP_CREATED", + "TENANT_PRINCIPAL_APP_IN_PROGRESS", + "TENANT_PRINCIPAL_APP_FAILED", + "TENANT_PRINCIPAL_CREATED", + "TENANT_PRINCIPAL_IN_PROGRESS", + "TENANT_PRINCIPAL_FAILED", + "TENANT_PRINCIPAL_CREDENTIAL_CREATED", + "TENANT_PRINCIPAL_CREDENTIAL_IN_PROGRESS", + "TENANT_PRINCIPAL_CREDENTIAL_FAILED", + "ADMIN_ROLE_DEFINITION_CREATED", + "ADMIN_ROLE_DEFINITION_IN_PROGRESS", + "ADMIN_ROLE_DEFINITION_FAILED", + "PRINCIPAL_ADMIN_ROLE_CREATED", + "PRINCIPAL_ADMIN_ROLE_IN_PROGRESS", + "PRINCIPAL_ADMIN_ROLE_FAILED", + "TENANT_ADMIN_OWNERSHIP_CREATED", + "TENANT_ADMIN_OWNERSHIP_IN_PROGRESS", + "TENANT_ADMIN_OWNERSHIP_FAILED", + "TENANT_PRINCIPAL_OWNERSHIP_CREATED", + "TENANT_PRINCIPAL_OWNERSHIP_IN_PROGRESS", + "TENANT_PRINCIPAL_OWNERSHIP_FAILED", + name="fsmstates", + native_enum=False, + ), + existing_nullable=False, + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column( + "portfolio_state_machines", + "state", + type_=sa.Enum( + "UNSTARTED", + "STARTING", + "STARTED", + "COMPLETED", + "FAILED", + "TENANT_CREATED", + "TENANT_IN_PROGRESS", + "TENANT_FAILED", + "BILLING_PROFILE_CREATION_CREATED", + "BILLING_PROFILE_CREATION_IN_PROGRESS", + "BILLING_PROFILE_CREATION_FAILED", + "BILLING_PROFILE_VERIFICATION_CREATED", + "BILLING_PROFILE_VERIFICATION_IN_PROGRESS", + "BILLING_PROFILE_VERIFICATION_FAILED", + "BILLING_PROFILE_TENANT_ACCESS_CREATED", + "BILLING_PROFILE_TENANT_ACCESS_IN_PROGRESS", + "BILLING_PROFILE_TENANT_ACCESS_FAILED", + "TASK_ORDER_BILLING_CREATION_CREATED", + "TASK_ORDER_BILLING_CREATION_IN_PROGRESS", + "TASK_ORDER_BILLING_CREATION_FAILED", + "TASK_ORDER_BILLING_VERIFICATION_CREATED", + "TASK_ORDER_BILLING_VERIFICATION_IN_PROGRESS", + "TASK_ORDER_BILLING_VERIFICATION_FAILED", + "BILLING_INSTRUCTION_CREATED", + "BILLING_INSTRUCTION_IN_PROGRESS", + "BILLING_INSTRUCTION_FAILED", + "TENANT_PRINCIPAL_APP_CREATED", + "TENANT_PRINCIPAL_APP_IN_PROGRESS", + "TENANT_PRINCIPAL_APP_FAILED", + "TENANT_PRINCIPAL_CREATED", + "TENANT_PRINCIPAL_IN_PROGRESS", + "TENANT_PRINCIPAL_FAILED", + "TENANT_PRINCIPAL_CREDENTIAL_CREATED", + "TENANT_PRINCIPAL_CREDENTIAL_IN_PROGRESS", + "TENANT_PRINCIPAL_CREDENTIAL_FAILED", + "ADMIN_ROLE_DEFINITION_CREATED", + "ADMIN_ROLE_DEFINITION_IN_PROGRESS", + "ADMIN_ROLE_DEFINITION_FAILED", + "PRINCIPAL_ADMIN_ROLE_CREATED", + "PRINCIPAL_ADMIN_ROLE_IN_PROGRESS", + "PRINCIPAL_ADMIN_ROLE_FAILED", + "TENANT_ADMIN_OWNERSHIP_CREATED", + "TENANT_ADMIN_OWNERSHIP_IN_PROGRESS", + "TENANT_ADMIN_OWNERSHIP_FAILED", + "TENANT_PRINCIPAL_OWNERSHIP_CREATED", + "TENANT_PRINCIPAL_OWNERSHIP_IN_PROGRESS", + "TENANT_PRINCIPAL_OWNERSHIP_FAILED", + name="fsmstates", + native_enum=False, + ), + existing_type=sa.Enum( + "UNSTARTED", + "STARTING", + "STARTED", + "COMPLETED", + "FAILED", + "TENANT_CREATED", + "TENANT_IN_PROGRESS", + "TENANT_FAILED", + "BILLING_PROFILE_CREATION_CREATED", + "BILLING_PROFILE_CREATION_IN_PROGRESS", + "BILLING_PROFILE_CREATION_FAILED", + "BILLING_PROFILE_VERIFICATION_CREATED", + "BILLING_PROFILE_VERIFICATION_IN_PROGRESS", + "BILLING_PROFILE_VERIFICATION_FAILED", + "BILLING_PROFILE_TENANT_ACCESS_CREATED", + "BILLING_PROFILE_TENANT_ACCESS_IN_PROGRESS", + "BILLING_PROFILE_TENANT_ACCESS_FAILED", + "TASK_ORDER_BILLING_CREATION_CREATED", + "TASK_ORDER_BILLING_CREATION_IN_PROGRESS", + "TASK_ORDER_BILLING_CREATION_FAILED", + "TASK_ORDER_BILLING_VERIFICATION_CREATED", + "TASK_ORDER_BILLING_VERIFICATION_IN_PROGRESS", + "TASK_ORDER_BILLING_VERIFICATION_FAILED", + "BILLING_INSTRUCTION_CREATED", + "BILLING_INSTRUCTION_IN_PROGRESS", + "BILLING_INSTRUCTION_FAILED", + "PRODUCT_PURCHASE_CREATED", + "PRODUCT_PURCHASE_IN_PROGRESS", + "PRODUCT_PURCHASE_FAILED", + "PRODUCT_PURCHASE_VERIFICATION_CREATED", + "PRODUCT_PURCHASE_VERIFICATION_IN_PROGRESS", + "PRODUCT_PURCHASE_VERIFICATION_FAILED", + "TENANT_PRINCIPAL_APP_CREATED", + "TENANT_PRINCIPAL_APP_IN_PROGRESS", + "TENANT_PRINCIPAL_APP_FAILED", + "TENANT_PRINCIPAL_CREATED", + "TENANT_PRINCIPAL_IN_PROGRESS", + "TENANT_PRINCIPAL_FAILED", + "TENANT_PRINCIPAL_CREDENTIAL_CREATED", + "TENANT_PRINCIPAL_CREDENTIAL_IN_PROGRESS", + "TENANT_PRINCIPAL_CREDENTIAL_FAILED", + "ADMIN_ROLE_DEFINITION_CREATED", + "ADMIN_ROLE_DEFINITION_IN_PROGRESS", + "ADMIN_ROLE_DEFINITION_FAILED", + "PRINCIPAL_ADMIN_ROLE_CREATED", + "PRINCIPAL_ADMIN_ROLE_IN_PROGRESS", + "PRINCIPAL_ADMIN_ROLE_FAILED", + "INITIAL_MGMT_GROUP_CREATED", + "INITIAL_MGMT_GROUP_IN_PROGRESS", + "INITIAL_MGMT_GROUP_FAILED", + "INITIAL_MGMT_GROUP_VERIFICATION_CREATED", + "INITIAL_MGMT_GROUP_VERIFICATION_IN_PROGRESS", + "INITIAL_MGMT_GROUP_VERIFICATION_FAILED", + "TENANT_ADMIN_OWNERSHIP_CREATED", + "TENANT_ADMIN_OWNERSHIP_IN_PROGRESS", + "TENANT_ADMIN_OWNERSHIP_FAILED", + "TENANT_PRINCIPAL_OWNERSHIP_CREATED", + "TENANT_PRINCIPAL_OWNERSHIP_IN_PROGRESS", + "TENANT_PRINCIPAL_OWNERSHIP_FAILED", + name="fsmstates", + native_enum=False, + ), + existing_nullable=False, + ) + diff --git a/atst/app.py b/atst/app.py index 05578827..04aed44d 100644 --- a/atst/app.py +++ b/atst/app.py @@ -157,7 +157,6 @@ def map_config(config): **config["default"], "USE_AUDIT_LOG": config["default"].getboolean("USE_AUDIT_LOG"), "ENV": config["default"]["ENVIRONMENT"], - "BROKER_URL": config["default"]["REDIS_URI"], "DEBUG": config["default"].getboolean("DEBUG"), "DEBUG_MAILER": config["default"].getboolean("DEBUG_MAILER"), "SQLALCHEMY_ECHO": config["default"].getboolean("SQLALCHEMY_ECHO"), @@ -233,13 +232,34 @@ def make_config(direct_config=None): config.set("default", "DATABASE_URI", database_uri) # Assemble REDIS_URI value + redis_use_tls = config["default"].getboolean("REDIS_TLS") redis_uri = "redis{}://{}:{}@{}".format( # pragma: allowlist secret - ("s" if config["default"].getboolean("REDIS_TLS") else ""), + ("s" if redis_use_tls else ""), (config.get("default", "REDIS_USER") or ""), (config.get("default", "REDIS_PASSWORD") or ""), config.get("default", "REDIS_HOST"), ) + celery_uri = redis_uri + if redis_use_tls: + tls_mode = config.get("default", "REDIS_SSLMODE") + tls_mode_str = tls_mode.lower() if tls_mode else "none" + redis_uri = f"{redis_uri}/?ssl_cert_reqs={tls_mode_str}" + + # TODO: Kombu, one of Celery's dependencies, still requires + # that ssl_cert_reqs be passed as the string version of an + # option on the ssl module. We can clean this up and use + # the REDIS_URI for both when this PR to Kombu is released: + # https://github.com/celery/kombu/pull/1139 + kombu_modes = { + "none": "CERT_NONE", + "required": "CERT_REQUIRED", + "optional": "CERT_OPTIONAL", + } + celery_tls_mode_str = kombu_modes[tls_mode_str] + celery_uri = f"{celery_uri}/?ssl_cert_reqs={celery_tls_mode_str}" + config.set("default", "REDIS_URI", redis_uri) + config.set("default", "BROKER_URL", celery_uri) return map_config(config) diff --git a/atst/domain/csp/cloud/azure_cloud_provider.py b/atst/domain/csp/cloud/azure_cloud_provider.py index 1d921fde..42b6018e 100644 --- a/atst/domain/csp/cloud/azure_cloud_provider.py +++ b/atst/domain/csp/cloud/azure_cloud_provider.py @@ -1,6 +1,5 @@ import json from secrets import token_urlsafe -from typing import Any, Dict from uuid import uuid4 from atst.utils import sha256_hex @@ -25,6 +24,10 @@ from .models import ( BillingProfileVerificationCSPPayload, BillingProfileVerificationCSPResult, CostManagementQueryCSPResult, + InitialMgmtGroupCSPPayload, + InitialMgmtGroupCSPResult, + InitialMgmtGroupVerificationCSPPayload, + InitialMgmtGroupVerificationCSPResult, EnvironmentCSPPayload, EnvironmentCSPResult, KeyVaultCredentials, @@ -57,6 +60,8 @@ from .models import ( TenantPrincipalOwnershipCSPResult, UserCSPPayload, UserCSPResult, + UserRoleCSPPayload, + UserRoleCSPResult, ) from .policy import AzurePolicyManager @@ -103,10 +108,14 @@ class AzureCloudProvider(CloudProviderInterface): self.secret_key = config["AZURE_SECRET_KEY"] self.tenant_id = config["AZURE_TENANT_ID"] self.vault_url = config["AZURE_VAULT_URL"] - self.ps_client_id = config["POWERSHELL_CLIENT_ID"] - self.owner_role_def_id = config["AZURE_OWNER_ROLE_DEF_ID"] + self.ps_client_id = config["AZURE_POWERSHELL_CLIENT_ID"] self.graph_resource = config["AZURE_GRAPH_RESOURCE"] self.default_aadp_qty = config["AZURE_AADP_QTY"] + self.roles = { + "owner": config["AZURE_ROLE_DEF_ID_OWNER"], + "contributor": config["AZURE_ROLE_DEF_ID_CONTRIBUTOR"], + "billing": config["AZURE_ROLE_DEF_ID_BILLING_READER"], + } if azure_sdk_provider is None: self.sdk = AzureSDKProvider() @@ -189,6 +198,38 @@ class AzureCloudProvider(CloudProviderInterface): return ApplicationCSPResult(**response) + def create_initial_mgmt_group(self, payload: InitialMgmtGroupCSPPayload): + creds = self._source_creds(payload.tenant_id) + credentials = self._get_credential_obj( + { + "client_id": creds.root_sp_client_id, + "secret_key": creds.root_sp_key, + "tenant_id": creds.root_tenant_id, + }, + resource=self.sdk.cloud.endpoints.resource_manager, + ) + response = self._create_management_group( + credentials, payload.management_group_name, payload.display_name, + ) + + return InitialMgmtGroupCSPResult(**response) + + def create_initial_mgmt_group_verification( + self, payload: InitialMgmtGroupVerificationCSPPayload + ): + creds = self._source_creds(payload.tenant_id) + credentials = self._get_credential_obj( + { + "client_id": creds.root_sp_client_id, + "secret_key": creds.root_sp_key, + "tenant_id": creds.root_tenant_id, + }, + resource=self.sdk.cloud.endpoints.resource_manager, + ) + + response = self._get_management_group(credentials, payload.tenant_id,) + return InitialMgmtGroupVerificationCSPResult(**response.result()) + def _create_management_group( self, credentials, management_group_id, display_name, parent_id=None, ): @@ -215,6 +256,11 @@ class AzureCloudProvider(CloudProviderInterface): # instead? return create_request.result() + def _get_management_group(self, credentials, management_group_id): + mgmgt_group_client = self.sdk.managementgroups.ManagementGroupsAPI(credentials) + response = mgmgt_group_client.management_groups.get(management_group_id) + return response + def _create_policy_definition( self, credentials, subscription_id, management_group_id, properties, ): @@ -580,7 +626,7 @@ class AzureCloudProvider(CloudProviderInterface): def create_tenant_admin_ownership(self, payload: TenantAdminOwnershipCSPPayload): mgmt_token = self._get_elevated_management_token(payload.tenant_id) - role_definition_id = f"/providers/Microsoft.Management/managementGroups/{payload.tenant_id}/providers/Microsoft.Authorization/roleDefinitions/{self.owner_role_def_id}" + role_definition_id = f"/providers/Microsoft.Management/managementGroups/{payload.tenant_id}/providers/Microsoft.Authorization/roleDefinitions/{self.roles['owner']}" request_body = { "properties": { @@ -608,7 +654,7 @@ class AzureCloudProvider(CloudProviderInterface): mgmt_token = self._get_elevated_management_token(payload.tenant_id) # NOTE: the tenant_id is also the id of the root management group, once it is created - role_definition_id = f"/providers/Microsoft.Management/managementGroups/{payload.tenant_id}/providers/Microsoft.Authorization/roleDefinitions/{self.owner_role_def_id}" + role_definition_id = f"/providers/Microsoft.Management/managementGroups/{payload.tenant_id}/providers/Microsoft.Authorization/roleDefinitions/{self.roles['owner']}" request_body = { "properties": { @@ -895,6 +941,40 @@ class AzureCloudProvider(CloudProviderInterface): f"Failed update user email: {response.json()}" ) + def create_user_role(self, payload: UserRoleCSPPayload): + graph_token = self._get_tenant_principal_token(payload.tenant_id) + if graph_token is None: + raise AuthenticationException( + "Could not resolve graph token for tenant admin" + ) + + role_guid = self.roles[payload.role] + role_definition_id = f"/providers/Microsoft.Management/managementGroups/{payload.management_group_id}/providers/Microsoft.Authorization/roleDefinitions/{role_guid}" + + request_body = { + "properties": { + "roleDefinitionId": role_definition_id, + "principalId": payload.user_object_id, + } + } + + auth_header = { + "Authorization": f"Bearer {graph_token}", + } + + assignment_guid = str(uuid4()) + + url = f"{self.sdk.cloud.endpoints.resource_manager}/providers/Microsoft.Management/managementGroups/{payload.management_group_id}/providers/Microsoft.Authorization/roleAssignments/{assignment_guid}?api-version=2015-07-01" + + response = self.sdk.requests.put(url, headers=auth_header, json=request_body) + + if response.ok: + return UserRoleCSPResult(**response.json()) + else: + raise UserProvisioningException( + f"Failed to create user role assignment: {response.json()}" + ) + def _extract_subscription_id(self, subscription_url): sub_id_match = SUBSCRIPTION_ID_REGEX.match(subscription_url) @@ -1026,12 +1106,10 @@ class AzureCloudProvider(CloudProviderInterface): def update_tenant_creds(self, tenant_id, secret: KeyVaultCredentials): hashed = sha256_hex(tenant_id) - new_secrets = secret.dict() curr_secrets = self._source_tenant_creds(tenant_id) - updated_secrets: Dict[str, Any] = {**curr_secrets.dict(), **new_secrets} - us = KeyVaultCredentials(**updated_secrets) - self.set_secret(hashed, json.dumps(us.dict())) - return us + updated_secrets = curr_secrets.merge_credentials(secret) + self.set_secret(hashed, json.dumps(updated_secrets.dict())) + return updated_secrets def _source_tenant_creds(self, tenant_id) -> KeyVaultCredentials: hashed = sha256_hex(tenant_id) @@ -1060,7 +1138,7 @@ class AzureCloudProvider(CloudProviderInterface): "timeframe": "Custom", "timePeriod": {"from": payload.from_date, "to": payload.to_date,}, "dataset": { - "granularity": "Daily", + "granularity": "Monthly", "aggregation": {"totalCost": {"name": "PreTaxCost", "function": "Sum"}}, "grouping": [{"type": "Dimension", "name": "InvoiceId"}], }, diff --git a/atst/domain/csp/cloud/cloud_provider_interface.py b/atst/domain/csp/cloud/cloud_provider_interface.py index 88b55f96..250ac6ef 100644 --- a/atst/domain/csp/cloud/cloud_provider_interface.py +++ b/atst/domain/csp/cloud/cloud_provider_interface.py @@ -1,7 +1,7 @@ from typing import Dict -class CloudProviderInterface: +class CloudProviderInterface: # pragma: no cover def set_secret(self, secret_key: str, secret_value: str): raise NotImplementedError() diff --git a/atst/domain/csp/cloud/mock_cloud_provider.py b/atst/domain/csp/cloud/mock_cloud_provider.py index 7ec0636f..b33600ae 100644 --- a/atst/domain/csp/cloud/mock_cloud_provider.py +++ b/atst/domain/csp/cloud/mock_cloud_provider.py @@ -1,4 +1,5 @@ from uuid import uuid4 +import pendulum from .cloud_provider_interface import CloudProviderInterface from .exceptions import ( @@ -23,6 +24,10 @@ from .models import ( BillingProfileTenantAccessCSPResult, BillingProfileVerificationCSPPayload, BillingProfileVerificationCSPResult, + InitialMgmtGroupCSPPayload, + InitialMgmtGroupCSPResult, + InitialMgmtGroupVerificationCSPPayload, + InitialMgmtGroupVerificationCSPResult, CostManagementQueryCSPResult, CostManagementQueryProperties, ProductPurchaseCSPPayload, @@ -280,6 +285,29 @@ class MockCloudProvider(CloudProviderInterface): } ) + def create_initial_mgmt_group(self, payload: InitialMgmtGroupCSPPayload): + self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) + self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) + self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) + + return InitialMgmtGroupCSPResult( + id=f"{AZURE_MGMNT_PATH}{payload.management_group_name}", + ) + + def create_initial_mgmt_group_verification( + self, payload: InitialMgmtGroupVerificationCSPPayload + ): + self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) + self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) + self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) + + return InitialMgmtGroupVerificationCSPResult( + **dict( + id="Test Id" + # id=f"{AZURE_MGMNT_PATH}{payload.management_group_name}" + ) + ) + def create_product_purchase(self, payload: ProductPurchaseCSPPayload): self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) @@ -459,15 +487,26 @@ class MockCloudProvider(CloudProviderInterface): self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) object_id = str(uuid4()) + start_of_month = pendulum.today(tz="utc").start_of("month").replace(tzinfo=None) + this_month = start_of_month.to_atom_string() + last_month = start_of_month.subtract(months=1).to_atom_string() + two_months_ago = start_of_month.subtract(months=2).to_atom_string() + properties = CostManagementQueryProperties( **dict( columns=[ {"name": "PreTaxCost", "type": "Number"}, - {"name": "UsageDate", "type": "Number"}, + {"name": "BillingMonth", "type": "Datetime"}, {"name": "InvoiceId", "type": "String"}, {"name": "Currency", "type": "String"}, ], - rows=[], + rows=[ + [1.0, two_months_ago, "", "USD"], + [500.0, two_months_ago, "e05009w9sf", "USD"], + [50.0, last_month, "", "USD"], + [1000.0, last_month, "e0500a4qhw", "USD"], + [500.0, this_month, "", "USD"], + ], ) ) diff --git a/atst/domain/csp/cloud/models.py b/atst/domain/csp/cloud/models.py index 358f7934..74702309 100644 --- a/atst/domain/csp/cloud/models.py +++ b/atst/domain/csp/cloud/models.py @@ -1,3 +1,4 @@ +from enum import Enum from secrets import token_urlsafe from typing import Dict, List, Optional from uuid import uuid4 @@ -320,7 +321,7 @@ class ManagementGroupCSPPayload(AliasModel): tenant_id: str management_group_name: Optional[str] display_name: str - parent_id: str + parent_id: Optional[str] @validator("management_group_name", pre=True, always=True) def supply_management_group_name_default(cls, name): @@ -340,16 +341,25 @@ class ManagementGroupCSPPayload(AliasModel): @validator("parent_id", pre=True, always=True) def enforce_parent_id_pattern(cls, id_): - if AZURE_MGMNT_PATH not in id_: - return f"{AZURE_MGMNT_PATH}{id_}" - else: - return id_ + if id_: + if AZURE_MGMNT_PATH not in id_: + return f"{AZURE_MGMNT_PATH}{id_}" + else: + return id_ class ManagementGroupCSPResponse(AliasModel): id: str +class ManagementGroupGetCSPPayload(BaseCSPPayload): + management_group_name: str + + +class ManagementGroupGetCSPResponse(AliasModel): + id: str + + class ApplicationCSPPayload(ManagementGroupCSPPayload): pass @@ -358,6 +368,22 @@ class ApplicationCSPResult(ManagementGroupCSPResponse): pass +class InitialMgmtGroupCSPPayload(ManagementGroupCSPPayload): + pass + + +class InitialMgmtGroupCSPResult(ManagementGroupCSPResponse): + pass + + +class InitialMgmtGroupVerificationCSPPayload(ManagementGroupGetCSPPayload): + pass + + +class InitialMgmtGroupVerificationCSPResult(ManagementGroupGetCSPResponse): + pass + + class EnvironmentCSPPayload(ManagementGroupCSPPayload): pass @@ -417,6 +443,15 @@ class KeyVaultCredentials(BaseModel): return values + def merge_credentials( + self, new_creds: "KeyVaultCredentials" + ) -> "KeyVaultCredentials": + updated_creds = {k: v for k, v in new_creds.dict().items() if v} + old_creds = self.dict() + old_creds.update(updated_creds) + + return KeyVaultCredentials(**old_creds) + class SubscriptionCreationCSPPayload(BaseCSPPayload): display_name: str @@ -509,6 +544,21 @@ class UserCSPResult(AliasModel): id: str +class UserRoleCSPPayload(BaseCSPPayload): + class Roles(str, Enum): + owner = "owner" + contributor = "contributor" + billing = "billing" + + management_group_id: str + role: Roles + user_object_id: str + + +class UserRoleCSPResult(AliasModel): + id: str + + class QueryColumn(AliasModel): name: str type: str diff --git a/atst/domain/csp/reports.py b/atst/domain/csp/reports.py index 3f9ccbf8..700947f7 100644 --- a/atst/domain/csp/reports.py +++ b/atst/domain/csp/reports.py @@ -1,6 +1,6 @@ -from collections import defaultdict import json from decimal import Decimal +import pendulum def load_fixture_data(): @@ -11,128 +11,25 @@ def load_fixture_data(): class MockReportingProvider: FIXTURE_SPEND_DATA = load_fixture_data() - @classmethod - def get_portfolio_monthly_spending(cls, portfolio): - """ - returns an array of application and environment spending for the - portfolio. Applications and their nested environments are sorted in - alphabetical order by name. - [ - { - name - this_month - last_month - total - environments [ - { - name - this_month - last_month - total - } - ] - } - ] - """ - fixture_apps = cls.FIXTURE_SPEND_DATA.get(portfolio.name, {}).get( - "applications", [] - ) +def prepare_azure_reporting_data(rows: list): + """ + Returns a dict representing invoiced and estimated funds for a portfolio given + a list of rows from CostManagementQueryCSPResult.properties.rows + { + invoiced: Decimal, + estimated: Decimal + } + """ - for application in portfolio.applications: - if application.name not in [app["name"] for app in fixture_apps]: - fixture_apps.append({"name": application.name, "environments": []}) + estimated = [] + while rows: + if pendulum.parse(rows[-1][1]) >= pendulum.now(tz="utc").start_of("month"): + estimated.append(rows.pop()) + else: + break - return sorted( - [ - cls._get_application_monthly_totals(portfolio, fixture_app) - for fixture_app in fixture_apps - if fixture_app["name"] - in [application.name for application in portfolio.applications] - ], - key=lambda app: app["name"], - ) - - @classmethod - def _get_environment_monthly_totals(cls, environment): - """ - returns a dictionary that represents spending totals for an environment e.g. - { - name - this_month - last_month - total - } - """ - return { - "name": environment["name"], - "this_month": sum(environment["spending"]["this_month"].values()), - "last_month": sum(environment["spending"]["last_month"].values()), - "total": sum(environment["spending"]["total"].values()), - } - - @classmethod - def _get_application_monthly_totals(cls, portfolio, fixture_app): - """ - returns a dictionary that represents spending totals for an application - and its environments e.g. - { - name - this_month - last_month - total - environments: [ - { - name - this_month - last_month - total - } - ] - } - """ - application_envs = [ - env - for env in portfolio.all_environments - if env.application.name == fixture_app["name"] - ] - - environments = [ - cls._get_environment_monthly_totals(env) - for env in fixture_app["environments"] - if env["name"] in [e.name for e in application_envs] - ] - - for env in application_envs: - if env.name not in [env["name"] for env in environments]: - environments.append({"name": env.name}) - - return { - "name": fixture_app["name"], - "this_month": sum(env.get("this_month", 0) for env in environments), - "last_month": sum(env.get("last_month", 0) for env in environments), - "total": sum(env.get("total", 0) for env in environments), - "environments": sorted(environments, key=lambda env: env["name"]), - } - - @classmethod - def get_spending_by_JEDI_clin(cls, portfolio): - """ - returns an dictionary of spending per JEDI CLIN for a portfolio - { - jedi_clin: { - invoiced - estimated - }, - } - """ - if portfolio.name in cls.FIXTURE_SPEND_DATA: - CLIN_spend_dict = defaultdict(lambda: defaultdict(Decimal)) - for application in cls.FIXTURE_SPEND_DATA[portfolio.name]["applications"]: - for environment in application["environments"]: - for clin, spend in environment["spending"]["this_month"].items(): - CLIN_spend_dict[clin]["estimated"] += Decimal(spend) - for clin, spend in environment["spending"]["total"].items(): - CLIN_spend_dict[clin]["invoiced"] += Decimal(spend) - return CLIN_spend_dict - return {} + return dict( + invoiced=Decimal(sum([row[0] for row in rows])), + estimated=Decimal(sum([row[0] for row in estimated])), + ) diff --git a/atst/domain/environment_roles.py b/atst/domain/environment_roles.py index f0b600c6..cc942f2d 100644 --- a/atst/domain/environment_roles.py +++ b/atst/domain/environment_roles.py @@ -90,14 +90,18 @@ class EnvironmentRoles(object): ) @classmethod - def get_environment_roles_pending_creation(cls) -> List[UUID]: + def get_pending_creation(cls) -> List[UUID]: results = ( db.session.query(EnvironmentRole.id) .join(Environment) .join(ApplicationRole) .filter(Environment.deleted == False) - .filter(EnvironmentRole.status == EnvironmentRole.Status.PENDING) - .filter(ApplicationRole.status == ApplicationRoleStatus.ACTIVE) + .filter(EnvironmentRole.deleted == False) + .filter(ApplicationRole.deleted == False) + .filter(ApplicationRole.cloud_id != None) + .filter(ApplicationRole.status != ApplicationRoleStatus.DISABLED) + .filter(EnvironmentRole.status != EnvironmentRole.Status.DISABLED) + .filter(EnvironmentRole.cloud_id.is_(None)) .all() ) return [id_ for id_, in results] @@ -106,7 +110,7 @@ class EnvironmentRoles(object): def disable(cls, environment_role_id): environment_role = EnvironmentRoles.get_by_id(environment_role_id) - if environment_role.csp_user_id and not environment_role.environment.cloud_id: + if environment_role.cloud_id and not environment_role.environment.cloud_id: tenant_id = environment_role.environment.application.portfolio.csp_data.get( "tenant_id" ) diff --git a/atst/domain/reports.py b/atst/domain/reports.py index 99b229e3..fc619649 100644 --- a/atst/domain/reports.py +++ b/atst/domain/reports.py @@ -1,12 +1,13 @@ from flask import current_app -from itertools import groupby +from atst.domain.csp.cloud.models import ( + ReportingCSPPayload, + CostManagementQueryCSPResult, +) +from atst.domain.csp.reports import prepare_azure_reporting_data +import pendulum class Reports: - @classmethod - def monthly_spending(cls, portfolio): - return current_app.csp.reports.get_portfolio_monthly_spending(portfolio) - @classmethod def expired_task_orders(cls, portfolio): return [ @@ -14,31 +15,19 @@ class Reports: ] @classmethod - def obligated_funds_by_JEDI_clin(cls, portfolio): - clin_spending = current_app.csp.reports.get_spending_by_JEDI_clin(portfolio) - active_clins = portfolio.active_clins - for jedi_clin, clins in groupby( - active_clins, key=lambda clin: clin.jedi_clin_type - ): - if not clin_spending.get(jedi_clin.name): - clin_spending[jedi_clin.name] = {} - clin_spending[jedi_clin.name]["obligated"] = sum( - clin.obligated_amount for clin in clins - ) + def get_portfolio_spending(cls, portfolio): + # TODO: Extend this function to make from_date and to_date configurable + from_date = pendulum.now().subtract(years=1).add(days=1).format("YYYY-MM-DD") + to_date = pendulum.now().format("YYYY-MM-DD") + rows = [] - output = [] - for clin in clin_spending.keys(): - invoiced = clin_spending[clin].get("invoiced", 0) - estimated = clin_spending[clin].get("estimated", 0) - obligated = clin_spending[clin].get("obligated", 0) - remaining = obligated - (invoiced + estimated) - output.append( - { - "name": clin, - "invoiced": invoiced, - "estimated": estimated, - "obligated": obligated, - "remaining": remaining, - } + if portfolio.csp_data: + payload = ReportingCSPPayload( + from_date=from_date, to_date=to_date, **portfolio.csp_data ) - return output + response: CostManagementQueryCSPResult = current_app.csp.cloud.get_reporting_data( + payload + ) + rows = response.properties.rows + + return prepare_azure_reporting_data(rows) diff --git a/atst/filters.py b/atst/filters.py index 3508f1e9..84191017 100644 --- a/atst/filters.py +++ b/atst/filters.py @@ -5,7 +5,7 @@ from flask import render_template from jinja2 import contextfilter from jinja2.exceptions import TemplateNotFound from urllib.parse import urlparse, urlunparse, parse_qs, urlencode -from decimal import DivisionByZero as DivisionByZeroException +from decimal import DivisionByZero as DivisionByZeroException, InvalidOperation def iconSvg(name): @@ -43,7 +43,7 @@ def obligatedFundingGraphWidth(values): numerator, denominator = values try: return (numerator / denominator) * 100 - except DivisionByZeroException: + except (DivisionByZeroException, InvalidOperation): return 0 diff --git a/atst/jobs.py b/atst/jobs.py index 46575a3d..0d462f0f 100644 --- a/atst/jobs.py +++ b/atst/jobs.py @@ -12,10 +12,12 @@ from atst.domain.csp.cloud.models import ( ApplicationCSPPayload, EnvironmentCSPPayload, UserCSPPayload, + UserRoleCSPPayload, ) from atst.domain.environments import Environments +from atst.domain.environment_roles import EnvironmentRoles from atst.domain.portfolios import Portfolios -from atst.models import JobFailure +from atst.models import CSPRole, JobFailure from atst.domain.task_orders import TaskOrders from atst.models.utils import claim_for_update, claim_many_for_update from atst.queue import celery @@ -124,13 +126,46 @@ def do_create_environment(csp: CloudProviderInterface, environment_id=None): payload = EnvironmentCSPPayload( tenant_id=tenant_id, display_name=environment.name, parent_id=parent_id ) - env_result = csp.create_environment(payload) environment.cloud_id = env_result.id db.session.add(environment) db.session.commit() +def do_create_environment_role(csp: CloudProviderInterface, environment_role_id=None): + env_role = EnvironmentRoles.get_by_id(environment_role_id) + + with claim_for_update(env_role) as env_role: + + if env_role.cloud_id is not None: + return + + env = env_role.environment + csp_details = env.application.portfolio.csp_data + app_role = env_role.application_role + + role = None + if env_role.role == CSPRole.ADMIN: + role = UserRoleCSPPayload.Roles.owner + elif env_role.role == CSPRole.BILLING_READ: + role = UserRoleCSPPayload.Roles.billing + elif env_role.role == CSPRole.CONTRIBUTOR: + role = UserRoleCSPPayload.Roles.contributor + + payload = UserRoleCSPPayload( + tenant_id=csp_details.get("tenant_id"), + management_group_id=env.cloud_id, + user_object_id=app_role.cloud_id, + role=role, + ) + result = csp.create_user_role(payload) + + env_role.cloud_id = result.id + db.session.add(env_role) + db.session.commit() + # TODO: should send notification email to the user, maybe with their portal login name + + def render_email(template_path, context): return app.jinja_env.get_template(template_path).render(context) @@ -165,6 +200,16 @@ def create_user(self, application_role_ids=None): ) +@celery.task(bind=True, base=RecordFailure) +def create_environment_role(self, environment_role_id=None): + do_work( + do_create_environment_role, + self, + app.csp.cloud, + environment_role_id=environment_role_id, + ) + + @celery.task(bind=True, base=RecordFailure) def create_environment(self, environment_id=None): do_work(do_create_environment, self, app.csp.cloud, environment_id=environment_id) @@ -191,6 +236,12 @@ def dispatch_create_user(self): create_user.delay(application_role_ids=application_role_ids) +@celery.task(bind=True) +def dispatch_create_environment_role(self): + for environment_role_id in EnvironmentRoles.get_pending_creation(): + create_environment_role.delay(environment_role_id=environment_role_id) + + @celery.task(bind=True) def dispatch_create_environment(self): for environment_id in Environments.get_environments_pending_creation( diff --git a/atst/models/environment_role.py b/atst/models/environment_role.py index 56fe78d4..871f39a1 100644 --- a/atst/models/environment_role.py +++ b/atst/models/environment_role.py @@ -36,7 +36,7 @@ class EnvironmentRole( ) application_role = relationship("ApplicationRole") - csp_user_id = Column(String()) + cloud_id = Column(String()) class Status(Enum): PENDING = "pending" diff --git a/atst/models/mixins/state_machines.py b/atst/models/mixins/state_machines.py index a7edf268..64355d54 100644 --- a/atst/models/mixins/state_machines.py +++ b/atst/models/mixins/state_machines.py @@ -24,6 +24,8 @@ class AzureStages(Enum): TENANT_PRINCIPAL_CREDENTIAL = "tenant principal credential" ADMIN_ROLE_DEFINITION = "admin role definition" PRINCIPAL_ADMIN_ROLE = "tenant principal admin" + INITIAL_MGMT_GROUP = "initial management group" + INITIAL_MGMT_GROUP_VERIFICATION = "initial management group verification" TENANT_ADMIN_OWNERSHIP = "tenant admin ownership" TENANT_PRINCIPAL_OWNERSHIP = "tenant principial ownership" diff --git a/atst/models/portfolio.py b/atst/models/portfolio.py index 2d4f8d80..8c9694a6 100644 --- a/atst/models/portfolio.py +++ b/atst/models/portfolio.py @@ -94,6 +94,12 @@ class Portfolio( def active_task_orders(self): return [task_order for task_order in self.task_orders if task_order.is_active] + @property + def total_obligated_funds(self): + return sum( + (task_order.total_obligated_funds for task_order in self.active_task_orders) + ) + @property def funding_duration(self): """ diff --git a/atst/models/portfolio_state_machine.py b/atst/models/portfolio_state_machine.py index e154b71a..96192d18 100644 --- a/atst/models/portfolio_state_machine.py +++ b/atst/models/portfolio_state_machine.py @@ -157,37 +157,38 @@ class PortfolioStateMachine( print(exc.json()) app.logger.info(payload) self.fail_stage(stage) + else: + # TODO: Determine best place to do this, maybe @reconstructor + self.csp = app.csp.cloud - # TODO: Determine best place to do this, maybe @reconstructor - self.csp = app.csp.cloud + try: + func_name = f"create_{stage}" + response = getattr(self.csp, func_name)(payload_data) + if self.portfolio.csp_data is None: + self.portfolio.csp_data = {} + self.portfolio.csp_data.update(response.dict()) + db.session.add(self.portfolio) + db.session.commit() + except PydanticValidationError as exc: + app.logger.error( + f"Failed to cast response to valid result class {self.__repr__()}:", + exc_info=1, + ) + app.logger.info(exc.json()) + print(exc.json()) + app.logger.info(payload_data) + # TODO: Ensure that failing the stage does not preclude a Celery retry + self.fail_stage(stage) + # TODO: catch and handle general CSP exception here + except (ConnectionException, UnknownServerException) as exc: + app.logger.error( + f"CSP api call. Caught exception for {self.__repr__()}.", + exc_info=1, + ) + # TODO: Ensure that failing the stage does not preclude a Celery retry + self.fail_stage(stage) - try: - func_name = f"create_{stage}" - response = getattr(self.csp, func_name)(payload_data) - if self.portfolio.csp_data is None: - self.portfolio.csp_data = {} - self.portfolio.csp_data.update(response.dict()) - db.session.add(self.portfolio) - db.session.commit() - except PydanticValidationError as exc: - app.logger.error( - f"Failed to cast response to valid result class {self.__repr__()}:", - exc_info=1, - ) - app.logger.info(exc.json()) - print(exc.json()) - app.logger.info(payload_data) - # TODO: Ensure that failing the stage does not preclude a Celery retry - self.fail_stage(stage) - # TODO: catch and handle general CSP exception here - except (ConnectionException, UnknownServerException) as exc: - app.logger.error( - f"CSP api call. Caught exception for {self.__repr__()}.", exc_info=1, - ) - # TODO: Ensure that failing the stage does not preclude a Celery retry - self.fail_stage(stage) - - self.finish_stage(stage) + self.finish_stage(stage) def is_csp_data_valid(self, event): """ diff --git a/atst/models/task_order.py b/atst/models/task_order.py index d6aa63f8..3ab493a9 100644 --- a/atst/models/task_order.py +++ b/atst/models/task_order.py @@ -25,7 +25,6 @@ SORT_ORDERING = [ Status.DRAFT, Status.UPCOMING, Status.EXPIRED, - Status.UNSIGNED, ] @@ -148,7 +147,10 @@ class TaskOrder(Base, mixins.TimestampsMixin): @property def display_status(self): - return self.status.value + if self.status == Status.UNSIGNED: + return Status.DRAFT.value + else: + return self.status.value @property def portfolio_name(self): diff --git a/atst/queue.py b/atst/queue.py index 0ea910fb..10bcb350 100644 --- a/atst/queue.py +++ b/atst/queue.py @@ -23,6 +23,10 @@ def update_celery(celery, app): "task": "atst.jobs.dispatch_create_user", "schedule": 60, }, + "beat-dispatch_create_environment_role": { + "task": "atst.jobs.dispatch_create_environment_role", + "schedule": 60, + }, } class ContextTask(celery.Task): diff --git a/atst/routes/portfolios/index.py b/atst/routes/portfolios/index.py index f9e7d5cf..795d4b70 100644 --- a/atst/routes/portfolios/index.py +++ b/atst/routes/portfolios/index.py @@ -34,25 +34,25 @@ def create_portfolio(): @user_can(Permissions.VIEW_PORTFOLIO_REPORTS, message="view portfolio reports") def reports(portfolio_id): portfolio = Portfolios.get(g.current_user, portfolio_id) + spending = Reports.get_portfolio_spending(portfolio) + obligated = portfolio.total_obligated_funds + remaining = obligated - (spending["invoiced"] + spending["estimated"]) - current_obligated_funds = Reports.obligated_funds_by_JEDI_clin(portfolio) + current_obligated_funds = { + **spending, + "obligated": obligated, + "remaining": remaining, + } - if any(map(lambda clin: clin["remaining"] < 0, current_obligated_funds)): + if current_obligated_funds["remaining"] < 0: flash("insufficient_funds") - # wrapped in str() because the sum of obligated funds returns a Decimal object - total_portfolio_value = str( - sum( - task_order.total_obligated_funds - for task_order in portfolio.active_task_orders - ) - ) return render_template( "portfolios/reports/index.html", portfolio=portfolio, - total_portfolio_value=total_portfolio_value, + # wrapped in str() because the sum of obligated funds returns a Decimal object + total_portfolio_value=str(portfolio.total_obligated_funds), current_obligated_funds=current_obligated_funds, expired_task_orders=Reports.expired_task_orders(portfolio), - monthly_spending=Reports.monthly_spending(portfolio), retrieved=datetime.now(), # mocked datetime of reporting data retrival ) diff --git a/config/base.ini b/config/base.ini index 1f4c732a..727172d8 100644 --- a/config/base.ini +++ b/config/base.ini @@ -1,9 +1,19 @@ [default] ASSETS_URL +AZURE_AADP_QTY=5 AZURE_ACCOUNT_NAME -AZURE_STORAGE_KEY -AZURE_TO_BUCKET_NAME +AZURE_CLIENT_ID +AZURE_GRAPH_RESOURCE="https://graph.microsoft.com/" AZURE_POLICY_LOCATION=policies +AZURE_POWERSHELL_CLIENT_ID +AZURE_ROLE_DEF_ID_BILLING_READER="fa23ad8b-c56e-40d8-ac0c-ce449e1d2c64" +AZURE_ROLE_DEF_ID_CONTRIBUTOR="b24988ac-6180-42a0-ab88-20f7382dd24c" +AZURE_ROLE_DEF_ID_OWNER="8e3af657-a8ff-443c-a75c-2fe8c4bcb635" +AZURE_SECRET_KEY +AZURE_STORAGE_KEY +AZURE_TENANT_ID +AZURE_TO_BUCKET_NAME +AZURE_VAULT_URL BLOB_STORAGE_URL=http://localhost:8000/ CAC_URL = http://localhost:8000/login-redirect CA_CHAIN = ssl/server-certs/ca-chain.pem @@ -38,14 +48,15 @@ PGUSER = postgres PORT=8000 REDIS_HOST=localhost:6379 REDIS_PASSWORD +REDIS_SSLMODE REDIS_TLS=False REDIS_USER SECRET_KEY = change_me_into_something_secret SERVER_NAME -SESSION_COOKIE_NAME=atat SESSION_COOKIE_DOMAIN -SESSION_KEY_PREFIX=session: +SESSION_COOKIE_NAME=atat SESSION_COOKIE_SECURE=false +SESSION_KEY_PREFIX=session: SESSION_TYPE = redis SESSION_USE_SIGNER = True SQLALCHEMY_ECHO = False diff --git a/deploy/azure/kustomization.yaml b/deploy/azure/kustomization.yaml index d0162394..b46021b0 100644 --- a/deploy/azure/kustomization.yaml +++ b/deploy/azure/kustomization.yaml @@ -10,6 +10,5 @@ resources: - volume-claim.yml - nginx-client-ca-bundle.yml - acme-challenges.yml - - aadpodidentity.yml - nginx-snippets.yml - autoscaling.yml diff --git a/deploy/overlays/cloudzero-dev/envvars.yml b/deploy/overlays/cloudzero-dev/envvars.yml index 179811ed..ded47f7b 100644 --- a/deploy/overlays/cloudzero-dev/envvars.yml +++ b/deploy/overlays/cloudzero-dev/envvars.yml @@ -4,19 +4,30 @@ kind: ConfigMap metadata: name: atst-worker-envvars data: + AZURE_ACCOUNT_NAME: jeditasksatat CELERY_DEFAULT_QUEUE: celery-staging - SERVER_NAME: staging.atat.code.mil FLASK_ENV: staging + PGDATABASE: cloudzero_jedidev_atat + PGHOST: 191.238.6.43 + PGUSER: atat@cloudzero-jedidev-sql + PGSSLMODE: require + REDIS_HOST: 10.1.3.34:6380 + SERVER_NAME: dev.atat.cloud.mil --- apiVersion: v1 kind: ConfigMap metadata: name: atst-envvars data: - ASSETS_URL: https://atat-cdn-staging.azureedge.net/ - CDN_ORIGIN: https://staging.atat.code.mil + ASSETS_URL: "" + AZURE_ACCOUNT_NAME: jeditasksatat + CAC_URL: https://auth-dev.atat.cloud.mil + CDN_ORIGIN: https://dev.atat.cloud.mil CELERY_DEFAULT_QUEUE: celery-staging FLASK_ENV: staging - STATIC_URL: https://atat-cdn-staging.azureedge.net/static/ - PGHOST: cloudzero-dev-sql.postgres.database.azure.com - REDIS_HOST: cloudzero-dev-redis.redis.cache.windows.net:6380 + PGDATABASE: cloudzero_jedidev_atat + PGHOST: 191.238.6.43 + PGUSER: atat@cloudzero-jedidev-sql + PGSSLMODE: require + REDIS_HOST: 10.1.3.34:6380 + SESSION_COOKIE_DOMAIN: atat.cloud.mil diff --git a/deploy/overlays/cloudzero-dev/flex_vol.yml b/deploy/overlays/cloudzero-dev/flex_vol.yml index a3c65df7..990d21e5 100644 --- a/deploy/overlays/cloudzero-dev/flex_vol.yml +++ b/deploy/overlays/cloudzero-dev/flex_vol.yml @@ -9,23 +9,19 @@ spec: - name: nginx-secret flexVolume: options: - keyvaultname: "cloudzero-dev-keyvault" - # keyvaultobjectnames: "dhparam4096;cert;cert" - keyvaultobjectnames: "foo" - keyvaultobjectaliases: "FOO" - keyvaultobjecttypes: "secret" - usevmmanagedidentity: "true" usepodidentity: "false" + usevmmanagedidentity: "true" + vmmanagedidentityclientid: $VMSS_CLIENT_ID + keyvaultname: "cz-jedidev-keyvault" + keyvaultobjectnames: "dhparam4096;ATATCERT;ATATCERT" - name: flask-secret flexVolume: options: - keyvaultname: "cloudzero-dev-keyvault" - # keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY" - keyvaultobjectnames: "master-PGPASSWORD" - keyvaultobjectaliases: "PGPASSWORD" - keyvaultobjecttypes: "secret" - usevmmanagedidentity: "true" usepodidentity: "false" + usevmmanagedidentity: "true" + vmmanagedidentityclientid: $VMSS_CLIENT_ID + keyvaultname: "cz-jedidev-keyvault" + keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY" --- apiVersion: extensions/v1beta1 kind: Deployment @@ -38,10 +34,11 @@ spec: - name: flask-secret flexVolume: options: - keyvaultname: "cloudzero-dev-keyvault" - keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY" - usevmmanagedidentity: "true" usepodidentity: "false" + usevmmanagedidentity: "true" + vmmanagedidentityclientid: $VMSS_CLIENT_ID + keyvaultname: "cz-jedidev-keyvault" + keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY" --- apiVersion: extensions/v1beta1 kind: Deployment @@ -54,10 +51,11 @@ spec: - name: flask-secret flexVolume: options: - keyvaultname: "cloudzero-dev-keyvault" - keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY" - usevmmanagedidentity: "true" usepodidentity: "false" + usevmmanagedidentity: "true" + vmmanagedidentityclientid: $VMSS_CLIENT_ID + keyvaultname: "cz-jedidev-keyvault" + keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY" --- apiVersion: batch/v1beta1 kind: CronJob @@ -72,7 +70,8 @@ spec: - name: flask-secret flexVolume: options: - keyvaultname: "cloudzero-dev-keyvault" - keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY" - usevmmanagedidentity: "true" usepodidentity: "false" + usevmmanagedidentity: "true" + vmmanagedidentityclientid: $VMSS_CLIENT_ID + keyvaultname: "cz-jedidev-keyvault" + keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY" diff --git a/deploy/overlays/cloudzero-dev/kustomization.yaml b/deploy/overlays/cloudzero-dev/kustomization.yaml index 24705531..65262fbe 100644 --- a/deploy/overlays/cloudzero-dev/kustomization.yaml +++ b/deploy/overlays/cloudzero-dev/kustomization.yaml @@ -1,9 +1,8 @@ -namespace: staging +namespace: cloudzero-dev bases: - ../../azure/ resources: - namespace.yml - - reset-cron-job.yml patchesStrategicMerge: - ports.yml - envvars.yml diff --git a/deploy/overlays/cloudzero-dev/namespace.yml b/deploy/overlays/cloudzero-dev/namespace.yml index ee38adfb..242c3a2f 100644 --- a/deploy/overlays/cloudzero-dev/namespace.yml +++ b/deploy/overlays/cloudzero-dev/namespace.yml @@ -1,4 +1,4 @@ apiVersion: v1 kind: Namespace metadata: - name: staging + name: cloudzero-dev diff --git a/deploy/overlays/cloudzero-dev/ports.yml b/deploy/overlays/cloudzero-dev/ports.yml index 8dbbd0f1..5225cf3c 100644 --- a/deploy/overlays/cloudzero-dev/ports.yml +++ b/deploy/overlays/cloudzero-dev/ports.yml @@ -5,7 +5,7 @@ metadata: name: atst-main annotations: service.beta.kubernetes.io/azure-load-balancer-internal: "true" - service.beta.kubernetes.io/azure-load-balancer-internal-subnet: "cloudzero-dev-public" + service.beta.kubernetes.io/azure-load-balancer-internal-subnet: "cloudzero-jedidev-public" spec: loadBalancerIP: "" ports: @@ -22,7 +22,7 @@ metadata: name: atst-auth annotations: service.beta.kubernetes.io/azure-load-balancer-internal: "true" - service.beta.kubernetes.io/azure-load-balancer-internal-subnet: "cloudzero-dev-public" + service.beta.kubernetes.io/azure-load-balancer-internal-subnet: "cloudzero-jedidev-public" spec: loadBalancerIP: "" ports: diff --git a/deploy/overlays/cloudzero-dev/reset-cron-job.yml b/deploy/overlays/cloudzero-dev/reset-cron-job.yml deleted file mode 100644 index b4792e5d..00000000 --- a/deploy/overlays/cloudzero-dev/reset-cron-job.yml +++ /dev/null @@ -1,46 +0,0 @@ -apiVersion: batch/v1beta1 -kind: CronJob -metadata: - name: reset-db - namespace: atat -spec: - schedule: "0 4 * * *" - concurrencyPolicy: Replace - successfulJobsHistoryLimit: 1 - jobTemplate: - spec: - template: - metadata: - labels: - app: atst - role: reset-db - aadpodidbinding: atat-kv-id-binding - spec: - restartPolicy: OnFailure - containers: - - name: reset - image: $CONTAINER_IMAGE - command: [ - "/bin/sh", "-c" - ] - args: [ - "/opt/atat/atst/.venv/bin/python", - "/opt/atat/atst/script/reset_database.py" - ] - envFrom: - - configMapRef: - name: atst-worker-envvars - volumeMounts: - - name: flask-secret - mountPath: "/config" - volumes: - - name: flask-secret - flexVolume: - driver: "azure/kv" - options: - usepodidentity: "true" - keyvaultname: "atat-vault-test" - keyvaultobjectnames: "staging-AZURE-STORAGE-KEY;staging-MAIL-PASSWORD;staging-PGPASSWORD;staging-REDIS-PASSWORD;staging-SECRET-KEY" - keyvaultobjectaliases: "AZURE_STORAGE_KEY;MAIL_PASSWORD;PGPASSWORD;REDIS_PASSWORD;SECRET_KEY" - keyvaultobjecttypes: "secret;secret;secret;secret;key" - tenantid: $TENANT_ID diff --git a/deploy/overlays/migration-cloudzero-dev/kustomization.yaml b/deploy/overlays/migration-cloudzero-dev/kustomization.yaml new file mode 100644 index 00000000..b12c0b88 --- /dev/null +++ b/deploy/overlays/migration-cloudzero-dev/kustomization.yaml @@ -0,0 +1,5 @@ +namespace: cloudzero-dev +bases: + - ../../shared/ +patchesStrategicMerge: + - migration.yaml diff --git a/deploy/overlays/migration-cloudzero-dev/migration.yaml b/deploy/overlays/migration-cloudzero-dev/migration.yaml new file mode 100644 index 00000000..53a39dcc --- /dev/null +++ b/deploy/overlays/migration-cloudzero-dev/migration.yaml @@ -0,0 +1,16 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: migration +spec: + template: + spec: + volumes: + - name: flask-secret + flexVolume: + options: + usepodidentity: "false" + usevmmanagedidentity: "true" + vmmanagedidentityclientid: $VMSS_CLIENT_ID + keyvaultname: "cz-jedidev-keyvault" + keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY" diff --git a/deploy/shared/kustomization.yaml b/deploy/shared/kustomization.yaml new file mode 100644 index 00000000..38dddc7e --- /dev/null +++ b/deploy/shared/kustomization.yaml @@ -0,0 +1,3 @@ +namespace: atat +resources: + - migration.yaml diff --git a/js/components/forms/multi_step_modal_form.js b/js/components/forms/multi_step_modal_form.js index 0dfcaba0..5cc5f4df 100644 --- a/js/components/forms/multi_step_modal_form.js +++ b/js/components/forms/multi_step_modal_form.js @@ -34,8 +34,10 @@ export default { methods: { next: function() { + this.submitted = true if (this.validateFields()) { this.step += 1 + this.submitted = false } }, previous: function() { diff --git a/js/components/sidenav_toggler.js b/js/components/sidenav_toggler.js index 11717849..d545b3ed 100644 --- a/js/components/sidenav_toggler.js +++ b/js/components/sidenav_toggler.js @@ -1,5 +1,6 @@ import ExpandSidenavMixin from '../mixins/expand_sidenav' import ToggleMixin from '../mixins/toggle' +import { sidenavCookieName } from '../lib/constants' export default { name: 'sidenav-toggler', @@ -14,7 +15,7 @@ export default { toggle: function(e) { e.preventDefault() this.isVisible = !this.isVisible - document.cookie = this.cookieName + '=' + this.isVisible + '; path=/' + document.cookie = sidenavCookieName + '=' + this.isVisible + '; path=/' this.$parent.$emit('sidenavToggle', this.isVisible) }, }, diff --git a/js/components/toggle_menu.js b/js/components/toggle_menu.js index e17a201a..6c23ce06 100644 --- a/js/components/toggle_menu.js +++ b/js/components/toggle_menu.js @@ -5,6 +5,13 @@ export default { mixins: [ToggleMixin], + props: { + defaultVisible: { + type: Boolean, + default: false, + }, + }, + methods: { toggle: function(e) { if (this.$el.contains(e.target)) { diff --git a/js/lib/constants.js b/js/lib/constants.js new file mode 100644 index 00000000..b4de4fcf --- /dev/null +++ b/js/lib/constants.js @@ -0,0 +1 @@ +export const sidenavCookieName = 'expandSidenav' diff --git a/js/mixins/expand_sidenav.js b/js/mixins/expand_sidenav.js index 7553b7d4..2af9c87a 100644 --- a/js/mixins/expand_sidenav.js +++ b/js/mixins/expand_sidenav.js @@ -1,11 +1,12 @@ +import { sidenavCookieName } from '../lib/constants' + export default { props: { - cookieName: 'expandSidenav', defaultVisible: { type: Boolean, default: function() { - if (document.cookie.match(this.cookieName)) { - return !!document.cookie.match(this.cookieName + ' *= *true') + if (document.cookie.match(sidenavCookieName)) { + return !!document.cookie.match(sidenavCookieName + ' *= *true') } else { return true } diff --git a/js/mixins/form.js b/js/mixins/form.js index 45aa6261..070466de 100644 --- a/js/mixins/form.js +++ b/js/mixins/form.js @@ -15,6 +15,7 @@ export default { return { changed: this.hasChanges, valid: false, + submitted: false, } }, @@ -36,15 +37,16 @@ export default { handleSubmit: function(event) { if (!this.valid) { event.preventDefault() + this.submitted = true } }, }, computed: { canSave: function() { - if (this.changed && this.valid) { + if (this.changed && this.valid && !this.submitted) { return true - } else if (this.enableSave && this.valid) { + } else if (this.enableSave && this.valid && !this.submitted) { return true } else { return false diff --git a/script/database_setup.py b/script/database_setup.py index 7784be05..e4964516 100644 --- a/script/database_setup.py +++ b/script/database_setup.py @@ -16,16 +16,14 @@ from reset_database import reset_database def database_setup(username, password, dbname, ccpo_users): + print("Applying schema and seeding roles and permissions.") + reset_database() + print( f"Creating Postgres user role for '{username}' and granting all privileges to database '{dbname}'." ) - try: - _create_database_user(username, password, dbname) - except sqlalchemy.exc.ProgrammingError as err: - print(f"Postgres user role '{username}' already exists.") + _create_database_user(username, password, dbname) - print("Applying schema and seeding roles and permissions.") - reset_database() print("Creating initial set of CCPO users.") _add_ccpo_users(ccpo_users) @@ -47,6 +45,22 @@ def _create_database_user(username, password, dbname): f"ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL PRIVILEGES ON FUNCTIONS TO {username}; \n" ) + try: + # TODO: make this more configurable + engine.execute(f"GRANT {username} TO azure_pg_admin;") + except sqlalchemy.exc.ProgrammingError as err: + print(f"Cannot grant new role {username} to azure_pg_admin") + + for table in meta.tables: + engine.execute(f"ALTER TABLE {table} OWNER TO {username};\n") + + sequence_results = engine.execute( + "SELECT c.relname FROM pg_class c WHERE c.relkind = 'S';" + ).fetchall() + sequences = [p[0] for p in sequence_results] + for sequence in sequences: + engine.execute(f"ALTER SEQUENCE {sequence} OWNER TO {username};\n") + trans.commit() diff --git a/script/k8s_config b/script/k8s_config index b489c942..36dc5ef7 100755 --- a/script/k8s_config +++ b/script/k8s_config @@ -13,6 +13,7 @@ SETTINGS=( AUTH_DOMAIN KV_MI_ID KV_MI_CLIENT_ID + VMSS_CLIENT_ID TENANT_ID ) diff --git a/templates/components/accordion.html b/templates/components/accordion.html index 9ec92099..cdf72901 100644 --- a/templates/components/accordion.html +++ b/templates/components/accordion.html @@ -6,8 +6,12 @@ heading_tag="h2", heading_classes="", content_tag="div", - content_classes="") %} - + content_classes="", + default_visible=False) %} + <{{wrapper_tag}} class="{{ wrapper_classes }}"> <{{heading_tag}} class="accordion__button {{ heading_classes }}">