diff --git a/.secrets.baseline b/.secrets.baseline index 657683b7..4e393738 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": "^.secrets.baseline$|^.*pgsslrootcert.yml$", "lines": null }, - "generated_at": "2020-02-04T21:00:49Z", + "generated_at": "2020-02-10T21:40:38Z", "plugins_used": [ { "base64_limit": 4.5, diff --git a/alembic/versions/418b52c1cedf_change_to_environment_roles_cloud_id.py b/alembic/versions/418b52c1cedf_change_to_environment_roles_cloud_id.py index 8d522769..93f11712 100644 --- a/alembic/versions/418b52c1cedf_change_to_environment_roles_cloud_id.py +++ b/alembic/versions/418b52c1cedf_change_to_environment_roles_cloud_id.py @@ -1,7 +1,7 @@ """change to environment_roles.cloud_Id Revision ID: 418b52c1cedf -Revises: 567bfb019a87 +Revises: 542bd3215dec Create Date: 2020-02-05 13:40:37.870183 """ @@ -11,7 +11,7 @@ import sqlalchemy as sa # revision identifiers, used by Alembic. revision = '418b52c1cedf' # pragma: allowlist secret -down_revision = '567bfb019a87' # pragma: allowlist secret +down_revision = '542bd3215dec' # pragma: allowlist secret branch_labels = None depends_on = None diff --git a/alembic/versions/542bd3215dec_state_machine_stage_added.py b/alembic/versions/542bd3215dec_state_machine_stage_added.py new file mode 100644 index 00000000..8a3410af --- /dev/null +++ b/alembic/versions/542bd3215dec_state_machine_stage_added.py @@ -0,0 +1,264 @@ +"""state machine stage added. + +Revision ID: 542bd3215dec +Revises: 567bfb019a87 +Create Date: 2020-02-06 12:01:58.077840 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "542bd3215dec" # pragma: allowlist secret +down_revision = "567bfb019a87" # pragma: allowlist secret +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column( + "portfolio_state_machines", + "state", + type_=sa.Enum( + "UNSTARTED", + "STARTING", + "STARTED", + "COMPLETED", + "FAILED", + "TENANT_CREATED", + "TENANT_IN_PROGRESS", + "TENANT_FAILED", + "BILLING_PROFILE_CREATION_CREATED", + "BILLING_PROFILE_CREATION_IN_PROGRESS", + "BILLING_PROFILE_CREATION_FAILED", + "BILLING_PROFILE_VERIFICATION_CREATED", + "BILLING_PROFILE_VERIFICATION_IN_PROGRESS", + "BILLING_PROFILE_VERIFICATION_FAILED", + "BILLING_PROFILE_TENANT_ACCESS_CREATED", + "BILLING_PROFILE_TENANT_ACCESS_IN_PROGRESS", + "BILLING_PROFILE_TENANT_ACCESS_FAILED", + "TASK_ORDER_BILLING_CREATION_CREATED", + "TASK_ORDER_BILLING_CREATION_IN_PROGRESS", + "TASK_ORDER_BILLING_CREATION_FAILED", + "TASK_ORDER_BILLING_VERIFICATION_CREATED", + "TASK_ORDER_BILLING_VERIFICATION_IN_PROGRESS", + "TASK_ORDER_BILLING_VERIFICATION_FAILED", + "BILLING_INSTRUCTION_CREATED", + "BILLING_INSTRUCTION_IN_PROGRESS", + "BILLING_INSTRUCTION_FAILED", + "PRODUCT_PURCHASE_CREATED", + "PRODUCT_PURCHASE_IN_PROGRESS", + "PRODUCT_PURCHASE_FAILED", + "PRODUCT_PURCHASE_VERIFICATION_CREATED", + "PRODUCT_PURCHASE_VERIFICATION_IN_PROGRESS", + "PRODUCT_PURCHASE_VERIFICATION_FAILED", + "TENANT_PRINCIPAL_APP_CREATED", + "TENANT_PRINCIPAL_APP_IN_PROGRESS", + "TENANT_PRINCIPAL_APP_FAILED", + "TENANT_PRINCIPAL_CREATED", + "TENANT_PRINCIPAL_IN_PROGRESS", + "TENANT_PRINCIPAL_FAILED", + "TENANT_PRINCIPAL_CREDENTIAL_CREATED", + "TENANT_PRINCIPAL_CREDENTIAL_IN_PROGRESS", + "TENANT_PRINCIPAL_CREDENTIAL_FAILED", + "ADMIN_ROLE_DEFINITION_CREATED", + "ADMIN_ROLE_DEFINITION_IN_PROGRESS", + "ADMIN_ROLE_DEFINITION_FAILED", + "PRINCIPAL_ADMIN_ROLE_CREATED", + "PRINCIPAL_ADMIN_ROLE_IN_PROGRESS", + "PRINCIPAL_ADMIN_ROLE_FAILED", + "INITIAL_MGMT_GROUP_CREATED", + "INITIAL_MGMT_GROUP_IN_PROGRESS", + "INITIAL_MGMT_GROUP_FAILED", + "INITIAL_MGMT_GROUP_VERIFICATION_CREATED", + "INITIAL_MGMT_GROUP_VERIFICATION_IN_PROGRESS", + "INITIAL_MGMT_GROUP_VERIFICATION_FAILED", + "TENANT_ADMIN_OWNERSHIP_CREATED", + "TENANT_ADMIN_OWNERSHIP_IN_PROGRESS", + "TENANT_ADMIN_OWNERSHIP_FAILED", + "TENANT_PRINCIPAL_OWNERSHIP_CREATED", + "TENANT_PRINCIPAL_OWNERSHIP_IN_PROGRESS", + "TENANT_PRINCIPAL_OWNERSHIP_FAILED", + name="fsmstates", + native_enum=False, + ), + existing_type=sa.Enum( + "UNSTARTED", + "STARTING", + "STARTED", + "COMPLETED", + "FAILED", + "TENANT_CREATED", + "TENANT_IN_PROGRESS", + "TENANT_FAILED", + "BILLING_PROFILE_CREATION_CREATED", + "BILLING_PROFILE_CREATION_IN_PROGRESS", + "BILLING_PROFILE_CREATION_FAILED", + "BILLING_PROFILE_VERIFICATION_CREATED", + "BILLING_PROFILE_VERIFICATION_IN_PROGRESS", + "BILLING_PROFILE_VERIFICATION_FAILED", + "BILLING_PROFILE_TENANT_ACCESS_CREATED", + "BILLING_PROFILE_TENANT_ACCESS_IN_PROGRESS", + "BILLING_PROFILE_TENANT_ACCESS_FAILED", + "TASK_ORDER_BILLING_CREATION_CREATED", + "TASK_ORDER_BILLING_CREATION_IN_PROGRESS", + "TASK_ORDER_BILLING_CREATION_FAILED", + "TASK_ORDER_BILLING_VERIFICATION_CREATED", + "TASK_ORDER_BILLING_VERIFICATION_IN_PROGRESS", + "TASK_ORDER_BILLING_VERIFICATION_FAILED", + "BILLING_INSTRUCTION_CREATED", + "BILLING_INSTRUCTION_IN_PROGRESS", + "BILLING_INSTRUCTION_FAILED", + "TENANT_PRINCIPAL_APP_CREATED", + "TENANT_PRINCIPAL_APP_IN_PROGRESS", + "TENANT_PRINCIPAL_APP_FAILED", + "TENANT_PRINCIPAL_CREATED", + "TENANT_PRINCIPAL_IN_PROGRESS", + "TENANT_PRINCIPAL_FAILED", + "TENANT_PRINCIPAL_CREDENTIAL_CREATED", + "TENANT_PRINCIPAL_CREDENTIAL_IN_PROGRESS", + "TENANT_PRINCIPAL_CREDENTIAL_FAILED", + "ADMIN_ROLE_DEFINITION_CREATED", + "ADMIN_ROLE_DEFINITION_IN_PROGRESS", + "ADMIN_ROLE_DEFINITION_FAILED", + "PRINCIPAL_ADMIN_ROLE_CREATED", + "PRINCIPAL_ADMIN_ROLE_IN_PROGRESS", + "PRINCIPAL_ADMIN_ROLE_FAILED", + "TENANT_ADMIN_OWNERSHIP_CREATED", + "TENANT_ADMIN_OWNERSHIP_IN_PROGRESS", + "TENANT_ADMIN_OWNERSHIP_FAILED", + "TENANT_PRINCIPAL_OWNERSHIP_CREATED", + "TENANT_PRINCIPAL_OWNERSHIP_IN_PROGRESS", + "TENANT_PRINCIPAL_OWNERSHIP_FAILED", + name="fsmstates", + native_enum=False, + ), + existing_nullable=False, + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column( + "portfolio_state_machines", + "state", + type_=sa.Enum( + "UNSTARTED", + "STARTING", + "STARTED", + "COMPLETED", + "FAILED", + "TENANT_CREATED", + "TENANT_IN_PROGRESS", + "TENANT_FAILED", + "BILLING_PROFILE_CREATION_CREATED", + "BILLING_PROFILE_CREATION_IN_PROGRESS", + "BILLING_PROFILE_CREATION_FAILED", + "BILLING_PROFILE_VERIFICATION_CREATED", + "BILLING_PROFILE_VERIFICATION_IN_PROGRESS", + "BILLING_PROFILE_VERIFICATION_FAILED", + "BILLING_PROFILE_TENANT_ACCESS_CREATED", + "BILLING_PROFILE_TENANT_ACCESS_IN_PROGRESS", + "BILLING_PROFILE_TENANT_ACCESS_FAILED", + "TASK_ORDER_BILLING_CREATION_CREATED", + "TASK_ORDER_BILLING_CREATION_IN_PROGRESS", + "TASK_ORDER_BILLING_CREATION_FAILED", + "TASK_ORDER_BILLING_VERIFICATION_CREATED", + "TASK_ORDER_BILLING_VERIFICATION_IN_PROGRESS", + "TASK_ORDER_BILLING_VERIFICATION_FAILED", + "BILLING_INSTRUCTION_CREATED", + "BILLING_INSTRUCTION_IN_PROGRESS", + "BILLING_INSTRUCTION_FAILED", + "TENANT_PRINCIPAL_APP_CREATED", + "TENANT_PRINCIPAL_APP_IN_PROGRESS", + "TENANT_PRINCIPAL_APP_FAILED", + "TENANT_PRINCIPAL_CREATED", + "TENANT_PRINCIPAL_IN_PROGRESS", + "TENANT_PRINCIPAL_FAILED", + "TENANT_PRINCIPAL_CREDENTIAL_CREATED", + "TENANT_PRINCIPAL_CREDENTIAL_IN_PROGRESS", + "TENANT_PRINCIPAL_CREDENTIAL_FAILED", + "ADMIN_ROLE_DEFINITION_CREATED", + "ADMIN_ROLE_DEFINITION_IN_PROGRESS", + "ADMIN_ROLE_DEFINITION_FAILED", + "PRINCIPAL_ADMIN_ROLE_CREATED", + "PRINCIPAL_ADMIN_ROLE_IN_PROGRESS", + "PRINCIPAL_ADMIN_ROLE_FAILED", + "TENANT_ADMIN_OWNERSHIP_CREATED", + "TENANT_ADMIN_OWNERSHIP_IN_PROGRESS", + "TENANT_ADMIN_OWNERSHIP_FAILED", + "TENANT_PRINCIPAL_OWNERSHIP_CREATED", + "TENANT_PRINCIPAL_OWNERSHIP_IN_PROGRESS", + "TENANT_PRINCIPAL_OWNERSHIP_FAILED", + name="fsmstates", + native_enum=False, + ), + existing_type=sa.Enum( + "UNSTARTED", + "STARTING", + "STARTED", + "COMPLETED", + "FAILED", + "TENANT_CREATED", + "TENANT_IN_PROGRESS", + "TENANT_FAILED", + "BILLING_PROFILE_CREATION_CREATED", + "BILLING_PROFILE_CREATION_IN_PROGRESS", + "BILLING_PROFILE_CREATION_FAILED", + "BILLING_PROFILE_VERIFICATION_CREATED", + "BILLING_PROFILE_VERIFICATION_IN_PROGRESS", + "BILLING_PROFILE_VERIFICATION_FAILED", + "BILLING_PROFILE_TENANT_ACCESS_CREATED", + "BILLING_PROFILE_TENANT_ACCESS_IN_PROGRESS", + "BILLING_PROFILE_TENANT_ACCESS_FAILED", + "TASK_ORDER_BILLING_CREATION_CREATED", + "TASK_ORDER_BILLING_CREATION_IN_PROGRESS", + "TASK_ORDER_BILLING_CREATION_FAILED", + "TASK_ORDER_BILLING_VERIFICATION_CREATED", + "TASK_ORDER_BILLING_VERIFICATION_IN_PROGRESS", + "TASK_ORDER_BILLING_VERIFICATION_FAILED", + "BILLING_INSTRUCTION_CREATED", + "BILLING_INSTRUCTION_IN_PROGRESS", + "BILLING_INSTRUCTION_FAILED", + "PRODUCT_PURCHASE_CREATED", + "PRODUCT_PURCHASE_IN_PROGRESS", + "PRODUCT_PURCHASE_FAILED", + "PRODUCT_PURCHASE_VERIFICATION_CREATED", + "PRODUCT_PURCHASE_VERIFICATION_IN_PROGRESS", + "PRODUCT_PURCHASE_VERIFICATION_FAILED", + "TENANT_PRINCIPAL_APP_CREATED", + "TENANT_PRINCIPAL_APP_IN_PROGRESS", + "TENANT_PRINCIPAL_APP_FAILED", + "TENANT_PRINCIPAL_CREATED", + "TENANT_PRINCIPAL_IN_PROGRESS", + "TENANT_PRINCIPAL_FAILED", + "TENANT_PRINCIPAL_CREDENTIAL_CREATED", + "TENANT_PRINCIPAL_CREDENTIAL_IN_PROGRESS", + "TENANT_PRINCIPAL_CREDENTIAL_FAILED", + "ADMIN_ROLE_DEFINITION_CREATED", + "ADMIN_ROLE_DEFINITION_IN_PROGRESS", + "ADMIN_ROLE_DEFINITION_FAILED", + "PRINCIPAL_ADMIN_ROLE_CREATED", + "PRINCIPAL_ADMIN_ROLE_IN_PROGRESS", + "PRINCIPAL_ADMIN_ROLE_FAILED", + "INITIAL_MGMT_GROUP_CREATED", + "INITIAL_MGMT_GROUP_IN_PROGRESS", + "INITIAL_MGMT_GROUP_FAILED", + "INITIAL_MGMT_GROUP_VERIFICATION_CREATED", + "INITIAL_MGMT_GROUP_VERIFICATION_IN_PROGRESS", + "INITIAL_MGMT_GROUP_VERIFICATION_FAILED", + "TENANT_ADMIN_OWNERSHIP_CREATED", + "TENANT_ADMIN_OWNERSHIP_IN_PROGRESS", + "TENANT_ADMIN_OWNERSHIP_FAILED", + "TENANT_PRINCIPAL_OWNERSHIP_CREATED", + "TENANT_PRINCIPAL_OWNERSHIP_IN_PROGRESS", + "TENANT_PRINCIPAL_OWNERSHIP_FAILED", + name="fsmstates", + native_enum=False, + ), + existing_nullable=False, + ) + diff --git a/atst/app.py b/atst/app.py index 05578827..04aed44d 100644 --- a/atst/app.py +++ b/atst/app.py @@ -157,7 +157,6 @@ def map_config(config): **config["default"], "USE_AUDIT_LOG": config["default"].getboolean("USE_AUDIT_LOG"), "ENV": config["default"]["ENVIRONMENT"], - "BROKER_URL": config["default"]["REDIS_URI"], "DEBUG": config["default"].getboolean("DEBUG"), "DEBUG_MAILER": config["default"].getboolean("DEBUG_MAILER"), "SQLALCHEMY_ECHO": config["default"].getboolean("SQLALCHEMY_ECHO"), @@ -233,13 +232,34 @@ def make_config(direct_config=None): config.set("default", "DATABASE_URI", database_uri) # Assemble REDIS_URI value + redis_use_tls = config["default"].getboolean("REDIS_TLS") redis_uri = "redis{}://{}:{}@{}".format( # pragma: allowlist secret - ("s" if config["default"].getboolean("REDIS_TLS") else ""), + ("s" if redis_use_tls else ""), (config.get("default", "REDIS_USER") or ""), (config.get("default", "REDIS_PASSWORD") or ""), config.get("default", "REDIS_HOST"), ) + celery_uri = redis_uri + if redis_use_tls: + tls_mode = config.get("default", "REDIS_SSLMODE") + tls_mode_str = tls_mode.lower() if tls_mode else "none" + redis_uri = f"{redis_uri}/?ssl_cert_reqs={tls_mode_str}" + + # TODO: Kombu, one of Celery's dependencies, still requires + # that ssl_cert_reqs be passed as the string version of an + # option on the ssl module. We can clean this up and use + # the REDIS_URI for both when this PR to Kombu is released: + # https://github.com/celery/kombu/pull/1139 + kombu_modes = { + "none": "CERT_NONE", + "required": "CERT_REQUIRED", + "optional": "CERT_OPTIONAL", + } + celery_tls_mode_str = kombu_modes[tls_mode_str] + celery_uri = f"{celery_uri}/?ssl_cert_reqs={celery_tls_mode_str}" + config.set("default", "REDIS_URI", redis_uri) + config.set("default", "BROKER_URL", celery_uri) return map_config(config) diff --git a/atst/domain/csp/cloud/azure_cloud_provider.py b/atst/domain/csp/cloud/azure_cloud_provider.py index 5d9ca275..7c976b8d 100644 --- a/atst/domain/csp/cloud/azure_cloud_provider.py +++ b/atst/domain/csp/cloud/azure_cloud_provider.py @@ -1,6 +1,5 @@ import json from secrets import token_urlsafe -from typing import Any, Dict from uuid import uuid4 from atst.utils import sha256_hex @@ -27,6 +26,10 @@ from .models import ( BillingProfileVerificationCSPPayload, BillingProfileVerificationCSPResult, CostManagementQueryCSPResult, + InitialMgmtGroupCSPPayload, + InitialMgmtGroupCSPResult, + InitialMgmtGroupVerificationCSPPayload, + InitialMgmtGroupVerificationCSPResult, EnvironmentCSPPayload, EnvironmentCSPResult, KeyVaultCredentials, @@ -197,6 +200,38 @@ class AzureCloudProvider(CloudProviderInterface): return ApplicationCSPResult(**response) + def create_initial_mgmt_group(self, payload: InitialMgmtGroupCSPPayload): + creds = self._source_creds(payload.tenant_id) + credentials = self._get_credential_obj( + { + "client_id": creds.root_sp_client_id, + "secret_key": creds.root_sp_key, + "tenant_id": creds.root_tenant_id, + }, + resource=self.sdk.cloud.endpoints.resource_manager, + ) + response = self._create_management_group( + credentials, payload.management_group_name, payload.display_name, + ) + + return InitialMgmtGroupCSPResult(**response) + + def create_initial_mgmt_group_verification( + self, payload: InitialMgmtGroupVerificationCSPPayload + ): + creds = self._source_creds(payload.tenant_id) + credentials = self._get_credential_obj( + { + "client_id": creds.root_sp_client_id, + "secret_key": creds.root_sp_key, + "tenant_id": creds.root_tenant_id, + }, + resource=self.sdk.cloud.endpoints.resource_manager, + ) + + response = self._get_management_group(credentials, payload.tenant_id,) + return InitialMgmtGroupVerificationCSPResult(**response.result()) + def _create_management_group( self, credentials, management_group_id, display_name, parent_id=None, ): @@ -223,6 +258,11 @@ class AzureCloudProvider(CloudProviderInterface): # instead? return create_request.result() + def _get_management_group(self, credentials, management_group_id): + mgmgt_group_client = self.sdk.managementgroups.ManagementGroupsAPI(credentials) + response = mgmgt_group_client.management_groups.get(management_group_id) + return response + def _create_policy_definition( self, credentials, subscription_id, management_group_id, properties, ): @@ -1127,12 +1167,10 @@ class AzureCloudProvider(CloudProviderInterface): def update_tenant_creds(self, tenant_id, secret: KeyVaultCredentials): hashed = sha256_hex(tenant_id) - new_secrets = secret.dict() curr_secrets = self._source_tenant_creds(tenant_id) - updated_secrets: Dict[str, Any] = {**curr_secrets.dict(), **new_secrets} - us = KeyVaultCredentials(**updated_secrets) - self.set_secret(hashed, json.dumps(us.dict())) - return us + updated_secrets = curr_secrets.merge_credentials(secret) + self.set_secret(hashed, json.dumps(updated_secrets.dict())) + return updated_secrets def _source_tenant_creds(self, tenant_id) -> KeyVaultCredentials: hashed = sha256_hex(tenant_id) @@ -1161,7 +1199,7 @@ class AzureCloudProvider(CloudProviderInterface): "timeframe": "Custom", "timePeriod": {"from": payload.from_date, "to": payload.to_date,}, "dataset": { - "granularity": "Daily", + "granularity": "Monthly", "aggregation": {"totalCost": {"name": "PreTaxCost", "function": "Sum"}}, "grouping": [{"type": "Dimension", "name": "InvoiceId"}], }, diff --git a/atst/domain/csp/cloud/mock_cloud_provider.py b/atst/domain/csp/cloud/mock_cloud_provider.py index 3454ec74..7371357a 100644 --- a/atst/domain/csp/cloud/mock_cloud_provider.py +++ b/atst/domain/csp/cloud/mock_cloud_provider.py @@ -1,4 +1,5 @@ from uuid import uuid4 +import pendulum from .cloud_provider_interface import CloudProviderInterface from .exceptions import ( @@ -25,6 +26,10 @@ from .models import ( BillingProfileTenantAccessCSPResult, BillingProfileVerificationCSPPayload, BillingProfileVerificationCSPResult, + InitialMgmtGroupCSPPayload, + InitialMgmtGroupCSPResult, + InitialMgmtGroupVerificationCSPPayload, + InitialMgmtGroupVerificationCSPResult, CostManagementQueryCSPResult, CostManagementQueryProperties, ProductPurchaseCSPPayload, @@ -282,6 +287,29 @@ class MockCloudProvider(CloudProviderInterface): } ) + def create_initial_mgmt_group(self, payload: InitialMgmtGroupCSPPayload): + self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) + self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) + self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) + + return InitialMgmtGroupCSPResult( + id=f"{AZURE_MGMNT_PATH}{payload.management_group_name}", + ) + + def create_initial_mgmt_group_verification( + self, payload: InitialMgmtGroupVerificationCSPPayload + ): + self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) + self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) + self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) + + return InitialMgmtGroupVerificationCSPResult( + **dict( + id="Test Id" + # id=f"{AZURE_MGMNT_PATH}{payload.management_group_name}" + ) + ) + def create_product_purchase(self, payload: ProductPurchaseCSPPayload): self._maybe_raise(self.NETWORK_FAILURE_PCT, self.NETWORK_EXCEPTION) self._maybe_raise(self.SERVER_FAILURE_PCT, self.SERVER_EXCEPTION) @@ -468,15 +496,26 @@ class MockCloudProvider(CloudProviderInterface): self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) object_id = str(uuid4()) + start_of_month = pendulum.today(tz="utc").start_of("month").replace(tzinfo=None) + this_month = start_of_month.to_atom_string() + last_month = start_of_month.subtract(months=1).to_atom_string() + two_months_ago = start_of_month.subtract(months=2).to_atom_string() + properties = CostManagementQueryProperties( **dict( columns=[ {"name": "PreTaxCost", "type": "Number"}, - {"name": "UsageDate", "type": "Number"}, + {"name": "BillingMonth", "type": "Datetime"}, {"name": "InvoiceId", "type": "String"}, {"name": "Currency", "type": "String"}, ], - rows=[], + rows=[ + [1.0, two_months_ago, "", "USD"], + [500.0, two_months_ago, "e05009w9sf", "USD"], + [50.0, last_month, "", "USD"], + [1000.0, last_month, "e0500a4qhw", "USD"], + [500.0, this_month, "", "USD"], + ], ) ) diff --git a/atst/domain/csp/cloud/models.py b/atst/domain/csp/cloud/models.py index 4656c432..859158a0 100644 --- a/atst/domain/csp/cloud/models.py +++ b/atst/domain/csp/cloud/models.py @@ -321,7 +321,7 @@ class ManagementGroupCSPPayload(AliasModel): tenant_id: str management_group_name: Optional[str] display_name: str - parent_id: str + parent_id: Optional[str] @validator("management_group_name", pre=True, always=True) def supply_management_group_name_default(cls, name): @@ -341,16 +341,25 @@ class ManagementGroupCSPPayload(AliasModel): @validator("parent_id", pre=True, always=True) def enforce_parent_id_pattern(cls, id_): - if AZURE_MGMNT_PATH not in id_: - return f"{AZURE_MGMNT_PATH}{id_}" - else: - return id_ + if id_: + if AZURE_MGMNT_PATH not in id_: + return f"{AZURE_MGMNT_PATH}{id_}" + else: + return id_ class ManagementGroupCSPResponse(AliasModel): id: str +class ManagementGroupGetCSPPayload(BaseCSPPayload): + management_group_name: str + + +class ManagementGroupGetCSPResponse(AliasModel): + id: str + + class ApplicationCSPPayload(ManagementGroupCSPPayload): pass @@ -359,6 +368,22 @@ class ApplicationCSPResult(ManagementGroupCSPResponse): pass +class InitialMgmtGroupCSPPayload(ManagementGroupCSPPayload): + pass + + +class InitialMgmtGroupCSPResult(ManagementGroupCSPResponse): + pass + + +class InitialMgmtGroupVerificationCSPPayload(ManagementGroupGetCSPPayload): + pass + + +class InitialMgmtGroupVerificationCSPResult(ManagementGroupGetCSPResponse): + pass + + class EnvironmentCSPPayload(ManagementGroupCSPPayload): pass @@ -418,6 +443,15 @@ class KeyVaultCredentials(BaseModel): return values + def merge_credentials( + self, new_creds: "KeyVaultCredentials" + ) -> "KeyVaultCredentials": + updated_creds = {k: v for k, v in new_creds.dict().items() if v} + old_creds = self.dict() + old_creds.update(updated_creds) + + return KeyVaultCredentials(**old_creds) + class SubscriptionCreationCSPPayload(BaseCSPPayload): display_name: str diff --git a/atst/domain/csp/reports.py b/atst/domain/csp/reports.py index 3f9ccbf8..700947f7 100644 --- a/atst/domain/csp/reports.py +++ b/atst/domain/csp/reports.py @@ -1,6 +1,6 @@ -from collections import defaultdict import json from decimal import Decimal +import pendulum def load_fixture_data(): @@ -11,128 +11,25 @@ def load_fixture_data(): class MockReportingProvider: FIXTURE_SPEND_DATA = load_fixture_data() - @classmethod - def get_portfolio_monthly_spending(cls, portfolio): - """ - returns an array of application and environment spending for the - portfolio. Applications and their nested environments are sorted in - alphabetical order by name. - [ - { - name - this_month - last_month - total - environments [ - { - name - this_month - last_month - total - } - ] - } - ] - """ - fixture_apps = cls.FIXTURE_SPEND_DATA.get(portfolio.name, {}).get( - "applications", [] - ) +def prepare_azure_reporting_data(rows: list): + """ + Returns a dict representing invoiced and estimated funds for a portfolio given + a list of rows from CostManagementQueryCSPResult.properties.rows + { + invoiced: Decimal, + estimated: Decimal + } + """ - for application in portfolio.applications: - if application.name not in [app["name"] for app in fixture_apps]: - fixture_apps.append({"name": application.name, "environments": []}) + estimated = [] + while rows: + if pendulum.parse(rows[-1][1]) >= pendulum.now(tz="utc").start_of("month"): + estimated.append(rows.pop()) + else: + break - return sorted( - [ - cls._get_application_monthly_totals(portfolio, fixture_app) - for fixture_app in fixture_apps - if fixture_app["name"] - in [application.name for application in portfolio.applications] - ], - key=lambda app: app["name"], - ) - - @classmethod - def _get_environment_monthly_totals(cls, environment): - """ - returns a dictionary that represents spending totals for an environment e.g. - { - name - this_month - last_month - total - } - """ - return { - "name": environment["name"], - "this_month": sum(environment["spending"]["this_month"].values()), - "last_month": sum(environment["spending"]["last_month"].values()), - "total": sum(environment["spending"]["total"].values()), - } - - @classmethod - def _get_application_monthly_totals(cls, portfolio, fixture_app): - """ - returns a dictionary that represents spending totals for an application - and its environments e.g. - { - name - this_month - last_month - total - environments: [ - { - name - this_month - last_month - total - } - ] - } - """ - application_envs = [ - env - for env in portfolio.all_environments - if env.application.name == fixture_app["name"] - ] - - environments = [ - cls._get_environment_monthly_totals(env) - for env in fixture_app["environments"] - if env["name"] in [e.name for e in application_envs] - ] - - for env in application_envs: - if env.name not in [env["name"] for env in environments]: - environments.append({"name": env.name}) - - return { - "name": fixture_app["name"], - "this_month": sum(env.get("this_month", 0) for env in environments), - "last_month": sum(env.get("last_month", 0) for env in environments), - "total": sum(env.get("total", 0) for env in environments), - "environments": sorted(environments, key=lambda env: env["name"]), - } - - @classmethod - def get_spending_by_JEDI_clin(cls, portfolio): - """ - returns an dictionary of spending per JEDI CLIN for a portfolio - { - jedi_clin: { - invoiced - estimated - }, - } - """ - if portfolio.name in cls.FIXTURE_SPEND_DATA: - CLIN_spend_dict = defaultdict(lambda: defaultdict(Decimal)) - for application in cls.FIXTURE_SPEND_DATA[portfolio.name]["applications"]: - for environment in application["environments"]: - for clin, spend in environment["spending"]["this_month"].items(): - CLIN_spend_dict[clin]["estimated"] += Decimal(spend) - for clin, spend in environment["spending"]["total"].items(): - CLIN_spend_dict[clin]["invoiced"] += Decimal(spend) - return CLIN_spend_dict - return {} + return dict( + invoiced=Decimal(sum([row[0] for row in rows])), + estimated=Decimal(sum([row[0] for row in estimated])), + ) diff --git a/atst/domain/reports.py b/atst/domain/reports.py index 99b229e3..fc619649 100644 --- a/atst/domain/reports.py +++ b/atst/domain/reports.py @@ -1,12 +1,13 @@ from flask import current_app -from itertools import groupby +from atst.domain.csp.cloud.models import ( + ReportingCSPPayload, + CostManagementQueryCSPResult, +) +from atst.domain.csp.reports import prepare_azure_reporting_data +import pendulum class Reports: - @classmethod - def monthly_spending(cls, portfolio): - return current_app.csp.reports.get_portfolio_monthly_spending(portfolio) - @classmethod def expired_task_orders(cls, portfolio): return [ @@ -14,31 +15,19 @@ class Reports: ] @classmethod - def obligated_funds_by_JEDI_clin(cls, portfolio): - clin_spending = current_app.csp.reports.get_spending_by_JEDI_clin(portfolio) - active_clins = portfolio.active_clins - for jedi_clin, clins in groupby( - active_clins, key=lambda clin: clin.jedi_clin_type - ): - if not clin_spending.get(jedi_clin.name): - clin_spending[jedi_clin.name] = {} - clin_spending[jedi_clin.name]["obligated"] = sum( - clin.obligated_amount for clin in clins - ) + def get_portfolio_spending(cls, portfolio): + # TODO: Extend this function to make from_date and to_date configurable + from_date = pendulum.now().subtract(years=1).add(days=1).format("YYYY-MM-DD") + to_date = pendulum.now().format("YYYY-MM-DD") + rows = [] - output = [] - for clin in clin_spending.keys(): - invoiced = clin_spending[clin].get("invoiced", 0) - estimated = clin_spending[clin].get("estimated", 0) - obligated = clin_spending[clin].get("obligated", 0) - remaining = obligated - (invoiced + estimated) - output.append( - { - "name": clin, - "invoiced": invoiced, - "estimated": estimated, - "obligated": obligated, - "remaining": remaining, - } + if portfolio.csp_data: + payload = ReportingCSPPayload( + from_date=from_date, to_date=to_date, **portfolio.csp_data ) - return output + response: CostManagementQueryCSPResult = current_app.csp.cloud.get_reporting_data( + payload + ) + rows = response.properties.rows + + return prepare_azure_reporting_data(rows) diff --git a/atst/filters.py b/atst/filters.py index 3508f1e9..84191017 100644 --- a/atst/filters.py +++ b/atst/filters.py @@ -5,7 +5,7 @@ from flask import render_template from jinja2 import contextfilter from jinja2.exceptions import TemplateNotFound from urllib.parse import urlparse, urlunparse, parse_qs, urlencode -from decimal import DivisionByZero as DivisionByZeroException +from decimal import DivisionByZero as DivisionByZeroException, InvalidOperation def iconSvg(name): @@ -43,7 +43,7 @@ def obligatedFundingGraphWidth(values): numerator, denominator = values try: return (numerator / denominator) * 100 - except DivisionByZeroException: + except (DivisionByZeroException, InvalidOperation): return 0 diff --git a/atst/models/mixins/state_machines.py b/atst/models/mixins/state_machines.py index 8342853a..a01771b5 100644 --- a/atst/models/mixins/state_machines.py +++ b/atst/models/mixins/state_machines.py @@ -24,6 +24,8 @@ class AzureStages(Enum): TENANT_PRINCIPAL_CREDENTIAL = "tenant principal credential" ADMIN_ROLE_DEFINITION = "admin role definition" PRINCIPAL_ADMIN_ROLE = "tenant principal admin" + INITIAL_MGMT_GROUP = "initial management group" + INITIAL_MGMT_GROUP_VERIFICATION = "initial management group verification" TENANT_ADMIN_OWNERSHIP = "tenant admin ownership" TENANT_PRINCIPAL_OWNERSHIP = "tenant principial ownership" BILLING_OWNER = "billing owner" diff --git a/atst/models/portfolio.py b/atst/models/portfolio.py index 5a8f0f1e..2ddcaa41 100644 --- a/atst/models/portfolio.py +++ b/atst/models/portfolio.py @@ -89,6 +89,12 @@ class Portfolio( def active_task_orders(self): return [task_order for task_order in self.task_orders if task_order.is_active] + @property + def total_obligated_funds(self): + return sum( + (task_order.total_obligated_funds for task_order in self.active_task_orders) + ) + @property def funding_duration(self): """ diff --git a/atst/models/portfolio_state_machine.py b/atst/models/portfolio_state_machine.py index f5c1a461..4098c06c 100644 --- a/atst/models/portfolio_state_machine.py +++ b/atst/models/portfolio_state_machine.py @@ -155,37 +155,38 @@ class PortfolioStateMachine( print(exc.json()) app.logger.info(payload) self.fail_stage(stage) + else: + # TODO: Determine best place to do this, maybe @reconstructor + self.csp = app.csp.cloud - # TODO: Determine best place to do this, maybe @reconstructor - self.csp = app.csp.cloud + try: + func_name = f"create_{stage}" + response = getattr(self.csp, func_name)(payload_data) + if self.portfolio.csp_data is None: + self.portfolio.csp_data = {} + self.portfolio.csp_data.update(response.dict()) + db.session.add(self.portfolio) + db.session.commit() + except PydanticValidationError as exc: + app.logger.error( + f"Failed to cast response to valid result class {self.__repr__()}:", + exc_info=1, + ) + app.logger.info(exc.json()) + print(exc.json()) + app.logger.info(payload_data) + # TODO: Ensure that failing the stage does not preclude a Celery retry + self.fail_stage(stage) + # TODO: catch and handle general CSP exception here + except (ConnectionException, UnknownServerException) as exc: + app.logger.error( + f"CSP api call. Caught exception for {self.__repr__()}.", + exc_info=1, + ) + # TODO: Ensure that failing the stage does not preclude a Celery retry + self.fail_stage(stage) - try: - func_name = f"create_{stage}" - response = getattr(self.csp, func_name)(payload_data) - if self.portfolio.csp_data is None: - self.portfolio.csp_data = {} - self.portfolio.csp_data.update(response.dict()) - db.session.add(self.portfolio) - db.session.commit() - except PydanticValidationError as exc: - app.logger.error( - f"Failed to cast response to valid result class {self.__repr__()}:", - exc_info=1, - ) - app.logger.info(exc.json()) - print(exc.json()) - app.logger.info(payload_data) - # TODO: Ensure that failing the stage does not preclude a Celery retry - self.fail_stage(stage) - # TODO: catch and handle general CSP exception here - except (ConnectionException, UnknownServerException) as exc: - app.logger.error( - f"CSP api call. Caught exception for {self.__repr__()}.", exc_info=1, - ) - # TODO: Ensure that failing the stage does not preclude a Celery retry - self.fail_stage(stage) - - self.finish_stage(stage) + self.finish_stage(stage) def is_csp_data_valid(self, event): """ diff --git a/atst/models/task_order.py b/atst/models/task_order.py index d6aa63f8..3ab493a9 100644 --- a/atst/models/task_order.py +++ b/atst/models/task_order.py @@ -25,7 +25,6 @@ SORT_ORDERING = [ Status.DRAFT, Status.UPCOMING, Status.EXPIRED, - Status.UNSIGNED, ] @@ -148,7 +147,10 @@ class TaskOrder(Base, mixins.TimestampsMixin): @property def display_status(self): - return self.status.value + if self.status == Status.UNSIGNED: + return Status.DRAFT.value + else: + return self.status.value @property def portfolio_name(self): diff --git a/atst/routes/portfolios/index.py b/atst/routes/portfolios/index.py index f9e7d5cf..795d4b70 100644 --- a/atst/routes/portfolios/index.py +++ b/atst/routes/portfolios/index.py @@ -34,25 +34,25 @@ def create_portfolio(): @user_can(Permissions.VIEW_PORTFOLIO_REPORTS, message="view portfolio reports") def reports(portfolio_id): portfolio = Portfolios.get(g.current_user, portfolio_id) + spending = Reports.get_portfolio_spending(portfolio) + obligated = portfolio.total_obligated_funds + remaining = obligated - (spending["invoiced"] + spending["estimated"]) - current_obligated_funds = Reports.obligated_funds_by_JEDI_clin(portfolio) + current_obligated_funds = { + **spending, + "obligated": obligated, + "remaining": remaining, + } - if any(map(lambda clin: clin["remaining"] < 0, current_obligated_funds)): + if current_obligated_funds["remaining"] < 0: flash("insufficient_funds") - # wrapped in str() because the sum of obligated funds returns a Decimal object - total_portfolio_value = str( - sum( - task_order.total_obligated_funds - for task_order in portfolio.active_task_orders - ) - ) return render_template( "portfolios/reports/index.html", portfolio=portfolio, - total_portfolio_value=total_portfolio_value, + # wrapped in str() because the sum of obligated funds returns a Decimal object + total_portfolio_value=str(portfolio.total_obligated_funds), current_obligated_funds=current_obligated_funds, expired_task_orders=Reports.expired_task_orders(portfolio), - monthly_spending=Reports.monthly_spending(portfolio), retrieved=datetime.now(), # mocked datetime of reporting data retrival ) diff --git a/config/base.ini b/config/base.ini index 0105eea5..727172d8 100644 --- a/config/base.ini +++ b/config/base.ini @@ -48,6 +48,7 @@ PGUSER = postgres PORT=8000 REDIS_HOST=localhost:6379 REDIS_PASSWORD +REDIS_SSLMODE REDIS_TLS=False REDIS_USER SECRET_KEY = change_me_into_something_secret diff --git a/deploy/azure/kustomization.yaml b/deploy/azure/kustomization.yaml index d0162394..b46021b0 100644 --- a/deploy/azure/kustomization.yaml +++ b/deploy/azure/kustomization.yaml @@ -10,6 +10,5 @@ resources: - volume-claim.yml - nginx-client-ca-bundle.yml - acme-challenges.yml - - aadpodidentity.yml - nginx-snippets.yml - autoscaling.yml diff --git a/deploy/overlays/cloudzero-dev/envvars.yml b/deploy/overlays/cloudzero-dev/envvars.yml index 179811ed..ded47f7b 100644 --- a/deploy/overlays/cloudzero-dev/envvars.yml +++ b/deploy/overlays/cloudzero-dev/envvars.yml @@ -4,19 +4,30 @@ kind: ConfigMap metadata: name: atst-worker-envvars data: + AZURE_ACCOUNT_NAME: jeditasksatat CELERY_DEFAULT_QUEUE: celery-staging - SERVER_NAME: staging.atat.code.mil FLASK_ENV: staging + PGDATABASE: cloudzero_jedidev_atat + PGHOST: 191.238.6.43 + PGUSER: atat@cloudzero-jedidev-sql + PGSSLMODE: require + REDIS_HOST: 10.1.3.34:6380 + SERVER_NAME: dev.atat.cloud.mil --- apiVersion: v1 kind: ConfigMap metadata: name: atst-envvars data: - ASSETS_URL: https://atat-cdn-staging.azureedge.net/ - CDN_ORIGIN: https://staging.atat.code.mil + ASSETS_URL: "" + AZURE_ACCOUNT_NAME: jeditasksatat + CAC_URL: https://auth-dev.atat.cloud.mil + CDN_ORIGIN: https://dev.atat.cloud.mil CELERY_DEFAULT_QUEUE: celery-staging FLASK_ENV: staging - STATIC_URL: https://atat-cdn-staging.azureedge.net/static/ - PGHOST: cloudzero-dev-sql.postgres.database.azure.com - REDIS_HOST: cloudzero-dev-redis.redis.cache.windows.net:6380 + PGDATABASE: cloudzero_jedidev_atat + PGHOST: 191.238.6.43 + PGUSER: atat@cloudzero-jedidev-sql + PGSSLMODE: require + REDIS_HOST: 10.1.3.34:6380 + SESSION_COOKIE_DOMAIN: atat.cloud.mil diff --git a/deploy/overlays/cloudzero-dev/flex_vol.yml b/deploy/overlays/cloudzero-dev/flex_vol.yml index a3c65df7..990d21e5 100644 --- a/deploy/overlays/cloudzero-dev/flex_vol.yml +++ b/deploy/overlays/cloudzero-dev/flex_vol.yml @@ -9,23 +9,19 @@ spec: - name: nginx-secret flexVolume: options: - keyvaultname: "cloudzero-dev-keyvault" - # keyvaultobjectnames: "dhparam4096;cert;cert" - keyvaultobjectnames: "foo" - keyvaultobjectaliases: "FOO" - keyvaultobjecttypes: "secret" - usevmmanagedidentity: "true" usepodidentity: "false" + usevmmanagedidentity: "true" + vmmanagedidentityclientid: $VMSS_CLIENT_ID + keyvaultname: "cz-jedidev-keyvault" + keyvaultobjectnames: "dhparam4096;ATATCERT;ATATCERT" - name: flask-secret flexVolume: options: - keyvaultname: "cloudzero-dev-keyvault" - # keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY" - keyvaultobjectnames: "master-PGPASSWORD" - keyvaultobjectaliases: "PGPASSWORD" - keyvaultobjecttypes: "secret" - usevmmanagedidentity: "true" usepodidentity: "false" + usevmmanagedidentity: "true" + vmmanagedidentityclientid: $VMSS_CLIENT_ID + keyvaultname: "cz-jedidev-keyvault" + keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY" --- apiVersion: extensions/v1beta1 kind: Deployment @@ -38,10 +34,11 @@ spec: - name: flask-secret flexVolume: options: - keyvaultname: "cloudzero-dev-keyvault" - keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY" - usevmmanagedidentity: "true" usepodidentity: "false" + usevmmanagedidentity: "true" + vmmanagedidentityclientid: $VMSS_CLIENT_ID + keyvaultname: "cz-jedidev-keyvault" + keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY" --- apiVersion: extensions/v1beta1 kind: Deployment @@ -54,10 +51,11 @@ spec: - name: flask-secret flexVolume: options: - keyvaultname: "cloudzero-dev-keyvault" - keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY" - usevmmanagedidentity: "true" usepodidentity: "false" + usevmmanagedidentity: "true" + vmmanagedidentityclientid: $VMSS_CLIENT_ID + keyvaultname: "cz-jedidev-keyvault" + keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY" --- apiVersion: batch/v1beta1 kind: CronJob @@ -72,7 +70,8 @@ spec: - name: flask-secret flexVolume: options: - keyvaultname: "cloudzero-dev-keyvault" - keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY" - usevmmanagedidentity: "true" usepodidentity: "false" + usevmmanagedidentity: "true" + vmmanagedidentityclientid: $VMSS_CLIENT_ID + keyvaultname: "cz-jedidev-keyvault" + keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY" diff --git a/deploy/overlays/cloudzero-dev/kustomization.yaml b/deploy/overlays/cloudzero-dev/kustomization.yaml index 24705531..65262fbe 100644 --- a/deploy/overlays/cloudzero-dev/kustomization.yaml +++ b/deploy/overlays/cloudzero-dev/kustomization.yaml @@ -1,9 +1,8 @@ -namespace: staging +namespace: cloudzero-dev bases: - ../../azure/ resources: - namespace.yml - - reset-cron-job.yml patchesStrategicMerge: - ports.yml - envvars.yml diff --git a/deploy/overlays/cloudzero-dev/namespace.yml b/deploy/overlays/cloudzero-dev/namespace.yml index ee38adfb..242c3a2f 100644 --- a/deploy/overlays/cloudzero-dev/namespace.yml +++ b/deploy/overlays/cloudzero-dev/namespace.yml @@ -1,4 +1,4 @@ apiVersion: v1 kind: Namespace metadata: - name: staging + name: cloudzero-dev diff --git a/deploy/overlays/cloudzero-dev/ports.yml b/deploy/overlays/cloudzero-dev/ports.yml index 8dbbd0f1..5225cf3c 100644 --- a/deploy/overlays/cloudzero-dev/ports.yml +++ b/deploy/overlays/cloudzero-dev/ports.yml @@ -5,7 +5,7 @@ metadata: name: atst-main annotations: service.beta.kubernetes.io/azure-load-balancer-internal: "true" - service.beta.kubernetes.io/azure-load-balancer-internal-subnet: "cloudzero-dev-public" + service.beta.kubernetes.io/azure-load-balancer-internal-subnet: "cloudzero-jedidev-public" spec: loadBalancerIP: "" ports: @@ -22,7 +22,7 @@ metadata: name: atst-auth annotations: service.beta.kubernetes.io/azure-load-balancer-internal: "true" - service.beta.kubernetes.io/azure-load-balancer-internal-subnet: "cloudzero-dev-public" + service.beta.kubernetes.io/azure-load-balancer-internal-subnet: "cloudzero-jedidev-public" spec: loadBalancerIP: "" ports: diff --git a/deploy/overlays/cloudzero-dev/reset-cron-job.yml b/deploy/overlays/cloudzero-dev/reset-cron-job.yml deleted file mode 100644 index b4792e5d..00000000 --- a/deploy/overlays/cloudzero-dev/reset-cron-job.yml +++ /dev/null @@ -1,46 +0,0 @@ -apiVersion: batch/v1beta1 -kind: CronJob -metadata: - name: reset-db - namespace: atat -spec: - schedule: "0 4 * * *" - concurrencyPolicy: Replace - successfulJobsHistoryLimit: 1 - jobTemplate: - spec: - template: - metadata: - labels: - app: atst - role: reset-db - aadpodidbinding: atat-kv-id-binding - spec: - restartPolicy: OnFailure - containers: - - name: reset - image: $CONTAINER_IMAGE - command: [ - "/bin/sh", "-c" - ] - args: [ - "/opt/atat/atst/.venv/bin/python", - "/opt/atat/atst/script/reset_database.py" - ] - envFrom: - - configMapRef: - name: atst-worker-envvars - volumeMounts: - - name: flask-secret - mountPath: "/config" - volumes: - - name: flask-secret - flexVolume: - driver: "azure/kv" - options: - usepodidentity: "true" - keyvaultname: "atat-vault-test" - keyvaultobjectnames: "staging-AZURE-STORAGE-KEY;staging-MAIL-PASSWORD;staging-PGPASSWORD;staging-REDIS-PASSWORD;staging-SECRET-KEY" - keyvaultobjectaliases: "AZURE_STORAGE_KEY;MAIL_PASSWORD;PGPASSWORD;REDIS_PASSWORD;SECRET_KEY" - keyvaultobjecttypes: "secret;secret;secret;secret;key" - tenantid: $TENANT_ID diff --git a/deploy/overlays/migration-cloudzero-dev/kustomization.yaml b/deploy/overlays/migration-cloudzero-dev/kustomization.yaml new file mode 100644 index 00000000..b12c0b88 --- /dev/null +++ b/deploy/overlays/migration-cloudzero-dev/kustomization.yaml @@ -0,0 +1,5 @@ +namespace: cloudzero-dev +bases: + - ../../shared/ +patchesStrategicMerge: + - migration.yaml diff --git a/deploy/overlays/migration-cloudzero-dev/migration.yaml b/deploy/overlays/migration-cloudzero-dev/migration.yaml new file mode 100644 index 00000000..53a39dcc --- /dev/null +++ b/deploy/overlays/migration-cloudzero-dev/migration.yaml @@ -0,0 +1,16 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: migration +spec: + template: + spec: + volumes: + - name: flask-secret + flexVolume: + options: + usepodidentity: "false" + usevmmanagedidentity: "true" + vmmanagedidentityclientid: $VMSS_CLIENT_ID + keyvaultname: "cz-jedidev-keyvault" + keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY" diff --git a/deploy/shared/kustomization.yaml b/deploy/shared/kustomization.yaml new file mode 100644 index 00000000..38dddc7e --- /dev/null +++ b/deploy/shared/kustomization.yaml @@ -0,0 +1,3 @@ +namespace: atat +resources: + - migration.yaml diff --git a/js/components/forms/multi_step_modal_form.js b/js/components/forms/multi_step_modal_form.js index 0dfcaba0..5cc5f4df 100644 --- a/js/components/forms/multi_step_modal_form.js +++ b/js/components/forms/multi_step_modal_form.js @@ -34,8 +34,10 @@ export default { methods: { next: function() { + this.submitted = true if (this.validateFields()) { this.step += 1 + this.submitted = false } }, previous: function() { diff --git a/js/components/sidenav_toggler.js b/js/components/sidenav_toggler.js index 11717849..d545b3ed 100644 --- a/js/components/sidenav_toggler.js +++ b/js/components/sidenav_toggler.js @@ -1,5 +1,6 @@ import ExpandSidenavMixin from '../mixins/expand_sidenav' import ToggleMixin from '../mixins/toggle' +import { sidenavCookieName } from '../lib/constants' export default { name: 'sidenav-toggler', @@ -14,7 +15,7 @@ export default { toggle: function(e) { e.preventDefault() this.isVisible = !this.isVisible - document.cookie = this.cookieName + '=' + this.isVisible + '; path=/' + document.cookie = sidenavCookieName + '=' + this.isVisible + '; path=/' this.$parent.$emit('sidenavToggle', this.isVisible) }, }, diff --git a/js/components/toggle_menu.js b/js/components/toggle_menu.js index e17a201a..6c23ce06 100644 --- a/js/components/toggle_menu.js +++ b/js/components/toggle_menu.js @@ -5,6 +5,13 @@ export default { mixins: [ToggleMixin], + props: { + defaultVisible: { + type: Boolean, + default: false, + }, + }, + methods: { toggle: function(e) { if (this.$el.contains(e.target)) { diff --git a/js/lib/constants.js b/js/lib/constants.js new file mode 100644 index 00000000..b4de4fcf --- /dev/null +++ b/js/lib/constants.js @@ -0,0 +1 @@ +export const sidenavCookieName = 'expandSidenav' diff --git a/js/mixins/expand_sidenav.js b/js/mixins/expand_sidenav.js index 7553b7d4..2af9c87a 100644 --- a/js/mixins/expand_sidenav.js +++ b/js/mixins/expand_sidenav.js @@ -1,11 +1,12 @@ +import { sidenavCookieName } from '../lib/constants' + export default { props: { - cookieName: 'expandSidenav', defaultVisible: { type: Boolean, default: function() { - if (document.cookie.match(this.cookieName)) { - return !!document.cookie.match(this.cookieName + ' *= *true') + if (document.cookie.match(sidenavCookieName)) { + return !!document.cookie.match(sidenavCookieName + ' *= *true') } else { return true } diff --git a/js/mixins/form.js b/js/mixins/form.js index 45aa6261..070466de 100644 --- a/js/mixins/form.js +++ b/js/mixins/form.js @@ -15,6 +15,7 @@ export default { return { changed: this.hasChanges, valid: false, + submitted: false, } }, @@ -36,15 +37,16 @@ export default { handleSubmit: function(event) { if (!this.valid) { event.preventDefault() + this.submitted = true } }, }, computed: { canSave: function() { - if (this.changed && this.valid) { + if (this.changed && this.valid && !this.submitted) { return true - } else if (this.enableSave && this.valid) { + } else if (this.enableSave && this.valid && !this.submitted) { return true } else { return false diff --git a/script/database_setup.py b/script/database_setup.py index 7784be05..e4964516 100644 --- a/script/database_setup.py +++ b/script/database_setup.py @@ -16,16 +16,14 @@ from reset_database import reset_database def database_setup(username, password, dbname, ccpo_users): + print("Applying schema and seeding roles and permissions.") + reset_database() + print( f"Creating Postgres user role for '{username}' and granting all privileges to database '{dbname}'." ) - try: - _create_database_user(username, password, dbname) - except sqlalchemy.exc.ProgrammingError as err: - print(f"Postgres user role '{username}' already exists.") + _create_database_user(username, password, dbname) - print("Applying schema and seeding roles and permissions.") - reset_database() print("Creating initial set of CCPO users.") _add_ccpo_users(ccpo_users) @@ -47,6 +45,22 @@ def _create_database_user(username, password, dbname): f"ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL PRIVILEGES ON FUNCTIONS TO {username}; \n" ) + try: + # TODO: make this more configurable + engine.execute(f"GRANT {username} TO azure_pg_admin;") + except sqlalchemy.exc.ProgrammingError as err: + print(f"Cannot grant new role {username} to azure_pg_admin") + + for table in meta.tables: + engine.execute(f"ALTER TABLE {table} OWNER TO {username};\n") + + sequence_results = engine.execute( + "SELECT c.relname FROM pg_class c WHERE c.relkind = 'S';" + ).fetchall() + sequences = [p[0] for p in sequence_results] + for sequence in sequences: + engine.execute(f"ALTER SEQUENCE {sequence} OWNER TO {username};\n") + trans.commit() diff --git a/script/k8s_config b/script/k8s_config index b489c942..36dc5ef7 100755 --- a/script/k8s_config +++ b/script/k8s_config @@ -13,6 +13,7 @@ SETTINGS=( AUTH_DOMAIN KV_MI_ID KV_MI_CLIENT_ID + VMSS_CLIENT_ID TENANT_ID ) diff --git a/templates/components/accordion.html b/templates/components/accordion.html index 9ec92099..cdf72901 100644 --- a/templates/components/accordion.html +++ b/templates/components/accordion.html @@ -6,8 +6,12 @@ heading_tag="h2", heading_classes="", content_tag="div", - content_classes="") %} - + content_classes="", + default_visible=False) %} + <{{wrapper_tag}} class="{{ wrapper_classes }}"> <{{heading_tag}} class="accordion__button {{ heading_classes }}">