diff --git a/.secrets.baseline b/.secrets.baseline index a233e4cf..f7145df8 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": "^.secrets.baseline$|^.*pgsslrootcert.yml$", "lines": null }, - "generated_at": "2020-01-27T19:24:43Z", + "generated_at": "2020-02-10T21:40:38Z", "plugins_used": [ { "base64_limit": 4.5, @@ -82,7 +82,7 @@ "hashed_secret": "afc848c316af1a89d49826c5ae9d00ed769415f3", "is_secret": false, "is_verified": false, - "line_number": 32, + "line_number": 33, "type": "Secret Keyword" } ], diff --git a/atst/app.py b/atst/app.py index 05578827..04aed44d 100644 --- a/atst/app.py +++ b/atst/app.py @@ -157,7 +157,6 @@ def map_config(config): **config["default"], "USE_AUDIT_LOG": config["default"].getboolean("USE_AUDIT_LOG"), "ENV": config["default"]["ENVIRONMENT"], - "BROKER_URL": config["default"]["REDIS_URI"], "DEBUG": config["default"].getboolean("DEBUG"), "DEBUG_MAILER": config["default"].getboolean("DEBUG_MAILER"), "SQLALCHEMY_ECHO": config["default"].getboolean("SQLALCHEMY_ECHO"), @@ -233,13 +232,34 @@ def make_config(direct_config=None): config.set("default", "DATABASE_URI", database_uri) # Assemble REDIS_URI value + redis_use_tls = config["default"].getboolean("REDIS_TLS") redis_uri = "redis{}://{}:{}@{}".format( # pragma: allowlist secret - ("s" if config["default"].getboolean("REDIS_TLS") else ""), + ("s" if redis_use_tls else ""), (config.get("default", "REDIS_USER") or ""), (config.get("default", "REDIS_PASSWORD") or ""), config.get("default", "REDIS_HOST"), ) + celery_uri = redis_uri + if redis_use_tls: + tls_mode = config.get("default", "REDIS_SSLMODE") + tls_mode_str = tls_mode.lower() if tls_mode else "none" + redis_uri = f"{redis_uri}/?ssl_cert_reqs={tls_mode_str}" + + # TODO: Kombu, one of Celery's dependencies, still requires + # that ssl_cert_reqs be passed as the string version of an + # option on the ssl module. We can clean this up and use + # the REDIS_URI for both when this PR to Kombu is released: + # https://github.com/celery/kombu/pull/1139 + kombu_modes = { + "none": "CERT_NONE", + "required": "CERT_REQUIRED", + "optional": "CERT_OPTIONAL", + } + celery_tls_mode_str = kombu_modes[tls_mode_str] + celery_uri = f"{celery_uri}/?ssl_cert_reqs={celery_tls_mode_str}" + config.set("default", "REDIS_URI", redis_uri) + config.set("default", "BROKER_URL", celery_uri) return map_config(config) diff --git a/atst/domain/csp/cloud/azure_cloud_provider.py b/atst/domain/csp/cloud/azure_cloud_provider.py index 1d921fde..425fe649 100644 --- a/atst/domain/csp/cloud/azure_cloud_provider.py +++ b/atst/domain/csp/cloud/azure_cloud_provider.py @@ -1,6 +1,5 @@ import json from secrets import token_urlsafe -from typing import Any, Dict from uuid import uuid4 from atst.utils import sha256_hex @@ -1026,12 +1025,10 @@ class AzureCloudProvider(CloudProviderInterface): def update_tenant_creds(self, tenant_id, secret: KeyVaultCredentials): hashed = sha256_hex(tenant_id) - new_secrets = secret.dict() curr_secrets = self._source_tenant_creds(tenant_id) - updated_secrets: Dict[str, Any] = {**curr_secrets.dict(), **new_secrets} - us = KeyVaultCredentials(**updated_secrets) - self.set_secret(hashed, json.dumps(us.dict())) - return us + updated_secrets = curr_secrets.merge_credentials(secret) + self.set_secret(hashed, json.dumps(updated_secrets.dict())) + return updated_secrets def _source_tenant_creds(self, tenant_id) -> KeyVaultCredentials: hashed = sha256_hex(tenant_id) @@ -1060,7 +1057,7 @@ class AzureCloudProvider(CloudProviderInterface): "timeframe": "Custom", "timePeriod": {"from": payload.from_date, "to": payload.to_date,}, "dataset": { - "granularity": "Daily", + "granularity": "Monthly", "aggregation": {"totalCost": {"name": "PreTaxCost", "function": "Sum"}}, "grouping": [{"type": "Dimension", "name": "InvoiceId"}], }, diff --git a/atst/domain/csp/cloud/mock_cloud_provider.py b/atst/domain/csp/cloud/mock_cloud_provider.py index 7ec0636f..da1cccd8 100644 --- a/atst/domain/csp/cloud/mock_cloud_provider.py +++ b/atst/domain/csp/cloud/mock_cloud_provider.py @@ -1,4 +1,5 @@ from uuid import uuid4 +import pendulum from .cloud_provider_interface import CloudProviderInterface from .exceptions import ( @@ -459,15 +460,26 @@ class MockCloudProvider(CloudProviderInterface): self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION) object_id = str(uuid4()) + start_of_month = pendulum.today(tz="utc").start_of("month").replace(tzinfo=None) + this_month = start_of_month.to_atom_string() + last_month = start_of_month.subtract(months=1).to_atom_string() + two_months_ago = start_of_month.subtract(months=2).to_atom_string() + properties = CostManagementQueryProperties( **dict( columns=[ {"name": "PreTaxCost", "type": "Number"}, - {"name": "UsageDate", "type": "Number"}, + {"name": "BillingMonth", "type": "Datetime"}, {"name": "InvoiceId", "type": "String"}, {"name": "Currency", "type": "String"}, ], - rows=[], + rows=[ + [1.0, two_months_ago, "", "USD"], + [500.0, two_months_ago, "e05009w9sf", "USD"], + [50.0, last_month, "", "USD"], + [1000.0, last_month, "e0500a4qhw", "USD"], + [500.0, this_month, "", "USD"], + ], ) ) diff --git a/atst/domain/csp/cloud/models.py b/atst/domain/csp/cloud/models.py index 358f7934..25521bb9 100644 --- a/atst/domain/csp/cloud/models.py +++ b/atst/domain/csp/cloud/models.py @@ -417,6 +417,15 @@ class KeyVaultCredentials(BaseModel): return values + def merge_credentials( + self, new_creds: "KeyVaultCredentials" + ) -> "KeyVaultCredentials": + updated_creds = {k: v for k, v in new_creds.dict().items() if v} + old_creds = self.dict() + old_creds.update(updated_creds) + + return KeyVaultCredentials(**old_creds) + class SubscriptionCreationCSPPayload(BaseCSPPayload): display_name: str diff --git a/atst/domain/csp/reports.py b/atst/domain/csp/reports.py index 3f9ccbf8..700947f7 100644 --- a/atst/domain/csp/reports.py +++ b/atst/domain/csp/reports.py @@ -1,6 +1,6 @@ -from collections import defaultdict import json from decimal import Decimal +import pendulum def load_fixture_data(): @@ -11,128 +11,25 @@ def load_fixture_data(): class MockReportingProvider: FIXTURE_SPEND_DATA = load_fixture_data() - @classmethod - def get_portfolio_monthly_spending(cls, portfolio): - """ - returns an array of application and environment spending for the - portfolio. Applications and their nested environments are sorted in - alphabetical order by name. - [ - { - name - this_month - last_month - total - environments [ - { - name - this_month - last_month - total - } - ] - } - ] - """ - fixture_apps = cls.FIXTURE_SPEND_DATA.get(portfolio.name, {}).get( - "applications", [] - ) +def prepare_azure_reporting_data(rows: list): + """ + Returns a dict representing invoiced and estimated funds for a portfolio given + a list of rows from CostManagementQueryCSPResult.properties.rows + { + invoiced: Decimal, + estimated: Decimal + } + """ - for application in portfolio.applications: - if application.name not in [app["name"] for app in fixture_apps]: - fixture_apps.append({"name": application.name, "environments": []}) + estimated = [] + while rows: + if pendulum.parse(rows[-1][1]) >= pendulum.now(tz="utc").start_of("month"): + estimated.append(rows.pop()) + else: + break - return sorted( - [ - cls._get_application_monthly_totals(portfolio, fixture_app) - for fixture_app in fixture_apps - if fixture_app["name"] - in [application.name for application in portfolio.applications] - ], - key=lambda app: app["name"], - ) - - @classmethod - def _get_environment_monthly_totals(cls, environment): - """ - returns a dictionary that represents spending totals for an environment e.g. - { - name - this_month - last_month - total - } - """ - return { - "name": environment["name"], - "this_month": sum(environment["spending"]["this_month"].values()), - "last_month": sum(environment["spending"]["last_month"].values()), - "total": sum(environment["spending"]["total"].values()), - } - - @classmethod - def _get_application_monthly_totals(cls, portfolio, fixture_app): - """ - returns a dictionary that represents spending totals for an application - and its environments e.g. - { - name - this_month - last_month - total - environments: [ - { - name - this_month - last_month - total - } - ] - } - """ - application_envs = [ - env - for env in portfolio.all_environments - if env.application.name == fixture_app["name"] - ] - - environments = [ - cls._get_environment_monthly_totals(env) - for env in fixture_app["environments"] - if env["name"] in [e.name for e in application_envs] - ] - - for env in application_envs: - if env.name not in [env["name"] for env in environments]: - environments.append({"name": env.name}) - - return { - "name": fixture_app["name"], - "this_month": sum(env.get("this_month", 0) for env in environments), - "last_month": sum(env.get("last_month", 0) for env in environments), - "total": sum(env.get("total", 0) for env in environments), - "environments": sorted(environments, key=lambda env: env["name"]), - } - - @classmethod - def get_spending_by_JEDI_clin(cls, portfolio): - """ - returns an dictionary of spending per JEDI CLIN for a portfolio - { - jedi_clin: { - invoiced - estimated - }, - } - """ - if portfolio.name in cls.FIXTURE_SPEND_DATA: - CLIN_spend_dict = defaultdict(lambda: defaultdict(Decimal)) - for application in cls.FIXTURE_SPEND_DATA[portfolio.name]["applications"]: - for environment in application["environments"]: - for clin, spend in environment["spending"]["this_month"].items(): - CLIN_spend_dict[clin]["estimated"] += Decimal(spend) - for clin, spend in environment["spending"]["total"].items(): - CLIN_spend_dict[clin]["invoiced"] += Decimal(spend) - return CLIN_spend_dict - return {} + return dict( + invoiced=Decimal(sum([row[0] for row in rows])), + estimated=Decimal(sum([row[0] for row in estimated])), + ) diff --git a/atst/domain/reports.py b/atst/domain/reports.py index 99b229e3..fc619649 100644 --- a/atst/domain/reports.py +++ b/atst/domain/reports.py @@ -1,12 +1,13 @@ from flask import current_app -from itertools import groupby +from atst.domain.csp.cloud.models import ( + ReportingCSPPayload, + CostManagementQueryCSPResult, +) +from atst.domain.csp.reports import prepare_azure_reporting_data +import pendulum class Reports: - @classmethod - def monthly_spending(cls, portfolio): - return current_app.csp.reports.get_portfolio_monthly_spending(portfolio) - @classmethod def expired_task_orders(cls, portfolio): return [ @@ -14,31 +15,19 @@ class Reports: ] @classmethod - def obligated_funds_by_JEDI_clin(cls, portfolio): - clin_spending = current_app.csp.reports.get_spending_by_JEDI_clin(portfolio) - active_clins = portfolio.active_clins - for jedi_clin, clins in groupby( - active_clins, key=lambda clin: clin.jedi_clin_type - ): - if not clin_spending.get(jedi_clin.name): - clin_spending[jedi_clin.name] = {} - clin_spending[jedi_clin.name]["obligated"] = sum( - clin.obligated_amount for clin in clins - ) + def get_portfolio_spending(cls, portfolio): + # TODO: Extend this function to make from_date and to_date configurable + from_date = pendulum.now().subtract(years=1).add(days=1).format("YYYY-MM-DD") + to_date = pendulum.now().format("YYYY-MM-DD") + rows = [] - output = [] - for clin in clin_spending.keys(): - invoiced = clin_spending[clin].get("invoiced", 0) - estimated = clin_spending[clin].get("estimated", 0) - obligated = clin_spending[clin].get("obligated", 0) - remaining = obligated - (invoiced + estimated) - output.append( - { - "name": clin, - "invoiced": invoiced, - "estimated": estimated, - "obligated": obligated, - "remaining": remaining, - } + if portfolio.csp_data: + payload = ReportingCSPPayload( + from_date=from_date, to_date=to_date, **portfolio.csp_data ) - return output + response: CostManagementQueryCSPResult = current_app.csp.cloud.get_reporting_data( + payload + ) + rows = response.properties.rows + + return prepare_azure_reporting_data(rows) diff --git a/atst/filters.py b/atst/filters.py index 3508f1e9..84191017 100644 --- a/atst/filters.py +++ b/atst/filters.py @@ -5,7 +5,7 @@ from flask import render_template from jinja2 import contextfilter from jinja2.exceptions import TemplateNotFound from urllib.parse import urlparse, urlunparse, parse_qs, urlencode -from decimal import DivisionByZero as DivisionByZeroException +from decimal import DivisionByZero as DivisionByZeroException, InvalidOperation def iconSvg(name): @@ -43,7 +43,7 @@ def obligatedFundingGraphWidth(values): numerator, denominator = values try: return (numerator / denominator) * 100 - except DivisionByZeroException: + except (DivisionByZeroException, InvalidOperation): return 0 diff --git a/atst/models/portfolio.py b/atst/models/portfolio.py index 5a8f0f1e..2ddcaa41 100644 --- a/atst/models/portfolio.py +++ b/atst/models/portfolio.py @@ -89,6 +89,12 @@ class Portfolio( def active_task_orders(self): return [task_order for task_order in self.task_orders if task_order.is_active] + @property + def total_obligated_funds(self): + return sum( + (task_order.total_obligated_funds for task_order in self.active_task_orders) + ) + @property def funding_duration(self): """ diff --git a/atst/models/task_order.py b/atst/models/task_order.py index d6aa63f8..3ab493a9 100644 --- a/atst/models/task_order.py +++ b/atst/models/task_order.py @@ -25,7 +25,6 @@ SORT_ORDERING = [ Status.DRAFT, Status.UPCOMING, Status.EXPIRED, - Status.UNSIGNED, ] @@ -148,7 +147,10 @@ class TaskOrder(Base, mixins.TimestampsMixin): @property def display_status(self): - return self.status.value + if self.status == Status.UNSIGNED: + return Status.DRAFT.value + else: + return self.status.value @property def portfolio_name(self): diff --git a/atst/routes/portfolios/index.py b/atst/routes/portfolios/index.py index f9e7d5cf..795d4b70 100644 --- a/atst/routes/portfolios/index.py +++ b/atst/routes/portfolios/index.py @@ -34,25 +34,25 @@ def create_portfolio(): @user_can(Permissions.VIEW_PORTFOLIO_REPORTS, message="view portfolio reports") def reports(portfolio_id): portfolio = Portfolios.get(g.current_user, portfolio_id) + spending = Reports.get_portfolio_spending(portfolio) + obligated = portfolio.total_obligated_funds + remaining = obligated - (spending["invoiced"] + spending["estimated"]) - current_obligated_funds = Reports.obligated_funds_by_JEDI_clin(portfolio) + current_obligated_funds = { + **spending, + "obligated": obligated, + "remaining": remaining, + } - if any(map(lambda clin: clin["remaining"] < 0, current_obligated_funds)): + if current_obligated_funds["remaining"] < 0: flash("insufficient_funds") - # wrapped in str() because the sum of obligated funds returns a Decimal object - total_portfolio_value = str( - sum( - task_order.total_obligated_funds - for task_order in portfolio.active_task_orders - ) - ) return render_template( "portfolios/reports/index.html", portfolio=portfolio, - total_portfolio_value=total_portfolio_value, + # wrapped in str() because the sum of obligated funds returns a Decimal object + total_portfolio_value=str(portfolio.total_obligated_funds), current_obligated_funds=current_obligated_funds, expired_task_orders=Reports.expired_task_orders(portfolio), - monthly_spending=Reports.monthly_spending(portfolio), retrieved=datetime.now(), # mocked datetime of reporting data retrival ) diff --git a/config/base.ini b/config/base.ini index 1f4c732a..55482741 100644 --- a/config/base.ini +++ b/config/base.ini @@ -38,6 +38,7 @@ PGUSER = postgres PORT=8000 REDIS_HOST=localhost:6379 REDIS_PASSWORD +REDIS_SSLMODE REDIS_TLS=False REDIS_USER SECRET_KEY = change_me_into_something_secret diff --git a/deploy/azure/kustomization.yaml b/deploy/azure/kustomization.yaml index d0162394..b46021b0 100644 --- a/deploy/azure/kustomization.yaml +++ b/deploy/azure/kustomization.yaml @@ -10,6 +10,5 @@ resources: - volume-claim.yml - nginx-client-ca-bundle.yml - acme-challenges.yml - - aadpodidentity.yml - nginx-snippets.yml - autoscaling.yml diff --git a/deploy/overlays/cloudzero-dev/envvars.yml b/deploy/overlays/cloudzero-dev/envvars.yml index 179811ed..ded47f7b 100644 --- a/deploy/overlays/cloudzero-dev/envvars.yml +++ b/deploy/overlays/cloudzero-dev/envvars.yml @@ -4,19 +4,30 @@ kind: ConfigMap metadata: name: atst-worker-envvars data: + AZURE_ACCOUNT_NAME: jeditasksatat CELERY_DEFAULT_QUEUE: celery-staging - SERVER_NAME: staging.atat.code.mil FLASK_ENV: staging + PGDATABASE: cloudzero_jedidev_atat + PGHOST: 191.238.6.43 + PGUSER: atat@cloudzero-jedidev-sql + PGSSLMODE: require + REDIS_HOST: 10.1.3.34:6380 + SERVER_NAME: dev.atat.cloud.mil --- apiVersion: v1 kind: ConfigMap metadata: name: atst-envvars data: - ASSETS_URL: https://atat-cdn-staging.azureedge.net/ - CDN_ORIGIN: https://staging.atat.code.mil + ASSETS_URL: "" + AZURE_ACCOUNT_NAME: jeditasksatat + CAC_URL: https://auth-dev.atat.cloud.mil + CDN_ORIGIN: https://dev.atat.cloud.mil CELERY_DEFAULT_QUEUE: celery-staging FLASK_ENV: staging - STATIC_URL: https://atat-cdn-staging.azureedge.net/static/ - PGHOST: cloudzero-dev-sql.postgres.database.azure.com - REDIS_HOST: cloudzero-dev-redis.redis.cache.windows.net:6380 + PGDATABASE: cloudzero_jedidev_atat + PGHOST: 191.238.6.43 + PGUSER: atat@cloudzero-jedidev-sql + PGSSLMODE: require + REDIS_HOST: 10.1.3.34:6380 + SESSION_COOKIE_DOMAIN: atat.cloud.mil diff --git a/deploy/overlays/cloudzero-dev/flex_vol.yml b/deploy/overlays/cloudzero-dev/flex_vol.yml index a3c65df7..990d21e5 100644 --- a/deploy/overlays/cloudzero-dev/flex_vol.yml +++ b/deploy/overlays/cloudzero-dev/flex_vol.yml @@ -9,23 +9,19 @@ spec: - name: nginx-secret flexVolume: options: - keyvaultname: "cloudzero-dev-keyvault" - # keyvaultobjectnames: "dhparam4096;cert;cert" - keyvaultobjectnames: "foo" - keyvaultobjectaliases: "FOO" - keyvaultobjecttypes: "secret" - usevmmanagedidentity: "true" usepodidentity: "false" + usevmmanagedidentity: "true" + vmmanagedidentityclientid: $VMSS_CLIENT_ID + keyvaultname: "cz-jedidev-keyvault" + keyvaultobjectnames: "dhparam4096;ATATCERT;ATATCERT" - name: flask-secret flexVolume: options: - keyvaultname: "cloudzero-dev-keyvault" - # keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY" - keyvaultobjectnames: "master-PGPASSWORD" - keyvaultobjectaliases: "PGPASSWORD" - keyvaultobjecttypes: "secret" - usevmmanagedidentity: "true" usepodidentity: "false" + usevmmanagedidentity: "true" + vmmanagedidentityclientid: $VMSS_CLIENT_ID + keyvaultname: "cz-jedidev-keyvault" + keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY" --- apiVersion: extensions/v1beta1 kind: Deployment @@ -38,10 +34,11 @@ spec: - name: flask-secret flexVolume: options: - keyvaultname: "cloudzero-dev-keyvault" - keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY" - usevmmanagedidentity: "true" usepodidentity: "false" + usevmmanagedidentity: "true" + vmmanagedidentityclientid: $VMSS_CLIENT_ID + keyvaultname: "cz-jedidev-keyvault" + keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY" --- apiVersion: extensions/v1beta1 kind: Deployment @@ -54,10 +51,11 @@ spec: - name: flask-secret flexVolume: options: - keyvaultname: "cloudzero-dev-keyvault" - keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY" - usevmmanagedidentity: "true" usepodidentity: "false" + usevmmanagedidentity: "true" + vmmanagedidentityclientid: $VMSS_CLIENT_ID + keyvaultname: "cz-jedidev-keyvault" + keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY" --- apiVersion: batch/v1beta1 kind: CronJob @@ -72,7 +70,8 @@ spec: - name: flask-secret flexVolume: options: - keyvaultname: "cloudzero-dev-keyvault" - keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY" - usevmmanagedidentity: "true" usepodidentity: "false" + usevmmanagedidentity: "true" + vmmanagedidentityclientid: $VMSS_CLIENT_ID + keyvaultname: "cz-jedidev-keyvault" + keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY" diff --git a/deploy/overlays/cloudzero-dev/kustomization.yaml b/deploy/overlays/cloudzero-dev/kustomization.yaml index 24705531..65262fbe 100644 --- a/deploy/overlays/cloudzero-dev/kustomization.yaml +++ b/deploy/overlays/cloudzero-dev/kustomization.yaml @@ -1,9 +1,8 @@ -namespace: staging +namespace: cloudzero-dev bases: - ../../azure/ resources: - namespace.yml - - reset-cron-job.yml patchesStrategicMerge: - ports.yml - envvars.yml diff --git a/deploy/overlays/cloudzero-dev/namespace.yml b/deploy/overlays/cloudzero-dev/namespace.yml index ee38adfb..242c3a2f 100644 --- a/deploy/overlays/cloudzero-dev/namespace.yml +++ b/deploy/overlays/cloudzero-dev/namespace.yml @@ -1,4 +1,4 @@ apiVersion: v1 kind: Namespace metadata: - name: staging + name: cloudzero-dev diff --git a/deploy/overlays/cloudzero-dev/ports.yml b/deploy/overlays/cloudzero-dev/ports.yml index 8dbbd0f1..5225cf3c 100644 --- a/deploy/overlays/cloudzero-dev/ports.yml +++ b/deploy/overlays/cloudzero-dev/ports.yml @@ -5,7 +5,7 @@ metadata: name: atst-main annotations: service.beta.kubernetes.io/azure-load-balancer-internal: "true" - service.beta.kubernetes.io/azure-load-balancer-internal-subnet: "cloudzero-dev-public" + service.beta.kubernetes.io/azure-load-balancer-internal-subnet: "cloudzero-jedidev-public" spec: loadBalancerIP: "" ports: @@ -22,7 +22,7 @@ metadata: name: atst-auth annotations: service.beta.kubernetes.io/azure-load-balancer-internal: "true" - service.beta.kubernetes.io/azure-load-balancer-internal-subnet: "cloudzero-dev-public" + service.beta.kubernetes.io/azure-load-balancer-internal-subnet: "cloudzero-jedidev-public" spec: loadBalancerIP: "" ports: diff --git a/deploy/overlays/cloudzero-dev/reset-cron-job.yml b/deploy/overlays/cloudzero-dev/reset-cron-job.yml deleted file mode 100644 index b4792e5d..00000000 --- a/deploy/overlays/cloudzero-dev/reset-cron-job.yml +++ /dev/null @@ -1,46 +0,0 @@ -apiVersion: batch/v1beta1 -kind: CronJob -metadata: - name: reset-db - namespace: atat -spec: - schedule: "0 4 * * *" - concurrencyPolicy: Replace - successfulJobsHistoryLimit: 1 - jobTemplate: - spec: - template: - metadata: - labels: - app: atst - role: reset-db - aadpodidbinding: atat-kv-id-binding - spec: - restartPolicy: OnFailure - containers: - - name: reset - image: $CONTAINER_IMAGE - command: [ - "/bin/sh", "-c" - ] - args: [ - "/opt/atat/atst/.venv/bin/python", - "/opt/atat/atst/script/reset_database.py" - ] - envFrom: - - configMapRef: - name: atst-worker-envvars - volumeMounts: - - name: flask-secret - mountPath: "/config" - volumes: - - name: flask-secret - flexVolume: - driver: "azure/kv" - options: - usepodidentity: "true" - keyvaultname: "atat-vault-test" - keyvaultobjectnames: "staging-AZURE-STORAGE-KEY;staging-MAIL-PASSWORD;staging-PGPASSWORD;staging-REDIS-PASSWORD;staging-SECRET-KEY" - keyvaultobjectaliases: "AZURE_STORAGE_KEY;MAIL_PASSWORD;PGPASSWORD;REDIS_PASSWORD;SECRET_KEY" - keyvaultobjecttypes: "secret;secret;secret;secret;key" - tenantid: $TENANT_ID diff --git a/deploy/overlays/migration-cloudzero-dev/kustomization.yaml b/deploy/overlays/migration-cloudzero-dev/kustomization.yaml new file mode 100644 index 00000000..b12c0b88 --- /dev/null +++ b/deploy/overlays/migration-cloudzero-dev/kustomization.yaml @@ -0,0 +1,5 @@ +namespace: cloudzero-dev +bases: + - ../../shared/ +patchesStrategicMerge: + - migration.yaml diff --git a/deploy/overlays/migration-cloudzero-dev/migration.yaml b/deploy/overlays/migration-cloudzero-dev/migration.yaml new file mode 100644 index 00000000..53a39dcc --- /dev/null +++ b/deploy/overlays/migration-cloudzero-dev/migration.yaml @@ -0,0 +1,16 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: migration +spec: + template: + spec: + volumes: + - name: flask-secret + flexVolume: + options: + usepodidentity: "false" + usevmmanagedidentity: "true" + vmmanagedidentityclientid: $VMSS_CLIENT_ID + keyvaultname: "cz-jedidev-keyvault" + keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY" diff --git a/deploy/shared/kustomization.yaml b/deploy/shared/kustomization.yaml new file mode 100644 index 00000000..38dddc7e --- /dev/null +++ b/deploy/shared/kustomization.yaml @@ -0,0 +1,3 @@ +namespace: atat +resources: + - migration.yaml diff --git a/js/components/sidenav_toggler.js b/js/components/sidenav_toggler.js index 11717849..d545b3ed 100644 --- a/js/components/sidenav_toggler.js +++ b/js/components/sidenav_toggler.js @@ -1,5 +1,6 @@ import ExpandSidenavMixin from '../mixins/expand_sidenav' import ToggleMixin from '../mixins/toggle' +import { sidenavCookieName } from '../lib/constants' export default { name: 'sidenav-toggler', @@ -14,7 +15,7 @@ export default { toggle: function(e) { e.preventDefault() this.isVisible = !this.isVisible - document.cookie = this.cookieName + '=' + this.isVisible + '; path=/' + document.cookie = sidenavCookieName + '=' + this.isVisible + '; path=/' this.$parent.$emit('sidenavToggle', this.isVisible) }, }, diff --git a/js/lib/constants.js b/js/lib/constants.js new file mode 100644 index 00000000..b4de4fcf --- /dev/null +++ b/js/lib/constants.js @@ -0,0 +1 @@ +export const sidenavCookieName = 'expandSidenav' diff --git a/js/mixins/expand_sidenav.js b/js/mixins/expand_sidenav.js index 7553b7d4..2af9c87a 100644 --- a/js/mixins/expand_sidenav.js +++ b/js/mixins/expand_sidenav.js @@ -1,11 +1,12 @@ +import { sidenavCookieName } from '../lib/constants' + export default { props: { - cookieName: 'expandSidenav', defaultVisible: { type: Boolean, default: function() { - if (document.cookie.match(this.cookieName)) { - return !!document.cookie.match(this.cookieName + ' *= *true') + if (document.cookie.match(sidenavCookieName)) { + return !!document.cookie.match(sidenavCookieName + ' *= *true') } else { return true } diff --git a/script/database_setup.py b/script/database_setup.py index 7784be05..e4964516 100644 --- a/script/database_setup.py +++ b/script/database_setup.py @@ -16,16 +16,14 @@ from reset_database import reset_database def database_setup(username, password, dbname, ccpo_users): + print("Applying schema and seeding roles and permissions.") + reset_database() + print( f"Creating Postgres user role for '{username}' and granting all privileges to database '{dbname}'." ) - try: - _create_database_user(username, password, dbname) - except sqlalchemy.exc.ProgrammingError as err: - print(f"Postgres user role '{username}' already exists.") + _create_database_user(username, password, dbname) - print("Applying schema and seeding roles and permissions.") - reset_database() print("Creating initial set of CCPO users.") _add_ccpo_users(ccpo_users) @@ -47,6 +45,22 @@ def _create_database_user(username, password, dbname): f"ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL PRIVILEGES ON FUNCTIONS TO {username}; \n" ) + try: + # TODO: make this more configurable + engine.execute(f"GRANT {username} TO azure_pg_admin;") + except sqlalchemy.exc.ProgrammingError as err: + print(f"Cannot grant new role {username} to azure_pg_admin") + + for table in meta.tables: + engine.execute(f"ALTER TABLE {table} OWNER TO {username};\n") + + sequence_results = engine.execute( + "SELECT c.relname FROM pg_class c WHERE c.relkind = 'S';" + ).fetchall() + sequences = [p[0] for p in sequence_results] + for sequence in sequences: + engine.execute(f"ALTER SEQUENCE {sequence} OWNER TO {username};\n") + trans.commit() diff --git a/script/k8s_config b/script/k8s_config index b489c942..36dc5ef7 100755 --- a/script/k8s_config +++ b/script/k8s_config @@ -13,6 +13,7 @@ SETTINGS=( AUTH_DOMAIN KV_MI_ID KV_MI_CLIENT_ID + VMSS_CLIENT_ID TENANT_ID ) diff --git a/templates/components/accordion.html b/templates/components/accordion.html index 9ec92099..cdf72901 100644 --- a/templates/components/accordion.html +++ b/templates/components/accordion.html @@ -6,8 +6,12 @@ heading_tag="h2", heading_classes="", content_tag="div", - content_classes="") %} - + content_classes="", + default_visible=False) %} + <{{wrapper_tag}} class="{{ wrapper_classes }}"> <{{heading_tag}} class="accordion__button {{ heading_classes }}">