pk/atst
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'staging' into fix-double-submit

Browse Source
This commit is contained in:
leigh-mil 2020-02-11 10:31:53 -05:00 committed by GitHub
commit 858312b2e0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
39 changed files with 401 additions and 393 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.secrets.baseline
View File

@ -3,7 +3,7 @@
"files": "^.secrets.baseline$|^.*pgsslrootcert.yml$",
"lines": null
},
"generated_at": "2020-01-27T19:24:43Z",
"generated_at": "2020-02-10T21:40:38Z",
"plugins_used": [
{
"base64_limit": 4.5,
@ -82,7 +82,7 @@
"hashed_secret": "afc848c316af1a89d49826c5ae9d00ed769415f3",
"is_secret": false,
"is_verified": false,
"line_number": 32,
"line_number": 33,
"type": "Secret Keyword"
}
],

24
atst/app.py
View File

@ -157,7 +157,6 @@ def map_config(config):
**config["default"],
"USE_AUDIT_LOG": config["default"].getboolean("USE_AUDIT_LOG"),
"ENV": config["default"]["ENVIRONMENT"],
"BROKER_URL": config["default"]["REDIS_URI"],
"DEBUG": config["default"].getboolean("DEBUG"),
"DEBUG_MAILER": config["default"].getboolean("DEBUG_MAILER"),
"SQLALCHEMY_ECHO": config["default"].getboolean("SQLALCHEMY_ECHO"),
@ -233,13 +232,34 @@ def make_config(direct_config=None):
config.set("default", "DATABASE_URI", database_uri)
# Assemble REDIS_URI value
redis_use_tls = config["default"].getboolean("REDIS_TLS")
redis_uri = "redis{}://{}:{}@{}".format( # pragma: allowlist secret
("s" if config["default"].getboolean("REDIS_TLS") else ""),
("s" if redis_use_tls else ""),
(config.get("default", "REDIS_USER") or ""),
(config.get("default", "REDIS_PASSWORD") or ""),
config.get("default", "REDIS_HOST"),
)
celery_uri = redis_uri
if redis_use_tls:
tls_mode = config.get("default", "REDIS_SSLMODE")
tls_mode_str = tls_mode.lower() if tls_mode else "none"
redis_uri = f"{redis_uri}/?ssl_cert_reqs={tls_mode_str}"
# TODO: Kombu, one of Celery's dependencies, still requires
# that ssl_cert_reqs be passed as the string version of an
# option on the ssl module. We can clean this up and use
# the REDIS_URI for both when this PR to Kombu is released:
# https://github.com/celery/kombu/pull/1139
kombu_modes = {
"none": "CERT_NONE",
"required": "CERT_REQUIRED",
"optional": "CERT_OPTIONAL",
}
celery_tls_mode_str = kombu_modes[tls_mode_str]
celery_uri = f"{celery_uri}/?ssl_cert_reqs={celery_tls_mode_str}"
config.set("default", "REDIS_URI", redis_uri)
config.set("default", "BROKER_URL", celery_uri)
return map_config(config)

11
atst/domain/csp/cloud/azure_cloud_provider.py
View File

@ -1,6 +1,5 @@
import json
from secrets import token_urlsafe
from typing import Any, Dict
from uuid import uuid4
from atst.utils import sha256_hex
@ -1026,12 +1025,10 @@ class AzureCloudProvider(CloudProviderInterface):
def update_tenant_creds(self, tenant_id, secret: KeyVaultCredentials):
hashed = sha256_hex(tenant_id)
new_secrets = secret.dict()
curr_secrets = self._source_tenant_creds(tenant_id)
updated_secrets: Dict[str, Any] = {**curr_secrets.dict(), **new_secrets}
us = KeyVaultCredentials(**updated_secrets)
self.set_secret(hashed, json.dumps(us.dict()))
return us
updated_secrets = curr_secrets.merge_credentials(secret)
self.set_secret(hashed, json.dumps(updated_secrets.dict()))
return updated_secrets
def _source_tenant_creds(self, tenant_id) -> KeyVaultCredentials:
hashed = sha256_hex(tenant_id)
@ -1060,7 +1057,7 @@ class AzureCloudProvider(CloudProviderInterface):
"timeframe": "Custom",
"timePeriod": {"from": payload.from_date, "to": payload.to_date,},
"dataset": {
"granularity": "Daily",
"granularity": "Monthly",
"aggregation": {"totalCost": {"name": "PreTaxCost", "function": "Sum"}},
"grouping": [{"type": "Dimension", "name": "InvoiceId"}],
},

16
atst/domain/csp/cloud/mock_cloud_provider.py
View File

@ -1,4 +1,5 @@
from uuid import uuid4
import pendulum
from .cloud_provider_interface import CloudProviderInterface
from .exceptions import (
@ -459,15 +460,26 @@ class MockCloudProvider(CloudProviderInterface):
self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION)
object_id = str(uuid4())
start_of_month = pendulum.today(tz="utc").start_of("month").replace(tzinfo=None)
this_month = start_of_month.to_atom_string()
last_month = start_of_month.subtract(months=1).to_atom_string()
two_months_ago = start_of_month.subtract(months=2).to_atom_string()
properties = CostManagementQueryProperties(
**dict(
columns=[
{"name": "PreTaxCost", "type": "Number"},
{"name": "UsageDate", "type": "Number"},
{"name": "BillingMonth", "type": "Datetime"},
{"name": "InvoiceId", "type": "String"},
{"name": "Currency", "type": "String"},
],
rows=[],
rows=[
[1.0, two_months_ago, "", "USD"],
[500.0, two_months_ago, "e05009w9sf", "USD"],
[50.0, last_month, "", "USD"],
[1000.0, last_month, "e0500a4qhw", "USD"],
[500.0, this_month, "", "USD"],
],
)
)

9
atst/domain/csp/cloud/models.py
View File

@ -417,6 +417,15 @@ class KeyVaultCredentials(BaseModel):
return values
def merge_credentials(
self, new_creds: "KeyVaultCredentials"
) -> "KeyVaultCredentials":
updated_creds = {k: v for k, v in new_creds.dict().items() if v}
old_creds = self.dict()
old_creds.update(updated_creds)
return KeyVaultCredentials(**old_creds)
class SubscriptionCreationCSPPayload(BaseCSPPayload):
display_name: str

143
atst/domain/csp/reports.py
View File

@ -1,6 +1,6 @@
from collections import defaultdict
import json
from decimal import Decimal
import pendulum
def load_fixture_data():
@ -11,128 +11,25 @@ def load_fixture_data():
class MockReportingProvider:
FIXTURE_SPEND_DATA = load_fixture_data()
@classmethod
def get_portfolio_monthly_spending(cls, portfolio):
"""
returns an array of application and environment spending for the
portfolio. Applications and their nested environments are sorted in
alphabetical order by name.
[
{
name
this_month
last_month
total
environments [
{
name
this_month
last_month
total
}
]
}
]
"""
fixture_apps = cls.FIXTURE_SPEND_DATA.get(portfolio.name, {}).get(
"applications", []
)
def prepare_azure_reporting_data(rows: list):
"""
Returns a dict representing invoiced and estimated funds for a portfolio given
a list of rows from CostManagementQueryCSPResult.properties.rows
{
invoiced: Decimal,
estimated: Decimal
}
"""
for application in portfolio.applications:
if application.name not in [app["name"] for app in fixture_apps]:
fixture_apps.append({"name": application.name, "environments": []})
estimated = []
while rows:
if pendulum.parse(rows[-1][1]) >= pendulum.now(tz="utc").start_of("month"):
estimated.append(rows.pop())
else:
break
return sorted(
[
cls._get_application_monthly_totals(portfolio, fixture_app)
for fixture_app in fixture_apps
if fixture_app["name"]
in [application.name for application in portfolio.applications]
],
key=lambda app: app["name"],
)
@classmethod
def _get_environment_monthly_totals(cls, environment):
"""
returns a dictionary that represents spending totals for an environment e.g.
{
name
this_month
last_month
total
}
"""
return {
"name": environment["name"],
"this_month": sum(environment["spending"]["this_month"].values()),
"last_month": sum(environment["spending"]["last_month"].values()),
"total": sum(environment["spending"]["total"].values()),
}
@classmethod
def _get_application_monthly_totals(cls, portfolio, fixture_app):
"""
returns a dictionary that represents spending totals for an application
and its environments e.g.
{
name
this_month
last_month
total
environments: [
{
name
this_month
last_month
total
}
]
}
"""
application_envs = [
env
for env in portfolio.all_environments
if env.application.name == fixture_app["name"]
]
environments = [
cls._get_environment_monthly_totals(env)
for env in fixture_app["environments"]
if env["name"] in [e.name for e in application_envs]
]
for env in application_envs:
if env.name not in [env["name"] for env in environments]:
environments.append({"name": env.name})
return {
"name": fixture_app["name"],
"this_month": sum(env.get("this_month", 0) for env in environments),
"last_month": sum(env.get("last_month", 0) for env in environments),
"total": sum(env.get("total", 0) for env in environments),
"environments": sorted(environments, key=lambda env: env["name"]),
}
@classmethod
def get_spending_by_JEDI_clin(cls, portfolio):
"""
returns an dictionary of spending per JEDI CLIN for a portfolio
{
jedi_clin: {
invoiced
estimated
},
}
"""
if portfolio.name in cls.FIXTURE_SPEND_DATA:
CLIN_spend_dict = defaultdict(lambda: defaultdict(Decimal))
for application in cls.FIXTURE_SPEND_DATA[portfolio.name]["applications"]:
for environment in application["environments"]:
for clin, spend in environment["spending"]["this_month"].items():
CLIN_spend_dict[clin]["estimated"] += Decimal(spend)
for clin, spend in environment["spending"]["total"].items():
CLIN_spend_dict[clin]["invoiced"] += Decimal(spend)
return CLIN_spend_dict
return {}
return dict(
invoiced=Decimal(sum([row[0] for row in rows])),
estimated=Decimal(sum([row[0] for row in estimated])),
)

51
atst/domain/reports.py
View File

@ -1,12 +1,13 @@
from flask import current_app
from itertools import groupby
from atst.domain.csp.cloud.models import (
ReportingCSPPayload,
CostManagementQueryCSPResult,
)
from atst.domain.csp.reports import prepare_azure_reporting_data
import pendulum
class Reports:
@classmethod
def monthly_spending(cls, portfolio):
return current_app.csp.reports.get_portfolio_monthly_spending(portfolio)
@classmethod
def expired_task_orders(cls, portfolio):
return [
@ -14,31 +15,19 @@ class Reports:
]
@classmethod
def obligated_funds_by_JEDI_clin(cls, portfolio):
clin_spending = current_app.csp.reports.get_spending_by_JEDI_clin(portfolio)
active_clins = portfolio.active_clins
for jedi_clin, clins in groupby(
active_clins, key=lambda clin: clin.jedi_clin_type
):
if not clin_spending.get(jedi_clin.name):
clin_spending[jedi_clin.name] = {}
clin_spending[jedi_clin.name]["obligated"] = sum(
clin.obligated_amount for clin in clins
)
def get_portfolio_spending(cls, portfolio):
# TODO: Extend this function to make from_date and to_date configurable
from_date = pendulum.now().subtract(years=1).add(days=1).format("YYYY-MM-DD")
to_date = pendulum.now().format("YYYY-MM-DD")
rows = []
output = []
for clin in clin_spending.keys():
invoiced = clin_spending[clin].get("invoiced", 0)
estimated = clin_spending[clin].get("estimated", 0)
obligated = clin_spending[clin].get("obligated", 0)
remaining = obligated - (invoiced + estimated)
output.append(
{
"name": clin,
"invoiced": invoiced,
"estimated": estimated,
"obligated": obligated,
"remaining": remaining,
}
if portfolio.csp_data:
payload = ReportingCSPPayload(
from_date=from_date, to_date=to_date, **portfolio.csp_data
)
return output
response: CostManagementQueryCSPResult = current_app.csp.cloud.get_reporting_data(
payload
)
rows = response.properties.rows
return prepare_azure_reporting_data(rows)

4
atst/filters.py
View File

@ -5,7 +5,7 @@ from flask import render_template
from jinja2 import contextfilter
from jinja2.exceptions import TemplateNotFound
from urllib.parse import urlparse, urlunparse, parse_qs, urlencode
from decimal import DivisionByZero as DivisionByZeroException
from decimal import DivisionByZero as DivisionByZeroException, InvalidOperation
def iconSvg(name):
@ -43,7 +43,7 @@ def obligatedFundingGraphWidth(values):
numerator, denominator = values
try:
return (numerator / denominator) * 100
except DivisionByZeroException:
except (DivisionByZeroException, InvalidOperation):
return 0

6
atst/models/portfolio.py
View File

@ -89,6 +89,12 @@ class Portfolio(
def active_task_orders(self):
return [task_order for task_order in self.task_orders if task_order.is_active]
@property
def total_obligated_funds(self):
return sum(
(task_order.total_obligated_funds for task_order in self.active_task_orders)
)
@property
def funding_duration(self):
"""

6
atst/models/task_order.py
View File

@ -25,7 +25,6 @@ SORT_ORDERING = [
Status.DRAFT,
Status.UPCOMING,
Status.EXPIRED,
Status.UNSIGNED,
]
@ -148,7 +147,10 @@ class TaskOrder(Base, mixins.TimestampsMixin):
@property
def display_status(self):
return self.status.value
if self.status == Status.UNSIGNED:
return Status.DRAFT.value
else:
return self.status.value
@property
def portfolio_name(self):

22
atst/routes/portfolios/index.py
View File

@ -34,25 +34,25 @@ def create_portfolio():
@user_can(Permissions.VIEW_PORTFOLIO_REPORTS, message="view portfolio reports")
def reports(portfolio_id):
portfolio = Portfolios.get(g.current_user, portfolio_id)
spending = Reports.get_portfolio_spending(portfolio)
obligated = portfolio.total_obligated_funds
remaining = obligated - (spending["invoiced"] + spending["estimated"])
current_obligated_funds = Reports.obligated_funds_by_JEDI_clin(portfolio)
current_obligated_funds = {
**spending,
"obligated": obligated,
"remaining": remaining,
}
if any(map(lambda clin: clin["remaining"] < 0, current_obligated_funds)):
if current_obligated_funds["remaining"] < 0:
flash("insufficient_funds")
# wrapped in str() because the sum of obligated funds returns a Decimal object
total_portfolio_value = str(
sum(
task_order.total_obligated_funds
for task_order in portfolio.active_task_orders
)
)
return render_template(
"portfolios/reports/index.html",
portfolio=portfolio,
total_portfolio_value=total_portfolio_value,
# wrapped in str() because the sum of obligated funds returns a Decimal object
total_portfolio_value=str(portfolio.total_obligated_funds),
current_obligated_funds=current_obligated_funds,
expired_task_orders=Reports.expired_task_orders(portfolio),
monthly_spending=Reports.monthly_spending(portfolio),
retrieved=datetime.now(), # mocked datetime of reporting data retrival
)

1
config/base.ini
View File

@ -38,6 +38,7 @@ PGUSER = postgres
PORT=8000
REDIS_HOST=localhost:6379
REDIS_PASSWORD
REDIS_SSLMODE
REDIS_TLS=False
REDIS_USER
SECRET_KEY = change_me_into_something_secret

1
deploy/azure/kustomization.yaml
View File

@ -10,6 +10,5 @@ resources:
- volume-claim.yml
- nginx-client-ca-bundle.yml
- acme-challenges.yml
- aadpodidentity.yml
- nginx-snippets.yml
- autoscaling.yml

23
deploy/overlays/cloudzero-dev/envvars.yml
View File

@ -4,19 +4,30 @@ kind: ConfigMap
metadata:
name: atst-worker-envvars
data:
AZURE_ACCOUNT_NAME: jeditasksatat
CELERY_DEFAULT_QUEUE: celery-staging
SERVER_NAME: staging.atat.code.mil
FLASK_ENV: staging
PGDATABASE: cloudzero_jedidev_atat
PGHOST: 191.238.6.43
PGUSER: atat@cloudzero-jedidev-sql
PGSSLMODE: require
REDIS_HOST: 10.1.3.34:6380
SERVER_NAME: dev.atat.cloud.mil
---
apiVersion: v1
kind: ConfigMap
metadata:
name: atst-envvars
data:
ASSETS_URL: https://atat-cdn-staging.azureedge.net/
CDN_ORIGIN: https://staging.atat.code.mil
ASSETS_URL: ""
AZURE_ACCOUNT_NAME: jeditasksatat
CAC_URL: https://auth-dev.atat.cloud.mil
CDN_ORIGIN: https://dev.atat.cloud.mil
CELERY_DEFAULT_QUEUE: celery-staging
FLASK_ENV: staging
STATIC_URL: https://atat-cdn-staging.azureedge.net/static/
PGHOST: cloudzero-dev-sql.postgres.database.azure.com
REDIS_HOST: cloudzero-dev-redis.redis.cache.windows.net:6380
PGDATABASE: cloudzero_jedidev_atat
PGHOST: 191.238.6.43
PGUSER: atat@cloudzero-jedidev-sql
PGSSLMODE: require
REDIS_HOST: 10.1.3.34:6380
SESSION_COOKIE_DOMAIN: atat.cloud.mil

41
deploy/overlays/cloudzero-dev/flex_vol.yml
View File

@ -9,23 +9,19 @@ spec:
- name: nginx-secret
flexVolume:
options:
keyvaultname: "cloudzero-dev-keyvault"
# keyvaultobjectnames: "dhparam4096;cert;cert"
keyvaultobjectnames: "foo"
keyvaultobjectaliases: "FOO"
keyvaultobjecttypes: "secret"
usevmmanagedidentity: "true"
usepodidentity: "false"
usevmmanagedidentity: "true"
vmmanagedidentityclientid: $VMSS_CLIENT_ID
keyvaultname: "cz-jedidev-keyvault"
keyvaultobjectnames: "dhparam4096;ATATCERT;ATATCERT"
- name: flask-secret
flexVolume:
options:
keyvaultname: "cloudzero-dev-keyvault"
# keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY"
keyvaultobjectnames: "master-PGPASSWORD"
keyvaultobjectaliases: "PGPASSWORD"
keyvaultobjecttypes: "secret"
usevmmanagedidentity: "true"
usepodidentity: "false"
usevmmanagedidentity: "true"
vmmanagedidentityclientid: $VMSS_CLIENT_ID
keyvaultname: "cz-jedidev-keyvault"
keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY"
---
apiVersion: extensions/v1beta1
kind: Deployment
@ -38,10 +34,11 @@ spec:
- name: flask-secret
flexVolume:
options:
keyvaultname: "cloudzero-dev-keyvault"
keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY"
usevmmanagedidentity: "true"
usepodidentity: "false"
usevmmanagedidentity: "true"
vmmanagedidentityclientid: $VMSS_CLIENT_ID
keyvaultname: "cz-jedidev-keyvault"
keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY"
---
apiVersion: extensions/v1beta1
kind: Deployment
@ -54,10 +51,11 @@ spec:
- name: flask-secret
flexVolume:
options:
keyvaultname: "cloudzero-dev-keyvault"
keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY"
usevmmanagedidentity: "true"
usepodidentity: "false"
usevmmanagedidentity: "true"
vmmanagedidentityclientid: $VMSS_CLIENT_ID
keyvaultname: "cz-jedidev-keyvault"
keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY"
---
apiVersion: batch/v1beta1
kind: CronJob
@ -72,7 +70,8 @@ spec:
- name: flask-secret
flexVolume:
options:
keyvaultname: "cloudzero-dev-keyvault"
keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY"
usevmmanagedidentity: "true"
usepodidentity: "false"
usevmmanagedidentity: "true"
vmmanagedidentityclientid: $VMSS_CLIENT_ID
keyvaultname: "cz-jedidev-keyvault"
keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY"

3
deploy/overlays/cloudzero-dev/kustomization.yaml
View File

@ -1,9 +1,8 @@
namespace: staging
namespace: cloudzero-dev
bases:
- ../../azure/
resources:
- namespace.yml
- reset-cron-job.yml
patchesStrategicMerge:
- ports.yml
- envvars.yml

2
deploy/overlays/cloudzero-dev/namespace.yml
View File

@ -1,4 +1,4 @@
apiVersion: v1
kind: Namespace
metadata:
name: staging
name: cloudzero-dev

4
deploy/overlays/cloudzero-dev/ports.yml
View File

@ -5,7 +5,7 @@ metadata:
name: atst-main
annotations:
service.beta.kubernetes.io/azure-load-balancer-internal: "true"
service.beta.kubernetes.io/azure-load-balancer-internal-subnet: "cloudzero-dev-public"
service.beta.kubernetes.io/azure-load-balancer-internal-subnet: "cloudzero-jedidev-public"
spec:
loadBalancerIP: ""
ports:
@ -22,7 +22,7 @@ metadata:
name: atst-auth
annotations:
service.beta.kubernetes.io/azure-load-balancer-internal: "true"
service.beta.kubernetes.io/azure-load-balancer-internal-subnet: "cloudzero-dev-public"
service.beta.kubernetes.io/azure-load-balancer-internal-subnet: "cloudzero-jedidev-public"
spec:
loadBalancerIP: ""
ports:

46
deploy/overlays/cloudzero-dev/reset-cron-job.yml
View File

@ -1,46 +0,0 @@
apiVersion: batch/v1beta1
kind: CronJob
metadata:
name: reset-db
namespace: atat
spec:
schedule: "0 4 * * *"
concurrencyPolicy: Replace
successfulJobsHistoryLimit: 1
jobTemplate:
spec:
template:
metadata:
labels:
app: atst
role: reset-db
aadpodidbinding: atat-kv-id-binding
spec:
restartPolicy: OnFailure
containers:
- name: reset
image: $CONTAINER_IMAGE
command: [
"/bin/sh", "-c"
]
args: [
"/opt/atat/atst/.venv/bin/python",
"/opt/atat/atst/script/reset_database.py"
]
envFrom:
- configMapRef:
name: atst-worker-envvars
volumeMounts:
- name: flask-secret
mountPath: "/config"
volumes:
- name: flask-secret
flexVolume:
driver: "azure/kv"
options:
usepodidentity: "true"
keyvaultname: "atat-vault-test"
keyvaultobjectnames: "staging-AZURE-STORAGE-KEY;staging-MAIL-PASSWORD;staging-PGPASSWORD;staging-REDIS-PASSWORD;staging-SECRET-KEY"
keyvaultobjectaliases: "AZURE_STORAGE_KEY;MAIL_PASSWORD;PGPASSWORD;REDIS_PASSWORD;SECRET_KEY"
keyvaultobjecttypes: "secret;secret;secret;secret;key"
tenantid: $TENANT_ID

5
deploy/overlays/migration-cloudzero-dev/kustomization.yaml Normal file
View File

@ -0,0 +1,5 @@
namespace: cloudzero-dev
bases:
- ../../shared/
patchesStrategicMerge:
- migration.yaml

16
deploy/overlays/migration-cloudzero-dev/migration.yaml Normal file
View File

@ -0,0 +1,16 @@
apiVersion: batch/v1
kind: Job
metadata:
name: migration
spec:
template:
spec:
volumes:
- name: flask-secret
flexVolume:
options:
usepodidentity: "false"
usevmmanagedidentity: "true"
vmmanagedidentityclientid: $VMSS_CLIENT_ID
keyvaultname: "cz-jedidev-keyvault"
keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY"

3
deploy/shared/kustomization.yaml Normal file
View File

@ -0,0 +1,3 @@
namespace: atat
resources:
- migration.yaml

3
js/components/sidenav_toggler.js
View File

@ -1,5 +1,6 @@
import ExpandSidenavMixin from '../mixins/expand_sidenav'
import ToggleMixin from '../mixins/toggle'
import { sidenavCookieName } from '../lib/constants'
export default {
name: 'sidenav-toggler',
@ -14,7 +15,7 @@ export default {
toggle: function(e) {
e.preventDefault()
this.isVisible = !this.isVisible
document.cookie = this.cookieName + '=' + this.isVisible + '; path=/'
document.cookie = sidenavCookieName + '=' + this.isVisible + '; path=/'
this.$parent.$emit('sidenavToggle', this.isVisible)
},
},

1
js/lib/constants.js Normal file
View File

@ -0,0 +1 @@
export const sidenavCookieName = 'expandSidenav'

7
js/mixins/expand_sidenav.js
View File

@ -1,11 +1,12 @@
import { sidenavCookieName } from '../lib/constants'
export default {
props: {
cookieName: 'expandSidenav',
defaultVisible: {
type: Boolean,
default: function() {
if (document.cookie.match(this.cookieName)) {
return !!document.cookie.match(this.cookieName + ' *= *true')
if (document.cookie.match(sidenavCookieName)) {
return !!document.cookie.match(sidenavCookieName + ' *= *true')
} else {
return true
}

26
script/database_setup.py
View File

@ -16,16 +16,14 @@ from reset_database import reset_database
def database_setup(username, password, dbname, ccpo_users):
print("Applying schema and seeding roles and permissions.")
reset_database()
print(
f"Creating Postgres user role for '{username}' and granting all privileges to database '{dbname}'."
)
try:
_create_database_user(username, password, dbname)
except sqlalchemy.exc.ProgrammingError as err:
print(f"Postgres user role '{username}' already exists.")
_create_database_user(username, password, dbname)
print("Applying schema and seeding roles and permissions.")
reset_database()
print("Creating initial set of CCPO users.")
_add_ccpo_users(ccpo_users)
@ -47,6 +45,22 @@ def _create_database_user(username, password, dbname):
f"ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL PRIVILEGES ON FUNCTIONS TO {username}; \n"
)
try:
# TODO: make this more configurable
engine.execute(f"GRANT {username} TO azure_pg_admin;")
except sqlalchemy.exc.ProgrammingError as err:
print(f"Cannot grant new role {username} to azure_pg_admin")
for table in meta.tables:
engine.execute(f"ALTER TABLE {table} OWNER TO {username};\n")
sequence_results = engine.execute(
"SELECT c.relname FROM pg_class c WHERE c.relkind = 'S';"
).fetchall()
sequences = [p[0] for p in sequence_results]
for sequence in sequences:
engine.execute(f"ALTER SEQUENCE {sequence} OWNER TO {username};\n")
trans.commit()

1
script/k8s_config
View File

@ -13,6 +13,7 @@ SETTINGS=(
AUTH_DOMAIN
KV_MI_ID
KV_MI_CLIENT_ID
VMSS_CLIENT_ID
TENANT_ID
)

8
templates/components/accordion.html
View File

@ -6,8 +6,12 @@
heading_tag="h2",
heading_classes="",
content_tag="div",
content_classes="") %}
<accordion v-cloak inline-template>
content_classes="",
default_visible=False) %}
<accordion
v-cloak
inline-template
v-bind:default-visible='{{ default_visible | string | lower }}'>
<{{wrapper_tag}} class="{{ wrapper_classes }}">
<{{heading_tag}} class="accordion__button {{ heading_classes }}">
<button

8
templates/portfolios/reports/expired_task_orders.html
View File

@ -16,13 +16,12 @@
<th>PoP</th>
<th>CLIN Value</th>
<th>Amount Obligated</th>
<th>Amount Unspent</th>
</tr>
</thead>
<tbody>
{% for task_order in expired_task_orders %}
<tr>
<td colspan="5">
<td colspan="4">
<span class="h4 reporting-expended-funding__header">Task Order</span> <a href="{{ url_for("task_orders.view_task_order", task_order_id=task_order.id) }}">
{{ task_order.number }} {{ Icon("caret_right", classes="icon--tiny icon--blue" ) }}
</a>
@ -39,9 +38,8 @@
-
{{ clin.end_date | formattedDate(formatter="%b %d, %Y") }}
</td>
<td>{{ clin.total_amount | dollars }}</td>
<td>{{ clin.obligated_amount | dollars }}</td>
<td>{{ 0 | dollars }}</td>
<td class="table-cell--align-right">{{ clin.total_amount | dollars }}</td>
<td class="table-cell--align-right">{{ clin.obligated_amount | dollars }}</td>
<tr>
{% endfor %}
{% endfor %}

2
templates/portfolios/reports/index.html
View File

@ -13,7 +13,5 @@
<hr>
{% include "portfolios/reports/obligated_funds.html" %}
{% include "portfolios/reports/expired_task_orders.html" %}
<hr>
{% include "portfolios/reports/application_and_env_spending.html" %}
</div>
{% endblock %}

93
templates/portfolios/reports/obligated_funds.html
View File

@ -7,61 +7,56 @@
</header>
<div class='panel'>
<div class='panel__content jedi-clin-funding'>
{% for JEDI_clin in current_obligated_funds | sort(attribute='name')%}
<div class="jedi-clin-funding__clin-wrapper">
<h3 class="h5 jedi-clin-funding__header">
{{ "JEDICLINType.{}".format(JEDI_clin.name) | translate }}
</h3>
<p class="jedi-clin-funding__subheader">Total obligated amount: {{ JEDI_clin.obligated | dollars }}</p>
<div class="jedi-clin-funding__graph">
{% if JEDI_clin.remaining < 0 %}
<span style="width:100%" class="jedi-clin-funding__graph-bar jedi-clin-funding__graph-bar--insufficient"></span>
{% else %}
{% set invoiced_width = (JEDI_clin.invoiced, JEDI_clin.obligated) | obligatedFundingGraphWidth %}
{% if invoiced_width %}
<span style="width:{{ invoiced_width }}%"
class="jedi-clin-funding__graph-bar jedi-clin-funding__graph-bar--invoiced">
</span>
{% endif %}
{% set estimated_width = (JEDI_clin.estimated, JEDI_clin.obligated) | obligatedFundingGraphWidth %}
{% if estimated_width %}
<span style="width:{{ (JEDI_clin.estimated, JEDI_clin.obligated) | obligatedFundingGraphWidth }}%"
class="jedi-clin-funding__graph-bar jedi-clin-funding__graph-bar--estimated">
</span>
{% endif %}
<span style="width:{{ (JEDI_clin.remaining, JEDI_clin.obligated) | obligatedFundingGraphWidth }}%"
class="jedi-clin-funding__graph-bar jedi-clin-funding__graph-bar--remaining">
<div class="jedi-clin-funding__clin-wrapper">
<h3 class="h5 jedi-clin-funding__header">
Total obligated amount: {{ current_obligated_funds.obligated | dollars }}
</h3>
<div class="jedi-clin-funding__graph">
{% if current_obligated_funds.remaining < 0 %}
<span style="width:100%" class="jedi-clin-funding__graph-bar jedi-clin-funding__graph-bar--insufficient"></span>
{% else %}
{% set invoiced_width = (current_obligated_funds.invoiced, current_obligated_funds.obligated) | obligatedFundingGraphWidth %}
{% if invoiced_width %}
<span style="width:{{ invoiced_width }}%"
class="jedi-clin-funding__graph-bar jedi-clin-funding__graph-bar--invoiced">
</span>
{% endif %}
{% set estimated_width = (current_obligated_funds.estimated, current_obligated_funds.obligated) | obligatedFundingGraphWidth %}
{% if estimated_width %}
<span style="width:{{ (current_obligated_funds.estimated, current_obligated_funds.obligated) | obligatedFundingGraphWidth }}%"
class="jedi-clin-funding__graph-bar jedi-clin-funding__graph-bar--estimated">
</span>
{% endif %}
<span style="width:{{ (current_obligated_funds.remaining, current_obligated_funds.obligated) | obligatedFundingGraphWidth }}%"
class="jedi-clin-funding__graph-bar jedi-clin-funding__graph-bar--remaining">
</span>
{% endif %}
</div>
<div class="jedi-clin-funding__graph-values">
<div class="jedi-clin-funding__meta">
<p class="jedi-clin-funding__meta-header">
<span class="jedi-clin-funding__meta-key jedi-clin-funding__meta-key--invoiced"></span>
Invoiced expended funds:
</p>
<p class="h3 jedi-clin-funding__meta-value">{{ current_obligated_funds.invoiced | dollars }}</p>
</div>
<div class="jedi-clin-funding__graph-values">
<div class="jedi-clin-funding__meta">
<p class="jedi-clin-funding__meta-header">
<span class="jedi-clin-funding__meta-key jedi-clin-funding__meta-key--invoiced"></span>
Invoiced expended funds:
</p>
<p class="h3 jedi-clin-funding__meta-value">{{ JEDI_clin.invoiced | dollars }}</p>
</div>
<div class="jedi-clin-funding__meta">
<p class="jedi-clin-funding__meta-header">
<span class="jedi-clin-funding__meta-key jedi-clin-funding__meta-key--estimated"></span>
Estimated expended funds:
</p>
<p class="h3 jedi-clin-funding__meta-value">{{ JEDI_clin.estimated | dollars }}</p>
</div>
<div class="jedi-clin-funding__meta">
<p class="jedi-clin-funding__meta-header">
<span class="jedi-clin-funding__meta-key jedi-clin-funding__meta-key--{{"remaining" if JEDI_clin.remaining > 0 else "insufficient"}}"></span>
Remaining funds:
</p>
<p class="h3 jedi-clin-funding__meta-value {% if JEDI_clin.remaining < 0 %}text-danger{% endif %}">{{ JEDI_clin.remaining | dollars }}</p>
</div>
<div class="jedi-clin-funding__meta">
<p class="jedi-clin-funding__meta-header">
<span class="jedi-clin-funding__meta-key jedi-clin-funding__meta-key--estimated"></span>
Estimated expended funds:
</p>
<p class="h3 jedi-clin-funding__meta-value">{{ current_obligated_funds.estimated | dollars }}</p>
</div>
<div class="jedi-clin-funding__meta">
<p class="jedi-clin-funding__meta-header">
<span class="jedi-clin-funding__meta-key jedi-clin-funding__meta-key--{{"remaining" if current_obligated_funds.remaining > 0 else "insufficient"}}"></span>
Remaining funds:
</p>
<p class="h3 jedi-clin-funding__meta-value {% if current_obligated_funds.remaining < 0 %}text-danger{% endif %}">{{ current_obligated_funds.remaining | dollars }}</p>
</div>
</div>
{% endfor %}
</div>
<div class="jedi-clin-funding__active-task-orders">
<h3 class="h4">
Active Task Orders

10
templates/task_orders/index.html
View File

@ -14,9 +14,15 @@
{% macro TaskOrderList(task_orders, status) %}
{% set show_task_orders = task_orders|length > 0 %}
<div class="accordion">
{% call Accordion(title=("task_orders.status_list_title"|translate({'status': status})), id=status, heading_tag="h4") %}
{% if task_orders|length > 0 %}
{% call Accordion(
title=("task_orders.status_list_title"|translate({'status': status})),
id=status,
heading_tag="h4",
default_visible=show_task_orders
) %}
{% if show_task_orders %}
{% for task_order in task_orders %}
{% set to_number %}
{% if task_order.number != None %}

93
tests/domain/cloud/reports/test_reports.py
View File

@ -1,58 +1,47 @@
from atst.domain.csp.reports import MockReportingProvider
from atst.domain.csp.reports import prepare_azure_reporting_data
from tests.factories import PortfolioFactory
from decimal import Decimal
import pendulum
def test_get_environment_monthly_totals():
environment = {
"name": "Test Environment",
"spending": {
"this_month": {"JEDI_CLIN_1": 100, "JEDI_CLIN_2": 100},
"last_month": {"JEDI_CLIN_1": 200, "JEDI_CLIN_2": 200},
"total": {"JEDI_CLIN_1": 1000, "JEDI_CLIN_2": 1000},
},
}
totals = MockReportingProvider._get_environment_monthly_totals(environment)
assert totals == {
"name": "Test Environment",
"this_month": 200,
"last_month": 400,
"total": 2000,
}
class TestPrepareAzureData:
start_of_month = pendulum.today(tz="utc").start_of("month").replace(tzinfo=None)
next_month = start_of_month.add(months=1).to_atom_string()
this_month = start_of_month.to_atom_string()
last_month = start_of_month.subtract(months=1).to_atom_string()
two_months_ago = last_month = start_of_month.subtract(months=2).to_atom_string()
def test_estimated_and_invoiced(self):
rows = [
[150.0, self.two_months_ago, "", "USD"],
[100.0, self.last_month, "e0500a4qhw", "USD"],
[50.0, self.this_month, "", "USD"],
[50.0, self.next_month, "", "USD"],
]
output = prepare_azure_reporting_data(rows)
def test_get_application_monthly_totals():
portfolio = PortfolioFactory.create(
applications=[
{"name": "Test Application", "environments": [{"name": "Z"}, {"name": "A"}]}
],
)
application = {
"name": "Test Application",
"environments": [
{
"name": "Z",
"spending": {
"this_month": {"JEDI_CLIN_1": 50, "JEDI_CLIN_2": 50},
"last_month": {"JEDI_CLIN_1": 150, "JEDI_CLIN_2": 150},
"total": {"JEDI_CLIN_1": 250, "JEDI_CLIN_2": 250},
},
},
{
"name": "A",
"spending": {
"this_month": {"JEDI_CLIN_1": 100, "JEDI_CLIN_2": 100},
"last_month": {"JEDI_CLIN_1": 200, "JEDI_CLIN_2": 200},
"total": {"JEDI_CLIN_1": 1000, "JEDI_CLIN_2": 1000},
},
},
],
}
assert output.get("invoiced") == Decimal(250.0)
assert output.get("estimated") == Decimal(100.0)
totals = MockReportingProvider._get_application_monthly_totals(
portfolio, application
)
assert totals["name"] == "Test Application"
assert totals["this_month"] == 300
assert totals["last_month"] == 700
assert totals["total"] == 2500
assert [env["name"] for env in totals["environments"]] == ["A", "Z"]
def test_just_estimated(self):
rows = [
[100.0, self.this_month, "", "USD"],
]
output = prepare_azure_reporting_data(rows)
assert output.get("invoiced") == Decimal(0.0)
assert output.get("estimated") == Decimal(100.0)
def test_just_invoiced(self):
rows = [
[100.0, self.last_month, "", "USD"],
]
output = prepare_azure_reporting_data(rows)
assert output.get("invoiced") == Decimal(100.0)
assert output.get("estimated") == Decimal(0.0)
def test_no_rows(self):
output = prepare_azure_reporting_data([])
assert output.get("invoiced") == Decimal(0.0)
assert output.get("estimated") == Decimal(0.0)

21
tests/domain/cloud/test_azure_csp.py
View File

@ -25,6 +25,7 @@ from atst.domain.csp.cloud.models import (
CostManagementQueryCSPResult,
EnvironmentCSPPayload,
EnvironmentCSPResult,
KeyVaultCredentials,
PrincipalAdminRoleCSPPayload,
PrincipalAdminRoleCSPResult,
ProductPurchaseCSPPayload,
@ -938,3 +939,23 @@ def test_create_user(mock_azure: AzureCloudProvider):
result = mock_azure.create_user(payload)
assert result.id == "id"
def test_update_tenant_creds(mock_azure: AzureCloudProvider):
with patch.object(
AzureCloudProvider, "set_secret", wraps=mock_azure.set_secret,
) as set_secret:
set_secret.return_value = None
existing_secrets = {
"tenant_id": "mytenant",
"tenant_admin_username": "admin",
"tenant_admin_password": "foo", # pragma: allowlist secret
}
mock_azure = mock_get_secret(mock_azure, json.dumps(existing_secrets))
mock_new_secrets = KeyVaultCredentials(**MOCK_CREDS)
updated_secret = mock_azure.update_tenant_creds("mytenant", mock_new_secrets)
assert updated_secret == KeyVaultCredentials(
**{**existing_secrets, **MOCK_CREDS}
)

20
tests/domain/cloud/test_models.py
View File

@ -100,6 +100,26 @@ def test_KeyVaultCredentials_enforce_root_creds():
)
def test_KeyVaultCredentials_merge_credentials():
old_secret = KeyVaultCredentials(
tenant_id="foo",
tenant_admin_username="bar",
tenant_admin_password="baz", # pragma: allowlist secret
)
new_secret = KeyVaultCredentials(
tenant_id="foo", tenant_sp_client_id="bip", tenant_sp_key="bop"
)
expected_update = KeyVaultCredentials(
tenant_id="foo",
tenant_admin_username="bar",
tenant_admin_password="baz", # pragma: allowlist secret
tenant_sp_client_id="bip",
tenant_sp_key="bop",
)
assert old_secret.merge_credentials(new_secret) == expected_update
user_payload = {
"tenant_id": "123",
"display_name": "Han Solo",

35
tests/domain/test_reports.py
View File

@ -1,8 +1,31 @@
# TODO: Implement when we get real reporting data
def test_expired_task_orders():
pass
import pytest
from atst.domain.reports import Reports
from tests.factories import PortfolioFactory
from decimal import Decimal
# TODO: Implement when we get real reporting data
def test_obligated_funds_by_JEDI_clin():
pass
@pytest.fixture(scope="function")
def portfolio():
portfolio = PortfolioFactory.create()
return portfolio
class TestGetPortfolioSpending:
csp_data = {
"tenant_id": "",
"billing_profile_properties": {
"invoice_sections": [{"invoice_section_id": "",}]
},
}
def test_with_csp_data(self, portfolio):
portfolio.csp_data = self.csp_data
data = Reports.get_portfolio_spending(portfolio)
assert data["invoiced"] == Decimal(1551.0)
assert data["estimated"] == Decimal(500.0)
def test_without_csp_data(self, portfolio):
data = Reports.get_portfolio_spending(portfolio)
assert data["invoiced"] == Decimal(0)
assert data["estimated"] == Decimal(0)

5
tests/domain/test_task_orders.py
View File

@ -149,11 +149,12 @@ def test_task_order_sort_by_status():
]
sorted_by_status = TaskOrders.sort_by_status(initial_to_list)
assert len(sorted_by_status["Draft"]) == 3
assert len(sorted_by_status["Draft"]) == 4
assert len(sorted_by_status["Active"]) == 1
assert len(sorted_by_status["Upcoming"]) == 1
assert len(sorted_by_status["Expired"]) == 2
assert len(sorted_by_status["Unsigned"]) == 1
with pytest.raises(KeyError):
sorted_by_status["Unsigned"]
assert list(sorted_by_status.keys()) == [status.value for status in SORT_ORDERING]

16
tests/test_app.py
View File

@ -7,6 +7,7 @@ from atst.app import (
make_crl_validator,
apply_config_from_directory,
apply_config_from_environment,
make_config,
)
@ -67,3 +68,18 @@ def test_apply_config_from_environment_skips_unknown_settings(
monkeypatch.setenv("FLARF", "MAYO")
apply_config_from_environment(config_object)
assert "FLARF" not in config_object.options("default")
class TestMakeConfig:
def test_redis_ssl_connection(self):
config = make_config({"REDIS_TLS": True})
uri = config.get("REDIS_URI")
assert "rediss" in uri
assert "ssl_cert_reqs" in uri
def test_non_redis_ssl_connection(self):
config = make_config({"REDIS_TLS": False})
uri = config.get("REDIS_URI")
assert "rediss" not in uri
assert "redis" in uri
assert "ssl_cert_reqs" not in uri

4
translations.yaml
View File

@ -313,7 +313,7 @@ forms:
upload_error: There was an error uploading your file. Please try again. If you encounter repeated problems uploading this file, please contact CCPO.
size_error: "The file you have selected is too large. Please choose a file no larger than {file_size_limit}MB."
filename_error: File names can only contain the characters A-Z, 0-9, space, hyphen, underscore, and period.
number_description: 13-Digit Task Order Number
number_description: Task Order Number
pop_errors:
date_order: PoP start date must be before end date.
range: Date must be between {start} and {end}.
@ -530,7 +530,7 @@ task_orders:
form:
add_clin: Add Another CLIN
add_to_header: Enter the Task Order number
add_to_description: Please input your 13-digit Task Order number. This number may be listed under "Order Number" if your Contracting Officer used form 1149, or "Delivery Order/Call No." if form 1155 was used. Moving forward, this portion of funding will be referenced by the recorded Task Order number.
add_to_description: Please input your Task Order number. This number may be listed under "Order Number" if your Contracting Officer used form 1149, or "Delivery Order/Call No." if form 1155 was used. Moving forward, this portion of funding will be referenced by the recorded Task Order number.
builder_base:
cancel_modal: Do you want to save this draft?
delete_draft: No, delete it