Merge branch 'staging' into redis-ssl-verify

This commit is contained in:
dandds 2020-02-10 17:02:26 -05:00 committed by GitHub
commit f0ddc9b2aa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 191 additions and 284 deletions

View File

@ -1060,7 +1060,7 @@ class AzureCloudProvider(CloudProviderInterface):
"timeframe": "Custom",
"timePeriod": {"from": payload.from_date, "to": payload.to_date,},
"dataset": {
"granularity": "Daily",
"granularity": "Monthly",
"aggregation": {"totalCost": {"name": "PreTaxCost", "function": "Sum"}},
"grouping": [{"type": "Dimension", "name": "InvoiceId"}],
},

View File

@ -1,4 +1,5 @@
from uuid import uuid4
import pendulum
from .cloud_provider_interface import CloudProviderInterface
from .exceptions import (
@ -459,15 +460,26 @@ class MockCloudProvider(CloudProviderInterface):
self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION)
object_id = str(uuid4())
start_of_month = pendulum.today(tz="utc").start_of("month").replace(tzinfo=None)
this_month = start_of_month.to_atom_string()
last_month = start_of_month.subtract(months=1).to_atom_string()
two_months_ago = start_of_month.subtract(months=2).to_atom_string()
properties = CostManagementQueryProperties(
**dict(
columns=[
{"name": "PreTaxCost", "type": "Number"},
{"name": "UsageDate", "type": "Number"},
{"name": "BillingMonth", "type": "Datetime"},
{"name": "InvoiceId", "type": "String"},
{"name": "Currency", "type": "String"},
],
rows=[],
rows=[
[1.0, two_months_ago, "", "USD"],
[500.0, two_months_ago, "e05009w9sf", "USD"],
[50.0, last_month, "", "USD"],
[1000.0, last_month, "e0500a4qhw", "USD"],
[500.0, this_month, "", "USD"],
],
)
)

View File

@ -1,6 +1,6 @@
from collections import defaultdict
import json
from decimal import Decimal
import pendulum
def load_fixture_data():
@ -11,128 +11,25 @@ def load_fixture_data():
class MockReportingProvider:
FIXTURE_SPEND_DATA = load_fixture_data()
@classmethod
def get_portfolio_monthly_spending(cls, portfolio):
"""
returns an array of application and environment spending for the
portfolio. Applications and their nested environments are sorted in
alphabetical order by name.
[
{
name
this_month
last_month
total
environments [
{
name
this_month
last_month
total
}
]
}
]
"""
fixture_apps = cls.FIXTURE_SPEND_DATA.get(portfolio.name, {}).get(
"applications", []
)
def prepare_azure_reporting_data(rows: list):
"""
Returns a dict representing invoiced and estimated funds for a portfolio given
a list of rows from CostManagementQueryCSPResult.properties.rows
{
invoiced: Decimal,
estimated: Decimal
}
"""
for application in portfolio.applications:
if application.name not in [app["name"] for app in fixture_apps]:
fixture_apps.append({"name": application.name, "environments": []})
estimated = []
while rows:
if pendulum.parse(rows[-1][1]) >= pendulum.now(tz="utc").start_of("month"):
estimated.append(rows.pop())
else:
break
return sorted(
[
cls._get_application_monthly_totals(portfolio, fixture_app)
for fixture_app in fixture_apps
if fixture_app["name"]
in [application.name for application in portfolio.applications]
],
key=lambda app: app["name"],
)
@classmethod
def _get_environment_monthly_totals(cls, environment):
"""
returns a dictionary that represents spending totals for an environment e.g.
{
name
this_month
last_month
total
}
"""
return {
"name": environment["name"],
"this_month": sum(environment["spending"]["this_month"].values()),
"last_month": sum(environment["spending"]["last_month"].values()),
"total": sum(environment["spending"]["total"].values()),
}
@classmethod
def _get_application_monthly_totals(cls, portfolio, fixture_app):
"""
returns a dictionary that represents spending totals for an application
and its environments e.g.
{
name
this_month
last_month
total
environments: [
{
name
this_month
last_month
total
}
]
}
"""
application_envs = [
env
for env in portfolio.all_environments
if env.application.name == fixture_app["name"]
]
environments = [
cls._get_environment_monthly_totals(env)
for env in fixture_app["environments"]
if env["name"] in [e.name for e in application_envs]
]
for env in application_envs:
if env.name not in [env["name"] for env in environments]:
environments.append({"name": env.name})
return {
"name": fixture_app["name"],
"this_month": sum(env.get("this_month", 0) for env in environments),
"last_month": sum(env.get("last_month", 0) for env in environments),
"total": sum(env.get("total", 0) for env in environments),
"environments": sorted(environments, key=lambda env: env["name"]),
}
@classmethod
def get_spending_by_JEDI_clin(cls, portfolio):
"""
returns an dictionary of spending per JEDI CLIN for a portfolio
{
jedi_clin: {
invoiced
estimated
},
}
"""
if portfolio.name in cls.FIXTURE_SPEND_DATA:
CLIN_spend_dict = defaultdict(lambda: defaultdict(Decimal))
for application in cls.FIXTURE_SPEND_DATA[portfolio.name]["applications"]:
for environment in application["environments"]:
for clin, spend in environment["spending"]["this_month"].items():
CLIN_spend_dict[clin]["estimated"] += Decimal(spend)
for clin, spend in environment["spending"]["total"].items():
CLIN_spend_dict[clin]["invoiced"] += Decimal(spend)
return CLIN_spend_dict
return {}
return dict(
invoiced=Decimal(sum([row[0] for row in rows])),
estimated=Decimal(sum([row[0] for row in estimated])),
)

View File

@ -1,12 +1,13 @@
from flask import current_app
from itertools import groupby
from atst.domain.csp.cloud.models import (
ReportingCSPPayload,
CostManagementQueryCSPResult,
)
from atst.domain.csp.reports import prepare_azure_reporting_data
import pendulum
class Reports:
@classmethod
def monthly_spending(cls, portfolio):
return current_app.csp.reports.get_portfolio_monthly_spending(portfolio)
@classmethod
def expired_task_orders(cls, portfolio):
return [
@ -14,31 +15,19 @@ class Reports:
]
@classmethod
def obligated_funds_by_JEDI_clin(cls, portfolio):
clin_spending = current_app.csp.reports.get_spending_by_JEDI_clin(portfolio)
active_clins = portfolio.active_clins
for jedi_clin, clins in groupby(
active_clins, key=lambda clin: clin.jedi_clin_type
):
if not clin_spending.get(jedi_clin.name):
clin_spending[jedi_clin.name] = {}
clin_spending[jedi_clin.name]["obligated"] = sum(
clin.obligated_amount for clin in clins
)
def get_portfolio_spending(cls, portfolio):
# TODO: Extend this function to make from_date and to_date configurable
from_date = pendulum.now().subtract(years=1).add(days=1).format("YYYY-MM-DD")
to_date = pendulum.now().format("YYYY-MM-DD")
rows = []
output = []
for clin in clin_spending.keys():
invoiced = clin_spending[clin].get("invoiced", 0)
estimated = clin_spending[clin].get("estimated", 0)
obligated = clin_spending[clin].get("obligated", 0)
remaining = obligated - (invoiced + estimated)
output.append(
{
"name": clin,
"invoiced": invoiced,
"estimated": estimated,
"obligated": obligated,
"remaining": remaining,
}
if portfolio.csp_data:
payload = ReportingCSPPayload(
from_date=from_date, to_date=to_date, **portfolio.csp_data
)
return output
response: CostManagementQueryCSPResult = current_app.csp.cloud.get_reporting_data(
payload
)
rows = response.properties.rows
return prepare_azure_reporting_data(rows)

View File

@ -5,7 +5,7 @@ from flask import render_template
from jinja2 import contextfilter
from jinja2.exceptions import TemplateNotFound
from urllib.parse import urlparse, urlunparse, parse_qs, urlencode
from decimal import DivisionByZero as DivisionByZeroException
from decimal import DivisionByZero as DivisionByZeroException, InvalidOperation
def iconSvg(name):
@ -43,7 +43,7 @@ def obligatedFundingGraphWidth(values):
numerator, denominator = values
try:
return (numerator / denominator) * 100
except DivisionByZeroException:
except (DivisionByZeroException, InvalidOperation):
return 0

View File

@ -89,6 +89,12 @@ class Portfolio(
def active_task_orders(self):
return [task_order for task_order in self.task_orders if task_order.is_active]
@property
def total_obligated_funds(self):
return sum(
(task_order.total_obligated_funds for task_order in self.active_task_orders)
)
@property
def funding_duration(self):
"""

View File

@ -34,25 +34,25 @@ def create_portfolio():
@user_can(Permissions.VIEW_PORTFOLIO_REPORTS, message="view portfolio reports")
def reports(portfolio_id):
portfolio = Portfolios.get(g.current_user, portfolio_id)
spending = Reports.get_portfolio_spending(portfolio)
obligated = portfolio.total_obligated_funds
remaining = obligated - (spending["invoiced"] + spending["estimated"])
current_obligated_funds = Reports.obligated_funds_by_JEDI_clin(portfolio)
current_obligated_funds = {
**spending,
"obligated": obligated,
"remaining": remaining,
}
if any(map(lambda clin: clin["remaining"] < 0, current_obligated_funds)):
if current_obligated_funds["remaining"] < 0:
flash("insufficient_funds")
# wrapped in str() because the sum of obligated funds returns a Decimal object
total_portfolio_value = str(
sum(
task_order.total_obligated_funds
for task_order in portfolio.active_task_orders
)
)
return render_template(
"portfolios/reports/index.html",
portfolio=portfolio,
total_portfolio_value=total_portfolio_value,
# wrapped in str() because the sum of obligated funds returns a Decimal object
total_portfolio_value=str(portfolio.total_obligated_funds),
current_obligated_funds=current_obligated_funds,
expired_task_orders=Reports.expired_task_orders(portfolio),
monthly_spending=Reports.monthly_spending(portfolio),
retrieved=datetime.now(), # mocked datetime of reporting data retrival
)

View File

@ -16,13 +16,12 @@
<th>PoP</th>
<th>CLIN Value</th>
<th>Amount Obligated</th>
<th>Amount Unspent</th>
</tr>
</thead>
<tbody>
{% for task_order in expired_task_orders %}
<tr>
<td colspan="5">
<td colspan="4">
<span class="h4 reporting-expended-funding__header">Task Order</span> <a href="{{ url_for("task_orders.view_task_order", task_order_id=task_order.id) }}">
{{ task_order.number }} {{ Icon("caret_right", classes="icon--tiny icon--blue" ) }}
</a>
@ -39,9 +38,8 @@
-
{{ clin.end_date | formattedDate(formatter="%b %d, %Y") }}
</td>
<td>{{ clin.total_amount | dollars }}</td>
<td>{{ clin.obligated_amount | dollars }}</td>
<td>{{ 0 | dollars }}</td>
<td class="table-cell--align-right">{{ clin.total_amount | dollars }}</td>
<td class="table-cell--align-right">{{ clin.obligated_amount | dollars }}</td>
<tr>
{% endfor %}
{% endfor %}

View File

@ -13,7 +13,5 @@
<hr>
{% include "portfolios/reports/obligated_funds.html" %}
{% include "portfolios/reports/expired_task_orders.html" %}
<hr>
{% include "portfolios/reports/application_and_env_spending.html" %}
</div>
{% endblock %}

View File

@ -7,61 +7,56 @@
</header>
<div class='panel'>
<div class='panel__content jedi-clin-funding'>
{% for JEDI_clin in current_obligated_funds | sort(attribute='name')%}
<div class="jedi-clin-funding__clin-wrapper">
<h3 class="h5 jedi-clin-funding__header">
{{ "JEDICLINType.{}".format(JEDI_clin.name) | translate }}
</h3>
<p class="jedi-clin-funding__subheader">Total obligated amount: {{ JEDI_clin.obligated | dollars }}</p>
<div class="jedi-clin-funding__graph">
{% if JEDI_clin.remaining < 0 %}
<span style="width:100%" class="jedi-clin-funding__graph-bar jedi-clin-funding__graph-bar--insufficient"></span>
{% else %}
{% set invoiced_width = (JEDI_clin.invoiced, JEDI_clin.obligated) | obligatedFundingGraphWidth %}
{% if invoiced_width %}
<span style="width:{{ invoiced_width }}%"
class="jedi-clin-funding__graph-bar jedi-clin-funding__graph-bar--invoiced">
</span>
{% endif %}
{% set estimated_width = (JEDI_clin.estimated, JEDI_clin.obligated) | obligatedFundingGraphWidth %}
{% if estimated_width %}
<span style="width:{{ (JEDI_clin.estimated, JEDI_clin.obligated) | obligatedFundingGraphWidth }}%"
class="jedi-clin-funding__graph-bar jedi-clin-funding__graph-bar--estimated">
</span>
{% endif %}
<span style="width:{{ (JEDI_clin.remaining, JEDI_clin.obligated) | obligatedFundingGraphWidth }}%"
class="jedi-clin-funding__graph-bar jedi-clin-funding__graph-bar--remaining">
<div class="jedi-clin-funding__clin-wrapper">
<h3 class="h5 jedi-clin-funding__header">
Total obligated amount: {{ current_obligated_funds.obligated | dollars }}
</h3>
<div class="jedi-clin-funding__graph">
{% if current_obligated_funds.remaining < 0 %}
<span style="width:100%" class="jedi-clin-funding__graph-bar jedi-clin-funding__graph-bar--insufficient"></span>
{% else %}
{% set invoiced_width = (current_obligated_funds.invoiced, current_obligated_funds.obligated) | obligatedFundingGraphWidth %}
{% if invoiced_width %}
<span style="width:{{ invoiced_width }}%"
class="jedi-clin-funding__graph-bar jedi-clin-funding__graph-bar--invoiced">
</span>
{% endif %}
{% set estimated_width = (current_obligated_funds.estimated, current_obligated_funds.obligated) | obligatedFundingGraphWidth %}
{% if estimated_width %}
<span style="width:{{ (current_obligated_funds.estimated, current_obligated_funds.obligated) | obligatedFundingGraphWidth }}%"
class="jedi-clin-funding__graph-bar jedi-clin-funding__graph-bar--estimated">
</span>
{% endif %}
<span style="width:{{ (current_obligated_funds.remaining, current_obligated_funds.obligated) | obligatedFundingGraphWidth }}%"
class="jedi-clin-funding__graph-bar jedi-clin-funding__graph-bar--remaining">
</span>
{% endif %}
</div>
<div class="jedi-clin-funding__graph-values">
<div class="jedi-clin-funding__meta">
<p class="jedi-clin-funding__meta-header">
<span class="jedi-clin-funding__meta-key jedi-clin-funding__meta-key--invoiced"></span>
Invoiced expended funds:
</p>
<p class="h3 jedi-clin-funding__meta-value">{{ current_obligated_funds.invoiced | dollars }}</p>
</div>
<div class="jedi-clin-funding__graph-values">
<div class="jedi-clin-funding__meta">
<p class="jedi-clin-funding__meta-header">
<span class="jedi-clin-funding__meta-key jedi-clin-funding__meta-key--invoiced"></span>
Invoiced expended funds:
</p>
<p class="h3 jedi-clin-funding__meta-value">{{ JEDI_clin.invoiced | dollars }}</p>
</div>
<div class="jedi-clin-funding__meta">
<p class="jedi-clin-funding__meta-header">
<span class="jedi-clin-funding__meta-key jedi-clin-funding__meta-key--estimated"></span>
Estimated expended funds:
</p>
<p class="h3 jedi-clin-funding__meta-value">{{ JEDI_clin.estimated | dollars }}</p>
</div>
<div class="jedi-clin-funding__meta">
<p class="jedi-clin-funding__meta-header">
<span class="jedi-clin-funding__meta-key jedi-clin-funding__meta-key--{{"remaining" if JEDI_clin.remaining > 0 else "insufficient"}}"></span>
Remaining funds:
</p>
<p class="h3 jedi-clin-funding__meta-value {% if JEDI_clin.remaining < 0 %}text-danger{% endif %}">{{ JEDI_clin.remaining | dollars }}</p>
</div>
<div class="jedi-clin-funding__meta">
<p class="jedi-clin-funding__meta-header">
<span class="jedi-clin-funding__meta-key jedi-clin-funding__meta-key--estimated"></span>
Estimated expended funds:
</p>
<p class="h3 jedi-clin-funding__meta-value">{{ current_obligated_funds.estimated | dollars }}</p>
</div>
<div class="jedi-clin-funding__meta">
<p class="jedi-clin-funding__meta-header">
<span class="jedi-clin-funding__meta-key jedi-clin-funding__meta-key--{{"remaining" if current_obligated_funds.remaining > 0 else "insufficient"}}"></span>
Remaining funds:
</p>
<p class="h3 jedi-clin-funding__meta-value {% if current_obligated_funds.remaining < 0 %}text-danger{% endif %}">{{ current_obligated_funds.remaining | dollars }}</p>
</div>
</div>
{% endfor %}
</div>
<div class="jedi-clin-funding__active-task-orders">
<h3 class="h4">
Active Task Orders

View File

@ -1,58 +1,47 @@
from atst.domain.csp.reports import MockReportingProvider
from atst.domain.csp.reports import prepare_azure_reporting_data
from tests.factories import PortfolioFactory
from decimal import Decimal
import pendulum
def test_get_environment_monthly_totals():
environment = {
"name": "Test Environment",
"spending": {
"this_month": {"JEDI_CLIN_1": 100, "JEDI_CLIN_2": 100},
"last_month": {"JEDI_CLIN_1": 200, "JEDI_CLIN_2": 200},
"total": {"JEDI_CLIN_1": 1000, "JEDI_CLIN_2": 1000},
},
}
totals = MockReportingProvider._get_environment_monthly_totals(environment)
assert totals == {
"name": "Test Environment",
"this_month": 200,
"last_month": 400,
"total": 2000,
}
class TestPrepareAzureData:
start_of_month = pendulum.today(tz="utc").start_of("month").replace(tzinfo=None)
next_month = start_of_month.add(months=1).to_atom_string()
this_month = start_of_month.to_atom_string()
last_month = start_of_month.subtract(months=1).to_atom_string()
two_months_ago = last_month = start_of_month.subtract(months=2).to_atom_string()
def test_estimated_and_invoiced(self):
rows = [
[150.0, self.two_months_ago, "", "USD"],
[100.0, self.last_month, "e0500a4qhw", "USD"],
[50.0, self.this_month, "", "USD"],
[50.0, self.next_month, "", "USD"],
]
output = prepare_azure_reporting_data(rows)
def test_get_application_monthly_totals():
portfolio = PortfolioFactory.create(
applications=[
{"name": "Test Application", "environments": [{"name": "Z"}, {"name": "A"}]}
],
)
application = {
"name": "Test Application",
"environments": [
{
"name": "Z",
"spending": {
"this_month": {"JEDI_CLIN_1": 50, "JEDI_CLIN_2": 50},
"last_month": {"JEDI_CLIN_1": 150, "JEDI_CLIN_2": 150},
"total": {"JEDI_CLIN_1": 250, "JEDI_CLIN_2": 250},
},
},
{
"name": "A",
"spending": {
"this_month": {"JEDI_CLIN_1": 100, "JEDI_CLIN_2": 100},
"last_month": {"JEDI_CLIN_1": 200, "JEDI_CLIN_2": 200},
"total": {"JEDI_CLIN_1": 1000, "JEDI_CLIN_2": 1000},
},
},
],
}
assert output.get("invoiced") == Decimal(250.0)
assert output.get("estimated") == Decimal(100.0)
totals = MockReportingProvider._get_application_monthly_totals(
portfolio, application
)
assert totals["name"] == "Test Application"
assert totals["this_month"] == 300
assert totals["last_month"] == 700
assert totals["total"] == 2500
assert [env["name"] for env in totals["environments"]] == ["A", "Z"]
def test_just_estimated(self):
rows = [
[100.0, self.this_month, "", "USD"],
]
output = prepare_azure_reporting_data(rows)
assert output.get("invoiced") == Decimal(0.0)
assert output.get("estimated") == Decimal(100.0)
def test_just_invoiced(self):
rows = [
[100.0, self.last_month, "", "USD"],
]
output = prepare_azure_reporting_data(rows)
assert output.get("invoiced") == Decimal(100.0)
assert output.get("estimated") == Decimal(0.0)
def test_no_rows(self):
output = prepare_azure_reporting_data([])
assert output.get("invoiced") == Decimal(0.0)
assert output.get("estimated") == Decimal(0.0)

View File

@ -1,8 +1,31 @@
# TODO: Implement when we get real reporting data
def test_expired_task_orders():
pass
import pytest
from atst.domain.reports import Reports
from tests.factories import PortfolioFactory
from decimal import Decimal
# TODO: Implement when we get real reporting data
def test_obligated_funds_by_JEDI_clin():
pass
@pytest.fixture(scope="function")
def portfolio():
portfolio = PortfolioFactory.create()
return portfolio
class TestGetPortfolioSpending:
csp_data = {
"tenant_id": "",
"billing_profile_properties": {
"invoice_sections": [{"invoice_section_id": "",}]
},
}
def test_with_csp_data(self, portfolio):
portfolio.csp_data = self.csp_data
data = Reports.get_portfolio_spending(portfolio)
assert data["invoiced"] == Decimal(1551.0)
assert data["estimated"] == Decimal(500.0)
def test_without_csp_data(self, portfolio):
data = Reports.get_portfolio_spending(portfolio)
assert data["invoiced"] == Decimal(0)
assert data["estimated"] == Decimal(0)