Merge branch 'staging' into redis-ssl-verify

atst/domain/csp/cloud/azure_cloud_provider.py

@@ -1060,7 +1060,7 @@ class AzureCloudProvider(CloudProviderInterface):
             "timeframe": "Custom",
             "timePeriod": {"from": payload.from_date, "to": payload.to_date,},
             "dataset": {
-                "granularity": "Daily",
+                "granularity": "Monthly",
                 "aggregation": {"totalCost": {"name": "PreTaxCost", "function": "Sum"}},
                 "grouping": [{"type": "Dimension", "name": "InvoiceId"}],
             },

atst/domain/csp/cloud/mock_cloud_provider.py

@@ -1,4 +1,5 @@
 from uuid import uuid4
+import pendulum
 
 from .cloud_provider_interface import CloudProviderInterface
 from .exceptions import (
@@ -459,15 +460,26 @@ class MockCloudProvider(CloudProviderInterface):
         self._maybe_raise(self.UNAUTHORIZED_RATE, self.AUTHORIZATION_EXCEPTION)
         object_id = str(uuid4())
 
+        start_of_month = pendulum.today(tz="utc").start_of("month").replace(tzinfo=None)
+        this_month = start_of_month.to_atom_string()
+        last_month = start_of_month.subtract(months=1).to_atom_string()
+        two_months_ago = start_of_month.subtract(months=2).to_atom_string()
+
         properties = CostManagementQueryProperties(
             **dict(
                 columns=[
                     {"name": "PreTaxCost", "type": "Number"},
-                    {"name": "UsageDate", "type": "Number"},
+                    {"name": "BillingMonth", "type": "Datetime"},
                     {"name": "InvoiceId", "type": "String"},
                     {"name": "Currency", "type": "String"},
                 ],
-                rows=[],
+                rows=[
+                    [1.0, two_months_ago, "", "USD"],
+                    [500.0, two_months_ago, "e05009w9sf", "USD"],
+                    [50.0, last_month, "", "USD"],
+                    [1000.0, last_month, "e0500a4qhw", "USD"],
+                    [500.0, this_month, "", "USD"],
+                ],
             )
         )
 

atst/domain/csp/reports.py

@@ -1,6 +1,6 @@
-from collections import defaultdict
 import json
 from decimal import Decimal
+import pendulum
 
 
 def load_fixture_data():
@@ -11,128 +11,25 @@ def load_fixture_data():
 class MockReportingProvider:
     FIXTURE_SPEND_DATA = load_fixture_data()
 
-    @classmethod
-    def get_portfolio_monthly_spending(cls, portfolio):
-        """
-        returns an array of application and environment spending for the 
-        portfolio. Applications and their nested environments are sorted in
-        alphabetical order by name.
-        [
-            {
-                name
-                this_month
-                last_month
-                total
-                environments [
-                    {
-                        name
-                        this_month
-                        last_month
-                        total
-                    }
-                ]
-            }
-        ]
-        """
 
-        fixture_apps = cls.FIXTURE_SPEND_DATA.get(portfolio.name, {}).get(
+def prepare_azure_reporting_data(rows: list):
-            "applications", []
+    """
-        )
+    Returns a dict representing invoiced and estimated funds for a portfolio given
+    a list of rows from CostManagementQueryCSPResult.properties.rows
+    {
+        invoiced: Decimal,
+        estimated: Decimal
+    }
+    """
 
-        for application in portfolio.applications:
+    estimated = []
-            if application.name not in [app["name"] for app in fixture_apps]:
+    while rows:
-                fixture_apps.append({"name": application.name, "environments": []})
+        if pendulum.parse(rows[-1][1]) >= pendulum.now(tz="utc").start_of("month"):
+            estimated.append(rows.pop())
+        else:
+            break
 
-        return sorted(
+    return dict(
-            [
+        invoiced=Decimal(sum([row[0] for row in rows])),
-                cls._get_application_monthly_totals(portfolio, fixture_app)
+        estimated=Decimal(sum([row[0] for row in estimated])),
-                for fixture_app in fixture_apps
+    )
-                if fixture_app["name"]
-                in [application.name for application in portfolio.applications]
-            ],
-            key=lambda app: app["name"],
-        )
-
-    @classmethod
-    def _get_environment_monthly_totals(cls, environment):
-        """
-        returns a dictionary that represents spending totals for an environment e.g. 
-        {
-            name
-            this_month
-            last_month
-            total
-        }
-        """
-        return {
-            "name": environment["name"],
-            "this_month": sum(environment["spending"]["this_month"].values()),
-            "last_month": sum(environment["spending"]["last_month"].values()),
-            "total": sum(environment["spending"]["total"].values()),
-        }
-
-    @classmethod
-    def _get_application_monthly_totals(cls, portfolio, fixture_app):
-        """
-        returns a dictionary that represents spending totals for an application 
-        and its environments e.g. 
-            {
-                name
-                this_month
-                last_month
-                total
-                environments: [
-                    {
-                        name
-                        this_month
-                        last_month
-                        total
-                    }
-                ]
-            }
-        """
-        application_envs = [
-            env
-            for env in portfolio.all_environments
-            if env.application.name == fixture_app["name"]
-        ]
-
-        environments = [
-            cls._get_environment_monthly_totals(env)
-            for env in fixture_app["environments"]
-            if env["name"] in [e.name for e in application_envs]
-        ]
-
-        for env in application_envs:
-            if env.name not in [env["name"] for env in environments]:
-                environments.append({"name": env.name})
-
-        return {
-            "name": fixture_app["name"],
-            "this_month": sum(env.get("this_month", 0) for env in environments),
-            "last_month": sum(env.get("last_month", 0) for env in environments),
-            "total": sum(env.get("total", 0) for env in environments),
-            "environments": sorted(environments, key=lambda env: env["name"]),
-        }
-
-    @classmethod
-    def get_spending_by_JEDI_clin(cls, portfolio):
-        """
-        returns an dictionary of spending per JEDI CLIN for a portfolio
-        {
-            jedi_clin: {
-                invoiced
-                estimated
-            },
-        }
-        """
-        if portfolio.name in cls.FIXTURE_SPEND_DATA:
-            CLIN_spend_dict = defaultdict(lambda: defaultdict(Decimal))
-            for application in cls.FIXTURE_SPEND_DATA[portfolio.name]["applications"]:
-                for environment in application["environments"]:
-                    for clin, spend in environment["spending"]["this_month"].items():
-                        CLIN_spend_dict[clin]["estimated"] += Decimal(spend)
-                    for clin, spend in environment["spending"]["total"].items():
-                        CLIN_spend_dict[clin]["invoiced"] += Decimal(spend)
-            return CLIN_spend_dict
-        return {}

atst/domain/reports.py

@@ -1,12 +1,13 @@
 from flask import current_app
-from itertools import groupby
+from atst.domain.csp.cloud.models import (
+    ReportingCSPPayload,
+    CostManagementQueryCSPResult,
+)
+from atst.domain.csp.reports import prepare_azure_reporting_data
+import pendulum
 
 
 class Reports:
-    @classmethod
-    def monthly_spending(cls, portfolio):
-        return current_app.csp.reports.get_portfolio_monthly_spending(portfolio)
-
     @classmethod
     def expired_task_orders(cls, portfolio):
         return [
@@ -14,31 +15,19 @@ class Reports:
         ]
 
     @classmethod
-    def obligated_funds_by_JEDI_clin(cls, portfolio):
+    def get_portfolio_spending(cls, portfolio):
-        clin_spending = current_app.csp.reports.get_spending_by_JEDI_clin(portfolio)
+        # TODO: Extend this function to make from_date and to_date configurable
-        active_clins = portfolio.active_clins
+        from_date = pendulum.now().subtract(years=1).add(days=1).format("YYYY-MM-DD")
-        for jedi_clin, clins in groupby(
+        to_date = pendulum.now().format("YYYY-MM-DD")
-            active_clins, key=lambda clin: clin.jedi_clin_type
+        rows = []
-        ):
-            if not clin_spending.get(jedi_clin.name):
-                clin_spending[jedi_clin.name] = {}
-            clin_spending[jedi_clin.name]["obligated"] = sum(
-                clin.obligated_amount for clin in clins
-            )
 
-        output = []
+        if portfolio.csp_data:
-        for clin in clin_spending.keys():
+            payload = ReportingCSPPayload(
-            invoiced = clin_spending[clin].get("invoiced", 0)
+                from_date=from_date, to_date=to_date, **portfolio.csp_data
-            estimated = clin_spending[clin].get("estimated", 0)
-            obligated = clin_spending[clin].get("obligated", 0)
-            remaining = obligated - (invoiced + estimated)
-            output.append(
-                {
-                    "name": clin,
-                    "invoiced": invoiced,
-                    "estimated": estimated,
-                    "obligated": obligated,
-                    "remaining": remaining,
-                }
             )
-        return output
+            response: CostManagementQueryCSPResult = current_app.csp.cloud.get_reporting_data(
+                payload
+            )
+            rows = response.properties.rows
+
+        return prepare_azure_reporting_data(rows)

atst/filters.py

@@ -5,7 +5,7 @@ from flask import render_template
 from jinja2 import contextfilter
 from jinja2.exceptions import TemplateNotFound
 from urllib.parse import urlparse, urlunparse, parse_qs, urlencode
-from decimal import DivisionByZero as DivisionByZeroException
+from decimal import DivisionByZero as DivisionByZeroException, InvalidOperation
 
 
 def iconSvg(name):
@@ -43,7 +43,7 @@ def obligatedFundingGraphWidth(values):
     numerator, denominator = values
     try:
         return (numerator / denominator) * 100
-    except DivisionByZeroException:
+    except (DivisionByZeroException, InvalidOperation):
         return 0
 
 

atst/models/portfolio.py

@@ -89,6 +89,12 @@ class Portfolio(
     def active_task_orders(self):
         return [task_order for task_order in self.task_orders if task_order.is_active]
 
+    @property
+    def total_obligated_funds(self):
+        return sum(
+            (task_order.total_obligated_funds for task_order in self.active_task_orders)
+        )
+
     @property
     def funding_duration(self):
         """

atst/routes/portfolios/index.py

@@ -34,25 +34,25 @@ def create_portfolio():
 @user_can(Permissions.VIEW_PORTFOLIO_REPORTS, message="view portfolio reports")
 def reports(portfolio_id):
     portfolio = Portfolios.get(g.current_user, portfolio_id)
+    spending = Reports.get_portfolio_spending(portfolio)
+    obligated = portfolio.total_obligated_funds
+    remaining = obligated - (spending["invoiced"] + spending["estimated"])
 
-    current_obligated_funds = Reports.obligated_funds_by_JEDI_clin(portfolio)
+    current_obligated_funds = {
+        **spending,
+        "obligated": obligated,
+        "remaining": remaining,
+    }
 
-    if any(map(lambda clin: clin["remaining"] < 0, current_obligated_funds)):
+    if current_obligated_funds["remaining"] < 0:
         flash("insufficient_funds")
 
-    # wrapped in str() because the sum of obligated funds returns a Decimal object
-    total_portfolio_value = str(
-        sum(
-            task_order.total_obligated_funds
-            for task_order in portfolio.active_task_orders
-        )
-    )
     return render_template(
         "portfolios/reports/index.html",
         portfolio=portfolio,
-        total_portfolio_value=total_portfolio_value,
+        # wrapped in str() because the sum of obligated funds returns a Decimal object
+        total_portfolio_value=str(portfolio.total_obligated_funds),
         current_obligated_funds=current_obligated_funds,
         expired_task_orders=Reports.expired_task_orders(portfolio),
-        monthly_spending=Reports.monthly_spending(portfolio),
         retrieved=datetime.now(),  # mocked datetime of reporting data retrival
     )

templates/portfolios/reports/expired_task_orders.html

@@ -16,13 +16,12 @@
             <th>PoP</th>
             <th>CLIN Value</th>
             <th>Amount Obligated</th>
-            <th>Amount Unspent</th>
           </tr>
         </thead>
         <tbody>
         {% for task_order in expired_task_orders %}
           <tr>
-            <td colspan="5">
+            <td colspan="4">
               <span class="h4 reporting-expended-funding__header">Task Order</span> <a href="{{ url_for("task_orders.view_task_order", task_order_id=task_order.id) }}">
                 {{ task_order.number }} {{ Icon("caret_right", classes="icon--tiny icon--blue" ) }}
               </a>
@@ -39,9 +38,8 @@
                 - 
                 {{ clin.end_date | formattedDate(formatter="%b %d, %Y") }}
               </td>
-              <td>{{ clin.total_amount | dollars }}</td>
+              <td class="table-cell--align-right">{{ clin.total_amount | dollars }}</td>
-              <td>{{ clin.obligated_amount | dollars }}</td>
+              <td class="table-cell--align-right">{{ clin.obligated_amount | dollars }}</td>
-              <td>{{ 0 | dollars }}</td>
             <tr>
           {% endfor %}
         {% endfor %}  

templates/portfolios/reports/index.html

@@ -13,7 +13,5 @@
     <hr>
     {% include "portfolios/reports/obligated_funds.html" %}
     {% include "portfolios/reports/expired_task_orders.html" %}
-    <hr>
-    {% include "portfolios/reports/application_and_env_spending.html" %}
   </div>
 {% endblock %}

templates/portfolios/reports/obligated_funds.html

@@ -7,61 +7,56 @@
   </header>
   <div class='panel'>
     <div class='panel__content jedi-clin-funding'>
-      {% for JEDI_clin in current_obligated_funds | sort(attribute='name')%}
+      <div class="jedi-clin-funding__clin-wrapper">
-        <div class="jedi-clin-funding__clin-wrapper">
+        <h3 class="h5 jedi-clin-funding__header">
-          <h3 class="h5 jedi-clin-funding__header">
+          Total obligated amount: {{ current_obligated_funds.obligated | dollars }}
-            {{ "JEDICLINType.{}".format(JEDI_clin.name) | translate }}
+        </h3>
-          </h3>
+        <div class="jedi-clin-funding__graph">
-          <p class="jedi-clin-funding__subheader">Total obligated amount: {{ JEDI_clin.obligated | dollars }}</p>
+          {% if current_obligated_funds.remaining < 0  %}
-          <div class="jedi-clin-funding__graph">
+            <span style="width:100%" class="jedi-clin-funding__graph-bar jedi-clin-funding__graph-bar--insufficient"></span>
-            {% if JEDI_clin.remaining < 0  %}
+          {% else %}
-              <span style="width:100%" class="jedi-clin-funding__graph-bar jedi-clin-funding__graph-bar--insufficient"></span>
+            {% set invoiced_width = (current_obligated_funds.invoiced, current_obligated_funds.obligated) | obligatedFundingGraphWidth %}
-            {% else %}
+            {% if invoiced_width %}
-              {% set invoiced_width = (JEDI_clin.invoiced, JEDI_clin.obligated) | obligatedFundingGraphWidth %}
+              <span style="width:{{ invoiced_width }}%" 
-              {% if invoiced_width %}
+                    class="jedi-clin-funding__graph-bar jedi-clin-funding__graph-bar--invoiced">
-                <span style="width:{{ invoiced_width }}%" 
-                      class="jedi-clin-funding__graph-bar jedi-clin-funding__graph-bar--invoiced">
-                </span>
-              {% endif %}
-              
-              {% set estimated_width = (JEDI_clin.estimated, JEDI_clin.obligated) | obligatedFundingGraphWidth %}
-              {% if estimated_width %}
-                <span style="width:{{ (JEDI_clin.estimated, JEDI_clin.obligated) | obligatedFundingGraphWidth }}%" 
-                      class="jedi-clin-funding__graph-bar jedi-clin-funding__graph-bar--estimated">
-                </span>
-              {% endif %}
-              <span style="width:{{ (JEDI_clin.remaining, JEDI_clin.obligated) | obligatedFundingGraphWidth }}%" 
-                class="jedi-clin-funding__graph-bar jedi-clin-funding__graph-bar--remaining">
               </span>
             {% endif %}
             
-            
+            {% set estimated_width = (current_obligated_funds.estimated, current_obligated_funds.obligated) | obligatedFundingGraphWidth %}
+            {% if estimated_width %}
+              <span style="width:{{ (current_obligated_funds.estimated, current_obligated_funds.obligated) | obligatedFundingGraphWidth }}%" 
+                    class="jedi-clin-funding__graph-bar jedi-clin-funding__graph-bar--estimated">
+              </span>
+            {% endif %}
+            <span style="width:{{ (current_obligated_funds.remaining, current_obligated_funds.obligated) | obligatedFundingGraphWidth }}%" 
+              class="jedi-clin-funding__graph-bar jedi-clin-funding__graph-bar--remaining">
+            </span>
+          {% endif %}
+        </div>
+        <div class="jedi-clin-funding__graph-values">
+          <div class="jedi-clin-funding__meta">
+            <p class="jedi-clin-funding__meta-header">
+              <span class="jedi-clin-funding__meta-key jedi-clin-funding__meta-key--invoiced"></span>
+              Invoiced expended funds:
+            </p>
+            <p class="h3 jedi-clin-funding__meta-value">{{ current_obligated_funds.invoiced | dollars }}</p>
           </div>
-          <div class="jedi-clin-funding__graph-values">
+          <div class="jedi-clin-funding__meta">
-            <div class="jedi-clin-funding__meta">
+            <p class="jedi-clin-funding__meta-header">
-              <p class="jedi-clin-funding__meta-header">
+              <span class="jedi-clin-funding__meta-key jedi-clin-funding__meta-key--estimated"></span>
-                <span class="jedi-clin-funding__meta-key jedi-clin-funding__meta-key--invoiced"></span>
+              Estimated expended funds:
-                Invoiced expended funds:
+            </p>
-              </p>
+            <p class="h3 jedi-clin-funding__meta-value">{{ current_obligated_funds.estimated | dollars }}</p>
-              <p class="h3 jedi-clin-funding__meta-value">{{ JEDI_clin.invoiced | dollars }}</p>
+          </div>
-            </div>
+          <div class="jedi-clin-funding__meta">
-            <div class="jedi-clin-funding__meta">
+            <p class="jedi-clin-funding__meta-header">
-              <p class="jedi-clin-funding__meta-header">
+              <span class="jedi-clin-funding__meta-key jedi-clin-funding__meta-key--{{"remaining" if current_obligated_funds.remaining > 0 else "insufficient"}}"></span>
-                <span class="jedi-clin-funding__meta-key jedi-clin-funding__meta-key--estimated"></span>
+              Remaining funds:
-                Estimated expended funds:
+            </p>
-              </p>
+            <p class="h3 jedi-clin-funding__meta-value {% if current_obligated_funds.remaining < 0 %}text-danger{% endif %}">{{ current_obligated_funds.remaining | dollars }}</p>
-              <p class="h3 jedi-clin-funding__meta-value">{{ JEDI_clin.estimated | dollars }}</p>
-            </div>
-            <div class="jedi-clin-funding__meta">
-              <p class="jedi-clin-funding__meta-header">
-                <span class="jedi-clin-funding__meta-key jedi-clin-funding__meta-key--{{"remaining" if JEDI_clin.remaining > 0 else "insufficient"}}"></span>
-                Remaining funds:
-              </p>
-              <p class="h3 jedi-clin-funding__meta-value {% if JEDI_clin.remaining < 0 %}text-danger{% endif %}">{{ JEDI_clin.remaining | dollars }}</p>
-            </div>
           </div>
         </div>
-      {% endfor %}
+      </div>
       <div class="jedi-clin-funding__active-task-orders">
         <h3 class="h4">
           Active Task Orders

tests/domain/cloud/reports/test_reports.py

@@ -1,58 +1,47 @@
-from atst.domain.csp.reports import MockReportingProvider
+from atst.domain.csp.reports import prepare_azure_reporting_data
 from tests.factories import PortfolioFactory
+from decimal import Decimal
+import pendulum
 
 
-def test_get_environment_monthly_totals():
+class TestPrepareAzureData:
-    environment = {
+    start_of_month = pendulum.today(tz="utc").start_of("month").replace(tzinfo=None)
-        "name": "Test Environment",
+    next_month = start_of_month.add(months=1).to_atom_string()
-        "spending": {
+    this_month = start_of_month.to_atom_string()
-            "this_month": {"JEDI_CLIN_1": 100, "JEDI_CLIN_2": 100},
+    last_month = start_of_month.subtract(months=1).to_atom_string()
-            "last_month": {"JEDI_CLIN_1": 200, "JEDI_CLIN_2": 200},
+    two_months_ago = last_month = start_of_month.subtract(months=2).to_atom_string()
-            "total": {"JEDI_CLIN_1": 1000, "JEDI_CLIN_2": 1000},
-        },
-    }
-    totals = MockReportingProvider._get_environment_monthly_totals(environment)
-    assert totals == {
-        "name": "Test Environment",
-        "this_month": 200,
-        "last_month": 400,
-        "total": 2000,
-    }
 
+    def test_estimated_and_invoiced(self):
+        rows = [
+            [150.0, self.two_months_ago, "", "USD"],
+            [100.0, self.last_month, "e0500a4qhw", "USD"],
+            [50.0, self.this_month, "", "USD"],
+            [50.0, self.next_month, "", "USD"],
+        ]
+        output = prepare_azure_reporting_data(rows)
 
-def test_get_application_monthly_totals():
+        assert output.get("invoiced") == Decimal(250.0)
-    portfolio = PortfolioFactory.create(
+        assert output.get("estimated") == Decimal(100.0)
-        applications=[
-            {"name": "Test Application", "environments": [{"name": "Z"}, {"name": "A"}]}
-        ],
-    )
-    application = {
-        "name": "Test Application",
-        "environments": [
-            {
-                "name": "Z",
-                "spending": {
-                    "this_month": {"JEDI_CLIN_1": 50, "JEDI_CLIN_2": 50},
-                    "last_month": {"JEDI_CLIN_1": 150, "JEDI_CLIN_2": 150},
-                    "total": {"JEDI_CLIN_1": 250, "JEDI_CLIN_2": 250},
-                },
-            },
-            {
-                "name": "A",
-                "spending": {
-                    "this_month": {"JEDI_CLIN_1": 100, "JEDI_CLIN_2": 100},
-                    "last_month": {"JEDI_CLIN_1": 200, "JEDI_CLIN_2": 200},
-                    "total": {"JEDI_CLIN_1": 1000, "JEDI_CLIN_2": 1000},
-                },
-            },
-        ],
-    }
 
-    totals = MockReportingProvider._get_application_monthly_totals(
+    def test_just_estimated(self):
-        portfolio, application
+        rows = [
-    )
+            [100.0, self.this_month, "", "USD"],
-    assert totals["name"] == "Test Application"
+        ]
-    assert totals["this_month"] == 300
+        output = prepare_azure_reporting_data(rows)
-    assert totals["last_month"] == 700
+
-    assert totals["total"] == 2500
+        assert output.get("invoiced") == Decimal(0.0)
-    assert [env["name"] for env in totals["environments"]] == ["A", "Z"]
+        assert output.get("estimated") == Decimal(100.0)
+
+    def test_just_invoiced(self):
+        rows = [
+            [100.0, self.last_month, "", "USD"],
+        ]
+        output = prepare_azure_reporting_data(rows)
+
+        assert output.get("invoiced") == Decimal(100.0)
+        assert output.get("estimated") == Decimal(0.0)
+
+    def test_no_rows(self):
+        output = prepare_azure_reporting_data([])
+        assert output.get("invoiced") == Decimal(0.0)
+        assert output.get("estimated") == Decimal(0.0)

tests/domain/test_reports.py

@@ -1,8 +1,31 @@
-# TODO: Implement when we get real reporting data
+import pytest
-def test_expired_task_orders():
+
-    pass
+from atst.domain.reports import Reports
+from tests.factories import PortfolioFactory
+from decimal import Decimal
 
 
-# TODO: Implement when we get real reporting data
+@pytest.fixture(scope="function")
-def test_obligated_funds_by_JEDI_clin():
+def portfolio():
-    pass
+    portfolio = PortfolioFactory.create()
+    return portfolio
+
+
+class TestGetPortfolioSpending:
+    csp_data = {
+        "tenant_id": "",
+        "billing_profile_properties": {
+            "invoice_sections": [{"invoice_section_id": "",}]
+        },
+    }
+
+    def test_with_csp_data(self, portfolio):
+        portfolio.csp_data = self.csp_data
+        data = Reports.get_portfolio_spending(portfolio)
+        assert data["invoiced"] == Decimal(1551.0)
+        assert data["estimated"] == Decimal(500.0)
+
+    def test_without_csp_data(self, portfolio):
+        data = Reports.get_portfolio_spending(portfolio)
+        assert data["invoiced"] == Decimal(0)
+        assert data["estimated"] == Decimal(0)