From aa2d353260a409fed8df203a76adb203753bd74a Mon Sep 17 00:00:00 2001 From: leigh-mil Date: Tue, 11 Feb 2020 15:13:17 -0500 Subject: [PATCH 1/5] Add query for finding new CLINs that have not been sent. Use fixtures in tests. --- atst/domain/task_orders.py | 10 +++++++ tests/domain/test_task_orders.py | 48 ++++++++++++++++++++++---------- 2 files changed, 44 insertions(+), 14 deletions(-) diff --git a/atst/domain/task_orders.py b/atst/domain/task_orders.py index 44d6b47e..7a538c2d 100644 --- a/atst/domain/task_orders.py +++ b/atst/domain/task_orders.py @@ -90,3 +90,13 @@ class TaskOrders(BaseDomainClass): ) .all() ) + + @classmethod + def get_clins_for_create_billing_instructions(cls): + return ( + db.session.query(CLIN) + .filter( + CLIN.last_sent_at.is_(None), CLIN.start_date < pendulum.now(tz="UTC") + ) + .all() + ) diff --git a/tests/domain/test_task_orders.py b/tests/domain/test_task_orders.py index 5999677a..4b56b030 100644 --- a/tests/domain/test_task_orders.py +++ b/tests/domain/test_task_orders.py @@ -9,6 +9,27 @@ from atst.models.task_order import TaskOrder, SORT_ORDERING, Status from tests.factories import TaskOrderFactory, CLINFactory, PortfolioFactory +@pytest.fixture +def new_task_order(): + return TaskOrderFactory.create(create_clins=[{}]) + + +@pytest.fixture +def updated_task_order(): + return TaskOrderFactory.create( + create_clins=[{"last_sent_at": pendulum.date(2020, 2, 1)}], + pdf_last_sent_at=pendulum.date(2020, 1, 1), + ) + + +@pytest.fixture +def sent_task_order(): + return TaskOrderFactory.create( + create_clins=[{"last_sent_at": pendulum.date(2020, 1, 1)}], + pdf_last_sent_at=pendulum.date(2020, 1, 1), + ) + + def test_create_adds_clins(): portfolio = PortfolioFactory.create() clins = [ @@ -181,19 +202,18 @@ def test_allows_alphanumeric_number(): assert TaskOrders.create(portfolio.id, number, [], None) -def test_get_for_send_task_order_files(): - new_to = TaskOrderFactory.create(create_clins=[{}]) - updated_to = TaskOrderFactory.create( - create_clins=[{"last_sent_at": pendulum.datetime(2020, 2, 1)}], - pdf_last_sent_at=pendulum.datetime(2020, 1, 1), - ) - sent_to = TaskOrderFactory.create( - create_clins=[{"last_sent_at": pendulum.datetime(2020, 1, 1)}], - pdf_last_sent_at=pendulum.datetime(2020, 1, 1), - ) - +def test_get_for_send_task_order_files( + new_task_order, updated_task_order, sent_task_order +): updated_and_new_task_orders = TaskOrders.get_for_send_task_order_files() assert len(updated_and_new_task_orders) == 2 - assert sent_to not in updated_and_new_task_orders - assert updated_to in updated_and_new_task_orders - assert new_to in updated_and_new_task_orders + assert sent_task_order not in updated_and_new_task_orders + assert updated_task_order in updated_and_new_task_orders + assert new_task_order in updated_and_new_task_orders + + +def test_get_clins_for_create_billing_instructions(new_task_order, sent_task_order): + new_clins = TaskOrders.get_clins_for_create_billing_instructions() + assert len(new_clins) == 1 + assert new_task_order.clins[0] in new_clins + assert sent_task_order.clins[0] not in new_clins From 6ef3265cb5f0e1ba76c51afef50e6c36f93c289f Mon Sep 17 00:00:00 2001 From: leigh-mil Date: Wed, 12 Feb 2020 11:20:21 -0500 Subject: [PATCH 2/5] Create celery task for create_billing_instruction --- atst/domain/csp/cloud/models.py | 8 +++++ atst/jobs.py | 29 +++++++++++++++++ atst/models/clin.py | 6 +++- tests/test_jobs.py | 58 ++++++++++++++++++++++++++++++++- 4 files changed, 99 insertions(+), 2 deletions(-) diff --git a/atst/domain/csp/cloud/models.py b/atst/domain/csp/cloud/models.py index 27f2c9c7..5bb056d7 100644 --- a/atst/domain/csp/cloud/models.py +++ b/atst/domain/csp/cloud/models.py @@ -220,6 +220,14 @@ class BillingInstructionCSPPayload(BaseCSPPayload): billing_account_name: str billing_profile_name: str + class Config: + fields = { + "initial_clin_amount": "obligated_amount", + "initial_clin_start_date": "start_date", + "initial_clin_end_date": "end_date", + "initial_clin_type": "number", + } + class BillingInstructionCSPResult(AliasModel): reported_clin_name: str diff --git a/atst/jobs.py b/atst/jobs.py index c125bcb5..e5805875 100644 --- a/atst/jobs.py +++ b/atst/jobs.py @@ -10,6 +10,7 @@ from atst.domain.csp.cloud import CloudProviderInterface from atst.domain.csp.cloud.exceptions import GeneralCSPException from atst.domain.csp.cloud.models import ( ApplicationCSPPayload, + BillingInstructionCSPPayload, EnvironmentCSPPayload, UserCSPPayload, UserRoleCSPPayload, @@ -317,3 +318,31 @@ def send_task_order_files(self): db.session.add(task_order) db.session.commit() + + +@celery.task(bind=True) +def create_billing_instruction(self): + clins = TaskOrders.get_clins_for_create_billing_instructions() + for clin in clins: + portfolio = clin.task_order.portfolio + clin_data = clin.to_dictionary() + portfolio_data = portfolio.to_dictionary() + + payload = BillingInstructionCSPPayload( + tenant_id=portfolio.csp_data.get("tenant_id"), + billing_account_name=portfolio.csp_data.get("billing_account_name"), + billing_profile_name=portfolio.csp_data.get("billing_profile_name"), + **clin_data, + **portfolio_data, + ) + + try: + app.csp.cloud.create_billing_instruction(payload) + except (AzureError) as err: + app.logger.exception(err) + continue + + clin.last_sent_at = pendulum.now(tz="UTC") + db.session.add(clin) + + db.session.commit() diff --git a/atst/models/clin.py b/atst/models/clin.py index accab107..440ee0a0 100644 --- a/atst/models/clin.py +++ b/atst/models/clin.py @@ -66,11 +66,15 @@ class CLIN(Base, mixins.TimestampsMixin): ) def to_dictionary(self): - return { + data = { c.name: getattr(self, c.name) for c in self.__table__.columns if c.name not in ["id"] } + data["start_date"] = str(data["start_date"]) + data["end_date"] = str(data["end_date"]) + + return data @property def is_active(self): diff --git a/tests/test_jobs.py b/tests/test_jobs.py index ff681c7e..a7c6aced 100644 --- a/tests/test_jobs.py +++ b/tests/test_jobs.py @@ -6,7 +6,8 @@ from smtplib import SMTPException from azure.core.exceptions import AzureError from atst.domain.csp.cloud import MockCloudProvider -from atst.domain.csp.cloud.models import UserRoleCSPResult +from atst.domain.csp.cloud.models import BillingInstructionCSPPayload, UserRoleCSPResult +from atst.domain.portfolios import Portfolios from atst.models import ApplicationRoleStatus, Portfolio, FSMStates from atst.jobs import ( @@ -16,6 +17,7 @@ from atst.jobs import ( dispatch_create_user, dispatch_create_environment_role, dispatch_provision_portfolio, + create_billing_instruction, create_environment, do_create_user, do_provision_portfolio, @@ -489,3 +491,57 @@ class TestSendTaskOrderFiles: # Check that pdf_last_sent_at has not been updated assert not task_order.pdf_last_sent_at + + +class TestCreateBillingInstructions: + def test_update_clin_last_sent_at(self, session): + # create portfolio with one active clin + start_date = pendulum.now().subtract(days=1) + portfolio = PortfolioFactory.create( + csp_data={ + "tenant_id": str(uuid4()), + "billing_account_name": "fake", + "billing_profile_name": "fake", + }, + task_orders=[{"create_clins": [{"start_date": start_date}]}], + ) + unsent_clin = portfolio.task_orders[0].clins[0] + + assert not unsent_clin.last_sent_at + + # The session needs to be nested to prevent detached SQLAlchemy instance + session.begin_nested() + create_billing_instruction() + session.rollback() + + # check that last_sent_at has been updated + assert unsent_clin.last_sent_at + + def test_failure(self, monkeypatch, session): + def _create_billing_instruction(MockCloudProvider, object_name): + raise AzureError("something went wrong") + + monkeypatch.setattr( + "atst.domain.csp.cloud.MockCloudProvider.create_billing_instruction", + _create_billing_instruction, + ) + + # create portfolio with one active clin + start_date = pendulum.now().subtract(days=1) + portfolio = PortfolioFactory.create( + csp_data={ + "tenant_id": str(uuid4()), + "billing_account_name": "fake", + "billing_profile_name": "fake", + }, + task_orders=[{"create_clins": [{"start_date": start_date}]}], + ) + unsent_clin = portfolio.task_orders[0].clins[0] + + # The session needs to be nested to prevent detached SQLAlchemy instance + session.begin_nested() + create_billing_instruction() + session.rollback() + + # check that last_sent_at has not been updated + assert not unsent_clin.last_sent_at From 4dc5f2aa9130748b4b9fca0b46c86a91fe32c606 Mon Sep 17 00:00:00 2001 From: leigh-mil Date: Wed, 12 Feb 2020 12:05:16 -0500 Subject: [PATCH 3/5] Add beat task to queue --- atst/queue.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/atst/queue.py b/atst/queue.py index 5ed2f136..dcd123d1 100644 --- a/atst/queue.py +++ b/atst/queue.py @@ -31,6 +31,10 @@ def update_celery(celery, app): "task": "atst.jobs.send_task_order_files", "schedule": 60, }, + "beat-create_billing_instruction": { + "task": "atst.jobs.create_billing_instruction", + "schedule": 60, + }, } class ContextTask(celery.Task): From 5c7dfc428ea3ea50438d3d75d93f5725f7395e11 Mon Sep 17 00:00:00 2001 From: leigh-mil Date: Tue, 18 Feb 2020 15:30:19 -0500 Subject: [PATCH 4/5] Add new tests and refactor existing tests --- tests/test_jobs.py | 47 +++++++++++++++++++++++++++++++++------------- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/tests/test_jobs.py b/tests/test_jobs.py index a7c6aced..8031d2aa 100644 --- a/tests/test_jobs.py +++ b/tests/test_jobs.py @@ -30,6 +30,7 @@ from atst.jobs import ( from tests.factories import ( ApplicationFactory, ApplicationRoleFactory, + CLINFactory, EnvironmentFactory, EnvironmentRoleFactory, PortfolioFactory, @@ -494,8 +495,8 @@ class TestSendTaskOrderFiles: class TestCreateBillingInstructions: - def test_update_clin_last_sent_at(self, session): - # create portfolio with one active clin + @pytest.fixture + def unsent_clin(self): start_date = pendulum.now().subtract(days=1) portfolio = PortfolioFactory.create( csp_data={ @@ -505,19 +506,20 @@ class TestCreateBillingInstructions: }, task_orders=[{"create_clins": [{"start_date": start_date}]}], ) - unsent_clin = portfolio.task_orders[0].clins[0] + return portfolio.task_orders[0].clins[0] + def test_update_clin_last_sent_at(self, session, unsent_clin): assert not unsent_clin.last_sent_at # The session needs to be nested to prevent detached SQLAlchemy instance session.begin_nested() create_billing_instruction() - session.rollback() # check that last_sent_at has been updated assert unsent_clin.last_sent_at + session.rollback() - def test_failure(self, monkeypatch, session): + def test_failure(self, monkeypatch, session, unsent_clin): def _create_billing_instruction(MockCloudProvider, object_name): raise AzureError("something went wrong") @@ -526,22 +528,41 @@ class TestCreateBillingInstructions: _create_billing_instruction, ) - # create portfolio with one active clin - start_date = pendulum.now().subtract(days=1) + # The session needs to be nested to prevent detached SQLAlchemy instance + session.begin_nested() + create_billing_instruction() + + # check that last_sent_at has not been updated + assert not unsent_clin.last_sent_at + session.rollback() + + def test_task_order_with_multiple_clins(self, session): + start_date = pendulum.now(tz="UTC").subtract(days=1) portfolio = PortfolioFactory.create( csp_data={ "tenant_id": str(uuid4()), "billing_account_name": "fake", "billing_profile_name": "fake", }, - task_orders=[{"create_clins": [{"start_date": start_date}]}], + task_orders=[ + { + "create_clins": [ + {"start_date": start_date, "last_sent_at": start_date} + ] + } + ], ) - unsent_clin = portfolio.task_orders[0].clins[0] + task_order = portfolio.task_orders[0] + sent_clin = task_order.clins[0] + + # Add new CLIN to the Task Order + new_clin = CLINFactory.create(task_order=task_order) + assert not new_clin.last_sent_at - # The session needs to be nested to prevent detached SQLAlchemy instance session.begin_nested() create_billing_instruction() - session.rollback() - # check that last_sent_at has not been updated - assert not unsent_clin.last_sent_at + # check that last_sent_at has been update for the new clin only + assert new_clin.last_sent_at + assert sent_clin.last_sent_at != new_clin.last_sent_at + session.rollback() From 0bda0c481e64f8050b78a37ee4862f70fc21e7ff Mon Sep 17 00:00:00 2001 From: leigh-mil Date: Wed, 19 Feb 2020 11:10:20 -0500 Subject: [PATCH 5/5] Explicitly pass in kwargs instead of splatting clin and portfolio data --- atst/domain/csp/cloud/models.py | 8 -------- atst/jobs.py | 9 +++++---- atst/models/clin.py | 2 -- tests/test_jobs.py | 1 + 4 files changed, 6 insertions(+), 14 deletions(-) diff --git a/atst/domain/csp/cloud/models.py b/atst/domain/csp/cloud/models.py index 5bb056d7..27f2c9c7 100644 --- a/atst/domain/csp/cloud/models.py +++ b/atst/domain/csp/cloud/models.py @@ -220,14 +220,6 @@ class BillingInstructionCSPPayload(BaseCSPPayload): billing_account_name: str billing_profile_name: str - class Config: - fields = { - "initial_clin_amount": "obligated_amount", - "initial_clin_start_date": "start_date", - "initial_clin_end_date": "end_date", - "initial_clin_type": "number", - } - class BillingInstructionCSPResult(AliasModel): reported_clin_name: str diff --git a/atst/jobs.py b/atst/jobs.py index e5805875..855af5b2 100644 --- a/atst/jobs.py +++ b/atst/jobs.py @@ -325,15 +325,16 @@ def create_billing_instruction(self): clins = TaskOrders.get_clins_for_create_billing_instructions() for clin in clins: portfolio = clin.task_order.portfolio - clin_data = clin.to_dictionary() - portfolio_data = portfolio.to_dictionary() payload = BillingInstructionCSPPayload( tenant_id=portfolio.csp_data.get("tenant_id"), billing_account_name=portfolio.csp_data.get("billing_account_name"), billing_profile_name=portfolio.csp_data.get("billing_profile_name"), - **clin_data, - **portfolio_data, + initial_clin_amount=clin.obligated_amount, + initial_clin_start_date=str(clin.start_date), + initial_clin_end_date=str(clin.end_date), + initial_clin_type=clin.number, + initial_task_order_id=str(clin.task_order_id), ) try: diff --git a/atst/models/clin.py b/atst/models/clin.py index 440ee0a0..a96e907f 100644 --- a/atst/models/clin.py +++ b/atst/models/clin.py @@ -71,8 +71,6 @@ class CLIN(Base, mixins.TimestampsMixin): for c in self.__table__.columns if c.name not in ["id"] } - data["start_date"] = str(data["start_date"]) - data["end_date"] = str(data["end_date"]) return data diff --git a/tests/test_jobs.py b/tests/test_jobs.py index 8031d2aa..eba01f9b 100644 --- a/tests/test_jobs.py +++ b/tests/test_jobs.py @@ -561,6 +561,7 @@ class TestCreateBillingInstructions: session.begin_nested() create_billing_instruction() + session.add(sent_clin) # check that last_sent_at has been update for the new clin only assert new_clin.last_sent_at