From e9bf806dc6d7bf8ea302a1c45b5495be1a3aa717 Mon Sep 17 00:00:00 2001 From: richard-dds Date: Thu, 5 Sep 2019 12:49:15 -0400 Subject: [PATCH 01/14] Environment provisioning celery tasks Failing test Break env provisioning task into 3 separate tasks Make env creation task idempotent Test other env provisioning tasks DRY tasks --- .../502e79c55d2d_add_environment_csp_info.py | 30 +++++++ atst/jobs.py | 81 ++++++++++++++++++- atst/models/environment.py | 3 + tests/test_jobs.py | 62 +++++++++++++- 4 files changed, 174 insertions(+), 2 deletions(-) create mode 100644 alembic/versions/502e79c55d2d_add_environment_csp_info.py diff --git a/alembic/versions/502e79c55d2d_add_environment_csp_info.py b/alembic/versions/502e79c55d2d_add_environment_csp_info.py new file mode 100644 index 00000000..3804e3a9 --- /dev/null +++ b/alembic/versions/502e79c55d2d_add_environment_csp_info.py @@ -0,0 +1,30 @@ +"""add Environment csp info + +Revision ID: 502e79c55d2d +Revises: 4a3122ffe898 +Create Date: 2019-09-05 13:54:23.840512 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = '502e79c55d2d' +down_revision = '4a3122ffe898' # pragma: allowlist secret +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('environments', sa.Column('baseline_info', postgresql.JSONB(astext_type=sa.Text()), nullable=True)) + op.add_column('environments', sa.Column('root_user_info', postgresql.JSONB(astext_type=sa.Text()), nullable=True)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('environments', 'root_user_info') + op.drop_column('environments', 'baseline_info') + # ### end Alembic commands ### diff --git a/atst/jobs.py b/atst/jobs.py index 2d4440b1..ceea19db 100644 --- a/atst/jobs.py +++ b/atst/jobs.py @@ -1,8 +1,11 @@ from flask import current_app as app -from atst.queue import celery from atst.database import db +from atst.queue import celery from atst.models import EnvironmentJobFailure, EnvironmentRoleJobFailure +from atst.domain.csp.cloud import CloudProviderInterface, GeneralCSPException +from atst.domain.environments import Environments +from atst.domain.users import Users class RecordEnvironmentFailure(celery.Task): @@ -38,3 +41,79 @@ def send_notification_mail(recipients, subject, body): ) ) app.mailer.send(recipients, subject, body) + + +def do_create_environment( + csp: CloudProviderInterface, environment_id=None, atat_user_id=None +): + environment = Environments.get(environment_id) + + if environment.cloud_id is not None: + # TODO: Return value for this? + return + + user = Users.get(atat_user_id) + + # we'll need to do some checking in this job for cases where it's retrying + # when a failure occured after some successful steps + # (e.g. if environment.cloud_id is not None, then we can skip first step) + + # credentials either from a given user or pulled from config? + # if using global creds, do we need to log what user authorized action? + atat_root_creds = csp.root_creds() + + # user is needed because baseline root account in the environment will + # be assigned to the requesting user, open question how to handle duplicate + # email addresses across new environments + csp_environment_id = csp.create_environment(atat_root_creds, user, environment) + environment.cloud_id = csp_environment_id + db.session.add(environment) + db.session.commit() + + +def do_create_atat_admin_user(csp: CloudProviderInterface, environment_id=None): + environment = Environments.get(environment_id) + atat_root_creds = csp.root_creds() + + atat_remote_root_user = csp.create_atat_admin_user( + atat_root_creds, environment.cloud_id + ) + environment.root_user_info = atat_remote_root_user + db.session.add(environment) + db.session.commit() + + +def do_create_environment_baseline(csp: CloudProviderInterface, environment_id=None): + environment = Environments.get(environment_id) + + # ASAP switch to use remote root user for provisioning + atat_remote_root_creds = environment.root_user_info["credentials"] + + baseline_info = csp.create_environment_baseline( + atat_remote_root_creds, environment.cloud_id + ) + environment.baseline_info = baseline_info + db.session.add(environment) + db.session.commit() + + +def do_work(fn, task, csp, **kwargs): + try: + fn(csp, **kwargs) + except GeneralCSPException as e: + raise task.retry(exc=e) + + +@celery.task(bind=True) +def create_environment(self, environment_id=None, atat_user_id=None): + do_work(do_create_environment, self, app.csp.cloud, **kwargs) + + +@celery.task(bind=True) +def create_atat_admin_user(self, environment_id=None): + do_work(do_create_atat_admin_user, self, app.csp.cloud, **kwargs) + + +@celery.task(bind=True) +def create_environment_baseline(self, environment_id=None): + do_work(do_create_environment_baseline, self, app.csp.cloud, **kwargs) diff --git a/atst/models/environment.py b/atst/models/environment.py index 1c85484a..d85550c6 100644 --- a/atst/models/environment.py +++ b/atst/models/environment.py @@ -1,5 +1,6 @@ from sqlalchemy import Column, ForeignKey, String from sqlalchemy.orm import relationship +from sqlalchemy.dialects.postgresql import JSONB from atst.models import Base from atst.models.types import Id @@ -18,6 +19,8 @@ class Environment( application = relationship("Application") cloud_id = Column(String) + root_user_info = Column(JSONB) + baseline_info = Column(JSONB) job_failures = relationship("EnvironmentJobFailure") diff --git a/tests/test_jobs.py b/tests/test_jobs.py index 4ad5220c..e1ef988f 100644 --- a/tests/test_jobs.py +++ b/tests/test_jobs.py @@ -1,8 +1,16 @@ import pytest from atst.jobs import RecordEnvironmentFailure, RecordEnvironmentRoleFailure +from tests.factories import EnvironmentFactory, EnvironmentRoleFactory, UserFactory +from uuid import uuid4 +from unittest.mock import Mock -from tests.factories import EnvironmentFactory, EnvironmentRoleFactory +from atst.jobs import ( + do_create_environment, + do_create_atat_admin_user, + do_create_environment_baseline, +) +from atst.models import Environment def test_environment_job_failure(celery_app, celery_worker): @@ -39,3 +47,55 @@ def test_environment_role_job_failure(celery_app, celery_worker): assert role.job_failures job_failure = role.job_failures[0] assert job_failure.task == task +from tests.factories import EnvironmentFactory, UserFactory +from atst.domain.csp.cloud import MockCloudProvider + + +@pytest.fixture(autouse=True, scope="function") +def csp(): + return Mock(wraps=MockCloudProvider({}, with_delay=False, with_failure=False)) + + +def test_create_environment_job(session, csp): + user = UserFactory.create() + environment = EnvironmentFactory.create() + do_create_environment(csp, environment.id, user.id) + + environment_id = environment.id + del environment + + updated_environment = session.query(Environment).get(environment_id) + + assert updated_environment.cloud_id + + +def test_create_environment_job_is_idempotent(csp, session): + user = UserFactory.create() + environment = EnvironmentFactory.create(cloud_id=uuid4().hex) + do_create_environment(csp, environment.id, user.id) + + csp.create_environment.assert_not_called() + + +def test_create_atat_admin_user(csp, session): + environment = EnvironmentFactory.create(cloud_id="something") + do_create_atat_admin_user(csp, environment.id) + + environment_id = environment.id + del environment + updated_environment = session.query(Environment).get(environment_id) + + assert updated_environment.root_user_info + + +def test_create_environment_baseline(csp, session): + environment = EnvironmentFactory.create( + root_user_info={"credentials": csp.root_creds()} + ) + do_create_environment_baseline(csp, environment.id) + + environment_id = environment.id + del environment + updated_environment = session.query(Environment).get(environment_id) + + assert updated_environment.baseline_info From 4a5ca1cd27a55ffa719fb6d668f395885cf7d882 Mon Sep 17 00:00:00 2001 From: richard-dds Date: Fri, 6 Sep 2019 13:39:32 -0400 Subject: [PATCH 02/14] Add query for environments pending creation --- atst/jobs.py | 21 ++++++++++++++++++ tests/test_jobs.py | 55 ++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 72 insertions(+), 4 deletions(-) diff --git a/atst/jobs.py b/atst/jobs.py index ceea19db..8977cd7e 100644 --- a/atst/jobs.py +++ b/atst/jobs.py @@ -6,6 +6,7 @@ from atst.models import EnvironmentJobFailure, EnvironmentRoleJobFailure from atst.domain.csp.cloud import CloudProviderInterface, GeneralCSPException from atst.domain.environments import Environments from atst.domain.users import Users +from atst.models import Application, Portfolio, TaskOrder, CLIN, Environment class RecordEnvironmentFailure(celery.Task): @@ -117,3 +118,23 @@ def create_atat_admin_user(self, environment_id=None): @celery.task(bind=True) def create_environment_baseline(self, environment_id=None): do_work(do_create_environment_baseline, self, app.csp.cloud, **kwargs) + + +def environments_to_create(now): + query = ( + db.session.query(Environment.id) + .join(Application) + .join(Portfolio) + .join(TaskOrder) + .join(CLIN) + .filter(CLIN.start_date <= now) + .filter(CLIN.end_date > now) + .filter(Environment.cloud_id == None) + ) + return [environment_id for (environment_id,) in query.all()] + + +@celery.task(bind=True) +def dispatch_create_environment(self): + for environment_id in environments_to_create(pendulum.now()): + create_environment.delay(environment_id=environment_id, atat_user_id="TODO") diff --git a/tests/test_jobs.py b/tests/test_jobs.py index e1ef988f..8a2bf19d 100644 --- a/tests/test_jobs.py +++ b/tests/test_jobs.py @@ -1,18 +1,26 @@ +import pendulum import pytest from atst.jobs import RecordEnvironmentFailure, RecordEnvironmentRoleFailure from tests.factories import EnvironmentFactory, EnvironmentRoleFactory, UserFactory from uuid import uuid4 from unittest.mock import Mock - +from tests.factories import ( + UserFactory, + PortfolioFactory, + CLINFactory, + TaskOrderFactory, + EnvironmentFactory +) +from atst.domain.csp.cloud import MockCloudProvider from atst.jobs import ( do_create_environment, do_create_atat_admin_user, do_create_environment_baseline, + environments_to_create, ) from atst.models import Environment - def test_environment_job_failure(celery_app, celery_worker): @celery_app.task(bind=True, base=RecordEnvironmentFailure) def _fail_hard(self, environment_id=None): @@ -47,8 +55,10 @@ def test_environment_role_job_failure(celery_app, celery_worker): assert role.job_failures job_failure = role.job_failures[0] assert job_failure.task == task -from tests.factories import EnvironmentFactory, UserFactory -from atst.domain.csp.cloud import MockCloudProvider + +now = pendulum.now() +yesterday = now.subtract(days=1) +tomorrow = now.add(days=1) @pytest.fixture(autouse=True, scope="function") @@ -99,3 +109,40 @@ def test_create_environment_baseline(csp, session): updated_environment = session.query(Environment).get(environment_id) assert updated_environment.baseline_info + + +def create_portfolio_with_clins(start_and_end_dates): + return PortfolioFactory.create( + applications=[ + { + "name": "Mos Eisley", + "description": "Where Han shot first", + "environments": [{"name": "thebar"}], + } + ], + task_orders=[ + TaskOrderFactory.create( + clins=[ + CLINFactory.create(start_date=start_date, end_date=end_date) + for (start_date, end_date) in start_and_end_dates + ] + ) + ], + ) + + +def test_dispatch_query_with_expired_clins(session): + create_portfolio_with_clins([(yesterday, yesterday)]) + assert len(environments_to_create(pendulum.now())) == 0 + + +def test_dispatch_query_with_active_clins(session): + portfolio = create_portfolio_with_clins([(yesterday, tomorrow)]) + environments_to_create(pendulum.now()) == [ + portfolio.applications[0].environments[0].id + ] + + +def test_dispatch_query_with_future_clins(session): + create_portfolio_with_clins([(tomorrow, tomorrow)]) + assert len(environments_to_create(pendulum.now())) == 0 From e3ea2d2fe51b095672547104ae342d1ef75b71c9 Mon Sep 17 00:00:00 2001 From: richard-dds Date: Mon, 9 Sep 2019 11:39:09 -0400 Subject: [PATCH 03/14] Move "envs pending creation" query to Environments domain --- atst/domain/environments.py | 16 +++++++++++- atst/jobs.py | 18 +++---------- tests/domain/test_environments.py | 43 ++++++++++++++++++++++++++++++- tests/test_jobs.py | 40 +++------------------------- 4 files changed, 63 insertions(+), 54 deletions(-) diff --git a/atst/domain/environments.py b/atst/domain/environments.py index e21d4cb9..10ab15f9 100644 --- a/atst/domain/environments.py +++ b/atst/domain/environments.py @@ -1,7 +1,7 @@ from sqlalchemy.orm.exc import NoResultFound from atst.database import db -from atst.models.environment import Environment +from atst.models import Environment, Application, Portfolio, TaskOrder, CLIN from atst.domain.environment_roles import EnvironmentRoles from atst.domain.application_roles import ApplicationRoles @@ -102,3 +102,17 @@ class Environments(object): # TODO: How do we work around environment deletion being a largely manual process in the CSPs return environment + + @classmethod + def get_environments_pending_creation(cls, now) -> [str]: + query = ( + db.session.query(Environment.id) + .join(Application) + .join(Portfolio) + .join(TaskOrder) + .join(CLIN) + .filter(CLIN.start_date <= now) + .filter(CLIN.end_date > now) + .filter(Environment.cloud_id == None) + ) + return [environment_id for (environment_id,) in query.all()] diff --git a/atst/jobs.py b/atst/jobs.py index 8977cd7e..6e983a6c 100644 --- a/atst/jobs.py +++ b/atst/jobs.py @@ -120,21 +120,9 @@ def create_environment_baseline(self, environment_id=None): do_work(do_create_environment_baseline, self, app.csp.cloud, **kwargs) -def environments_to_create(now): - query = ( - db.session.query(Environment.id) - .join(Application) - .join(Portfolio) - .join(TaskOrder) - .join(CLIN) - .filter(CLIN.start_date <= now) - .filter(CLIN.end_date > now) - .filter(Environment.cloud_id == None) - ) - return [environment_id for (environment_id,) in query.all()] - - @celery.task(bind=True) def dispatch_create_environment(self): - for environment_id in environments_to_create(pendulum.now()): + for environment_id in Environments.get_environments_pending_creation( + pendulum.now() + ): create_environment.delay(environment_id=environment_id, atat_user_id="TODO") diff --git a/tests/domain/test_environments.py b/tests/domain/test_environments.py index 4589b179..58ea011b 100644 --- a/tests/domain/test_environments.py +++ b/tests/domain/test_environments.py @@ -1,4 +1,5 @@ import pytest +import pendulum from atst.domain.environments import Environments from atst.domain.environment_roles import EnvironmentRoles @@ -7,11 +8,12 @@ from atst.models.environment_role import CSPRole from tests.factories import ( ApplicationFactory, - UserFactory, PortfolioFactory, EnvironmentFactory, EnvironmentRoleFactory, ApplicationRoleFactory, + TaskOrderFactory, + CLINFactory, ) @@ -132,3 +134,42 @@ def test_update_environment(): assert environment.name is not "name 2" Environments.update(environment, name="name 2") assert environment.name == "name 2" + + +class TestGetEnvironmentsPendingCreate: + NOW = pendulum.now() + YESTERDAY = NOW.subtract(days=1) + TOMORROW = NOW.add(days=1) + + def create_portfolio_with_clins(self, start_and_end_dates): + return PortfolioFactory.create( + applications=[ + { + "name": "Mos Eisley", + "description": "Where Han shot first", + "environments": [{"name": "thebar"}], + } + ], + task_orders=[ + TaskOrderFactory.create( + clins=[ + CLINFactory.create(start_date=start_date, end_date=end_date) + for (start_date, end_date) in start_and_end_dates + ] + ) + ], + ) + + def test_with_expired_clins(self, session): + self.create_portfolio_with_clins([(self.YESTERDAY, self.YESTERDAY)]) + assert len(Environments.get_environments_pending_creation(self.NOW)) == 0 + + def test_with_active_clins(self, session): + portfolio = self.create_portfolio_with_clins([(self.YESTERDAY, self.TOMORROW)]) + Environments.get_environments_pending_creation(self.NOW) == [ + portfolio.applications[0].environments[0].id + ] + + def test_with_future_clins(self, session): + self.create_portfolio_with_clins([(self.TOMORROW, self.TOMORROW)]) + assert len(Environments.get_environments_pending_creation(self.NOW)) == 0 diff --git a/tests/test_jobs.py b/tests/test_jobs.py index 8a2bf19d..409663d1 100644 --- a/tests/test_jobs.py +++ b/tests/test_jobs.py @@ -5,6 +5,8 @@ from atst.jobs import RecordEnvironmentFailure, RecordEnvironmentRoleFailure from tests.factories import EnvironmentFactory, EnvironmentRoleFactory, UserFactory from uuid import uuid4 from unittest.mock import Mock + +from atst.models import Environment from tests.factories import ( UserFactory, PortfolioFactory, @@ -59,6 +61,7 @@ def test_environment_role_job_failure(celery_app, celery_worker): now = pendulum.now() yesterday = now.subtract(days=1) tomorrow = now.add(days=1) +from atst.domain.environments import Environments @pytest.fixture(autouse=True, scope="function") @@ -109,40 +112,3 @@ def test_create_environment_baseline(csp, session): updated_environment = session.query(Environment).get(environment_id) assert updated_environment.baseline_info - - -def create_portfolio_with_clins(start_and_end_dates): - return PortfolioFactory.create( - applications=[ - { - "name": "Mos Eisley", - "description": "Where Han shot first", - "environments": [{"name": "thebar"}], - } - ], - task_orders=[ - TaskOrderFactory.create( - clins=[ - CLINFactory.create(start_date=start_date, end_date=end_date) - for (start_date, end_date) in start_and_end_dates - ] - ) - ], - ) - - -def test_dispatch_query_with_expired_clins(session): - create_portfolio_with_clins([(yesterday, yesterday)]) - assert len(environments_to_create(pendulum.now())) == 0 - - -def test_dispatch_query_with_active_clins(session): - portfolio = create_portfolio_with_clins([(yesterday, tomorrow)]) - environments_to_create(pendulum.now()) == [ - portfolio.applications[0].environments[0].id - ] - - -def test_dispatch_query_with_future_clins(session): - create_portfolio_with_clins([(tomorrow, tomorrow)]) - assert len(environments_to_create(pendulum.now())) == 0 From 13861ad998a403dcc8ec89e0c240dcc9467cce5c Mon Sep 17 00:00:00 2001 From: richard-dds Date: Mon, 9 Sep 2019 16:00:08 -0400 Subject: [PATCH 04/14] Add query for getting environments pending atat user creation --- atst/domain/environments.py | 19 +++++++++-- atst/jobs.py | 8 +++++ tests/domain/test_environments.py | 53 +++++++++++++++++++++++++++---- 3 files changed, 71 insertions(+), 9 deletions(-) diff --git a/atst/domain/environments.py b/atst/domain/environments.py index 10ab15f9..2d3bd4a8 100644 --- a/atst/domain/environments.py +++ b/atst/domain/environments.py @@ -1,3 +1,4 @@ +from sqlalchemy import text from sqlalchemy.orm.exc import NoResultFound from atst.database import db @@ -104,8 +105,8 @@ class Environments(object): return environment @classmethod - def get_environments_pending_creation(cls, now) -> [str]: - query = ( + def base_provision_query(cls, now): + return ( db.session.query(Environment.id) .join(Application) .join(Portfolio) @@ -113,6 +114,18 @@ class Environments(object): .join(CLIN) .filter(CLIN.start_date <= now) .filter(CLIN.end_date > now) - .filter(Environment.cloud_id == None) + ) + + @classmethod + def get_environments_pending_creation(cls, now) -> [str]: + query = cls.base_provision_query(now).filter(Environment.cloud_id == None) + return [environment_id for (environment_id,) in query.all()] + + @classmethod + def get_environments_pending_atat_user_creation(cls, now) -> [str]: + query = ( + cls.base_provision_query(now) + .filter(Environment.cloud_id != None) + .filter(Environment.root_user_info == text("'null'")) ) return [environment_id for (environment_id,) in query.all()] diff --git a/atst/jobs.py b/atst/jobs.py index 6e983a6c..50a06588 100644 --- a/atst/jobs.py +++ b/atst/jobs.py @@ -126,3 +126,11 @@ def dispatch_create_environment(self): pendulum.now() ): create_environment.delay(environment_id=environment_id, atat_user_id="TODO") + + +@celery.task(bind=True) +def dispatch_create_atat_admin_user(self): + for environment_id in Environments.get_environments_pending_atat_user_creation( + pendulum.now() + ): + create_atat_admin_user.delay(environment_id=environment_id, atat_user_id="TODO") diff --git a/tests/domain/test_environments.py b/tests/domain/test_environments.py index 58ea011b..e7049d37 100644 --- a/tests/domain/test_environments.py +++ b/tests/domain/test_environments.py @@ -1,5 +1,6 @@ import pytest import pendulum +from uuid import uuid4 from atst.domain.environments import Environments from atst.domain.environment_roles import EnvironmentRoles @@ -136,18 +137,27 @@ def test_update_environment(): assert environment.name == "name 2" -class TestGetEnvironmentsPendingCreate: - NOW = pendulum.now() - YESTERDAY = NOW.subtract(days=1) - TOMORROW = NOW.add(days=1) +class EnvQueryTest: + @property + def NOW(self): + return pendulum.now() - def create_portfolio_with_clins(self, start_and_end_dates): + @property + def YESTERDAY(self): + return self.NOW.subtract(days=1) + + @property + def TOMORROW(self): + return self.NOW.add(days=1) + + def create_portfolio_with_clins(self, start_and_end_dates, env_data=None): + env_data = env_data or {} return PortfolioFactory.create( applications=[ { "name": "Mos Eisley", "description": "Where Han shot first", - "environments": [{"name": "thebar"}], + "environments": [{"name": "thebar", **env_data}], } ], task_orders=[ @@ -160,6 +170,8 @@ class TestGetEnvironmentsPendingCreate: ], ) + +class TestGetEnvironmentsPendingCreate(EnvQueryTest): def test_with_expired_clins(self, session): self.create_portfolio_with_clins([(self.YESTERDAY, self.YESTERDAY)]) assert len(Environments.get_environments_pending_creation(self.NOW)) == 0 @@ -173,3 +185,32 @@ class TestGetEnvironmentsPendingCreate: def test_with_future_clins(self, session): self.create_portfolio_with_clins([(self.TOMORROW, self.TOMORROW)]) assert len(Environments.get_environments_pending_creation(self.NOW)) == 0 + + +class TestGetEnvironmentsPendingAtatUserCreation(EnvQueryTest): + def test_with_provisioned_environment(self): + self.create_portfolio_with_clins( + [(self.YESTERDAY, self.TOMORROW)], + {"cloud_id": uuid4().hex, "root_user_info": {}}, + ) + assert ( + len(Environments.get_environments_pending_atat_user_creation(self.NOW)) == 0 + ) + + def test_with_unprovisioned_environment(self): + self.create_portfolio_with_clins( + [(self.YESTERDAY, self.TOMORROW)], + {"cloud_id": uuid4().hex, "root_user_info": None}, + ) + assert ( + len(Environments.get_environments_pending_atat_user_creation(self.NOW)) == 1 + ) + + def test_with_unprovisioned_expired_clins_environment(self): + self.create_portfolio_with_clins( + [(self.YESTERDAY, self.YESTERDAY)], + {"cloud_id": uuid4().hex, "root_user_info": None}, + ) + assert ( + len(Environments.get_environments_pending_atat_user_creation(self.NOW)) == 0 + ) From c00f13de2c4c9a578e4596c49b32c83940534a22 Mon Sep 17 00:00:00 2001 From: richard-dds Date: Mon, 9 Sep 2019 16:07:07 -0400 Subject: [PATCH 05/14] Add query for getting environments pending baseline creation --- atst/domain/environments.py | 10 +++++++++ tests/domain/test_environments.py | 37 +++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/atst/domain/environments.py b/atst/domain/environments.py index 2d3bd4a8..b4bd2edd 100644 --- a/atst/domain/environments.py +++ b/atst/domain/environments.py @@ -129,3 +129,13 @@ class Environments(object): .filter(Environment.root_user_info == text("'null'")) ) return [environment_id for (environment_id,) in query.all()] + + @classmethod + def get_environments_pending_baseline_creation(cls, now) -> [str]: + query = ( + cls.base_provision_query(now) + .filter(Environment.cloud_id != None) + .filter(Environment.root_user_info != text("'null'")) + .filter(Environment.baseline_info == text("'null'")) + ) + return [environment_id for (environment_id,) in query.all()] diff --git a/tests/domain/test_environments.py b/tests/domain/test_environments.py index e7049d37..36624403 100644 --- a/tests/domain/test_environments.py +++ b/tests/domain/test_environments.py @@ -214,3 +214,40 @@ class TestGetEnvironmentsPendingAtatUserCreation(EnvQueryTest): assert ( len(Environments.get_environments_pending_atat_user_creation(self.NOW)) == 0 ) + + +class TestGetEnvironmentsPendingBaselineCreation(EnvQueryTest): + def test_with_provisioned_environment(self): + self.create_portfolio_with_clins( + [(self.YESTERDAY, self.TOMORROW)], + { + "cloud_id": uuid4().hex, + "root_user_info": {"foo": "bar"}, + "baseline_info": {"foo": "bar"}, + }, + ) + assert ( + len(Environments.get_environments_pending_baseline_creation(self.NOW)) == 0 + ) + + def test_with_unprovisioned_environment(self): + self.create_portfolio_with_clins( + [(self.YESTERDAY, self.TOMORROW)], + { + "cloud_id": uuid4().hex, + "root_user_info": {"foo": "bar"}, + "baseline_info": None, + }, + ) + assert ( + len(Environments.get_environments_pending_baseline_creation(self.NOW)) == 1 + ) + + def test_with_unprovisioned_expired_clins_environment(self): + self.create_portfolio_with_clins( + [(self.YESTERDAY, self.YESTERDAY)], + {"cloud_id": uuid4().hex, "root_user_info": {"foo": "bar"}}, + ) + assert ( + len(Environments.get_environments_pending_baseline_creation(self.NOW)) == 0 + ) From ac0c19449015c9371fd2e176e6aafa5e56695aeb Mon Sep 17 00:00:00 2001 From: richard-dds Date: Tue, 10 Sep 2019 10:23:36 -0400 Subject: [PATCH 06/14] Add celery beat worker --- .gitignore | 3 +++ atst/queue.py | 1 + script/dev_queue | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index cf9591a0..36a8261e 100644 --- a/.gitignore +++ b/.gitignore @@ -62,3 +62,6 @@ browserstacklocal # configuration files override.ini atst-overrides.ini + +# binary file created by celery beat +celerybeat-schedule diff --git a/atst/queue.py b/atst/queue.py index e3ae049d..d71532b1 100644 --- a/atst/queue.py +++ b/atst/queue.py @@ -5,6 +5,7 @@ celery = Celery(__name__) def update_celery(celery, app): celery.conf.update(app.config) + celery.conf.CELERYBEAT_SCHEDULE = {} class ContextTask(celery.Task): def __call__(self, *args, **kwargs): diff --git a/script/dev_queue b/script/dev_queue index 670dd96b..4e8720ee 100755 --- a/script/dev_queue +++ b/script/dev_queue @@ -4,7 +4,7 @@ set -e -WORKER="pipenv run celery -A celery_worker.celery worker --loglevel=info" +WORKER="pipenv run celery -A celery_worker.celery worker --loglevel=info -B" if [[ `command -v entr` ]]; then find atst | entr -r $WORKER From 4405ed91d58094c13ecb8cea07c596e2be8178c5 Mon Sep 17 00:00:00 2001 From: richard-dds Date: Tue, 10 Sep 2019 13:24:55 -0400 Subject: [PATCH 07/14] Fix migration path --- .../502e79c55d2d_add_environment_csp_info.py | 2 +- atst/jobs.py | 1 - tests/test_jobs.py | 17 +++++------------ 3 files changed, 6 insertions(+), 14 deletions(-) diff --git a/alembic/versions/502e79c55d2d_add_environment_csp_info.py b/alembic/versions/502e79c55d2d_add_environment_csp_info.py index 3804e3a9..ab1f4555 100644 --- a/alembic/versions/502e79c55d2d_add_environment_csp_info.py +++ b/alembic/versions/502e79c55d2d_add_environment_csp_info.py @@ -11,7 +11,7 @@ from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = '502e79c55d2d' -down_revision = '4a3122ffe898' # pragma: allowlist secret +down_revision = '30ea1cb20807' # pragma: allowlist secret branch_labels = None depends_on = None diff --git a/atst/jobs.py b/atst/jobs.py index 50a06588..2c95f235 100644 --- a/atst/jobs.py +++ b/atst/jobs.py @@ -6,7 +6,6 @@ from atst.models import EnvironmentJobFailure, EnvironmentRoleJobFailure from atst.domain.csp.cloud import CloudProviderInterface, GeneralCSPException from atst.domain.environments import Environments from atst.domain.users import Users -from atst.models import Application, Portfolio, TaskOrder, CLIN, Environment class RecordEnvironmentFailure(celery.Task): diff --git a/tests/test_jobs.py b/tests/test_jobs.py index 409663d1..be4f13fe 100644 --- a/tests/test_jobs.py +++ b/tests/test_jobs.py @@ -1,27 +1,19 @@ import pendulum import pytest - -from atst.jobs import RecordEnvironmentFailure, RecordEnvironmentRoleFailure -from tests.factories import EnvironmentFactory, EnvironmentRoleFactory, UserFactory from uuid import uuid4 from unittest.mock import Mock from atst.models import Environment -from tests.factories import ( - UserFactory, - PortfolioFactory, - CLINFactory, - TaskOrderFactory, - EnvironmentFactory -) from atst.domain.csp.cloud import MockCloudProvider from atst.jobs import ( + RecordEnvironmentFailure, + RecordEnvironmentRoleFailure, do_create_environment, do_create_atat_admin_user, do_create_environment_baseline, - environments_to_create, ) -from atst.models import Environment +from tests.factories import EnvironmentFactory, EnvironmentRoleFactory, UserFactory + def test_environment_job_failure(celery_app, celery_worker): @celery_app.task(bind=True, base=RecordEnvironmentFailure) @@ -58,6 +50,7 @@ def test_environment_role_job_failure(celery_app, celery_worker): job_failure = role.job_failures[0] assert job_failure.task == task + now = pendulum.now() yesterday = now.subtract(days=1) tomorrow = now.add(days=1) From e65c1d69b6465774dfdb33cd3967f4ecc51474fa Mon Sep 17 00:00:00 2001 From: richard-dds Date: Tue, 10 Sep 2019 13:25:27 -0400 Subject: [PATCH 08/14] Add Environment.creator_role relation --- ...ab6c8243cb_add_environment_creator_role.py | 30 +++++++++++++++++++ atst/models/environment.py | 3 ++ 2 files changed, 33 insertions(+) create mode 100644 alembic/versions/cfab6c8243cb_add_environment_creator_role.py diff --git a/alembic/versions/cfab6c8243cb_add_environment_creator_role.py b/alembic/versions/cfab6c8243cb_add_environment_creator_role.py new file mode 100644 index 00000000..33a7a2a0 --- /dev/null +++ b/alembic/versions/cfab6c8243cb_add_environment_creator_role.py @@ -0,0 +1,30 @@ +"""add Environment creator_role + +Revision ID: cfab6c8243cb +Revises: 502e79c55d2d +Create Date: 2019-09-10 11:21:43.252592 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = 'cfab6c8243cb' # pragma: allowlist secret +down_revision = '502e79c55d2d' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('environments', sa.Column('creator_role_id', postgresql.UUID(as_uuid=True), nullable=False)) + op.create_foreign_key("fk_application_roles_id", 'environments', 'application_roles', ['creator_role_id'], ['id']) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint("fk_application_roles_id", 'environments', type_='foreignkey') + op.drop_column('environments', 'creator_role_id') + # ### end Alembic commands ### diff --git a/atst/models/environment.py b/atst/models/environment.py index d85550c6..fde36934 100644 --- a/atst/models/environment.py +++ b/atst/models/environment.py @@ -18,6 +18,9 @@ class Environment( application_id = Column(ForeignKey("applications.id"), nullable=False) application = relationship("Application") + creator_role_id = Column(ForeignKey("application_roles.id"), nullable=False) + creator = relationship("ApplicationRole") + cloud_id = Column(String) root_user_info = Column(JSONB) baseline_info = Column(JSONB) From f6cb6f2a3128bbecabde5cbf91559e81b9d43415 Mon Sep 17 00:00:00 2001 From: richard-dds Date: Tue, 10 Sep 2019 14:16:38 -0400 Subject: [PATCH 09/14] Change Environment.creator relation from ApplicationRole to User --- ...fab6c8243cb_add_environment_creator_role.py | 8 ++++---- atst/domain/applications.py | 4 ++-- atst/domain/environments.py | 8 ++++---- atst/models/environment.py | 7 +++++-- atst/routes/applications/new.py | 3 ++- atst/routes/applications/settings.py | 4 +++- script/seed_sample.py | 3 ++- tests/domain/test_applications.py | 2 +- tests/domain/test_environments.py | 4 +++- tests/domain/test_portfolios.py | 18 +++++++++++++++--- tests/factories.py | 1 + tests/models/test_environments.py | 6 +++++- tests/routes/applications/test_settings.py | 3 +++ 13 files changed, 50 insertions(+), 21 deletions(-) diff --git a/alembic/versions/cfab6c8243cb_add_environment_creator_role.py b/alembic/versions/cfab6c8243cb_add_environment_creator_role.py index 33a7a2a0..b96c78e2 100644 --- a/alembic/versions/cfab6c8243cb_add_environment_creator_role.py +++ b/alembic/versions/cfab6c8243cb_add_environment_creator_role.py @@ -18,13 +18,13 @@ depends_on = None def upgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.add_column('environments', sa.Column('creator_role_id', postgresql.UUID(as_uuid=True), nullable=False)) - op.create_foreign_key("fk_application_roles_id", 'environments', 'application_roles', ['creator_role_id'], ['id']) + op.add_column('environments', sa.Column('creator_id', postgresql.UUID(as_uuid=True), nullable=False)) + op.create_foreign_key("fk_users_id", 'environments', 'users', ['creator_id'], ['id']) # ### end Alembic commands ### def downgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.drop_constraint("fk_application_roles_id", 'environments', type_='foreignkey') - op.drop_column('environments', 'creator_role_id') + op.drop_constraint("fk_users_id", 'environments', type_='foreignkey') + op.drop_column('environments', 'creator_id') # ### end Alembic commands ### diff --git a/atst/domain/applications.py b/atst/domain/applications.py index dc519a0a..0c063332 100644 --- a/atst/domain/applications.py +++ b/atst/domain/applications.py @@ -19,13 +19,13 @@ class Applications(BaseDomainClass): resource_name = "application" @classmethod - def create(cls, portfolio, name, description, environment_names): + def create(cls, user, portfolio, name, description, environment_names): application = Application( portfolio=portfolio, name=name, description=description ) db.session.add(application) - Environments.create_many(application, environment_names) + Environments.create_many(user, application, environment_names) db.session.commit() return application diff --git a/atst/domain/environments.py b/atst/domain/environments.py index b4bd2edd..3a740807 100644 --- a/atst/domain/environments.py +++ b/atst/domain/environments.py @@ -11,17 +11,17 @@ from .exceptions import NotFoundError class Environments(object): @classmethod - def create(cls, application, name): - environment = Environment(application=application, name=name) + def create(cls, user, application, name): + environment = Environment(application=application, name=name, creator=user) db.session.add(environment) db.session.commit() return environment @classmethod - def create_many(cls, application, names): + def create_many(cls, user, application, names): environments = [] for name in names: - environment = Environments.create(application, name) + environment = Environments.create(user, application, name) environments.append(environment) db.session.add_all(environments) diff --git a/atst/models/environment.py b/atst/models/environment.py index fde36934..02e87f85 100644 --- a/atst/models/environment.py +++ b/atst/models/environment.py @@ -18,8 +18,11 @@ class Environment( application_id = Column(ForeignKey("applications.id"), nullable=False) application = relationship("Application") - creator_role_id = Column(ForeignKey("application_roles.id"), nullable=False) - creator = relationship("ApplicationRole") + # User user.id as the foreign key here beacuse the Environment creator may + # not have an application role. We may need to revisit this if we receive any + # requirements around tracking an environment's custodian. + creator_id = Column(ForeignKey("users.id"), nullable=False) + creator = relationship("User") cloud_id = Column(String) root_user_info = Column(JSONB) diff --git a/atst/routes/applications/new.py b/atst/routes/applications/new.py index 070aa940..2d04d7a3 100644 --- a/atst/routes/applications/new.py +++ b/atst/routes/applications/new.py @@ -1,4 +1,4 @@ -from flask import redirect, render_template, request as http_request, url_for +from flask import redirect, render_template, request as http_request, url_for, g from . import applications_bp from atst.domain.applications import Applications @@ -24,6 +24,7 @@ def create(portfolio_id): if form.validate(): application_data = form.data Applications.create( + g.current_user, portfolio, application_data["name"], application_data["description"], diff --git a/atst/routes/applications/settings.py b/atst/routes/applications/settings.py index e586e317..ee7d844c 100644 --- a/atst/routes/applications/settings.py +++ b/atst/routes/applications/settings.py @@ -230,7 +230,9 @@ def new_environment(application_id): env_form = EditEnvironmentForm(formdata=http_request.form) if env_form.validate(): - Environments.create(application=application, name=env_form.name.data) + Environments.create( + g.current_user, application=application, name=env_form.name.data + ) flash("environment_added", environment_name=env_form.data["name"]) diff --git a/script/seed_sample.py b/script/seed_sample.py index e5fde801..a242d20d 100644 --- a/script/seed_sample.py +++ b/script/seed_sample.py @@ -220,6 +220,7 @@ def add_applications_to_portfolio(portfolio): applications = random_applications() for application_data in applications: application = Applications.create( + portfolio.owner, portfolio=portfolio, name=application_data["name"], description=application_data["description"], @@ -278,7 +279,7 @@ def create_demo_portfolio(name, data): portfolio=portfolio, name=mock_application.name, description="" ) env_names = [env.name for env in mock_application.environments] - envs = Environments.create_many(application, env_names) + envs = Environments.create_many(portfolio.owner, application, env_names) db.session.add(application) db.session.commit() diff --git a/tests/domain/test_applications.py b/tests/domain/test_applications.py index acbddc11..4e25ab4e 100644 --- a/tests/domain/test_applications.py +++ b/tests/domain/test_applications.py @@ -21,7 +21,7 @@ from tests.factories import ( def test_create_application_with_multiple_environments(): portfolio = PortfolioFactory.create() application = Applications.create( - portfolio, "My Test Application", "Test", ["dev", "prod"] + portfolio.owner, portfolio, "My Test Application", "Test", ["dev", "prod"] ) assert application.portfolio == portfolio diff --git a/tests/domain/test_environments.py b/tests/domain/test_environments.py index 36624403..a87551a4 100644 --- a/tests/domain/test_environments.py +++ b/tests/domain/test_environments.py @@ -21,7 +21,9 @@ from tests.factories import ( @pytest.mark.skip(reason="Reinstate and update once jobs api is up") def test_create_environments(): application = ApplicationFactory.create() - environments = Environments.create_many(application, ["Staging", "Production"]) + environments = Environments.create_many( + application.portfolio.owner, application, ["Staging", "Production"] + ) for env in environments: assert env.cloud_id is not None diff --git a/tests/domain/test_portfolios.py b/tests/domain/test_portfolios.py index e69ee45d..80dade92 100644 --- a/tests/domain/test_portfolios.py +++ b/tests/domain/test_portfolios.py @@ -71,7 +71,11 @@ def test_update_portfolio_role_role(portfolio, portfolio_owner): def test_scoped_portfolio_for_admin_missing_view_apps_perms(portfolio_owner, portfolio): Applications.create( - portfolio, "My Application 2", "My application 2", ["dev", "staging", "prod"] + portfolio.owner, + portfolio, + "My Application 2", + "My application 2", + ["dev", "staging", "prod"], ) restricted_admin = UserFactory.create() PortfolioRoleFactory.create( @@ -90,7 +94,11 @@ def test_scoped_portfolio_returns_all_applications_for_portfolio_admin( ): for _ in range(5): Applications.create( - portfolio, "My Application", "My application", ["dev", "staging", "prod"] + portfolio.owner, + portfolio, + "My Application", + "My application", + ["dev", "staging", "prod"], ) admin = UserFactory.create() @@ -109,7 +117,11 @@ def test_scoped_portfolio_returns_all_applications_for_portfolio_owner( ): for _ in range(5): Applications.create( - portfolio, "My Application", "My application", ["dev", "staging", "prod"] + portfolio.owner, + portfolio, + "My Application", + "My application", + ["dev", "staging", "prod"], ) scoped_portfolio = Portfolios.get(portfolio_owner, portfolio.id) diff --git a/tests/factories.py b/tests/factories.py index 7dd4a2f6..99ada8f4 100644 --- a/tests/factories.py +++ b/tests/factories.py @@ -185,6 +185,7 @@ class EnvironmentFactory(Base): name = factory.Faker("domain_word") application = factory.SubFactory(ApplicationFactory) + creator = factory.SubFactory(UserFactory) @classmethod def _create(cls, model_class, *args, **kwargs): diff --git a/tests/models/test_environments.py b/tests/models/test_environments.py index 9bc1c8b5..170b5162 100644 --- a/tests/models/test_environments.py +++ b/tests/models/test_environments.py @@ -12,7 +12,11 @@ def test_add_user_to_environment(): portfolio = PortfolioFactory.create(owner=owner) application = Applications.create( - portfolio, "my test application", "It's mine.", ["dev", "staging", "prod"] + portfolio.owner, + portfolio, + "my test application", + "It's mine.", + ["dev", "staging", "prod"], ) dev_environment = application.environments[0] diff --git a/tests/routes/applications/test_settings.py b/tests/routes/applications/test_settings.py index 9222167d..51ac6d91 100644 --- a/tests/routes/applications/test_settings.py +++ b/tests/routes/applications/test_settings.py @@ -71,6 +71,7 @@ def test_update_environment_failure(client, user_session): def test_application_settings(client, user_session): portfolio = PortfolioFactory.create() application = Applications.create( + portfolio.owner, portfolio, "Snazzy Application", "A new application for me and my friends", @@ -86,6 +87,7 @@ def test_application_settings(client, user_session): def test_edit_application_environments_obj(app, client, user_session): portfolio = PortfolioFactory.create() application = Applications.create( + portfolio.owner, portfolio, "Snazzy Application", "A new application for me and my friends", @@ -127,6 +129,7 @@ def test_edit_application_environments_obj(app, client, user_session): def test_data_for_app_env_roles_form(app, client, user_session): portfolio = PortfolioFactory.create() application = Applications.create( + portfolio.owner, portfolio, "Snazzy Application", "A new application for me and my friends", From 6b7db2ca46f83a49bbc8eb7a1980ad938abf6e95 Mon Sep 17 00:00:00 2001 From: richard-dds Date: Tue, 10 Sep 2019 14:33:35 -0400 Subject: [PATCH 10/14] Better ergonomics for creating factory portfolios w/ TOs --- tests/domain/test_environments.py | 8 ++++---- tests/domain/test_task_orders.py | 6 ++++-- tests/factories.py | 9 ++++++++- tests/routes/task_orders/test_new.py | 2 +- 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/tests/domain/test_environments.py b/tests/domain/test_environments.py index a87551a4..f963f92b 100644 --- a/tests/domain/test_environments.py +++ b/tests/domain/test_environments.py @@ -163,12 +163,12 @@ class EnvQueryTest: } ], task_orders=[ - TaskOrderFactory.create( - clins=[ - CLINFactory.create(start_date=start_date, end_date=end_date) + { + "create_clins": [ + {"start_date": start_date, "end_date": end_date} for (start_date, end_date) in start_and_end_dates ] - ) + } ], ) diff --git a/tests/domain/test_task_orders.py b/tests/domain/test_task_orders.py index e718d58f..6cfd91fd 100644 --- a/tests/domain/test_task_orders.py +++ b/tests/domain/test_task_orders.py @@ -135,7 +135,7 @@ def test_update_adds_clins(): def test_update_does_not_duplicate_clins(): task_order = TaskOrderFactory.create( - number="3453453456", create_clins=["123", "456"] + number="3453453456", create_clins=[{"number": "123"}, {"number": "456"}] ) clins = [ { @@ -165,7 +165,9 @@ def test_update_does_not_duplicate_clins(): def test_delete_task_order_with_clins(session): - task_order = TaskOrderFactory.create(create_clins=[1, 2, 3]) + task_order = TaskOrderFactory.create( + create_clins=[{"number": 1}, {"number": 2}, {"number": 3}] + ) TaskOrders.delete(task_order.id) assert not session.query( diff --git a/tests/factories.py b/tests/factories.py index 99ada8f4..27902558 100644 --- a/tests/factories.py +++ b/tests/factories.py @@ -118,6 +118,7 @@ class PortfolioFactory(Base): with_applications = kwargs.pop("applications", []) owner = kwargs.pop("owner", UserFactory.create()) members = kwargs.pop("members", []) + with_task_orders = kwargs.pop("task_orders", []) portfolio = super()._create(model_class, *args, **kwargs) @@ -126,6 +127,11 @@ class PortfolioFactory(Base): for p in with_applications ] + task_orders = [ + TaskOrderFactory.create(portfolio=portfolio, **to) + for to in with_task_orders + ] + PortfolioRoleFactory.create( portfolio=portfolio, user=owner, @@ -154,6 +160,7 @@ class PortfolioFactory(Base): ) portfolio.applications = applications + portfolio.task_orders = task_orders return portfolio @@ -279,7 +286,7 @@ class TaskOrderFactory(Base): task_order = super()._create(model_class, *args, **kwargs) for clin in create_clins: - CLINFactory.create(task_order=task_order, number=clin) + CLINFactory.create(task_order=task_order, **clin) return task_order diff --git a/tests/routes/task_orders/test_new.py b/tests/routes/task_orders/test_new.py index 4d64ed38..d2e1aae0 100644 --- a/tests/routes/task_orders/test_new.py +++ b/tests/routes/task_orders/test_new.py @@ -29,7 +29,7 @@ def completed_task_order(): task_order = TaskOrderFactory.create( creator=portfolio.owner, portfolio=portfolio, - create_clins=["1234567890123456789012345678901234567890123"], + create_clins=[{"number": "1234567890123456789012345678901234567890123"}], ) return task_order From 1a9c34d856935d1388a8bd67b22add0c43ea378b Mon Sep 17 00:00:00 2001 From: richard-dds Date: Tue, 10 Sep 2019 15:48:23 -0400 Subject: [PATCH 11/14] Fix and test environment dispatch tasks --- atst/domain/environments.py | 35 +++++++---- atst/jobs.py | 21 +++++-- tests/routes/task_orders/test_new.py | 2 +- tests/test_access.py | 2 +- tests/test_jobs.py | 93 +++++++++++++++++++++++++++- 5 files changed, 132 insertions(+), 21 deletions(-) diff --git a/atst/domain/environments.py b/atst/domain/environments.py index 3a740807..be42d3de 100644 --- a/atst/domain/environments.py +++ b/atst/domain/environments.py @@ -1,5 +1,7 @@ from sqlalchemy import text from sqlalchemy.orm.exc import NoResultFound +from sqlalchemy.orm import load_only +from typing import List from atst.database import db from atst.models import Environment, Application, Portfolio, TaskOrder, CLIN @@ -107,35 +109,44 @@ class Environments(object): @classmethod def base_provision_query(cls, now): return ( - db.session.query(Environment.id) + db.session.query(Environment) .join(Application) .join(Portfolio) .join(TaskOrder) .join(CLIN) .filter(CLIN.start_date <= now) .filter(CLIN.end_date > now) + # select only these columns + .options(load_only("id", "creator_id")) ) @classmethod - def get_environments_pending_creation(cls, now) -> [str]: - query = cls.base_provision_query(now).filter(Environment.cloud_id == None) - return [environment_id for (environment_id,) in query.all()] + def get_environments_pending_creation(cls, now) -> List[Environment]: + """ + Any environment with an active CLIN that doesn't yet have a `cloud_id`. + """ + return cls.base_provision_query(now).filter(Environment.cloud_id == None).all() @classmethod - def get_environments_pending_atat_user_creation(cls, now) -> [str]: - query = ( + def get_environments_pending_atat_user_creation(cls, now) -> List[Environment]: + """ + Any environment with an active CLIN that has a cloud_id but no `root_user_info`. + """ + return ( cls.base_provision_query(now) .filter(Environment.cloud_id != None) .filter(Environment.root_user_info == text("'null'")) - ) - return [environment_id for (environment_id,) in query.all()] + ).all() @classmethod - def get_environments_pending_baseline_creation(cls, now) -> [str]: - query = ( + def get_environments_pending_baseline_creation(cls, now) -> List[Environment]: + """ + Any environment with an active CLIN that has a `cloud_id` and `root_user_info` + but no `baseline_info`. + """ + return ( cls.base_provision_query(now) .filter(Environment.cloud_id != None) .filter(Environment.root_user_info != text("'null'")) .filter(Environment.baseline_info == text("'null'")) - ) - return [environment_id for (environment_id,) in query.all()] + ).all() diff --git a/atst/jobs.py b/atst/jobs.py index 2c95f235..050cb4ca 100644 --- a/atst/jobs.py +++ b/atst/jobs.py @@ -1,4 +1,5 @@ from flask import current_app as app +import pendulum from atst.database import db from atst.queue import celery @@ -121,15 +122,23 @@ def create_environment_baseline(self, environment_id=None): @celery.task(bind=True) def dispatch_create_environment(self): - for environment_id in Environments.get_environments_pending_creation( - pendulum.now() - ): - create_environment.delay(environment_id=environment_id, atat_user_id="TODO") + for environment in Environments.get_environments_pending_creation(pendulum.now()): + create_environment.delay( + environment_id=environment.id, atat_user_id=environment.creator_id + ) @celery.task(bind=True) def dispatch_create_atat_admin_user(self): - for environment_id in Environments.get_environments_pending_atat_user_creation( + for environment in Environments.get_environments_pending_atat_user_creation( pendulum.now() ): - create_atat_admin_user.delay(environment_id=environment_id, atat_user_id="TODO") + create_atat_admin_user.delay(environment_id=environment.id) + + +@celery.task(bind=True) +def dispatch_create_environment_baseline(self): + for environment in Environments.get_environments_pending_baseline_creation( + pendulum.now() + ): + create_environment_baseline.delay(environment_id=environment.id) diff --git a/tests/routes/task_orders/test_new.py b/tests/routes/task_orders/test_new.py index d2e1aae0..3682fc1e 100644 --- a/tests/routes/task_orders/test_new.py +++ b/tests/routes/task_orders/test_new.py @@ -329,7 +329,7 @@ def test_task_orders_submit_task_order(client, user_session, task_order): ({"_pdf": None, "number": "", "clins": []}, "step_1"), ({"number": "", "clins": []}, "step_2"), ({"number": "1234567890123", "clins": []}, "step_3"), - ({"number": "1234567890123", "create_clins": [1]}, "step_4"), + ({"number": "1234567890123", "create_clins": [{"number": 1}]}, "step_4"), ], ) def test_task_orders_edit_redirects_to_latest_incomplete_step( diff --git a/tests/test_access.py b/tests/test_access.py index 8a43b97a..50f14575 100644 --- a/tests/test_access.py +++ b/tests/test_access.py @@ -541,7 +541,7 @@ def test_task_orders_new_get_routes(get_url_assert_status): task_order = TaskOrderFactory.create( creator=owner, portfolio=portfolio, - create_clins=["1234567890123456789012345678901234567890123"], + create_clins=[{"number": "1234567890123456789012345678901234567890123"}], ) for route in get_routes: diff --git a/tests/test_jobs.py b/tests/test_jobs.py index be4f13fe..749b5ad5 100644 --- a/tests/test_jobs.py +++ b/tests/test_jobs.py @@ -11,8 +11,16 @@ from atst.jobs import ( do_create_environment, do_create_atat_admin_user, do_create_environment_baseline, + dispatch_create_environment, + dispatch_create_atat_admin_user, + dispatch_create_environment_baseline, +) +from tests.factories import ( + EnvironmentFactory, + EnvironmentRoleFactory, + UserFactory, + PortfolioFactory, ) -from tests.factories import EnvironmentFactory, EnvironmentRoleFactory, UserFactory def test_environment_job_failure(celery_app, celery_worker): @@ -105,3 +113,86 @@ def test_create_environment_baseline(csp, session): updated_environment = session.query(Environment).get(environment_id) assert updated_environment.baseline_info + + +def test_dispatch_create_environment(session, monkeypatch): + portfolio = PortfolioFactory.create( + applications=[{"environments": [{}]}], + task_orders=[ + { + "create_clins": [ + { + "start_date": pendulum.now().subtract(days=1), + "end_date": pendulum.now().add(days=1), + } + ] + } + ], + ) + mock = Mock() + monkeypatch.setattr("atst.jobs.create_environment", mock) + environment = portfolio.applications[0].environments[0] + + dispatch_create_environment.run() + + mock.delay.assert_called_once_with( + environment_id=environment.id, atat_user_id=environment.creator_id + ) + + +def test_dispatch_create_atat_admin_user(session, monkeypatch): + portfolio = PortfolioFactory.create( + applications=[ + {"environments": [{"cloud_id": uuid4().hex, "root_user_info": None}]} + ], + task_orders=[ + { + "create_clins": [ + { + "start_date": pendulum.now().subtract(days=1), + "end_date": pendulum.now().add(days=1), + } + ] + } + ], + ) + mock = Mock() + monkeypatch.setattr("atst.jobs.create_atat_admin_user", mock) + environment = portfolio.applications[0].environments[0] + + dispatch_create_atat_admin_user.run() + + mock.delay.assert_called_once_with(environment_id=environment.id) + + +def test_dispatch_create_environment_baseline(session, monkeypatch): + portfolio = PortfolioFactory.create( + applications=[ + { + "environments": [ + { + "cloud_id": uuid4().hex, + "root_user_info": {}, + "baseline_info": None, + } + ] + } + ], + task_orders=[ + { + "create_clins": [ + { + "start_date": pendulum.now().subtract(days=1), + "end_date": pendulum.now().add(days=1), + } + ] + } + ], + ) + mock = Mock() + monkeypatch.setattr("atst.jobs.create_environment_baseline", mock) + environment = portfolio.applications[0].environments[0] + + dispatch_create_environment_baseline.run() + + mock.delay.assert_called_once_with(environment_id=environment.id) From 365a50efce412e87768f40858d28c2f8ff65a719 Mon Sep 17 00:00:00 2001 From: richard-dds Date: Tue, 10 Sep 2019 16:47:45 -0400 Subject: [PATCH 12/14] Add Environments.provisioning_status --- atst/models/environment.py | 16 ++++++++++++++++ tests/models/test_environments.py | 29 ++++++++++++++++++++++++++++- 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/atst/models/environment.py b/atst/models/environment.py index 02e87f85..fcdea074 100644 --- a/atst/models/environment.py +++ b/atst/models/environment.py @@ -1,6 +1,7 @@ from sqlalchemy import Column, ForeignKey, String from sqlalchemy.orm import relationship from sqlalchemy.dialects.postgresql import JSONB +from enum import Enum from atst.models import Base from atst.models.types import Id @@ -30,6 +31,10 @@ class Environment( job_failures = relationship("EnvironmentJobFailure") + class ProvisioningStatus(Enum): + PENDING = "pending" + COMPLETED = "completed" + @property def users(self): return {r.application_role.user for r in self.roles} @@ -50,6 +55,17 @@ class Environment( def portfolio_id(self): return self.application.portfolio_id + @property + def provisioning_status(self) -> ProvisioningStatus: + if ( + self.cloud_id is None + or self.root_user_info is None + or self.baseline_info is None + ): + return self.ProvisioningStatus.PENDING + else: + return self.ProvisioningStatus.COMPLETED + def __repr__(self): return "".format( self.name, diff --git a/tests/models/test_environments.py b/tests/models/test_environments.py index 170b5162..2c2453f6 100644 --- a/tests/models/test_environments.py +++ b/tests/models/test_environments.py @@ -1,6 +1,7 @@ +import pytest + from atst.models import AuditEvent from atst.models.environment_role import CSPRole -from atst.domain.environments import Environments from atst.domain.applications import Applications from tests.factories import * @@ -46,3 +47,29 @@ def test_audit_event_for_environment_deletion(session): before, after = update_event.changed_state["deleted"] assert not before assert after + + +@pytest.mark.parametrize( + "env_data,expected_status", + [ + [ + {"cloud_id": None, "root_user_info": None, "baseline_info": None}, + Environment.ProvisioningStatus.PENDING, + ], + [ + {"cloud_id": 1, "root_user_info": None, "baseline_info": None}, + Environment.ProvisioningStatus.PENDING, + ], + [ + {"cloud_id": 1, "root_user_info": {}, "baseline_info": None}, + Environment.ProvisioningStatus.PENDING, + ], + [ + {"cloud_id": 1, "root_user_info": {}, "baseline_info": {}}, + Environment.ProvisioningStatus.COMPLETED, + ], + ], +) +def test_environment_provisioning_status(env_data, expected_status): + environment = EnvironmentFactory.create(**env_data) + assert environment.provisioning_status == expected_status From bc9426015ce598447455e0fdee8f7b3693ef2c63 Mon Sep 17 00:00:00 2001 From: richard-dds Date: Tue, 10 Sep 2019 17:07:46 -0400 Subject: [PATCH 13/14] Add another test for Environments.get_environments_pending_creation --- tests/domain/test_environments.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/domain/test_environments.py b/tests/domain/test_environments.py index f963f92b..4c44c1ec 100644 --- a/tests/domain/test_environments.py +++ b/tests/domain/test_environments.py @@ -188,6 +188,12 @@ class TestGetEnvironmentsPendingCreate(EnvQueryTest): self.create_portfolio_with_clins([(self.TOMORROW, self.TOMORROW)]) assert len(Environments.get_environments_pending_creation(self.NOW)) == 0 + def test_with_already_provisioned_env(self, session): + self.create_portfolio_with_clins( + [(self.YESTERDAY, self.TOMORROW)], env_data={"cloud_id": uuid4().hex} + ) + assert len(Environments.get_environments_pending_creation(self.NOW)) == 0 + class TestGetEnvironmentsPendingAtatUserCreation(EnvQueryTest): def test_with_provisioned_environment(self): From 6100b6e479ef9c6606b90f1121ca3ff45730f6f6 Mon Sep 17 00:00:00 2001 From: richard-dds Date: Wed, 11 Sep 2019 15:36:07 -0400 Subject: [PATCH 14/14] Fix create_environments test --- tests/domain/test_environments.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/domain/test_environments.py b/tests/domain/test_environments.py index 4c44c1ec..f528d6f0 100644 --- a/tests/domain/test_environments.py +++ b/tests/domain/test_environments.py @@ -13,19 +13,16 @@ from tests.factories import ( EnvironmentFactory, EnvironmentRoleFactory, ApplicationRoleFactory, - TaskOrderFactory, - CLINFactory, ) -@pytest.mark.skip(reason="Reinstate and update once jobs api is up") def test_create_environments(): application = ApplicationFactory.create() environments = Environments.create_many( application.portfolio.owner, application, ["Staging", "Production"] ) for env in environments: - assert env.cloud_id is not None + assert env.cloud_id is None def test_update_env_role():