Merge branch 'staging' into environment-role-creation

Dockerfile

@@ -101,5 +101,7 @@ RUN mkdir /var/run/uwsgi && \
       chown -R atst:atat /var/run/uwsgi && \
       chown -R atst:atat "${APP_DIR}"
 
+RUN update-ca-certificates
+
 # Run as the unprivileged APP user
 USER atst

alembic/versions/418b52c1cedf_change_to_environment_roles_cloud_id.py

@@ -1,7 +1,7 @@
 """change to environment_roles.cloud_Id
 
 Revision ID: 418b52c1cedf
-Revises: 0039308c6351
+Revises: 567bfb019a87
 Create Date: 2020-02-05 13:40:37.870183
 
 """
@@ -11,7 +11,7 @@ import sqlalchemy as sa
 
 # revision identifiers, used by Alembic.
 revision = '418b52c1cedf' # pragma: allowlist secret
-down_revision = '0039308c6351' # pragma: allowlist secret
+down_revision = '567bfb019a87' # pragma: allowlist secret
 branch_labels = None
 depends_on = None
 

alembic/versions/567bfb019a87_add_last_sent_column_to_clins_and_pdf_.py

@@ -0,0 +1,29 @@
+"""add last_sent column to clins and pdf_last_sent to task_orders
+
+Revision ID: 567bfb019a87
+Revises: 0039308c6351
+Create Date: 2020-01-31 14:06:21.926019
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = '567bfb019a87' # pragma: allowlist secret
+down_revision = '0039308c6351' # pragma: allowlist secret
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('clins', sa.Column('last_sent_at', sa.DateTime(), nullable=True))
+    op.add_column('task_orders', sa.Column('pdf_last_sent_at', sa.DateTime(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('task_orders', 'pdf_last_sent_at')
+    op.drop_column('clins', 'last_sent_at')
+    # ### end Alembic commands ###

atst/domain/csp/files.py

@@ -43,10 +43,12 @@ class AzureFileService(FileService):
 
         from azure.storage.common import CloudStorageAccount
         from azure.storage.blob import BlobSasPermissions
+        from azure.storage.blob.models import BlobPermissions
         from azure.storage.blob.blockblobservice import BlockBlobService
 
         self.CloudStorageAccount = CloudStorageAccount
         self.BlobSasPermissions = BlobSasPermissions
+        self.BlobPermissions = BlobPermissions
         self.BlockBlobService = BlockBlobService
 
     def get_token(self):
@@ -72,20 +74,22 @@ class AzureFileService(FileService):
         return ({"token": sas_token}, object_name)
 
     def generate_download_link(self, object_name, filename):
-        account = self.CloudStorageAccount(
+        block_blob_service = self.BlockBlobService(
             account_name=self.account_name, account_key=self.storage_key
         )
-        bbs = account.create_block_blob_service()
-        sas_token = bbs.generate_blob_shared_access_signature(
-            self.container_name,
-            object_name,
-            permission=self.BlobSasPermissions(read=True),
+        sas_token = block_blob_service.generate_blob_shared_access_signature(
+            container_name=self.container_name,
+            blob_name=object_name,
+            permission=self.BlobPermissions(read=True),
             expiry=datetime.utcnow() + self.timeout,
             content_disposition=f"attachment; filename={filename}",
             protocol="https",
         )
-        return bbs.make_blob_url(
-            self.container_name, object_name, protocol="https", sas_token=sas_token
+        return block_blob_service.make_blob_url(
+            container_name=self.container_name,
+            blob_name=object_name,
+            protocol="https",
+            sas_token=sas_token,
         )
 
     def download_task_order(self, object_name):

atst/domain/task_orders.py

@@ -1,4 +1,5 @@
-import datetime
+from datetime import datetime
+from sqlalchemy import or_
 
 from atst.database import db
 from atst.models.clin import CLIN
@@ -40,7 +41,7 @@ class TaskOrders(BaseDomainClass):
     @classmethod
     def sign(cls, task_order, signer_dod_id):
         task_order.signer_dod_id = signer_dod_id
-        task_order.signed_at = datetime.datetime.now()
+        task_order.signed_at = datetime.now()
 
         db.session.add(task_order)
         db.session.commit()
@@ -76,3 +77,17 @@ class TaskOrders(BaseDomainClass):
         task_order = TaskOrders.get(task_order_id)
         db.session.delete(task_order)
         db.session.commit()
+
+    @classmethod
+    def get_for_send_task_order_files(cls):
+        return (
+            db.session.query(TaskOrder)
+            .join(CLIN)
+            .filter(
+                or_(
+                    TaskOrder.pdf_last_sent_at < CLIN.last_sent_at,
+                    TaskOrder.pdf_last_sent_at.is_(None),
+                )
+            )
+            .all()
+        )

atst/jobs.py

@@ -1,5 +1,7 @@
 import pendulum
 from flask import current_app as app
+from smtplib import SMTPException
+from azure.core.exceptions import AzureError
 
 from atst.database import db
 from atst.domain.application_roles import ApplicationRoles
@@ -16,8 +18,10 @@ from atst.domain.environments import Environments
 from atst.domain.environment_roles import EnvironmentRoles
 from atst.domain.portfolios import Portfolios
 from atst.models import CSPRole, JobFailure
+from atst.domain.task_orders import TaskOrders
 from atst.models.utils import claim_for_update, claim_many_for_update
 from atst.queue import celery
+from atst.utils.localization import translate
 
 
 class RecordFailure(celery.Task):
@@ -45,8 +49,8 @@ class RecordFailure(celery.Task):
 
 
 @celery.task(ignore_result=True)
-def send_mail(recipients, subject, body):
-    app.mailer.send(recipients, subject, body)
+def send_mail(recipients, subject, body, attachments=[]):
+    app.mailer.send(recipients, subject, body, attachments)
 
 
 @celery.task(ignore_result=True)
@@ -244,3 +248,39 @@ def dispatch_create_environment(self):
         pendulum.now()
     ):
         create_environment.delay(environment_id=environment_id)
+
+
+@celery.task(bind=True)
+def dispatch_create_atat_admin_user(self):
+    for environment_id in Environments.get_environments_pending_atat_user_creation(
+        pendulum.now()
+    ):
+        create_atat_admin_user.delay(environment_id=environment_id)
+
+
+@celery.task(bind=True)
+def dispatch_send_task_order_files(self):
+    task_orders = TaskOrders.get_for_send_task_order_files()
+    recipients = [app.config.get("MICROSOFT_TASK_ORDER_EMAIL_ADDRESS")]
+
+    for task_order in task_orders:
+        subject = translate(
+            "email.task_order_sent.subject", {"to_number": task_order.number}
+        )
+        body = translate("email.task_order_sent.body", {"to_number": task_order.number})
+
+        try:
+            file = app.csp.files.download_task_order(task_order.pdf.object_name)
+            file["maintype"] = "application"
+            file["subtype"] = "pdf"
+            send_mail(
+                recipients=recipients, subject=subject, body=body, attachments=[file]
+            )
+        except (AzureError, SMTPException) as err:
+            app.logger.exception(err)
+            continue
+
+        task_order.pdf_last_sent_at = pendulum.now()
+        db.session.add(task_order)
+
+    db.session.commit()

atst/models/clin.py

@@ -1,5 +1,13 @@
 from enum import Enum
-from sqlalchemy import Column, Date, Enum as SQLAEnum, ForeignKey, Numeric, String
+from sqlalchemy import (
+    Column,
+    Date,
+    DateTime,
+    Enum as SQLAEnum,
+    ForeignKey,
+    Numeric,
+    String,
+)
 from sqlalchemy.orm import relationship
 from datetime import date
 
@@ -29,6 +37,7 @@ class CLIN(Base, mixins.TimestampsMixin):
     total_amount = Column(Numeric(scale=2), nullable=False)
     obligated_amount = Column(Numeric(scale=2), nullable=False)
     jedi_clin_type = Column(SQLAEnum(JEDICLINType, native_enum=False), nullable=False)
+    last_sent_at = Column(DateTime)
 
     #
     # NOTE: For now obligated CLINS are CLIN 1 + CLIN 3

atst/models/environment.py

@@ -61,6 +61,10 @@ class Environment(
     def portfolio_id(self):
         return self.application.portfolio_id
 
+    @property
+    def is_pending(self):
+        return self.cloud_id is None
+
     def __repr__(self):
         return "<Environment(name='{}', num_users='{}', application='{}', portfolio='{}', id='{}')>".format(
             self.name,

atst/models/task_order.py

@@ -39,6 +39,7 @@ class TaskOrder(Base, mixins.TimestampsMixin):
 
     pdf_attachment_id = Column(ForeignKey("attachments.id"))
     _pdf = relationship("Attachment", foreign_keys=[pdf_attachment_id])
+    pdf_last_sent_at = Column(DateTime)
     number = Column(String, unique=True,)  # Task Order Number
     signer_dod_id = Column(String)
     signed_at = Column(DateTime)

atst/routes/applications/settings.py

@@ -39,6 +39,7 @@ def get_environments_obj_for_app(application):
             {
                 "id": env.id,
                 "name": env.name,
+                "pending": env.is_pending,
                 "edit_form": EditEnvironmentForm(obj=env),
                 "member_count": len(env.roles),
                 "members": sorted(

atst/utils/context_processors.py

@@ -19,6 +19,9 @@ from atst.models import (
 def get_resources_from_context(view_args):
     query = None
 
+    if view_args is None:
+        view_args = {}
+
     if "portfolio_token" in view_args:
         query = (
             db.session.query(Portfolio)

js/components/upload_input.js

@@ -17,7 +17,7 @@ export default {
     filename: {
       type: String,
     },
-    objectName: {
+    initialObjectName: {
       type: String,
     },
     initialErrors: {
@@ -42,6 +42,7 @@ export default {
       filenameError: false,
       downloadLink: '',
       fileSizeLimit: this.sizeLimit,
+      objectName: this.initialObjectName,
     }
   },
 
@@ -72,6 +73,7 @@ export default {
       const response = await uploader.upload(file)
       if (uploadResponseOkay(response)) {
         this.attachment = e.target.value
+        this.objectName = uploader.objectName
         this.$refs.attachmentFilename.value = file.name
         this.$refs.attachmentObjectName.value = response.objectName
         this.$refs.attachmentInput.disabled = true

js/lib/input_validations.js

@@ -9,6 +9,12 @@ export default {
     unmask: [],
     validationError: 'Please enter a response',
   },
+  applicationName: {
+    mask: false,
+    match: /^[A-Za-z0-9\-_,'".\s]{4,100}$$/,
+    unmask: [],
+    validationError: 'Application names can be between 4-100 characters',
+  },
   clinNumber: {
     mask: false,
     match: /^\d{4}$/,
@@ -42,14 +48,14 @@ export default {
   },
   defaultStringField: {
     mask: false,
-    match: /^[A-Za-z0-9\-_ \.]{1,100}$/,
+    match: /^[A-Za-z0-9\-_,'".\s]{1,1000}$/,
     unmask: [],
     validationError:
       'Please enter a response of no more than 100 alphanumeric characters',
   },
   defaultTextAreaField: {
     mask: false,
-    match: /^[A-Za-z0-9\-_ \.]{1,1000}$/,
+    match: /^[A-Za-z0-9\-_,'".\s]{1,1000}$/,
     unmask: [],
     validationError:
       'Please enter a response of no more than 1000 alphanumeric characters',
@@ -94,7 +100,7 @@ export default {
   },
   portfolioName: {
     mask: false,
-    match: /^.{4,100}$/,
+    match: /^[A-Za-z0-9\-_,'".\s]{4,100}$$/,
     unmask: [],
     validationError: 'Portfolio names can be between 4-100 characters',
   },

styles/core/_util.scss

@@ -98,3 +98,7 @@ hr {
 .usa-section {
   padding: 0;
 }
+
+.form {
+  margin-bottom: $action-footer-height + $large-spacing;
+}

styles/core/_variables.scss

@@ -20,6 +20,7 @@ $max-panel-width: 90rem;
 $home-pg-icon-width: 6rem;
 $large-spacing: 4rem;
 $max-page-width: $max-panel-width + $sidenav-expanded-width + $large-spacing;
+$action-footer-height: 6rem;
 
 /*
  * USWDS Variables

styles/elements/_action_group.scss

@@ -42,6 +42,7 @@
   border-top: 1px solid $color-gray-lighter;
   z-index: 1;
   width: 100%;
+  height: $action-footer-height;
 
   &.action-group-footer--expand-offset {
     padding-left: $sidenav-expanded-width;

styles/elements/_inputs.scss

@@ -228,6 +228,7 @@
 
   &--validation {
     &--anything,
+    &--applicationName,
     &--portfolioName,
     &--requiredField,
     &--defaultStringField,

styles/sections/_task_order.scss

@@ -1,6 +1,5 @@
 .task-order {
   margin-top: $gap * 4;
-  margin-bottom: $footer-height;
   width: 900px;
 
   &__amount {

templates/applications/fragments/environments.html

@@ -47,7 +47,7 @@
                         <span>
                           {{ env['name'] }}
                         </span>
-                        {{ Label(type="pending_creation", classes='label--below')}}
+                        {{ Label(type="pending_creation")}}
                       {%- endif %}
                       {% if user_can(permissions.EDIT_ENVIRONMENT) -%}
                         {{

templates/applications/fragments/members.html

@@ -14,7 +14,7 @@
   action_new,
   action_update) %}
 
-  <h3  id="application-members">
+  <h3 id="application-members">
     {{ 'portfolios.applications.settings.team_members' | translate }}
   </h3>
 
@@ -22,7 +22,7 @@
       {% include "fragments/flash.html" %}
   {% endif %}
 
-  <div class="panel">
+  <div class="panel{% if new_member_form%} form{% endif %}">
   {% if not application.members %}
     <div class='empty-state empty-state--centered empty-state--white panel__content'>
       <p class='empty-state__message'>

templates/applications/new/step_1.html

@@ -22,11 +22,11 @@
   {% include "fragments/flash.html" %}
 
   <base-form inline-template :enable-save="true">
-    <form method="POST" action="{{ action }}" v-on:submit="handleSubmit">
+    <form method="POST" action="{{ action }}" v-on:submit="handleSubmit" class="form">
       {{ form.csrf_token }}
       <div class="form-row">
         <div class="form-col">
-          {{ TextInput(form.name, validation="name", optional=False) }}
+          {{ TextInput(form.name, validation="applicationName", optional=False) }}
           {{ ('portfolios.applications.new.step_1_form_help_text.name' | translate | safe) }}
         </div>
       </div>

templates/applications/new/step_2.html

@@ -21,7 +21,7 @@
   </p>
   <hr>
   <application-environments inline-template v-bind:initial-data='{{ form.data|tojson }}'>
-    <form method="POST" action="{{ url_for('applications.update_new_application_step_2', portfolio_id=portfolio.id, application_id=application.id) }}" v-on:submit="handleSubmit">
+    <form method="POST" action="{{ url_for('applications.update_new_application_step_2', portfolio_id=portfolio.id, application_id=application.id) }}" v-on:submit="handleSubmit" class="form">
       <div class="subheading">{{ 'portfolios.applications.environments_heading' | translate }}</div>
       <div class="panel">
         <div class="panel__content">

templates/applications/settings.html

@@ -22,7 +22,7 @@
     <base-form inline-template>
       <form method="POST" action="{{ url_for('applications.update', application_id=application.id) }}" class="col col--half">
         {{ application_form.csrf_token }}
-        {{ TextInput(application_form.name, validation="name", optional=False) }}
+        {{ TextInput(application_form.name, validation="applicationName", optional=False) }}
         {{ TextInput(application_form.description, validation="defaultTextAreaField", paragraph=True, optional=True, showOptional=False) }}
         <div class="action-group action-group--tight">
           {{ SaveButton(text='common.save_changes'|translate) }}

templates/components/upload_input.html

@@ -5,7 +5,7 @@
   inline-template
   {% if not field.errors %}
     v-bind:filename='{{ field.filename.data | tojson }}'
-    v-bind:object-name='{{ field.object_name.data | tojson }}'
+    v-bind:initial-object-name='{{ field.object_name.data | tojson }}'
   {% else %}
     v-bind:initial-errors='true'
   {% endif %}
@@ -46,7 +46,7 @@
           v-bind:value="attachment"
           type="file">
           <input type="hidden" name="{{ field.filename.name }}" id="{{ field.filename.name }}" ref="attachmentFilename">
-          <input type="hidden" name="{{ field.object_name.name }}" id="{{ field.object_name.name }}" ref="attachmentObjectName">
+          <input type="hidden" name="{{ field.object_name.name }}" id="{{ field.object_name.name }}" ref="attachmentObjectName" v-bind:value='objectName'>
       </div>
       <template v-if="uploadError">
         <span class="usa-input__message">{{ "forms.task_order.upload_error" | translate }}</span>

templates/portfolios/new/step_1.html

@@ -19,11 +19,11 @@
   {{ StickyCTA(text="portfolios.new.cta_step_1"|translate, context=("portfolios.new.sticky_header_context"|translate({"step": "1"}) )) }}
   <base-form inline-template>
     <div class="row">
-      <form id="portfolio-create" class="col" action="{{ url_for('portfolios.create_portfolio') }}" method="POST">
+      <form id="portfolio-create" class="col form" action="{{ url_for('portfolios.create_portfolio') }}" method="POST">
         {{ form.csrf_token }}
         <div class="form-row form-row--bordered">
           <div class="form-col">
-            {{ TextInput(form.name, validation="name", optional=False, classes="form-col") }}
+            {{ TextInput(form.name, validation="portfolioName", optional=False, classes="form-col") }}
             {{"forms.portfolio.name.help_text" | translate | safe }}
           </div>
         </div>

templates/task_orders/builder_base.html

@@ -28,7 +28,7 @@
 
       {% include "fragments/flash.html" %}
 
-      <div class="task-order">
+      <div class="task-order form">
         {% block to_builder_form_field %}{% endblock %}
       </div>
       <div

tests/domain/test_task_orders.py

@@ -1,5 +1,5 @@
 import pytest
-from datetime import date, timedelta
+from datetime import date, datetime, timedelta
 from decimal import Decimal
 
 from atst.domain.exceptions import AlreadyExistsError
@@ -178,3 +178,21 @@ def test_allows_alphanumeric_number():
 
     for number in valid_to_numbers:
         assert TaskOrders.create(portfolio.id, number, [], None)
+
+
+def test_get_for_send_task_order_files():
+    new_to = TaskOrderFactory.create(create_clins=[{}])
+    updated_to = TaskOrderFactory.create(
+        create_clins=[{"last_sent_at": datetime(2020, 2, 1)}],
+        pdf_last_sent_at=datetime(2020, 1, 1),
+    )
+    sent_to = TaskOrderFactory.create(
+        create_clins=[{"last_sent_at": datetime(2020, 1, 1)}],
+        pdf_last_sent_at=datetime(2020, 1, 1),
+    )
+
+    updated_and_new_task_orders = TaskOrders.get_for_send_task_order_files()
+    assert len(updated_and_new_task_orders) == 2
+    assert sent_to not in updated_and_new_task_orders
+    assert updated_to in updated_and_new_task_orders
+    assert new_to in updated_and_new_task_orders

tests/factories.py

@@ -322,6 +322,7 @@ class TaskOrderFactory(Base):
     number = factory.LazyFunction(random_task_order_number)
     signed_at = None
     _pdf = factory.SubFactory(AttachmentFactory)
+    pdf_last_sent_at = None
 
     @classmethod
     def _create(cls, model_class, *args, **kwargs):
@@ -347,6 +348,7 @@ class CLINFactory(Base):
     jedi_clin_type = factory.LazyFunction(
         lambda *args: random.choice(list(clin.JEDICLINType))
     )
+    last_sent_at = None
 
 
 class NotificationRecipientFactory(Base):

tests/test_jobs.py

@@ -2,6 +2,8 @@ import pendulum
 import pytest
 from uuid import uuid4
 from unittest.mock import Mock, MagicMock
+from smtplib import SMTPException
+from azure.core.exceptions import AzureError
 
 from atst.domain.csp.cloud import MockCloudProvider
 from atst.domain.csp.cloud.models import UserRoleCSPResult
@@ -15,6 +17,7 @@ from atst.jobs import (
     dispatch_create_user,
     dispatch_create_environment_role,
     dispatch_provision_portfolio,
+    dispatch_send_task_order_files,
     create_environment,
     do_create_user,
     do_provision_portfolio,
@@ -23,15 +26,17 @@ from atst.jobs import (
     do_create_application,
 )
 from tests.factories import (
+    ApplicationFactory,
+    ApplicationRoleFactory,
     EnvironmentFactory,
     EnvironmentRoleFactory,
     PortfolioFactory,
     PortfolioStateMachineFactory,
-    ApplicationFactory,
-    ApplicationRoleFactory,
+    TaskOrderFactory,
     UserFactory,
 )
 from atst.models import CSPRole, EnvironmentRole, ApplicationRoleStatus, JobFailure
+from atst.utils.localization import translate
 
 
 @pytest.fixture(autouse=True, scope="function")
@@ -326,3 +331,84 @@ def test_create_environment_role():
     do_create_environment_role(csp, environment_role_id=env_role.id)
 
     assert env_role.cloud_id == "a-cloud-id"
+
+
+# TODO: Refactor the tests related to dispatch_send_task_order_files() into a class
+# and separate the success test into two tests
+def test_dispatch_send_task_order_files(monkeypatch, app):
+    mock = Mock()
+    monkeypatch.setattr("atst.jobs.send_mail", mock)
+
+    def _download_task_order(MockFileService, object_name):
+        return {"name": object_name}
+
+    monkeypatch.setattr(
+        "atst.domain.csp.files.MockFileService.download_task_order",
+        _download_task_order,
+    )
+
+    # Create 3 new Task Orders
+    for i in range(3):
+        TaskOrderFactory.create(create_clins=[{"number": "0001"}])
+
+    dispatch_send_task_order_files.run()
+
+    # Check that send_with_attachment was called once for each task order
+    assert mock.call_count == 3
+    mock.reset_mock()
+
+    # Create new TO
+    task_order = TaskOrderFactory.create(create_clins=[{"number": "0001"}])
+    assert not task_order.pdf_last_sent_at
+
+    dispatch_send_task_order_files.run()
+
+    # Check that send_with_attachment was called with correct kwargs
+    mock.assert_called_once_with(
+        recipients=[app.config.get("MICROSOFT_TASK_ORDER_EMAIL_ADDRESS")],
+        subject=translate(
+            "email.task_order_sent.subject", {"to_number": task_order.number}
+        ),
+        body=translate("email.task_order_sent.body", {"to_number": task_order.number}),
+        attachments=[
+            {
+                "name": task_order.pdf.object_name,
+                "maintype": "application",
+                "subtype": "pdf",
+            }
+        ],
+    )
+
+    assert task_order.pdf_last_sent_at
+
+
+def test_dispatch_send_task_order_files_send_failure(monkeypatch):
+    def _raise_smtp_exception(**kwargs):
+        raise SMTPException
+
+    monkeypatch.setattr("atst.jobs.send_mail", _raise_smtp_exception)
+
+    task_order = TaskOrderFactory.create(create_clins=[{"number": "0001"}])
+    dispatch_send_task_order_files.run()
+
+    # Check that pdf_last_sent_at has not been updated
+    assert not task_order.pdf_last_sent_at
+
+
+def test_dispatch_send_task_order_files_download_failure(monkeypatch):
+    mock = Mock()
+    monkeypatch.setattr("atst.jobs.send_mail", mock)
+
+    def _download_task_order(MockFileService, object_name):
+        raise AzureError("something went wrong")
+
+    monkeypatch.setattr(
+        "atst.domain.csp.files.MockFileService.download_task_order",
+        _download_task_order,
+    )
+
+    task_order = TaskOrderFactory.create(create_clins=[{"number": "0002"}])
+    dispatch_send_task_order_files.run()
+
+    # Check that pdf_last_sent_at has not been updated
+    assert not task_order.pdf_last_sent_at