Merge pull request #238 from dod-ccpo/request-schema-#159719829

Request schema #159719829

atst/domain/date.py

@@ -1,12 +0,0 @@
-import pendulum
-
-
-def parse_date(data):
-    date_formats = ["YYYY-MM-DD", "MM/DD/YYYY"]
-    for _format in date_formats:
-        try:
-            return pendulum.from_format(data, _format).date()
-        except (ValueError, pendulum.parsing.exceptions.ParserError):
-            pass
-
-    raise ValueError("Unable to parse string {}".format(data))

atst/domain/requests.py

@@ -2,35 +2,31 @@ from enum import Enum
 from sqlalchemy import exists, and_, exc
 from sqlalchemy.sql import text
 from sqlalchemy.orm.exc import NoResultFound
-from sqlalchemy.orm.attributes import flag_modified
 from werkzeug.datastructures import FileStorage
+import dateutil
 
 from atst.database import db
 from atst.domain.authz import Authorization
 from atst.domain.task_orders import TaskOrders
 from atst.domain.workspaces import Workspaces
 from atst.models.request import Request
+from atst.models.request_revision import RequestRevision
 from atst.models.request_status_event import RequestStatusEvent, RequestStatus
+from atst.utils import deep_merge
 
 from .exceptions import NotFoundError, UnauthorizedError
 
 
-def deep_merge(source, destination: dict):
-    """
-    Merge source dict into destination dict recursively.
-    """
-
-    def _deep_merge(a, b):
-        for key, value in a.items():
-            if isinstance(value, dict):
-                node = b.setdefault(key, {})
-                _deep_merge(value, node)
-            else:
-                b[key] = value
-
-        return b
-
-    return _deep_merge(source, dict(destination))
+def create_revision_from_request_body(body):
+    body = {k: v for p in body.values() for k, v in p.items()}
+    DATES = ["start_date", "date_latest_training"]
+    coerced_timestamps = {
+        k: dateutil.parser.parse(v)
+        for k, v in body.items()
+        if k in DATES and isinstance(v, str)
+    }
+    body = {**body, **coerced_timestamps}
+    return RequestRevision(**body)
 
 
 class Requests(object):
@@ -39,7 +35,8 @@ class Requests(object):
 
     @classmethod
     def create(cls, creator, body):
-        request = Request(creator=creator, body=body)
+        revision = create_revision_from_request_body(body)
+        request = Request(creator=creator, revisions=[revision])
         request = Requests.set_status(request, RequestStatus.STARTED)
 
         db.session.add(request)
@@ -105,7 +102,10 @@ class Requests(object):
     @classmethod
     def update(cls, request_id, request_delta):
         request = Requests._get_with_lock(request_id)
-        request = Requests._merge_body(request, request_delta)
+
+        new_body = deep_merge(request_delta, request.body)
+        revision = create_revision_from_request_body(new_body)
+        request.revisions.append(revision)
 
         db.session.add(request)
         db.session.commit()
@@ -127,16 +127,6 @@ class Requests(object):
         except NoResultFound:
             raise NotFoundError()
 
-    @classmethod
-    def _merge_body(cls, request, request_delta):
-        request.body = deep_merge(request_delta, request.body)
-
-        # Without this, sqlalchemy won't notice the change to request.body,
-        # since it doesn't track dictionary mutations by default.
-        flag_modified(request, "body")
-
-        return request
-
     @classmethod
     def approve_and_create_workspace(cls, request):
         approved_request = Requests.set_status(request, RequestStatus.APPROVED)
@@ -149,7 +139,9 @@ class Requests(object):
 
     @classmethod
     def set_status(cls, request: Request, status: RequestStatus):
-        status_event = RequestStatusEvent(new_status=status)
+        status_event = RequestStatusEvent(
+            new_status=status, revision=request.latest_revision
+        )
         request.status_events.append(status_event)
         return request
 
@@ -256,12 +248,7 @@ WHERE requests_with_status.status = :status
         if task_order:
             request.task_order = task_order
 
-        request = Requests._merge_body(
-            request, {"financial_verification": request_data}
-        )
-
-        db.session.add(request)
-        db.session.commit()
+        request = Requests.update(request.id, {"financial_verification": request_data})
 
         return request
 

atst/forms/fields.py

@@ -1,23 +1,6 @@
-from wtforms.fields.html5 import DateField
 from wtforms.fields import Field, SelectField as SelectField_
 from wtforms.widgets import TextArea
 
-from atst.domain.date import parse_date
-
-
-class DateField(DateField):
-    def _value(self):
-        if self.data:
-            return parse_date(self.data)
-        else:
-            return None
-
-    def process_formdata(self, values):
-        if values:
-            self.data = values[0]
-        else:
-            self.data = []
-
 
 class NewlineListField(Field):
     widget = TextArea()

atst/forms/org.py

@@ -1,9 +1,9 @@
-from wtforms.fields.html5 import EmailField, TelField
+from wtforms.fields.html5 import DateField, EmailField, TelField
 from wtforms.fields import RadioField, StringField
 from wtforms.validators import Required, Email
 import pendulum
 
-from .fields import DateField, SelectField
+from .fields import SelectField
 from .forms import ValidatedForm
 from .validators import DateRange, PhoneNumber, Alphabet
 from .data import SERVICE_BRANCHES
@@ -60,4 +60,5 @@ class OrgForm(ValidatedForm):
                 message="Must be a date within the last year.",
             ),
         ],
+        format="%m/%d/%Y",
     )

atst/forms/request.py

@@ -1,8 +1,8 @@
-from wtforms.fields.html5 import IntegerField
+from wtforms.fields.html5 import DateField, IntegerField
 from wtforms.fields import RadioField, TextAreaField
 from wtforms.validators import Optional, Required
 
-from .fields import DateField, SelectField
+from .fields import SelectField
 from .forms import ValidatedForm
 from .data import (
     SERVICE_BRANCHES,
@@ -135,4 +135,5 @@ class RequestForm(ValidatedForm):
     start_date = DateField(
         description="When do you expect to start using the JEDI Cloud (not for billing purposes)?",
         validators=[Required()],
+        format="%m/%d/%Y",
     )

atst/forms/validators.py

@@ -2,13 +2,11 @@ import re
 from wtforms.validators import ValidationError
 import pendulum
 
-from atst.domain.date import parse_date
-
 
 def DateRange(lower_bound=None, upper_bound=None, message=None):
     def _date_range(form, field):
         now = pendulum.now().date()
-        date = parse_date(field.data)
+        date = field.data
 
         if lower_bound is not None:
             if (now - lower_bound) > date:

atst/models/__init__.py

@@ -14,3 +14,4 @@ from .workspace import Workspace
 from .project import Project
 from .environment import Environment
 from .attachment import Attachment
+from .request_revision import RequestRevision

atst/models/request.py

@@ -1,6 +1,5 @@
 from sqlalchemy import Column, func, ForeignKey
 from sqlalchemy.types import DateTime
-from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.orm import relationship
 import pendulum
 
@@ -8,6 +7,23 @@ from atst.models import Base
 from atst.models.types import Id
 from atst.models.request_status_event import RequestStatus
 from atst.utils import first_or_none
+from atst.models.request_revision import RequestRevision
+
+
+def map_properties_to_dict(properties, instance):
+    return {
+        field: getattr(instance, field)
+        for field in properties
+        if getattr(instance, field) is not None
+    }
+
+
+def update_dict_with_properties(instance, body, top_level_key, properties):
+    new_properties = map_properties_to_dict(properties, instance)
+    if new_properties:
+        body[top_level_key] = new_properties
+
+    return body
 
 
 class Request(Base):
@@ -15,7 +31,6 @@ class Request(Base):
 
     id = Id()
     time_created = Column(DateTime(timezone=True), server_default=func.now())
-    body = Column(JSONB)
     status_events = relationship(
         "RequestStatusEvent", backref="request", order_by="RequestStatusEvent.sequence"
     )
@@ -28,6 +43,78 @@ class Request(Base):
     task_order_id = Column(ForeignKey("task_order.id"))
     task_order = relationship("TaskOrder")
 
+    revisions = relationship(
+        "RequestRevision", back_populates="request", order_by="RequestRevision.sequence"
+    )
+
+    @property
+    def latest_revision(self):
+        if self.revisions:
+            return self.revisions[-1]
+
+        else:
+            return RequestRevision(request=self)
+
+    PRIMARY_POC_FIELDS = ["am_poc", "dodid_poc", "email_poc", "fname_poc", "lname_poc"]
+    DETAILS_OF_USE_FIELDS = [
+        "jedi_usage",
+        "start_date",
+        "cloud_native",
+        "dollar_value",
+        "dod_component",
+        "data_transfers",
+        "expected_completion_date",
+        "jedi_migration",
+        "num_software_systems",
+        "number_user_sessions",
+        "average_daily_traffic",
+        "engineering_assessment",
+        "technical_support_team",
+        "estimated_monthly_spend",
+        "average_daily_traffic_gb",
+        "rationalization_software_systems",
+        "organization_providing_assistance",
+    ]
+    INFORMATION_ABOUT_YOU_FIELDS = [
+        "citizenship",
+        "designation",
+        "phone_number",
+        "email_request",
+        "fname_request",
+        "lname_request",
+        "service_branch",
+        "date_latest_training",
+    ]
+    FINANCIAL_VERIFICATION_FIELDS = [
+        "pe_id",
+        "task_order_number",
+        "fname_co",
+        "lname_co",
+        "email_co",
+        "office_co",
+        "fname_cor",
+        "lname_cor",
+        "email_cor",
+        "office_cor",
+        "uii_ids",
+        "treasury_code",
+        "ba_code",
+    ]
+
+    @property
+    def body(self):
+        current = self.latest_revision
+        body = {}
+        for top_level_key, properties in [
+            ("primary_poc", Request.PRIMARY_POC_FIELDS),
+            ("details_of_use", Request.DETAILS_OF_USE_FIELDS),
+            ("information_about_you", Request.INFORMATION_ABOUT_YOU_FIELDS),
+            ("financial_verification", Request.FINANCIAL_VERIFICATION_FIELDS),
+        ]:
+            body = update_dict_with_properties(current, body, top_level_key, properties)
+
+        return body
+
     @property
     def status(self):
         return self.status_events[-1].new_status
@@ -38,7 +125,7 @@ class Request(Base):
 
     @property
     def annual_spend(self):
-        monthly = self.body.get("details_of_use", {}).get("estimated_monthly_spend", 0)
+        monthly = self.latest_revision.estimated_monthly_spend or 0
         return monthly * 12
 
     @property

atst/models/request_revision.py

@@ -0,0 +1,78 @@
+from sqlalchemy import (
+    Column,
+    ForeignKey,
+    String,
+    Boolean,
+    Integer,
+    Date,
+    BigInteger,
+    Sequence,
+)
+from sqlalchemy.orm import relationship
+from sqlalchemy.dialects.postgresql import ARRAY
+
+from atst.models import Base
+from atst.models.mixins import TimestampsMixin
+from atst.models.types import Id
+
+
+class RequestRevision(Base, TimestampsMixin):
+    __tablename__ = "request_revisions"
+
+    id = Id()
+    request_id = Column(ForeignKey("requests.id"), nullable=False)
+    request = relationship("Request", back_populates="revisions")
+    sequence = Column(
+        BigInteger, Sequence("request_revisions_sequence_seq"), nullable=False
+    )
+
+    # primary_poc
+    am_poc = Column(Boolean, default=False)
+    dodid_poc = Column(String)
+    email_poc = Column(String)
+    fname_poc = Column(String)
+    lname_poc = Column(String)
+
+    # details_of_use
+    jedi_usage = Column(String)
+    start_date = Column(Date())
+    cloud_native = Column(String)
+    dollar_value = Column(Integer)
+    dod_component = Column(String)
+    data_transfers = Column(String)
+    expected_completion_date = Column(String)
+    jedi_migration = Column(String)
+    num_software_systems = Column(Integer)
+    number_user_sessions = Column(Integer)
+    average_daily_traffic = Column(Integer)
+    engineering_assessment = Column(String)
+    technical_support_team = Column(String)
+    estimated_monthly_spend = Column(Integer)
+    average_daily_traffic_gb = Column(Integer)
+    rationalization_software_systems = Column(String)
+    organization_providing_assistance = Column(String)
+
+    # information_about_you
+    citizenship = Column(String)
+    designation = Column(String)
+    phone_number = Column(String)
+    email_request = Column(String)
+    fname_request = Column(String)
+    lname_request = Column(String)
+    service_branch = Column(String)
+    date_latest_training = Column(Date())
+
+    # financial_verification
+    pe_id = Column(String)
+    task_order_number = Column(String)
+    fname_co = Column(String)
+    lname_co = Column(String)
+    email_co = Column(String)
+    office_co = Column(String)
+    fname_cor = Column(String)
+    lname_cor = Column(String)
+    email_cor = Column(String)
+    office_cor = Column(String)
+    uii_ids = Column(ARRAY(String))
+    treasury_code = Column(String)
+    ba_code = Column(String)

atst/models/request_status_event.py

@@ -1,5 +1,6 @@
 from enum import Enum
 from sqlalchemy import Column, func, ForeignKey, Enum as SQLAEnum
+from sqlalchemy.orm import relationship
 from sqlalchemy.types import DateTime, BigInteger
 from sqlalchemy.schema import Sequence
 from sqlalchemy.dialects.postgresql import UUID
@@ -31,6 +32,8 @@ class RequestStatusEvent(Base):
     sequence = Column(
         BigInteger, Sequence("request_status_events_sequence_seq"), nullable=False
     )
+    request_revision_id = Column(ForeignKey("request_revisions.id"), nullable=False)
+    revision = relationship("RequestRevision")
 
     @property
     def displayname(self):

atst/utils.py

@@ -1,2 +1,20 @@
 def first_or_none(predicate, lst):
     return next((x for x in lst if predicate(x)), None)
+
+
+def deep_merge(source, destination: dict):
+    """
+    Merge source dict into destination dict recursively.
+    """
+
+    def _deep_merge(a, b):
+        for key, value in a.items():
+            if isinstance(value, dict):
+                node = b.setdefault(key, {})
+                _deep_merge(value, node)
+            else:
+                b[key] = value
+
+        return b
+
+    return _deep_merge(source, dict(destination))