From 7ace430deea207fa303167b1da6e94ef4c01161b Mon Sep 17 00:00:00 2001 From: richard-dds Date: Tue, 12 Nov 2019 12:53:17 -0500 Subject: [PATCH 01/12] Use url_for for all static paths --- package.json | 1 + templates/about.html | 3 ++- templates/base.html | 2 +- templates/base_public.html | 2 +- templates/error_base.html | 2 +- templates/help/docs/getting-started.html | 2 +- templates/home.html | 2 +- 7 files changed, 8 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index 3b7f50c4..8521623c 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ "scripts": { "watch": "parcel watch js/index.js -d static/assets --public-url /static/assets -o index.js --no-autoinstall", "build": "parcel build js/index.js -d static/assets --public-url /static/assets -o index.js", + "build-prod": "parcel build js/index.js -d static/assets --public-url $CDN_URL -o index.js", "test": "jest", "test:coverage": "jest --coverage --collectCoverageFrom='js/**/*.js'", "test:watch": "jest --watch --no-cache" diff --git a/templates/about.html b/templates/about.html index b7ebce79..6e24b662 100644 --- a/templates/about.html +++ b/templates/about.html @@ -6,7 +6,8 @@
diff --git a/templates/base.html b/templates/base.html index c06a05fb..fabcb92c 100644 --- a/templates/base.html +++ b/templates/base.html @@ -8,7 +8,7 @@ {% assets "css" %} {% endassets %} - + diff --git a/templates/base_public.html b/templates/base_public.html index 248c302a..7b872471 100644 --- a/templates/base_public.html +++ b/templates/base_public.html @@ -10,7 +10,7 @@ {% assets "css" %} {% endassets %} - + diff --git a/templates/error_base.html b/templates/error_base.html index 253a62f8..92be8e60 100644 --- a/templates/error_base.html +++ b/templates/error_base.html @@ -7,7 +7,7 @@ {% assets "css" %} {% endassets %} - + diff --git a/templates/help/docs/getting-started.html b/templates/help/docs/getting-started.html index deb9b935..bf32e037 100644 --- a/templates/help/docs/getting-started.html +++ b/templates/help/docs/getting-started.html @@ -132,6 +132,6 @@

Here’s an example:
Application A has a development environment, production environment, and sandbox environment. The cloud resources in the development environment are grouped and accessed separately from the production environment and sandbox environment.

-AT-AT FAQs Content +AT-AT FAQs Content {% endblock %} diff --git a/templates/home.html b/templates/home.html index bcfc5c04..91774138 100644 --- a/templates/home.html +++ b/templates/home.html @@ -94,7 +94,7 @@
- JEDI heirarchy diagram + JEDI heirarchy diagram
From c10aedd4dc432f6069fa668e108c352c10a96bd5 Mon Sep 17 00:00:00 2001 From: richard-dds Date: Wed, 13 Nov 2019 16:58:28 -0500 Subject: [PATCH 02/12] Override flask's url_for --- atst/app.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/atst/app.py b/atst/app.py index 0ba56929..3aed098f 100644 --- a/atst/app.py +++ b/atst/app.py @@ -2,11 +2,12 @@ import os import re from configparser import ConfigParser from datetime import datetime -from flask import Flask, request, g, session +from flask import Flask, request, g, session, url_for as flask_url_for from flask_session import Session import redis from unipath import Path from flask_wtf.csrf import CSRFProtect +from urllib.parse import urljoin from atst.database import db from atst.assets import environment as assets_environment @@ -63,6 +64,7 @@ def make_app(config): make_flask_callbacks(app) register_filters(app) + register_jinja_globals(app) make_csp_provider(app, config.get("CSP", "mock")) make_crl_validator(app) make_mailer(app) @@ -293,3 +295,16 @@ def apply_json_logger(): "root": {"level": "INFO", "handlers": ["wsgi"]}, } ) + +def register_jinja_globals(app): + static_url = app.config.get("STATIC_URL", "/static/") + + def _url_for(endpoint, **values): + if endpoint == "static": + filename = values["filename"] + return urljoin(static_url, filename) + else: + return flask_url_for(endpoint, **values) + + + app.jinja_env.globals["url_for"] = _url_for From 4c5a4f29b8e0078b608d41917cc87a70e8837e0c Mon Sep 17 00:00:00 2001 From: richard-dds Date: Wed, 13 Nov 2019 17:01:06 -0500 Subject: [PATCH 03/12] Allow static_url domain in prod --- atst/app.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/atst/app.py b/atst/app.py index 3aed098f..22acbb31 100644 --- a/atst/app.py +++ b/atst/app.py @@ -127,6 +127,8 @@ def make_flask_callbacks(app): def set_default_headers(app): # pragma: no cover + static_url = app.config.get("STATIC_URL") + @app.after_request def _set_security_headers(response): response.headers[ @@ -143,7 +145,7 @@ def set_default_headers(app): # pragma: no cover else: response.headers[ "Content-Security-Policy" - ] = "default-src 'self' 'unsafe-eval' 'unsafe-inline'" + ] = f"default-src 'self' 'unsafe-eval' 'unsafe-inline' {static_url}" return response From 5e9a3f0ba685aa2c77d5693e0ac7184cc8e6bb28 Mon Sep 17 00:00:00 2001 From: richard-dds Date: Wed, 13 Nov 2019 17:01:37 -0500 Subject: [PATCH 04/12] Add STATIC_URL and ASSET_URL config in base.ini --- .secrets.baseline | 9 ++++++++- config/base.ini | 8 +++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index 9353d013..b943273c 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -98,7 +98,14 @@ "hashed_secret": "afc848c316af1a89d49826c5ae9d00ed769415f3", "is_secret": false, "is_verified": false, - "line_number": 20, + "line_number": 22, + "type": "Secret Keyword" + }, + { + "hashed_secret": "abcdb568713c255c81376829da20004ba9463fd3", + "is_secret": false, + "is_verified": false, + "line_number": 30, "type": "Secret Keyword" } ], diff --git a/config/base.ini b/config/base.ini index b68e526c..e0177c55 100644 --- a/config/base.ini +++ b/config/base.ini @@ -1,9 +1,8 @@ [default] -USE_AUDIT_LOG = false +ASSETS_URL CAC_URL = http://localhost:8000/login-redirect CA_CHAIN = ssl/server-certs/ca-chain.pem CLASSIFIED = false -CONTRACT_START_DATE = 2019-09-14 CONTRACT_END_DATE = 2022-09-14 CELERY_DEFAULT_QUEUE=celery DISABLE_CRL_CHECK = false @@ -11,7 +10,9 @@ CRL_FAIL_OPEN = false CRL_STORAGE_CONTAINER = crls CSP=mock DEBUG = true +DISABLE_CRL_CHECK = false ENVIRONMENT = dev +LIMIT_CONCURRENT_SESSIONS = false LOG_JSON = false PERMANENT_SESSION_LIFETIME = 1800 PE_NUMBER_CSV_URL = http://c95e1ebb198426ee57b8-174bb05a294821bedbf46b6384fe9b1f.r31.cf5.rackcdn.com/penumbers.csv @@ -32,5 +33,6 @@ SESSION_COOKIE_NAME=atat SESSION_TYPE = redis SESSION_USE_SIGNER = True SQLALCHEMY_ECHO = False +STATIC_URL=/static/ +USE_AUDIT_LOG = false WTF_CSRF_ENABLED = true -LIMIT_CONCURRENT_SESSIONS = false From 8ddd9fab96f85cb601ac9a75953695931e049434 Mon Sep 17 00:00:00 2001 From: richard-dds Date: Wed, 13 Nov 2019 17:02:01 -0500 Subject: [PATCH 05/12] Add docker arg for CDN_URL --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index bf9b26be..2d78b0fc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,7 @@ FROM python:3.7.3-alpine3.9 AS builder ARG CSP +ARG CDN_URL=/static/assets/ ENV TZ UTC RUN mkdir -p /install/.venv From 4c693cf9c0025a2d4e0f9ab8f5bfbdcd2b65f609 Mon Sep 17 00:00:00 2001 From: richard-dds Date: Thu, 14 Nov 2019 16:19:57 -0500 Subject: [PATCH 06/12] Fix bad rebase --- .secrets.baseline | 9 +-------- config/base.ini | 5 +++-- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index b943273c..dd2428f1 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -98,14 +98,7 @@ "hashed_secret": "afc848c316af1a89d49826c5ae9d00ed769415f3", "is_secret": false, "is_verified": false, - "line_number": 22, - "type": "Secret Keyword" - }, - { - "hashed_secret": "abcdb568713c255c81376829da20004ba9463fd3", - "is_secret": false, - "is_verified": false, - "line_number": 30, + "line_number": 21, "type": "Secret Keyword" } ], diff --git a/config/base.ini b/config/base.ini index e0177c55..9812e433 100644 --- a/config/base.ini +++ b/config/base.ini @@ -2,15 +2,16 @@ ASSETS_URL CAC_URL = http://localhost:8000/login-redirect CA_CHAIN = ssl/server-certs/ca-chain.pem +CELERY_DEFAULT_QUEUE=celery CLASSIFIED = false CONTRACT_END_DATE = 2022-09-14 -CELERY_DEFAULT_QUEUE=celery -DISABLE_CRL_CHECK = false +CONTRACT_START_DATE = 2019-09-14 CRL_FAIL_OPEN = false CRL_STORAGE_CONTAINER = crls CSP=mock DEBUG = true DISABLE_CRL_CHECK = false +DISABLE_CRL_CHECK = false ENVIRONMENT = dev LIMIT_CONCURRENT_SESSIONS = false LOG_JSON = false From 5ca20222fc0e22fed967ff92ff2961df085d5af0 Mon Sep 17 00:00:00 2001 From: richard-dds Date: Thu, 14 Nov 2019 16:23:54 -0500 Subject: [PATCH 07/12] Run build-prod in CI --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 2d78b0fc..e11a8723 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,7 +42,7 @@ RUN ./script/write_dotenv && \ yarn install && \ rm -r ./static/fonts/ &> /dev/null || true && \ cp -rf ./node_modules/uswds/src/fonts ./static/ && \ - yarn build + yarn build-prod ## NEW IMAGE FROM python:3.7.3-alpine3.9 From e29163f65a3b62e16cf3a1ec93d5147042dea409 Mon Sep 17 00:00:00 2001 From: richard-dds Date: Thu, 14 Nov 2019 16:44:37 -0500 Subject: [PATCH 08/12] Add CDN config for prod --- deploy/azure/atst-envvars-configmap.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/deploy/azure/atst-envvars-configmap.yml b/deploy/azure/atst-envvars-configmap.yml index b521e8d2..61f81fa8 100644 --- a/deploy/azure/atst-envvars-configmap.yml +++ b/deploy/azure/atst-envvars-configmap.yml @@ -5,12 +5,14 @@ metadata: name: atst-envvars namespace: atat data: - CSP: azure + ASSETS_URL: https://atat-cdn.azureedge.net/ CELERY_DEFAULT_QUEUE: celery-master + CSP: azure FLASK_ENV: dev LOG_JSON: "true" OVERRIDE_CONFIG_FULLPATH: /opt/atat/atst/atst-overrides.ini PGSSLMODE: verify-full PGSSLROOTCERT: /opt/atat/atst/ssl/pgsslrootcert.crt + STATIC_URL: https://atat-cdn.azureedge.net/static/ TZ: UTC UWSGI_CONFIG_FULLPATH: /opt/atat/atst/uwsgi.ini From 8e12c6bfbdd214590d378012d1560869ee5c96b9 Mon Sep 17 00:00:00 2001 From: richard-dds Date: Thu, 14 Nov 2019 16:44:50 -0500 Subject: [PATCH 09/12] Add CDN config for staging --- deploy/overlays/staging/envvars.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deploy/overlays/staging/envvars.yml b/deploy/overlays/staging/envvars.yml index 64feb666..abbd10c9 100644 --- a/deploy/overlays/staging/envvars.yml +++ b/deploy/overlays/staging/envvars.yml @@ -12,4 +12,6 @@ kind: ConfigMap metadata: name: atst-envvars data: + ASSETS_URL: https://atat-cdn-staging.azureedge.net/ CELERY_DEFAULT_QUEUE: celery-staging + STATIC_URL: https://atat-cdn-staging.azureedge.net/static/ From 293854ece6b1a03f854e016b85e6918738d09b6b Mon Sep 17 00:00:00 2001 From: dandds Date: Thu, 14 Nov 2019 17:06:28 -0500 Subject: [PATCH 10/12] CircleCI config for passing CDN url to the Docker image. The workflows need to split since we're passing different parameters to the Docker build at the beginning of the workflow. --- .circleci/config.yml | 92 +++++++++++++++++++++++++++++++++----------- atst/app.py | 2 +- config/base.ini | 1 - 3 files changed, 71 insertions(+), 24 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 10ed47a8..a962400f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -54,6 +54,22 @@ commands: name: Apply the default permission sets command: docker run --network atat -e PGDATABASE=<< parameters.pgdatabase >> << parameters.container_env >> atat:builder .venv/bin/python script/seed_roles.py + docker-build: + parameters: + cdn_url: + type: string + steps: + - checkout + - setup_remote_docker: + docker_layer_caching: true + version: 18.06.0-ce + - run: + name: Build image + command: | + docker build . --build-arg CSP=azure --build-arg CDN_URL=<< parameters.cdn_url >> -f ./Dockerfile -t atat:builder --target builder + docker build . --build-arg CSP=azure --build-arg CDN_URL=<< parameters.cdn_url >> -f ./Dockerfile -t atat:latest + - cache_docker_image + deploy: parameters: namespace: @@ -124,20 +140,19 @@ commands: kubectl set image cronjobs.batch/crls crls=<< parameters.tag >> --namespace=<< parameters.namespace >> jobs: - docker-build: + docker-build-staging: docker: - image: docker:18.06.0-ce-git steps: - - checkout - - setup_remote_docker: - docker_layer_caching: true - version: 18.06.0-ce - - run: - name: Build image - command: | - docker build . --target builder --build-arg CSP=azure -f ./Dockerfile -t atat:builder - docker build . --build-arg CSP=azure -f ./Dockerfile -t atat:latest - - cache_docker_image + - docker-build: + cdn_url: https://atat-cdn-staging.azureedge.net/ + + docker-build-master: + docker: + - image: docker:18.06.0-ce-git + steps: + - docker-build: + cdn_url: https://atat-cdn.azureedge.net/ test: docker: @@ -194,6 +209,19 @@ jobs: --virtualenv /install/.venv \ --http-socket :8000 " + - run: + name: Wait for ATAT container to be available + command: | + docker pull curlimages/curl:latest + docker run --network atat \ + curlimages/curl:latest \ + curl --connect-timeout 3 \ + --max-time 5 \ + --retry 120 \ + --retry-connrefused \ + --retry-delay 1 \ + --retry-max-time 120 \ + test-atat:8000 - run: name: Execute Ghost Inspector test suite command: | @@ -257,29 +285,49 @@ workflows: version: 2 run-tests: jobs: - - docker-build + - docker-build-staging - test: requires: - - docker-build + - docker-build-staging - integration-tests: requires: - - docker-build - - deploy-staging: - requires: - - test - - integration-tests + - docker-build-staging + + build-staging: + jobs: + - docker-build-staging: filters: branches: only: - staging - - deploy-master: + - test: + requires: + - docker-build-staging + - integration-tests: + requires: + - docker-build-staging + - deploy-staging: requires: - test - integration-tests + + build-master: + jobs: + - docker-build-master: filters: branches: only: - master + - test: + requires: + - docker-build-master + - integration-tests: + requires: + - docker-build-master + - deploy-master: + requires: + - test + - integration-tests test-crl-parser: triggers: @@ -288,9 +336,9 @@ workflows: filters: branches: only: - - master + - staging jobs: - - docker-build + - docker-build-staging - test-crl-parser: requires: - - docker-build + - docker-build-staging diff --git a/atst/app.py b/atst/app.py index 22acbb31..83163a88 100644 --- a/atst/app.py +++ b/atst/app.py @@ -298,6 +298,7 @@ def apply_json_logger(): } ) + def register_jinja_globals(app): static_url = app.config.get("STATIC_URL", "/static/") @@ -308,5 +309,4 @@ def register_jinja_globals(app): else: return flask_url_for(endpoint, **values) - app.jinja_env.globals["url_for"] = _url_for diff --git a/config/base.ini b/config/base.ini index 9812e433..105aeb4c 100644 --- a/config/base.ini +++ b/config/base.ini @@ -11,7 +11,6 @@ CRL_STORAGE_CONTAINER = crls CSP=mock DEBUG = true DISABLE_CRL_CHECK = false -DISABLE_CRL_CHECK = false ENVIRONMENT = dev LIMIT_CONCURRENT_SESSIONS = false LOG_JSON = false From c6187466a3d7ada8b7e08e1a44b4d4b40a258ab3 Mon Sep 17 00:00:00 2001 From: dandds Date: Mon, 18 Nov 2019 11:22:39 -0500 Subject: [PATCH 11/12] Configure staging with different FLASK_ENV, include sub-route for CDN_URL. --- .circleci/config.yml | 4 ++-- deploy/overlays/staging/envvars.yml | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a962400f..1374452f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -145,14 +145,14 @@ jobs: - image: docker:18.06.0-ce-git steps: - docker-build: - cdn_url: https://atat-cdn-staging.azureedge.net/ + cdn_url: https://atat-cdn-staging.azureedge.net/static/assets docker-build-master: docker: - image: docker:18.06.0-ce-git steps: - docker-build: - cdn_url: https://atat-cdn.azureedge.net/ + cdn_url: https://atat-cdn.azureedge.net/static/assets test: docker: diff --git a/deploy/overlays/staging/envvars.yml b/deploy/overlays/staging/envvars.yml index abbd10c9..86ea42b7 100644 --- a/deploy/overlays/staging/envvars.yml +++ b/deploy/overlays/staging/envvars.yml @@ -6,6 +6,7 @@ metadata: data: CELERY_DEFAULT_QUEUE: celery-staging SERVER_NAME: staging.atat.code.mil + FLASK_ENV: staging --- apiVersion: v1 kind: ConfigMap @@ -14,4 +15,5 @@ metadata: data: ASSETS_URL: https://atat-cdn-staging.azureedge.net/ CELERY_DEFAULT_QUEUE: celery-staging + FLASK_ENV: staging STATIC_URL: https://atat-cdn-staging.azureedge.net/static/ From 08fc53022377430051c5e1d2a53a39d7a2e43496 Mon Sep 17 00:00:00 2001 From: dandds Date: Tue, 19 Nov 2019 13:20:20 -0500 Subject: [PATCH 12/12] Add config value for CDN origin. This value is set as the Access-Control-Allow-Origin header value for the application. When using Azure CDN, the CDN will consume this header when it populates its cache and use it on subsequent requests. It would be possible to make this the same as the Flask SERVER_NAME value. We explicitly set SERVER_NAME for Celery worker processes because they need that information to contruct URLs outside of the request cycle (Flask can infer the server name within a request cycle). I decided not to rely on SERVER_NAME though because it has side effects: - It determines what `url_for` uses as the host domain (which would be fine). - It makes it so that the Flask app can only server requests to that domain (probably fine, but it felt like too big a side effect). Additionally, SERVER_NAME does not include the scheme. For all of these reasons I opted to make CDN_ORIGIN a separate config value. --- atst/app.py | 1 + config/base.ini | 1 + deploy/azure/atst-envvars-configmap.yml | 1 + deploy/overlays/staging/envvars.yml | 1 + 4 files changed, 4 insertions(+) diff --git a/atst/app.py b/atst/app.py index 83163a88..346ffe8b 100644 --- a/atst/app.py +++ b/atst/app.py @@ -137,6 +137,7 @@ def set_default_headers(app): # pragma: no cover response.headers["X-Content-Type-Options"] = "nosniff" response.headers["X-Frame-Options"] = "SAMEORIGIN" response.headers["X-XSS-Protection"] = "1; mode=block" + response.headers["Access-Control-Allow-Origin"] = app.config.get("CDN_ORIGIN") if ENV == "dev": response.headers[ diff --git a/config/base.ini b/config/base.ini index 105aeb4c..76e0f03e 100644 --- a/config/base.ini +++ b/config/base.ini @@ -2,6 +2,7 @@ ASSETS_URL CAC_URL = http://localhost:8000/login-redirect CA_CHAIN = ssl/server-certs/ca-chain.pem +CDN_ORIGIN=http://localhost:8000 CELERY_DEFAULT_QUEUE=celery CLASSIFIED = false CONTRACT_END_DATE = 2022-09-14 diff --git a/deploy/azure/atst-envvars-configmap.yml b/deploy/azure/atst-envvars-configmap.yml index 61f81fa8..4870e6a6 100644 --- a/deploy/azure/atst-envvars-configmap.yml +++ b/deploy/azure/atst-envvars-configmap.yml @@ -7,6 +7,7 @@ metadata: data: ASSETS_URL: https://atat-cdn.azureedge.net/ CELERY_DEFAULT_QUEUE: celery-master + CDN_ORIGIN: https://azure.atat.code.mil CSP: azure FLASK_ENV: dev LOG_JSON: "true" diff --git a/deploy/overlays/staging/envvars.yml b/deploy/overlays/staging/envvars.yml index 86ea42b7..99721ec9 100644 --- a/deploy/overlays/staging/envvars.yml +++ b/deploy/overlays/staging/envvars.yml @@ -14,6 +14,7 @@ metadata: name: atst-envvars data: ASSETS_URL: https://atat-cdn-staging.azureedge.net/ + CDN_ORIGIN: https://staging.atat.code.mil CELERY_DEFAULT_QUEUE: celery-staging FLASK_ENV: staging STATIC_URL: https://atat-cdn-staging.azureedge.net/static/