From 65b0b95ebaadefeb4705a5e4ae90761c415828ce Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Thu, 30 Aug 2018 11:40:05 -0400 Subject: [PATCH 01/26] Ensure ALL tests are run --- script/cibuild | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/cibuild b/script/cibuild index 55ac15cb..2d3b099f 100755 --- a/script/cibuild +++ b/script/cibuild @@ -11,7 +11,7 @@ export FLASK_ENV=ci RESET_DB="true" # Define all relevant python files and directories for this app -PYTHON_FILES="./app.py ./atst ./config" +PYTHON_FILES="./app.py ./atst/** ./config" # Enable Python testing RUN_PYTHON_TESTS="true" From 8af2f5b99e3507d99abc2f15b60e6073bfd297b3 Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Thu, 30 Aug 2018 11:56:46 -0400 Subject: [PATCH 02/26] Add ability to override any config setting via ENV var --- atst/app.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/atst/app.py b/atst/app.py index f0e5ffc6..aa6a8158 100644 --- a/atst/app.py +++ b/atst/app.py @@ -114,6 +114,13 @@ def make_config(): # ENV_CONFIG will override values in BASE_CONFIG. config.read(config_files) + # Check for ENV variables as a final source of overrides + # ENV vars must be named "ATST_" + for confsetting in config.options("default"): + env_override = os.getenv("ATST_" + confsetting.upper()) + if env_override: + config.set("default", confsetting, env_override) + # Assemble DATABASE_URI value database_uri = ( "postgres://" From a6b5dd8c535a3be8ad1203d4b4e2f026e046d554 Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Thu, 30 Aug 2018 11:56:59 -0400 Subject: [PATCH 03/26] Add circleci config --- .circleci/config.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000..208744dc --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,14 @@ +version: 2.0 +jobs: + build: + docker: + - image: registry.atat.codes:443/atat-app-builder:circleci + environment: + ATST_PGHOST: localhost + ATST_PGUSER: root + ATST_PGDATABASE: circle_test + - image: circleci/postgres:9.6.5-alpine-ram + steps: + - checkout + - run: ./script/setup + - run: ./script/cibuild From 0bcc0617d606398e3b3d9bdd831e0088d5c201ad Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Thu, 30 Aug 2018 12:03:54 -0400 Subject: [PATCH 04/26] Add registry credentials --- .circleci/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 208744dc..68653d25 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3,6 +3,9 @@ jobs: build: docker: - image: registry.atat.codes:443/atat-app-builder:circleci + auth: + username: $REGISTRY_USERNAME + password: $REGISTRY_PASSWORD environment: ATST_PGHOST: localhost ATST_PGUSER: root From fece8f223265747298f3454c4b01493fd25817d5 Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Thu, 30 Aug 2018 12:16:48 -0400 Subject: [PATCH 05/26] Convert submodule URL to https --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index e2e55be7..5ae3e48e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "script/include"] path = script/include - url = git@github.com:dod-ccpo/scriptz.git + url = https://github.com:dod-ccpo/scriptz.git branch = master From b3cc9313c3b4d102dc60ff85d2cc8f69f395667e Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Thu, 30 Aug 2018 12:16:59 -0400 Subject: [PATCH 06/26] Ensure submodules are cloned --- .circleci/config.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 68653d25..bc3add8b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -13,5 +13,9 @@ jobs: - image: circleci/postgres:9.6.5-alpine-ram steps: - checkout + - run: + name: "Clone Submodules" + command: | + git submodule update --init --recursive - run: ./script/setup - run: ./script/cibuild From d58b677dad3a1ca24dac297d39646e4a6bf31245 Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Thu, 30 Aug 2018 12:19:43 -0400 Subject: [PATCH 07/26] Fix syntax --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 5ae3e48e..0e4a0b6f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "script/include"] path = script/include - url = https://github.com:dod-ccpo/scriptz.git + url = https://github.com/dod-ccpo/scriptz.git branch = master From d4fd1abf611394b0382b1017dfdde4309daf9399 Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Thu, 30 Aug 2018 13:51:01 -0400 Subject: [PATCH 08/26] Temp disable DB vars from ini files --- .circleci/config.yml | 3 +++ script/get_db_settings | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index bc3add8b..e5d51c4a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,6 +10,9 @@ jobs: ATST_PGHOST: localhost ATST_PGUSER: root ATST_PGDATABASE: circle_test + PGHOST: localhost + PGUSER: root + PGDATABASE: circle_test - image: circleci/postgres:9.6.5-alpine-ram steps: - checkout diff --git a/script/get_db_settings b/script/get_db_settings index c96970a8..da185819 100755 --- a/script/get_db_settings +++ b/script/get_db_settings @@ -5,4 +5,4 @@ source "$(dirname "${0}")"/../script/include/global_header.inc.sh # Run the shared get_db_settings script -source ./script/include/run_get_db_settings.sh +#source ./script/include/run_get_db_settings.sh From 1d084d17cea9fa31a3cb82a6a168ed6d6321905c Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Thu, 30 Aug 2018 14:04:24 -0400 Subject: [PATCH 09/26] Add redis for test dependencies --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index e5d51c4a..db3baf71 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,6 +14,7 @@ jobs: PGUSER: root PGDATABASE: circle_test - image: circleci/postgres:9.6.5-alpine-ram + - image: circleci/redis:4-alpine3.8 steps: - checkout - run: From 73f724823598473472d08f7ad936924ee180526f Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Thu, 30 Aug 2018 14:10:36 -0400 Subject: [PATCH 10/26] Remove yarn build (already done by setup script) --- script/cibuild | 3 --- 1 file changed, 3 deletions(-) diff --git a/script/cibuild b/script/cibuild index 2d3b099f..7df80828 100755 --- a/script/cibuild +++ b/script/cibuild @@ -16,9 +16,6 @@ PYTHON_FILES="./app.py ./atst/** ./config" # Enable Python testing RUN_PYTHON_TESTS="true" -# Compile assets -yarn build - # Check python formatting source ./script/format check From 3fc8f6e9709775051202503e2e1d15cdc7271dc7 Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Thu, 30 Aug 2018 14:11:37 -0400 Subject: [PATCH 11/26] Remove extra config settings --- config/ci.ini | 3 --- 1 file changed, 3 deletions(-) diff --git a/config/ci.ini b/config/ci.ini index 3ee7f480..4f955c0c 100644 --- a/config/ci.ini +++ b/config/ci.ini @@ -1,6 +1,3 @@ [default] -PGHOST = postgreshost -PGDATABASE = atat_test -REDIS_URI = redis://redishost:6379 CRL_DIRECTORY = tests/fixtures/crl WTF_CSRF_ENABLED = false From dd578f6f3f7600a39be101ecb632aea6de49ad21 Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Thu, 30 Aug 2018 14:35:26 -0400 Subject: [PATCH 12/26] Add crl sync and crl file caching --- .circleci/config.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index db3baf71..41a805ea 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -22,4 +22,12 @@ jobs: command: | git submodule update --init --recursive - run: ./script/setup + - restore_cache: + keys: + - disa-crls + - run: ./script/sync-crls + - save_cache: + paths: + - ./crl + key: disa-crls - run: ./script/cibuild From 604bfc92f2678247e55acebf4d807fa149f48f96 Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Thu, 30 Aug 2018 16:17:24 -0400 Subject: [PATCH 13/26] Remove db reset; setup does that for ci --- script/cibuild | 3 --- 1 file changed, 3 deletions(-) diff --git a/script/cibuild b/script/cibuild index 7df80828..1493008c 100755 --- a/script/cibuild +++ b/script/cibuild @@ -7,9 +7,6 @@ source "$(dirname "${0}")"/../script/include/global_header.inc.sh # Run lint/style checks and unit tests export FLASK_ENV=ci -# Enable database resetting -RESET_DB="true" - # Define all relevant python files and directories for this app PYTHON_FILES="./app.py ./atst/** ./config" From cfbe37a53551565ffd6fea11c3e50c10f7ab6df0 Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Thu, 30 Aug 2018 16:34:48 -0400 Subject: [PATCH 14/26] Convert to just using straight key names as env var names --- atst/app.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/atst/app.py b/atst/app.py index aa6a8158..e882c5cc 100644 --- a/atst/app.py +++ b/atst/app.py @@ -115,9 +115,8 @@ def make_config(): config.read(config_files) # Check for ENV variables as a final source of overrides - # ENV vars must be named "ATST_" for confsetting in config.options("default"): - env_override = os.getenv("ATST_" + confsetting.upper()) + env_override = os.getenv(confsetting.upper()) if env_override: config.set("default", confsetting, env_override) From 17787aaa7fc621e8c60c5439597b5df69a4b3937 Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Thu, 30 Aug 2018 16:45:00 -0400 Subject: [PATCH 15/26] Update scripts code --- script/include | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/include b/script/include index c44ca507..b3122025 160000 --- a/script/include +++ b/script/include @@ -1 +1 @@ -Subproject commit c44ca5070da78fd522a2e485aaa225cc638e11d3 +Subproject commit b31220250ae97978e3d54f21e67128e90ef180c5 From b1b201b8ca08324795019c5e476525d42ce28b0e Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Thu, 30 Aug 2018 16:45:22 -0400 Subject: [PATCH 16/26] Reenable DB setting processing --- script/get_db_settings | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get_db_settings b/script/get_db_settings index da185819..c96970a8 100755 --- a/script/get_db_settings +++ b/script/get_db_settings @@ -5,4 +5,4 @@ source "$(dirname "${0}")"/../script/include/global_header.inc.sh # Run the shared get_db_settings script -#source ./script/include/run_get_db_settings.sh +source ./script/include/run_get_db_settings.sh From 6fb4e4d1798e3a17deb9e620e5f3e9cf85ad8ab7 Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Thu, 30 Aug 2018 17:11:24 -0400 Subject: [PATCH 17/26] Add some caching to the build --- .circleci/config.yml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 41a805ea..3d5c7c5d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -21,7 +21,40 @@ jobs: name: "Clone Submodules" command: | git submodule update --init --recursive + - restore_cache: + name: "Load Cache: Pipenv References" + keys: + - pipenv-v1-{{ .Branch }}-{{ checksum "Pipefile.lock" }} + - pipenv-v1-{{ .Branch }}- + - pipenv-v1- + - restore_cache: + name: "Load Cache: Python Venv" + keys: + - venv-v1-{{ .Branch }}-{{ checksum "Pipefile.lock" }} + - venv-v1-{{ .Branch }}- + - venv-v1- + - restore_cache: + name: "Load Cache: Yarn" + keys: + - yarn-v1-{{ .Branch }}-{{ checksum "yarn.lock" }} + - yarn-v1-{{ .Branch }}- + - yarn-v1- - run: ./script/setup + - save_cache: + name: "Save Cache: Pipenv Refrences" + paths: + - ~/.local/share + key: pipenv-v1-{{ .Branch }}-{{ checksum "Pipefile.lock" }} + - save_cache: + name: "Save Cache: Python Venv" + paths: + - ./.venv + key: venv-v1-{{ .Branch }}-{{ checksum "Pipefile.lock" }} + - save_cache: + name: "Save Cache: Yarn" + paths: + - ~/.cache/yarn + key: yarn-v1-{{ .Branch }}-{{ checksum "yarn.lock" }} - restore_cache: keys: - disa-crls From 1c6ce3b2f317e7661c65ee321dc9aaeb23bed226 Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Thu, 30 Aug 2018 17:15:19 -0400 Subject: [PATCH 18/26] Fix file reference --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3d5c7c5d..b2d1aed4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -24,13 +24,13 @@ jobs: - restore_cache: name: "Load Cache: Pipenv References" keys: - - pipenv-v1-{{ .Branch }}-{{ checksum "Pipefile.lock" }} + - pipenv-v1-{{ .Branch }}-{{ checksum "Pipfile.lock" }} - pipenv-v1-{{ .Branch }}- - pipenv-v1- - restore_cache: name: "Load Cache: Python Venv" keys: - - venv-v1-{{ .Branch }}-{{ checksum "Pipefile.lock" }} + - venv-v1-{{ .Branch }}-{{ checksum "Pipfile.lock" }} - venv-v1-{{ .Branch }}- - venv-v1- - restore_cache: @@ -44,12 +44,12 @@ jobs: name: "Save Cache: Pipenv Refrences" paths: - ~/.local/share - key: pipenv-v1-{{ .Branch }}-{{ checksum "Pipefile.lock" }} + key: pipenv-v1-{{ .Branch }}-{{ checksum "Pipfile.lock" }} - save_cache: name: "Save Cache: Python Venv" paths: - ./.venv - key: venv-v1-{{ .Branch }}-{{ checksum "Pipefile.lock" }} + key: venv-v1-{{ .Branch }}-{{ checksum "Pipfile.lock" }} - save_cache: name: "Save Cache: Yarn" paths: From d59f85cc01b0a507501050960baeabade29a4661 Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Thu, 30 Aug 2018 17:16:53 -0400 Subject: [PATCH 19/26] Add REDIS_URL env var --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index b2d1aed4..60c422b0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -13,6 +13,7 @@ jobs: PGHOST: localhost PGUSER: root PGDATABASE: circle_test + REDIS_URL: redis://localhost:6379 - image: circleci/postgres:9.6.5-alpine-ram - image: circleci/redis:4-alpine3.8 steps: From b4cf32f845798c4943339b8480372159038f83a1 Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Thu, 30 Aug 2018 17:17:08 -0400 Subject: [PATCH 20/26] Restore necessary CI setttings for Travis --- config/ci.ini | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config/ci.ini b/config/ci.ini index 4f955c0c..3ee7f480 100644 --- a/config/ci.ini +++ b/config/ci.ini @@ -1,3 +1,6 @@ [default] +PGHOST = postgreshost +PGDATABASE = atat_test +REDIS_URI = redis://redishost:6379 CRL_DIRECTORY = tests/fixtures/crl WTF_CSRF_ENABLED = false From e486d9f53bde9526349001e6b3123c9466972772 Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Thu, 30 Aug 2018 17:58:21 -0400 Subject: [PATCH 21/26] Fix var name --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 60c422b0..b48384fa 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -13,7 +13,7 @@ jobs: PGHOST: localhost PGUSER: root PGDATABASE: circle_test - REDIS_URL: redis://localhost:6379 + REDIS_URI: redis://localhost:6379 - image: circleci/postgres:9.6.5-alpine-ram - image: circleci/redis:4-alpine3.8 steps: From 77c07a8d1e547f0091ec658963bb0618a126652c Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Thu, 30 Aug 2018 18:02:04 -0400 Subject: [PATCH 22/26] Update scriptz code --- script/include | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/include b/script/include index b3122025..216717ac 160000 --- a/script/include +++ b/script/include @@ -1 +1 @@ -Subproject commit b31220250ae97978e3d54f21e67128e90ef180c5 +Subproject commit 216717ac4980ff508099f0bd8d6698137f2d0d56 From eaf9a90c060c0f876a31f9618d04942dae0eb7d4 Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Thu, 30 Aug 2018 18:38:34 -0400 Subject: [PATCH 23/26] Remove unused vars --- .circleci/config.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b48384fa..aaedf043 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,9 +7,6 @@ jobs: username: $REGISTRY_USERNAME password: $REGISTRY_PASSWORD environment: - ATST_PGHOST: localhost - ATST_PGUSER: root - ATST_PGDATABASE: circle_test PGHOST: localhost PGUSER: root PGDATABASE: circle_test From 28a27c518a648919a40ab2c582db1eb75b61213e Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Thu, 30 Aug 2018 18:39:29 -0400 Subject: [PATCH 24/26] Enable venv caching --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index aaedf043..45770805 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,6 +7,7 @@ jobs: username: $REGISTRY_USERNAME password: $REGISTRY_PASSWORD environment: + KEEP_EXISTING_VENV: true PGHOST: localhost PGUSER: root PGDATABASE: circle_test From f5f104d553f8cde28d7f9b16f423784373c5e0f1 Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Thu, 30 Aug 2018 18:39:43 -0400 Subject: [PATCH 25/26] Update scripz version --- script/include | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/include b/script/include index 216717ac..22b70975 160000 --- a/script/include +++ b/script/include @@ -1 +1 @@ -Subproject commit 216717ac4980ff508099f0bd8d6698137f2d0d56 +Subproject commit 22b709756bb3d56ced713dbcb79a2b7fa7bb866e From 9d356c3344e2708079d1d545e610f5b522bfc74f Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Thu, 30 Aug 2018 18:51:33 -0400 Subject: [PATCH 26/26] Update scripz version --- script/include | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/include b/script/include index 22b70975..c55d1eb5 160000 --- a/script/include +++ b/script/include @@ -1 +1 @@ -Subproject commit 22b709756bb3d56ced713dbcb79a2b7fa7bb866e +Subproject commit c55d1eb5db0255784e46aec1006e4419465ef945