From 649987a33b5c582cedc985d2a8db14a270daba03 Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Fri, 6 Jul 2018 13:07:29 -0400 Subject: [PATCH 01/47] Add file with basic command all scripts should run --- script/include/global_header.inc.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100755 script/include/global_header.inc.sh diff --git a/script/include/global_header.inc.sh b/script/include/global_header.inc.sh new file mode 100755 index 00000000..67a048b2 --- /dev/null +++ b/script/include/global_header.inc.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +# scriptz/global_header.inc: Any basic things that should be executed at the +# beginning of any and every script + +# If any command fails, immediately exit the script +set -e + +# Ensure the working directory is the app root directory +cd "$(dirname "${0}")/.." + +# Source all function definition files +source ./script/include/*_functions.inc.sh From 04d073329a15ad7b3be6f6da84e90a89181b96b6 Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Fri, 6 Jul 2018 13:07:45 -0400 Subject: [PATCH 02/47] Add file with general helper functions for use by other scripts --- script/include/helper_functions.inc.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 script/include/helper_functions.inc.sh diff --git a/script/include/helper_functions.inc.sh b/script/include/helper_functions.inc.sh new file mode 100644 index 00000000..18968aae --- /dev/null +++ b/script/include/helper_functions.inc.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# helper_functions.inc.sh: General helper functions + +# Check pip to see if the given package is installed +# (returns 0 if installed, 2 if not installed) +check_pip_for () { + return $(pip list --format=columns --disable-pip-version-check | \ + grep -Fe "${1}" >/dev/null 2>&1) +} From 16de4f829e2d99191bf70511e76cadc17144fd83 Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Fri, 6 Jul 2018 13:08:02 -0400 Subject: [PATCH 03/47] Add file to contain functions used by setup --- script/include/setup_functions.inc.sh | 49 +++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 script/include/setup_functions.inc.sh diff --git a/script/include/setup_functions.inc.sh b/script/include/setup_functions.inc.sh new file mode 100644 index 00000000..7c4a6741 --- /dev/null +++ b/script/include/setup_functions.inc.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +# setup_functions.inc.sh: Functions used by the setup script + +install_pipenv() { + exit_code=0 + + # Ensure we are not in a virtual env already + if [ -z "${VIRTUAL_ENV+xxxx}" ]; then + if ! check_pip_for pipenv; then + # pipenv is not installed, so install it + echo "Installing pipenv..." + pip install pipenv + # Capture pip exit code + exit_code="${?}" + fi + fi + + return "${exit_code}" +} + +create_virtual_environment() { + default_python_version=3.6 + # Parse out the required Python version from the Pipfile + python_version=$(grep python_version ./Pipfile | cut -d '"' -f 2) + + # If we ended up with an empty string for the required Python version, + # specify the default version + if [ -z "${python_version}" ]; then + python_version="${default_python_version}" + fi + + # Create a new virtual environment for the app + # The environment will be in a directory called .venv off the app + # root directory + echo "Creating virtual environment using Python version ${python_version}..." + return $(PIPENV_VENV_IN_PROJECT=true pipenv --python "${python_version}") +} + +install_sass() { + if ! type sass >/dev/null; then + if type gem >/dev/null; then + echo 'Installing a sass compiler (gem)...' + gem install sass + else + echo 'Could not install a sass compiler. Please install a version of sass.' + fi + fi +} From c4283fe1a4e55dfbe74b64c485117eaebada9d7d Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Fri, 6 Jul 2018 13:17:03 -0400 Subject: [PATCH 04/47] Remove interpreter specification These script fragments should only ever be sourced rather then directly executed --- script/include/global_header.inc.sh | 6 ++---- script/include/helper_functions.inc.sh | 3 +-- script/include/setup_functions.inc.sh | 2 -- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/script/include/global_header.inc.sh b/script/include/global_header.inc.sh index 67a048b2..443b5c03 100755 --- a/script/include/global_header.inc.sh +++ b/script/include/global_header.inc.sh @@ -1,7 +1,5 @@ -#!/bin/bash - -# scriptz/global_header.inc: Any basic things that should be executed at the -# beginning of any and every script +# global_header.inc: Any basic things that should be executed at the +# beginning of any and every script # If any command fails, immediately exit the script set -e diff --git a/script/include/helper_functions.inc.sh b/script/include/helper_functions.inc.sh index 18968aae..3c427a0f 100644 --- a/script/include/helper_functions.inc.sh +++ b/script/include/helper_functions.inc.sh @@ -1,10 +1,9 @@ -#!/bin/bash - # helper_functions.inc.sh: General helper functions # Check pip to see if the given package is installed # (returns 0 if installed, 2 if not installed) check_pip_for () { + # Use 'pip list' to see if the requested package is already installed return $(pip list --format=columns --disable-pip-version-check | \ grep -Fe "${1}" >/dev/null 2>&1) } diff --git a/script/include/setup_functions.inc.sh b/script/include/setup_functions.inc.sh index 7c4a6741..50b12af3 100644 --- a/script/include/setup_functions.inc.sh +++ b/script/include/setup_functions.inc.sh @@ -1,5 +1,3 @@ -#!/bin/bash - # setup_functions.inc.sh: Functions used by the setup script install_pipenv() { From 516feb2cd4975f546136395b54d97988018ad91a Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Fri, 6 Jul 2018 13:18:51 -0400 Subject: [PATCH 05/47] Add script that runs all the setup commands --- script/include/run_setup | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100755 script/include/run_setup diff --git a/script/include/run_setup b/script/include/run_setup new file mode 100755 index 00000000..b9240644 --- /dev/null +++ b/script/include/run_setup @@ -0,0 +1,24 @@ +# include/setup: Set up application for the first time after cloning, or set it +# back to the initial first unused state. + +# If CREATE_VENV is not set, set it to "true" +if [ -z "${CREATE_VENV+is_set}" ]; then + CREATE_VENV="true" +fi + +# If INSTALL_SASS is not set, set it to "false" +if [ -z "${INSTALL_SASS+is_set}" ]; then + INSTALL_SASS="false" +fi + +if [ "${CREATE_VENV}" = "true" ]; then + install_pipenv + create_virtual_environment +fi + +if [ "${INSTALL_SASS}" = "true" ]; then + install_sass +fi + +# Install application dependencies +source ./script/bootstrap From 1ea184f049461820921443bd53b12a3784913555 Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Fri, 6 Jul 2018 13:19:35 -0400 Subject: [PATCH 06/47] Update script to use shared fragments from the include dir --- script/setup | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/script/setup b/script/setup index 0fae92e6..925ced94 100755 --- a/script/setup +++ b/script/setup @@ -3,24 +3,10 @@ # script/setup: Set up application for the first time after cloning, or set it # back to the initial first unused state. -# If a command fails, exit the script -set -e +source "$(dirname "${0}")"/../script/include/global_header.inc.sh -# Ensure we are in the app root directory (not the /script directory) -cd "$(dirname "${0}")/.." +# Turn on sass compiler installation +INSTALL_SASS="true" -# Install virtualenv -pip install pipenv -pipenv --python 3.6 - -if ! type sass > /dev/null; then - if type gem > /dev/null; then - echo 'installing a sass compiler...' - gem install sass - else - echo 'Could not install a sass compiler. Please install a version of sass.' - fi -fi - -# Install application dependencies -script/bootstrap +# Run the shared setup script +source ./script/include/run_setup From 958f453442dad73254a900d5ed4a54c0e2d4029c Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Mon, 9 Jul 2018 11:04:26 -0400 Subject: [PATCH 07/47] Fix function snippet sourcing --- script/include/global_header.inc.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/script/include/global_header.inc.sh b/script/include/global_header.inc.sh index 443b5c03..571967fd 100755 --- a/script/include/global_header.inc.sh +++ b/script/include/global_header.inc.sh @@ -8,4 +8,8 @@ set -e cd "$(dirname "${0}")/.." # Source all function definition files -source ./script/include/*_functions.inc.sh + +for function_snippet in ./script/include/*_functions.inc.sh +do + source "${function_snippet}" +done From 6e4c5911c33de82169096426894e64a2c18f6bfb Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Mon, 9 Jul 2018 11:04:39 -0400 Subject: [PATCH 08/47] Add comments --- script/include/run_setup | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/script/include/run_setup b/script/include/run_setup index b9240644..25e5fc7f 100755 --- a/script/include/run_setup +++ b/script/include/run_setup @@ -1,6 +1,7 @@ # include/setup: Set up application for the first time after cloning, or set it # back to the initial first unused state. +## Option defaults # If CREATE_VENV is not set, set it to "true" if [ -z "${CREATE_VENV+is_set}" ]; then CREATE_VENV="true" @@ -11,6 +12,7 @@ if [ -z "${INSTALL_SASS+is_set}" ]; then INSTALL_SASS="false" fi +## Main if [ "${CREATE_VENV}" = "true" ]; then install_pipenv create_virtual_environment @@ -21,4 +23,4 @@ if [ "${INSTALL_SASS}" = "true" ]; then fi # Install application dependencies -source ./script/bootstrap +./script/bootstrap From 6b31da0a0b62e5667076cf52d62d85090595ce65 Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Mon, 9 Jul 2018 12:10:42 -0400 Subject: [PATCH 09/47] Update global: only source the helper_functions --- script/include/global_header.inc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/include/global_header.inc.sh b/script/include/global_header.inc.sh index 571967fd..01e1709d 100755 --- a/script/include/global_header.inc.sh +++ b/script/include/global_header.inc.sh @@ -9,7 +9,7 @@ cd "$(dirname "${0}")/.." # Source all function definition files -for function_snippet in ./script/include/*_functions.inc.sh +for function_snippet in ./script/include/helper_functions.inc.sh do source "${function_snippet}" done From 05c42e5988f850db0a0419366a35eb3b1b57ccac Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Mon, 9 Jul 2018 12:10:56 -0400 Subject: [PATCH 10/47] Update helper_functions - Modify check pip to make it clear it checks the system python - Add generic sounding "run_command", in case we switch away from pipenv --- script/include/helper_functions.inc.sh | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/script/include/helper_functions.inc.sh b/script/include/helper_functions.inc.sh index 3c427a0f..57415ba1 100644 --- a/script/include/helper_functions.inc.sh +++ b/script/include/helper_functions.inc.sh @@ -2,8 +2,18 @@ # Check pip to see if the given package is installed # (returns 0 if installed, 2 if not installed) -check_pip_for () { +check_system_pip_for () { + local package_name="${1}" + # Use 'pip list' to see if the requested package is already installed - return $(pip list --format=columns --disable-pip-version-check | \ - grep -Fe "${1}" >/dev/null 2>&1) + pip list --format=columns --disable-pip-version-check | \ + grep -Fe "${package_name}" >/dev/null 2>&1 + return $? +} + +# Used whenever an environment sensitive command is being run +run_command () { + local cmd="${1}" + pipenv ${cmd} + return $? } From adb41337e3f701a502fe0551a96de4d9e7de56b2 Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Mon, 9 Jul 2018 12:12:53 -0400 Subject: [PATCH 11/47] Fix function name which was just changed --- script/include/setup_functions.inc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/include/setup_functions.inc.sh b/script/include/setup_functions.inc.sh index 50b12af3..ea26d23a 100644 --- a/script/include/setup_functions.inc.sh +++ b/script/include/setup_functions.inc.sh @@ -5,7 +5,7 @@ install_pipenv() { # Ensure we are not in a virtual env already if [ -z "${VIRTUAL_ENV+xxxx}" ]; then - if ! check_pip_for pipenv; then + if ! check_system_pip_for pipenv; then # pipenv is not installed, so install it echo "Installing pipenv..." pip install pipenv From b23c6b15bbc8854a9476ee5d3775e819be030c7a Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Mon, 9 Jul 2018 12:13:48 -0400 Subject: [PATCH 12/47] Update script to include sourcing setup specific functions --- script/include/run_setup | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/script/include/run_setup b/script/include/run_setup index 25e5fc7f..97fa0808 100755 --- a/script/include/run_setup +++ b/script/include/run_setup @@ -1,7 +1,10 @@ # include/setup: Set up application for the first time after cloning, or set it # back to the initial first unused state. -## Option defaults +# Load setup functions +source ./script/include/setup_functions.inc.sh + +## Set option defaults # If CREATE_VENV is not set, set it to "true" if [ -z "${CREATE_VENV+is_set}" ]; then CREATE_VENV="true" From 6fa5d7aad1cffefa4d7e9394461c2cb35364e05a Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Mon, 9 Jul 2018 12:16:04 -0400 Subject: [PATCH 13/47] Add file containing bootstrap specific functions --- script/include/bootstrap_functions.inc.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 script/include/bootstrap_functions.inc.sh diff --git a/script/include/bootstrap_functions.inc.sh b/script/include/bootstrap_functions.inc.sh new file mode 100644 index 00000000..33e22bf8 --- /dev/null +++ b/script/include/bootstrap_functions.inc.sh @@ -0,0 +1,12 @@ +# bootstrap_functions.inc.sh: Functions used by the bootstrap script + +install_python_packages() { + local install_flags="${1}" + pipenv install ${install_flags} + return $? +} + +install_node_packages() { + npm install + return $? +} From 617c154d01036fe95b16d7a778fa5bc97c491541 Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Mon, 9 Jul 2018 12:16:30 -0400 Subject: [PATCH 14/47] Add script to execute bootstrap logic --- script/include/run_bootstrap | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100755 script/include/run_bootstrap diff --git a/script/include/run_bootstrap b/script/include/run_bootstrap new file mode 100755 index 00000000..2ac48ca9 --- /dev/null +++ b/script/include/run_bootstrap @@ -0,0 +1,24 @@ +# include/run_bootstrap: Install application dependencies + +# Load bootstrap functions +source ./script/include/bootstrap_functions.inc.sh + +## Set option defaults +# If PIPENV_INSTALL_FLAGS is not set, give it the default value of "--dev" +if [ -z "${PIPENV_INSTALL_FLAGS+is_set}" ]; then + CREATE_VENV="--dev" +fi + +## Main + +if [ "${INSTALL_PYTHON_PACKAGES}" = "true" ]; then + install_python_packages "${PIPENV_INSTALL_FLAGS}" +fi + +if [ "${INSTALL_NODE_PACKAGES}" = "true" ]; then + install_node_packages +fi + +if [ -n "${COMPILE_SASS_CMD}" ]; then + run_command "${COMPILE_SASS_CMD}" +fi From cb0564e3e7be5c35d081c5730c4df559a69eb283 Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Mon, 9 Jul 2018 12:16:56 -0400 Subject: [PATCH 15/47] Update script to use shared functions and logic --- script/bootstrap | 33 +++++++-------------------------- 1 file changed, 7 insertions(+), 26 deletions(-) diff --git a/script/bootstrap b/script/bootstrap index 78f9391f..7b569239 100755 --- a/script/bootstrap +++ b/script/bootstrap @@ -3,33 +3,14 @@ # script/bootstrap: Resolve all dependencies that the application requires to # run. -# If a command fails, exit the script -set -e +source "$(dirname "${0}")"/../script/include/global_header.inc.sh -# Ensure we are in the app root directory (not the /script directory) -cd "$(dirname "${0}")/.." +# Set sass compiling command for this app +COMPILE_SASS_CMD="webassets -m atst.assets build" -if [ -z "${CIBUILD+xxxx}" ]; then - CMD_PREFIX='pipenv run ' -fi -PIP_CMD="${CMD_PREFIX}pip" -WEBASSETS_CMD="${CMD_PREFIX}webassets" +# Run the shared bootstrap script +source ./script/include/run_bootstrap -PIPENV_INSTALL_FLAGS='--dev' -if [ -n "${CIBUILD}" ]; then - PIPENV_INSTALL_FLAGS+=' --system --ignore-pipfile' -fi - -# Install Python dependencies -${PIP_CMD} install --upgrade pip -pipenv install ${PIPENV_INSTALL_FLAGS} - -# Install uswds node module and dependencies -npm install - -# Relink uswds fonts into the /static directory +# Link USWDS fonts into the /static directory rm -f ./static/fonts -ln -s ../node_modules/uswds/src/fonts ./static/fonts - -# Precompile assets for deployment -${WEBASSETS_CMD} -m atst.assets build +ln -s ../node/modules/uswds/src/fonts ./static/fonts From a31e833fa43e136d83373e479c58f6c3ac3e3d57 Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Mon, 9 Jul 2018 12:56:13 -0400 Subject: [PATCH 16/47] Fix pienv run command by actually adding "run" --- script/include/helper_functions.inc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/include/helper_functions.inc.sh b/script/include/helper_functions.inc.sh index 57415ba1..96218f9a 100644 --- a/script/include/helper_functions.inc.sh +++ b/script/include/helper_functions.inc.sh @@ -14,6 +14,6 @@ check_system_pip_for () { # Used whenever an environment sensitive command is being run run_command () { local cmd="${1}" - pipenv ${cmd} + pipenv run ${cmd} return $? } From d70481e425699d8014185c21e29b46b5cd0b30be Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Mon, 9 Jul 2018 12:56:32 -0400 Subject: [PATCH 17/47] Fix variable name --- script/include/run_bootstrap | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/include/run_bootstrap b/script/include/run_bootstrap index 2ac48ca9..a3d11798 100755 --- a/script/include/run_bootstrap +++ b/script/include/run_bootstrap @@ -6,7 +6,7 @@ source ./script/include/bootstrap_functions.inc.sh ## Set option defaults # If PIPENV_INSTALL_FLAGS is not set, give it the default value of "--dev" if [ -z "${PIPENV_INSTALL_FLAGS+is_set}" ]; then - CREATE_VENV="--dev" + PIPENV_INSTALL_FLAGS="--dev" fi ## Main From 2ba3c68921306119aafb3962634c17431f833c91 Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Mon, 9 Jul 2018 12:56:55 -0400 Subject: [PATCH 18/47] Add file containing functions used by the test script --- script/include/test_functions.inc.sh | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 script/include/test_functions.inc.sh diff --git a/script/include/test_functions.inc.sh b/script/include/test_functions.inc.sh new file mode 100644 index 00000000..23e231d8 --- /dev/null +++ b/script/include/test_functions.inc.sh @@ -0,0 +1,18 @@ +# test_functions.inc.sh: Functions used by the run_test script + +run_python_lint() { + local python_files="${1}" + run_command "pylint ${python_files}" + return $? +} + +run_python_static_analysis() { + local python_files="${1}" + run_command "bandit -c ./.bandit_config -r ${python_files}" + return $? +} + +run_python_unit_tests() { + run_command "python -m pytest -s" + return $? +} From b8d3a4536b65522facb67a640f7991ccf4918721 Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Mon, 9 Jul 2018 12:57:22 -0400 Subject: [PATCH 19/47] Add test script to execute code checks and unit tests --- script/include/run_test | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100755 script/include/run_test diff --git a/script/include/run_test b/script/include/run_test new file mode 100755 index 00000000..4187c2d1 --- /dev/null +++ b/script/include/run_test @@ -0,0 +1,17 @@ +# include/run_test: Execute code checkers and unit tests + +# Load test functions +source ./script/include/test_functions.inc.sh + +## Set option defaults +# If PYTHON_FILES is not set, give it the default value of "app.py" +if [ -z "${PYTHON_FILES+is_set}" ]; then + PYTHON_FILES="app.py" +fi + +## Main +if [ "${RUN_PYTHON_TESTS}" = "true" ]; then + run_python_lint "${PYTHON_FILES}" + run_python_static_analysis "${PYTHON_FILES}" + run_python_unit_tests "${PYTHON_FILES}" +fi From 53f27b2ea2c3e5df47caeeb92d89f5b26b578c36 Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Mon, 9 Jul 2018 12:57:45 -0400 Subject: [PATCH 20/47] Enable pythong and node dependency installation --- script/bootstrap | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/script/bootstrap b/script/bootstrap index 7b569239..e30b69da 100755 --- a/script/bootstrap +++ b/script/bootstrap @@ -8,6 +8,10 @@ source "$(dirname "${0}")"/../script/include/global_header.inc.sh # Set sass compiling command for this app COMPILE_SASS_CMD="webassets -m atst.assets build" +# Enable python and node package installation +INSTALL_PYTHON_PACKAGES="true" +INSTALL_NODE_PACKAGES="true" + # Run the shared bootstrap script source ./script/include/run_bootstrap From 9e89f75b92b44c448984c2bb57481d796cd7573c Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Mon, 9 Jul 2018 12:58:05 -0400 Subject: [PATCH 21/47] Update to use shared test script and functions --- script/test | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/script/test b/script/test index fe2d7ecb..1f6c3380 100755 --- a/script/test +++ b/script/test @@ -2,22 +2,13 @@ # script/test: Run static code checks and unit tests -# If a command fails, exit the script -set -e +source "$(dirname "${0}")"/../script/include/global_header.inc.sh -# Ensure we are in the app root directory (not the /script directory) -cd "$(dirname "${0}")/.." +# Define all relevant python files and directories for this app +PYTHON_FILES="./app.py ./atst ./config" -if [ -z "${SKIP_PIPENV+xxxx}" ]; then - CMD_PREFIX='pipenv run ' -fi -PYLINT_CMD="${CMD_PREFIX}pylint" -PYTHON_CMD="${CMD_PREFIX}python" +# Enable Python testing +RUN_PYTHON_TESTS="true" -# Run lint check -echo "Running lint..." -${PYLINT_CMD} app.py atst/ tests/ - -# Run unit tests -echo "Running unit tests..." -${PYTHON_CMD} -m pytest -s $* +# Run the shared test script +source ./script/include/run_test From 09cf59ccd18ec949b44cd433623b997a3f9308bc Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Mon, 9 Jul 2018 12:58:32 -0400 Subject: [PATCH 22/47] Add default config file for bandit --- .bandit_config | 156 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 .bandit_config diff --git a/.bandit_config b/.bandit_config new file mode 100644 index 00000000..ff68cb98 --- /dev/null +++ b/.bandit_config @@ -0,0 +1,156 @@ +### This config may optionally select a subset of tests to run or skip by +### filling out the 'tests' and 'skips' lists given below. If no tests are +### specified for inclusion then it is assumed all tests are desired. The skips +### set will remove specific tests from the include set. +### Note that the same test ID should not appear in both 'tests' and 'skips', +### this would be nonsensical and is detected by Bandit at runtime. + +# (optional) list included test IDs here, eg '[B101, B406]': +tests: + +# (optional) list skipped test IDs here, eg '[B101, B406]': +skips: + +### (optional) plugin settings - some test plugins require configuration data +### that may be given here, per-plugin. All bandit test plugins have a built in +### set of sensible defaults and these will be used if no configuration is +### provided. It is not necessary to provide settings for every (or any) plugin +### if the defaults are acceptable. + +any_other_function_with_shell_equals_true: + no_shell: [os.execl, os.execle, os.execlp, os.execlpe, os.execv, os.execve, os.execvp, + os.execvpe, os.spawnl, os.spawnle, os.spawnlp, os.spawnlpe, os.spawnv, os.spawnve, + os.spawnvp, os.spawnvpe, os.startfile] + shell: [os.system, os.popen, os.popen2, os.popen3, os.popen4, popen2.popen2, popen2.popen3, + popen2.popen4, popen2.Popen3, popen2.Popen4, commands.getoutput, commands.getstatusoutput] + subprocess: [subprocess.Popen, subprocess.call, subprocess.check_call, subprocess.check_output, + utils.execute, utils.execute_with_timeout] +execute_with_run_as_root_equals_true: + function_names: [ceilometer.utils.execute, cinder.utils.execute, neutron.agent.linux.utils.execute, + nova.utils.execute, nova.utils.trycmd] +hardcoded_tmp_directory: + tmp_dirs: [/tmp, /var/tmp, /dev/shm] +linux_commands_wildcard_injection: + no_shell: [os.execl, os.execle, os.execlp, os.execlpe, os.execv, os.execve, os.execvp, + os.execvpe, os.spawnl, os.spawnle, os.spawnlp, os.spawnlpe, os.spawnv, os.spawnve, + os.spawnvp, os.spawnvpe, os.startfile] + shell: [os.system, os.popen, os.popen2, os.popen3, os.popen4, popen2.popen2, popen2.popen3, + popen2.popen4, popen2.Popen3, popen2.Popen4, commands.getoutput, commands.getstatusoutput] + subprocess: [subprocess.Popen, subprocess.call, subprocess.check_call, subprocess.check_output, + utils.execute, utils.execute_with_timeout] +password_config_option_not_marked_secret: + function_names: [oslo.config.cfg.StrOpt, oslo_config.cfg.StrOpt] +ssl_with_bad_defaults: + bad_protocol_versions: [PROTOCOL_SSLv2, SSLv2_METHOD, SSLv23_METHOD, PROTOCOL_SSLv3, + PROTOCOL_TLSv1, SSLv3_METHOD, TLSv1_METHOD] +ssl_with_bad_version: + bad_protocol_versions: [PROTOCOL_SSLv2, SSLv2_METHOD, SSLv23_METHOD, PROTOCOL_SSLv3, + PROTOCOL_TLSv1, SSLv3_METHOD, TLSv1_METHOD] +start_process_with_a_shell: + no_shell: [os.execl, os.execle, os.execlp, os.execlpe, os.execv, os.execve, os.execvp, + os.execvpe, os.spawnl, os.spawnle, os.spawnlp, os.spawnlpe, os.spawnv, os.spawnve, + os.spawnvp, os.spawnvpe, os.startfile] + shell: [os.system, os.popen, os.popen2, os.popen3, os.popen4, popen2.popen2, popen2.popen3, + popen2.popen4, popen2.Popen3, popen2.Popen4, commands.getoutput, commands.getstatusoutput] + subprocess: [subprocess.Popen, subprocess.call, subprocess.check_call, subprocess.check_output, + utils.execute, utils.execute_with_timeout] +start_process_with_no_shell: + no_shell: [os.execl, os.execle, os.execlp, os.execlpe, os.execv, os.execve, os.execvp, + os.execvpe, os.spawnl, os.spawnle, os.spawnlp, os.spawnlpe, os.spawnv, os.spawnve, + os.spawnvp, os.spawnvpe, os.startfile] + shell: [os.system, os.popen, os.popen2, os.popen3, os.popen4, popen2.popen2, popen2.popen3, + popen2.popen4, popen2.Popen3, popen2.Popen4, commands.getoutput, commands.getstatusoutput] + subprocess: [subprocess.Popen, subprocess.call, subprocess.check_call, subprocess.check_output, + utils.execute, utils.execute_with_timeout] +start_process_with_partial_path: + no_shell: [os.execl, os.execle, os.execlp, os.execlpe, os.execv, os.execve, os.execvp, + os.execvpe, os.spawnl, os.spawnle, os.spawnlp, os.spawnlpe, os.spawnv, os.spawnve, + os.spawnvp, os.spawnvpe, os.startfile] + shell: [os.system, os.popen, os.popen2, os.popen3, os.popen4, popen2.popen2, popen2.popen3, + popen2.popen4, popen2.Popen3, popen2.Popen4, commands.getoutput, commands.getstatusoutput] + subprocess: [subprocess.Popen, subprocess.call, subprocess.check_call, subprocess.check_output, + utils.execute, utils.execute_with_timeout] +subprocess_popen_with_shell_equals_true: + no_shell: [os.execl, os.execle, os.execlp, os.execlpe, os.execv, os.execve, os.execvp, + os.execvpe, os.spawnl, os.spawnle, os.spawnlp, os.spawnlpe, os.spawnv, os.spawnve, + os.spawnvp, os.spawnvpe, os.startfile] + shell: [os.system, os.popen, os.popen2, os.popen3, os.popen4, popen2.popen2, popen2.popen3, + popen2.popen4, popen2.Popen3, popen2.Popen4, commands.getoutput, commands.getstatusoutput] + subprocess: [subprocess.Popen, subprocess.call, subprocess.check_call, subprocess.check_output, + utils.execute, utils.execute_with_timeout] +subprocess_without_shell_equals_true: + no_shell: [os.execl, os.execle, os.execlp, os.execlpe, os.execv, os.execve, os.execvp, + os.execvpe, os.spawnl, os.spawnle, os.spawnlp, os.spawnlpe, os.spawnv, os.spawnve, + os.spawnvp, os.spawnvpe, os.startfile] + shell: [os.system, os.popen, os.popen2, os.popen3, os.popen4, popen2.popen2, popen2.popen3, + popen2.popen4, popen2.Popen3, popen2.Popen4, commands.getoutput, commands.getstatusoutput] + subprocess: [subprocess.Popen, subprocess.call, subprocess.check_call, subprocess.check_output, + utils.execute, utils.execute_with_timeout] +try_except_continue: {check_typed_exception: false} +try_except_pass: {check_typed_exception: false} + +### Reference of Available tests: +# B101 : assert_used +# B102 : exec_used +# B103 : set_bad_file_permissions +# B104 : hardcoded_bind_all_interfaces +# B105 : hardcoded_password_string +# B106 : hardcoded_password_funcarg +# B107 : hardcoded_password_default +# B108 : hardcoded_tmp_directory +# B109 : password_config_option_not_marked_secret +# B110 : try_except_pass +# B111 : execute_with_run_as_root_equals_true +# B112 : try_except_continue +# B201 : flask_debug_true +# B301 : pickle +# B302 : marshal +# B303 : md5 +# B304 : ciphers +# B305 : cipher_modes +# B306 : mktemp_q +# B307 : eval +# B308 : mark_safe +# B309 : httpsconnection +# B310 : urllib_urlopen +# B311 : random +# B312 : telnetlib +# B313 : xml_bad_cElementTree +# B314 : xml_bad_ElementTree +# B315 : xml_bad_expatreader +# B316 : xml_bad_expatbuilder +# B317 : xml_bad_sax +# B318 : xml_bad_minidom +# B319 : xml_bad_pulldom +# B320 : xml_bad_etree +# B321 : ftplib +# B322 : input +# B401 : import_telnetlib +# B402 : import_ftplib +# B403 : import_pickle +# B404 : import_subprocess +# B405 : import_xml_etree +# B406 : import_xml_sax +# B407 : import_xml_expat +# B408 : import_xml_minidom +# B409 : import_xml_pulldom +# B410 : import_lxml +# B411 : import_xmlrpclib +# B412 : import_httpoxy +# B501 : request_with_no_cert_validation +# B502 : ssl_with_bad_version +# B503 : ssl_with_bad_defaults +# B504 : ssl_with_no_version +# B505 : weak_cryptographic_key +# B506 : yaml_load +# B601 : paramiko_calls +# B602 : subprocess_popen_with_shell_equals_true +# B603 : subprocess_without_shell_equals_true +# B604 : any_other_function_with_shell_equals_true +# B605 : start_process_with_a_shell +# B606 : start_process_with_no_shell +# B607 : start_process_with_partial_path +# B608 : hardcoded_sql_expressions +# B609 : linux_commands_wildcard_injection +# B701 : jinja2_autoescape_false +# B702 : use_of_mako_templates From 27cfb3442f49ab7251b71918439cb2a83e4262a9 Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Mon, 9 Jul 2018 13:00:29 -0400 Subject: [PATCH 23/47] Update to use shared header --- script/cibuild | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/script/cibuild b/script/cibuild index c9fd2975..e9d01564 100755 --- a/script/cibuild +++ b/script/cibuild @@ -2,15 +2,7 @@ # script/cibuild: Run CI related checks and tests -# If a command fails, exit the script -set -e - -# Ensure we are in the app root directory (not the /script directory) -cd "$(dirname "${0}")/.." +source "$(dirname "${0}")"/../script/include/global_header.inc.sh # Run lint/style checks and unit tests -script/test - -# Run static code analysis security checks -# (excluding the tests and node_modules subdirs) -bandit -r . -x node_modules,tests +source ./script/test From 1a5f9dfc49b5c74cad6910df77a472b87eeb19e9 Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Mon, 9 Jul 2018 14:57:45 -0400 Subject: [PATCH 24/47] Move docker files and update travis.yml accordingly --- .travis.yml | 6 +++--- {docker => deploy/docker}/prod/Dockerfile | 0 {docker => deploy/docker}/tester/Dockerfile | 0 3 files changed, 3 insertions(+), 3 deletions(-) rename {docker => deploy/docker}/prod/Dockerfile (100%) rename {docker => deploy/docker}/tester/Dockerfile (100%) diff --git a/.travis.yml b/.travis.yml index 9b5500d2..486de5cc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,15 +7,15 @@ env: - TESTER_IMAGE_NAME=atst-tester - PROD_IMAGE_NAME=atst-prod -before_install: +before_script: - docker login -u $ATAT_DOCKER_REGISTRY_USERNAME -p $ATAT_DOCKER_REGISTRY_PASSWORD $ATAT_DOCKER_REGISTRY_URL - - docker build --tag "${TESTER_IMAGE_NAME}" . -f docker/tester/Dockerfile + - docker build --tag "${TESTER_IMAGE_NAME}" . -f deploy/docker/tester/Dockerfile script: - docker run "${TESTER_IMAGE_NAME}" before_deploy: - - docker build --tag "${PROD_IMAGE_NAME}" . -f docker/prod/Dockerfile + - docker build --tag "${PROD_IMAGE_NAME}" . -f deploy/docker/prod/Dockerfile - git_sha="$(git rev-parse --short HEAD)" - remote_image_name="${ATAT_DOCKER_REGISTRY_URL}/${PROD_IMAGE_NAME}:${git_sha}" - docker tag "${PROD_IMAGE_NAME}" "${remote_image_name}" diff --git a/docker/prod/Dockerfile b/deploy/docker/prod/Dockerfile similarity index 100% rename from docker/prod/Dockerfile rename to deploy/docker/prod/Dockerfile diff --git a/docker/tester/Dockerfile b/deploy/docker/tester/Dockerfile similarity index 100% rename from docker/tester/Dockerfile rename to deploy/docker/tester/Dockerfile From ada0595f36681538bbb11ad19706f009462c2d7b Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Mon, 9 Jul 2018 15:13:27 -0400 Subject: [PATCH 25/47] Add alpine setup functions and sharable script --- script/include/alpine_setup_functions.inc.sh | 32 ++++++++++++++++++++ script/include/run_alpine_setup | 25 +++++++++++++++ 2 files changed, 57 insertions(+) create mode 100755 script/include/alpine_setup_functions.inc.sh create mode 100755 script/include/run_alpine_setup diff --git a/script/include/alpine_setup_functions.inc.sh b/script/include/alpine_setup_functions.inc.sh new file mode 100755 index 00000000..d28b88a3 --- /dev/null +++ b/script/include/alpine_setup_functions.inc.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +# alpine_setup_functions: Functions used by the run_alpine_setup script + +update_system_packages() { + apk update + apk upgrade +} + +install_package() { + local package_name=${1} + + apk add ${1} + return $? +} + +add_group() { + local group_name="${1}" + local gid="${2}" + + addgroup -g "${gid}" -S "${group_name}" + return $? +} + +add_user() { + local username="${1}" + local primary_group="${2}" + local uid="${3}" + + adduser -u "${3}" -D -S -G "${primary_group}" "${username}" + return $? +} diff --git a/script/include/run_alpine_setup b/script/include/run_alpine_setup new file mode 100755 index 00000000..c094521a --- /dev/null +++ b/script/include/run_alpine_setup @@ -0,0 +1,25 @@ +# run_alpine_setup: Install basic system requirements for an app to run + +# Load alpine setup functions +source ./script/include/alpine_setup_functions.inc.sh + +## Set option defaults +# If GROUP information is incomplete, use the default one +if [ -z "${APP_GROUP+is_set}" ] || \ + [ -z "${APP_GID+is_set}" ]; then + APP_GROUP="atat" + APP_GROUP_ID="8000" +fi + +# If USER information is incomplete, error out +if [ -z "${APP_USER+is_set}" ] || \ + [ -z "${APP_UID+is_set}" ]; then + exit 1 +fi + +## Main +update_system_packages +install_package "bash" +install_package "dumb-init" +add_group "${APP_GROUP}" "${APP_GID}" +add_user "${APP_USER}" "${APP_GROUP}" "${APP_UID}" From c370063f8d021dd52171f0469028b7989e0eb8ca Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Mon, 9 Jul 2018 15:14:05 -0400 Subject: [PATCH 26/47] Update to use shared script --- script/alpine_setup | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/script/alpine_setup b/script/alpine_setup index c029d19b..28f836c2 100755 --- a/script/alpine_setup +++ b/script/alpine_setup @@ -3,20 +3,11 @@ # script/alpine_setup: Adds all the system packages, directors, users, etc. # required to run the application on Alpine -# If a command fails, exit the script -set -e +source "$(dirname "${0}")"/../script/include/global_header.inc.sh -# Ensure we are in the app root directory (not the /script directory) -cd "$(dirname "${0}")/.." +# Set app specific items +APP_USER="atst" +APP_UID="8010" -APP_USER=${1} -APP_GROUP=${2} - -apk update -apk upgrade - -apk add bash -apk add dumb-init - -addgroup -g 8000 -S "${APP_GROUP}" -adduser -u 8010 -D -S -G "${APP_GROUP}" "${APP_USER}" +# Run the shared alpine setup script +source ./script/include/run_alpine_setup From 5159e3e163d62e54166705576a9fc222a83d5fc8 Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Mon, 9 Jul 2018 15:14:44 -0400 Subject: [PATCH 27/47] Remove script params; no longer required to pass these values --- deploy/docker/tester/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/docker/tester/Dockerfile b/deploy/docker/tester/Dockerfile index 30c6cc11..69a835e0 100644 --- a/deploy/docker/tester/Dockerfile +++ b/deploy/docker/tester/Dockerfile @@ -28,7 +28,7 @@ COPY script/alpine_setup ./script/ # Add required system packages and app user RUN set -x ; \ - script/alpine_setup "${APP_USER}" "${APP_GROUP}" + script/alpine_setup ### Items that will change almost every build ############################################# From 2ec2eefb4bc22c5287bd07591c43286cb18ec459 Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Mon, 9 Jul 2018 15:28:41 -0400 Subject: [PATCH 28/47] Fix variable name --- script/include/run_alpine_setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/include/run_alpine_setup b/script/include/run_alpine_setup index c094521a..532d613b 100755 --- a/script/include/run_alpine_setup +++ b/script/include/run_alpine_setup @@ -8,7 +8,7 @@ source ./script/include/alpine_setup_functions.inc.sh if [ -z "${APP_GROUP+is_set}" ] || \ [ -z "${APP_GID+is_set}" ]; then APP_GROUP="atat" - APP_GROUP_ID="8000" + APP_GID="8000" fi # If USER information is incomplete, error out From 6a4069853a343db612c58022998ab51099765f91 Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Mon, 9 Jul 2018 15:29:30 -0400 Subject: [PATCH 29/47] Update first copy to include ALL scripts --- deploy/docker/tester/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/docker/tester/Dockerfile b/deploy/docker/tester/Dockerfile index 69a835e0..c490c6ec 100644 --- a/deploy/docker/tester/Dockerfile +++ b/deploy/docker/tester/Dockerfile @@ -23,8 +23,8 @@ RUN set -x ; \ # Set working dir WORKDIR ${APP_DIR} -# Copy over alpine setup script -COPY script/alpine_setup ./script/ +# Copy over setup scripts +COPY script/ ./script/ # Add required system packages and app user RUN set -x ; \ From 10fa0a7ffad1d4e67fa5d1a861f013e3d640249d Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Tue, 10 Jul 2018 09:30:02 -0400 Subject: [PATCH 30/47] Update script to use global header file --- script/update | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/script/update b/script/update index 752ad3ad..8cb772c1 100755 --- a/script/update +++ b/script/update @@ -1,10 +1,8 @@ #!/bin/bash -# If a command fails, exit the script -set -e +# script/update: Update dependencies -# Ensure we are in the app root directory (not the /script directory) -cd "$(dirname "${0}")/.." +source "$(dirname "${0}")"/../script/include/global_header.inc.sh -# Update dependencies -script/bootstrap +# Run the bootstrap script +source ./script/bootstrap From 2b95291f936f5988c9cb4924b65044b0d1ad9248 Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Tue, 10 Jul 2018 09:46:44 -0400 Subject: [PATCH 31/47] Update script to use global header and no trap The server script should just give a clean, foreground execution of the app. A second script has been added to a dev server launch to try to get a backgrounding wrapper in place. --- script/server | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/script/server b/script/server index fa78012d..47e7cbaa 100755 --- a/script/server +++ b/script/server @@ -1,26 +1,8 @@ #!/bin/bash -reap() { - kill -TERM $child - sleep 0.1 - exit -} +# script/server: Launch the server -trap reap TERM INT - -# If a command fails, exit the script -set -e - -# Ensure we are in the app root directory (not the /script directory) -cd "$(dirname "${0}")/.." - -if [ -z "${SKIP_PIPENV+xxxx}" ]; then - CMD_PREFIX='pipenv run ' -fi -PYTHON_CMD="${CMD_PREFIX}python" +source "$(dirname "${0}")"/../script/include/global_header.inc.sh # Launch the app -${PYTHON_CMD} app.py ${@} & -child=$! - -wait $child +run_command "./app.py ${LAUNCH_ARGS}" From 2b426cb784d204305db94795dc15ea4c12a96abc Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Tue, 10 Jul 2018 09:48:16 -0400 Subject: [PATCH 32/47] Add script to background a server launch --- script/dev_server_WIP | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100755 script/dev_server_WIP diff --git a/script/dev_server_WIP b/script/dev_server_WIP new file mode 100755 index 00000000..f0a64ae4 --- /dev/null +++ b/script/dev_server_WIP @@ -0,0 +1,26 @@ +#!/bin/bash + +# script/local_server: Launch a local dev version of the server + +source "$(dirname "${0}")"/../script/include/global_header.inc.sh + +# Create a function to trap signals with +reap() { + kill -TERM "${child}" + sleep 0.1 + exit +} + +trap reap TERM INT + +# Set server launch related environment variables +DEBUG=1 +LAUNCH_ARGS="$*" + +# Launch the app +source ./script/server + +# Capture the PID of the child process +child=$! + +wait $child From 263fcd1c35796b04b5aea24ba56ab466d93878b4 Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Tue, 10 Jul 2018 09:59:12 -0400 Subject: [PATCH 33/47] Update comments --- script/dev_server_WIP | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/script/dev_server_WIP b/script/dev_server_WIP index f0a64ae4..202df4fe 100755 --- a/script/dev_server_WIP +++ b/script/dev_server_WIP @@ -1,6 +1,10 @@ #!/bin/bash -# script/local_server: Launch a local dev version of the server +# script/dev_server: Launch a local dev version of the server in the background + +# +# WIP +# source "$(dirname "${0}")"/../script/include/global_header.inc.sh @@ -11,6 +15,7 @@ reap() { exit } +# Register trap trap reap TERM INT # Set server launch related environment variables From 7adc36eed3f435c54dfe85c92c1df170a2a912bc Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Tue, 10 Jul 2018 10:07:38 -0400 Subject: [PATCH 34/47] Remove interpretor; file should be sourced not executed --- script/include/alpine_setup_functions.inc.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/script/include/alpine_setup_functions.inc.sh b/script/include/alpine_setup_functions.inc.sh index d28b88a3..19d60985 100755 --- a/script/include/alpine_setup_functions.inc.sh +++ b/script/include/alpine_setup_functions.inc.sh @@ -1,5 +1,3 @@ -#!/bin/sh - # alpine_setup_functions: Functions used by the run_alpine_setup script update_system_packages() { From d2c610fbf7cbf7751b892e979bc964efd87881eb Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Tue, 10 Jul 2018 10:08:05 -0400 Subject: [PATCH 35/47] Remove loop: only sourcing one file now --- script/include/global_header.inc.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/script/include/global_header.inc.sh b/script/include/global_header.inc.sh index 01e1709d..45989ee1 100755 --- a/script/include/global_header.inc.sh +++ b/script/include/global_header.inc.sh @@ -9,7 +9,4 @@ cd "$(dirname "${0}")/.." # Source all function definition files -for function_snippet in ./script/include/helper_functions.inc.sh -do - source "${function_snippet}" -done +source ./script/include/helper_functions.inc.sh From 40494e8f634b04d744d630cdecf88f60e68da65a Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Tue, 10 Jul 2018 10:08:45 -0400 Subject: [PATCH 36/47] Standardize code styling --- script/include/bootstrap_functions.inc.sh | 1 + script/include/helper_functions.inc.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/script/include/bootstrap_functions.inc.sh b/script/include/bootstrap_functions.inc.sh index 33e22bf8..599fc207 100644 --- a/script/include/bootstrap_functions.inc.sh +++ b/script/include/bootstrap_functions.inc.sh @@ -2,6 +2,7 @@ install_python_packages() { local install_flags="${1}" + pipenv install ${install_flags} return $? } diff --git a/script/include/helper_functions.inc.sh b/script/include/helper_functions.inc.sh index 96218f9a..e24594c2 100644 --- a/script/include/helper_functions.inc.sh +++ b/script/include/helper_functions.inc.sh @@ -14,6 +14,7 @@ check_system_pip_for () { # Used whenever an environment sensitive command is being run run_command () { local cmd="${1}" + pipenv run ${cmd} return $? } From e23b767a721d4bb8c42e47c0f245596aac39bf3d Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Tue, 10 Jul 2018 10:10:38 -0400 Subject: [PATCH 37/47] Standardize code styling --- script/include/run_bootstrap | 3 +-- script/include/run_test | 2 +- script/include/test_functions.inc.sh | 2 ++ 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/script/include/run_bootstrap b/script/include/run_bootstrap index a3d11798..9115b004 100755 --- a/script/include/run_bootstrap +++ b/script/include/run_bootstrap @@ -1,4 +1,4 @@ -# include/run_bootstrap: Install application dependencies +# run_bootstrap: Install application dependencies # Load bootstrap functions source ./script/include/bootstrap_functions.inc.sh @@ -10,7 +10,6 @@ if [ -z "${PIPENV_INSTALL_FLAGS+is_set}" ]; then fi ## Main - if [ "${INSTALL_PYTHON_PACKAGES}" = "true" ]; then install_python_packages "${PIPENV_INSTALL_FLAGS}" fi diff --git a/script/include/run_test b/script/include/run_test index 4187c2d1..d5f6bc5f 100755 --- a/script/include/run_test +++ b/script/include/run_test @@ -1,4 +1,4 @@ -# include/run_test: Execute code checkers and unit tests +# run_test: Execute code checkers and unit tests # Load test functions source ./script/include/test_functions.inc.sh diff --git a/script/include/test_functions.inc.sh b/script/include/test_functions.inc.sh index 23e231d8..622c0bfb 100644 --- a/script/include/test_functions.inc.sh +++ b/script/include/test_functions.inc.sh @@ -2,12 +2,14 @@ run_python_lint() { local python_files="${1}" + run_command "pylint ${python_files}" return $? } run_python_static_analysis() { local python_files="${1}" + run_command "bandit -c ./.bandit_config -r ${python_files}" return $? } From d2eff80de917e6e7f91005d045770ef91447cb3d Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Tue, 10 Jul 2018 10:10:59 -0400 Subject: [PATCH 38/47] Add error message output before aborting --- script/include/run_alpine_setup | 1 + 1 file changed, 1 insertion(+) diff --git a/script/include/run_alpine_setup b/script/include/run_alpine_setup index 532d613b..837f7e67 100755 --- a/script/include/run_alpine_setup +++ b/script/include/run_alpine_setup @@ -14,6 +14,7 @@ fi # If USER information is incomplete, error out if [ -z "${APP_USER+is_set}" ] || \ [ -z "${APP_UID+is_set}" ]; then + echo "ERROR: Missing app user information! Received: ${APP_USER}:${APP_UID}" exit 1 fi From 89e14268e933825744e72120db58a65b90a5982d Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Tue, 10 Jul 2018 10:11:30 -0400 Subject: [PATCH 39/47] Fix function return value and clean up code style - Fix create_virtual_environment return code; use pipenv exit code not pipenv output - Clean up code style in install_pipenv --- script/include/setup_functions.inc.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/script/include/setup_functions.inc.sh b/script/include/setup_functions.inc.sh index ea26d23a..ef0ff935 100644 --- a/script/include/setup_functions.inc.sh +++ b/script/include/setup_functions.inc.sh @@ -1,20 +1,20 @@ # setup_functions.inc.sh: Functions used by the setup script install_pipenv() { - exit_code=0 + return_code=0 # Ensure we are not in a virtual env already - if [ -z "${VIRTUAL_ENV+xxxx}" ]; then + if [ -z "${VIRTUAL_ENV+is_set}" ]; then if ! check_system_pip_for pipenv; then # pipenv is not installed, so install it echo "Installing pipenv..." pip install pipenv # Capture pip exit code - exit_code="${?}" + return_code="${?}" fi fi - return "${exit_code}" + return "${return_code}" } create_virtual_environment() { @@ -32,7 +32,8 @@ create_virtual_environment() { # The environment will be in a directory called .venv off the app # root directory echo "Creating virtual environment using Python version ${python_version}..." - return $(PIPENV_VENV_IN_PROJECT=true pipenv --python "${python_version}") + PIPENV_VENV_IN_PROJECT=true pipenv --python "${python_version}" + return $? } install_sass() { From bbe9880db367d62fb5e448f08fca0a33042d4101 Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Tue, 10 Jul 2018 10:13:24 -0400 Subject: [PATCH 40/47] Ensure node modules get a fresh install on reset --- script/include/run_setup | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/script/include/run_setup b/script/include/run_setup index 97fa0808..fd026477 100755 --- a/script/include/run_setup +++ b/script/include/run_setup @@ -1,5 +1,5 @@ -# include/setup: Set up application for the first time after cloning, or set it -# back to the initial first unused state. +# setup: Set up application for the first time after cloning, or set it +# back to the initial first unused state. # Load setup functions source ./script/include/setup_functions.inc.sh @@ -16,6 +16,9 @@ if [ -z "${INSTALL_SASS+is_set}" ]; then fi ## Main +# Remove any existing node modules as part of initial app setup or reset +rm -rf ./node_modules + if [ "${CREATE_VENV}" = "true" ]; then install_pipenv create_virtual_environment From 628b3dc562961a1591159470fe40f12a6700a474 Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Tue, 10 Jul 2018 10:52:55 -0400 Subject: [PATCH 41/47] Modify wrapper to use PGID when terminating app --- script/dev_server_WIP | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/script/dev_server_WIP b/script/dev_server_WIP index 202df4fe..1f5f75a1 100755 --- a/script/dev_server_WIP +++ b/script/dev_server_WIP @@ -8,24 +8,25 @@ source "$(dirname "${0}")"/../script/include/global_header.inc.sh -# Create a function to trap signals with +# Create a function to run after a trap is triggered reap() { - kill -TERM "${child}" + kill -s SIGTERM -- "-$$" sleep 0.1 exit } -# Register trap -trap reap TERM INT +# Register trapping of SIGTERM and SIGINT +trap reap SIGTERM SIGINT + +# Display the script PID, which will also be the process group ID for all +# child processes +echo "Process Group: $$" # Set server launch related environment variables DEBUG=1 LAUNCH_ARGS="$*" + # Launch the app -source ./script/server - -# Capture the PID of the child process -child=$! - -wait $child +source ./script/server & +wait From f5f809c1673f6288ff22ba759540683ba1013674 Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Tue, 10 Jul 2018 10:54:27 -0400 Subject: [PATCH 42/47] Account for new script names and forced pipenv --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 88698d19..64996466 100644 --- a/README.md +++ b/README.md @@ -19,19 +19,19 @@ Additionally, ATST requires a redis instance for session management. Have redis ## Running (development) -To start the app and watch for changes: +To start the app locally in the foreground and watch for changes: - DEBUG=1 script/server + script/dev_server ## Testing -To run unit tests: +To run all linting and tests: script/test -or +To run only the unit tests: - python -m pytest + pipenv run python -m pytest ## Notes From 9dbc6cd1801fe80cf8c2025d6caa107ae6485f07 Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Tue, 10 Jul 2018 10:54:38 -0400 Subject: [PATCH 43/47] Rename file: no longer a work in progress --- script/{dev_server_WIP => dev_server} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename script/{dev_server_WIP => dev_server} (100%) diff --git a/script/dev_server_WIP b/script/dev_server similarity index 100% rename from script/dev_server_WIP rename to script/dev_server From 0535a45bd97b2d032ee6cec3d892ad46af13b686 Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Tue, 10 Jul 2018 12:01:56 -0400 Subject: [PATCH 44/47] Check for pipenv and error out if not present --- script/include/run_setup | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/script/include/run_setup b/script/include/run_setup index fd026477..2ff7eb52 100755 --- a/script/include/run_setup +++ b/script/include/run_setup @@ -20,7 +20,11 @@ fi rm -rf ./node_modules if [ "${CREATE_VENV}" = "true" ]; then - install_pipenv + # Ensure pipenv is installed + if ! check_system_pip_for pipenv; then + echo "ERROR: pipenv is required but is not present" + exit 1 + fi create_virtual_environment fi From 2864300a4e7ef27eaa3152ee2e94213a534f745a Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Tue, 10 Jul 2018 14:55:02 -0400 Subject: [PATCH 45/47] Add function for installing pip packages in the venv --- script/include/helper_functions.inc.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/script/include/helper_functions.inc.sh b/script/include/helper_functions.inc.sh index e24594c2..84792652 100644 --- a/script/include/helper_functions.inc.sh +++ b/script/include/helper_functions.inc.sh @@ -11,6 +11,14 @@ check_system_pip_for () { return $? } +pip_install () { + local packages="${1}" + local flags="${2}" + + run_command "pip install ${flags} ${packages}" + return $? +} + # Used whenever an environment sensitive command is being run run_command () { local cmd="${1}" From d620fb6cc08a3a637040f65d3816712fd940eff6 Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Tue, 10 Jul 2018 14:55:26 -0400 Subject: [PATCH 46/47] Add pip update in venv --- script/include/run_setup | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/script/include/run_setup b/script/include/run_setup index 2ff7eb52..c4c691bb 100755 --- a/script/include/run_setup +++ b/script/include/run_setup @@ -15,6 +15,11 @@ if [ -z "${INSTALL_SASS+is_set}" ]; then INSTALL_SASS="false" fi +# If PIP_VERSION is not set, set it to "10.*" +if [ -z "${PIP_VERSION+is_set}" ]; then + PIP_VERSION="10.*" +fi + ## Main # Remove any existing node modules as part of initial app setup or reset rm -rf ./node_modules @@ -26,6 +31,7 @@ if [ "${CREATE_VENV}" = "true" ]; then exit 1 fi create_virtual_environment + pip_install "pip==${PIP_VERSION}" "--upgrade" fi if [ "${INSTALL_SASS}" = "true" ]; then From fc62dcdb65b5126be94a87083302a96fef9c6d08 Mon Sep 17 00:00:00 2001 From: Devon Mackay Date: Tue, 10 Jul 2018 15:20:21 -0400 Subject: [PATCH 47/47] Update pipenv check to just test the binary --- script/include/run_setup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/include/run_setup b/script/include/run_setup index c4c691bb..376802f8 100755 --- a/script/include/run_setup +++ b/script/include/run_setup @@ -26,8 +26,8 @@ rm -rf ./node_modules if [ "${CREATE_VENV}" = "true" ]; then # Ensure pipenv is installed - if ! check_system_pip_for pipenv; then - echo "ERROR: pipenv is required but is not present" + if ! pipenv --version >/dev/null 2>&1 ; then + echo "ERROR: pipenv is malfunctioning or not present" exit 1 fi create_virtual_environment