Merge pull request #64 from dod-ccpo/script-rework

Script standardization and shared functions
This commit is contained in:
Devon 2018-07-10 15:35:03 -04:00 committed by GitHub
commit 74eae52869
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 495 additions and 123 deletions

156
.bandit_config Normal file
View File

@ -0,0 +1,156 @@
### This config may optionally select a subset of tests to run or skip by
### filling out the 'tests' and 'skips' lists given below. If no tests are
### specified for inclusion then it is assumed all tests are desired. The skips
### set will remove specific tests from the include set.
### Note that the same test ID should not appear in both 'tests' and 'skips',
### this would be nonsensical and is detected by Bandit at runtime.
# (optional) list included test IDs here, eg '[B101, B406]':
tests:
# (optional) list skipped test IDs here, eg '[B101, B406]':
skips:
### (optional) plugin settings - some test plugins require configuration data
### that may be given here, per-plugin. All bandit test plugins have a built in
### set of sensible defaults and these will be used if no configuration is
### provided. It is not necessary to provide settings for every (or any) plugin
### if the defaults are acceptable.
any_other_function_with_shell_equals_true:
no_shell: [os.execl, os.execle, os.execlp, os.execlpe, os.execv, os.execve, os.execvp,
os.execvpe, os.spawnl, os.spawnle, os.spawnlp, os.spawnlpe, os.spawnv, os.spawnve,
os.spawnvp, os.spawnvpe, os.startfile]
shell: [os.system, os.popen, os.popen2, os.popen3, os.popen4, popen2.popen2, popen2.popen3,
popen2.popen4, popen2.Popen3, popen2.Popen4, commands.getoutput, commands.getstatusoutput]
subprocess: [subprocess.Popen, subprocess.call, subprocess.check_call, subprocess.check_output,
utils.execute, utils.execute_with_timeout]
execute_with_run_as_root_equals_true:
function_names: [ceilometer.utils.execute, cinder.utils.execute, neutron.agent.linux.utils.execute,
nova.utils.execute, nova.utils.trycmd]
hardcoded_tmp_directory:
tmp_dirs: [/tmp, /var/tmp, /dev/shm]
linux_commands_wildcard_injection:
no_shell: [os.execl, os.execle, os.execlp, os.execlpe, os.execv, os.execve, os.execvp,
os.execvpe, os.spawnl, os.spawnle, os.spawnlp, os.spawnlpe, os.spawnv, os.spawnve,
os.spawnvp, os.spawnvpe, os.startfile]
shell: [os.system, os.popen, os.popen2, os.popen3, os.popen4, popen2.popen2, popen2.popen3,
popen2.popen4, popen2.Popen3, popen2.Popen4, commands.getoutput, commands.getstatusoutput]
subprocess: [subprocess.Popen, subprocess.call, subprocess.check_call, subprocess.check_output,
utils.execute, utils.execute_with_timeout]
password_config_option_not_marked_secret:
function_names: [oslo.config.cfg.StrOpt, oslo_config.cfg.StrOpt]
ssl_with_bad_defaults:
bad_protocol_versions: [PROTOCOL_SSLv2, SSLv2_METHOD, SSLv23_METHOD, PROTOCOL_SSLv3,
PROTOCOL_TLSv1, SSLv3_METHOD, TLSv1_METHOD]
ssl_with_bad_version:
bad_protocol_versions: [PROTOCOL_SSLv2, SSLv2_METHOD, SSLv23_METHOD, PROTOCOL_SSLv3,
PROTOCOL_TLSv1, SSLv3_METHOD, TLSv1_METHOD]
start_process_with_a_shell:
no_shell: [os.execl, os.execle, os.execlp, os.execlpe, os.execv, os.execve, os.execvp,
os.execvpe, os.spawnl, os.spawnle, os.spawnlp, os.spawnlpe, os.spawnv, os.spawnve,
os.spawnvp, os.spawnvpe, os.startfile]
shell: [os.system, os.popen, os.popen2, os.popen3, os.popen4, popen2.popen2, popen2.popen3,
popen2.popen4, popen2.Popen3, popen2.Popen4, commands.getoutput, commands.getstatusoutput]
subprocess: [subprocess.Popen, subprocess.call, subprocess.check_call, subprocess.check_output,
utils.execute, utils.execute_with_timeout]
start_process_with_no_shell:
no_shell: [os.execl, os.execle, os.execlp, os.execlpe, os.execv, os.execve, os.execvp,
os.execvpe, os.spawnl, os.spawnle, os.spawnlp, os.spawnlpe, os.spawnv, os.spawnve,
os.spawnvp, os.spawnvpe, os.startfile]
shell: [os.system, os.popen, os.popen2, os.popen3, os.popen4, popen2.popen2, popen2.popen3,
popen2.popen4, popen2.Popen3, popen2.Popen4, commands.getoutput, commands.getstatusoutput]
subprocess: [subprocess.Popen, subprocess.call, subprocess.check_call, subprocess.check_output,
utils.execute, utils.execute_with_timeout]
start_process_with_partial_path:
no_shell: [os.execl, os.execle, os.execlp, os.execlpe, os.execv, os.execve, os.execvp,
os.execvpe, os.spawnl, os.spawnle, os.spawnlp, os.spawnlpe, os.spawnv, os.spawnve,
os.spawnvp, os.spawnvpe, os.startfile]
shell: [os.system, os.popen, os.popen2, os.popen3, os.popen4, popen2.popen2, popen2.popen3,
popen2.popen4, popen2.Popen3, popen2.Popen4, commands.getoutput, commands.getstatusoutput]
subprocess: [subprocess.Popen, subprocess.call, subprocess.check_call, subprocess.check_output,
utils.execute, utils.execute_with_timeout]
subprocess_popen_with_shell_equals_true:
no_shell: [os.execl, os.execle, os.execlp, os.execlpe, os.execv, os.execve, os.execvp,
os.execvpe, os.spawnl, os.spawnle, os.spawnlp, os.spawnlpe, os.spawnv, os.spawnve,
os.spawnvp, os.spawnvpe, os.startfile]
shell: [os.system, os.popen, os.popen2, os.popen3, os.popen4, popen2.popen2, popen2.popen3,
popen2.popen4, popen2.Popen3, popen2.Popen4, commands.getoutput, commands.getstatusoutput]
subprocess: [subprocess.Popen, subprocess.call, subprocess.check_call, subprocess.check_output,
utils.execute, utils.execute_with_timeout]
subprocess_without_shell_equals_true:
no_shell: [os.execl, os.execle, os.execlp, os.execlpe, os.execv, os.execve, os.execvp,
os.execvpe, os.spawnl, os.spawnle, os.spawnlp, os.spawnlpe, os.spawnv, os.spawnve,
os.spawnvp, os.spawnvpe, os.startfile]
shell: [os.system, os.popen, os.popen2, os.popen3, os.popen4, popen2.popen2, popen2.popen3,
popen2.popen4, popen2.Popen3, popen2.Popen4, commands.getoutput, commands.getstatusoutput]
subprocess: [subprocess.Popen, subprocess.call, subprocess.check_call, subprocess.check_output,
utils.execute, utils.execute_with_timeout]
try_except_continue: {check_typed_exception: false}
try_except_pass: {check_typed_exception: false}
### Reference of Available tests:
# B101 : assert_used
# B102 : exec_used
# B103 : set_bad_file_permissions
# B104 : hardcoded_bind_all_interfaces
# B105 : hardcoded_password_string
# B106 : hardcoded_password_funcarg
# B107 : hardcoded_password_default
# B108 : hardcoded_tmp_directory
# B109 : password_config_option_not_marked_secret
# B110 : try_except_pass
# B111 : execute_with_run_as_root_equals_true
# B112 : try_except_continue
# B201 : flask_debug_true
# B301 : pickle
# B302 : marshal
# B303 : md5
# B304 : ciphers
# B305 : cipher_modes
# B306 : mktemp_q
# B307 : eval
# B308 : mark_safe
# B309 : httpsconnection
# B310 : urllib_urlopen
# B311 : random
# B312 : telnetlib
# B313 : xml_bad_cElementTree
# B314 : xml_bad_ElementTree
# B315 : xml_bad_expatreader
# B316 : xml_bad_expatbuilder
# B317 : xml_bad_sax
# B318 : xml_bad_minidom
# B319 : xml_bad_pulldom
# B320 : xml_bad_etree
# B321 : ftplib
# B322 : input
# B401 : import_telnetlib
# B402 : import_ftplib
# B403 : import_pickle
# B404 : import_subprocess
# B405 : import_xml_etree
# B406 : import_xml_sax
# B407 : import_xml_expat
# B408 : import_xml_minidom
# B409 : import_xml_pulldom
# B410 : import_lxml
# B411 : import_xmlrpclib
# B412 : import_httpoxy
# B501 : request_with_no_cert_validation
# B502 : ssl_with_bad_version
# B503 : ssl_with_bad_defaults
# B504 : ssl_with_no_version
# B505 : weak_cryptographic_key
# B506 : yaml_load
# B601 : paramiko_calls
# B602 : subprocess_popen_with_shell_equals_true
# B603 : subprocess_without_shell_equals_true
# B604 : any_other_function_with_shell_equals_true
# B605 : start_process_with_a_shell
# B606 : start_process_with_no_shell
# B607 : start_process_with_partial_path
# B608 : hardcoded_sql_expressions
# B609 : linux_commands_wildcard_injection
# B701 : jinja2_autoescape_false
# B702 : use_of_mako_templates

View File

@ -7,15 +7,15 @@ env:
- TESTER_IMAGE_NAME=atst-tester
- PROD_IMAGE_NAME=atst-prod
before_install:
before_script:
- docker login -u $ATAT_DOCKER_REGISTRY_USERNAME -p $ATAT_DOCKER_REGISTRY_PASSWORD $ATAT_DOCKER_REGISTRY_URL
- docker build --tag "${TESTER_IMAGE_NAME}" . -f docker/tester/Dockerfile
- docker build --tag "${TESTER_IMAGE_NAME}" . -f deploy/docker/tester/Dockerfile
script:
- docker run "${TESTER_IMAGE_NAME}"
before_deploy:
- docker build --tag "${PROD_IMAGE_NAME}" . -f docker/prod/Dockerfile
- docker build --tag "${PROD_IMAGE_NAME}" . -f deploy/docker/prod/Dockerfile
- git_sha="$(git rev-parse --short HEAD)"
- remote_image_name="${ATAT_DOCKER_REGISTRY_URL}/${PROD_IMAGE_NAME}:${git_sha}"
- docker tag "${PROD_IMAGE_NAME}" "${remote_image_name}"

View File

@ -19,19 +19,19 @@ Additionally, ATST requires a redis instance for session management. Have redis
## Running (development)
To start the app and watch for changes:
To start the app locally in the foreground and watch for changes:
DEBUG=1 script/server
script/dev_server
## Testing
To run unit tests:
To run all linting and tests:
script/test
or
To run only the unit tests:
python -m pytest
pipenv run python -m pytest
## Notes

View File

@ -23,12 +23,12 @@ RUN set -x ; \
# Set working dir
WORKDIR ${APP_DIR}
# Copy over alpine setup script
COPY script/alpine_setup ./script/
# Copy over setup scripts
COPY script/ ./script/
# Add required system packages and app user
RUN set -x ; \
script/alpine_setup "${APP_USER}" "${APP_GROUP}"
script/alpine_setup
### Items that will change almost every build
#############################################

View File

@ -3,20 +3,11 @@
# script/alpine_setup: Adds all the system packages, directors, users, etc.
# required to run the application on Alpine
# If a command fails, exit the script
set -e
source "$(dirname "${0}")"/../script/include/global_header.inc.sh
# Ensure we are in the app root directory (not the /script directory)
cd "$(dirname "${0}")/.."
# Set app specific items
APP_USER="atst"
APP_UID="8010"
APP_USER=${1}
APP_GROUP=${2}
apk update
apk upgrade
apk add bash
apk add dumb-init
addgroup -g 8000 -S "${APP_GROUP}"
adduser -u 8010 -D -S -G "${APP_GROUP}" "${APP_USER}"
# Run the shared alpine setup script
source ./script/include/run_alpine_setup

View File

@ -3,33 +3,18 @@
# script/bootstrap: Resolve all dependencies that the application requires to
# run.
# If a command fails, exit the script
set -e
source "$(dirname "${0}")"/../script/include/global_header.inc.sh
# Ensure we are in the app root directory (not the /script directory)
cd "$(dirname "${0}")/.."
# Set sass compiling command for this app
COMPILE_SASS_CMD="webassets -m atst.assets build"
if [ -z "${CIBUILD+xxxx}" ]; then
CMD_PREFIX='pipenv run '
fi
PIP_CMD="${CMD_PREFIX}pip"
WEBASSETS_CMD="${CMD_PREFIX}webassets"
# Enable python and node package installation
INSTALL_PYTHON_PACKAGES="true"
INSTALL_NODE_PACKAGES="true"
PIPENV_INSTALL_FLAGS='--dev'
if [ -n "${CIBUILD}" ]; then
PIPENV_INSTALL_FLAGS+=' --system --ignore-pipfile'
fi
# Run the shared bootstrap script
source ./script/include/run_bootstrap
# Install Python dependencies
${PIP_CMD} install --upgrade pip
pipenv install ${PIPENV_INSTALL_FLAGS}
# Install uswds node module and dependencies
npm install
# Relink uswds fonts into the /static directory
# Link USWDS fonts into the /static directory
rm -f ./static/fonts
ln -s ../node_modules/uswds/src/fonts ./static/fonts
# Precompile assets for deployment
${WEBASSETS_CMD} -m atst.assets build
ln -s ../node/modules/uswds/src/fonts ./static/fonts

View File

@ -2,15 +2,7 @@
# script/cibuild: Run CI related checks and tests
# If a command fails, exit the script
set -e
# Ensure we are in the app root directory (not the /script directory)
cd "$(dirname "${0}")/.."
source "$(dirname "${0}")"/../script/include/global_header.inc.sh
# Run lint/style checks and unit tests
script/test
# Run static code analysis security checks
# (excluding the tests and node_modules subdirs)
bandit -r . -x node_modules,tests
source ./script/test

32
script/dev_server Executable file
View File

@ -0,0 +1,32 @@
#!/bin/bash
# script/dev_server: Launch a local dev version of the server in the background
#
# WIP
#
source "$(dirname "${0}")"/../script/include/global_header.inc.sh
# Create a function to run after a trap is triggered
reap() {
kill -s SIGTERM -- "-$$"
sleep 0.1
exit
}
# Register trapping of SIGTERM and SIGINT
trap reap SIGTERM SIGINT
# Display the script PID, which will also be the process group ID for all
# child processes
echo "Process Group: $$"
# Set server launch related environment variables
DEBUG=1
LAUNCH_ARGS="$*"
# Launch the app
source ./script/server &
wait

View File

@ -0,0 +1,30 @@
# alpine_setup_functions: Functions used by the run_alpine_setup script
update_system_packages() {
apk update
apk upgrade
}
install_package() {
local package_name=${1}
apk add ${1}
return $?
}
add_group() {
local group_name="${1}"
local gid="${2}"
addgroup -g "${gid}" -S "${group_name}"
return $?
}
add_user() {
local username="${1}"
local primary_group="${2}"
local uid="${3}"
adduser -u "${3}" -D -S -G "${primary_group}" "${username}"
return $?
}

View File

@ -0,0 +1,13 @@
# bootstrap_functions.inc.sh: Functions used by the bootstrap script
install_python_packages() {
local install_flags="${1}"
pipenv install ${install_flags}
return $?
}
install_node_packages() {
npm install
return $?
}

View File

@ -0,0 +1,12 @@
# global_header.inc: Any basic things that should be executed at the
# beginning of any and every script
# If any command fails, immediately exit the script
set -e
# Ensure the working directory is the app root directory
cd "$(dirname "${0}")/.."
# Source all function definition files
source ./script/include/helper_functions.inc.sh

View File

@ -0,0 +1,28 @@
# helper_functions.inc.sh: General helper functions
# Check pip to see if the given package is installed
# (returns 0 if installed, 2 if not installed)
check_system_pip_for () {
local package_name="${1}"
# Use 'pip list' to see if the requested package is already installed
pip list --format=columns --disable-pip-version-check | \
grep -Fe "${package_name}" >/dev/null 2>&1
return $?
}
pip_install () {
local packages="${1}"
local flags="${2}"
run_command "pip install ${flags} ${packages}"
return $?
}
# Used whenever an environment sensitive command is being run
run_command () {
local cmd="${1}"
pipenv run ${cmd}
return $?
}

26
script/include/run_alpine_setup Executable file
View File

@ -0,0 +1,26 @@
# run_alpine_setup: Install basic system requirements for an app to run
# Load alpine setup functions
source ./script/include/alpine_setup_functions.inc.sh
## Set option defaults
# If GROUP information is incomplete, use the default one
if [ -z "${APP_GROUP+is_set}" ] || \
[ -z "${APP_GID+is_set}" ]; then
APP_GROUP="atat"
APP_GID="8000"
fi
# If USER information is incomplete, error out
if [ -z "${APP_USER+is_set}" ] || \
[ -z "${APP_UID+is_set}" ]; then
echo "ERROR: Missing app user information! Received: ${APP_USER}:${APP_UID}"
exit 1
fi
## Main
update_system_packages
install_package "bash"
install_package "dumb-init"
add_group "${APP_GROUP}" "${APP_GID}"
add_user "${APP_USER}" "${APP_GROUP}" "${APP_UID}"

23
script/include/run_bootstrap Executable file
View File

@ -0,0 +1,23 @@
# run_bootstrap: Install application dependencies
# Load bootstrap functions
source ./script/include/bootstrap_functions.inc.sh
## Set option defaults
# If PIPENV_INSTALL_FLAGS is not set, give it the default value of "--dev"
if [ -z "${PIPENV_INSTALL_FLAGS+is_set}" ]; then
PIPENV_INSTALL_FLAGS="--dev"
fi
## Main
if [ "${INSTALL_PYTHON_PACKAGES}" = "true" ]; then
install_python_packages "${PIPENV_INSTALL_FLAGS}"
fi
if [ "${INSTALL_NODE_PACKAGES}" = "true" ]; then
install_node_packages
fi
if [ -n "${COMPILE_SASS_CMD}" ]; then
run_command "${COMPILE_SASS_CMD}"
fi

42
script/include/run_setup Executable file
View File

@ -0,0 +1,42 @@
# setup: Set up application for the first time after cloning, or set it
# back to the initial first unused state.
# Load setup functions
source ./script/include/setup_functions.inc.sh
## Set option defaults
# If CREATE_VENV is not set, set it to "true"
if [ -z "${CREATE_VENV+is_set}" ]; then
CREATE_VENV="true"
fi
# If INSTALL_SASS is not set, set it to "false"
if [ -z "${INSTALL_SASS+is_set}" ]; then
INSTALL_SASS="false"
fi
# If PIP_VERSION is not set, set it to "10.*"
if [ -z "${PIP_VERSION+is_set}" ]; then
PIP_VERSION="10.*"
fi
## Main
# Remove any existing node modules as part of initial app setup or reset
rm -rf ./node_modules
if [ "${CREATE_VENV}" = "true" ]; then
# Ensure pipenv is installed
if ! pipenv --version >/dev/null 2>&1 ; then
echo "ERROR: pipenv is malfunctioning or not present"
exit 1
fi
create_virtual_environment
pip_install "pip==${PIP_VERSION}" "--upgrade"
fi
if [ "${INSTALL_SASS}" = "true" ]; then
install_sass
fi
# Install application dependencies
./script/bootstrap

17
script/include/run_test Executable file
View File

@ -0,0 +1,17 @@
# run_test: Execute code checkers and unit tests
# Load test functions
source ./script/include/test_functions.inc.sh
## Set option defaults
# If PYTHON_FILES is not set, give it the default value of "app.py"
if [ -z "${PYTHON_FILES+is_set}" ]; then
PYTHON_FILES="app.py"
fi
## Main
if [ "${RUN_PYTHON_TESTS}" = "true" ]; then
run_python_lint "${PYTHON_FILES}"
run_python_static_analysis "${PYTHON_FILES}"
run_python_unit_tests "${PYTHON_FILES}"
fi

View File

@ -0,0 +1,48 @@
# setup_functions.inc.sh: Functions used by the setup script
install_pipenv() {
return_code=0
# Ensure we are not in a virtual env already
if [ -z "${VIRTUAL_ENV+is_set}" ]; then
if ! check_system_pip_for pipenv; then
# pipenv is not installed, so install it
echo "Installing pipenv..."
pip install pipenv
# Capture pip exit code
return_code="${?}"
fi
fi
return "${return_code}"
}
create_virtual_environment() {
default_python_version=3.6
# Parse out the required Python version from the Pipfile
python_version=$(grep python_version ./Pipfile | cut -d '"' -f 2)
# If we ended up with an empty string for the required Python version,
# specify the default version
if [ -z "${python_version}" ]; then
python_version="${default_python_version}"
fi
# Create a new virtual environment for the app
# The environment will be in a directory called .venv off the app
# root directory
echo "Creating virtual environment using Python version ${python_version}..."
PIPENV_VENV_IN_PROJECT=true pipenv --python "${python_version}"
return $?
}
install_sass() {
if ! type sass >/dev/null; then
if type gem >/dev/null; then
echo 'Installing a sass compiler (gem)...'
gem install sass
else
echo 'Could not install a sass compiler. Please install a version of sass.'
fi
fi
}

View File

@ -0,0 +1,20 @@
# test_functions.inc.sh: Functions used by the run_test script
run_python_lint() {
local python_files="${1}"
run_command "pylint ${python_files}"
return $?
}
run_python_static_analysis() {
local python_files="${1}"
run_command "bandit -c ./.bandit_config -r ${python_files}"
return $?
}
run_python_unit_tests() {
run_command "python -m pytest -s"
return $?
}

View File

@ -1,26 +1,8 @@
#!/bin/bash
reap() {
kill -TERM $child
sleep 0.1
exit
}
# script/server: Launch the server
trap reap TERM INT
# If a command fails, exit the script
set -e
# Ensure we are in the app root directory (not the /script directory)
cd "$(dirname "${0}")/.."
if [ -z "${SKIP_PIPENV+xxxx}" ]; then
CMD_PREFIX='pipenv run '
fi
PYTHON_CMD="${CMD_PREFIX}python"
source "$(dirname "${0}")"/../script/include/global_header.inc.sh
# Launch the app
${PYTHON_CMD} app.py ${@} &
child=$!
wait $child
run_command "./app.py ${LAUNCH_ARGS}"

View File

@ -3,24 +3,10 @@
# script/setup: Set up application for the first time after cloning, or set it
# back to the initial first unused state.
# If a command fails, exit the script
set -e
source "$(dirname "${0}")"/../script/include/global_header.inc.sh
# Ensure we are in the app root directory (not the /script directory)
cd "$(dirname "${0}")/.."
# Turn on sass compiler installation
INSTALL_SASS="true"
# Install virtualenv
pip install pipenv
pipenv --python 3.6
if ! type sass > /dev/null; then
if type gem > /dev/null; then
echo 'installing a sass compiler...'
gem install sass
else
echo 'Could not install a sass compiler. Please install a version of sass.'
fi
fi
# Install application dependencies
script/bootstrap
# Run the shared setup script
source ./script/include/run_setup

View File

@ -2,22 +2,13 @@
# script/test: Run static code checks and unit tests
# If a command fails, exit the script
set -e
source "$(dirname "${0}")"/../script/include/global_header.inc.sh
# Ensure we are in the app root directory (not the /script directory)
cd "$(dirname "${0}")/.."
# Define all relevant python files and directories for this app
PYTHON_FILES="./app.py ./atst ./config"
if [ -z "${SKIP_PIPENV+xxxx}" ]; then
CMD_PREFIX='pipenv run '
fi
PYLINT_CMD="${CMD_PREFIX}pylint"
PYTHON_CMD="${CMD_PREFIX}python"
# Enable Python testing
RUN_PYTHON_TESTS="true"
# Run lint check
echo "Running lint..."
${PYLINT_CMD} app.py atst/ tests/
# Run unit tests
echo "Running unit tests..."
${PYTHON_CMD} -m pytest -s $*
# Run the shared test script
source ./script/include/run_test

View File

@ -1,10 +1,8 @@
#!/bin/bash
# If a command fails, exit the script
set -e
# script/update: Update dependencies
# Ensure we are in the app root directory (not the /script directory)
cd "$(dirname "${0}")/.."
source "$(dirname "${0}")"/../script/include/global_header.inc.sh
# Update dependencies
script/bootstrap
# Run the bootstrap script
source ./script/bootstrap