diff --git a/.gitmodules b/.gitmodules index 0e4a0b6f..e69de29b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +0,0 @@ -[submodule "script/include"] - path = script/include - url = https://github.com/dod-ccpo/scriptz.git - branch = master diff --git a/script/include b/script/include deleted file mode 160000 index bb072dd1..00000000 --- a/script/include +++ /dev/null @@ -1 +0,0 @@ -Subproject commit bb072dd10190a05c77973b85e9d4f8f1a7fccdde diff --git a/script/include/README.md b/script/include/README.md new file mode 100644 index 00000000..730e3358 --- /dev/null +++ b/script/include/README.md @@ -0,0 +1,60 @@ +# scriptz + +These script fragments are shared between all of the ATAT applications, powering +their /script files. See +[Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all) +and an ATAT application repository for details. + +- `global_header.inc.sh`: Run by all /script files at the start of the script. +- `helper_functions.inc.sh`: Contains general helper functions; sourced by +global_header.inc.sh +- `run_setup`: Main logic run by most /script/setup scripts, after they set any +relevant environment variables. Initializes the local environment to run the app. +- `run_bootstrap`: Main logic run by most /script/bootstrap scripts, after they set +any relevant environment variables. Installs application dependencies. +- `run_test`: Main logic run by most /script/test scripts, after they set any +relevant environment variables. Executes lint, static analysis, and unit tests. +- `run_update`: Main logic run by most /script/update scripts, after they set any + relevant environment variables. Executes application dependency updates, and + potentially DB migrations. +- `run_dev_server`: Main logic run by most /script/dev_server scripts, after any + relevant environment variables are set. Launches an application server in DEBUG + mode in the background. +- `run_alpine_setup`: Main logic run by most /script/alpine_setup scripts, after +they set any relevant environment variables. Adds Alpine Linux specific system +dependencies required to run the app (used by Docker builds). +- `setup_functions.inc.sh`: Sourced by run_setup; contains setup specific +functions. +- `bootstrap_functions.inc.sh`: Sourced by run_bootstrap; contains bootstrap +specific functions. +- `test_functions.inc.sh`: Sourced by run_test; contains test specific +functions. +- `update_functions.inc.sh`: Sourced by run_update; contains update specific + functions. +- `dev_server_functions.inc.sh`: Sourced by run_dev_server; contains dev_server + specific functions. +- `alpine_setup_functions.inc.sh`: Sourced by run_alpine_setup; contains +alpine_setup specific functions. + +## Requirements + +- **[Python 3.6](https://www.python.org/downloads/)** +- **[Pipenv](https://docs.pipenv.org/install/#installing-pipenv)** + +**Note:** None of these script fragments are designed to be run by themselves. +They should be sourced by the relevant bash script in an application's /script +directory. + +#### Ubuntu 18.04 Instructions + +Install pip: +``` +sudo apt install python3-distutils +curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py +python3 get-pip.py --user +``` + +Install pipenv: +``` +pip install --user pipenv +``` diff --git a/script/include/alpine_setup_functions.inc.sh b/script/include/alpine_setup_functions.inc.sh new file mode 100644 index 00000000..a352515c --- /dev/null +++ b/script/include/alpine_setup_functions.inc.sh @@ -0,0 +1,36 @@ +# alpine_setup_functions: Functions used by the run_alpine_setup script + +update_system_packages() { + local apk_cache_dir="/etc/apk/cache" + + apk update + apk upgrade + + if [ -d "${apk_cache_dir}" ] || [ -L "${apk_cache_dir}" ]; then + apk cache clean + fi +} + +install_package() { + local package_name=${1} + + apk add "${package_name}" + return $? +} + +add_group() { + local group_name="${1}" + local gid="${2}" + + addgroup -g "${gid}" -S "${group_name}" + return $? +} + +add_user() { + local username="${1}" + local primary_group="${2}" + local uid="${3}" + + adduser -u "${uid}" -D -S -G "${primary_group}" "${username}" + return $? +} diff --git a/script/include/bootstrap_functions.inc.sh b/script/include/bootstrap_functions.inc.sh new file mode 100644 index 00000000..f00460c1 --- /dev/null +++ b/script/include/bootstrap_functions.inc.sh @@ -0,0 +1,13 @@ +# bootstrap_functions.inc.sh: Functions used by the bootstrap script + +install_python_packages() { + local install_flags="${1}" + + pipenv install ${install_flags} + return $? +} + +install_node_packages() { + yarn install + return $? +} diff --git a/script/include/dev_server_functions.inc.sh b/script/include/dev_server_functions.inc.sh new file mode 100644 index 00000000..2947f4d4 --- /dev/null +++ b/script/include/dev_server_functions.inc.sh @@ -0,0 +1,8 @@ +# dev_server_functions.inc.sh: Functions used by the dev_server script + +# Used to ensure a SIGTERM is sent to all process in the current process group +reap() { + kill -s SIGTERM -- "-$$" + sleep 0.1 + exit +} diff --git a/script/include/get_db_settings_functions.inc.sh b/script/include/get_db_settings_functions.inc.sh new file mode 100644 index 00000000..61bc1d73 --- /dev/null +++ b/script/include/get_db_settings_functions.inc.sh @@ -0,0 +1,15 @@ +# set_db_env_vars.inc.sh: Functions used by the set_db_env_vars script + +set_db_env_vars () { + local config_file="${1}" + + # Read in and export all name=value pairs where the name starts with PG + # unless we already have an env var with that value + # (NOTE: all whitespaces are removed from each line) + while read -r DBVAR + do + if ! $(env | grep -qFle "${DBVAR%%=*}"); then + eval "export ${DBVAR}" + fi + done < <(grep -Ee '^PG' "${config_file}" | sed 's/ //g') +} diff --git a/script/include/global_header.inc.sh b/script/include/global_header.inc.sh new file mode 100644 index 00000000..45989ee1 --- /dev/null +++ b/script/include/global_header.inc.sh @@ -0,0 +1,12 @@ +# global_header.inc: Any basic things that should be executed at the +# beginning of any and every script + +# If any command fails, immediately exit the script +set -e + +# Ensure the working directory is the app root directory +cd "$(dirname "${0}")/.." + +# Source all function definition files + +source ./script/include/helper_functions.inc.sh diff --git a/script/include/helper_functions.inc.sh b/script/include/helper_functions.inc.sh new file mode 100644 index 00000000..44ca5882 --- /dev/null +++ b/script/include/helper_functions.inc.sh @@ -0,0 +1,55 @@ +# helper_functions.inc.sh: General helper functions + +# Check pip to see if the given package is installed +# (returns 0 if installed, 2 if not installed) +check_system_pip_for () { + local package_name="${1}" + + # Use 'pip list' to see if the requested package is already installed + pip list --format=columns --disable-pip-version-check | \ + grep -Fe "${package_name}" >/dev/null 2>&1 + return $? +} + +pip_install () { + local packages="${1}" + local flags="${2}" + + run_command "pip install ${flags} ${packages}" + return $? +} + +# Used whenever an environment sensitive command is being run +run_command () { + local cmd="${1}" + + pipenv run ${cmd} + return $? +} + +migrate_db() { + # Run migrations + run_command "alembic upgrade head" +} + +seed_db() { + run_command "python ./script/seed_roles.py" +} + +reset_db() { + local database_name="${1}" + + # If the DB exists, drop it + set +e + dropdb "${database_name}" + set -e + + # Create a fresh DB + createdb "${database_name}" + + # Run migrations + migrate_db + + # Seed database data + seed_db +} diff --git a/script/include/run_alpine_setup b/script/include/run_alpine_setup new file mode 100644 index 00000000..cee4bcd6 --- /dev/null +++ b/script/include/run_alpine_setup @@ -0,0 +1,32 @@ +# run_alpine_setup: Install basic system requirements for an app to run + +# Load alpine setup functions +source ./script/include/alpine_setup_functions.inc.sh + +## Set option defaults +# If GROUP information is incomplete, use the default one +if [ -z "${APP_GROUP+is_set}" ] || \ + [ -z "${APP_GID+is_set}" ]; then + APP_GROUP="atat" + APP_GID="8000" +fi + +# If USER information is incomplete, error out +if [ -z "${APP_USER+is_set}" ] || \ + [ -z "${APP_UID+is_set}" ]; then + echo "ERROR: Missing app user information! Received: ${APP_USER}:${APP_UID}" + exit 1 +fi + +## Main +update_system_packages +install_package "bash" +install_package "dumb-init" +if [ ! -z "${ADDITIONAL_PACKAGES+is_set}" ]; then + for package in ${ADDITIONAL_PACKAGES} + do + install_package "${package}" + done +fi +add_group "${APP_GROUP}" "${APP_GID}" +add_user "${APP_USER}" "${APP_GROUP}" "${APP_UID}" diff --git a/script/include/run_bootstrap b/script/include/run_bootstrap new file mode 100644 index 00000000..90138546 --- /dev/null +++ b/script/include/run_bootstrap @@ -0,0 +1,29 @@ +# run_bootstrap: Install application dependencies + +# Load bootstrap functions +source ./script/include/bootstrap_functions.inc.sh + +## Set option defaults +# If PIPENV_INSTALL_FLAGS is not set, give it the default value of "--dev" +if [ -z "${PIPENV_INSTALL_FLAGS+is_set}" ]; then + PIPENV_INSTALL_FLAGS="--dev" +fi + +# If this is a CI build, only use the latest lock file for dep install +if [ "${CIBUILD}" = "true" ] || [ "${FLASK_ENV}" = "ci" ]; then + PIPENV_INSTALL_FLAGS+=" --ignore-pipfile" +fi + +## Main +if [ "${INSTALL_PYTHON_PACKAGES}" = "true" ]; then + install_python_packages "${PIPENV_INSTALL_FLAGS}" + pipenv clean --verbose +fi + +if [ "${INSTALL_NODE_PACKAGES}" = "true" ]; then + install_node_packages +fi + +if [ -n "${COMPILE_SASS_CMD}" ]; then + run_command "${COMPILE_SASS_CMD}" +fi diff --git a/script/include/run_dev_server b/script/include/run_dev_server new file mode 100644 index 00000000..909b1c9c --- /dev/null +++ b/script/include/run_dev_server @@ -0,0 +1,20 @@ +# run_dev_server: Run a dev version of the app server in the background +# + +# Load dev_server functions +source ./script/include/dev_server_functions.inc.sh + +# Register trapping of SIGTERM and SIGINT +trap reap SIGTERM SIGINT + +# Display the script PID, which will also be the process group ID for all +# child processes +echo "Process Group: $$" + +# Set server launch related environment variables +DEBUG=1 +LAUNCH_ARGS="$*" + +# Launch the app +source ./script/server & +wait diff --git a/script/include/run_get_db_settings.sh b/script/include/run_get_db_settings.sh new file mode 100644 index 00000000..67ae8819 --- /dev/null +++ b/script/include/run_get_db_settings.sh @@ -0,0 +1,26 @@ +# get_db_settings: Reads postgres related settings from all of the relevant +# config files and then sets them as environment variables + +# Load get_db_settings_functions functions +source ./script/include/get_db_settings_functions.inc.sh + + +# If an OVERRIDE config has been specified, and the file is present, +# read those values first +if [ ! -z "${OVERRIDE_CONFIG_FULLPATH+is_not_set}" ] && + [ -f "${OVERRIDE_CONFIG_FULLPATH}" ]; then + set_db_env_vars "${OVERRIDE_CONFIG_FULLPATH}" +fi + +# If FLASK_ENV is set, and a config file exists for it, +# allow it to set anything not already defined +if [ "${FLASK_ENV}x" != "x" ]; then + flask_env_config_file="./config/${FLASK_ENV}.ini" + + if [ -f "${flask_env_config_file}" ]; then + set_db_env_vars "${flask_env_config_file}" + fi +fi + +# Finish with the base config file, setting anything that is still unset +set_db_env_vars "./config/base.ini" diff --git a/script/include/run_setup b/script/include/run_setup new file mode 100644 index 00000000..3696d51a --- /dev/null +++ b/script/include/run_setup @@ -0,0 +1,73 @@ +# setup: Set up application for the first time after cloning, or set it +# back to the initial first unused state. + +# Load setup functions +source ./script/include/setup_functions.inc.sh + +## Set option defaults +# If CREATE_VENV is not set, set it to "true" +if [ -z "${CREATE_VENV+is_set}" ]; then + CREATE_VENV="true" +fi + +# If INSTALL_SASS is not set, set it to "false" +if [ -z "${INSTALL_SASS+is_set}" ]; then + INSTALL_SASS="false" +fi + +# If PIP_VERSION is not set, set it to "10.*" +if [ -z "${PIP_VERSION+is_set}" ]; then + PIP_VERSION="10.*" +fi + +# If RESET_DB is not set, set it to "false" +if [ -z "${RESET_DB+is_set}" ]; then + RESET_DB="false" +fi + +# If KEEP_EXISTING_VENV is not set, set it to "false" +if [ -z "${KEEP_EXISTING_VENV+is_set}" ]; then + KEEP_EXISTING_VENV="false" +fi + +## Main +# Remove any existing node modules as part of initial app setup or reset +rm -rf ./node_modules + +if [ "${CREATE_VENV}" = "true" ]; then + # Ensure pipenv is installed + if ! pipenv --version >/dev/null 2>&1 ; then + echo "ERROR: pipenv is malfunctioning or not present" + exit 1 + fi + + python_version=$(grep python_version ./Pipfile | cut -d '"' -f 2) + if ! check_for_existing_virtual_environment "${python_version}" || \ + [ "${KEEP_EXISTING_VENV}" = "false" ] + then + create_virtual_environment "${python_version}" + fi + + pip_install "pip==${PIP_VERSION}" "--upgrade" +fi + +if [ "${INSTALL_SASS}" = "true" ]; then + install_sass +fi + +# Install application dependencies +./script/bootstrap + +if [ "${RESET_DB}" = "true" ]; then + # Fetch postgres settings and set them as ENV vars + source ./script/get_db_settings + + if [ -n "${PGDATABASE}" ]; then + echo "Resetting database ${PGDATABASE}..." + # Reset the db + reset_db "${PGDATABASE}" + else + echo "ERROR: RESET_DB is set, but PGDATABASE is not!" + echo "Skipping database reset..." + fi +fi diff --git a/script/include/run_test b/script/include/run_test new file mode 100644 index 00000000..340b9eab --- /dev/null +++ b/script/include/run_test @@ -0,0 +1,49 @@ +# run_test: Execute code checkers and unit tests + +# Load test functions +source ./script/include/test_functions.inc.sh + +## Set option defaults +# If PYTHON_FILES is not set, give it the default value of "app.py" +if [ -z "${PYTHON_FILES+is_set}" ]; then + PYTHON_FILES="app.py" +fi + +# reset test database +if [ "${RESET_DB}" = "true" ]; then + # Fetch postgres settings and set them as ENV vars + source ./script/get_db_settings + + if [ -n "${PGDATABASE}" ]; then + echo "Resetting database ${PGDATABASE}..." + # Reset the db + reset_db "${PGDATABASE}" + else + echo "ERROR: RESET_DB is set, but PGDATABASE is not!" + echo "Skipping database reset..." + fi +fi + +## Main +if [ "${RUN_PYTHON_TESTS}" = "true" ]; then + python_test_status=0 + set +e + + run_python_lint "${PYTHON_FILES}" + ((python_test_status+=$?)) + run_python_static_analysis "${PYTHON_FILES}" + ((python_test_status+=$?)) + run_python_unit_tests "${PYTHON_FILES}" + ((python_test_status+=$?)) + + if [ "${python_test_status}" != "0" ]; then + echo "Failed to pass one or more Python checks" + exit ${python_test_status} + fi + + set -e +fi + +if [ "${RUN_JS_TESTS}" = "true" ]; then + run_javascript_tests +fi diff --git a/script/include/run_update b/script/include/run_update new file mode 100644 index 00000000..5a022f3e --- /dev/null +++ b/script/include/run_update @@ -0,0 +1,22 @@ +# update: Bring an existing application up-to-date +# + +# Load update functions +source ./script/include/update_functions.inc.sh + +## Set option defaults +# If MIGRATE_DB is not set, set it to "false" +if [ -z "${MIGRATE_DB+is_set}" ]; then + MIGRATE_DB="false" +fi + +## Main +# Update dependencies +source ./script/bootstrap + +# Update database schema +if [ "${MIGRATE_DB}" = "true" ]; then + migrate_db +fi + +seed_db diff --git a/script/include/setup_functions.inc.sh b/script/include/setup_functions.inc.sh new file mode 100644 index 00000000..92c5dfa5 --- /dev/null +++ b/script/include/setup_functions.inc.sh @@ -0,0 +1,58 @@ +# setup_functions.inc.sh: Functions used by the setup script + +install_pipenv() { + return_code=0 + + # Ensure we are not in a virtual env already + if [ -z "${VIRTUAL_ENV+is_set}" ]; then + if ! check_system_pip_for pipenv; then + # pipenv is not installed, so install it + echo "Installing pipenv..." + pip install pipenv + # Capture pip exit code + return_code="${?}" + fi + fi + + return "${return_code}" +} + +check_for_existing_virtual_environment() { + local python_version="${1}" + local target_python_version_regex="^Python ${python_version}" + + # Check for existing venv, and if one exists, save the Python version string + existing_venv_version=$($(pipenv --py) --version) + if [ "$?" = "0" ]; then + # Existing venv; see if the Python version matches + if [[ "${existing_venv_version}" =~ ${target_python_version_regex} ]]; then + # Version strings match, valid existing environment is present + return 0 + fi + fi + + # No valid virtual environment found + return 1 +} + +create_virtual_environment() { + local python_version="${1}" + + # Create a new virtual environment for the app + # The environment will be in a directory called .venv off the app + # root directory + echo "Creating virtual environment using Python version ${python_version}..." + PIPENV_VENV_IN_PROJECT=true pipenv --python "${python_version}" + return $? +} + +install_sass() { + if ! type sass >/dev/null; then + if type gem >/dev/null; then + echo 'Installing a sass compiler (gem)...' + gem install sass + else + echo 'Could not install a sass compiler. Please install a version of sass.' + fi + fi +} diff --git a/script/include/test_functions.inc.sh b/script/include/test_functions.inc.sh new file mode 100644 index 00000000..71574d98 --- /dev/null +++ b/script/include/test_functions.inc.sh @@ -0,0 +1,25 @@ +# test_functions.inc.sh: Functions used by the run_test script + +run_python_lint() { + local python_files="${1}" + + run_command "pylint ${python_files}" + return $? +} + +run_python_static_analysis() { + local python_files="${1}" + + run_command "bandit -c ./.bandit_config -r ${python_files}" + return $? +} + +run_python_unit_tests() { + run_command "python -m pytest -s" + return $? +} + +run_javascript_tests() { + run_command "yarn test:coverage" + return $? +} diff --git a/script/include/update_functions.inc.sh b/script/include/update_functions.inc.sh new file mode 100644 index 00000000..ac3776a6 --- /dev/null +++ b/script/include/update_functions.inc.sh @@ -0,0 +1 @@ +# update_functions.inc.sh: Functions used by the update script