Remove the scriptz submodule and hard-commit the files.

The submodule is a leftover from when this project was intended to work
as a series of microservices. It was meant to provide common
functionality to the builds for every microservice. That's no longer the
case, and the submodule is a pain-point both in on-boarding new
developers and running the Docker build.

.gitmodules

@@ -1,4 +0,0 @@
-[submodule "script/include"]
-	path = script/include
-	url = https://github.com/dod-ccpo/scriptz.git
-	branch = master

script/include

@@ -1 +0,0 @@
-Subproject commit bb072dd10190a05c77973b85e9d4f8f1a7fccdde

script/include/README.md

@@ -0,0 +1,60 @@
+# scriptz
+
+These script fragments are shared between all of the ATAT applications, powering
+their /script files. See 
+[Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all) 
+and an ATAT application repository for details.
+
+- `global_header.inc.sh`: Run by all /script files at the start of the script.
+- `helper_functions.inc.sh`: Contains general helper functions; sourced by 
+global_header.inc.sh
+- `run_setup`: Main logic run by most /script/setup scripts, after they set any
+relevant environment variables. Initializes the local environment to run the app.
+- `run_bootstrap`: Main logic run by most /script/bootstrap scripts, after they set 
+any relevant environment variables. Installs application dependencies.
+- `run_test`: Main logic run by most /script/test scripts, after they set any
+relevant environment variables. Executes lint, static analysis, and unit tests.
+- `run_update`: Main logic run by most /script/update scripts, after they set any
+ relevant environment variables. Executes application dependency updates, and
+ potentially DB migrations.
+- `run_dev_server`: Main logic run by most /script/dev_server scripts, after any
+ relevant environment variables are set. Launches an application server in DEBUG
+ mode in the background.
+- `run_alpine_setup`: Main logic run by most /script/alpine_setup scripts, after 
+they set any relevant environment variables. Adds Alpine Linux specific system 
+dependencies required to run the app (used by Docker builds).
+- `setup_functions.inc.sh`: Sourced by run_setup; contains setup specific 
+functions.
+- `bootstrap_functions.inc.sh`: Sourced by run_bootstrap; contains bootstrap 
+specific functions.
+- `test_functions.inc.sh`: Sourced by run_test; contains test specific 
+functions.
+- `update_functions.inc.sh`: Sourced by run_update; contains update specific
+ functions.
+- `dev_server_functions.inc.sh`: Sourced by run_dev_server; contains dev_server
+ specific functions.
+- `alpine_setup_functions.inc.sh`: Sourced by run_alpine_setup; contains 
+alpine_setup specific functions.
+
+## Requirements
+
+- **[Python 3.6](https://www.python.org/downloads/)**
+- **[Pipenv](https://docs.pipenv.org/install/#installing-pipenv)**
+
+**Note:** None of these script fragments are designed to be run by themselves. 
+They should be sourced by the relevant bash script in an application's /script 
+directory.
+
+#### Ubuntu 18.04 Instructions
+
+Install pip:
+```
+sudo apt install python3-distutils
+curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
+python3 get-pip.py --user
+```
+
+Install pipenv:
+```
+pip install --user pipenv
+```

script/include/alpine_setup_functions.inc.sh

@@ -0,0 +1,36 @@
+# alpine_setup_functions: Functions used by the run_alpine_setup script
+
+update_system_packages() {
+  local apk_cache_dir="/etc/apk/cache"
+
+  apk update
+  apk upgrade
+
+  if [ -d "${apk_cache_dir}" ] || [ -L "${apk_cache_dir}" ]; then
+    apk cache clean
+  fi
+}
+
+install_package() {
+  local package_name=${1}
+
+  apk add "${package_name}"
+  return $?
+}
+
+add_group() {
+  local group_name="${1}"
+  local gid="${2}"
+
+  addgroup -g "${gid}" -S "${group_name}"
+  return $?
+}
+
+add_user() {
+  local username="${1}"
+  local primary_group="${2}"
+  local uid="${3}"
+
+  adduser -u "${uid}" -D -S -G "${primary_group}" "${username}"
+  return $?
+}

script/include/bootstrap_functions.inc.sh

@@ -0,0 +1,13 @@
+# bootstrap_functions.inc.sh: Functions used by the bootstrap script
+
+install_python_packages() {
+  local install_flags="${1}"
+
+  pipenv install ${install_flags}
+  return $?
+}
+
+install_node_packages() {
+  yarn install
+  return $?
+}

script/include/dev_server_functions.inc.sh

@@ -0,0 +1,8 @@
+# dev_server_functions.inc.sh: Functions used by the dev_server script
+
+# Used to ensure a SIGTERM is sent to all process in the current process group
+reap() {
+   kill -s SIGTERM -- "-$$"
+   sleep 0.1
+   exit
+}

script/include/get_db_settings_functions.inc.sh

@@ -0,0 +1,15 @@
+# set_db_env_vars.inc.sh: Functions used by the set_db_env_vars script
+
+set_db_env_vars () {
+    local config_file="${1}"
+
+    # Read in and export all name=value pairs where the name starts with PG
+    # unless we already have an env var with that value
+    # (NOTE: all whitespaces are removed from each line)
+    while read -r DBVAR
+    do
+	if ! $(env | grep -qFle "${DBVAR%%=*}"); then
+            eval "export ${DBVAR}"
+	fi
+    done < <(grep -Ee '^PG' "${config_file}" | sed 's/ //g')
+}

script/include/global_header.inc.sh

@@ -0,0 +1,12 @@
+# global_header.inc: Any basic things that should be executed at the
+#                    beginning of any and every script
+
+# If any command fails, immediately exit the script
+set -e
+
+# Ensure the working directory is the app root directory
+cd "$(dirname "${0}")/.."
+
+# Source all function definition files
+
+source ./script/include/helper_functions.inc.sh

script/include/helper_functions.inc.sh

@@ -0,0 +1,55 @@
+# helper_functions.inc.sh: General helper functions
+
+# Check pip to see if the given package is installed
+# (returns 0 if installed, 2 if not installed)
+check_system_pip_for () {
+  local package_name="${1}"
+
+  # Use 'pip list' to see if the requested package is already installed
+  pip list --format=columns --disable-pip-version-check | \
+    grep -Fe "${package_name}" >/dev/null 2>&1
+  return $?
+}
+
+pip_install () {
+  local packages="${1}"
+  local flags="${2}"
+
+  run_command "pip install ${flags} ${packages}"
+  return $?
+}
+
+# Used whenever an environment sensitive command is being run
+run_command () {
+  local cmd="${1}"
+
+  pipenv run ${cmd}
+  return $?
+}
+
+migrate_db() {
+  # Run migrations
+  run_command "alembic upgrade head"
+}
+
+seed_db() {
+  run_command "python ./script/seed_roles.py"
+}
+
+reset_db() {
+  local database_name="${1}"
+
+  # If the DB exists, drop it
+  set +e
+  dropdb "${database_name}"
+  set -e
+
+  # Create a fresh DB
+  createdb "${database_name}"
+
+  # Run migrations
+  migrate_db
+
+  # Seed database data
+  seed_db
+}

script/include/run_alpine_setup

@@ -0,0 +1,32 @@
+# run_alpine_setup: Install basic system requirements for an app to run
+
+# Load alpine setup functions
+source ./script/include/alpine_setup_functions.inc.sh
+
+## Set option defaults
+# If GROUP information is incomplete, use the default one
+if [ -z "${APP_GROUP+is_set}" ] || \
+   [ -z "${APP_GID+is_set}" ]; then
+  APP_GROUP="atat"
+  APP_GID="8000"
+fi
+
+# If USER information is incomplete, error out
+if [ -z "${APP_USER+is_set}" ] || \
+   [ -z "${APP_UID+is_set}" ]; then
+  echo "ERROR: Missing app user information! Received: ${APP_USER}:${APP_UID}"
+  exit 1
+fi
+
+## Main
+update_system_packages
+install_package "bash"
+install_package "dumb-init"
+if [ ! -z "${ADDITIONAL_PACKAGES+is_set}" ]; then
+  for package in ${ADDITIONAL_PACKAGES}
+  do
+    install_package "${package}"
+  done
+fi
+add_group "${APP_GROUP}" "${APP_GID}"
+add_user "${APP_USER}" "${APP_GROUP}" "${APP_UID}"

script/include/run_bootstrap

@@ -0,0 +1,29 @@
+# run_bootstrap: Install application dependencies
+
+# Load bootstrap functions
+source ./script/include/bootstrap_functions.inc.sh
+
+## Set option defaults
+# If PIPENV_INSTALL_FLAGS is not set, give it the default value of "--dev"
+if [ -z "${PIPENV_INSTALL_FLAGS+is_set}" ]; then
+  PIPENV_INSTALL_FLAGS="--dev"
+fi
+
+# If this is a CI build, only use the latest lock file for dep install
+if [ "${CIBUILD}" = "true" ] || [ "${FLASK_ENV}" = "ci" ]; then
+  PIPENV_INSTALL_FLAGS+=" --ignore-pipfile"
+fi
+
+## Main
+if [ "${INSTALL_PYTHON_PACKAGES}" = "true" ]; then
+  install_python_packages "${PIPENV_INSTALL_FLAGS}"
+  pipenv clean --verbose
+fi
+
+if [ "${INSTALL_NODE_PACKAGES}" = "true" ]; then
+  install_node_packages
+fi
+
+if [ -n "${COMPILE_SASS_CMD}" ]; then
+  run_command "${COMPILE_SASS_CMD}"
+fi

script/include/run_dev_server

@@ -0,0 +1,20 @@
+#  run_dev_server: Run a dev version of the app server in the background
+#
+
+# Load dev_server functions
+source ./script/include/dev_server_functions.inc.sh
+
+# Register trapping of SIGTERM and SIGINT
+trap reap SIGTERM SIGINT
+
+# Display the script PID, which will also be the process group ID for all
+# child processes
+echo "Process Group: $$"
+
+# Set server launch related environment variables
+DEBUG=1
+LAUNCH_ARGS="$*"
+
+# Launch the app
+source ./script/server &
+wait

script/include/run_get_db_settings.sh

@@ -0,0 +1,26 @@
+# get_db_settings: Reads postgres related settings from all of the relevant
+#                  config files and then sets them as environment variables
+
+# Load get_db_settings_functions functions
+source ./script/include/get_db_settings_functions.inc.sh
+
+
+# If an OVERRIDE config has been specified, and the file is present, 
+# read those values first
+if [ ! -z "${OVERRIDE_CONFIG_FULLPATH+is_not_set}" ] && 
+   [ -f "${OVERRIDE_CONFIG_FULLPATH}" ]; then
+    set_db_env_vars "${OVERRIDE_CONFIG_FULLPATH}"
+fi
+
+# If FLASK_ENV is set, and a config file exists for it,
+# allow it to set anything not already defined
+if [ "${FLASK_ENV}x" != "x" ]; then
+    flask_env_config_file="./config/${FLASK_ENV}.ini"
+
+    if [ -f "${flask_env_config_file}" ]; then
+        set_db_env_vars "${flask_env_config_file}"
+    fi
+fi
+
+# Finish with the base config file, setting anything that is still unset
+set_db_env_vars "./config/base.ini"

script/include/run_setup

@@ -0,0 +1,73 @@
+# setup: Set up application for the first time after cloning, or set it
+#        back to the initial first unused state.
+
+# Load setup functions
+source ./script/include/setup_functions.inc.sh
+
+## Set option defaults
+# If CREATE_VENV is not set, set it to "true"
+if [ -z "${CREATE_VENV+is_set}" ]; then
+  CREATE_VENV="true"
+fi
+
+# If INSTALL_SASS is not set, set it to "false"
+if [ -z "${INSTALL_SASS+is_set}" ]; then
+  INSTALL_SASS="false"
+fi
+
+# If PIP_VERSION is not set, set it to "10.*"
+if [ -z "${PIP_VERSION+is_set}" ]; then
+  PIP_VERSION="10.*"
+fi
+
+# If RESET_DB is not set, set it to "false"
+if [ -z "${RESET_DB+is_set}" ]; then
+  RESET_DB="false"
+fi
+
+# If KEEP_EXISTING_VENV is not set, set it to "false"
+if [ -z "${KEEP_EXISTING_VENV+is_set}" ]; then
+  KEEP_EXISTING_VENV="false"
+fi
+
+## Main
+# Remove any existing node modules as part of initial app setup or reset
+rm -rf ./node_modules
+
+if [ "${CREATE_VENV}" = "true" ]; then
+  # Ensure pipenv is installed
+  if ! pipenv --version >/dev/null 2>&1 ; then
+    echo "ERROR: pipenv is malfunctioning or not present"
+    exit 1
+  fi
+
+  python_version=$(grep python_version ./Pipfile | cut -d '"' -f 2)
+  if ! check_for_existing_virtual_environment "${python_version}" || \
+       [ "${KEEP_EXISTING_VENV}" = "false" ]
+  then
+    create_virtual_environment "${python_version}"
+  fi
+
+  pip_install "pip==${PIP_VERSION}" "--upgrade"
+fi
+
+if [ "${INSTALL_SASS}" = "true" ]; then
+  install_sass
+fi
+
+# Install application dependencies
+./script/bootstrap
+
+if [ "${RESET_DB}" = "true" ]; then
+  # Fetch postgres settings and set them as ENV vars
+  source ./script/get_db_settings
+
+  if [ -n "${PGDATABASE}" ]; then
+    echo "Resetting database ${PGDATABASE}..."
+    # Reset the db
+    reset_db "${PGDATABASE}"
+  else
+    echo "ERROR: RESET_DB is set, but PGDATABASE is not!"
+    echo "Skipping database reset..."
+  fi
+fi

script/include/run_test

@@ -0,0 +1,49 @@
+# run_test: Execute code checkers and unit tests
+
+# Load test functions
+source ./script/include/test_functions.inc.sh
+
+## Set option defaults
+# If PYTHON_FILES is not set, give it the default value of "app.py"
+if [ -z "${PYTHON_FILES+is_set}" ]; then
+  PYTHON_FILES="app.py"
+fi
+
+# reset test database
+if [ "${RESET_DB}" = "true" ]; then
+  # Fetch postgres settings and set them as ENV vars
+  source ./script/get_db_settings
+
+  if [ -n "${PGDATABASE}" ]; then
+    echo "Resetting database ${PGDATABASE}..."
+    # Reset the db
+    reset_db "${PGDATABASE}"
+  else
+    echo "ERROR: RESET_DB is set, but PGDATABASE is not!"
+    echo "Skipping database reset..."
+  fi
+fi
+
+## Main
+if [ "${RUN_PYTHON_TESTS}" = "true" ]; then
+  python_test_status=0
+  set +e
+
+  run_python_lint "${PYTHON_FILES}"
+  ((python_test_status+=$?))
+  run_python_static_analysis "${PYTHON_FILES}"
+  ((python_test_status+=$?))
+  run_python_unit_tests "${PYTHON_FILES}"
+  ((python_test_status+=$?))
+
+  if [ "${python_test_status}" != "0" ]; then
+    echo "Failed to pass one or more Python checks"
+    exit ${python_test_status}
+  fi
+
+  set -e
+fi
+
+if [ "${RUN_JS_TESTS}" = "true" ]; then
+  run_javascript_tests
+fi

script/include/run_update

@@ -0,0 +1,22 @@
+#  update: Bring an existing application up-to-date
+#
+
+# Load update functions
+source ./script/include/update_functions.inc.sh
+
+## Set option defaults
+# If MIGRATE_DB is not set, set it to "false"
+if [ -z "${MIGRATE_DB+is_set}" ]; then
+  MIGRATE_DB="false"
+fi
+
+## Main
+# Update dependencies
+source ./script/bootstrap
+
+# Update database schema
+if [ "${MIGRATE_DB}" = "true" ]; then
+  migrate_db
+fi
+
+seed_db

script/include/setup_functions.inc.sh

@@ -0,0 +1,58 @@
+# setup_functions.inc.sh: Functions used by the setup script
+
+install_pipenv() {
+  return_code=0
+
+  # Ensure we are not in a virtual env already
+  if [ -z "${VIRTUAL_ENV+is_set}" ]; then
+    if ! check_system_pip_for pipenv; then
+      # pipenv is not installed, so install it
+      echo "Installing pipenv..."
+      pip install pipenv
+      # Capture pip exit code
+      return_code="${?}"
+    fi
+  fi
+
+  return "${return_code}"
+}
+
+check_for_existing_virtual_environment() {
+  local python_version="${1}"
+  local target_python_version_regex="^Python ${python_version}"
+
+  # Check for existing venv, and if one exists, save the Python version string
+  existing_venv_version=$($(pipenv --py) --version)
+  if [ "$?" = "0" ]; then
+    # Existing venv; see if the Python version matches
+    if [[ "${existing_venv_version}" =~ ${target_python_version_regex} ]]; then
+      # Version strings match, valid existing environment is present
+      return 0
+    fi
+  fi
+
+  # No valid virtual environment found
+  return 1
+}
+
+create_virtual_environment() {
+  local python_version="${1}"
+
+  # Create a new virtual environment for the app
+  # The environment will be in a directory called .venv off the app
+  # root directory
+  echo "Creating virtual environment using Python version ${python_version}..."
+  PIPENV_VENV_IN_PROJECT=true pipenv --python "${python_version}"
+  return $?
+}
+
+install_sass() {
+  if ! type sass >/dev/null; then
+    if type gem >/dev/null; then
+      echo 'Installing a sass compiler (gem)...'
+      gem install sass
+    else
+      echo 'Could not install a sass compiler. Please install a version of sass.'
+    fi
+  fi
+}

script/include/test_functions.inc.sh

@@ -0,0 +1,25 @@
+# test_functions.inc.sh: Functions used by the run_test script
+
+run_python_lint() {
+  local python_files="${1}"
+
+  run_command "pylint ${python_files}"
+  return $?
+}
+
+run_python_static_analysis() {
+  local python_files="${1}"
+
+  run_command "bandit -c ./.bandit_config -r ${python_files}"
+  return $?
+}
+
+run_python_unit_tests() {
+  run_command "python -m pytest -s"
+  return $?
+}
+
+run_javascript_tests() {
+  run_command "yarn test:coverage"
+  return $?
+}

script/include/update_functions.inc.sh

@@ -0,0 +1 @@
+# update_functions.inc.sh: Functions used by the update script