From 058ee575271fe13318900828f226c87bd722f055 Mon Sep 17 00:00:00 2001 From: dandds Date: Sat, 25 Jan 2020 12:21:52 -0500 Subject: [PATCH] Create database with separate script. Creating the ATAT database requires a separate connection to one of the default Postgres databases, like `postgres`. This updates the scripts and secrets-tool command to handle creating the database. It also removes database creation from Terraform and updates the documentation. --- .dockerignore | 7 +--- notes.md | 6 +++ script/create_database.py | 41 +++++++++++++++++++++ terraform/README.md | 31 +++++++++++++--- terraform/modules/postgres/main.tf | 8 ---- terraform/modules/postgres/outputs.tf | 3 -- terraform/secrets-tool/README.md | 11 ++++++ terraform/secrets-tool/commands/database.py | 23 ++++++++---- users.yml | 3 ++ 9 files changed, 105 insertions(+), 28 deletions(-) create mode 100644 notes.md create mode 100644 script/create_database.py create mode 100644 users.yml diff --git a/.dockerignore b/.dockerignore index 7b9644ad..5674e27e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -21,11 +21,8 @@ LICENSE # Skip envrc .envrc -# Skip ansible-container stuff -ansible* -container.yml -meta.yml -requirements.yml +# Skip terraform +terraform # Skip kubernetes and Docker config stuff deploy diff --git a/notes.md b/notes.md new file mode 100644 index 00000000..4c1dd84f --- /dev/null +++ b/notes.md @@ -0,0 +1,6 @@ +- for setting up the database: + - create database + - create postgres user password? could we do this as a key? + - create user secret in application key vault + - execute SQL to create user +- we need an initial image to seed ACR with diff --git a/script/create_database.py b/script/create_database.py new file mode 100644 index 00000000..f21a857e --- /dev/null +++ b/script/create_database.py @@ -0,0 +1,41 @@ +# Add root application dir to the python path +import os +import sys + +parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) +sys.path.append(parent_dir) + +import sqlalchemy + +from atst.app import make_config + + +def _root_connection(config, root_db): + # Assemble DATABASE_URI value + database_uri = "postgresql://{}:{}@{}:{}/{}".format( # pragma: allowlist secret + config.get("PGUSER"), + config.get("PGPASSWORD"), + config.get("PGHOST"), + config.get("PGPORT"), + root_db, + ) + engine = sqlalchemy.create_engine(database_uri) + return engine.connect() + + +def create_database(conn, dbname): + conn.execute("commit") + conn.execute(f"CREATE DATABASE {dbname};") + conn.close() + + return True + + +if __name__ == "__main__": + dbname = sys.argv[1] + config = make_config() + + conn = _root_connection(config, "postgres") + + print(f"Creating database {dbname}") + create_database(conn, dbname) diff --git a/terraform/README.md b/terraform/README.md index ec0fbdeb..8c8e7beb 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -1,11 +1,11 @@ # ATAT Terraform Welcome! You've found the ATAT IaC configurations. -ATAT is configured using terraform and a wrapper script called `secrets-tool`. With `terraform` we can configure infrastructure in a programatic way and ensure consistency across environments. +ATAT is configured using terraform and a wrapper script called `secrets-tool`. With `terraform` we can configure infrastructure in a programatic way and ensure consistency across environments. ## Directory Structure -**modules/** - Terraform modules. These are modules that can be re-used for multiple environments. +**modules/** - Terraform modules. These are modules that can be re-used for multiple environments. **providers/** - Specific environment configurations. (dev,production, etc) @@ -92,7 +92,7 @@ Check the output for errors. Sometimes the syntax is valid, but some of the conf # After running TF (Manual Steps) -## VM Scale Set +## VM Scale Set After running terraform, we need to make a manual change to the VM Scale Set that is used in the kubernetes. Terraform has a bug that is not applying this as of `v1.40` of the `azurerm` provider. In order to get the `SystemAssigned` identity to be set, it needs to be set manually in the console. @@ -253,7 +253,7 @@ Uncomment the `backend {}` section in the `provider.tf` file. Once uncommented, *Say `yes` to the question* -Now we need to update the Update `variables.tf` with the principals for the users in `admin_users` variable map. If these are not defined yet, just leave it as an empty set. +Now we need to update the Update `variables.tf` with the principals for the users in `admin_users` variable map. If these are not defined yet, just leave it as an empty set. Next, we'll create the operator keyvault. @@ -281,4 +281,25 @@ secrets-tool secrets --keyvault https://ops-jedidev-keyvault.vault.azure.net/ cr `terraform apply` -*[Configure AD for MFA](https://docs.microsoft.com/en-us/azure/vpn-gateway/openvpn-azure-ad-mfa)* \ No newline at end of file +*[Configure AD for MFA](https://docs.microsoft.com/en-us/azure/vpn-gateway/openvpn-azure-ad-mfa)* + +*Then we need an instance of the container* + +Change directories to the repo root. Ensure that you've checked out the staging or master branch: + +`docker build . --build-arg CSP=azure -f ./Dockerfile -t atat:latest` + +*Create secrets for ATAT database user* + +Change directories back to terraform/secrets-tool. There is a sample file there. Make sure you know the URL for the aplication Key Vault (distinct from the operator Key Vault). Run: + +`secrets-tool secrets --keyvault [application key vault URL] load -f ./postgres-user.yaml + +*Create the database, database user, schema, and initial data set* + + +This is discussed in more detail [here](https://github.com/dod-ccpo/atst/tree/staging/terraform/secrets-tool#setting-up-the-initial-atat-database). Be sure to read the requirements section. + +``` +secrets-tool database --keyvault [operator key vault URL] provision --app-keyvault [application key vault URL] --dbname jedidev-atat --dbhost [database host name] --ccpo-users /full/path/to/users.yml +``` diff --git a/terraform/modules/postgres/main.tf b/terraform/modules/postgres/main.tf index c3252264..29b6cc53 100644 --- a/terraform/modules/postgres/main.tf +++ b/terraform/modules/postgres/main.tf @@ -35,11 +35,3 @@ resource "azurerm_postgresql_virtual_network_rule" "sql" { subnet_id = var.subnet_id ignore_missing_vnet_service_endpoint = true } - -resource "azurerm_postgresql_database" "db" { - name = "${var.name}-${var.environment}-atat" - resource_group_name = azurerm_resource_group.sql.name - server_name = azurerm_postgresql_server.sql.name - charset = "UTF8" - collation = "en-US" -} diff --git a/terraform/modules/postgres/outputs.tf b/terraform/modules/postgres/outputs.tf index 1ff1dd65..e69de29b 100644 --- a/terraform/modules/postgres/outputs.tf +++ b/terraform/modules/postgres/outputs.tf @@ -1,3 +0,0 @@ -output "db_name" { - value = azurerm_postgresql_database.db.name -} diff --git a/terraform/secrets-tool/README.md b/terraform/secrets-tool/README.md index 9ce07497..bd407607 100644 --- a/terraform/secrets-tool/README.md +++ b/terraform/secrets-tool/README.md @@ -45,6 +45,17 @@ Requirements: - docker - A copy of the ATAT docker image. This can be built in the repo root with: `docker build . --build-arg CSP=azure -f ./Dockerfile -t atat:latest` - You need to know the hostname for the Postgres database. Your IP must either be whitelisted in its firewall rules or you must be behind the VPN. +- You will need a YAML file listing all the CCPO users to be added to the database, with the format: + +``` +- dod_id: "2323232323" + first_name: "Luke" + last_name: "Skywalker" +- dod_id: "5656565656" + first_name: "Han" + last_name: "Solo" +``` + - There should be a password for the ATAT database user in the application Key Vault, preferably named `PGPASSWORD`. You can load this by running `secrets-tool --keyvault [operator key vault url] load -f postgres-user.yml` and supplying YAML like: ``` diff --git a/terraform/secrets-tool/commands/database.py b/terraform/secrets-tool/commands/database.py index e181741c..98f404a5 100644 --- a/terraform/secrets-tool/commands/database.py +++ b/terraform/secrets-tool/commands/database.py @@ -116,19 +116,28 @@ def provision( logger.info("starting docker process") - cmd = ( - f"docker run -e PGHOST={dbhost}" - +f" -e PGPASSWORD=\"{root_password}\"" + create_database_cmd = ( + f"docker run -e PGHOST='{dbhost}'" + +f" -e PGPASSWORD='{root_password}'" +f" -e PGUSER='{root_name}@{dbhost}'" - +f" -e PGDATABASE=\"{dbname}\"" - +f" -e REDIS_HOST=host.docker.internal" + +f" -e PGDATABASE='{dbname}'" + +f" -e PGSSLMODE=require" + +f" {container}" + +f" .venv/bin/python script/create_database.py {dbname}" + ) + _run_cmd(create_database_cmd) + + seed_database_cmd = ( + f"docker run -e PGHOST='{dbhost}'" + +f" -e PGPASSWORD='{root_password}'" + +f" -e PGUSER='{root_name}@{dbhost}'" + +f" -e PGDATABASE='{dbname}'" +f" -e PGSSLMODE=require" +f" -v {ccpo_users}:/opt/atat/atst/users.yml" +f" {container}" +f" .venv/bin/python script/database_setup.py {user_username} '{user_password}' users.yml" ) - print(cmd) - _run_cmd(cmd) + _run_cmd(seed_database_cmd) database.add_command(provision) diff --git a/users.yml b/users.yml new file mode 100644 index 00000000..561031f2 --- /dev/null +++ b/users.yml @@ -0,0 +1,3 @@ +- dod_id: "2323232323" + first_name: "hi" + last_name: "bye"