secrets-tool command for bootstrapping database.

This additional secrets-tool command can be used to run the database
bootsrapping script (`script/database_setup.py`) inside an ATAT docker
container against the Azure database. It sources the necessary keys from
Key Vault.
This commit is contained in:
dandds 2020-01-23 15:35:35 -05:00
parent 49a1a219ae
commit a8f6befc17
6 changed files with 185 additions and 13 deletions

View File

@ -1,13 +1,11 @@
# Add root application dir to the python path # Add root application dir to the python path
import os import os
import sys import sys
from contextlib import contextmanager
parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
sys.path.append(parent_dir) sys.path.append(parent_dir)
import sqlalchemy import sqlalchemy
from alembic import config as alembic_config
import yaml import yaml
from atst.app import make_config, make_app from atst.app import make_config, make_app
@ -25,7 +23,6 @@ def database_setup(username, password, dbname, ccpo_users):
try: try:
_create_database_user(username, password, dbname) _create_database_user(username, password, dbname)
except sqlalchemy.exc.ProgrammingError as err: except sqlalchemy.exc.ProgrammingError as err:
raise err
print(f"Postgres user role '{username}' already exists.") print(f"Postgres user role '{username}' already exists.")
print("Applying schema and seeding roles and permissions.") print("Applying schema and seeding roles and permissions.")

View File

@ -15,7 +15,7 @@ With both usernames and passwords generated, the application only needs to make
Ex. Ex.
``` ```
{ {
'postgres_root_user': 'EzTEzSNLKQPHuJyPdPloIDCAlcibbl', 'postgres_root_user': 'EzTEzSNLKQPHuJyPdPloIDCAlcibbl',
'postgres_root_password': "2+[A@E4:C=ubb/#R#'n<p|wCW-|%q^" 'postgres_root_password': "2+[A@E4:C=ubb/#R#'n<p|wCW-|%q^"
} }
``` ```
@ -30,6 +30,40 @@ Terraform typically expects user defined secrets to be stored in either a file,
This provides a number of security benefits. First, secrets are not on disk. Secondly, users/operators never see the secrets fly by (passerbys or voyeurs that like to look over your shoulder when deploying to production) This provides a number of security benefits. First, secrets are not on disk. Secondly, users/operators never see the secrets fly by (passerbys or voyeurs that like to look over your shoulder when deploying to production)
## Setting up the initial ATAT database
This handles bootstrapping the ATAT database with a user, schema, and initial data.
It does the following:
- Sources the Postgres root user credentials
- Source the Postgres ATAT user password
- Runs a script inside an ATAT docker container to set up the initial database user, schema, and seed data in the database
Requirements:
- docker
- A copy of the ATAT docker image. This can be built in the repo root with: `docker build . --build-arg CSP=azure -f ./Dockerfile -t atat:latest`
- You need to know the hostname for the Postgres database. Your IP must either be whitelisted in its firewall rules or you must be behind the VPN.
- There should be a password for the ATAT database user in the application Key Vault, preferably named `PGPASSWORD`. You can load this by running `secrets-tool --keyvault [operator key vault url] load -f postgres-user.yml` and supplying YAML like:
```
---
- PGPASSWORD:
type: 'password'
length: 30
```
This command takes a lot of arguments. Run `secrets-tool database --keyvault [operator key vault url] provision -- help` to see the full list of available options.
The command supplies some defaults by assuming you've followed the patterns in sample-secrets.yml and elsewhere.
An example would be:
```
secrets-tool database --keyvault [operator key vault URL] provision --app-keyvault [application key vault URL] --dbname jedidev-atat --dbhost [database host name] --ccpo-users /full/path/to/users.yml
```
# Setup # Setup
*Requirements* *Requirements*
@ -76,4 +110,4 @@ secrets-tool secrets --keyvault https://operator-dev-keyvault.vault.azure.net/ l
This will fetch all secrets from the keyvault specified. `secrets-tool` then converts the keys to a variable name that terraform will look for. Essentially it prepends the keys found in KeyVault with `TF_VAR` and then executes terraform as a subprocess with the injected environment variables. This will fetch all secrets from the keyvault specified. `secrets-tool` then converts the keys to a variable name that terraform will look for. Essentially it prepends the keys found in KeyVault with `TF_VAR` and then executes terraform as a subprocess with the injected environment variables.
``` ```
secrets-tool terraform --keyvault https://operator-dev-keyvault.vault.azure.net/ plan secrets-tool terraform --keyvault https://operator-dev-keyvault.vault.azure.net/ plan
``` ```

View File

@ -0,0 +1,134 @@
import os
import click
import logging
import subprocess
from utils.keyvault.secrets import SecretsClient
logger = logging.getLogger(__name__)
def _run_cmd(command):
try:
env = os.environ.copy()
with subprocess.Popen(
command, env=env, stdout=subprocess.PIPE, shell=True
) as proc:
for line in proc.stdout:
logging.info(line.decode("utf-8"))
except Exception as e:
print(e)
@click.group()
@click.option("--keyvault", required=True, help="Specify the keyvault to operate on")
@click.pass_context
def database(ctx, keyvault):
ctx.ensure_object(dict)
ctx.obj["keyvault"] = keyvault
# root password, root username
@click.command("provision")
@click.option(
"--app-keyvault",
"app_keyvault",
required=True,
help="The username for the new Postgres user.",
)
@click.option(
"--user-username",
"user_username",
default="atat",
required=True,
help="The username for the new Postgres user.",
)
@click.option(
"--user-password-key",
"user_password_key",
default="PGPASSWORD",
required=True,
help="The name of the user's password key in the specified vault.",
)
@click.option(
"--root-username-key",
"root_username_key",
default="postgres-root-user",
required=True,
help="The name of the user's password key in the specified vault.",
)
@click.option(
"--root-password-key",
"root_password_key",
default="postgres-root-password",
required=True,
help="The name of the user's password key in the specified vault.",
)
@click.option(
"--dbname",
"dbname",
required=True,
help="The name of the database the user will be given full access to.",
)
@click.option(
"--dbhost",
"dbhost",
required=True,
help="The name of the database the user will be given full access to.",
)
@click.option(
"--container",
"container",
default="atat:latest",
required=True,
help="The container to run the provisioning command in.",
)
@click.option(
"--ccpo-users",
"ccpo_users",
required=True,
help="The full path to a YAML file listing CCPO users to be seeded to the database.",
)
@click.pass_context
def provision(
ctx,
app_keyvault,
user_username,
user_password_key,
root_username_key,
root_password_key,
dbname,
dbhost,
container,
ccpo_users,
):
"""
Set up the initial ATAT database.
"""
logger.info("obtaining postgres root user credentials")
operator_keyvault = SecretsClient(vault_url=ctx.obj["keyvault"])
root_password = operator_keyvault.get_secret(root_password_key)
root_name = operator_keyvault.get_secret(root_username_key)
logger.info("obtaining postgres database user password")
app_keyvault = SecretsClient(vault_url=app_keyvault)
user_password = app_keyvault.get_secret(user_password_key)
logger.info("starting docker process")
cmd = (
f"docker run -e PGHOST={dbhost}"
+f" -e PGPASSWORD=\"{root_password}\""
+f" -e PGUSER='{root_name}@{dbhost}'"
+f" -e PGDATABASE=\"{dbname}\""
+f" -e REDIS_HOST=host.docker.internal"
+f" -e PGSSLMODE=require"
+f" -v {ccpo_users}:/opt/atat/atst/users.yml"
+f" {container}"
+f" .venv/bin/python script/database_setup.py {user_username} '{user_password}' users.yml"
)
print(cmd)
_run_cmd(cmd)
database.add_command(provision)

View File

@ -0,0 +1,4 @@
---
- PGPASSWORD:
type: 'password'
length: 30

View File

@ -7,6 +7,7 @@ import logging
from commands.secrets import secrets from commands.secrets import secrets
from commands.terraform import terraform from commands.terraform import terraform
from commands.database import database
config.setup_logging() config.setup_logging()
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -21,6 +22,7 @@ def cli():
# Add additional command groups # Add additional command groups
cli.add_command(secrets) cli.add_command(secrets)
cli.add_command(terraform) cli.add_command(terraform)
cli.add_command(database)
if __name__ == "__main__": if __name__ == "__main__":
@ -41,12 +43,12 @@ if __name__ == "__main__":
val = keyvault.get_secret(secret) val = keyvault.get_secret(secret)
#print(val) #print(val)
os.environ[name] = val os.environ[name] = val
env = os.environ.copy() env = os.environ.copy()
command = "{} {}".format(PROCESS, sys.argv[1]) command = "{} {}".format(PROCESS, sys.argv[1])
with subprocess.Popen(command, env=env, stdout=subprocess.PIPE, shell=True) as proc: with subprocess.Popen(command, env=env, stdout=subprocess.PIPE, shell=True) as proc:
for line in proc.stdout: for line in proc.stdout:
logging.info(line.decode("utf-8") ) logging.info(line.decode("utf-8") )
except Exception as e: except Exception as e:
print(e, traceback.print_stack) print(e, traceback.print_stack)
''' '''

View File

@ -36,7 +36,7 @@ class SecretsLoader():
load the secrets in to keyvault load the secrets in to keyvault
""" """
def __init__(self, yaml_file: str, keyvault: object): def __init__(self, yaml_file: str, keyvault: object):
assert Path(yaml_file).exists() assert Path(yaml_file).exists()
self.yaml_file = yaml_file self.yaml_file = yaml_file
self.keyvault = keyvault self.keyvault = keyvault
self.config = dict() self.config = dict()
@ -47,7 +47,7 @@ class SecretsLoader():
def _load_yaml(self): def _load_yaml(self):
with open(self.yaml_file) as handle: with open(self.yaml_file) as handle:
self.config = yaml.load(handle, Loader=yaml.FullLoader) self.config = yaml.load(handle, Loader=yaml.FullLoader)
def _generate_secrets(self): def _generate_secrets(self):
secrets = GenerateSecrets(self.config).process_definition() secrets = GenerateSecrets(self.config).process_definition()
self.secrets = secrets self.secrets = secrets
@ -60,12 +60,14 @@ class SecretsLoader():
class GenerateSecrets(): class GenerateSecrets():
""" """
Read the secrets definition and generate requiesite Read the secrets definition and generate requiesite
secrets based on the type of secret and arguments secrets based on the type of secret and arguments
provided provided
""" """
def __init__(self, definitions: dict): def __init__(self, definitions: dict):
self.definitions = definitions self.definitions = definitions
most_punctuation = string.punctuation.replace("'", "").replace('"', "")
self.password_characters = string.ascii_letters + string.digits + most_punctuation
def process_definition(self): def process_definition(self):
""" """
@ -101,9 +103,8 @@ class GenerateSecrets():
# Types. Can be usernames, passwords, or in the future things like salted # Types. Can be usernames, passwords, or in the future things like salted
# tokens, uuid, or other specialized types # tokens, uuid, or other specialized types
def _generate_password(self, length: int): def _generate_password(self, length: int):
self.password_characters = string.ascii_letters + string.digits + string.punctuation
return ''.join(secrets.choice(self.password_characters) for i in range(length)) return ''.join(secrets.choice(self.password_characters) for i in range(length))
def _generate_username(self, length: int): def _generate_username(self, length: int):
self.username_characters = string.ascii_letters self.username_characters = string.ascii_letters
return ''.join(secrets.choice(self.username_characters) for i in range(length)) return ''.join(secrets.choice(self.username_characters) for i in range(length))