secrets-tool command for bootstrapping database.
This additional secrets-tool command can be used to run the database bootsrapping script (`script/database_setup.py`) inside an ATAT docker container against the Azure database. It sources the necessary keys from Key Vault.
This commit is contained in:
parent
49a1a219ae
commit
a8f6befc17
@ -1,13 +1,11 @@
|
|||||||
# Add root application dir to the python path
|
# Add root application dir to the python path
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
from contextlib import contextmanager
|
|
||||||
|
|
||||||
parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
|
parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
|
||||||
sys.path.append(parent_dir)
|
sys.path.append(parent_dir)
|
||||||
|
|
||||||
import sqlalchemy
|
import sqlalchemy
|
||||||
from alembic import config as alembic_config
|
|
||||||
import yaml
|
import yaml
|
||||||
|
|
||||||
from atst.app import make_config, make_app
|
from atst.app import make_config, make_app
|
||||||
@ -25,7 +23,6 @@ def database_setup(username, password, dbname, ccpo_users):
|
|||||||
try:
|
try:
|
||||||
_create_database_user(username, password, dbname)
|
_create_database_user(username, password, dbname)
|
||||||
except sqlalchemy.exc.ProgrammingError as err:
|
except sqlalchemy.exc.ProgrammingError as err:
|
||||||
raise err
|
|
||||||
print(f"Postgres user role '{username}' already exists.")
|
print(f"Postgres user role '{username}' already exists.")
|
||||||
|
|
||||||
print("Applying schema and seeding roles and permissions.")
|
print("Applying schema and seeding roles and permissions.")
|
||||||
|
@ -15,7 +15,7 @@ With both usernames and passwords generated, the application only needs to make
|
|||||||
Ex.
|
Ex.
|
||||||
```
|
```
|
||||||
{
|
{
|
||||||
'postgres_root_user': 'EzTEzSNLKQPHuJyPdPloIDCAlcibbl',
|
'postgres_root_user': 'EzTEzSNLKQPHuJyPdPloIDCAlcibbl',
|
||||||
'postgres_root_password': "2+[A@E4:C=ubb/#R#'n<p|wCW-|%q^"
|
'postgres_root_password': "2+[A@E4:C=ubb/#R#'n<p|wCW-|%q^"
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
@ -30,6 +30,40 @@ Terraform typically expects user defined secrets to be stored in either a file,
|
|||||||
|
|
||||||
This provides a number of security benefits. First, secrets are not on disk. Secondly, users/operators never see the secrets fly by (passerbys or voyeurs that like to look over your shoulder when deploying to production)
|
This provides a number of security benefits. First, secrets are not on disk. Secondly, users/operators never see the secrets fly by (passerbys or voyeurs that like to look over your shoulder when deploying to production)
|
||||||
|
|
||||||
|
## Setting up the initial ATAT database
|
||||||
|
|
||||||
|
This handles bootstrapping the ATAT database with a user, schema, and initial data.
|
||||||
|
|
||||||
|
It does the following:
|
||||||
|
|
||||||
|
- Sources the Postgres root user credentials
|
||||||
|
- Source the Postgres ATAT user password
|
||||||
|
- Runs a script inside an ATAT docker container to set up the initial database user, schema, and seed data in the database
|
||||||
|
|
||||||
|
Requirements:
|
||||||
|
|
||||||
|
- docker
|
||||||
|
- A copy of the ATAT docker image. This can be built in the repo root with: `docker build . --build-arg CSP=azure -f ./Dockerfile -t atat:latest`
|
||||||
|
- You need to know the hostname for the Postgres database. Your IP must either be whitelisted in its firewall rules or you must be behind the VPN.
|
||||||
|
- There should be a password for the ATAT database user in the application Key Vault, preferably named `PGPASSWORD`. You can load this by running `secrets-tool --keyvault [operator key vault url] load -f postgres-user.yml` and supplying YAML like:
|
||||||
|
|
||||||
|
```
|
||||||
|
---
|
||||||
|
- PGPASSWORD:
|
||||||
|
type: 'password'
|
||||||
|
length: 30
|
||||||
|
```
|
||||||
|
|
||||||
|
This command takes a lot of arguments. Run `secrets-tool database --keyvault [operator key vault url] provision -- help` to see the full list of available options.
|
||||||
|
|
||||||
|
The command supplies some defaults by assuming you've followed the patterns in sample-secrets.yml and elsewhere.
|
||||||
|
|
||||||
|
An example would be:
|
||||||
|
|
||||||
|
```
|
||||||
|
secrets-tool database --keyvault [operator key vault URL] provision --app-keyvault [application key vault URL] --dbname jedidev-atat --dbhost [database host name] --ccpo-users /full/path/to/users.yml
|
||||||
|
```
|
||||||
|
|
||||||
# Setup
|
# Setup
|
||||||
|
|
||||||
*Requirements*
|
*Requirements*
|
||||||
@ -76,4 +110,4 @@ secrets-tool secrets --keyvault https://operator-dev-keyvault.vault.azure.net/ l
|
|||||||
This will fetch all secrets from the keyvault specified. `secrets-tool` then converts the keys to a variable name that terraform will look for. Essentially it prepends the keys found in KeyVault with `TF_VAR` and then executes terraform as a subprocess with the injected environment variables.
|
This will fetch all secrets from the keyvault specified. `secrets-tool` then converts the keys to a variable name that terraform will look for. Essentially it prepends the keys found in KeyVault with `TF_VAR` and then executes terraform as a subprocess with the injected environment variables.
|
||||||
```
|
```
|
||||||
secrets-tool terraform --keyvault https://operator-dev-keyvault.vault.azure.net/ plan
|
secrets-tool terraform --keyvault https://operator-dev-keyvault.vault.azure.net/ plan
|
||||||
```
|
```
|
||||||
|
134
terraform/secrets-tool/commands/database.py
Normal file
134
terraform/secrets-tool/commands/database.py
Normal file
@ -0,0 +1,134 @@
|
|||||||
|
import os
|
||||||
|
import click
|
||||||
|
import logging
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
from utils.keyvault.secrets import SecretsClient
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def _run_cmd(command):
|
||||||
|
try:
|
||||||
|
env = os.environ.copy()
|
||||||
|
with subprocess.Popen(
|
||||||
|
command, env=env, stdout=subprocess.PIPE, shell=True
|
||||||
|
) as proc:
|
||||||
|
for line in proc.stdout:
|
||||||
|
logging.info(line.decode("utf-8"))
|
||||||
|
except Exception as e:
|
||||||
|
print(e)
|
||||||
|
|
||||||
|
|
||||||
|
@click.group()
|
||||||
|
@click.option("--keyvault", required=True, help="Specify the keyvault to operate on")
|
||||||
|
@click.pass_context
|
||||||
|
def database(ctx, keyvault):
|
||||||
|
ctx.ensure_object(dict)
|
||||||
|
ctx.obj["keyvault"] = keyvault
|
||||||
|
|
||||||
|
|
||||||
|
# root password, root username
|
||||||
|
@click.command("provision")
|
||||||
|
@click.option(
|
||||||
|
"--app-keyvault",
|
||||||
|
"app_keyvault",
|
||||||
|
required=True,
|
||||||
|
help="The username for the new Postgres user.",
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
"--user-username",
|
||||||
|
"user_username",
|
||||||
|
default="atat",
|
||||||
|
required=True,
|
||||||
|
help="The username for the new Postgres user.",
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
"--user-password-key",
|
||||||
|
"user_password_key",
|
||||||
|
default="PGPASSWORD",
|
||||||
|
required=True,
|
||||||
|
help="The name of the user's password key in the specified vault.",
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
"--root-username-key",
|
||||||
|
"root_username_key",
|
||||||
|
default="postgres-root-user",
|
||||||
|
required=True,
|
||||||
|
help="The name of the user's password key in the specified vault.",
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
"--root-password-key",
|
||||||
|
"root_password_key",
|
||||||
|
default="postgres-root-password",
|
||||||
|
required=True,
|
||||||
|
help="The name of the user's password key in the specified vault.",
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
"--dbname",
|
||||||
|
"dbname",
|
||||||
|
required=True,
|
||||||
|
help="The name of the database the user will be given full access to.",
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
"--dbhost",
|
||||||
|
"dbhost",
|
||||||
|
required=True,
|
||||||
|
help="The name of the database the user will be given full access to.",
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
"--container",
|
||||||
|
"container",
|
||||||
|
default="atat:latest",
|
||||||
|
required=True,
|
||||||
|
help="The container to run the provisioning command in.",
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
"--ccpo-users",
|
||||||
|
"ccpo_users",
|
||||||
|
required=True,
|
||||||
|
help="The full path to a YAML file listing CCPO users to be seeded to the database.",
|
||||||
|
)
|
||||||
|
@click.pass_context
|
||||||
|
def provision(
|
||||||
|
ctx,
|
||||||
|
app_keyvault,
|
||||||
|
user_username,
|
||||||
|
user_password_key,
|
||||||
|
root_username_key,
|
||||||
|
root_password_key,
|
||||||
|
dbname,
|
||||||
|
dbhost,
|
||||||
|
container,
|
||||||
|
ccpo_users,
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
Set up the initial ATAT database.
|
||||||
|
"""
|
||||||
|
logger.info("obtaining postgres root user credentials")
|
||||||
|
operator_keyvault = SecretsClient(vault_url=ctx.obj["keyvault"])
|
||||||
|
root_password = operator_keyvault.get_secret(root_password_key)
|
||||||
|
root_name = operator_keyvault.get_secret(root_username_key)
|
||||||
|
|
||||||
|
logger.info("obtaining postgres database user password")
|
||||||
|
app_keyvault = SecretsClient(vault_url=app_keyvault)
|
||||||
|
user_password = app_keyvault.get_secret(user_password_key)
|
||||||
|
|
||||||
|
logger.info("starting docker process")
|
||||||
|
|
||||||
|
cmd = (
|
||||||
|
f"docker run -e PGHOST={dbhost}"
|
||||||
|
+f" -e PGPASSWORD=\"{root_password}\""
|
||||||
|
+f" -e PGUSER='{root_name}@{dbhost}'"
|
||||||
|
+f" -e PGDATABASE=\"{dbname}\""
|
||||||
|
+f" -e REDIS_HOST=host.docker.internal"
|
||||||
|
+f" -e PGSSLMODE=require"
|
||||||
|
+f" -v {ccpo_users}:/opt/atat/atst/users.yml"
|
||||||
|
+f" {container}"
|
||||||
|
+f" .venv/bin/python script/database_setup.py {user_username} '{user_password}' users.yml"
|
||||||
|
)
|
||||||
|
print(cmd)
|
||||||
|
_run_cmd(cmd)
|
||||||
|
|
||||||
|
|
||||||
|
database.add_command(provision)
|
4
terraform/secrets-tool/postgres-user.yaml
Normal file
4
terraform/secrets-tool/postgres-user.yaml
Normal file
@ -0,0 +1,4 @@
|
|||||||
|
---
|
||||||
|
- PGPASSWORD:
|
||||||
|
type: 'password'
|
||||||
|
length: 30
|
@ -7,6 +7,7 @@ import logging
|
|||||||
|
|
||||||
from commands.secrets import secrets
|
from commands.secrets import secrets
|
||||||
from commands.terraform import terraform
|
from commands.terraform import terraform
|
||||||
|
from commands.database import database
|
||||||
|
|
||||||
config.setup_logging()
|
config.setup_logging()
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
@ -21,6 +22,7 @@ def cli():
|
|||||||
# Add additional command groups
|
# Add additional command groups
|
||||||
cli.add_command(secrets)
|
cli.add_command(secrets)
|
||||||
cli.add_command(terraform)
|
cli.add_command(terraform)
|
||||||
|
cli.add_command(database)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
@ -41,12 +43,12 @@ if __name__ == "__main__":
|
|||||||
val = keyvault.get_secret(secret)
|
val = keyvault.get_secret(secret)
|
||||||
#print(val)
|
#print(val)
|
||||||
os.environ[name] = val
|
os.environ[name] = val
|
||||||
env = os.environ.copy()
|
env = os.environ.copy()
|
||||||
command = "{} {}".format(PROCESS, sys.argv[1])
|
command = "{} {}".format(PROCESS, sys.argv[1])
|
||||||
with subprocess.Popen(command, env=env, stdout=subprocess.PIPE, shell=True) as proc:
|
with subprocess.Popen(command, env=env, stdout=subprocess.PIPE, shell=True) as proc:
|
||||||
for line in proc.stdout:
|
for line in proc.stdout:
|
||||||
logging.info(line.decode("utf-8") )
|
logging.info(line.decode("utf-8") )
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(e, traceback.print_stack)
|
print(e, traceback.print_stack)
|
||||||
'''
|
'''
|
||||||
|
@ -36,7 +36,7 @@ class SecretsLoader():
|
|||||||
load the secrets in to keyvault
|
load the secrets in to keyvault
|
||||||
"""
|
"""
|
||||||
def __init__(self, yaml_file: str, keyvault: object):
|
def __init__(self, yaml_file: str, keyvault: object):
|
||||||
assert Path(yaml_file).exists()
|
assert Path(yaml_file).exists()
|
||||||
self.yaml_file = yaml_file
|
self.yaml_file = yaml_file
|
||||||
self.keyvault = keyvault
|
self.keyvault = keyvault
|
||||||
self.config = dict()
|
self.config = dict()
|
||||||
@ -47,7 +47,7 @@ class SecretsLoader():
|
|||||||
def _load_yaml(self):
|
def _load_yaml(self):
|
||||||
with open(self.yaml_file) as handle:
|
with open(self.yaml_file) as handle:
|
||||||
self.config = yaml.load(handle, Loader=yaml.FullLoader)
|
self.config = yaml.load(handle, Loader=yaml.FullLoader)
|
||||||
|
|
||||||
def _generate_secrets(self):
|
def _generate_secrets(self):
|
||||||
secrets = GenerateSecrets(self.config).process_definition()
|
secrets = GenerateSecrets(self.config).process_definition()
|
||||||
self.secrets = secrets
|
self.secrets = secrets
|
||||||
@ -60,12 +60,14 @@ class SecretsLoader():
|
|||||||
|
|
||||||
class GenerateSecrets():
|
class GenerateSecrets():
|
||||||
"""
|
"""
|
||||||
Read the secrets definition and generate requiesite
|
Read the secrets definition and generate requiesite
|
||||||
secrets based on the type of secret and arguments
|
secrets based on the type of secret and arguments
|
||||||
provided
|
provided
|
||||||
"""
|
"""
|
||||||
def __init__(self, definitions: dict):
|
def __init__(self, definitions: dict):
|
||||||
self.definitions = definitions
|
self.definitions = definitions
|
||||||
|
most_punctuation = string.punctuation.replace("'", "").replace('"', "")
|
||||||
|
self.password_characters = string.ascii_letters + string.digits + most_punctuation
|
||||||
|
|
||||||
def process_definition(self):
|
def process_definition(self):
|
||||||
"""
|
"""
|
||||||
@ -101,9 +103,8 @@ class GenerateSecrets():
|
|||||||
# Types. Can be usernames, passwords, or in the future things like salted
|
# Types. Can be usernames, passwords, or in the future things like salted
|
||||||
# tokens, uuid, or other specialized types
|
# tokens, uuid, or other specialized types
|
||||||
def _generate_password(self, length: int):
|
def _generate_password(self, length: int):
|
||||||
self.password_characters = string.ascii_letters + string.digits + string.punctuation
|
|
||||||
return ''.join(secrets.choice(self.password_characters) for i in range(length))
|
return ''.join(secrets.choice(self.password_characters) for i in range(length))
|
||||||
|
|
||||||
def _generate_username(self, length: int):
|
def _generate_username(self, length: int):
|
||||||
self.username_characters = string.ascii_letters
|
self.username_characters = string.ascii_letters
|
||||||
return ''.join(secrets.choice(self.username_characters) for i in range(length))
|
return ''.join(secrets.choice(self.username_characters) for i in range(length))
|
||||||
|
Loading…
x
Reference in New Issue
Block a user