Merge pull request #1353 from dod-ccpo/take3

Database bootstrapping

.dockerignore

@@ -21,11 +21,8 @@ LICENSE
 # Skip envrc
 .envrc
 
-# Skip ansible-container stuff
-ansible*
-container.yml
-meta.yml
-requirements.yml
+# Skip terraform
+terraform
 
 # Skip kubernetes and Docker config stuff
 deploy

atst/domain/users.py

@@ -93,10 +93,13 @@ class Users(object):
         return user
 
     @classmethod
-    def give_ccpo_perms(cls, user):
+    def give_ccpo_perms(cls, user, commit=True):
         user.permission_sets = PermissionSets.get_all()
         db.session.add(user)
-        db.session.commit()
+
+        if commit:
+            db.session.commit()
+
         return user
 
     @classmethod

script/create_database.py

@@ -0,0 +1,41 @@
+# Add root application dir to the python path
+import os
+import sys
+
+parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+sys.path.append(parent_dir)
+
+import sqlalchemy
+
+from atst.app import make_config
+
+
+def _root_connection(config, root_db):
+    # Assemble DATABASE_URI value
+    database_uri = "postgresql://{}:{}@{}:{}/{}".format(  # pragma: allowlist secret
+        config.get("PGUSER"),
+        config.get("PGPASSWORD"),
+        config.get("PGHOST"),
+        config.get("PGPORT"),
+        root_db,
+    )
+    engine = sqlalchemy.create_engine(database_uri)
+    return engine.connect()
+
+
+def create_database(conn, dbname):
+    conn.execute("commit")
+    conn.execute(f"CREATE DATABASE {dbname};")
+    conn.close()
+
+    return True
+
+
+if __name__ == "__main__":
+    dbname = sys.argv[1]
+    config = make_config()
+
+    conn = _root_connection(config, "postgres")
+
+    print(f"Creating database {dbname}")
+    create_database(conn, dbname)

script/database_setup.py

@@ -0,0 +1,76 @@
+# Add root application dir to the python path
+import os
+import sys
+
+parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+sys.path.append(parent_dir)
+
+import sqlalchemy
+import yaml
+
+from atst.app import make_config, make_app
+from atst.database import db
+from atst.domain.users import Users
+from atst.models import User
+from reset_database import reset_database
+
+
+def database_setup(username, password, dbname, ccpo_users):
+    print(
+        f"Creating Postgres user role for '{username}' and granting all privileges to database '{dbname}'."
+    )
+    try:
+        _create_database_user(username, password, dbname)
+    except sqlalchemy.exc.ProgrammingError as err:
+        print(f"Postgres user role '{username}' already exists.")
+
+    print("Applying schema and seeding roles and permissions.")
+    reset_database()
+    print("Creating initial set of CCPO users.")
+    _add_ccpo_users(ccpo_users)
+
+
+def _create_database_user(username, password, dbname):
+    conn = db.engine.connect()
+
+    meta = sqlalchemy.MetaData(bind=conn)
+    meta.reflect()
+
+    trans = conn.begin()
+    engine = trans.connection.engine
+
+    engine.execute(
+        f"CREATE ROLE {username} WITH LOGIN NOSUPERUSER INHERIT NOCREATEDB NOCREATEROLE NOREPLICATION PASSWORD '{password}';\n"
+        f"GRANT ALL PRIVILEGES ON DATABASE {dbname} TO {username};\n"
+        f"ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL PRIVILEGES ON TABLES TO {username}; \n"
+        f"ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL PRIVILEGES ON SEQUENCES TO {username}; \n"
+        f"ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL PRIVILEGES ON FUNCTIONS TO {username}; \n"
+    )
+
+    trans.commit()
+
+
+def _add_ccpo_users(ccpo_users):
+    for user_data in ccpo_users:
+        user = User(**user_data)
+        Users.give_ccpo_perms(user, commit=False)
+        db.session.add(user)
+
+    db.session.commit()
+
+
+def _load_yaml(file_):
+    with open(file_) as f:
+        return yaml.safe_load(f)
+
+
+if __name__ == "__main__":
+    config = make_config({"DISABLE_CRL_CHECK": True, "DEBUG": False})
+    app = make_app(config)
+    with app.app_context():
+        dbname = config.get("PGDATABASE", "atat")
+        username = sys.argv[1]
+        password = sys.argv[2]
+        ccpo_user_file = sys.argv[3]
+        ccpo_users = _load_yaml(ccpo_user_file)
+        database_setup(username, password, dbname, ccpo_users)

script/reset_database.py

@@ -16,7 +16,9 @@ from atst.app import make_config, make_app
 def reset_database():
     conn = db.engine.connect()
 
-    meta = sqlalchemy.MetaData(bind=conn, reflect=True)
+    meta = sqlalchemy.MetaData(bind=conn)
+    meta.reflect()
+
     trans = conn.begin()
 
     # drop all tables

terraform/README.md

@@ -1,11 +1,11 @@
 # ATAT Terraform
 Welcome! You've found the ATAT IaC configurations.
 
-ATAT is configured using terraform and a wrapper script called `secrets-tool`. With `terraform` we can configure infrastructure in a programatic way and ensure consistency across environments. 
+ATAT is configured using terraform and a wrapper script called `secrets-tool`. With `terraform` we can configure infrastructure in a programatic way and ensure consistency across environments.
 
 ## Directory Structure
 
-**modules/** - Terraform modules. These are modules that can be re-used for multiple environments. 
+**modules/** - Terraform modules. These are modules that can be re-used for multiple environments.
 
 **providers/** - Specific environment configurations. (dev,production, etc)
 
@@ -92,7 +92,7 @@ Check the output for errors. Sometimes the syntax is valid, but some of the conf
 
 # After running TF (Manual Steps)
 
-## VM Scale Set 
+## VM Scale Set
 After running terraform, we need to make a manual change to the VM Scale Set that is used in the kubernetes. Terraform has a bug that is not applying this as of `v1.40` of the `azurerm` provider.
 
 In order to get the `SystemAssigned` identity to be set, it needs to be set manually in the console.
@@ -253,7 +253,7 @@ Uncomment the `backend {}` section in the `provider.tf` file. Once uncommented,
 
 *Say `yes` to the question*
 
-Now we need to update the Update `variables.tf` with the principals for the users in `admin_users` variable map. If these are not defined yet, just leave it as an empty set. 
+Now we need to update the Update `variables.tf` with the principals for the users in `admin_users` variable map. If these are not defined yet, just leave it as an empty set.
 
 Next, we'll create the operator keyvault.
 
@@ -281,4 +281,25 @@ secrets-tool secrets --keyvault https://ops-jedidev-keyvault.vault.azure.net/ cr
 
 `terraform apply`
 
-*[Configure AD for MFA](https://docs.microsoft.com/en-us/azure/vpn-gateway/openvpn-azure-ad-mfa)*
+*[Configure AD for MFA](https://docs.microsoft.com/en-us/azure/vpn-gateway/openvpn-azure-ad-mfa)*
+
+*Then we need an instance of the container*
+
+Change directories to the repo root. Ensure that you've checked out the staging or master branch:
+
+`docker build . --build-arg CSP=azure -f ./Dockerfile -t atat:latest`
+
+*Create secrets for ATAT database user*
+
+Change directories back to terraform/secrets-tool. There is a sample file there. Make sure you know the URL for the aplication Key Vault (distinct from the operator Key Vault). Run:
+
+`secrets-tool secrets --keyvault [application key vault URL] load -f ./postgres-user.yaml
+
+*Create the database, database user, schema, and initial data set*
+
+
+This is discussed in more detail [here](https://github.com/dod-ccpo/atst/tree/staging/terraform/secrets-tool#setting-up-the-initial-atat-database). Be sure to read the requirements section.
+
+```
+secrets-tool database --keyvault [operator key vault URL] provision --app-keyvault [application key vault URL] --dbname jedidev-atat --dbhost [database host name] --ccpo-users /full/path/to/users.yml
+```

terraform/modules/postgres/main.tf

@@ -35,11 +35,3 @@ resource "azurerm_postgresql_virtual_network_rule" "sql" {
   subnet_id                            = var.subnet_id
   ignore_missing_vnet_service_endpoint = true
 }
-
-resource "azurerm_postgresql_database" "db" {
-  name                = "${var.name}-${var.environment}-atat"
-  resource_group_name = azurerm_resource_group.sql.name
-  server_name         = azurerm_postgresql_server.sql.name
-  charset             = "UTF8"
-  collation           = "en-US"
-}

terraform/modules/postgres/outputs.tf

@@ -1,3 +0,0 @@
-output "db_name" {
-  value = azurerm_postgresql_database.db.name
-}

terraform/secrets-tool/README.md

@@ -15,7 +15,7 @@ With both usernames and passwords generated, the application only needs to make
 Ex.
 ```
 {
-    'postgres_root_user': 'EzTEzSNLKQPHuJyPdPloIDCAlcibbl',             
+    'postgres_root_user': 'EzTEzSNLKQPHuJyPdPloIDCAlcibbl',
     'postgres_root_password': "2+[A@E4:C=ubb/#R#'n<p|wCW-|%q^"
 }
 ```
@@ -30,6 +30,51 @@ Terraform typically expects user defined secrets to be stored in either a file,
 
 This provides a number of security benefits. First, secrets are not on disk. Secondly, users/operators never see the secrets fly by (passerbys or voyeurs that like to look over your shoulder when deploying to production)
 
+## Setting up the initial ATAT database
+
+This handles bootstrapping the ATAT database with a user, schema, and initial data.
+
+It does the following:
+
+- Sources the Postgres root user credentials
+- Source the Postgres ATAT user password
+- Runs a script inside an ATAT docker container to set up the initial database user, schema, and seed data in the database
+
+Requirements:
+
+- docker
+- A copy of the ATAT docker image. This can be built in the repo root with: `docker build . --build-arg CSP=azure -f ./Dockerfile -t atat:latest`
+- You need to know the hostname for the Postgres database. Your IP must either be whitelisted in its firewall rules or you must be behind the VPN.
+- You will need a YAML file listing all the CCPO users to be added to the database, with the format:
+
+```
+- dod_id: "2323232323"
+  first_name: "Luke"
+  last_name: "Skywalker"
+- dod_id: "5656565656"
+  first_name: "Han"
+  last_name: "Solo"
+```
+
+- There should be a password for the ATAT database user in the application Key Vault, preferably named `PGPASSWORD`. You can load this by running `secrets-tool --keyvault [operator key vault url] load -f postgres-user.yml` and supplying YAML like:
+
+```
+---
+- PGPASSWORD:
+    type: 'password'
+    length: 30
+```
+
+This command takes a lot of arguments. Run `secrets-tool database --keyvault [operator key vault url] provision -- help` to see the full list of available options.
+
+The command supplies some defaults by assuming you've followed the patterns in sample-secrets.yml and elsewhere.
+
+An example would be:
+
+```
+secrets-tool database --keyvault [operator key vault URL] provision --app-keyvault [application key vault URL] --dbname jedidev-atat --dbhost [database host name] --ccpo-users /full/path/to/users.yml
+```
+
 # Setup
 
 *Requirements*
@@ -76,4 +121,4 @@ secrets-tool secrets --keyvault https://operator-dev-keyvault.vault.azure.net/ l
 This will fetch all secrets from the keyvault specified. `secrets-tool` then converts the keys to a variable name that terraform will look for. Essentially it prepends the keys found in KeyVault with `TF_VAR` and then executes terraform as a subprocess with the injected environment variables.
 ```
 secrets-tool terraform --keyvault https://operator-dev-keyvault.vault.azure.net/ plan
-```
+```

terraform/secrets-tool/commands/database.py

@@ -0,0 +1,143 @@
+import os
+import click
+import logging
+import subprocess
+
+from utils.keyvault.secrets import SecretsClient
+
+logger = logging.getLogger(__name__)
+
+
+def _run_cmd(command):
+    try:
+        env = os.environ.copy()
+        with subprocess.Popen(
+            command, env=env, stdout=subprocess.PIPE, shell=True
+        ) as proc:
+            for line in proc.stdout:
+                logging.info(line.decode("utf-8"))
+    except Exception as e:
+        print(e)
+
+
+@click.group()
+@click.option("--keyvault", required=True, help="Specify the keyvault to operate on")
+@click.pass_context
+def database(ctx, keyvault):
+    ctx.ensure_object(dict)
+    ctx.obj["keyvault"] = keyvault
+
+
+# root password, root username
+@click.command("provision")
+@click.option(
+    "--app-keyvault",
+    "app_keyvault",
+    required=True,
+    help="The username for the new Postgres user.",
+)
+@click.option(
+    "--user-username",
+    "user_username",
+    default="atat",
+    required=True,
+    help="The username for the new Postgres user.",
+)
+@click.option(
+    "--user-password-key",
+    "user_password_key",
+    default="PGPASSWORD",
+    required=True,
+    help="The name of the user's password key in the specified vault.",
+)
+@click.option(
+    "--root-username-key",
+    "root_username_key",
+    default="postgres-root-user",
+    required=True,
+    help="The name of the user's password key in the specified vault.",
+)
+@click.option(
+    "--root-password-key",
+    "root_password_key",
+    default="postgres-root-password",
+    required=True,
+    help="The name of the user's password key in the specified vault.",
+)
+@click.option(
+    "--dbname",
+    "dbname",
+    required=True,
+    help="The name of the database the user will be given full access to.",
+)
+@click.option(
+    "--dbhost",
+    "dbhost",
+    required=True,
+    help="The name of the database the user will be given full access to.",
+)
+@click.option(
+    "--container",
+    "container",
+    default="atat:latest",
+    required=True,
+    help="The container to run the provisioning command in.",
+)
+@click.option(
+    "--ccpo-users",
+    "ccpo_users",
+    required=True,
+    help="The full path to a YAML file listing CCPO users to be seeded to the database.",
+)
+@click.pass_context
+def provision(
+    ctx,
+    app_keyvault,
+    user_username,
+    user_password_key,
+    root_username_key,
+    root_password_key,
+    dbname,
+    dbhost,
+    container,
+    ccpo_users,
+):
+    """
+    Set up the initial ATAT database.
+    """
+    logger.info("obtaining postgres root user credentials")
+    operator_keyvault = SecretsClient(vault_url=ctx.obj["keyvault"])
+    root_password = operator_keyvault.get_secret(root_password_key)
+    root_name = operator_keyvault.get_secret(root_username_key)
+
+    logger.info("obtaining postgres database user password")
+    app_keyvault = SecretsClient(vault_url=app_keyvault)
+    user_password = app_keyvault.get_secret(user_password_key)
+
+    logger.info("starting docker process")
+
+    create_database_cmd = (
+        f"docker run -e PGHOST='{dbhost}'"
+        f" -e PGPASSWORD='{root_password}'"
+        f" -e PGUSER='{root_name}@{dbhost}'"
+        f" -e PGDATABASE='{dbname}'"
+        f" -e PGSSLMODE=require"
+        f" {container}"
+        f" .venv/bin/python script/create_database.py {dbname}"
+    )
+    _run_cmd(create_database_cmd)
+
+    seed_database_cmd = (
+        f"docker run -e PGHOST='{dbhost}'"
+        f" -e PGPASSWORD='{root_password}'"
+        f" -e PGUSER='{root_name}@{dbhost}'"
+        f" -e PGDATABASE='{dbname}'"
+        f" -e PGSSLMODE=require"
+        f" -v {ccpo_users}:/opt/atat/atst/users.yml"
+        f" {container}"
+        f" .venv/bin/python script/database_setup.py {user_username} '{user_password}' users.yml"
+    )
+    _run_cmd(seed_database_cmd)
+
+
+database.add_command(provision)

terraform/secrets-tool/postgres-user.yaml

@@ -0,0 +1,4 @@
+---
+- PGPASSWORD:
+    type: 'password'
+    length: 30

terraform/secrets-tool/secrets-tool

@@ -7,6 +7,7 @@ import logging
 
 from commands.secrets import secrets
 from commands.terraform import terraform
+from commands.database import database
 
 config.setup_logging()
 logger = logging.getLogger(__name__)
@@ -21,6 +22,7 @@ def cli():
 # Add additional command groups
 cli.add_command(secrets)
 cli.add_command(terraform)
+cli.add_command(database)
 
 
 if __name__ == "__main__":
@@ -41,12 +43,12 @@ if __name__ == "__main__":
             val = keyvault.get_secret(secret)
             #print(val)
             os.environ[name] = val
-        env = os.environ.copy() 
+        env = os.environ.copy()
         command = "{} {}".format(PROCESS, sys.argv[1])
         with subprocess.Popen(command, env=env, stdout=subprocess.PIPE, shell=True) as proc:
             for line in proc.stdout:
                 logging.info(line.decode("utf-8") )
-        
+
     except Exception as e:
         print(e, traceback.print_stack)
-'''
+'''

terraform/secrets-tool/utils/keyvault/secrets.py

@@ -36,7 +36,7 @@ class SecretsLoader():
     load the secrets in to keyvault
     """
     def __init__(self, yaml_file: str, keyvault: object):
-        assert Path(yaml_file).exists() 
+        assert Path(yaml_file).exists()
         self.yaml_file = yaml_file
         self.keyvault = keyvault
         self.config = dict()
@@ -47,7 +47,7 @@ class SecretsLoader():
     def _load_yaml(self):
         with open(self.yaml_file) as handle:
             self.config = yaml.load(handle, Loader=yaml.FullLoader)
-    
+
     def _generate_secrets(self):
         secrets = GenerateSecrets(self.config).process_definition()
         self.secrets = secrets
@@ -60,12 +60,14 @@ class SecretsLoader():
 
 class GenerateSecrets():
     """
-    Read the secrets definition and generate requiesite 
+    Read the secrets definition and generate requiesite
     secrets based on the type of secret and arguments
     provided
     """
     def __init__(self, definitions: dict):
         self.definitions = definitions
+        most_punctuation = string.punctuation.replace("'", "").replace('"', "")
+        self.password_characters = string.ascii_letters + string.digits + most_punctuation
 
     def process_definition(self):
         """
@@ -101,9 +103,8 @@ class GenerateSecrets():
     # Types. Can be usernames, passwords, or in the future things like salted
     # tokens, uuid, or other specialized types
     def _generate_password(self, length: int):
-        self.password_characters = string.ascii_letters + string.digits + string.punctuation
         return ''.join(secrets.choice(self.password_characters) for i in range(length))
-    
+
     def _generate_username(self, length: int):
         self.username_characters = string.ascii_letters
         return ''.join(secrets.choice(self.username_characters) for i in range(length))