diff --git a/.dockerignore b/.dockerignore index 7b9644ad..5674e27e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -21,11 +21,8 @@ LICENSE # Skip envrc .envrc -# Skip ansible-container stuff -ansible* -container.yml -meta.yml -requirements.yml +# Skip terraform +terraform # Skip kubernetes and Docker config stuff deploy diff --git a/atst/domain/users.py b/atst/domain/users.py index 5e09ce22..e5fdbad7 100644 --- a/atst/domain/users.py +++ b/atst/domain/users.py @@ -93,10 +93,13 @@ class Users(object): return user @classmethod - def give_ccpo_perms(cls, user): + def give_ccpo_perms(cls, user, commit=True): user.permission_sets = PermissionSets.get_all() db.session.add(user) - db.session.commit() + + if commit: + db.session.commit() + return user @classmethod diff --git a/script/create_database.py b/script/create_database.py new file mode 100644 index 00000000..f21a857e --- /dev/null +++ b/script/create_database.py @@ -0,0 +1,41 @@ +# Add root application dir to the python path +import os +import sys + +parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) +sys.path.append(parent_dir) + +import sqlalchemy + +from atst.app import make_config + + +def _root_connection(config, root_db): + # Assemble DATABASE_URI value + database_uri = "postgresql://{}:{}@{}:{}/{}".format( # pragma: allowlist secret + config.get("PGUSER"), + config.get("PGPASSWORD"), + config.get("PGHOST"), + config.get("PGPORT"), + root_db, + ) + engine = sqlalchemy.create_engine(database_uri) + return engine.connect() + + +def create_database(conn, dbname): + conn.execute("commit") + conn.execute(f"CREATE DATABASE {dbname};") + conn.close() + + return True + + +if __name__ == "__main__": + dbname = sys.argv[1] + config = make_config() + + conn = _root_connection(config, "postgres") + + print(f"Creating database {dbname}") + create_database(conn, dbname) diff --git a/script/database_setup.py b/script/database_setup.py new file mode 100644 index 00000000..7784be05 --- /dev/null +++ b/script/database_setup.py @@ -0,0 +1,76 @@ +# Add root application dir to the python path +import os +import sys + +parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) +sys.path.append(parent_dir) + +import sqlalchemy +import yaml + +from atst.app import make_config, make_app +from atst.database import db +from atst.domain.users import Users +from atst.models import User +from reset_database import reset_database + + +def database_setup(username, password, dbname, ccpo_users): + print( + f"Creating Postgres user role for '{username}' and granting all privileges to database '{dbname}'." + ) + try: + _create_database_user(username, password, dbname) + except sqlalchemy.exc.ProgrammingError as err: + print(f"Postgres user role '{username}' already exists.") + + print("Applying schema and seeding roles and permissions.") + reset_database() + print("Creating initial set of CCPO users.") + _add_ccpo_users(ccpo_users) + + +def _create_database_user(username, password, dbname): + conn = db.engine.connect() + + meta = sqlalchemy.MetaData(bind=conn) + meta.reflect() + + trans = conn.begin() + engine = trans.connection.engine + + engine.execute( + f"CREATE ROLE {username} WITH LOGIN NOSUPERUSER INHERIT NOCREATEDB NOCREATEROLE NOREPLICATION PASSWORD '{password}';\n" + f"GRANT ALL PRIVILEGES ON DATABASE {dbname} TO {username};\n" + f"ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL PRIVILEGES ON TABLES TO {username}; \n" + f"ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL PRIVILEGES ON SEQUENCES TO {username}; \n" + f"ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL PRIVILEGES ON FUNCTIONS TO {username}; \n" + ) + + trans.commit() + + +def _add_ccpo_users(ccpo_users): + for user_data in ccpo_users: + user = User(**user_data) + Users.give_ccpo_perms(user, commit=False) + db.session.add(user) + + db.session.commit() + + +def _load_yaml(file_): + with open(file_) as f: + return yaml.safe_load(f) + + +if __name__ == "__main__": + config = make_config({"DISABLE_CRL_CHECK": True, "DEBUG": False}) + app = make_app(config) + with app.app_context(): + dbname = config.get("PGDATABASE", "atat") + username = sys.argv[1] + password = sys.argv[2] + ccpo_user_file = sys.argv[3] + ccpo_users = _load_yaml(ccpo_user_file) + database_setup(username, password, dbname, ccpo_users) diff --git a/script/reset_database.py b/script/reset_database.py index cfa63298..dda1c1ba 100644 --- a/script/reset_database.py +++ b/script/reset_database.py @@ -16,7 +16,9 @@ from atst.app import make_config, make_app def reset_database(): conn = db.engine.connect() - meta = sqlalchemy.MetaData(bind=conn, reflect=True) + meta = sqlalchemy.MetaData(bind=conn) + meta.reflect() + trans = conn.begin() # drop all tables diff --git a/terraform/README.md b/terraform/README.md index ec0fbdeb..8c8e7beb 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -1,11 +1,11 @@ # ATAT Terraform Welcome! You've found the ATAT IaC configurations. -ATAT is configured using terraform and a wrapper script called `secrets-tool`. With `terraform` we can configure infrastructure in a programatic way and ensure consistency across environments. +ATAT is configured using terraform and a wrapper script called `secrets-tool`. With `terraform` we can configure infrastructure in a programatic way and ensure consistency across environments. ## Directory Structure -**modules/** - Terraform modules. These are modules that can be re-used for multiple environments. +**modules/** - Terraform modules. These are modules that can be re-used for multiple environments. **providers/** - Specific environment configurations. (dev,production, etc) @@ -92,7 +92,7 @@ Check the output for errors. Sometimes the syntax is valid, but some of the conf # After running TF (Manual Steps) -## VM Scale Set +## VM Scale Set After running terraform, we need to make a manual change to the VM Scale Set that is used in the kubernetes. Terraform has a bug that is not applying this as of `v1.40` of the `azurerm` provider. In order to get the `SystemAssigned` identity to be set, it needs to be set manually in the console. @@ -253,7 +253,7 @@ Uncomment the `backend {}` section in the `provider.tf` file. Once uncommented, *Say `yes` to the question* -Now we need to update the Update `variables.tf` with the principals for the users in `admin_users` variable map. If these are not defined yet, just leave it as an empty set. +Now we need to update the Update `variables.tf` with the principals for the users in `admin_users` variable map. If these are not defined yet, just leave it as an empty set. Next, we'll create the operator keyvault. @@ -281,4 +281,25 @@ secrets-tool secrets --keyvault https://ops-jedidev-keyvault.vault.azure.net/ cr `terraform apply` -*[Configure AD for MFA](https://docs.microsoft.com/en-us/azure/vpn-gateway/openvpn-azure-ad-mfa)* \ No newline at end of file +*[Configure AD for MFA](https://docs.microsoft.com/en-us/azure/vpn-gateway/openvpn-azure-ad-mfa)* + +*Then we need an instance of the container* + +Change directories to the repo root. Ensure that you've checked out the staging or master branch: + +`docker build . --build-arg CSP=azure -f ./Dockerfile -t atat:latest` + +*Create secrets for ATAT database user* + +Change directories back to terraform/secrets-tool. There is a sample file there. Make sure you know the URL for the aplication Key Vault (distinct from the operator Key Vault). Run: + +`secrets-tool secrets --keyvault [application key vault URL] load -f ./postgres-user.yaml + +*Create the database, database user, schema, and initial data set* + + +This is discussed in more detail [here](https://github.com/dod-ccpo/atst/tree/staging/terraform/secrets-tool#setting-up-the-initial-atat-database). Be sure to read the requirements section. + +``` +secrets-tool database --keyvault [operator key vault URL] provision --app-keyvault [application key vault URL] --dbname jedidev-atat --dbhost [database host name] --ccpo-users /full/path/to/users.yml +``` diff --git a/terraform/modules/postgres/main.tf b/terraform/modules/postgres/main.tf index c3252264..29b6cc53 100644 --- a/terraform/modules/postgres/main.tf +++ b/terraform/modules/postgres/main.tf @@ -35,11 +35,3 @@ resource "azurerm_postgresql_virtual_network_rule" "sql" { subnet_id = var.subnet_id ignore_missing_vnet_service_endpoint = true } - -resource "azurerm_postgresql_database" "db" { - name = "${var.name}-${var.environment}-atat" - resource_group_name = azurerm_resource_group.sql.name - server_name = azurerm_postgresql_server.sql.name - charset = "UTF8" - collation = "en-US" -} diff --git a/terraform/modules/postgres/outputs.tf b/terraform/modules/postgres/outputs.tf index 1ff1dd65..e69de29b 100644 --- a/terraform/modules/postgres/outputs.tf +++ b/terraform/modules/postgres/outputs.tf @@ -1,3 +0,0 @@ -output "db_name" { - value = azurerm_postgresql_database.db.name -} diff --git a/terraform/secrets-tool/README.md b/terraform/secrets-tool/README.md index 28b44817..bd407607 100644 --- a/terraform/secrets-tool/README.md +++ b/terraform/secrets-tool/README.md @@ -15,7 +15,7 @@ With both usernames and passwords generated, the application only needs to make Ex. ``` { - 'postgres_root_user': 'EzTEzSNLKQPHuJyPdPloIDCAlcibbl', + 'postgres_root_user': 'EzTEzSNLKQPHuJyPdPloIDCAlcibbl', 'postgres_root_password': "2+[A@E4:C=ubb/#R#'n