From d3bced5b5dbd0d56374a9141aa6f2198b9ef8878 Mon Sep 17 00:00:00 2001 From: Rob Gil Date: Sun, 15 Dec 2019 13:02:52 -0500 Subject: [PATCH 01/15] Adds CODEOWNERS file and configures terraform with an owner for approvals --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000..d38680ce --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +terraform/* @dandds From 955a1c483b8b75cef6a3f49a35f07a4fc05e3428 Mon Sep 17 00:00:00 2001 From: Rob Gil Date: Thu, 12 Dec 2019 11:58:31 -0500 Subject: [PATCH 02/15] 169163334 - Initial VPC TF and structure 169163334 - Make supernet configurable 169163334 - Makes DNS servers configurable 169163334 - Adds bucket for state storage 169163334 - Adds k8s, keyvault, azuread provider 169163334 - Adds route tables 169163334 - Adds route table associations 169163334 - Adds default routes to route tables and fixes route table association flapping --- terraform/.gitignore | 1 + terraform/modules/k8s/main.tf | 35 ++++++++++++ terraform/modules/k8s/outputs.tf | 0 terraform/modules/k8s/variables.tf | 35 ++++++++++++ terraform/modules/keyvault/main.tf | 44 +++++++++++++++ terraform/modules/keyvault/variables.tf | 19 +++++++ terraform/modules/vpc/main.tf | 72 +++++++++++++++++++++++++ terraform/modules/vpc/outputs.tf | 3 ++ terraform/modules/vpc/variables.tf | 43 +++++++++++++++ terraform/providers/dev/k8s.tf | 11 ++++ terraform/providers/dev/keyvault.tf | 7 +++ terraform/providers/dev/provider.tf | 17 ++++++ terraform/providers/dev/variables.tf | 56 +++++++++++++++++++ terraform/providers/dev/vpc.tf | 12 +++++ 14 files changed, 355 insertions(+) create mode 100644 terraform/.gitignore create mode 100644 terraform/modules/k8s/main.tf create mode 100644 terraform/modules/k8s/outputs.tf create mode 100644 terraform/modules/k8s/variables.tf create mode 100644 terraform/modules/keyvault/main.tf create mode 100644 terraform/modules/keyvault/variables.tf create mode 100644 terraform/modules/vpc/main.tf create mode 100644 terraform/modules/vpc/outputs.tf create mode 100644 terraform/modules/vpc/variables.tf create mode 100644 terraform/providers/dev/k8s.tf create mode 100644 terraform/providers/dev/keyvault.tf create mode 100644 terraform/providers/dev/provider.tf create mode 100644 terraform/providers/dev/variables.tf create mode 100644 terraform/providers/dev/vpc.tf diff --git a/terraform/.gitignore b/terraform/.gitignore new file mode 100644 index 00000000..3fa8c86b --- /dev/null +++ b/terraform/.gitignore @@ -0,0 +1 @@ +.terraform diff --git a/terraform/modules/k8s/main.tf b/terraform/modules/k8s/main.tf new file mode 100644 index 00000000..93e84141 --- /dev/null +++ b/terraform/modules/k8s/main.tf @@ -0,0 +1,35 @@ +resource "azurerm_resource_group" "k8s" { + name = "${var.name}-${var.environment}-vpc" + location = var.region +} + +resource "azurerm_kubernetes_cluster" "k8s" { + name = "${var.name}-${var.environment}-k8s" + location = azurerm_resource_group.k8s.location + resource_group_name = azurerm_resource_group.k8s.name + dns_prefix = var.k8s_dns_prefix + + service_principal { + client_id = "f05a4457-bd5e-4c63-98e1-89aab42645d0" + client_secret = "19b69e2c-9f55-4850-87cb-88c67a8dc811" + } + + default_node_pool { + name = "default" + vm_size = "Standard_D1_v2" + os_disk_size_gb = 30 + vnet_subnet_id = var.vnet_subnet_id + node_count = 1 + } + + lifecycle { + ignore_changes = [ + default_node_pool.0.node_count + ] + } + + tags = { + environment = var.environment + owner = var.owner + } +} \ No newline at end of file diff --git a/terraform/modules/k8s/outputs.tf b/terraform/modules/k8s/outputs.tf new file mode 100644 index 00000000..e69de29b diff --git a/terraform/modules/k8s/variables.tf b/terraform/modules/k8s/variables.tf new file mode 100644 index 00000000..7a3663ce --- /dev/null +++ b/terraform/modules/k8s/variables.tf @@ -0,0 +1,35 @@ +variable "region" { + type = string + description = "Region this module and resources will be created in" +} + +variable "name" { + type = string + description = "Unique name for the services in this module" +} + +variable "environment" { + type = string + description = "Environment these resources reside (prod, dev, staging, etc)" +} + +variable "owner" { + type = string + description = "Owner of the environment and resources created in this module" +} + +variable "k8s_dns_prefix" { + type = string + description = "A DNS prefix" +} + +variable "k8s_node_size" { + type = string + description = "The size of the instance to use in the node pools for k8s" + default = "Standard_A1_v2" +} + +variable "vnet_subnet_id" { + description = "Subnet to use for the default k8s pool" + type = string +} diff --git a/terraform/modules/keyvault/main.tf b/terraform/modules/keyvault/main.tf new file mode 100644 index 00000000..2eb1d6d1 --- /dev/null +++ b/terraform/modules/keyvault/main.tf @@ -0,0 +1,44 @@ +data "azurerm_client_config" "current" {} + +resource "azurerm_resource_group" "keyvault" { + name = "${var.name}-${var.environment}-rg" + location = var.region +} + +resource "random_id" "server" { + keepers = { + ami_id = 1 + } + + byte_length = 8 +} + +resource "azurerm_key_vault" "keyvault" { + name = "${var.name}-${var.environment}-keyvault" + location = azurerm_resource_group.keyvault.location + resource_group_name = azurerm_resource_group.keyvault.name + tenant_id = data.azurerm_client_config.current.tenant_id + + sku_name = "premium" + + access_policy { + tenant_id = data.azurerm_client_config.current.tenant_id + object_id = data.azurerm_client_config.current.service_principal_object_id + + key_permissions = [ + "create", + "get", + ] + + secret_permissions = [ + "set", + "get", + "delete", + ] + } + + tags = { + environment = var.environment + owner = var.owner + } +} \ No newline at end of file diff --git a/terraform/modules/keyvault/variables.tf b/terraform/modules/keyvault/variables.tf new file mode 100644 index 00000000..7ad8ab26 --- /dev/null +++ b/terraform/modules/keyvault/variables.tf @@ -0,0 +1,19 @@ +variable "region" { + type = string + description = "Region this module and resources will be created in" +} + +variable "name" { + type = string + description = "Unique name for the services in this module" +} + +variable "environment" { + type = string + description = "Environment these resources reside (prod, dev, staging, etc)" +} + +variable "owner" { + type = string + description = "Owner of this environment" +} diff --git a/terraform/modules/vpc/main.tf b/terraform/modules/vpc/main.tf new file mode 100644 index 00000000..e614b9e4 --- /dev/null +++ b/terraform/modules/vpc/main.tf @@ -0,0 +1,72 @@ +resource "azurerm_resource_group" "vpc" { + name = "${var.name}-${var.environment}-vpc" + location = var.region + + tags = { + environment = var.environment + owner = var.owner + } +} + +resource "azurerm_network_ddos_protection_plan" "vpc" { + count = var.ddos_enabled + name = "${var.name}-${var.environment}-ddos" + location = azurerm_resource_group.vpc.location + resource_group_name = azurerm_resource_group.vpc.name +} + +resource "azurerm_virtual_network" "vpc" { + name = "${var.name}-${var.environment}-network" + location = azurerm_resource_group.vpc.location + resource_group_name = azurerm_resource_group.vpc.name + address_space = ["${var.virtual_network}"] + dns_servers = var.dns_servers + + tags = { + environment = var.environment + owner = var.owner + } +} + +resource "azurerm_subnet" "subnet" { + for_each = var.networks + name = "${var.name}-${var.environment}-${each.key}" + resource_group_name = azurerm_resource_group.vpc.name + virtual_network_name = azurerm_virtual_network.vpc.name + address_prefix = element(split(",", each.value), 0) + + # See https://github.com/terraform-providers/terraform-provider-azurerm/issues/3471 + lifecycle { + ignore_changes = [route_table_id] + } + #delegation { + # name = "acctestdelegation" + # + # service_delegation { + # name = "Microsoft.ContainerInstance/containerGroups" + # actions = ["Microsoft.Network/virtualNetworks/subnets/action"] + # } + #} +} + +resource "azurerm_route_table" "route_table" { + for_each = var.route_tables + name = "${var.name}-${var.environment}-${each.key}" + location = azurerm_resource_group.vpc.location + resource_group_name = azurerm_resource_group.vpc.name +} + +resource "azurerm_subnet_route_table_association" "route_table" { + for_each = var.networks + subnet_id = azurerm_subnet.subnet[each.key].id + route_table_id = azurerm_route_table.route_table[each.key].id +} + +resource "azurerm_route" "route" { + for_each = var.route_tables + name = "${var.name}-${var.environment}-default" + resource_group_name = azurerm_resource_group.vpc.name + route_table_name = azurerm_route_table.route_table[each.key].name + address_prefix = "0.0.0.0/0" + next_hop_type = each.value +} diff --git a/terraform/modules/vpc/outputs.tf b/terraform/modules/vpc/outputs.tf new file mode 100644 index 00000000..eedaab6c --- /dev/null +++ b/terraform/modules/vpc/outputs.tf @@ -0,0 +1,3 @@ +output "subnets" { + value = azurerm_subnet.subnet["private"].id #FIXME - output should be a map +} diff --git a/terraform/modules/vpc/variables.tf b/terraform/modules/vpc/variables.tf new file mode 100644 index 00000000..ab2aa894 --- /dev/null +++ b/terraform/modules/vpc/variables.tf @@ -0,0 +1,43 @@ +variable "environment" { + description = "Environment (Prod,Dev,etc)" +} + +variable "region" { + description = "Region (useast2, etc)" + +} + +variable "name" { + description = "Name or prefix to use for all resources created by this module" +} + +variable "owner" { + description = "Owner of these resources" + +} + +variable "ddos_enabled" { + description = "Enable or disable DDoS Protection (1,0)" + default = "0" +} + +variable "virtual_network" { + description = "The supernet used for this VPC a.k.a Virtual Network" + type = string +} + +variable "networks" { + description = "A map of lists describing the network topology" + type = map +} + +variable "dns_servers" { + description = "DNS Server IPs for internal and public DNS lookups (must be on a defined subnet)" + type = list + +} + +variable "route_tables" { + type = map + description = "A map with the route tables to create" +} diff --git a/terraform/providers/dev/k8s.tf b/terraform/providers/dev/k8s.tf new file mode 100644 index 00000000..b41df8a4 --- /dev/null +++ b/terraform/providers/dev/k8s.tf @@ -0,0 +1,11 @@ +module "k8s" { + source = "../../modules/k8s" + region = var.region + name = var.name + environment = var.environment + owner = var.owner + k8s_dns_prefix = var.k8s_dns_prefix + k8s_node_size = var.k8s_node_size + vnet_subnet_id = module.vpc.subnets #FIXME - output from module.vpc.subnets should be map +} + diff --git a/terraform/providers/dev/keyvault.tf b/terraform/providers/dev/keyvault.tf new file mode 100644 index 00000000..96545568 --- /dev/null +++ b/terraform/providers/dev/keyvault.tf @@ -0,0 +1,7 @@ +#module "keyvault" { +# source = "../../modules/keyvault" +# name = var.name +# region = var.region +# owner = var.owner +# environment = var.environment +#} diff --git a/terraform/providers/dev/provider.tf b/terraform/providers/dev/provider.tf new file mode 100644 index 00000000..0d225638 --- /dev/null +++ b/terraform/providers/dev/provider.tf @@ -0,0 +1,17 @@ +provider "azurerm" { + version = "=1.38.0" +} + +provider "azuread" { + # Whilst version is optional, we /strongly recommend/ using it to pin the version of the Provider being used + version = "=0.7.0" +} + +terraform { + backend "azurerm" { + resource_group_name = "cloudzero-dev-tfstate" + storage_account_name = "cloudzerodevtfstate" + container_name = "tfstate" + key = "dev.terraform.tfstate" + } +} diff --git a/terraform/providers/dev/variables.tf b/terraform/providers/dev/variables.tf new file mode 100644 index 00000000..3ea68131 --- /dev/null +++ b/terraform/providers/dev/variables.tf @@ -0,0 +1,56 @@ +variable "environment" { + default = "dev" +} + +variable "region" { + default = "eastus2" + +} + +variable "owner" { + default = "dev" +} + +variable "name" { + default = "cloudzero" +} + +variable "virtual_network" { + type = string + default = "10.1.0.0/16" +} + + +variable "networks" { + type = map + default = { + #format + #name = "CIDR, route table, Security Group Name" + public = "10.1.1.0/24,public" # LBs + private = "10.1.2.0/24,private" # k8s, postgres, redis, dns, ad + } +} + +variable "route_tables" { + description = "Route tables and their default routes" + type = map + default = { + public = "Internet" + private = "VnetLocal" + } +} + +variable "dns_servers" { + type = list + default = ["10.1.2.4", "10.1.2.5"] +} + +variable "k8s_node_size" { + type = string + default = "Standard_A1_v2" +} + +variable "k8s_dns_prefix" { + type = string + default = "atat" +} diff --git a/terraform/providers/dev/vpc.tf b/terraform/providers/dev/vpc.tf new file mode 100644 index 00000000..0b930a0d --- /dev/null +++ b/terraform/providers/dev/vpc.tf @@ -0,0 +1,12 @@ +module "vpc" { + source = "../../modules/vpc/" + environment = var.environment + region = var.region + virtual_network = var.virtual_network + networks = var.networks + route_tables = var.route_tables + owner = var.owner + name = var.name + dns_servers = var.dns_servers +} + From f104803b6ddc1fdf745906444b83bcafe10e3739 Mon Sep 17 00:00:00 2001 From: Rob Gil Date: Sun, 15 Dec 2019 12:48:40 -0500 Subject: [PATCH 03/15] 169163334 - Adds postgres module and configures dev to run pg Adds the postgres module and configures it in the development environment. --- terraform/modules/postgres/main.tf | 37 ++++++++++++ terraform/modules/postgres/outputs.tf | 0 terraform/modules/postgres/variables.tf | 75 +++++++++++++++++++++++++ terraform/providers/dev/postgres.tf | 8 +++ 4 files changed, 120 insertions(+) create mode 100644 terraform/modules/postgres/main.tf create mode 100644 terraform/modules/postgres/outputs.tf create mode 100644 terraform/modules/postgres/variables.tf create mode 100644 terraform/providers/dev/postgres.tf diff --git a/terraform/modules/postgres/main.tf b/terraform/modules/postgres/main.tf new file mode 100644 index 00000000..9db23be9 --- /dev/null +++ b/terraform/modules/postgres/main.tf @@ -0,0 +1,37 @@ +resource "azurerm_resource_group" "sql" { + name = "${var.name}-${var.environment}-postgres" + location = var.region +} + +resource "azurerm_postgresql_server" "sql" { + name = "${var.name}-${var.environment}-sql" + location = azurerm_resource_group.sql.location + resource_group_name = azurerm_resource_group.sql.name + + sku { + name = var.sku_name + capacity = var.sku_capacity + tier = var.sku_tier + family = var.sku_family + } + + storage_profile { + storage_mb = var.storage_mb + backup_retention_days = var.storage_backup_retention_days + geo_redundant_backup = var.storage_geo_redundant_backup + auto_grow = var.stroage_auto_grow + } + + administrator_login = "sqladmindude" + administrator_login_password = "eI0l7yswwtuhHpwzoVjwRKdAcuGNsg" + version = "11" + ssl_enforcement = "Enabled" +} + +resource "azurerm_postgresql_virtual_network_rule" "sql" { + name = "postgresql-vnet-rule" + resource_group_name = azurerm_resource_group.sql.name + server_name = azurerm_postgresql_server.sql.name + subnet_id = var.subnet_id + ignore_missing_vnet_service_endpoint = true +} \ No newline at end of file diff --git a/terraform/modules/postgres/outputs.tf b/terraform/modules/postgres/outputs.tf new file mode 100644 index 00000000..e69de29b diff --git a/terraform/modules/postgres/variables.tf b/terraform/modules/postgres/variables.tf new file mode 100644 index 00000000..91af61bc --- /dev/null +++ b/terraform/modules/postgres/variables.tf @@ -0,0 +1,75 @@ +variable "region" { + type = string + description = "Region this module and resources will be created in" +} + +variable "name" { + type = string + description = "Unique name for the services in this module" +} + +variable "environment" { + type = string + description = "Environment these resources reside (prod, dev, staging, etc)" +} + +variable "owner" { + type = string + description = "Owner of the environment and resources created in this module" +} + +variable "subnet_id" { + type = string + description = "Subnet the SQL server should run" +} + +variable "sku_name" { + type = string + description = "SKU name" + default = "GP_Gen5_2" +} + +variable "sku_capacity" { + type = string + description = "SKU Capacity" + default = "2" +} + +variable "sku_tier" { + type = string + description = "SKU Tier" + default = "GeneralPurpose" + +} + +variable "sku_family" { + type = string + description = "SKU Family" + default = "Gen5" +} + +variable "storage_mb" { + type = string + description = "Size in MB of the storage used for the sql server" + default = "5000" +} + + +variable "storage_backup_retention_days" { + type = string + description = "Storage backup retention (days)" + default = "7" +} + +variable "storage_geo_redundant_backup" { + type = string + description = "Geographic redundant backup (Enabled/Disabled)" + default = "Disabled" +} + +variable "storage_auto_grow" { + type = string + description = "Auto Grow? (Enabled/Disabled)" + default = "Enabled" +} + diff --git a/terraform/providers/dev/postgres.tf b/terraform/providers/dev/postgres.tf new file mode 100644 index 00000000..89f06e0d --- /dev/null +++ b/terraform/providers/dev/postgres.tf @@ -0,0 +1,8 @@ +module "sql" { + source = "../../modules/postgres" + name = var.name + owner = var.owner + environment = var.environment + region = var.region + subnet_id = module.vpc.subnets # FIXME - Should be a map of subnets and specify private +} From b11dc849f34994333cd2585c6cc8015b8d85bad2 Mon Sep 17 00:00:00 2001 From: Rob Gil Date: Sun, 15 Dec 2019 13:19:32 -0500 Subject: [PATCH 04/15] 169163334 - Adds more variables to postgres TF module --- terraform/modules/postgres/main.tf | 12 +++++------ terraform/modules/postgres/variables.tf | 27 ++++++++++++++++++++++++- 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/terraform/modules/postgres/main.tf b/terraform/modules/postgres/main.tf index 9db23be9..860ece56 100644 --- a/terraform/modules/postgres/main.tf +++ b/terraform/modules/postgres/main.tf @@ -19,17 +19,17 @@ resource "azurerm_postgresql_server" "sql" { storage_mb = var.storage_mb backup_retention_days = var.storage_backup_retention_days geo_redundant_backup = var.storage_geo_redundant_backup - auto_grow = var.stroage_auto_grow + auto_grow = var.storage_auto_grow } - administrator_login = "sqladmindude" - administrator_login_password = "eI0l7yswwtuhHpwzoVjwRKdAcuGNsg" - version = "11" - ssl_enforcement = "Enabled" + administrator_login = var.administrator_login + administrator_login_password = var.administrator_login_password + version = var.postgres_version + ssl_enforcement = var.ssl_enforcement } resource "azurerm_postgresql_virtual_network_rule" "sql" { - name = "postgresql-vnet-rule" + name = "${var.name}-${var.environment}-rule" resource_group_name = azurerm_resource_group.sql.name server_name = azurerm_postgresql_server.sql.name subnet_id = var.subnet_id diff --git a/terraform/modules/postgres/variables.tf b/terraform/modules/postgres/variables.tf index 91af61bc..3346ff8f 100644 --- a/terraform/modules/postgres/variables.tf +++ b/terraform/modules/postgres/variables.tf @@ -51,7 +51,7 @@ variable "sku_family" { variable "storage_mb" { type = string description = "Size in MB of the storage used for the sql server" - default = "5000" + default = "5120" } @@ -73,3 +73,28 @@ variable "storage_auto_grow" { default = "Enabled" } +variable "administrator_login" { + type = string + description = "Administrator login" + default = "sqladmindude" # FIXME - Remove with wrapper using KeyVault +} + +variable "administrator_login_password" { + type = string + description = "Administrator password" + default = "eI0l7yswwtuhHpwzoVjwRKdAcuGNsg" # FIXME - Remove with wrapper using KeyVault +} + + +variable "postgres_version" { + type = string + description = "Postgres version to use" + default = "11" +} + +variable "ssl_enforcement" { + type = string + description = "Enforce SSL (Enabled/Disable)" + default = "Enabled" +} + From fd6bf723db86b3ac9d5271f71a95bce7cc779833 Mon Sep 17 00:00:00 2001 From: Rob Gil Date: Sun, 15 Dec 2019 14:44:02 -0500 Subject: [PATCH 05/15] 169163334 - Enables KeyVault server in dev TF env This keyvault server will be used for db, redis, ad, k8s, and app secrets for this environment. --- terraform/modules/keyvault/main.tf | 26 +------------------------ terraform/modules/keyvault/variables.tf | 7 ++++++- terraform/providers/dev/keyvault.tf | 15 +++++++------- terraform/providers/dev/variables.tf | 5 +++++ 4 files changed, 20 insertions(+), 33 deletions(-) diff --git a/terraform/modules/keyvault/main.tf b/terraform/modules/keyvault/main.tf index 2eb1d6d1..131c7808 100644 --- a/terraform/modules/keyvault/main.tf +++ b/terraform/modules/keyvault/main.tf @@ -5,14 +5,6 @@ resource "azurerm_resource_group" "keyvault" { location = var.region } -resource "random_id" "server" { - keepers = { - ami_id = 1 - } - - byte_length = 8 -} - resource "azurerm_key_vault" "keyvault" { name = "${var.name}-${var.environment}-keyvault" location = azurerm_resource_group.keyvault.location @@ -21,24 +13,8 @@ resource "azurerm_key_vault" "keyvault" { sku_name = "premium" - access_policy { - tenant_id = data.azurerm_client_config.current.tenant_id - object_id = data.azurerm_client_config.current.service_principal_object_id - - key_permissions = [ - "create", - "get", - ] - - secret_permissions = [ - "set", - "get", - "delete", - ] - } - tags = { environment = var.environment owner = var.owner } -} \ No newline at end of file +} diff --git a/terraform/modules/keyvault/variables.tf b/terraform/modules/keyvault/variables.tf index 7ad8ab26..f6b7b429 100644 --- a/terraform/modules/keyvault/variables.tf +++ b/terraform/modules/keyvault/variables.tf @@ -14,6 +14,11 @@ variable "environment" { } variable "owner" { - type = string + type = string description = "Owner of this environment" } + +variable "tenant_id" { + type = string + description = "The Tenant ID" +} diff --git a/terraform/providers/dev/keyvault.tf b/terraform/providers/dev/keyvault.tf index 96545568..009cd93f 100644 --- a/terraform/providers/dev/keyvault.tf +++ b/terraform/providers/dev/keyvault.tf @@ -1,7 +1,8 @@ -#module "keyvault" { -# source = "../../modules/keyvault" -# name = var.name -# region = var.region -# owner = var.owner -# environment = var.environment -#} +module "keyvault" { + source = "../../modules/keyvault" + name = var.name + region = var.region + owner = var.owner + environment = var.environment + tenant_id = var.tenant_id +} diff --git a/terraform/providers/dev/variables.tf b/terraform/providers/dev/variables.tf index 3ea68131..7a9eea21 100644 --- a/terraform/providers/dev/variables.tf +++ b/terraform/providers/dev/variables.tf @@ -54,3 +54,8 @@ variable "k8s_dns_prefix" { type = string default = "atat" } + +variable "tenant_id" { + type = string + default = "b5ab0e1e-09f8-4258-afb7-fb17654bc5b3" +} From 3b05f9b830b59da2594b5fcdf4f62f91845442b4 Mon Sep 17 00:00:00 2001 From: Rob Gil Date: Sun, 15 Dec 2019 16:24:44 -0500 Subject: [PATCH 06/15] Adds rgil to keyvault access policy --- terraform/modules/keyvault/main.tf | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/terraform/modules/keyvault/main.tf b/terraform/modules/keyvault/main.tf index 131c7808..d4208e36 100644 --- a/terraform/modules/keyvault/main.tf +++ b/terraform/modules/keyvault/main.tf @@ -18,3 +18,23 @@ resource "azurerm_key_vault" "keyvault" { owner = var.owner } } + +resource "azurerm_key_vault_access_policy" "keyvault" { + key_vault_id = azurerm_key_vault.keyvault.id + + tenant_id = "b5ab0e1e-09f8-4258-afb7-fb17654bc5b3" + object_id = "2ca63d41-d058-4e06-aef6-eb517a53b631" + + key_permissions = [ + "get", + "list", + "create", + ] + + secret_permissions = [ + "get", + "list", + "set", + ] +} + From 1968d458dbe00e1efad4686fac5664ee7d3da4ed Mon Sep 17 00:00:00 2001 From: dandds Date: Mon, 16 Dec 2019 10:12:35 -0500 Subject: [PATCH 07/15] Display names for subscriptions are passed to create_subscription. We don't need to generate the display name for a new subscription inside the _create_subscription method. --- atst/domain/csp/cloud.py | 1 - 1 file changed, 1 deletion(-) diff --git a/atst/domain/csp/cloud.py b/atst/domain/csp/cloud.py index 2da48642..ea48ba4e 100644 --- a/atst/domain/csp/cloud.py +++ b/atst/domain/csp/cloud.py @@ -530,7 +530,6 @@ class AzureCloudProvider(CloudProviderInterface): ): sub_client = self.sdk.subscription.SubscriptionClient(credentials) - display_name = f"{environment.application.name}_{environment.name}_{environment.id}" # proposed format billing_profile_id = "?" # where do we source this? sku_id = AZURE_SKU_ID # These 2 seem like something that might be worthwhile to allow tiebacks to From 3457f51d99595028c2d8f13f57c0d2ecbc835ab6 Mon Sep 17 00:00:00 2001 From: dandds Date: Mon, 16 Dec 2019 10:47:22 -0500 Subject: [PATCH 08/15] CI script should fail hard. Right now, unit test failures in script/cibuild are not being emitted correctly. Instead, we'll just `set -e` at the top of the CI script so that failures are fast and obvious. --- script/cibuild | 1 + 1 file changed, 1 insertion(+) diff --git a/script/cibuild b/script/cibuild index 55bc7857..f0011051 100755 --- a/script/cibuild +++ b/script/cibuild @@ -1,4 +1,5 @@ #!/bin/bash +set -e # script/cibuild: Run CI related checks and tests From 22dd5d7b85105afceceeb0a4566ab7dc83ae79db Mon Sep 17 00:00:00 2001 From: leigh-mil Date: Fri, 13 Dec 2019 14:49:39 -0500 Subject: [PATCH 09/15] Add migration for enforcing uniqueness of an application name within a portfolio and update create/update Applicaiton domain methods. --- .secrets.baseline | 4 +-- ..._add_application_name_and_portfolio_id_.py | 28 +++++++++++++++++++ atst/domain/applications.py | 17 +++++++++-- atst/models/application.py | 7 ++++- tests/domain/test_applications.py | 21 +++++++++++++- tests/domain/test_portfolios.py | 5 ++-- tests/test_access.py | 23 ++++++++------- 7 files changed, 87 insertions(+), 18 deletions(-) create mode 100644 alembic/versions/c487d91f1a26_add_application_name_and_portfolio_id_.py diff --git a/.secrets.baseline b/.secrets.baseline index 05921baa..24361804 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": "^.secrets.baseline$|^.*pgsslrootcert.yml$", "lines": null }, - "generated_at": "2019-12-06T21:22:07Z", + "generated_at": "2019-12-13T20:38:57Z", "plugins_used": [ { "base64_limit": 4.5, @@ -170,7 +170,7 @@ "hashed_secret": "e4f14805dfd1e6af030359090c535e149e6b4207", "is_secret": false, "is_verified": false, - "line_number": 656, + "line_number": 659, "type": "Hex High Entropy String" } ] diff --git a/alembic/versions/c487d91f1a26_add_application_name_and_portfolio_id_.py b/alembic/versions/c487d91f1a26_add_application_name_and_portfolio_id_.py new file mode 100644 index 00000000..a0c32c2c --- /dev/null +++ b/alembic/versions/c487d91f1a26_add_application_name_and_portfolio_id_.py @@ -0,0 +1,28 @@ +"""add application name and portfolio_id unique constraint + +Revision ID: c487d91f1a26 +Revises: 3bd8552f1c57 +Create Date: 2019-12-13 14:33:23.952450 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'c487d91f1a26' # pragma: allowlist secret +down_revision = '3bd8552f1c57' # pragma: allowlist secret +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_unique_constraint('name_portfolio_id_unique_constraint', 'applications', ['name', 'portfolio_id']) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint('name_portfolio_id_unique_constraint', 'applications', type_='unique') + # ### end Alembic commands ### diff --git a/atst/domain/applications.py b/atst/domain/applications.py index 21e8782b..39cf7251 100644 --- a/atst/domain/applications.py +++ b/atst/domain/applications.py @@ -1,4 +1,7 @@ +from sqlalchemy.exc import IntegrityError + from . import BaseDomainClass +from .exceptions import AlreadyExistsError from flask import g from atst.database import db from atst.domain.application_roles import ApplicationRoles @@ -28,7 +31,12 @@ class Applications(BaseDomainClass): if environment_names: Environments.create_many(user, application, environment_names) - db.session.commit() + try: + db.session.commit() + except IntegrityError: + db.session.rollback() + raise AlreadyExistsError("application") + return application @classmethod @@ -54,7 +62,12 @@ class Applications(BaseDomainClass): g.current_user, application, new_data["environment_names"] ) db.session.add(application) - db.session.commit() + + try: + db.session.commit() + except IntegrityError: + db.session.rollback() + raise AlreadyExistsError("application") return application diff --git a/atst/models/application.py b/atst/models/application.py index b6cff6bd..415e5694 100644 --- a/atst/models/application.py +++ b/atst/models/application.py @@ -1,4 +1,4 @@ -from sqlalchemy import and_, Column, ForeignKey, String +from sqlalchemy import and_, Column, ForeignKey, String, UniqueConstraint from sqlalchemy.orm import relationship, synonym from atst.models.base import Base @@ -34,6 +34,11 @@ class Application( ), ) members = synonym("roles") + __table_args__ = ( + UniqueConstraint( + "name", "portfolio_id", name="name_portfolio_id_unique_constraint" + ), + ) @property def users(self): diff --git a/tests/domain/test_applications.py b/tests/domain/test_applications.py index 66f4ee01..50ae8038 100644 --- a/tests/domain/test_applications.py +++ b/tests/domain/test_applications.py @@ -5,7 +5,7 @@ from atst.models import CSPRole, ApplicationRoleStatus from atst.domain.application_roles import ApplicationRoles from atst.domain.applications import Applications from atst.domain.environment_roles import EnvironmentRoles -from atst.domain.exceptions import NotFoundError +from atst.domain.exceptions import AlreadyExistsError, NotFoundError from atst.domain.permission_sets import PermissionSets from tests.factories import ( @@ -177,3 +177,22 @@ def test_invite_to_nonexistent_environment(): {"environment_id": uuid4(), "role": CSPRole.BASIC_ACCESS.value}, ], ) + + +def test_create_does_not_duplicate_names_within_portfolio(): + portfolio = PortfolioFactory.create() + name = "An Awesome Application" + + assert Applications.create(portfolio.owner, portfolio, name, "") + with pytest.raises(AlreadyExistsError): + Applications.create(portfolio.owner, portfolio, name, "") + + +def test_update_does_not_duplicate_names_within_portfolio(): + portfolio = PortfolioFactory.create() + name = "An Awesome Application" + application = ApplicationFactory.create(portfolio=portfolio, name=name) + dupe_application = ApplicationFactory.create(portfolio=portfolio) + + with pytest.raises(AlreadyExistsError): + Applications.update(dupe_application, {"name": name}) diff --git a/tests/domain/test_portfolios.py b/tests/domain/test_portfolios.py index 41e3fc81..0390f69f 100644 --- a/tests/domain/test_portfolios.py +++ b/tests/domain/test_portfolios.py @@ -1,4 +1,5 @@ import pytest +import random from uuid import uuid4 from atst.domain.exceptions import NotFoundError, UnauthorizedError @@ -97,7 +98,7 @@ def test_scoped_portfolio_returns_all_applications_for_portfolio_admin( Applications.create( portfolio.owner, portfolio, - "My Application", + "My Application %s" % (random.randrange(1, 1000)), "My application", ["dev", "staging", "prod"], ) @@ -120,7 +121,7 @@ def test_scoped_portfolio_returns_all_applications_for_portfolio_owner( Applications.create( portfolio.owner, portfolio, - "My Application", + "My Application %s" % (random.randrange(1, 1000)), "My application", ["dev", "staging", "prod"], ) diff --git a/tests/test_access.py b/tests/test_access.py index 19aaf749..ad4bd5be 100644 --- a/tests/test_access.py +++ b/tests/test_access.py @@ -1,6 +1,7 @@ from unittest.mock import Mock import pytest +import random from flask import url_for, Response @@ -264,26 +265,28 @@ def test_applications_post_application_step_1(post_url_assert_status): rando = user_with() portfolio = PortfolioFactory.create(owner=owner) application = ApplicationFactory.create(portfolio=portfolio) - step_1_form_data = { - "name": "Test Application", - "description": "This is only a test", - } + + def _form_data(): + return { + "name": "Test Application %s" % (random.randrange(1, 1000)), + "description": "This is only a test", + } url = url_for( "applications.create_new_application_step_1", portfolio_id=portfolio.id ) - post_url_assert_status(ccpo, url, 302, data=step_1_form_data) - post_url_assert_status(owner, url, 302, data=step_1_form_data) - post_url_assert_status(rando, url, 404, data=step_1_form_data) + post_url_assert_status(ccpo, url, 302, data=_form_data()) + post_url_assert_status(owner, url, 302, data=_form_data()) + post_url_assert_status(rando, url, 404, data=_form_data()) url = url_for( "applications.update_new_application_step_1", portfolio_id=portfolio.id, application_id=application.id, ) - post_url_assert_status(ccpo, url, 302, data=step_1_form_data) - post_url_assert_status(owner, url, 302, data=step_1_form_data) - post_url_assert_status(rando, url, 404, data=step_1_form_data) + post_url_assert_status(ccpo, url, 302, data=_form_data()) + post_url_assert_status(owner, url, 302, data=_form_data()) + post_url_assert_status(rando, url, 404, data=_form_data()) # applications.view_new_application_step_2 From ffbf6122903db0ee6e3a47db136b8b1be57c129e Mon Sep 17 00:00:00 2001 From: leigh-mil Date: Fri, 13 Dec 2019 16:08:22 -0500 Subject: [PATCH 10/15] Update route to catch error when app name uniqueness is violated and display a error message --- atst/routes/applications/new.py | 38 ++++++++++++++++++++------- atst/utils/flash.py | 5 ++++ tests/routes/applications/test_new.py | 18 +++++++++++++ translations.yaml | 2 ++ 4 files changed, 53 insertions(+), 10 deletions(-) diff --git a/atst/routes/applications/new.py b/atst/routes/applications/new.py index a1a1a54c..e9238775 100644 --- a/atst/routes/applications/new.py +++ b/atst/routes/applications/new.py @@ -2,6 +2,7 @@ from flask import redirect, render_template, request as http_request, url_for, g from .blueprint import applications_bp from atst.domain.applications import Applications +from atst.domain.exceptions import AlreadyExistsError from atst.domain.portfolios import Portfolios from atst.forms.application import NameAndDescriptionForm, EnvironmentsForm from atst.domain.authz.decorator import user_can_access_decorator as user_can @@ -37,6 +38,31 @@ def render_new_application_form( return render_template(template, **render_args) +def update_application(form, application_id=None, portfolio_id=None): + if form.validate(): + application = None + try: + if application_id: + application = Applications.get(application_id) + application = Applications.update(application, form.data) + flash("application_updated", application_name=application.name) + else: + portfolio = Portfolios.get_for_update(portfolio_id) + application = Applications.create( + g.current_user, portfolio, **form.data + ) + flash("application_created", application_name=application.name) + + return application + + except AlreadyExistsError: + flash("application_name_error", name=form.data["name"]) + return False + + else: + return False + + @applications_bp.route("/portfolios//applications/new") @applications_bp.route("/applications//new/step_1") @user_can(Permissions.CREATE_APPLICATION, message="view create new application form") @@ -64,17 +90,9 @@ def create_or_update_new_application_step_1(portfolio_id=None, application_id=No form = get_new_application_form( {**http_request.form}, NameAndDescriptionForm, application_id ) + application = update_application(form, application_id, portfolio_id) - if form.validate(): - application = None - if application_id: - application = Applications.get(application_id) - application = Applications.update(application, form.data) - flash("application_updated", application_name=application.name) - else: - portfolio = Portfolios.get_for_update(portfolio_id) - application = Applications.create(g.current_user, portfolio, **form.data) - flash("application_created", application_name=application.name) + if application: return redirect( url_for( "applications.update_new_application_step_2", diff --git a/atst/utils/flash.py b/atst/utils/flash.py index fabba4d2..0de143bb 100644 --- a/atst/utils/flash.py +++ b/atst/utils/flash.py @@ -64,6 +64,11 @@ MESSAGES = { "message_template": "You have successfully updated the permissions for {{ user_name }}", "category": "success", }, + "application_name_error": { + "title_template": "", + "message_template": """{{ 'flash.application.name_error.message' | translate({ 'name': name }) }}""", + "category": "error", + }, "ccpo_user_added": { "title_template": translate("flash.success"), "message_template": "You have successfully given {{ user_name }} CCPO permissions.", diff --git a/tests/routes/applications/test_new.py b/tests/routes/applications/test_new.py index c4fb83ed..045ac19e 100644 --- a/tests/routes/applications/test_new.py +++ b/tests/routes/applications/test_new.py @@ -70,6 +70,24 @@ def test_post_name_and_description_for_update(client, session, user_session): assert application.description == "This is only a test" +def test_post_name_and_description_enforces_unique_name(client, user_session, session): + portfolio = PortfolioFactory.create() + name = "Test Application" + application = ApplicationFactory.create(portfolio=portfolio, name=name) + user_session(portfolio.owner) + + session.begin_nested() + response = client.post( + url_for( + "applications.create_new_application_step_1", portfolio_id=portfolio.id + ), + data={"name": name, "description": "This is only a test"}, + ) + session.rollback() + + assert response.status_code == 400 + + def test_get_environments(client, user_session): application = ApplicationFactory.create() user_session(application.portfolio.owner) diff --git a/translations.yaml b/translations.yaml index 1d876e97..6a678f48 100644 --- a/translations.yaml +++ b/translations.yaml @@ -114,6 +114,8 @@ flash: message: '{application_name} has been successfully created. You may continue on to provision environments and assign team members now, or come back and complete these tasks at a later time.' updated: 'You have successfully updated the {application_name} application.' deleted: 'You have successfully deleted the {application_name} application. To view the retained activity log, visit the portfolio administration page.' + name_error: + message: 'The application name {name} has already been used in this portfolio. Please enter a unique name.' delete_member_success: 'You have successfully deleted {member_name} from the portfolio.' deleted_member: Portfolio member deleted environment_added: 'The environment "{env_name}" has been added to the application.' From afad5362a1e74bc1985c6d6912ecfe0961e2c962 Mon Sep 17 00:00:00 2001 From: leigh-mil Date: Mon, 16 Dec 2019 10:21:45 -0500 Subject: [PATCH 11/15] Update name of UniqueContraint to include the table name --- .../c487d91f1a26_add_application_name_and_portfolio_id_.py | 4 ++-- atst/models/application.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/alembic/versions/c487d91f1a26_add_application_name_and_portfolio_id_.py b/alembic/versions/c487d91f1a26_add_application_name_and_portfolio_id_.py index a0c32c2c..f3cdc433 100644 --- a/alembic/versions/c487d91f1a26_add_application_name_and_portfolio_id_.py +++ b/alembic/versions/c487d91f1a26_add_application_name_and_portfolio_id_.py @@ -18,11 +18,11 @@ depends_on = None def upgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.create_unique_constraint('name_portfolio_id_unique_constraint', 'applications', ['name', 'portfolio_id']) + op.create_unique_constraint('applications_name_portfolio_id_key', 'applications', ['name', 'portfolio_id']) # ### end Alembic commands ### def downgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.drop_constraint('name_portfolio_id_unique_constraint', 'applications', type_='unique') + op.drop_constraint('applications_name_portfolio_id_key', 'applications', type_='unique') # ### end Alembic commands ### diff --git a/atst/models/application.py b/atst/models/application.py index 415e5694..a7bdadba 100644 --- a/atst/models/application.py +++ b/atst/models/application.py @@ -36,7 +36,7 @@ class Application( members = synonym("roles") __table_args__ = ( UniqueConstraint( - "name", "portfolio_id", name="name_portfolio_id_unique_constraint" + "name", "portfolio_id", name="applications_name_portfolio_id_key" ), ) From b927ef1b0ec06d9f9a58a1a6cd436b3bf504c9be Mon Sep 17 00:00:00 2001 From: leigh-mil Date: Mon, 16 Dec 2019 10:39:47 -0500 Subject: [PATCH 12/15] Create utility function for the pattern of committing to the database or raising AlreadyExistsError --- atst/domain/applications.py | 21 ++++----------------- atst/domain/task_orders.py | 19 +++---------------- atst/utils/__init__.py | 13 +++++++++++++ 3 files changed, 20 insertions(+), 33 deletions(-) diff --git a/atst/domain/applications.py b/atst/domain/applications.py index 39cf7251..c2321194 100644 --- a/atst/domain/applications.py +++ b/atst/domain/applications.py @@ -1,7 +1,4 @@ -from sqlalchemy.exc import IntegrityError - from . import BaseDomainClass -from .exceptions import AlreadyExistsError from flask import g from atst.database import db from atst.domain.application_roles import ApplicationRoles @@ -14,7 +11,7 @@ from atst.models import ( ApplicationRoleStatus, EnvironmentRole, ) -from atst.utils import first_or_none +from atst.utils import first_or_none, update_or_raise_already_exists_error class Applications(BaseDomainClass): @@ -31,12 +28,7 @@ class Applications(BaseDomainClass): if environment_names: Environments.create_many(user, application, environment_names) - try: - db.session.commit() - except IntegrityError: - db.session.rollback() - raise AlreadyExistsError("application") - + update_or_raise_already_exists_error(message="application") return application @classmethod @@ -61,14 +53,9 @@ class Applications(BaseDomainClass): Environments.create_many( g.current_user, application, new_data["environment_names"] ) + db.session.add(application) - - try: - db.session.commit() - except IntegrityError: - db.session.rollback() - raise AlreadyExistsError("application") - + update_or_raise_already_exists_error(message="application") return application @classmethod diff --git a/atst/domain/task_orders.py b/atst/domain/task_orders.py index 835590c9..8ad8b0f4 100644 --- a/atst/domain/task_orders.py +++ b/atst/domain/task_orders.py @@ -1,11 +1,10 @@ import datetime -from sqlalchemy.exc import IntegrityError from atst.database import db from atst.models.clin import CLIN from atst.models.task_order import TaskOrder, SORT_ORDERING from . import BaseDomainClass -from .exceptions import AlreadyExistsError +from atst.utils import update_or_raise_already_exists_error class TaskOrders(BaseDomainClass): @@ -16,15 +15,8 @@ class TaskOrders(BaseDomainClass): def create(cls, portfolio_id, number, clins, pdf): task_order = TaskOrder(portfolio_id=portfolio_id, number=number, pdf=pdf) db.session.add(task_order) - - try: - db.session.commit() - except IntegrityError: - db.session.rollback() - raise AlreadyExistsError("task_order") - + update_or_raise_already_exists_error(message="task_order") TaskOrders.create_clins(task_order.id, clins) - return task_order @classmethod @@ -42,12 +34,7 @@ class TaskOrders(BaseDomainClass): task_order.number = number db.session.add(task_order) - try: - db.session.commit() - except IntegrityError: - db.session.rollback() - raise AlreadyExistsError("task_order") - + update_or_raise_already_exists_error(message="task_order") return task_order @classmethod diff --git a/atst/utils/__init__.py b/atst/utils/__init__.py index 01988e10..9772af67 100644 --- a/atst/utils/__init__.py +++ b/atst/utils/__init__.py @@ -1,5 +1,10 @@ import re +from sqlalchemy.exc import IntegrityError + +from atst.database import db +from atst.domain.exceptions import AlreadyExistsError + def first_or_none(predicate, lst): return next((x for x in lst if predicate(x)), None) @@ -23,3 +28,11 @@ def camel_to_snake(camel_cased): def pick(keys, dct): _keys = set(keys) return {k: v for (k, v) in dct.items() if k in _keys} + + +def update_or_raise_already_exists_error(message): + try: + db.session.commit() + except IntegrityError: + db.session.rollback() + raise AlreadyExistsError(message) From 5f267e9cfaf7d9858da82518d71254b9e4972516 Mon Sep 17 00:00:00 2001 From: leigh-mil Date: Mon, 16 Dec 2019 14:40:17 -0500 Subject: [PATCH 13/15] Remove unused import --- .../c487d91f1a26_add_application_name_and_portfolio_id_.py | 1 - 1 file changed, 1 deletion(-) diff --git a/alembic/versions/c487d91f1a26_add_application_name_and_portfolio_id_.py b/alembic/versions/c487d91f1a26_add_application_name_and_portfolio_id_.py index f3cdc433..b70b4fde 100644 --- a/alembic/versions/c487d91f1a26_add_application_name_and_portfolio_id_.py +++ b/alembic/versions/c487d91f1a26_add_application_name_and_portfolio_id_.py @@ -6,7 +6,6 @@ Create Date: 2019-12-13 14:33:23.952450 """ from alembic import op -import sqlalchemy as sa # revision identifiers, used by Alembic. From dbe24fdf91db622fdfb8edbc9e7a09555cc82e7e Mon Sep 17 00:00:00 2001 From: Rob Gil Date: Mon, 16 Dec 2019 10:46:47 -0500 Subject: [PATCH 14/15] 169163334 - Adds initial terraform documentation --- terraform/README.md | 84 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 terraform/README.md diff --git a/terraform/README.md b/terraform/README.md new file mode 100644 index 00000000..1f40fadc --- /dev/null +++ b/terraform/README.md @@ -0,0 +1,84 @@ +# ATAT Terraform +Welcome! You've found the ATAT IaC configurations. + +ATAT is configured using terraform and a wrapper script called `secrets-tool`. With `terraform` we can configure infrastructure in a programatic way and ensure consistency across environments. + +## Directory Structure + +**modules/** - Terraform modules. These are modules that can be re-used for multiple environments. + +**providers/** - Specific environment configurations. (dev,production, etc) + +# Setup +Install the following requirements. + +I highly recommend [tfenv](https://github.com/tfutils/tfenv) which will help you manage versions of TF and install new ones as needed. It gives you the ability to switch back and forth between versions as necessary, especially when doing upgrades and managing multiple environments. Think of it like `pyenv`. + +Python is required for the `secrets-tool`. It is used to wrap terraform and pass secrets in to terraform from Azure KeyVault. This approach avoids leaving secrets on the filesystem in any way and allow for restricting access to secrets to specific operators. + +Azure CLI is necessary for creating some intial resources, but is also used by the Python Azure SDK to make calls in some cases. + +Requirements: +- [tfenv](https://github.com/tfutils/tfenv) +- Python 3.7 +- Python pip +- Python virtualenv # FIXME: Switch to `pipenv` +- [azure cli](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli?view=azure-cli-latest) + +# tfenv +`tfenv` will allow you to install TF versions. For example. + +``` +tfenv install 0.12.18 +``` +_0.12.18 at time of writing_ + + +To select a version to use +``` +tfenv use 0.12.18 +``` + +# Running Terraform +First, you'll need to log in to Azure. With the Azure CLI installed, you can run the following. + +``` +az login +``` + +Next, you'll need to initialize the environment. This process pulls down the terraform provider module from github as well as pulls in the modules that will be used by this provider/environment setup. + +``` +cd providers/dev/ +terraform init +``` + +Once initialized, you can run a plan. A `plan` compares the terraform definitions you have configured in the provider directory (Ex. `providers/dev`) with what is in the shared state file in the Azure Object Storage (which all providers are currently configured for). This then also compares it to the state of the services which are running in Azure. + +If nothing has been applied, you'll see all the resources defined in terraform as all new with a `+` next to the resource name. If the resource exists, but has changed, you'll see a `~` next to the resource and the delta of the change to be applied. + +If you're plan looks good, you can run the apply. +``` +terraform apply +``` + +Check the output for errors. Sometimes the syntax is valid, but some of the configuration may be wrong and only rejected by the Azure API at run time. If this is the case, fix your mistake, and re-run. + +# Shutting down and environment +To shutdown and remove an environment completely as to not incur any costs you would need to run a `terraform destroy`. + +``` +terraform destroy +``` + +**This will destroy all resources defined in the provider so use with caution!! This will include things like KeyVault, Postgres, and so on. You may lose data!!** + +# Advanced Terraform +## Targeted Apply +Sometimes you're writing a new module and don't want to make changes to anything else. In this case you can limit what TF changes. + +``` +terraform plan -target=module.vpc +``` + +In the above example, this will only run a plan (plan/apply/destroy) on the specific module. This can be a module, or resource. You can get a list of module and resources by running `terraform show`. From 9d282ee82aa65762d393c2c45c9fc6b2744099f1 Mon Sep 17 00:00:00 2001 From: dandds Date: Thu, 12 Dec 2019 11:27:41 -0500 Subject: [PATCH 15/15] K8s cronjob for resetting the database on staging. This K8s CronJob will run the script for resetting the database. It will only be applied to the staging site. --- Dockerfile | 3 +- deploy/overlays/staging/kustomization.yaml | 1 + deploy/overlays/staging/reset-cron-job.yml | 46 ++++++++++++++++++++++ 3 files changed, 48 insertions(+), 2 deletions(-) create mode 100644 deploy/overlays/staging/reset-cron-job.yml diff --git a/Dockerfile b/Dockerfile index 744c9739..1785b5d8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -84,8 +84,7 @@ COPY --from=builder /install/celery_worker.py ./celery_worker.py COPY --from=builder /install/config/ ./config/ COPY --from=builder /install/templates/ ./templates/ COPY --from=builder /install/translations.yaml . -COPY --from=builder /install/script/seed_roles.py ./script/seed_roles.py -COPY --from=builder /install/script/sync-crls ./script/sync-crls +COPY --from=builder /install/script/ ./script/ COPY --from=builder /install/static/ ./static/ COPY --from=builder /install/fixtures/ ./fixtures COPY --from=builder /install/uwsgi.ini . diff --git a/deploy/overlays/staging/kustomization.yaml b/deploy/overlays/staging/kustomization.yaml index 38251002..ee6f3a0c 100644 --- a/deploy/overlays/staging/kustomization.yaml +++ b/deploy/overlays/staging/kustomization.yaml @@ -3,6 +3,7 @@ bases: - ../../azure/ resources: - namespace.yml + - reset-cron-job.yml patchesStrategicMerge: - replica_count.yml - ports.yml diff --git a/deploy/overlays/staging/reset-cron-job.yml b/deploy/overlays/staging/reset-cron-job.yml new file mode 100644 index 00000000..b4792e5d --- /dev/null +++ b/deploy/overlays/staging/reset-cron-job.yml @@ -0,0 +1,46 @@ +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: reset-db + namespace: atat +spec: + schedule: "0 4 * * *" + concurrencyPolicy: Replace + successfulJobsHistoryLimit: 1 + jobTemplate: + spec: + template: + metadata: + labels: + app: atst + role: reset-db + aadpodidbinding: atat-kv-id-binding + spec: + restartPolicy: OnFailure + containers: + - name: reset + image: $CONTAINER_IMAGE + command: [ + "/bin/sh", "-c" + ] + args: [ + "/opt/atat/atst/.venv/bin/python", + "/opt/atat/atst/script/reset_database.py" + ] + envFrom: + - configMapRef: + name: atst-worker-envvars + volumeMounts: + - name: flask-secret + mountPath: "/config" + volumes: + - name: flask-secret + flexVolume: + driver: "azure/kv" + options: + usepodidentity: "true" + keyvaultname: "atat-vault-test" + keyvaultobjectnames: "staging-AZURE-STORAGE-KEY;staging-MAIL-PASSWORD;staging-PGPASSWORD;staging-REDIS-PASSWORD;staging-SECRET-KEY" + keyvaultobjectaliases: "AZURE_STORAGE_KEY;MAIL_PASSWORD;PGPASSWORD;REDIS_PASSWORD;SECRET_KEY" + keyvaultobjecttypes: "secret;secret;secret;secret;key" + tenantid: $TENANT_ID