Merge pull request #1350 from robgil-dds/170808212-storage-private-endpoint

Service Endpoints

terraform/README.md

@@ -281,3 +281,4 @@ secrets-tool secrets --keyvault https://ops-jedidev-keyvault.vault.azure.net/ cr
 
 `terraform apply`
 
+*[Configure AD for MFA](https://docs.microsoft.com/en-us/azure/vpn-gateway/openvpn-azure-ad-mfa)*

terraform/modules/bucket/main.tf

@@ -11,6 +11,20 @@ resource "azurerm_storage_account" "bucket" {
   account_replication_type = "LRS"
 }
 
+resource "azurerm_storage_account_network_rules" "acls" {
+  resource_group_name  = azurerm_resource_group.bucket.name
+  storage_account_name = azurerm_storage_account.bucket.name
+
+  default_action = var.policy
+
+  # Azure Storage CIDR ACLs do not accept /32 CIDR ranges.
+  ip_rules = [
+    for cidr in values(var.whitelist) : cidr
+  ]
+  virtual_network_subnet_ids = var.subnet_ids
+  bypass                     = ["AzureServices"]
+}
+
 resource "azurerm_storage_container" "bucket" {
   name                  = "content"
   storage_account_name  = azurerm_storage_account.bucket.name

terraform/modules/bucket/variables.tf

@@ -29,3 +29,20 @@ variable "service_name" {
   description = "Name of the service using this bucket"
   type        = string
 }
+
+variable "subnet_ids" {
+  description = "List of subnet_ids that will have access to this service"
+  type        = list
+}
+
+variable "policy" {
+  description = "The default policy for the network access rules (Allow/Deny)"
+  default     = "Deny"
+  type        = string
+}
+
+variable "whitelist" {
+  type        = map
+  description = "A map of whitelisted IPs and CIDR ranges. For single IPs, Azure expects just the IP, NOT a /32."
+  default     = {}
+}

terraform/modules/container_registry/main.tf

@@ -1,3 +1,7 @@
+locals {
+  whitelist = values(var.whitelist)
+}
+
 resource "azurerm_resource_group" "acr" {
   name     = "${var.name}-${var.environment}-acr"
   location = var.region
@@ -10,4 +14,30 @@ resource "azurerm_container_registry" "acr" {
   sku                 = var.sku
   admin_enabled       = var.admin_enabled
   #georeplication_locations = [azurerm_resource_group.acr.location, var.backup_region]
+
+  network_rule_set {
+    default_action = var.policy
+
+    ip_rule = [
+      for cidr in values(var.whitelist) : {
+        action   = "Allow"
+        ip_range = cidr
+      }
+    ]
+    # Dynamic rule should work, but doesn't - See https://github.com/hashicorp/terraform/issues/22340#issuecomment-518779733
+    #dynamic "ip_rule" {
+    #  for_each = values(var.whitelist)
+    #  content {
+    #    action   = "Allow"
+    #    ip_range = ip_rule.value
+    #  }
+    #}
+
+    virtual_network = [
+      for subnet in var.subnet_ids : {
+        action    = "Allow"
+        subnet_id = subnet.value
+      }
+    ]
+  }
 }

terraform/modules/container_registry/variables.tf

@@ -35,3 +35,20 @@ variable "admin_enabled" {
   default     = false
 
 }
+
+variable "subnet_ids" {
+  description = "List of subnet_ids that will have access to this service"
+  type        = list
+}
+
+variable "policy" {
+  description = "The default policy for the network access rules (Allow/Deny)"
+  default     = "Deny"
+  type        = string
+}
+
+variable "whitelist" {
+  type        = map
+  description = "A map of whitelisted IPs and CIDR ranges. For single IPs, Azure expects just the IP, NOT a /32."
+  default     = {}
+}

terraform/modules/keyvault/main.tf

@@ -13,6 +13,13 @@ resource "azurerm_key_vault" "keyvault" {
 
   sku_name = "premium"
 
+  network_acls {
+    default_action             = var.policy
+    bypass                     = "AzureServices"
+    virtual_network_subnet_ids = var.subnet_ids
+    ip_rules                   = values(var.whitelist)
+  }
+
   tags = {
     environment = var.environment
     owner       = var.owner

terraform/modules/keyvault/variables.tf

@@ -32,3 +32,20 @@ variable "admin_principals" {
   type        = map
   description = "A list of user principals who need access to manage the keyvault"
 }
+
+variable "subnet_ids" {
+  description = "List of subnet_ids that will have access to this service"
+  type        = list
+}
+
+variable "policy" {
+  description = "The default policy for the network access rules (Allow/Deny)"
+  default     = "Deny"
+  type        = string
+}
+
+variable "whitelist" {
+  type        = map
+  description = "A map of whitelisted IPs and CIDR ranges. For single IPs, Azure expects just the IP, NOT a /32."
+  default     = {}
+}

terraform/modules/postgres/main.tf

@@ -37,9 +37,9 @@ resource "azurerm_postgresql_virtual_network_rule" "sql" {
 }
 
 resource "azurerm_postgresql_database" "db" {
-  name                = "${var.environment}-atat"
+  name                = "${var.name}-${var.environment}-atat"
   resource_group_name = azurerm_resource_group.sql.name
   server_name         = azurerm_postgresql_server.sql.name
   charset             = "UTF8"
-  collation           = "en_US.utf8"
+  collation           = "en-US"
 }

terraform/modules/postgres/variables.tf

@@ -93,4 +93,3 @@ variable "ssl_enforcement" {
   description = "Enforce SSL (Enabled/Disable)"
   default     = "Enabled"
 }
-

terraform/modules/redis/main.tf

@@ -13,6 +13,7 @@ resource "azurerm_redis_cache" "redis" {
   sku_name            = var.sku_name
   enable_non_ssl_port = var.enable_non_ssl_port
   minimum_tls_version = var.minimum_tls_version
+  subnet_id           = var.subnet_id
 
   redis_configuration {
     enable_authentication = var.enable_authentication

terraform/modules/redis/variables.tf

@@ -22,35 +22,30 @@ variable "capacity" {
   type        = string
   default     = 2
   description = "The capacity of the redis cache"
-
 }
 
 variable "family" {
   type        = string
   default     = "C"
   description = "The subscription family for redis"
-
 }
 
 variable "sku_name" {
   type        = string
   default     = "Standard"
   description = "The sku to use"
-
 }
 
 variable "enable_non_ssl_port" {
   type        = bool
   default     = false
   description = "Enable non TLS port (default: false)"
-
 }
 
 variable "minimum_tls_version" {
   type        = string
   default     = "1.2"
   description = "Minimum TLS version to use"
-
 }
 
 variable "enable_authentication" {
@@ -58,3 +53,8 @@ variable "enable_authentication" {
   default     = true
   description = "Enable or disable authentication (default: true)"
 }
+
+variable "subnet_id" {
+  type        = string
+  description = "Subnet ID that the service_endpoint should reside"
+}

terraform/modules/vpc/main.tf

@@ -39,6 +39,8 @@ resource "azurerm_subnet" "subnet" {
   lifecycle {
     ignore_changes = [route_table_id]
   }
+
+  service_endpoints = split(",", var.service_endpoints[each.key])
   #delegation {
   #  name = "acctestdelegation"
   #
@@ -108,7 +110,7 @@ resource "azurerm_virtual_network_gateway" "vnet_gateway" {
   }
 
   vpn_client_configuration {
-    address_space        = ["172.16.1.0/24"]
+    address_space        = var.vpn_client_cidr
     vpn_client_protocols = ["OpenVPN"]
   }
 }

terraform/modules/vpc/outputs.tf

@@ -1,3 +1,9 @@
 output "subnets" {
-  value = azurerm_subnet.subnet["private"].id #FIXME - output should be a map
+  value = azurerm_subnet.subnet["private"].id #FIXED: this is now legacy, use subnet_list
 }
+
+output "subnet_list" {
+  value = {
+    for k, id in azurerm_subnet.subnet : k => id
+  }
+}

terraform/modules/vpc/variables.tf

@@ -46,3 +46,15 @@ variable "gateway_subnet" {
   type        = string
   description = "The Subnet CIDR that we'll use for the virtual_network_gateway 'GatewaySubnet'"
 }
+
+variable "service_endpoints" {
+  type        = map
+  description = "A map of the service endpoints and its mapping to subnets"
+
+}
+
+variable "vpn_client_cidr" {
+  type        = list
+  description = "The CIDR range used for clients on the VPN"
+  default     = ["172.16.0.0/16"]
+}

terraform/providers/dev/buckets.tf

@@ -1,3 +1,5 @@
+# Task order bucket is required to be accessible publicly by the users.
+# which is why the policy here is "Allow"
 module "task_order_bucket" {
   source       = "../../modules/bucket"
   service_name = "jeditasksatat"
@@ -5,8 +7,15 @@ module "task_order_bucket" {
   name         = var.name
   environment  = var.environment
   region       = var.region
+  policy       = "Allow"
+  subnet_ids   = [module.vpc.subnets]
+  whitelist    = var.storage_admin_whitelist
 }
 
+# TF State should be restricted to admins only, but IP protected
+# This has to be public due to a chicken/egg issue of VPN not 
+# existing until TF is run. If this bucket is private, you would
+# not be able to access it when running TF without being on a VPN.
 module "tf_state" {
   source       = "../../modules/bucket"
   service_name = "jedidevtfstate"
@@ -14,4 +23,7 @@ module "tf_state" {
   name         = var.name
   environment  = var.environment
   region       = var.region
+  policy       = "Deny"
+  subnet_ids   = []
+  whitelist    = var.storage_admin_whitelist
 }

terraform/providers/dev/container_registry.tf

@@ -5,4 +5,7 @@ module "container_registry" {
   environment   = var.environment
   owner         = var.owner
   backup_region = var.backup_region
+  policy        = "Deny"
+  subnet_ids    = []
+  whitelist     = var.admin_user_whitelist
 }

terraform/providers/dev/keyvault.tf

@@ -7,5 +7,8 @@ module "keyvault" {
   tenant_id        = var.tenant_id
   principal_id     = "f9bcbe58-8b73-4957-aee2-133dc3e58063"
   admin_principals = var.admin_users
+  policy           = "Deny"
+  subnet_ids       = [module.vpc.subnets]
+  whitelist        = var.admin_user_whitelist
 }
 

terraform/providers/dev/redis.tf

@@ -4,4 +4,7 @@ module "redis" {
   environment = var.environment
   region      = var.region
   name        = var.name
+  subnet_id   = module.vpc.subnet_list["redis"].id
+  sku_name    = "Premium"
+  family      = "P"
 }

terraform/providers/dev/secrets.tf

@@ -7,4 +7,7 @@ module "operator_keyvault" {
   tenant_id        = var.tenant_id
   principal_id     = ""
   admin_principals = var.admin_users
+  policy           = "Deny"
+  subnet_ids       = [module.vpc.subnets]
+  whitelist        = var.admin_user_whitelist
 }

terraform/providers/dev/variables.tf

@@ -32,7 +32,17 @@ variable "networks" {
     #format
     #name         = "CIDR, route table, Security Group Name"
     public  = "10.1.1.0/24,public"  # LBs
-    private = "10.1.2.0/24,private" # k8s, postgres, redis, dns, ad
+    private = "10.1.2.0/24,private" # k8s, postgres, keyvault
+    redis   = "10.1.3.0/24,private" # Redis
+  }
+}
+
+variable "service_endpoints" {
+  type = map
+  default = {
+    public  = "Microsoft.ContainerRegistry" # Not necessary but added to avoid infinite state loop
+    private = "Microsoft.Storage,Microsoft.KeyVault,Microsoft.ContainerRegistry,Microsoft.Sql"
+    redis   = "Microsoft.Storage,Microsoft.Sql" # FIXME: There is no Microsoft.Redis
   }
 }
 
@@ -48,6 +58,7 @@ variable "route_tables" {
   default = {
     public  = "Internet"
     private = "Internet"
+    redis   = "VnetLocal"
     #private = "VnetLocal"
   }
 }
@@ -79,3 +90,26 @@ variable "admin_users" {
     "Dan Corrigan" = "7e852ceb-eb0d-49b1-b71e-e9dcd1082ffc"
   }
 }
+
+variable "admin_user_whitelist" {
+  type = map
+  default = {
+    "Rob Gil"           = "66.220.238.246/32"
+    "Dan Corrigan Work" = "108.16.207.173/32"
+    "Dan Corrigan Home" = "71.162.221.27/32"
+  }
+}
+
+variable "storage_admin_whitelist" {
+  type = map
+  default = {
+    "Rob Gil"           = "66.220.238.246"
+    "Dan Corrigan Work" = "108.16.207.173"
+    "Dan Corrigan Home" = "71.162.221.27"
+  }
+}
+
+variable "vpn_client_cidr" {
+  type    = list
+  default = ["172.16.255.0/24"]
+}

terraform/providers/dev/vpc.tf

@@ -1,13 +1,15 @@
 module "vpc" {
-  source          = "../../modules/vpc/"
-  environment     = var.environment
-  region          = var.region
-  virtual_network = var.virtual_network
-  networks        = var.networks
-  gateway_subnet  = var.gateway_subnet
-  route_tables    = var.route_tables
-  owner           = var.owner
-  name            = var.name
-  dns_servers     = var.dns_servers
+  source            = "../../modules/vpc/"
+  environment       = var.environment
+  region            = var.region
+  virtual_network   = var.virtual_network
+  networks          = var.networks
+  gateway_subnet    = var.gateway_subnet
+  route_tables      = var.route_tables
+  owner             = var.owner
+  name              = var.name
+  dns_servers       = var.dns_servers
+  service_endpoints = var.service_endpoints
+  vpn_client_cidr   = var.vpn_client_cidr
 }