Merge pull request #1298 from robgil-dds/169163334-iam-for-scale-set

IAM policy for Azure VM Scale Set in k8s

terraform/README.md

@@ -51,6 +51,19 @@ For Ubuntu 19.10
 snap install powershell --classic
 ```
 
+# Preview Features
+To create all the resources we need for this environment we'll need to enable some _Preview_ features.
+
+This registers the specific feature for _SystemAssigned_ principals
+```
+az feature register --namespace Microsoft.ContainerService --name MSIPreview
+```
+
+To apply the registration, run the following
+```
+az provider register -n Microsoft.ContainerService
+```
+
 # Running Terraform
 First, you'll need to log in to Azure. With the Azure CLI installed, you can run the following.
 
@@ -76,6 +89,50 @@ terraform apply
 
 Check the output for errors. Sometimes the syntax is valid, but some of the configuration may be wrong and only rejected by the Azure API at run time. If this is the case, fix your mistake, and re-run.
 
+# After running TF (Manual Steps)
+
+## VM Scale Set 
+After running terraform, we need to make a manual change to the VM Scale Set that is used in the kubernetes. Terraform has a bug that is not applying this as of `v1.40` of the `azurerm` provider.
+
+In order to get the `SystemAssigned` identity to be set, it needs to be set manually in the console.
+
+Navigate to the VM Scale Set for the k8s cluster you're managing (in the console).
+
+![SystemAssigned Identity](images/system-assigned.png)
+_Just click the `Status` to `On`_
+
+## KeyVault Policy
+There is a bug (missing feature really) in the `azurerm` terraform provider which exposes the wrong `object_id/principal_id` in the `azurerm_kubernetes_cluster` output. The `id` that it exposes is the `object_id` of the cluster itself, and _not_ the Virtual Machine Scale Set SystemAssigned identity. This needs to be updated manually after running terraform for the first time.
+
+To update, just edit the `keyvault.tf`. Set the `principal_id` to the `object_id` of the Virtual Machine Scale set. This can be found in the Azure portal, or via cli.
+
+```
+az vmss list
+```
+In that list, find the scale set for the k8s cluster you're working on. You'll want the value of `principal_id`.
+
+
+The error looks like the following
+```
+  Warning  FailedMount  8s (x6 over 25s)   kubelet, aks-default-54410534-vmss000001  MountVolume.SetUp failed for volume "flask-secret" : mount command failed, status: Failure, reason: /etc/kubernetes/volumeplugins/azure~kv/azurekeyvault-flex
+volume failed, Access denied. Caller was not found on any access policy. r nCaller: appid=e6651156-7127-432d-9617-4425177c48f1;oid=f9bcbe58-8b73-4957-aee2-133dc3e58063;numgroups=0;iss=https://sts.windows.net/b5ab0e1e-09f8-4258-afb7-fb17654bc5
+b3/ r nVault: cloudzero-dev-keyvault;location=eastus2 InnerError={code:AccessDenied}
+```
+
+Final configuration will look like this.
+**keyvault.tf**
+```
+module "keyvault" {
+  source       = "../../modules/keyvault"
+  name         = var.name
+  region       = var.region
+  owner        = var.owner
+  environment  = var.environment
+  tenant_id    = var.tenant_id
+  principal_id = "f9bcbe58-8b73-4957-aee2-133dc3e58063"
+}
+```
+
 # Shutting down and environment
 To shutdown and remove an environment completely as to not incur any costs you would need to run a `terraform destroy`.
 

terraform/modules/k8s/main.tf

@@ -25,6 +25,9 @@ resource "azurerm_kubernetes_cluster" "k8s" {
     min_count             = var.min_count # FIXME: if auto_scaling disabled, set to 0
   }
 
+  identity {
+    type = "SystemAssigned"
+  }
   lifecycle {
     ignore_changes = [
       default_node_pool.0.node_count

terraform/modules/k8s/outputs.tf

@@ -0,0 +1,3 @@
+output "principal_id" {
+  value = azurerm_kubernetes_cluster.k8s.identity[0].principal_id
+}

terraform/modules/keyvault/main.tf

@@ -22,19 +22,15 @@ resource "azurerm_key_vault" "keyvault" {
 resource "azurerm_key_vault_access_policy" "keyvault" {
   key_vault_id = azurerm_key_vault.keyvault.id
 
-  tenant_id = "b5ab0e1e-09f8-4258-afb7-fb17654bc5b3"
-  object_id = "ca8cfc48-9995-4973-a8cc-6c7f755e84de"
+  tenant_id = data.azurerm_client_config.current.tenant_id
+  object_id = var.principal_id
 
   key_permissions = [
     "get",
-    "list",
-    "create",
   ]
 
   secret_permissions = [
     "get",
-    "list",
-    "set",
   ]
 }
 

terraform/modules/keyvault/variables.tf

@@ -22,3 +22,8 @@ variable "tenant_id" {
   type        = string
   description = "The Tenant ID"
 }
+
+variable "principal_id" {
+  type        = string
+  description = "The service principal_id of the k8s cluster"
+}

terraform/modules/vpc/main.tf

@@ -81,7 +81,7 @@ resource "azurerm_subnet" "gateway" {
 
 
 resource "azurerm_public_ip" "vpn_ip" {
-  name                = "test"
+  name                = "${var.name}-${var.environment}-vpn-ip"
   location            = azurerm_resource_group.vpc.location
   resource_group_name = azurerm_resource_group.vpc.name
 
@@ -89,7 +89,7 @@ resource "azurerm_public_ip" "vpn_ip" {
 }
 
 resource "azurerm_virtual_network_gateway" "vnet_gateway" {
-  name                = "test"
+  name                = "${var.name}-${var.environment}-gateway"
   location            = azurerm_resource_group.vpc.location
   resource_group_name = azurerm_resource_group.vpc.name
 

terraform/providers/dev/keyvault.tf

@@ -5,4 +5,5 @@ module "keyvault" {
   owner        = var.owner
   environment  = var.environment
   tenant_id    = var.tenant_id
+  principal_id = "f9bcbe58-8b73-4957-aee2-133dc3e58063"
 }

terraform/providers/dev/provider.tf

@@ -1,5 +1,5 @@
 provider "azurerm" {
-  version = "=1.38.0"
+  version = "=1.40.0"
 }
 
 provider "azuread" {