From d121a12429e076ee18fc582f720bfd8be2c732b4 Mon Sep 17 00:00:00 2001 From: dandds Date: Mon, 6 Jan 2020 12:20:49 -0500 Subject: [PATCH 1/9] Minimal config for cloudzero dev environment. This includes config for the VMSS assigned identity to authenticate for FlexVol purposes. Right now, some dummy keys are referenced in the config that we'll swap for the real ones later. This also includes config for specifying the subnet the load balancers should be in. --- deploy/overlays/cloudzero-dev/flex_vol.yml | 30 +++++++++++++++++----- deploy/overlays/cloudzero-dev/ports.yml | 6 +++++ 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/deploy/overlays/cloudzero-dev/flex_vol.yml b/deploy/overlays/cloudzero-dev/flex_vol.yml index 1da24f7a..a3c65df7 100644 --- a/deploy/overlays/cloudzero-dev/flex_vol.yml +++ b/deploy/overlays/cloudzero-dev/flex_vol.yml @@ -9,13 +9,23 @@ spec: - name: nginx-secret flexVolume: options: - keyvaultname: "atat-vault-test" - keyvaultobjectnames: "dhparam4096;cert;cert" + keyvaultname: "cloudzero-dev-keyvault" + # keyvaultobjectnames: "dhparam4096;cert;cert" + keyvaultobjectnames: "foo" + keyvaultobjectaliases: "FOO" + keyvaultobjecttypes: "secret" + usevmmanagedidentity: "true" + usepodidentity: "false" - name: flask-secret flexVolume: options: - keyvaultname: "atat-vault-test" - keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY" + keyvaultname: "cloudzero-dev-keyvault" + # keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY" + keyvaultobjectnames: "master-PGPASSWORD" + keyvaultobjectaliases: "PGPASSWORD" + keyvaultobjecttypes: "secret" + usevmmanagedidentity: "true" + usepodidentity: "false" --- apiVersion: extensions/v1beta1 kind: Deployment @@ -28,8 +38,10 @@ spec: - name: flask-secret flexVolume: options: - keyvaultname: "atat-vault-test" + keyvaultname: "cloudzero-dev-keyvault" keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY" + usevmmanagedidentity: "true" + usepodidentity: "false" --- apiVersion: extensions/v1beta1 kind: Deployment @@ -42,8 +54,10 @@ spec: - name: flask-secret flexVolume: options: - keyvaultname: "atat-vault-test" + keyvaultname: "cloudzero-dev-keyvault" keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY" + usevmmanagedidentity: "true" + usepodidentity: "false" --- apiVersion: batch/v1beta1 kind: CronJob @@ -58,5 +72,7 @@ spec: - name: flask-secret flexVolume: options: - keyvaultname: "atat-vault-test" + keyvaultname: "cloudzero-dev-keyvault" keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY" + usevmmanagedidentity: "true" + usepodidentity: "false" diff --git a/deploy/overlays/cloudzero-dev/ports.yml b/deploy/overlays/cloudzero-dev/ports.yml index 8f4ff72c..8dbbd0f1 100644 --- a/deploy/overlays/cloudzero-dev/ports.yml +++ b/deploy/overlays/cloudzero-dev/ports.yml @@ -3,6 +3,9 @@ apiVersion: v1 kind: Service metadata: name: atst-main + annotations: + service.beta.kubernetes.io/azure-load-balancer-internal: "true" + service.beta.kubernetes.io/azure-load-balancer-internal-subnet: "cloudzero-dev-public" spec: loadBalancerIP: "" ports: @@ -17,6 +20,9 @@ apiVersion: v1 kind: Service metadata: name: atst-auth + annotations: + service.beta.kubernetes.io/azure-load-balancer-internal: "true" + service.beta.kubernetes.io/azure-load-balancer-internal-subnet: "cloudzero-dev-public" spec: loadBalancerIP: "" ports: From 591682ab8705d81ea5b13ae9b3f3f2ebba229d90 Mon Sep 17 00:00:00 2001 From: dandds Date: Fri, 17 Jan 2020 19:29:24 -0500 Subject: [PATCH 2/9] Display client source IP correctly in logs. In order to display the client source IP, we need to make a configuration change to the cluster. Setting externalTrafficPolicy to "Local" preserves the client IP, per: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip --- deploy/azure/azure.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deploy/azure/azure.yml b/deploy/azure/azure.yml index 1300ed34..81612d5d 100644 --- a/deploy/azure/azure.yml +++ b/deploy/azure/azure.yml @@ -309,6 +309,7 @@ metadata: namespace: atat spec: loadBalancerIP: 13.92.235.6 + externalTrafficPolicy: Local ports: - port: 80 targetPort: 8342 @@ -329,6 +330,7 @@ metadata: namespace: atat spec: loadBalancerIP: 23.100.24.41 + externalTrafficPolicy: Local ports: - port: 80 targetPort: 8343 From 4eded230512550dfb7cafe55b8133acf6b501dce Mon Sep 17 00:00:00 2001 From: Rob Gil Date: Mon, 20 Jan 2020 14:04:51 -0500 Subject: [PATCH 3/9] Adds keyvault outputs --- terraform/modules/keyvault/outputs.tf | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 terraform/modules/keyvault/outputs.tf diff --git a/terraform/modules/keyvault/outputs.tf b/terraform/modules/keyvault/outputs.tf new file mode 100644 index 00000000..9e29252a --- /dev/null +++ b/terraform/modules/keyvault/outputs.tf @@ -0,0 +1,7 @@ +output "id" { + value = azurerm_key_vault.keyvault.id +} + +output "url" { + value = azurerm_key_vault.keyvault.vault_uri +} \ No newline at end of file From 636653a5ade773176e9c7163d53097820e260e23 Mon Sep 17 00:00:00 2001 From: Rob Gil Date: Mon, 20 Jan 2020 15:37:01 -0500 Subject: [PATCH 4/9] Additional quick steps on how to configure terraform --- terraform/README.md | 61 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/terraform/README.md b/terraform/README.md index 7b875d70..b488be91 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -206,4 +206,63 @@ https://login.microsoftonline.com/common/oauth2/authorize?client_id=41b23e61-6c1 TODO ## Downloading a client profile -TODO \ No newline at end of file +TODO + +# Quick Steps +Copy paste (mostly) + +*Edit provider.tf and turn off remote bucket temporarily (comment out backend {} section)* +``` +provider "azurerm" { + version = "=1.40.0" +} + +provider "azuread" { + # Whilst version is optional, we /strongly recommend/ using it to pin the version of the Provider being used + version = "=0.7.0" +} + +terraform { + #backend "azurerm" { + #resource_group_name = "cloudzero-dev-tfstate" + #storage_account_name = "cloudzerodevtfstate" + #container_name = "tfstate" + #key = "dev.terraform.tfstate" + #} +} +``` + +`terraform init` + +`terraform plan -target=module.tf_state` + +Ensure the state bucket is created. + +*create the container in the portal (or cli).* +This simply involves going to the bucket in the azure portal and creating the container. + +Now is the tricky part. For this, we will be switching from local state (files) to remote state (stored in the azure bucket) + +Uncomment the `backend {}` section in the `provider.tf` file. Once uncommented, we will re-run the init. This will attempt to copy the local state to the remote bucket. + +`terraform init` + +*Say `yes` to the question* + +Now we need to update the Update `variables.tf` with the principals for the users in `admin_users` variable map. If these are not defined yet, just leave it as an empty set. + +Next, we'll create the operator keyvault. + +`terraform plan -target=module.operator_keyvault` + +Lastly, we'll pre-populate some secrets using the secrets-tool. Follow the install/setup section in the README.md first. Then populate the secrets with a definition file as described in the following link. + + +https://github.com/dod-ccpo/atst/tree/staging/terraform/secrets-tool#populating-secrets-from-secrets-definition-file + +*Next we'll apply the rest of the TF configuration* + +`terraform plan` # Make sure this looks correct + +`terraform apply` + From 2008b4e6e3fee110dc899c2e97e5bb55cce44de1 Mon Sep 17 00:00:00 2001 From: raydds Date: Fri, 17 Jan 2020 11:21:06 -0500 Subject: [PATCH 5/9] Run UWSGI with 4 processes and 2 threads. Allocate more CPU/memory --- deploy/azure/autoscaling.yml | 4 ++-- deploy/azure/azure.yml | 19 +++++++++++++------ 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/deploy/azure/autoscaling.yml b/deploy/azure/autoscaling.yml index e208d943..8bab90f4 100644 --- a/deploy/azure/autoscaling.yml +++ b/deploy/azure/autoscaling.yml @@ -17,7 +17,7 @@ spec: - type: Resource resource: name: cpu - targetAverageUtilization: 60 + targetAverageUtilization: 70 --- apiVersion: autoscaling/v2beta1 kind: HorizontalPodAutoscaler @@ -37,4 +37,4 @@ spec: - type: Resource resource: name: cpu - targetAverageUtilization: 60 + targetAverageUtilization: 70 diff --git a/deploy/azure/azure.yml b/deploy/azure/azure.yml index 81612d5d..7feb0823 100644 --- a/deploy/azure/azure.yml +++ b/deploy/azure/azure.yml @@ -29,6 +29,13 @@ spec: containers: - name: atst image: $CONTAINER_IMAGE + env: + - name: UWSGI_PROCESSES + value: "4" + - name: UWSGI_THREADS + value: "2" + - name: UWSGI_ENABLE_THREADS + value: "1" envFrom: - configMapRef: name: atst-envvars @@ -50,11 +57,11 @@ spec: mountPath: "/config" resources: requests: - memory: 200Mi - cpu: 400m + memory: 400Mi + cpu: 1200m limits: - memory: 200Mi - cpu: 400m + memory: 400Mi + cpu: 1200m - name: nginx image: nginx:alpine ports: @@ -86,10 +93,10 @@ spec: resources: requests: memory: 20Mi - cpu: 10m + cpu: 100m limits: memory: 20Mi - cpu: 10m + cpu: 100m volumes: - name: nginx-client-ca-bundle configMap: From 14b9f3620440e9c684054f3a1b3d282b5e2332a7 Mon Sep 17 00:00:00 2001 From: raydds Date: Mon, 20 Jan 2020 14:06:54 -0500 Subject: [PATCH 6/9] Minimize staging --- deploy/overlays/staging/autoscaling.yml | 16 ++++++++++++++++ deploy/overlays/staging/kustomization.yaml | 1 + 2 files changed, 17 insertions(+) create mode 100644 deploy/overlays/staging/autoscaling.yml diff --git a/deploy/overlays/staging/autoscaling.yml b/deploy/overlays/staging/autoscaling.yml new file mode 100644 index 00000000..b7500c09 --- /dev/null +++ b/deploy/overlays/staging/autoscaling.yml @@ -0,0 +1,16 @@ +--- +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: atst +spec: + minReplicas: 1 + maxReplicas: 2 +--- +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: atst-worker +spec: + minReplicas: 1 + maxReplicas: 2 diff --git a/deploy/overlays/staging/kustomization.yaml b/deploy/overlays/staging/kustomization.yaml index 24705531..c1ef8fd2 100644 --- a/deploy/overlays/staging/kustomization.yaml +++ b/deploy/overlays/staging/kustomization.yaml @@ -5,6 +5,7 @@ resources: - namespace.yml - reset-cron-job.yml patchesStrategicMerge: + - autoscaling.yml - ports.yml - envvars.yml - flex_vol.yml From 35cd76adf5c36079873731c2ad21bff52e272edd Mon Sep 17 00:00:00 2001 From: raydds Date: Mon, 20 Jan 2020 14:07:08 -0500 Subject: [PATCH 7/9] Try to fit two web processes per node --- deploy/azure/autoscaling.yml | 4 ++-- deploy/azure/azure.yml | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/deploy/azure/autoscaling.yml b/deploy/azure/autoscaling.yml index 8bab90f4..e208d943 100644 --- a/deploy/azure/autoscaling.yml +++ b/deploy/azure/autoscaling.yml @@ -17,7 +17,7 @@ spec: - type: Resource resource: name: cpu - targetAverageUtilization: 70 + targetAverageUtilization: 60 --- apiVersion: autoscaling/v2beta1 kind: HorizontalPodAutoscaler @@ -37,4 +37,4 @@ spec: - type: Resource resource: name: cpu - targetAverageUtilization: 70 + targetAverageUtilization: 60 diff --git a/deploy/azure/azure.yml b/deploy/azure/azure.yml index 7feb0823..f988d5fc 100644 --- a/deploy/azure/azure.yml +++ b/deploy/azure/azure.yml @@ -31,7 +31,7 @@ spec: image: $CONTAINER_IMAGE env: - name: UWSGI_PROCESSES - value: "4" + value: "2" - name: UWSGI_THREADS value: "2" - name: UWSGI_ENABLE_THREADS @@ -58,10 +58,10 @@ spec: resources: requests: memory: 400Mi - cpu: 1200m + cpu: 940m limits: memory: 400Mi - cpu: 1200m + cpu: 940m - name: nginx image: nginx:alpine ports: @@ -93,10 +93,10 @@ spec: resources: requests: memory: 20Mi - cpu: 100m + cpu: 25m limits: memory: 20Mi - cpu: 100m + cpu: 25m volumes: - name: nginx-client-ca-bundle configMap: From 584b8853118a6b7111a07ea2e906e021a581c72c Mon Sep 17 00:00:00 2001 From: Rob Gil Date: Mon, 20 Jan 2020 16:10:55 -0500 Subject: [PATCH 8/9] Adds notes on AKS service_principal and preview features that must be enabled --- terraform/README.md | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/terraform/README.md b/terraform/README.md index b488be91..40460cb9 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -57,6 +57,7 @@ To create all the resources we need for this environment we'll need to enable so This registers the specific feature for _SystemAssigned_ principals ``` az feature register --namespace Microsoft.ContainerService --name MSIPreview +az feature register --namespace Microsoft.ContainerService --name NodePublicIPPreview ``` To apply the registration, run the following @@ -211,6 +212,9 @@ TODO # Quick Steps Copy paste (mostly) +*Register Preview features* +See [Registering Features](#Preview_Features) + *Edit provider.tf and turn off remote bucket temporarily (comment out backend {} section)* ``` provider "azurerm" { @@ -255,11 +259,22 @@ Next, we'll create the operator keyvault. `terraform plan -target=module.operator_keyvault` -Lastly, we'll pre-populate some secrets using the secrets-tool. Follow the install/setup section in the README.md first. Then populate the secrets with a definition file as described in the following link. - +Next, we'll pre-populate some secrets using the secrets-tool. Follow the install/setup section in the README.md first. Then populate the secrets with a definition file as described in the following link. https://github.com/dod-ccpo/atst/tree/staging/terraform/secrets-tool#populating-secrets-from-secrets-definition-file +*Create service principal for AKS* +``` +az ad sp create-for-rbac +``` +Take note of the output, you'll need it in the next step to store the secret and `client_id` in keyvault. + +This also involves using secrets-tool. Substitute your keyvault url. +``` +secrets-tool secrets --keyvault https://ops-jedidev-keyvault.vault.azure.net/ create --key k8s-client-id --value [value] +secrets-tool secrets --keyvault https://ops-jedidev-keyvault.vault.azure.net/ create --key k8s-client-secret --value [value] +``` + *Next we'll apply the rest of the TF configuration* `terraform plan` # Make sure this looks correct From 9f2bdd4a9f1bf95875f6abde200eabdb9c74f5da Mon Sep 17 00:00:00 2001 From: dandds Date: Mon, 20 Jan 2020 19:34:47 -0500 Subject: [PATCH 9/9] Updated dev environment for JEDI. - Updated environment name. - Updated variables. - AKS service principal creds moved to the operator Key Vault. --- .gitignore | 5 +++++ terraform/modules/k8s/main.tf | 6 +++--- terraform/modules/k8s/variables.tf | 10 ++++++++++ terraform/providers/dev/buckets.tf | 11 ++++++++++- terraform/providers/dev/k8s.tf | 12 ++++++++++++ terraform/providers/dev/keyvault.tf | 2 +- terraform/providers/dev/provider.tf | 4 ++-- terraform/providers/dev/secrets.tf | 2 +- terraform/providers/dev/variables.tf | 10 +++++----- 9 files changed, 49 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index d8e2290d..05e52e03 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,7 @@ static/buildinfo.* # local log files log/* +*.log config/dev.ini .env* @@ -74,3 +75,7 @@ celerybeat-schedule js/test_templates .mypy_cache/ + +# terraform +*.tfstate +*.backup diff --git a/terraform/modules/k8s/main.tf b/terraform/modules/k8s/main.tf index 9eb7b68d..060d50b8 100644 --- a/terraform/modules/k8s/main.tf +++ b/terraform/modules/k8s/main.tf @@ -10,8 +10,8 @@ resource "azurerm_kubernetes_cluster" "k8s" { dns_prefix = var.k8s_dns_prefix service_principal { - client_id = "f05a4457-bd5e-4c63-98e1-89aab42645d0" - client_secret = "19b69e2c-9f55-4850-87cb-88c67a8dc811" + client_id = var.client_id + client_secret = var.client_secret } default_node_pool { @@ -38,4 +38,4 @@ resource "azurerm_kubernetes_cluster" "k8s" { environment = var.environment owner = var.owner } -} \ No newline at end of file +} diff --git a/terraform/modules/k8s/variables.tf b/terraform/modules/k8s/variables.tf index 28677ec2..e8ca5a27 100644 --- a/terraform/modules/k8s/variables.tf +++ b/terraform/modules/k8s/variables.tf @@ -52,3 +52,13 @@ variable "min_count" { type = string description = "Minimum number of nodes to use in autoscaling. This requires `enable_auto_scaling` to be set to true" } + +variable "client_id" { + type = string + description = "The client ID for the Service Principal associated with the AKS cluster." +} + +variable "client_secret" { + type = string + description = "The client secret for the Service Principal associated with the AKS cluster." +} diff --git a/terraform/providers/dev/buckets.tf b/terraform/providers/dev/buckets.tf index 3ded916f..d58987fc 100644 --- a/terraform/providers/dev/buckets.tf +++ b/terraform/providers/dev/buckets.tf @@ -1,6 +1,15 @@ module "task_order_bucket" { source = "../../modules/bucket" - service_name = "tasksatat" + service_name = "jeditasksatat" + owner = var.owner + name = var.name + environment = var.environment + region = var.region +} + +module "tf_state" { + source = "../../modules/bucket" + service_name = "jedidevtfstate" owner = var.owner name = var.name environment = var.environment diff --git a/terraform/providers/dev/k8s.tf b/terraform/providers/dev/k8s.tf index 127b9306..7d415c9c 100644 --- a/terraform/providers/dev/k8s.tf +++ b/terraform/providers/dev/k8s.tf @@ -1,3 +1,13 @@ +data "azurerm_key_vault_secret" "k8s_client_id" { + name = "k8s-client-id" + key_vault_id = module.operator_keyvault.id +} + +data "azurerm_key_vault_secret" "k8s_client_secret" { + name = "k8s-client-secret" + key_vault_id = module.operator_keyvault.id +} + module "k8s" { source = "../../modules/k8s" region = var.region @@ -10,6 +20,8 @@ module "k8s" { enable_auto_scaling = true max_count = 5 min_count = 3 + client_id = data.azurerm_key_vault_secret.k8s_client_id.value + client_secret = data.azurerm_key_vault_secret.k8s_client_secret.value } #module "main_lb" { diff --git a/terraform/providers/dev/keyvault.tf b/terraform/providers/dev/keyvault.tf index aca74e78..75f7b13d 100644 --- a/terraform/providers/dev/keyvault.tf +++ b/terraform/providers/dev/keyvault.tf @@ -1,6 +1,6 @@ module "keyvault" { source = "../../modules/keyvault" - name = var.name + name = "cz" region = var.region owner = var.owner environment = var.environment diff --git a/terraform/providers/dev/provider.tf b/terraform/providers/dev/provider.tf index cd121d6d..7225b1e1 100644 --- a/terraform/providers/dev/provider.tf +++ b/terraform/providers/dev/provider.tf @@ -9,8 +9,8 @@ provider "azuread" { terraform { backend "azurerm" { - resource_group_name = "cloudzero-dev-tfstate" - storage_account_name = "cloudzerodevtfstate" + resource_group_name = "cloudzero-jedidev-jedidevtfstate" + storage_account_name = "jedidevtfstate" container_name = "tfstate" key = "dev.terraform.tfstate" } diff --git a/terraform/providers/dev/secrets.tf b/terraform/providers/dev/secrets.tf index 5ef43a81..bccdcf50 100644 --- a/terraform/providers/dev/secrets.tf +++ b/terraform/providers/dev/secrets.tf @@ -1,6 +1,6 @@ module "operator_keyvault" { source = "../../modules/keyvault" - name = "operator" + name = "ops" region = var.region owner = var.owner environment = var.environment diff --git a/terraform/providers/dev/variables.tf b/terraform/providers/dev/variables.tf index 24c59503..32ba5688 100644 --- a/terraform/providers/dev/variables.tf +++ b/terraform/providers/dev/variables.tf @@ -1,9 +1,9 @@ variable "environment" { - default = "dev" + default = "jedidev" } variable "region" { - default = "eastus2" + default = "eastus" } @@ -69,13 +69,13 @@ variable "k8s_dns_prefix" { variable "tenant_id" { type = string - default = "b5ab0e1e-09f8-4258-afb7-fb17654bc5b3" + default = "47f616e9-6ff5-4736-9b9e-b3f62c93a915" } variable "admin_users" { type = map default = { - "Rob Gil" = "2ca63d41-d058-4e06-aef6-eb517a53b631" - "Daniel Corrigan" = "d5bb69c2-3b88-4e96-b1a2-320400f1bf1b" + "Rob Gil" = "cef37d01-1acf-4085-96c8-da9d34d0237e" + "Dan Corrigan" = "7e852ceb-eb0d-49b1-b71e-e9dcd1082ffc" } }