diff --git a/.gitignore b/.gitignore index d8e2290d..05e52e03 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,7 @@ static/buildinfo.* # local log files log/* +*.log config/dev.ini .env* @@ -74,3 +75,7 @@ celerybeat-schedule js/test_templates .mypy_cache/ + +# terraform +*.tfstate +*.backup diff --git a/deploy/azure/azure.yml b/deploy/azure/azure.yml index 1300ed34..f988d5fc 100644 --- a/deploy/azure/azure.yml +++ b/deploy/azure/azure.yml @@ -29,6 +29,13 @@ spec: containers: - name: atst image: $CONTAINER_IMAGE + env: + - name: UWSGI_PROCESSES + value: "2" + - name: UWSGI_THREADS + value: "2" + - name: UWSGI_ENABLE_THREADS + value: "1" envFrom: - configMapRef: name: atst-envvars @@ -50,11 +57,11 @@ spec: mountPath: "/config" resources: requests: - memory: 200Mi - cpu: 400m + memory: 400Mi + cpu: 940m limits: - memory: 200Mi - cpu: 400m + memory: 400Mi + cpu: 940m - name: nginx image: nginx:alpine ports: @@ -86,10 +93,10 @@ spec: resources: requests: memory: 20Mi - cpu: 10m + cpu: 25m limits: memory: 20Mi - cpu: 10m + cpu: 25m volumes: - name: nginx-client-ca-bundle configMap: @@ -309,6 +316,7 @@ metadata: namespace: atat spec: loadBalancerIP: 13.92.235.6 + externalTrafficPolicy: Local ports: - port: 80 targetPort: 8342 @@ -329,6 +337,7 @@ metadata: namespace: atat spec: loadBalancerIP: 23.100.24.41 + externalTrafficPolicy: Local ports: - port: 80 targetPort: 8343 diff --git a/deploy/overlays/cloudzero-dev/flex_vol.yml b/deploy/overlays/cloudzero-dev/flex_vol.yml index 1da24f7a..a3c65df7 100644 --- a/deploy/overlays/cloudzero-dev/flex_vol.yml +++ b/deploy/overlays/cloudzero-dev/flex_vol.yml @@ -9,13 +9,23 @@ spec: - name: nginx-secret flexVolume: options: - keyvaultname: "atat-vault-test" - keyvaultobjectnames: "dhparam4096;cert;cert" + keyvaultname: "cloudzero-dev-keyvault" + # keyvaultobjectnames: "dhparam4096;cert;cert" + keyvaultobjectnames: "foo" + keyvaultobjectaliases: "FOO" + keyvaultobjecttypes: "secret" + usevmmanagedidentity: "true" + usepodidentity: "false" - name: flask-secret flexVolume: options: - keyvaultname: "atat-vault-test" - keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY" + keyvaultname: "cloudzero-dev-keyvault" + # keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY" + keyvaultobjectnames: "master-PGPASSWORD" + keyvaultobjectaliases: "PGPASSWORD" + keyvaultobjecttypes: "secret" + usevmmanagedidentity: "true" + usepodidentity: "false" --- apiVersion: extensions/v1beta1 kind: Deployment @@ -28,8 +38,10 @@ spec: - name: flask-secret flexVolume: options: - keyvaultname: "atat-vault-test" + keyvaultname: "cloudzero-dev-keyvault" keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY" + usevmmanagedidentity: "true" + usepodidentity: "false" --- apiVersion: extensions/v1beta1 kind: Deployment @@ -42,8 +54,10 @@ spec: - name: flask-secret flexVolume: options: - keyvaultname: "atat-vault-test" + keyvaultname: "cloudzero-dev-keyvault" keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY" + usevmmanagedidentity: "true" + usepodidentity: "false" --- apiVersion: batch/v1beta1 kind: CronJob @@ -58,5 +72,7 @@ spec: - name: flask-secret flexVolume: options: - keyvaultname: "atat-vault-test" + keyvaultname: "cloudzero-dev-keyvault" keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY" + usevmmanagedidentity: "true" + usepodidentity: "false" diff --git a/deploy/overlays/cloudzero-dev/ports.yml b/deploy/overlays/cloudzero-dev/ports.yml index 8f4ff72c..8dbbd0f1 100644 --- a/deploy/overlays/cloudzero-dev/ports.yml +++ b/deploy/overlays/cloudzero-dev/ports.yml @@ -3,6 +3,9 @@ apiVersion: v1 kind: Service metadata: name: atst-main + annotations: + service.beta.kubernetes.io/azure-load-balancer-internal: "true" + service.beta.kubernetes.io/azure-load-balancer-internal-subnet: "cloudzero-dev-public" spec: loadBalancerIP: "" ports: @@ -17,6 +20,9 @@ apiVersion: v1 kind: Service metadata: name: atst-auth + annotations: + service.beta.kubernetes.io/azure-load-balancer-internal: "true" + service.beta.kubernetes.io/azure-load-balancer-internal-subnet: "cloudzero-dev-public" spec: loadBalancerIP: "" ports: diff --git a/deploy/overlays/staging/autoscaling.yml b/deploy/overlays/staging/autoscaling.yml new file mode 100644 index 00000000..b7500c09 --- /dev/null +++ b/deploy/overlays/staging/autoscaling.yml @@ -0,0 +1,16 @@ +--- +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: atst +spec: + minReplicas: 1 + maxReplicas: 2 +--- +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: atst-worker +spec: + minReplicas: 1 + maxReplicas: 2 diff --git a/deploy/overlays/staging/kustomization.yaml b/deploy/overlays/staging/kustomization.yaml index 24705531..c1ef8fd2 100644 --- a/deploy/overlays/staging/kustomization.yaml +++ b/deploy/overlays/staging/kustomization.yaml @@ -5,6 +5,7 @@ resources: - namespace.yml - reset-cron-job.yml patchesStrategicMerge: + - autoscaling.yml - ports.yml - envvars.yml - flex_vol.yml diff --git a/terraform/README.md b/terraform/README.md index 7b875d70..40460cb9 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -57,6 +57,7 @@ To create all the resources we need for this environment we'll need to enable so This registers the specific feature for _SystemAssigned_ principals ``` az feature register --namespace Microsoft.ContainerService --name MSIPreview +az feature register --namespace Microsoft.ContainerService --name NodePublicIPPreview ``` To apply the registration, run the following @@ -206,4 +207,77 @@ https://login.microsoftonline.com/common/oauth2/authorize?client_id=41b23e61-6c1 TODO ## Downloading a client profile -TODO \ No newline at end of file +TODO + +# Quick Steps +Copy paste (mostly) + +*Register Preview features* +See [Registering Features](#Preview_Features) + +*Edit provider.tf and turn off remote bucket temporarily (comment out backend {} section)* +``` +provider "azurerm" { + version = "=1.40.0" +} + +provider "azuread" { + # Whilst version is optional, we /strongly recommend/ using it to pin the version of the Provider being used + version = "=0.7.0" +} + +terraform { + #backend "azurerm" { + #resource_group_name = "cloudzero-dev-tfstate" + #storage_account_name = "cloudzerodevtfstate" + #container_name = "tfstate" + #key = "dev.terraform.tfstate" + #} +} +``` + +`terraform init` + +`terraform plan -target=module.tf_state` + +Ensure the state bucket is created. + +*create the container in the portal (or cli).* +This simply involves going to the bucket in the azure portal and creating the container. + +Now is the tricky part. For this, we will be switching from local state (files) to remote state (stored in the azure bucket) + +Uncomment the `backend {}` section in the `provider.tf` file. Once uncommented, we will re-run the init. This will attempt to copy the local state to the remote bucket. + +`terraform init` + +*Say `yes` to the question* + +Now we need to update the Update `variables.tf` with the principals for the users in `admin_users` variable map. If these are not defined yet, just leave it as an empty set. + +Next, we'll create the operator keyvault. + +`terraform plan -target=module.operator_keyvault` + +Next, we'll pre-populate some secrets using the secrets-tool. Follow the install/setup section in the README.md first. Then populate the secrets with a definition file as described in the following link. + +https://github.com/dod-ccpo/atst/tree/staging/terraform/secrets-tool#populating-secrets-from-secrets-definition-file + +*Create service principal for AKS* +``` +az ad sp create-for-rbac +``` +Take note of the output, you'll need it in the next step to store the secret and `client_id` in keyvault. + +This also involves using secrets-tool. Substitute your keyvault url. +``` +secrets-tool secrets --keyvault https://ops-jedidev-keyvault.vault.azure.net/ create --key k8s-client-id --value [value] +secrets-tool secrets --keyvault https://ops-jedidev-keyvault.vault.azure.net/ create --key k8s-client-secret --value [value] +``` + +*Next we'll apply the rest of the TF configuration* + +`terraform plan` # Make sure this looks correct + +`terraform apply` + diff --git a/terraform/modules/k8s/main.tf b/terraform/modules/k8s/main.tf index 9eb7b68d..060d50b8 100644 --- a/terraform/modules/k8s/main.tf +++ b/terraform/modules/k8s/main.tf @@ -10,8 +10,8 @@ resource "azurerm_kubernetes_cluster" "k8s" { dns_prefix = var.k8s_dns_prefix service_principal { - client_id = "f05a4457-bd5e-4c63-98e1-89aab42645d0" - client_secret = "19b69e2c-9f55-4850-87cb-88c67a8dc811" + client_id = var.client_id + client_secret = var.client_secret } default_node_pool { @@ -38,4 +38,4 @@ resource "azurerm_kubernetes_cluster" "k8s" { environment = var.environment owner = var.owner } -} \ No newline at end of file +} diff --git a/terraform/modules/k8s/variables.tf b/terraform/modules/k8s/variables.tf index 28677ec2..e8ca5a27 100644 --- a/terraform/modules/k8s/variables.tf +++ b/terraform/modules/k8s/variables.tf @@ -52,3 +52,13 @@ variable "min_count" { type = string description = "Minimum number of nodes to use in autoscaling. This requires `enable_auto_scaling` to be set to true" } + +variable "client_id" { + type = string + description = "The client ID for the Service Principal associated with the AKS cluster." +} + +variable "client_secret" { + type = string + description = "The client secret for the Service Principal associated with the AKS cluster." +} diff --git a/terraform/modules/keyvault/outputs.tf b/terraform/modules/keyvault/outputs.tf new file mode 100644 index 00000000..9e29252a --- /dev/null +++ b/terraform/modules/keyvault/outputs.tf @@ -0,0 +1,7 @@ +output "id" { + value = azurerm_key_vault.keyvault.id +} + +output "url" { + value = azurerm_key_vault.keyvault.vault_uri +} \ No newline at end of file diff --git a/terraform/providers/dev/buckets.tf b/terraform/providers/dev/buckets.tf index 3ded916f..d58987fc 100644 --- a/terraform/providers/dev/buckets.tf +++ b/terraform/providers/dev/buckets.tf @@ -1,6 +1,15 @@ module "task_order_bucket" { source = "../../modules/bucket" - service_name = "tasksatat" + service_name = "jeditasksatat" + owner = var.owner + name = var.name + environment = var.environment + region = var.region +} + +module "tf_state" { + source = "../../modules/bucket" + service_name = "jedidevtfstate" owner = var.owner name = var.name environment = var.environment diff --git a/terraform/providers/dev/k8s.tf b/terraform/providers/dev/k8s.tf index 127b9306..7d415c9c 100644 --- a/terraform/providers/dev/k8s.tf +++ b/terraform/providers/dev/k8s.tf @@ -1,3 +1,13 @@ +data "azurerm_key_vault_secret" "k8s_client_id" { + name = "k8s-client-id" + key_vault_id = module.operator_keyvault.id +} + +data "azurerm_key_vault_secret" "k8s_client_secret" { + name = "k8s-client-secret" + key_vault_id = module.operator_keyvault.id +} + module "k8s" { source = "../../modules/k8s" region = var.region @@ -10,6 +20,8 @@ module "k8s" { enable_auto_scaling = true max_count = 5 min_count = 3 + client_id = data.azurerm_key_vault_secret.k8s_client_id.value + client_secret = data.azurerm_key_vault_secret.k8s_client_secret.value } #module "main_lb" { diff --git a/terraform/providers/dev/keyvault.tf b/terraform/providers/dev/keyvault.tf index aca74e78..75f7b13d 100644 --- a/terraform/providers/dev/keyvault.tf +++ b/terraform/providers/dev/keyvault.tf @@ -1,6 +1,6 @@ module "keyvault" { source = "../../modules/keyvault" - name = var.name + name = "cz" region = var.region owner = var.owner environment = var.environment diff --git a/terraform/providers/dev/provider.tf b/terraform/providers/dev/provider.tf index cd121d6d..7225b1e1 100644 --- a/terraform/providers/dev/provider.tf +++ b/terraform/providers/dev/provider.tf @@ -9,8 +9,8 @@ provider "azuread" { terraform { backend "azurerm" { - resource_group_name = "cloudzero-dev-tfstate" - storage_account_name = "cloudzerodevtfstate" + resource_group_name = "cloudzero-jedidev-jedidevtfstate" + storage_account_name = "jedidevtfstate" container_name = "tfstate" key = "dev.terraform.tfstate" } diff --git a/terraform/providers/dev/secrets.tf b/terraform/providers/dev/secrets.tf index 5ef43a81..bccdcf50 100644 --- a/terraform/providers/dev/secrets.tf +++ b/terraform/providers/dev/secrets.tf @@ -1,6 +1,6 @@ module "operator_keyvault" { source = "../../modules/keyvault" - name = "operator" + name = "ops" region = var.region owner = var.owner environment = var.environment diff --git a/terraform/providers/dev/variables.tf b/terraform/providers/dev/variables.tf index 24c59503..32ba5688 100644 --- a/terraform/providers/dev/variables.tf +++ b/terraform/providers/dev/variables.tf @@ -1,9 +1,9 @@ variable "environment" { - default = "dev" + default = "jedidev" } variable "region" { - default = "eastus2" + default = "eastus" } @@ -69,13 +69,13 @@ variable "k8s_dns_prefix" { variable "tenant_id" { type = string - default = "b5ab0e1e-09f8-4258-afb7-fb17654bc5b3" + default = "47f616e9-6ff5-4736-9b9e-b3f62c93a915" } variable "admin_users" { type = map default = { - "Rob Gil" = "2ca63d41-d058-4e06-aef6-eb517a53b631" - "Daniel Corrigan" = "d5bb69c2-3b88-4e96-b1a2-320400f1bf1b" + "Rob Gil" = "cef37d01-1acf-4085-96c8-da9d34d0237e" + "Dan Corrigan" = "7e852ceb-eb0d-49b1-b71e-e9dcd1082ffc" } }