Merge branch 'staging' into grid-styling

.gitignore

@@ -31,6 +31,7 @@ static/buildinfo.*
 
 # local log files
 log/*
+*.log
 
 config/dev.ini
 .env*
@@ -74,3 +75,7 @@ celerybeat-schedule
 js/test_templates
 
 .mypy_cache/
+
+# terraform
+*.tfstate
+*.backup

deploy/azure/azure.yml

@@ -29,6 +29,13 @@ spec:
       containers:
         - name: atst
           image: $CONTAINER_IMAGE
+          env:
+            - name: UWSGI_PROCESSES
+              value: "2"
+            - name: UWSGI_THREADS
+              value: "2"
+            - name: UWSGI_ENABLE_THREADS
+              value: "1"
           envFrom:
             - configMapRef:
                 name: atst-envvars
@@ -50,11 +57,11 @@ spec:
               mountPath: "/config"
           resources:
             requests:
-              memory: 200Mi
+              memory: 400Mi
-              cpu: 400m
+              cpu: 940m
             limits:
-              memory: 200Mi
+              memory: 400Mi
-              cpu: 400m
+              cpu: 940m
         - name: nginx
           image: nginx:alpine
           ports:
@@ -86,10 +93,10 @@ spec:
           resources:
             requests:
               memory: 20Mi
-              cpu: 10m
+              cpu: 25m
             limits:
               memory: 20Mi
-              cpu: 10m
+              cpu: 25m
       volumes:
         - name: nginx-client-ca-bundle
           configMap:
@@ -309,6 +316,7 @@ metadata:
   namespace: atat
 spec:
   loadBalancerIP: 13.92.235.6
+  externalTrafficPolicy: Local
   ports:
     - port: 80
       targetPort: 8342
@@ -329,6 +337,7 @@ metadata:
   namespace: atat
 spec:
   loadBalancerIP: 23.100.24.41
+  externalTrafficPolicy: Local
   ports:
     - port: 80
       targetPort: 8343

deploy/overlays/cloudzero-dev/flex_vol.yml

@@ -9,13 +9,23 @@ spec:
         - name: nginx-secret
           flexVolume:
             options:
-              keyvaultname: "atat-vault-test"
+              keyvaultname: "cloudzero-dev-keyvault"
-              keyvaultobjectnames: "dhparam4096;cert;cert"
+              # keyvaultobjectnames: "dhparam4096;cert;cert"
+              keyvaultobjectnames: "foo"
+              keyvaultobjectaliases: "FOO"
+              keyvaultobjecttypes: "secret"
+              usevmmanagedidentity: "true"
+              usepodidentity: "false"
         - name: flask-secret
           flexVolume:
             options:
-              keyvaultname: "atat-vault-test"
+              keyvaultname: "cloudzero-dev-keyvault"
-              keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY"
+              # keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY"
+              keyvaultobjectnames: "master-PGPASSWORD"
+              keyvaultobjectaliases: "PGPASSWORD"
+              keyvaultobjecttypes: "secret"
+              usevmmanagedidentity: "true"
+              usepodidentity: "false"
 ---
 apiVersion: extensions/v1beta1
 kind: Deployment
@@ -28,8 +38,10 @@ spec:
         - name: flask-secret
           flexVolume:
             options:
-              keyvaultname: "atat-vault-test"
+              keyvaultname: "cloudzero-dev-keyvault"
               keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY"
+              usevmmanagedidentity: "true"
+              usepodidentity: "false"
 ---
 apiVersion: extensions/v1beta1
 kind: Deployment
@@ -42,8 +54,10 @@ spec:
         - name: flask-secret
           flexVolume:
             options:
-              keyvaultname: "atat-vault-test"
+              keyvaultname: "cloudzero-dev-keyvault"
               keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY"
+              usevmmanagedidentity: "true"
+              usepodidentity: "false"
 ---
 apiVersion: batch/v1beta1
 kind: CronJob
@@ -58,5 +72,7 @@ spec:
             - name: flask-secret
               flexVolume:
                 options:
-                  keyvaultname: "atat-vault-test"
+                  keyvaultname: "cloudzero-dev-keyvault"
                   keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY"
+                  usevmmanagedidentity: "true"
+                  usepodidentity: "false"

deploy/overlays/cloudzero-dev/ports.yml

@@ -3,6 +3,9 @@ apiVersion: v1
 kind: Service
 metadata:
   name: atst-main
+  annotations:
+    service.beta.kubernetes.io/azure-load-balancer-internal: "true"
+    service.beta.kubernetes.io/azure-load-balancer-internal-subnet: "cloudzero-dev-public"
 spec:
   loadBalancerIP: ""
   ports:
@@ -17,6 +20,9 @@ apiVersion: v1
 kind: Service
 metadata:
   name: atst-auth
+  annotations:
+    service.beta.kubernetes.io/azure-load-balancer-internal: "true"
+    service.beta.kubernetes.io/azure-load-balancer-internal-subnet: "cloudzero-dev-public"
 spec:
   loadBalancerIP: ""
   ports:

deploy/overlays/staging/autoscaling.yml

@@ -0,0 +1,16 @@
+---
+apiVersion: autoscaling/v2beta1
+kind: HorizontalPodAutoscaler
+metadata:
+  name: atst
+spec:
+  minReplicas: 1
+  maxReplicas: 2
+---
+apiVersion: autoscaling/v2beta1
+kind: HorizontalPodAutoscaler
+metadata:
+  name: atst-worker
+spec:
+  minReplicas: 1
+  maxReplicas: 2

deploy/overlays/staging/kustomization.yaml

@@ -5,6 +5,7 @@ resources:
   - namespace.yml
   - reset-cron-job.yml
 patchesStrategicMerge:
+  - autoscaling.yml
   - ports.yml
   - envvars.yml
   - flex_vol.yml

terraform/README.md

@@ -57,6 +57,7 @@ To create all the resources we need for this environment we'll need to enable so
 This registers the specific feature for _SystemAssigned_ principals
 ```
 az feature register --namespace Microsoft.ContainerService --name MSIPreview
+az feature register --namespace Microsoft.ContainerService --name NodePublicIPPreview
 ```
 
 To apply the registration, run the following
@@ -207,3 +208,76 @@ TODO
 
 ## Downloading a client profile
 TODO
+
+# Quick Steps
+Copy paste (mostly)
+
+*Register Preview features*
+See [Registering Features](#Preview_Features)
+
+*Edit provider.tf and turn off remote bucket temporarily (comment out backend {} section)*
+```
+provider "azurerm" {
+  version = "=1.40.0"
+}
+
+provider "azuread" {
+  # Whilst version is optional, we /strongly recommend/ using it to pin the version of the Provider being used
+  version = "=0.7.0"
+}
+
+terraform {
+  #backend "azurerm" {
+    #resource_group_name  = "cloudzero-dev-tfstate"
+    #storage_account_name = "cloudzerodevtfstate"
+    #container_name       = "tfstate"
+    #key                  = "dev.terraform.tfstate"
+  #}
+}
+```
+
+`terraform init`
+
+`terraform plan -target=module.tf_state`
+
+Ensure the state bucket is created.
+
+*create the container in the portal (or cli).*
+This simply involves going to the bucket in the azure portal and creating the container.
+
+Now is the tricky part. For this, we will be switching from local state (files) to remote state (stored in the azure bucket)
+
+Uncomment the `backend {}` section in the `provider.tf` file. Once uncommented, we will re-run the init. This will attempt to copy the local state to the remote bucket.
+
+`terraform init`
+
+*Say `yes` to the question*
+
+Now we need to update the Update `variables.tf` with the principals for the users in `admin_users` variable map. If these are not defined yet, just leave it as an empty set. 
+
+Next, we'll create the operator keyvault.
+
+`terraform plan -target=module.operator_keyvault`
+
+Next, we'll pre-populate some secrets using the secrets-tool. Follow the install/setup section in the README.md first. Then populate the secrets with a definition file as described in the following link.
+
+https://github.com/dod-ccpo/atst/tree/staging/terraform/secrets-tool#populating-secrets-from-secrets-definition-file
+
+*Create service principal for AKS*
+```
+az ad sp create-for-rbac
+```
+Take note of the output, you'll need it in the next step to store the secret and `client_id` in keyvault.
+
+This also involves using secrets-tool. Substitute your keyvault url.
+```
+secrets-tool secrets --keyvault https://ops-jedidev-keyvault.vault.azure.net/ create --key k8s-client-id --value [value]
+secrets-tool secrets --keyvault https://ops-jedidev-keyvault.vault.azure.net/ create --key k8s-client-secret --value [value]
+```
+
+*Next we'll apply the rest of the TF configuration*
+
+`terraform plan` # Make sure this looks correct
+
+`terraform apply`
+

terraform/modules/k8s/main.tf

@@ -10,8 +10,8 @@ resource "azurerm_kubernetes_cluster" "k8s" {
   dns_prefix          = var.k8s_dns_prefix
 
   service_principal {
-    client_id     = "f05a4457-bd5e-4c63-98e1-89aab42645d0"
+    client_id     = var.client_id
-    client_secret = "19b69e2c-9f55-4850-87cb-88c67a8dc811"
+    client_secret = var.client_secret
   }
 
   default_node_pool {

terraform/modules/k8s/variables.tf

@@ -52,3 +52,13 @@ variable "min_count" {
   type        = string
   description = "Minimum number of nodes to use in autoscaling. This requires `enable_auto_scaling` to be set to true"
 }
+
+variable "client_id" {
+  type        = string
+  description = "The client ID for the Service Principal associated with the AKS cluster."
+}
+
+variable "client_secret" {
+  type        = string
+  description = "The client secret for the Service Principal associated with the AKS cluster."
+}

terraform/modules/keyvault/outputs.tf

@@ -0,0 +1,7 @@
+output "id" {
+  value = azurerm_key_vault.keyvault.id
+}
+
+output "url" {
+  value = azurerm_key_vault.keyvault.vault_uri
+}

terraform/providers/dev/buckets.tf

@@ -1,6 +1,15 @@
 module "task_order_bucket" {
   source       = "../../modules/bucket"
-  service_name = "tasksatat"
+  service_name = "jeditasksatat"
+  owner        = var.owner
+  name         = var.name
+  environment  = var.environment
+  region       = var.region
+}
+
+module "tf_state" {
+  source       = "../../modules/bucket"
+  service_name = "jedidevtfstate"
   owner        = var.owner
   name         = var.name
   environment  = var.environment

terraform/providers/dev/k8s.tf

@@ -1,3 +1,13 @@
+data "azurerm_key_vault_secret" "k8s_client_id" {
+  name         = "k8s-client-id"
+  key_vault_id = module.operator_keyvault.id
+}
+
+data "azurerm_key_vault_secret" "k8s_client_secret" {
+  name         = "k8s-client-secret"
+  key_vault_id = module.operator_keyvault.id
+}
+
 module "k8s" {
   source              = "../../modules/k8s"
   region              = var.region
@@ -10,6 +20,8 @@ module "k8s" {
   enable_auto_scaling = true
   max_count           = 5
   min_count           = 3
+  client_id           = data.azurerm_key_vault_secret.k8s_client_id.value
+  client_secret       = data.azurerm_key_vault_secret.k8s_client_secret.value
 }
 
 #module "main_lb" {

terraform/providers/dev/keyvault.tf

@@ -1,6 +1,6 @@
 module "keyvault" {
   source           = "../../modules/keyvault"
-  name             = var.name
+  name             = "cz"
   region           = var.region
   owner            = var.owner
   environment      = var.environment

terraform/providers/dev/provider.tf

@@ -9,8 +9,8 @@ provider "azuread" {
 
 terraform {
   backend "azurerm" {
-    resource_group_name  = "cloudzero-dev-tfstate"
+    resource_group_name  = "cloudzero-jedidev-jedidevtfstate"
-    storage_account_name = "cloudzerodevtfstate"
+    storage_account_name = "jedidevtfstate"
     container_name       = "tfstate"
     key                  = "dev.terraform.tfstate"
   }

terraform/providers/dev/secrets.tf

@@ -1,6 +1,6 @@
 module "operator_keyvault" {
   source           = "../../modules/keyvault"
-  name             = "operator"
+  name             = "ops"
   region           = var.region
   owner            = var.owner
   environment      = var.environment

terraform/providers/dev/variables.tf

@@ -1,9 +1,9 @@
 variable "environment" {
-  default = "dev"
+  default = "jedidev"
 }
 
 variable "region" {
-  default = "eastus2"
+  default = "eastus"
 
 }
 
@@ -69,13 +69,13 @@ variable "k8s_dns_prefix" {
 
 variable "tenant_id" {
   type    = string
-  default = "b5ab0e1e-09f8-4258-afb7-fb17654bc5b3"
+  default = "47f616e9-6ff5-4736-9b9e-b3f62c93a915"
 }
 
 variable "admin_users" {
   type = map
   default = {
-    "Rob Gil"         = "2ca63d41-d058-4e06-aef6-eb517a53b631"
+    "Rob Gil"      = "cef37d01-1acf-4085-96c8-da9d34d0237e"
-    "Daniel Corrigan" = "d5bb69c2-3b88-4e96-b1a2-320400f1bf1b"
+    "Dan Corrigan" = "7e852ceb-eb0d-49b1-b71e-e9dcd1082ffc"
   }
 }