Merge pull request #1276 from robgil-dds/169163334-dns-fix

169163334 - Adds public ips to k8s nodes for internet access

deploy/overlays/cloudzero-dev/envvars.yml

@@ -0,0 +1,22 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: atst-worker-envvars
+data:
+  CELERY_DEFAULT_QUEUE: celery-staging
+  SERVER_NAME: staging.atat.code.mil
+  FLASK_ENV: staging
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: atst-envvars
+data:
+  ASSETS_URL: https://atat-cdn-staging.azureedge.net/
+  CDN_ORIGIN: https://staging.atat.code.mil
+  CELERY_DEFAULT_QUEUE: celery-staging
+  FLASK_ENV: staging
+  STATIC_URL: https://atat-cdn-staging.azureedge.net/static/
+  PGHOST: cloudzero-dev-sql.postgres.database.azure.com
+  REDIS_HOST: cloudzero-dev-redis.redis.cache.windows.net:6380

deploy/overlays/cloudzero-dev/flex_vol.yml

@@ -0,0 +1,62 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: atst
+spec:
+  template:
+    spec:
+      volumes:
+        - name: nginx-secret
+          flexVolume:
+            options:
+              keyvaultname: "atat-vault-test"
+              keyvaultobjectnames: "dhparam4096;cert;cert"
+        - name: flask-secret
+          flexVolume:
+            options:
+              keyvaultname: "atat-vault-test"
+              keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY"
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: atst-worker
+spec:
+  template:
+    spec:
+      volumes:
+        - name: flask-secret
+          flexVolume:
+            options:
+              keyvaultname: "atat-vault-test"
+              keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY"
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: atst-beat
+spec:
+  template:
+    spec:
+      volumes:
+        - name: flask-secret
+          flexVolume:
+            options:
+              keyvaultname: "atat-vault-test"
+              keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY"
+---
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: crls
+spec:
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          volumes:
+            - name: flask-secret
+              flexVolume:
+                options:
+                  keyvaultname: "atat-vault-test"
+                  keyvaultobjectnames: "AZURE-STORAGE-KEY;MAIL-PASSWORD;PGPASSWORD;REDIS-PASSWORD;SECRET-KEY"

deploy/overlays/cloudzero-dev/json_ports.yml

@@ -0,0 +1,12 @@
+- op: replace
+  path: /spec/template/spec/containers/1/ports/0/containerPort
+  value: 9342
+- op: replace
+  path: /spec/template/spec/containers/1/ports/1/containerPort
+  value: 9442
+- op: replace
+  path: /spec/template/spec/containers/1/ports/2/containerPort
+  value: 9343
+- op: replace
+  path: /spec/template/spec/containers/1/ports/3/containerPort
+  value: 9443

deploy/overlays/cloudzero-dev/kustomization.yaml

@@ -0,0 +1,18 @@
+namespace: staging
+bases:
+  - ../../azure/
+resources:
+  - namespace.yml
+  - reset-cron-job.yml
+patchesStrategicMerge:
+  - replica_count.yml
+  - ports.yml
+  - envvars.yml
+  - flex_vol.yml
+patchesJson6902:
+  - target:
+      group: extensions
+      version: v1beta1
+      kind: Deployment
+      name: atst
+    path: json_ports.yml

deploy/overlays/cloudzero-dev/namespace.yml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: staging

deploy/overlays/cloudzero-dev/ports.yml

@@ -0,0 +1,28 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: atst-main
+spec:
+  loadBalancerIP: ""
+  ports:
+  - port: 80
+    targetPort: 9342
+    name: http
+  - port: 443
+    targetPort: 9442
+    name: https
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: atst-auth
+spec:
+  loadBalancerIP: ""
+  ports:
+  - port: 80
+    targetPort: 9343
+    name: http
+  - port: 443
+    targetPort: 9443
+    name: https

deploy/overlays/cloudzero-dev/replica_count.yml

@@ -0,0 +1,14 @@
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: atst
+spec:
+  replicas: 2
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: atst-worker
+spec:
+  replicas: 1

deploy/overlays/cloudzero-dev/reset-cron-job.yml

@@ -0,0 +1,46 @@
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: reset-db
+  namespace: atat
+spec:
+  schedule: "0 4 * * *"
+  concurrencyPolicy: Replace
+  successfulJobsHistoryLimit: 1
+  jobTemplate:
+    spec:
+      template:
+        metadata:
+          labels:
+            app: atst
+            role: reset-db
+            aadpodidbinding: atat-kv-id-binding
+        spec:
+          restartPolicy: OnFailure
+          containers:
+          - name: reset
+            image: $CONTAINER_IMAGE
+            command: [
+              "/bin/sh", "-c"
+            ]
+            args: [
+              "/opt/atat/atst/.venv/bin/python",
+              "/opt/atat/atst/script/reset_database.py"
+            ]
+            envFrom:
+            - configMapRef:
+                name: atst-worker-envvars
+            volumeMounts:
+              - name: flask-secret
+                mountPath: "/config"
+          volumes:
+            - name: flask-secret
+              flexVolume:
+                driver: "azure/kv"
+                options:
+                  usepodidentity: "true"
+                  keyvaultname: "atat-vault-test"
+                  keyvaultobjectnames: "staging-AZURE-STORAGE-KEY;staging-MAIL-PASSWORD;staging-PGPASSWORD;staging-REDIS-PASSWORD;staging-SECRET-KEY"
+                  keyvaultobjectaliases: "AZURE_STORAGE_KEY;MAIL_PASSWORD;PGPASSWORD;REDIS_PASSWORD;SECRET_KEY"
+                  keyvaultobjecttypes: "secret;secret;secret;secret;key"
+                  tenantid: $TENANT_ID

terraform/modules/k8s/main.tf

@@ -15,11 +15,12 @@ resource "azurerm_kubernetes_cluster" "k8s" {
   }
 
   default_node_pool {
-    name            = "default"
+    name                  = "default"
-    vm_size         = "Standard_D1_v2"
+    vm_size               = "Standard_D1_v2"
-    os_disk_size_gb = 30
+    os_disk_size_gb       = 30
-    vnet_subnet_id  = var.vnet_subnet_id
+    vnet_subnet_id        = var.vnet_subnet_id
-    node_count      = 1
+    node_count            = 1
+    enable_node_public_ip = true # Nodes need a public IP for external resources. FIXME: Switch to NAT Gateway if its available in our subscription
   }
 
   lifecycle {

terraform/providers/dev/variables.tf

@@ -47,13 +47,14 @@ variable "route_tables" {
   type        = map
   default = {
     public  = "Internet"
-    private = "VnetLocal"
+    private = "Internet"
+    #private = "VnetLocal"
   }
 }
 
 variable "dns_servers" {
   type    = list
-  default = ["10.1.2.4", "10.1.2.5"]
+  default = []
 }
 
 variable "k8s_node_size" {