Merge pull request #1109 from dod-ccpo/minikube-setup

Add setup script and configuration for Minikube cluster.

README.md

@@ -329,3 +329,51 @@ fi
 ```
 
 Also note that if the line number of a previously whitelisted secret changes, the whitelist file, `.secrets.baseline`, will be updated and needs to be committed.
+
+## Local Kubernetes Setup
+
+A modified version of the Kubernetes cluster can be deployed locally for
+testing and development purposes.
+
+It is strongly recommended that you backup your local K8s config (usually
+`~/.kube/config`) before launching Minikube for the first time.
+
+Before beginning:
+
+- install the [Docker CLI](https://docs.docker.com/v17.12/install/)
+- install [Minikube](https://kubernetes.io/docs/tasks/tools/install-minikube/)
+  (this will also require installing a Hypervisor, such as VirtualBox)
+
+### Setup
+
+Run
+
+```
+script/minikube_setup
+```
+
+Once the script exits successfully, run
+
+```
+minikube service list
+```
+
+### Access the site
+
+One of the two URLs given for the `atat-auth` service will load an HTTP version
+of the application.
+
+For HTTP basic auth, the username and password are both `minikube`.
+
+### Differences from the main config
+
+As of the time of writing, this setup does not include the following:
+
+- SSL/TLS or the complete DoD PKI
+- the cronjob for syncing CRLs and the peristent storage
+- production configuration
+
+In order for the application to run, the K8s config for Minikube includes an
+additional deployment resource called `datastores`. This includes Postgres
+and Redis containers. It also includes hard-coded versions of the K8s secrets
+used in the regular clusters.

deploy/minikube/atst-configmap.yml

@@ -0,0 +1,35 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: atst-config
+  namespace: atat
+data:
+  uwsgi-config: |-
+    [uwsgi]
+    callable = app
+    module = app
+    socket = /var/run/uwsgi/uwsgi.socket
+    plugin = python3
+    plugin = logfile
+    virtualenv = /opt/atat/atst/.venv
+    chmod-socket = 666
+
+    ; logger config
+
+    ; application logs: log without modifying
+    logger = secondlogger stdio
+    log-route = secondlogger atst
+    log-encoder = format:secondlogger ${msg}
+
+    ; default uWSGI messages (start, stop, etc.)
+    logger = default stdio
+    log-route = default ^((?!atst).)*$
+    log-encoder = json:default {"timestamp":"${strftime:%%FT%%T}","source":"uwsgi","severity":"DEBUG","message":"${msg}"}
+    log-encoder = nl
+
+    ; uWSGI request logs
+    logger-req = stdio
+    log-format = request_id=%(var.HTTP_X_REQUEST_ID), pid=%(pid), remote_add=%(addr), request=%(method) %(uri), status=%(status), body_bytes_sent=%(rsize), referer=%(referer), user_agent=%(uagent), http_x_forwarded_for=%(var.HTTP_X_FORWARDED_FOR)
+    log-req-encoder = json {"timestamp":"${strftime:%%FT%%T}","source":"req","severity":"INFO","message":"${msg}"}
+    log-req-encoder = nl

deploy/minikube/atst-envvars-configmap.yml

@@ -0,0 +1,15 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: atst-envvars
+  namespace: atat
+data:
+  TZ: UTC
+  FLASK_ENV: dev
+  OVERRIDE_CONFIG_FULLPATH: /opt/atat/atst/atst-overrides.ini
+  UWSGI_CONFIG_FULLPATH: /opt/atat/atst/uwsgi.ini
+  CRL_STORAGE_PROVIDER: CLOUDFILES
+  LOG_JSON: "true"
+  REDIS_URI: "redis://redis-svc:6379"
+  PGHOST: postgres-svc

deploy/minikube/atst-nginx-configmap.yml

@@ -0,0 +1,73 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: atst-nginx
+  namespace: atat
+data:
+  nginx-config: |-
+    server {
+        listen 8342;
+        server_name aws.atat.code.mil;
+        return 301 https://$host$request_uri;
+    }
+    server {
+        listen 8343;
+        server_name auth-aws.atat.code.mil;
+        return 301 https://$host$request_uri;
+    }
+    server {
+        server_name aws.atat.code.mil;
+        # access_log /var/log/nginx/access.log json;
+        listen 8442;
+        location /login-redirect {
+            return 301 https://auth-aws.atat.code.mil$request_uri;
+        }
+        location /login-dev {
+            try_files $uri @appbasicauth;
+        }
+        location / {
+            try_files $uri @app;
+        }
+        location @app {
+            include uwsgi_params;
+            uwsgi_pass unix:///var/run/uwsgi/uwsgi.socket;
+            uwsgi_param HTTP_X_REQUEST_ID $request_id;
+        }
+        location @appbasicauth {
+            include uwsgi_params;
+            uwsgi_pass unix:///var/run/uwsgi/uwsgi.socket;
+            auth_basic "Developer Access";
+            auth_basic_user_file /etc/nginx/.htpasswd;
+            uwsgi_param HTTP_X_REQUEST_ID $request_id;
+        }
+    }
+    server {
+        # access_log /var/log/nginx/access.log json;
+        server_name auth-aws.atat.code.mil;
+        listen 8443;
+        listen [::]:8443 ipv6only=on;
+        # Request and validate client certificate
+        ssl_verify_client on;
+        ssl_verify_depth 10;
+        ssl_client_certificate /etc/ssl/client-ca-bundle.pem;
+        # Guard against HTTPS -> HTTP downgrade
+        add_header Strict-Transport-Security "max-age=31536000; includeSubDomains; always";
+        location / {
+            return 301 https://aws.atat.code.mil$request_uri;
+        }
+        location /login-redirect {
+            try_files $uri @app;
+        }
+        location @app {
+            include uwsgi_params;
+            uwsgi_pass unix:///var/run/uwsgi/uwsgi.socket;
+            uwsgi_param HTTP_X_SSL_CLIENT_VERIFY $ssl_client_verify;
+            uwsgi_param HTTP_X_SSL_CLIENT_CERT $ssl_client_raw_cert;
+            uwsgi_param HTTP_X_SSL_CLIENT_S_DN $ssl_client_s_dn;
+            uwsgi_param HTTP_X_SSL_CLIENT_S_DN_LEGACY $ssl_client_s_dn_legacy;
+            uwsgi_param HTTP_X_SSL_CLIENT_I_DN $ssl_client_i_dn;
+            uwsgi_param HTTP_X_SSL_CLIENT_I_DN_LEGACY $ssl_client_i_dn_legacy;
+            uwsgi_param HTTP_X_REQUEST_ID $request_id;
+        }
+    }

deploy/minikube/atst-worker-envvars-configmap.yml

@@ -0,0 +1,12 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: atst-worker-envvars
+  namespace: atat
+data:
+  TZ: UTC
+  DISABLE_CRL_CHECK: "True"
+  CRL_STORAGE_PROVIDER: CLOUDFILES
+  REDIS_URI: "redis://redis-svc:6379"
+  PGHOST: postgres-svc

deploy/minikube/datastores.yml

@@ -0,0 +1,61 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: db-cache
+  name: datastores
+  namespace: atat
+spec:
+  selector:
+    matchLabels:
+      app: db-cache
+  replicas: 1
+  strategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: db-cache
+    spec:
+      securityContext:
+        fsGroup: 101
+      containers:
+        - name: postgres
+          image: postgres:11-alpine
+          imagePullPolicy: Never
+          ports:
+          - containerPort: 5432
+        - name: redis
+          image: redis:5.0-alpine
+          imagePullPolicy: Never
+          ports:
+          - containerPort: 6379
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: postgres-svc
+  namespace: atat
+spec:
+  ports:
+  - name: db-port
+    protocol: "TCP"
+    port: 5432
+    targetPort: 5432
+  selector:
+    app: db-cache
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: redis-svc
+  namespace: atat
+spec:
+  ports:
+  - name: cache-port
+    protocol: "TCP"
+    port: 6379
+    targetPort: 6379
+  selector:
+    app: db-cache

deploy/minikube/minikube.yml

@@ -0,0 +1,235 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: atst
+  name: atst
+  namespace: atat
+spec:
+  selector:
+    matchLabels:
+      role: web
+  replicas: 1
+  strategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: atst
+        role: web
+    spec:
+      securityContext:
+        fsGroup: 101
+      containers:
+        - name: atst
+          image: atat:latest
+          imagePullPolicy: Never
+          envFrom:
+          - configMapRef:
+              name: atst-envvars
+          volumeMounts:
+            - name: atst-config
+              mountPath: "/opt/atat/atst/atst-overrides.ini"
+              subPath: atst-overrides.ini
+            - name: nginx-client-ca-bundle
+              mountPath: "/opt/atat/atst/ssl/server-certs/ca-chain.pem"
+              subPath: client-ca-bundle.pem
+            - name: uwsgi-socket-dir
+              mountPath: "/var/run/uwsgi"
+        - name: nginx
+          image: nginx:alpine
+          imagePullPolicy: Never
+          ports:
+            - containerPort: 8342
+              name: main-upgrade
+            - containerPort: 8442
+              name: main
+            - containerPort: 8343
+              name: auth-upgrade
+            - containerPort: 8443
+              name: auth
+          volumeMounts:
+            - name: nginx-config
+              mountPath: "/etc/nginx/conf.d/atst.conf"
+              subPath: atst.conf
+            - name: uwsgi-socket-dir
+              mountPath: "/var/run/uwsgi"
+            - name: nginx-htpasswd
+              mountPath: "/etc/nginx/.htpasswd"
+              subPath: .htpasswd
+            - name: nginx-client-ca-bundle
+              mountPath: "/etc/ssl/"
+      volumes:
+        - name: atst-config
+          secret:
+            secretName: atst-config-ini
+            items:
+            - key: override.ini
+              path: atst-overrides.ini
+              mode: 0644
+        - name: nginx-client-ca-bundle
+          secret:
+            secretName: nginx-client-ca-bundle
+            items:
+            - key: client-ca-bundle.pem
+              path: client-ca-bundle.pem
+              mode: 0666
+        - name: nginx-config
+          configMap:
+            name: atst-nginx
+            items:
+            - key: nginx-config
+              path: atst.conf
+        - name: uwsgi-socket-dir
+          emptyDir:
+            medium: Memory
+        - name: nginx-htpasswd
+          secret:
+            secretName: atst-nginx-htpasswd
+            items:
+            - key: htpasswd
+              path: .htpasswd
+              mode: 0640
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: atst
+  name: atst-worker
+  namespace: atat
+spec:
+  selector:
+    matchLabels:
+      role: worker
+  replicas: 1
+  strategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: atst
+        role: worker
+    spec:
+      securityContext:
+        fsGroup: 101
+      containers:
+        - name: atst-worker
+          image: atat:latest
+          imagePullPolicy: Never
+          args: [
+            "/opt/atat/atst/.venv/bin/python",
+            "/opt/atat/atst/.venv/bin/celery",
+            "-A",
+            "celery_worker.celery",
+            "worker",
+            "--loglevel=info"
+          ]
+          envFrom:
+          - configMapRef:
+              name: atst-envvars
+          - configMapRef:
+              name: atst-worker-envvars
+          volumeMounts:
+            - name: atst-config
+              mountPath: "/opt/atat/atst/atst-overrides.ini"
+              subPath: atst-overrides.ini
+      volumes:
+        - name: atst-config
+          secret:
+            secretName: atst-config-ini
+            items:
+            - key: override.ini
+              path: atst-overrides.ini
+              mode: 0644
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: atst
+  name: atst-beat
+  namespace: atat
+spec:
+  selector:
+    matchLabels:
+      role: beat
+  replicas: 1
+  strategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: atst
+        role: beat
+    spec:
+      securityContext:
+        fsGroup: 101
+      containers:
+        - name: atst-beat
+          image: atat:latest
+          imagePullPolicy: Never
+          args: [
+            "/opt/atat/atst/.venv/bin/python",
+            "/opt/atat/atst/.venv/bin/celery",
+            "-A",
+            "celery_worker.celery",
+            "beat",
+            "--loglevel=info"
+          ]
+          envFrom:
+          - configMapRef:
+              name: atst-envvars
+          - configMapRef:
+              name: atst-worker-envvars
+          volumeMounts:
+            - name: atst-config
+              mountPath: "/opt/atat/atst/atst-overrides.ini"
+              subPath: atst-overrides.ini
+      volumes:
+        - name: atst-config
+          secret:
+            secretName: atst-config-ini
+            items:
+            - key: override.ini
+              path: atst-overrides.ini
+              mode: 0644
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: atst
+  name: atst-main
+  namespace: atat
+spec:
+  ports:
+  - port: 80
+    targetPort: 8342
+    name: http-main
+  - port: 443
+    targetPort: 8442
+    name: https-main
+  selector:
+    role: web
+  type: LoadBalancer
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: atst
+  name: atst-auth
+  namespace: atat
+spec:
+  ports:
+  - port: 80
+    targetPort: 8343
+    name: http-auth
+  - port: 443
+    targetPort: 8443
+    name: https-auth
+  selector:
+    role: web
+  type: LoadBalancer

script/minikube_setup

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# script/minikube_setup: Set up local AT-AT cluster on Minikube
+
+source "$(dirname "${0}")"/../script/include/global_header.inc.sh
+
+output_divider "Start Minikube"
+minikube start
+
+output_divider "Use Minikube Docker environment"
+eval $(minikube docker-env)
+
+output_divider "Build AT-AT Docker image for Minikube registry"
+docker build . -t atat:latest
+
+output_divider "Pull images for AT-AT cluster"
+docker pull redis:5.0-alpine
+docker pull postgres:11-alpine
+docker pull nginx:alpine
+
+output_divider "Apply AT-AT Kubernetes config to Minikube cluster"
+kubectl --context=minikube create namespace atat
+kubectl --context=minikube apply -f deploy/minikube/
+
+output_divider "Create database and apply migrations"
+# wait for the datastore deployment to become available
+kubectl --context=minikube -n atat wait --for=condition=Available deployment/datastores
+# postgres isn't necessarily running as soon as the pod is available, so wait a few
+sleep 3
+DB_POD=$(kubectl --context=minikube -n atat get pods -l app=db-cache -o custom-columns=NAME:.metadata.name --no-headers | sed -n 1p)
+ATST_POD=$(kubectl --context=minikube -n atat get pods -l app=atst -o custom-columns=NAME:.metadata.name --no-headers | sed -n 1p)
+kubectl --context=minikube -n atat exec -it $DB_POD -c postgres -- createdb -U postgres atat
+kubectl --context=minikube -n atat exec -it $ATST_POD -c atst -- .venv/bin/python .venv/bin/alembic upgrade head