Add additional k8s config for AWS and Azure.

- remove unused k8s environment config
- experimental config for azure and aws
- use nginx for proxy and add rq worker pod
This commit is contained in:
dandds
2019-07-15 06:10:34 -04:00
parent d32ce93271
commit 52829a1236
13 changed files with 117 additions and 451 deletions

View File

@@ -1,26 +0,0 @@
kind: Role
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
namespace: atat-test
name: atat-sample-update
rules:
- apiGroups: [""]
resources: ["pods"]
verbs: ["get", "list"]
- apiGroups: [""]
resources: ["pods/exec"]
verbs: ["create"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: atst-sample-role-binding
namespace: atat-test
subjects:
- kind: ServiceAccount
name: atat-deployer
namespace: atat
roleRef:
kind: Role
name: atat-sample-update
apiGroup: rbac.authorization.k8s.io

View File

@@ -1,35 +0,0 @@
---
apiVersion: v1
kind: ConfigMap
metadata:
name: atst-config
namespace: atat-test
data:
uwsgi-config: |-
[uwsgi]
callable = app
module = app
socket = /var/run/uwsgi/uwsgi.socket
plugin = python3
plugin = logfile
virtualenv = /opt/atat/atst/.venv
chmod-socket = 666
; logger config
; application logs: log without modifying
logger = secondlogger stdio
log-route = secondlogger atst
log-encoder = format:secondlogger ${msg}
; default uWSGI messages (start, stop, etc.)
logger = default stdio
log-route = default ^((?!atst).)*$
log-encoder = json:default {"timestamp":"${strftime:%%FT%%T}","source":"uwsgi","severity":"DEBUG","message":"${msg}"}
log-encoder = nl
; uWSGI request logs
logger-req = stdio
log-format = request_id=%(var.HTTP_X_REQUEST_ID), pid=%(pid), remote_add=%(addr), request=%(method) %(uri), status=%(status), body_bytes_sent=%(rsize), referer=%(referer), user_agent=%(uagent), http_x_forwarded_for=%(var.HTTP_X_FORWARDED_FOR)
log-req-encoder = json {"timestamp":"${strftime:%%FT%%T}","source":"req","severity":"INFO","message":"${msg}"}
log-req-encoder = nl

View File

@@ -1,15 +0,0 @@
---
apiVersion: v1
kind: ConfigMap
metadata:
name: atst-envvars
namespace: atat-test
data:
TZ: UTC
FLASK_ENV: dev
OVERRIDE_CONFIG_FULLPATH: /opt/atat/atst/atst-overrides.ini
UWSGI_CONFIG_FULLPATH: /opt/atat/atst/uwsgi-config.ini
RQ_QUEUES: atat-test
CRL_STORAGE_PROVIDER: CLOUDFILES
LOG_JSON: "true"
DEBUG: "false"

View File

@@ -1,99 +0,0 @@
---
apiVersion: v1
kind: ConfigMap
metadata:
name: atst-nginx
namespace: atat-test
data:
nginx-config: |-
server {
access_log /var/log/nginx/access.log json;
server_name test.atat.code.mil;
listen 8442;
listen [::]:8442 ipv6only=on;
if ($http_x_forwarded_proto != 'https') {
return 301 https://$host$request_uri;
}
location /login-redirect {
return 301 https://auth-test.atat.code.mil$request_uri;
}
location /login-dev {
try_files $uri @appbasicauth;
}
location / {
try_files $uri @app;
}
location @app {
include uwsgi_params;
uwsgi_pass unix:///var/run/uwsgi/uwsgi.socket;
uwsgi_param HTTP_X_REQUEST_ID $request_id;
}
location @appbasicauth {
include uwsgi_params;
uwsgi_pass unix:///var/run/uwsgi/uwsgi.socket;
auth_basic "Developer Access";
auth_basic_user_file /etc/nginx/.htpasswd;
uwsgi_param HTTP_X_REQUEST_ID $request_id;
}
}
server {
access_log /var/log/nginx/access.log json;
server_name auth-test.atat.code.mil;
listen 8443 ssl;
listen [::]:8443 ssl ipv6only=on;
# SSL server certificate and private key
ssl_certificate /etc/ssl/private/auth.atat.crt;
ssl_certificate_key /etc/ssl/private/auth.atat.key;
# Set SSL protocols, ciphers, and related options
ssl_protocols TLSv1.3 TLSv1.2;
ssl_ciphers 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256';
ssl_prefer_server_ciphers on;
ssl_ecdh_curve secp384r1;
ssl_dhparam /etc/ssl/dhparam.pem;
# SSL session options
ssl_session_timeout 4h;
ssl_session_cache shared:SSL:10m; # 1mb = ~4000 sessions
ssl_session_tickets off;
# OCSP Stapling
ssl_stapling on;
ssl_stapling_verify on;
resolver 8.8.8.8 8.8.4.4;
# Request and validate client certificate
ssl_verify_client on;
ssl_verify_depth 10;
ssl_client_certificate /etc/ssl/client-ca-bundle.pem;
# Guard against HTTPS -> HTTP downgrade
add_header Strict-Transport-Security "max-age=31536000; includeSubDomains; always";
location / {
return 301 https://test.atat.code.mil$request_uri;
}
location /login-redirect {
try_files $uri @app;
}
location @app {
include uwsgi_params;
uwsgi_pass unix:///var/run/uwsgi/uwsgi.socket;
uwsgi_param HTTP_X_SSL_CLIENT_VERIFY $ssl_client_verify;
uwsgi_param HTTP_X_SSL_CLIENT_CERT $ssl_client_raw_cert;
uwsgi_param HTTP_X_SSL_CLIENT_S_DN $ssl_client_s_dn;
uwsgi_param HTTP_X_SSL_CLIENT_S_DN_LEGACY $ssl_client_s_dn_legacy;
uwsgi_param HTTP_X_SSL_CLIENT_I_DN $ssl_client_i_dn;
uwsgi_param HTTP_X_SSL_CLIENT_I_DN_LEGACY $ssl_client_i_dn_legacy;
uwsgi_param HTTP_X_REQUEST_ID $request_id;
}
}
nginx-json-log-config: |-
log_format json escape=json
'{'
'"timestamp":"$time_iso8601",'
'"msec":"$msec",'
'"request_id":"$request_id",'
'"remote_addr":"$remote_addr",'
'"remote_user":"$remote_user",'
'"request":"$request",'
'"status":$status,'
'"body_bytes_sent":$body_bytes_sent,'
'"referer":"$http_referer",'
'"user_agent":"$http_user_agent",'
'"http_x_forwarded_for":"$http_x_forwarded_for"'
'}';

View File

@@ -1,10 +0,0 @@
---
apiVersion: v1
kind: ConfigMap
metadata:
name: atst-worker-envvars
namespace: atat-test
data:
TZ: UTC
DISABLE_CRL_CHECK: "True"
CRL_STORAGE_PROVIDER: CLOUDFILES

View File

@@ -1,236 +0,0 @@
---
apiVersion: v1
kind: Namespace
metadata:
name: atat-test
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
labels:
app: atst
name: atst
namespace: atat-test
spec:
selector:
matchLabels:
role: web
replicas: 1
strategy:
type: RollingUpdate
template:
metadata:
labels:
app: atst
role: web
spec:
securityContext:
fsGroup: 101
containers:
- name: atst
image: registry.atat.code.mil:443/atst-prod:575cfce5
resources:
requests:
memory: "2500Mi"
envFrom:
- configMapRef:
name: atst-envvars
volumeMounts:
- name: atst-config
mountPath: "/opt/atat/atst/atst-overrides.ini"
subPath: atst-overrides.ini
- name: uwsgi-config
mountPath: "/opt/atat/atst/uwsgi-config.ini"
subPath: uwsgi-config.ini
- name: uwsgi-socket-dir
mountPath: "/var/run/uwsgi"
- name: atst-nginx
image: nginx:alpine
ports:
- containerPort: 8442
name: http
- containerPort: 8443
name: https
volumeMounts:
- name: nginx-auth-tls
mountPath: "/etc/ssl/private"
- name: nginx-client-ca-bundle
mountPath: "/etc/ssl/client-ca-bundle.pem"
subPath: client-ca-bundle.pem
- name: nginx-config
mountPath: "/etc/nginx/conf.d/atst.conf"
subPath: atst.conf
- name: nginx-config
mountPath: "/etc/nginx/conf.d/00json_log.conf"
subPath: 00json_log.conf
- name: nginx-dhparam
mountPath: "/etc/ssl/dhparam.pem"
subPath: dhparam.pem
- name: nginx-htpasswd
mountPath: "/etc/nginx/.htpasswd"
subPath: .htpasswd
- name: uwsgi-socket-dir
mountPath: "/var/run/uwsgi"
imagePullSecrets:
- name: regcred
volumes:
- name: atst-config
secret:
secretName: atst-config-ini
items:
- key: override.ini
path: atst-overrides.ini
mode: 0644
- name: nginx-auth-tls
secret:
secretName: atst-auth-test-ingress-tls
items:
- key: tls.crt
path: auth.atat.crt
mode: 0644
- key: tls.key
path: auth.atat.key
mode: 0640
- name: nginx-client-ca-bundle
secret:
secretName: nginx-client-ca-bundle
items:
- key: client-ca-bundle.pem
path: client-ca-bundle.pem
mode: 0666
- name: nginx-config
configMap:
name: atst-nginx
items:
- key: nginx-config
path: atst.conf
- key: nginx-json-log-config
path: 00json_log.conf
- name: nginx-dhparam
secret:
secretName: dhparam-4096
items:
- key: dhparam.pem
path: dhparam.pem
mode: 0640
- name: nginx-htpasswd
secret:
secretName: atst-nginx-htpasswd
items:
- key: htpasswd
path: .htpasswd
mode: 0640
- name: uwsgi-config
configMap:
name: atst-config
items:
- key: uwsgi-config
path: uwsgi-config.ini
mode: 0644
- name: uwsgi-socket-dir
emptyDir:
medium: Memory
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
labels:
app: atst
name: atst-worker
namespace: atat-test
spec:
selector:
matchLabels:
role: worker
replicas: 1
strategy:
type: RollingUpdate
template:
metadata:
labels:
app: atst
role: worker
spec:
securityContext:
fsGroup: 101
containers:
- name: atst-worker
image: registry.atat.code.mil:443/atst-prod:575cfce5
args: ["/bin/bash", "-c", "/opt/atat/atst/script/rq_worker"]
resources:
requests:
memory: "500Mi"
envFrom:
- configMapRef:
name: atst-envvars
- configMapRef:
name: atst-worker-envvars
volumeMounts:
- name: atst-config
mountPath: "/opt/atat/atst/atst-overrides.ini"
subPath: atst-overrides.ini
imagePullSecrets:
- name: regcred
volumes:
- name: atst-config
secret:
secretName: atst-config-ini
items:
- key: override.ini
path: atst-overrides.ini
mode: 0644
---
apiVersion: v1
kind: Service
metadata:
labels:
app: atst
name: atst
namespace: atat-test
spec:
ports:
- name: http
port: 80
targetPort: 8442
selector:
role: web
---
apiVersion: v1
kind: Service
metadata:
labels:
app: atst
name: atst-auth
namespace: atat-test
spec:
type: NodePort
ports:
- name: https
protocol: TCP
nodePort: 32711
port: 8443
selector:
role: web
---
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: atst
namespace: atat-test
annotations:
kubernetes.io/tls-acme: "true"
kubernetes.io/ingress.class: "nginx"
nginx.ingress.kubernetes.io/proxy-body-size: 10m
spec:
tls:
- secretName: atst-test-ingress-tls
hosts:
- test.atat.code.mil
rules:
- host: test.atat.code.mil
http:
paths:
- path: /
backend:
serviceName: atst
servicePort: 80

View File

@@ -1,35 +0,0 @@
---
apiVersion: v1
kind: ConfigMap
metadata:
name: atst-config
namespace: atat-uat
data:
uwsgi-config: |-
[uwsgi]
callable = app
module = app
socket = /var/run/uwsgi/uwsgi.socket
plugin = python3
plugin = logfile
virtualenv = /opt/atat/atst/.venv
chmod-socket = 666
; logger config
; application logs: log without modifying
logger = secondlogger stdio
log-route = secondlogger atst
log-encoder = format:secondlogger ${msg}
; default uWSGI messages (start, stop, etc.)
logger = default stdio
log-route = default ^((?!atst).)*$
log-encoder = json:default {"timestamp":"${strftime:%%FT%%T}","source":"uwsgi","severity":"DEBUG","message":"${msg}"}
log-encoder = nl
; uWSGI request logs
logger-req = stdio
log-format = request_id=%(var.HTTP_X_REQUEST_ID), pid=%(pid), remote_add=%(addr), request=%(method) %(uri), status=%(status), body_bytes_sent=%(rsize), referer=%(referer), user_agent=%(uagent), http_x_forwarded_for=%(var.HTTP_X_FORWARDED_FOR)
log-req-encoder = json {"timestamp":"${strftime:%%FT%%T}","source":"req","severity":"INFO","message":"${msg}"}
log-req-encoder = nl

View File

@@ -1,14 +0,0 @@
---
apiVersion: v1
kind: ConfigMap
metadata:
name: atst-envvars
namespace: atat-uat
data:
TZ: UTC
FLASK_ENV: dev
OVERRIDE_CONFIG_FULLPATH: /opt/atat/atst/atst-overrides.ini
UWSGI_CONFIG_FULLPATH: /opt/atat/atst/uwsgi-config.ini
RQ_QUEUES: atat-uat
CRL_STORAGE_PROVIDER: CLOUDFILES
LOG_JSON: "true"

View File

@@ -1,99 +0,0 @@
---
apiVersion: v1
kind: ConfigMap
metadata:
name: atst-nginx
namespace: atat-uat
data:
nginx-config: |-
server {
access_log /var/log/nginx/access.log json;
server_name uat.atat.code.mil;
listen 8442;
listen [::]:8442 ipv6only=on;
if ($http_x_forwarded_proto != 'https') {
return 301 https://$host$request_uri;
}
location /login-redirect {
return 301 https://auth-uat.atat.code.mil$request_uri;
}
location /login-dev {
try_files $uri @appbasicauth;
}
location / {
try_files $uri @app;
}
location @app {
include uwsgi_params;
uwsgi_pass unix:///var/run/uwsgi/uwsgi.socket;
uwsgi_param HTTP_X_REQUEST_ID $request_id;
}
location @appbasicauth {
include uwsgi_params;
uwsgi_pass unix:///var/run/uwsgi/uwsgi.socket;
auth_basic "Developer Access";
auth_basic_user_file /etc/nginx/.htpasswd;
uwsgi_param HTTP_X_REQUEST_ID $request_id;
}
}
server {
access_log /var/log/nginx/access.log json;
server_name auth-uat.atat.code.mil;
listen 8443 ssl;
listen [::]:8443 ssl ipv6only=on;
# SSL server certificate and private key
ssl_certificate /etc/ssl/private/auth.atat.crt;
ssl_certificate_key /etc/ssl/private/auth.atat.key;
# Set SSL protocols, ciphers, and related options
ssl_protocols TLSv1.3 TLSv1.2;
ssl_ciphers 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256';
ssl_prefer_server_ciphers on;
ssl_ecdh_curve secp384r1;
ssl_dhparam /etc/ssl/dhparam.pem;
# SSL session options
ssl_session_timeout 4h;
ssl_session_cache shared:SSL:10m; # 1mb = ~4000 sessions
ssl_session_tickets off;
# OCSP Stapling
ssl_stapling on;
ssl_stapling_verify on;
resolver 8.8.8.8 8.8.4.4;
# Request and validate client certificate
ssl_verify_client on;
ssl_verify_depth 10;
ssl_client_certificate /etc/ssl/client-ca-bundle.pem;
# Guard against HTTPS -> HTTP downgrade
add_header Strict-Transport-Security "max-age=31536000; includeSubDomains; always";
location / {
return 301 https://uat.atat.code.mil$request_uri;
}
location /login-redirect {
try_files $uri @app;
}
location @app {
include uwsgi_params;
uwsgi_pass unix:///var/run/uwsgi/uwsgi.socket;
uwsgi_param HTTP_X_SSL_CLIENT_VERIFY $ssl_client_verify;
uwsgi_param HTTP_X_SSL_CLIENT_CERT $ssl_client_raw_cert;
uwsgi_param HTTP_X_SSL_CLIENT_S_DN $ssl_client_s_dn;
uwsgi_param HTTP_X_SSL_CLIENT_S_DN_LEGACY $ssl_client_s_dn_legacy;
uwsgi_param HTTP_X_SSL_CLIENT_I_DN $ssl_client_i_dn;
uwsgi_param HTTP_X_SSL_CLIENT_I_DN_LEGACY $ssl_client_i_dn_legacy;
uwsgi_param HTTP_X_REQUEST_ID $request_id;
}
}
nginx-json-log-config: |-
log_format json escape=json
'{'
'"timestamp":"$time_iso8601",'
'"msec":"$msec",'
'"request_id":"$request_id",'
'"remote_addr":"$remote_addr",'
'"remote_user":"$remote_user",'
'"request":"$request",'
'"status":$status,'
'"body_bytes_sent":$body_bytes_sent,'
'"referer":"$http_referer",'
'"user_agent":"$http_user_agent",'
'"http_x_forwarded_for":"$http_x_forwarded_for"'
'}';

View File

@@ -1,10 +0,0 @@
---
apiVersion: v1
kind: ConfigMap
metadata:
name: atst-worker-envvars
namespace: atat-uat
data:
TZ: UTC
DISABLE_CRL_CHECK: "True"
CRL_STORAGE_PROVIDER: CLOUDFILES

View File

@@ -1,239 +0,0 @@
---
apiVersion: v1
kind: Namespace
metadata:
name: atat-uat
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
labels:
app: atst
name: atst
namespace: atat-uat
spec:
selector:
matchLabels:
role: web
replicas: 1
strategy:
type: RollingUpdate
template:
metadata:
labels:
app: atst
role: web
spec:
securityContext:
fsGroup: 101
containers:
- name: atst
image: registry.atat.code.mil:443/atst-prod:03ee3438
resources:
requests:
memory: "2500Mi"
envFrom:
- configMapRef:
name: atst-envvars
volumeMounts:
- name: atst-config
mountPath: "/opt/atat/atst/atst-overrides.ini"
subPath: atst-overrides.ini
- name: nginx-client-ca-bundle
mountPath: "/opt/atat/atst/ssl/server-certs/ca-chain.pem"
subPath: client-ca-bundle.pem
- name: uwsgi-config
mountPath: "/opt/atat/atst/uwsgi-config.ini"
subPath: uwsgi-config.ini
- name: uwsgi-socket-dir
mountPath: "/var/run/uwsgi"
- name: atst-nginx
image: nginx:alpine
ports:
- containerPort: 8442
name: http
- containerPort: 8443
name: https
volumeMounts:
- name: nginx-auth-tls
mountPath: "/etc/ssl/private"
- name: nginx-client-ca-bundle
mountPath: "/etc/ssl/client-ca-bundle.pem"
subPath: client-ca-bundle.pem
- name: nginx-config
mountPath: "/etc/nginx/conf.d/atst.conf"
subPath: atst.conf
- name: nginx-config
mountPath: "/etc/nginx/conf.d/00json_log.conf"
subPath: 00json_log.conf
- name: nginx-dhparam
mountPath: "/etc/ssl/dhparam.pem"
subPath: dhparam.pem
- name: nginx-htpasswd
mountPath: "/etc/nginx/.htpasswd"
subPath: .htpasswd
- name: uwsgi-socket-dir
mountPath: "/var/run/uwsgi"
imagePullSecrets:
- name: regcred
volumes:
- name: atst-config
secret:
secretName: atst-config-ini
items:
- key: override.ini
path: atst-overrides.ini
mode: 0644
- name: nginx-auth-tls
secret:
secretName: atst-auth-uat-ingress-tls
items:
- key: tls.crt
path: auth.atat.crt
mode: 0644
- key: tls.key
path: auth.atat.key
mode: 0640
- name: nginx-client-ca-bundle
secret:
secretName: nginx-client-ca-bundle
items:
- key: client-ca-bundle.pem
path: client-ca-bundle.pem
mode: 0666
- name: nginx-config
configMap:
name: atst-nginx
items:
- key: nginx-config
path: atst.conf
- key: nginx-json-log-config
path: 00json_log.conf
- name: nginx-dhparam
secret:
secretName: dhparam-4096
items:
- key: dhparam.pem
path: dhparam.pem
mode: 0640
- name: nginx-htpasswd
secret:
secretName: atst-nginx-htpasswd
items:
- key: htpasswd
path: .htpasswd
mode: 0640
- name: uwsgi-config
configMap:
name: atst-config
items:
- key: uwsgi-config
path: uwsgi-config.ini
mode: 0644
- name: uwsgi-socket-dir
emptyDir:
medium: Memory
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
labels:
app: atst
name: atst-worker
namespace: atat-uat
spec:
selector:
matchLabels:
role: worker
replicas: 1
strategy:
type: RollingUpdate
template:
metadata:
labels:
app: atst
role: worker
spec:
securityContext:
fsGroup: 101
containers:
- name: atst-worker
image: registry.atat.code.mil:443/atst-prod:03ee3438
args: ["/bin/bash", "-c", "/opt/atat/atst/script/rq_worker"]
resources:
requests:
memory: "500Mi"
envFrom:
- configMapRef:
name: atst-envvars
- configMapRef:
name: atst-worker-envvars
volumeMounts:
- name: atst-config
mountPath: "/opt/atat/atst/atst-overrides.ini"
subPath: atst-overrides.ini
imagePullSecrets:
- name: regcred
volumes:
- name: atst-config
secret:
secretName: atst-config-ini
items:
- key: override.ini
path: atst-overrides.ini
mode: 0644
---
apiVersion: v1
kind: Service
metadata:
labels:
app: atst
name: atst
namespace: atat-uat
spec:
ports:
- name: http
port: 80
targetPort: 8442
selector:
role: web
---
apiVersion: v1
kind: Service
metadata:
labels:
app: atst
name: atst-auth
namespace: atat-uat
spec:
type: NodePort
ports:
- name: https
protocol: TCP
nodePort: 32701
port: 8443
selector:
role: web
---
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: atst
namespace: atat-uat
annotations:
kubernetes.io/tls-acme: "true"
kubernetes.io/ingress.class: "nginx"
nginx.ingress.kubernetes.io/proxy-body-size: 10m
spec:
tls:
- secretName: atst-uat-ingress-tls
hosts:
- uat.atat.code.mil
rules:
- host: uat.atat.code.mil
http:
paths:
- path: /
backend:
serviceName: atst
servicePort: 80