Add additional k8s config for AWS and Azure.

- remove unused k8s environment config
- experimental config for azure and aws
- use nginx for proxy and add rq worker pod
This commit is contained in:
dandds 2019-07-15 06:10:34 -04:00
parent d32ce93271
commit 52829a1236
13 changed files with 117 additions and 451 deletions

View File

@ -1,26 +0,0 @@
kind: Role
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
namespace: atat-test
name: atat-sample-update
rules:
- apiGroups: [""]
resources: ["pods"]
verbs: ["get", "list"]
- apiGroups: [""]
resources: ["pods/exec"]
verbs: ["create"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: atst-sample-role-binding
namespace: atat-test
subjects:
- kind: ServiceAccount
name: atat-deployer
namespace: atat
roleRef:
kind: Role
name: atat-sample-update
apiGroup: rbac.authorization.k8s.io

View File

@ -1,99 +0,0 @@
---
apiVersion: v1
kind: ConfigMap
metadata:
name: atst-nginx
namespace: atat-test
data:
nginx-config: |-
server {
access_log /var/log/nginx/access.log json;
server_name test.atat.code.mil;
listen 8442;
listen [::]:8442 ipv6only=on;
if ($http_x_forwarded_proto != 'https') {
return 301 https://$host$request_uri;
}
location /login-redirect {
return 301 https://auth-test.atat.code.mil$request_uri;
}
location /login-dev {
try_files $uri @appbasicauth;
}
location / {
try_files $uri @app;
}
location @app {
include uwsgi_params;
uwsgi_pass unix:///var/run/uwsgi/uwsgi.socket;
uwsgi_param HTTP_X_REQUEST_ID $request_id;
}
location @appbasicauth {
include uwsgi_params;
uwsgi_pass unix:///var/run/uwsgi/uwsgi.socket;
auth_basic "Developer Access";
auth_basic_user_file /etc/nginx/.htpasswd;
uwsgi_param HTTP_X_REQUEST_ID $request_id;
}
}
server {
access_log /var/log/nginx/access.log json;
server_name auth-test.atat.code.mil;
listen 8443 ssl;
listen [::]:8443 ssl ipv6only=on;
# SSL server certificate and private key
ssl_certificate /etc/ssl/private/auth.atat.crt;
ssl_certificate_key /etc/ssl/private/auth.atat.key;
# Set SSL protocols, ciphers, and related options
ssl_protocols TLSv1.3 TLSv1.2;
ssl_ciphers 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256';
ssl_prefer_server_ciphers on;
ssl_ecdh_curve secp384r1;
ssl_dhparam /etc/ssl/dhparam.pem;
# SSL session options
ssl_session_timeout 4h;
ssl_session_cache shared:SSL:10m; # 1mb = ~4000 sessions
ssl_session_tickets off;
# OCSP Stapling
ssl_stapling on;
ssl_stapling_verify on;
resolver 8.8.8.8 8.8.4.4;
# Request and validate client certificate
ssl_verify_client on;
ssl_verify_depth 10;
ssl_client_certificate /etc/ssl/client-ca-bundle.pem;
# Guard against HTTPS -> HTTP downgrade
add_header Strict-Transport-Security "max-age=31536000; includeSubDomains; always";
location / {
return 301 https://test.atat.code.mil$request_uri;
}
location /login-redirect {
try_files $uri @app;
}
location @app {
include uwsgi_params;
uwsgi_pass unix:///var/run/uwsgi/uwsgi.socket;
uwsgi_param HTTP_X_SSL_CLIENT_VERIFY $ssl_client_verify;
uwsgi_param HTTP_X_SSL_CLIENT_CERT $ssl_client_raw_cert;
uwsgi_param HTTP_X_SSL_CLIENT_S_DN $ssl_client_s_dn;
uwsgi_param HTTP_X_SSL_CLIENT_S_DN_LEGACY $ssl_client_s_dn_legacy;
uwsgi_param HTTP_X_SSL_CLIENT_I_DN $ssl_client_i_dn;
uwsgi_param HTTP_X_SSL_CLIENT_I_DN_LEGACY $ssl_client_i_dn_legacy;
uwsgi_param HTTP_X_REQUEST_ID $request_id;
}
}
nginx-json-log-config: |-
log_format json escape=json
'{'
'"timestamp":"$time_iso8601",'
'"msec":"$msec",'
'"request_id":"$request_id",'
'"remote_addr":"$remote_addr",'
'"remote_user":"$remote_user",'
'"request":"$request",'
'"status":$status,'
'"body_bytes_sent":$body_bytes_sent,'
'"referer":"$http_referer",'
'"user_agent":"$http_user_agent",'
'"http_x_forwarded_for":"$http_x_forwarded_for"'
'}';

View File

@ -1,99 +0,0 @@
---
apiVersion: v1
kind: ConfigMap
metadata:
name: atst-nginx
namespace: atat-uat
data:
nginx-config: |-
server {
access_log /var/log/nginx/access.log json;
server_name uat.atat.code.mil;
listen 8442;
listen [::]:8442 ipv6only=on;
if ($http_x_forwarded_proto != 'https') {
return 301 https://$host$request_uri;
}
location /login-redirect {
return 301 https://auth-uat.atat.code.mil$request_uri;
}
location /login-dev {
try_files $uri @appbasicauth;
}
location / {
try_files $uri @app;
}
location @app {
include uwsgi_params;
uwsgi_pass unix:///var/run/uwsgi/uwsgi.socket;
uwsgi_param HTTP_X_REQUEST_ID $request_id;
}
location @appbasicauth {
include uwsgi_params;
uwsgi_pass unix:///var/run/uwsgi/uwsgi.socket;
auth_basic "Developer Access";
auth_basic_user_file /etc/nginx/.htpasswd;
uwsgi_param HTTP_X_REQUEST_ID $request_id;
}
}
server {
access_log /var/log/nginx/access.log json;
server_name auth-uat.atat.code.mil;
listen 8443 ssl;
listen [::]:8443 ssl ipv6only=on;
# SSL server certificate and private key
ssl_certificate /etc/ssl/private/auth.atat.crt;
ssl_certificate_key /etc/ssl/private/auth.atat.key;
# Set SSL protocols, ciphers, and related options
ssl_protocols TLSv1.3 TLSv1.2;
ssl_ciphers 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256';
ssl_prefer_server_ciphers on;
ssl_ecdh_curve secp384r1;
ssl_dhparam /etc/ssl/dhparam.pem;
# SSL session options
ssl_session_timeout 4h;
ssl_session_cache shared:SSL:10m; # 1mb = ~4000 sessions
ssl_session_tickets off;
# OCSP Stapling
ssl_stapling on;
ssl_stapling_verify on;
resolver 8.8.8.8 8.8.4.4;
# Request and validate client certificate
ssl_verify_client on;
ssl_verify_depth 10;
ssl_client_certificate /etc/ssl/client-ca-bundle.pem;
# Guard against HTTPS -> HTTP downgrade
add_header Strict-Transport-Security "max-age=31536000; includeSubDomains; always";
location / {
return 301 https://uat.atat.code.mil$request_uri;
}
location /login-redirect {
try_files $uri @app;
}
location @app {
include uwsgi_params;
uwsgi_pass unix:///var/run/uwsgi/uwsgi.socket;
uwsgi_param HTTP_X_SSL_CLIENT_VERIFY $ssl_client_verify;
uwsgi_param HTTP_X_SSL_CLIENT_CERT $ssl_client_raw_cert;
uwsgi_param HTTP_X_SSL_CLIENT_S_DN $ssl_client_s_dn;
uwsgi_param HTTP_X_SSL_CLIENT_S_DN_LEGACY $ssl_client_s_dn_legacy;
uwsgi_param HTTP_X_SSL_CLIENT_I_DN $ssl_client_i_dn;
uwsgi_param HTTP_X_SSL_CLIENT_I_DN_LEGACY $ssl_client_i_dn_legacy;
uwsgi_param HTTP_X_REQUEST_ID $request_id;
}
}
nginx-json-log-config: |-
log_format json escape=json
'{'
'"timestamp":"$time_iso8601",'
'"msec":"$msec",'
'"request_id":"$request_id",'
'"remote_addr":"$remote_addr",'
'"remote_user":"$remote_user",'
'"request":"$request",'
'"status":$status,'
'"body_bytes_sent":$body_bytes_sent,'
'"referer":"$http_referer",'
'"user_agent":"$http_user_agent",'
'"http_x_forwarded_for":"$http_x_forwarded_for"'
'}';

View File

@ -3,7 +3,7 @@ apiVersion: v1
kind: ConfigMap
metadata:
name: atst-config
namespace: atat-uat
namespace: atat
data:
uwsgi-config: |-
[uwsgi]

View File

@ -3,12 +3,10 @@ apiVersion: v1
kind: ConfigMap
metadata:
name: atst-envvars
namespace: atat-uat
namespace: atat
data:
TZ: UTC
FLASK_ENV: dev
OVERRIDE_CONFIG_FULLPATH: /opt/atat/atst/atst-overrides.ini
UWSGI_CONFIG_FULLPATH: /opt/atat/atst/uwsgi-config.ini
RQ_QUEUES: atat-uat
UWSGI_CONFIG_FULLPATH: /opt/atat/atst/uwsgi.ini
CRL_STORAGE_PROVIDER: CLOUDFILES
LOG_JSON: "true"

View File

@ -0,0 +1,37 @@
---
apiVersion: v1
kind: ConfigMap
metadata:
name: atst-nginx
namespace: atat
data:
nginx-config: |-
server {
listen 8442;
server_name localhost;
location / {
try_files $uri @app;
}
location @app {
uwsgi_pass unix:///var/run/uwsgi/uwsgi.socket;
uwsgi_param QUERY_STRING $query_string;
uwsgi_param REQUEST_METHOD $request_method;
uwsgi_param CONTENT_TYPE $content_type;
uwsgi_param CONTENT_LENGTH $content_length;
uwsgi_param REQUEST_URI $request_uri;
uwsgi_param PATH_INFO $document_uri;
uwsgi_param DOCUMENT_ROOT $document_root;
uwsgi_param SERVER_PROTOCOL $server_protocol;
uwsgi_param REQUEST_SCHEME $scheme;
uwsgi_param HTTPS $https if_not_empty;
uwsgi_param REMOTE_ADDR $remote_addr;
uwsgi_param REMOTE_PORT $remote_port;
uwsgi_param SERVER_PORT $server_port;
uwsgi_param SERVER_NAME $server_name;
uwsgi_param HTTP_X_REQUEST_ID $request_id;
}
}

View File

@ -3,7 +3,7 @@ apiVersion: v1
kind: ConfigMap
metadata:
name: atst-worker-envvars
namespace: atat-uat
namespace: atat
data:
TZ: UTC
DISABLE_CRL_CHECK: "True"

View File

@ -2,7 +2,7 @@
apiVersion: v1
kind: Namespace
metadata:
name: atat-test
name: atat
---
apiVersion: extensions/v1beta1
kind: Deployment
@ -10,7 +10,7 @@ metadata:
labels:
app: atst
name: atst
namespace: atat-test
namespace: atat
spec:
selector:
matchLabels:
@ -28,10 +28,10 @@ spec:
fsGroup: 101
containers:
- name: atst
image: registry.atat.code.mil:443/atst-prod:575cfce5
image: 904153757533.dkr.ecr.us-east-2.amazonaws.com/atat:884d95ada21a5097f5c07f305d8e4e24d0f2a03f
resources:
requests:
memory: "2500Mi"
memory: "500Mi"
envFrom:
- configMapRef:
name: atst-envvars
@ -39,40 +39,22 @@ spec:
- name: atst-config
mountPath: "/opt/atat/atst/atst-overrides.ini"
subPath: atst-overrides.ini
- name: uwsgi-config
mountPath: "/opt/atat/atst/uwsgi-config.ini"
subPath: uwsgi-config.ini
- name: nginx-client-ca-bundle
mountPath: "/opt/atat/atst/ssl/server-certs/ca-chain.pem"
subPath: client-ca-bundle.pem
- name: uwsgi-socket-dir
mountPath: "/var/run/uwsgi"
- name: atst-nginx
- name: nginx
image: nginx:alpine
ports:
- containerPort: 8442
name: http
- containerPort: 8443
name: https
volumeMounts:
- name: nginx-auth-tls
mountPath: "/etc/ssl/private"
- name: nginx-client-ca-bundle
mountPath: "/etc/ssl/client-ca-bundle.pem"
subPath: client-ca-bundle.pem
- name: nginx-config
mountPath: "/etc/nginx/conf.d/atst.conf"
subPath: atst.conf
- name: nginx-config
mountPath: "/etc/nginx/conf.d/00json_log.conf"
subPath: 00json_log.conf
- name: nginx-dhparam
mountPath: "/etc/ssl/dhparam.pem"
subPath: dhparam.pem
- name: nginx-htpasswd
mountPath: "/etc/nginx/.htpasswd"
subPath: .htpasswd
- name: uwsgi-socket-dir
mountPath: "/var/run/uwsgi"
imagePullSecrets:
- name: regcred
volumes:
- name: atst-config
secret:
@ -81,16 +63,6 @@ spec:
- key: override.ini
path: atst-overrides.ini
mode: 0644
- name: nginx-auth-tls
secret:
secretName: atst-auth-test-ingress-tls
items:
- key: tls.crt
path: auth.atat.crt
mode: 0644
- key: tls.key
path: auth.atat.key
mode: 0640
- name: nginx-client-ca-bundle
secret:
secretName: nginx-client-ca-bundle
@ -104,29 +76,6 @@ spec:
items:
- key: nginx-config
path: atst.conf
- key: nginx-json-log-config
path: 00json_log.conf
- name: nginx-dhparam
secret:
secretName: dhparam-4096
items:
- key: dhparam.pem
path: dhparam.pem
mode: 0640
- name: nginx-htpasswd
secret:
secretName: atst-nginx-htpasswd
items:
- key: htpasswd
path: .htpasswd
mode: 0640
- name: uwsgi-config
configMap:
name: atst-config
items:
- key: uwsgi-config
path: uwsgi-config.ini
mode: 0644
- name: uwsgi-socket-dir
emptyDir:
medium: Memory
@ -137,7 +86,7 @@ metadata:
labels:
app: atst
name: atst-worker
namespace: atat-test
namespace: atat
spec:
selector:
matchLabels:
@ -155,8 +104,13 @@ spec:
fsGroup: 101
containers:
- name: atst-worker
image: registry.atat.code.mil:443/atst-prod:575cfce5
args: ["/bin/bash", "-c", "/opt/atat/atst/script/rq_worker"]
image: 904153757533.dkr.ecr.us-east-2.amazonaws.com/atat:884d95ada21a5097f5c07f305d8e4e24d0f2a03f
args: [
"/opt/atat/atst/.venv/bin/python",
"/opt/atat/atst/.venv/bin/flask",
"rq",
"worker"
]
resources:
requests:
memory: "500Mi"
@ -169,8 +123,6 @@ spec:
- name: atst-config
mountPath: "/opt/atat/atst/atst-overrides.ini"
subPath: atst-overrides.ini
imagePullSecrets:
- name: regcred
volumes:
- name: atst-config
secret:
@ -186,51 +138,11 @@ metadata:
labels:
app: atst
name: atst
namespace: atat-test
namespace: atat
spec:
ports:
- name: http
port: 80
- port: 80
targetPort: 8442
selector:
role: web
---
apiVersion: v1
kind: Service
metadata:
labels:
app: atst
name: atst-auth
namespace: atat-test
spec:
type: NodePort
ports:
- name: https
protocol: TCP
nodePort: 32711
port: 8443
selector:
role: web
---
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: atst
namespace: atat-test
annotations:
kubernetes.io/tls-acme: "true"
kubernetes.io/ingress.class: "nginx"
nginx.ingress.kubernetes.io/proxy-body-size: 10m
spec:
tls:
- secretName: atst-test-ingress-tls
hosts:
- test.atat.code.mil
rules:
- host: test.atat.code.mil
http:
paths:
- path: /
backend:
serviceName: atst
servicePort: 80
type: LoadBalancer

View File

@ -3,7 +3,7 @@ apiVersion: v1
kind: ConfigMap
metadata:
name: atst-config
namespace: atat-test
namespace: atat
data:
uwsgi-config: |-
[uwsgi]

View File

@ -3,13 +3,10 @@ apiVersion: v1
kind: ConfigMap
metadata:
name: atst-envvars
namespace: atat-test
namespace: atat
data:
TZ: UTC
FLASK_ENV: dev
OVERRIDE_CONFIG_FULLPATH: /opt/atat/atst/atst-overrides.ini
UWSGI_CONFIG_FULLPATH: /opt/atat/atst/uwsgi-config.ini
RQ_QUEUES: atat-test
UWSGI_CONFIG_FULLPATH: /opt/atat/atst/uwsgi.ini
CRL_STORAGE_PROVIDER: CLOUDFILES
LOG_JSON: "true"
DEBUG: "false"

View File

@ -0,0 +1,37 @@
---
apiVersion: v1
kind: ConfigMap
metadata:
name: atst-nginx
namespace: atat
data:
nginx-config: |-
server {
listen 8442;
server_name localhost;
location / {
try_files $uri @app;
}
location @app {
uwsgi_pass unix:///var/run/uwsgi/uwsgi.socket;
uwsgi_param QUERY_STRING $query_string;
uwsgi_param REQUEST_METHOD $request_method;
uwsgi_param CONTENT_TYPE $content_type;
uwsgi_param CONTENT_LENGTH $content_length;
uwsgi_param REQUEST_URI $request_uri;
uwsgi_param PATH_INFO $document_uri;
uwsgi_param DOCUMENT_ROOT $document_root;
uwsgi_param SERVER_PROTOCOL $server_protocol;
uwsgi_param REQUEST_SCHEME $scheme;
uwsgi_param HTTPS $https if_not_empty;
uwsgi_param REMOTE_ADDR $remote_addr;
uwsgi_param REMOTE_PORT $remote_port;
uwsgi_param SERVER_PORT $server_port;
uwsgi_param SERVER_NAME $server_name;
uwsgi_param HTTP_X_REQUEST_ID $request_id;
}
}

View File

@ -3,7 +3,7 @@ apiVersion: v1
kind: ConfigMap
metadata:
name: atst-worker-envvars
namespace: atat-test
namespace: atat
data:
TZ: UTC
DISABLE_CRL_CHECK: "True"

View File

@ -2,7 +2,7 @@
apiVersion: v1
kind: Namespace
metadata:
name: atat-uat
name: atat
---
apiVersion: extensions/v1beta1
kind: Deployment
@ -10,7 +10,7 @@ metadata:
labels:
app: atst
name: atst
namespace: atat-uat
namespace: atat
spec:
selector:
matchLabels:
@ -28,10 +28,10 @@ spec:
fsGroup: 101
containers:
- name: atst
image: registry.atat.code.mil:443/atst-prod:03ee3438
image: pwatat.azurecr.io/atat:884d95ada21a5097f5c07f305d8e4e24d0f2a03f
resources:
requests:
memory: "2500Mi"
memory: "500Mi"
envFrom:
- configMapRef:
name: atst-envvars
@ -42,40 +42,19 @@ spec:
- name: nginx-client-ca-bundle
mountPath: "/opt/atat/atst/ssl/server-certs/ca-chain.pem"
subPath: client-ca-bundle.pem
- name: uwsgi-config
mountPath: "/opt/atat/atst/uwsgi-config.ini"
subPath: uwsgi-config.ini
- name: uwsgi-socket-dir
mountPath: "/var/run/uwsgi"
- name: atst-nginx
- name: nginx
image: nginx:alpine
ports:
- containerPort: 8442
name: http
- containerPort: 8443
name: https
volumeMounts:
- name: nginx-auth-tls
mountPath: "/etc/ssl/private"
- name: nginx-client-ca-bundle
mountPath: "/etc/ssl/client-ca-bundle.pem"
subPath: client-ca-bundle.pem
- name: nginx-config
mountPath: "/etc/nginx/conf.d/atst.conf"
subPath: atst.conf
- name: nginx-config
mountPath: "/etc/nginx/conf.d/00json_log.conf"
subPath: 00json_log.conf
- name: nginx-dhparam
mountPath: "/etc/ssl/dhparam.pem"
subPath: dhparam.pem
- name: nginx-htpasswd
mountPath: "/etc/nginx/.htpasswd"
subPath: .htpasswd
- name: uwsgi-socket-dir
mountPath: "/var/run/uwsgi"
imagePullSecrets:
- name: regcred
volumes:
- name: atst-config
secret:
@ -84,16 +63,6 @@ spec:
- key: override.ini
path: atst-overrides.ini
mode: 0644
- name: nginx-auth-tls
secret:
secretName: atst-auth-uat-ingress-tls
items:
- key: tls.crt
path: auth.atat.crt
mode: 0644
- key: tls.key
path: auth.atat.key
mode: 0640
- name: nginx-client-ca-bundle
secret:
secretName: nginx-client-ca-bundle
@ -107,29 +76,6 @@ spec:
items:
- key: nginx-config
path: atst.conf
- key: nginx-json-log-config
path: 00json_log.conf
- name: nginx-dhparam
secret:
secretName: dhparam-4096
items:
- key: dhparam.pem
path: dhparam.pem
mode: 0640
- name: nginx-htpasswd
secret:
secretName: atst-nginx-htpasswd
items:
- key: htpasswd
path: .htpasswd
mode: 0640
- name: uwsgi-config
configMap:
name: atst-config
items:
- key: uwsgi-config
path: uwsgi-config.ini
mode: 0644
- name: uwsgi-socket-dir
emptyDir:
medium: Memory
@ -140,7 +86,7 @@ metadata:
labels:
app: atst
name: atst-worker
namespace: atat-uat
namespace: atat
spec:
selector:
matchLabels:
@ -158,8 +104,13 @@ spec:
fsGroup: 101
containers:
- name: atst-worker
image: registry.atat.code.mil:443/atst-prod:03ee3438
args: ["/bin/bash", "-c", "/opt/atat/atst/script/rq_worker"]
image: pwatat.azurecr.io/atat:884d95ada21a5097f5c07f305d8e4e24d0f2a03f
args: [
"/opt/atat/atst/.venv/bin/python",
"/opt/atat/atst/.venv/bin/flask",
"rq",
"worker"
]
resources:
requests:
memory: "500Mi"
@ -172,8 +123,6 @@ spec:
- name: atst-config
mountPath: "/opt/atat/atst/atst-overrides.ini"
subPath: atst-overrides.ini
imagePullSecrets:
- name: regcred
volumes:
- name: atst-config
secret:
@ -189,51 +138,11 @@ metadata:
labels:
app: atst
name: atst
namespace: atat-uat
namespace: atat
spec:
ports:
- name: http
port: 80
- port: 80
targetPort: 8442
selector:
role: web
---
apiVersion: v1
kind: Service
metadata:
labels:
app: atst
name: atst-auth
namespace: atat-uat
spec:
type: NodePort
ports:
- name: https
protocol: TCP
nodePort: 32701
port: 8443
selector:
role: web
---
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: atst
namespace: atat-uat
annotations:
kubernetes.io/tls-acme: "true"
kubernetes.io/ingress.class: "nginx"
nginx.ingress.kubernetes.io/proxy-body-size: 10m
spec:
tls:
- secretName: atst-uat-ingress-tls
hosts:
- uat.atat.code.mil
rules:
- host: uat.atat.code.mil
http:
paths:
- path: /
backend:
serviceName: atst
servicePort: 80
type: LoadBalancer