From bd3662e8ce8833b03be51f3e9c9c19b17e70b225 Mon Sep 17 00:00:00 2001 From: dandds Date: Tue, 6 Aug 2019 09:48:39 -0400 Subject: [PATCH] Add config for EFS persistent volume in AWS k8s cluster. Chose Elastic File Storage over EBS (Elastic Block Storage) because the latter can only be mounted into a single node. This adds the RBAC config and deployment for managing EFS mounts within the cluster. Largely depends on this efs-provisioner config: https://github.com/kubernetes-incubator/external-storage/tree/master/aws/efs The config has been hard-copied into the repo and updated for future reference. Note that the config requires an environment variable substitution and cannot be applied directly to the cluster. --- k8s/aws/aws.yml | 5 +++ k8s/aws/crls-sync.yaml | 43 +++++++++++++++++++++ k8s/aws/efs-rbac.yml | 66 ++++++++++++++++++++++++++++++++ k8s/aws/storage-class.yml | 80 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 194 insertions(+) create mode 100644 k8s/aws/crls-sync.yaml create mode 100644 k8s/aws/efs-rbac.yml create mode 100644 k8s/aws/storage-class.yml diff --git a/k8s/aws/aws.yml b/k8s/aws/aws.yml index 4fe84e70..8f8e46d0 100644 --- a/k8s/aws/aws.yml +++ b/k8s/aws/aws.yml @@ -44,6 +44,8 @@ spec: subPath: client-ca-bundle.pem - name: uwsgi-socket-dir mountPath: "/var/run/uwsgi" + - name: crls-vol + mountPath: "/opt/atat/atst/crls" - name: nginx image: nginx:alpine ports: @@ -109,6 +111,9 @@ spec: - key: tls.key path: atat.key mode: 0640 + - name: crls-vol + persistentVolumeClaim: + claimName: efs --- apiVersion: extensions/v1beta1 kind: Deployment diff --git a/k8s/aws/crls-sync.yaml b/k8s/aws/crls-sync.yaml new file mode 100644 index 00000000..2d7ee55e --- /dev/null +++ b/k8s/aws/crls-sync.yaml @@ -0,0 +1,43 @@ +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: crls + namespace: atat +spec: + schedule: "0 * * * *" + jobTemplate: + spec: + template: + spec: + restartPolicy: OnFailure + containers: + - name: crls + image: 904153757533.dkr.ecr.us-east-2.amazonaws.com/atat:8f1c8b5633ca70168837c885010e7d66d93562dc + command: [ + "/bin/sh", "-c" + ] + args: [ + "/opt/atat/atst/script/sync-crls", + ] + envFrom: + - configMapRef: + name: atst-envvars + - configMapRef: + name: atst-worker-envvars + volumeMounts: + - name: atst-config + mountPath: "/opt/atat/atst/atst-overrides.ini" + subPath: atst-overrides.ini + - name: crls-vol + mountPath: "/opt/atat/atst/crls" + volumes: + - name: atst-config + secret: + secretName: atst-config-ini + items: + - key: override.ini + path: atst-overrides.ini + mode: 0644 + - name: crls-vol + persistentVolumeClaim: + claimName: efs diff --git a/k8s/aws/efs-rbac.yml b/k8s/aws/efs-rbac.yml new file mode 100644 index 00000000..12496f02 --- /dev/null +++ b/k8s/aws/efs-rbac.yml @@ -0,0 +1,66 @@ +# This can't be run without substituting the EFSID environment variable. +# from https://github.com/kubernetes-incubator/external-storage/blob/master/aws/efs/deploy/rbac.yaml +--- +kind: ServiceAccount +apiVersion: v1 +metadata: + name: efs-provisioner +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: efs-provisioner-runner +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete", "describe"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "update", "patch"] + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "list", "watch", "create", "update", "patch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: run-efs-provisioner +subjects: + - kind: ServiceAccount + name: efs-provisioner + # replace with namespace where provisioner is deployed + namespace: atat +roleRef: + kind: ClusterRole + name: efs-provisioner-runner + apiGroup: rbac.authorization.k8s.io +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: leader-locking-efs-provisioner +rules: + - apiGroups: [""] + resources: ["endpoints"] + + verbs: ["get", "list", "watch", "create", "update", "patch"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: leader-locking-efs-provisioner +subjects: + - kind: ServiceAccount + name: efs-provisioner + # replace with namespace where provisioner is deployed + namespace: atat +roleRef: + kind: Role + name: leader-locking-efs-provisioner + apiGroup: rbac.authorization.k8s.io diff --git a/k8s/aws/storage-class.yml b/k8s/aws/storage-class.yml new file mode 100644 index 00000000..1df8d78c --- /dev/null +++ b/k8s/aws/storage-class.yml @@ -0,0 +1,80 @@ +# from https://github.com/kubernetes-incubator/external-storage/blob/master/aws/efs/deploy/manifest.yaml +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: efs-provisioner +data: + file.system.id: $EFSID + aws.region: us-east-2 + provisioner.name: example.com/aws-efs + dns.name: $EFSID.efs.us-east-2.amazonaws.com +--- +kind: Deployment +apiVersion: extensions/v1beta1 +metadata: + name: efs-provisioner +spec: + replicas: 1 + strategy: + type: Recreate + template: + metadata: + labels: + app: efs-provisioner + spec: + serviceAccountName: efs-provisioner + containers: + - name: efs-provisioner + image: quay.io/external_storage/efs-provisioner:latest + env: + - name: FILE_SYSTEM_ID + valueFrom: + configMapKeyRef: + name: efs-provisioner + key: file.system.id + - name: AWS_REGION + valueFrom: + configMapKeyRef: + name: efs-provisioner + key: aws.region + - name: DNS_NAME + valueFrom: + configMapKeyRef: + name: efs-provisioner + key: dns.name + optional: true + - name: PROVISIONER_NAME + valueFrom: + configMapKeyRef: + name: efs-provisioner + key: provisioner.name + volumeMounts: + - name: pv-volume + mountPath: /persistentvolumes + volumes: + - name: pv-volume + nfs: + server: $EFSID.efs.us-east-2.amazonaws.com + path: / +--- +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: aws-efs +provisioner: example.com/aws-efs +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: efs + annotations: + volume.beta.kubernetes.io/storage-class: "aws-efs" +spec: + accessModes: + - ReadWriteMany + storageClassName: aws-efs + resources: + requests: + storage: 1Mi +---