Initial FlexVol Setup

This commit is the first part of consuming secrets from the Azure Key Vault. This will set up the required services to consume Azure's RBAC controls in the cluster, an identity to read the secrets, and the tool (FlexVol) to mount the secrets.

deploy/azure/aadpodidentity.yml

@@ -0,0 +1,19 @@
+---
+apiVersion: "aadpodidentity.k8s.io/v1"
+kind: AzureIdentity
+metadata:
+  name: atat-kv-identity
+spec:
+  type: 0
+  ResourceID: $KV_MI_ID
+  ClientID: $KV_MI_CLIENT_ID
+---
+apiVersion: "aadpodidentity.k8s.io/v1"
+kind: AzureIdentityBinding
+metadata:
+  name: atat-key-vault-identity-binding
+spec:
+  AzureIdentity: atat-kv-identity
+  Selector: atat-kv-id-binding
+---
+

deploy/azure/keyvault/deployment-rbac.yml

@@ -0,0 +1,259 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: aad-pod-id-nmi-service-account
+  namespace: default
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: azureassignedidentities.aadpodidentity.k8s.io
+spec:
+  group: aadpodidentity.k8s.io
+  version: v1
+  names:
+    kind: AzureAssignedIdentity
+    plural: azureassignedidentities
+  scope: Namespaced
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: azureidentitybindings.aadpodidentity.k8s.io
+spec:
+  group: aadpodidentity.k8s.io
+  version: v1
+  names:
+    kind: AzureIdentityBinding
+    plural: azureidentitybindings
+  scope: Namespaced
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: azureidentities.aadpodidentity.k8s.io
+spec:
+  group: aadpodidentity.k8s.io
+  version: v1
+  names:
+    kind: AzureIdentity
+    singular: azureidentity
+    plural: azureidentities
+  scope: Namespaced
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: azurepodidentityexceptions.aadpodidentity.k8s.io
+spec:
+  group: aadpodidentity.k8s.io
+  version: v1
+  names:
+    kind: AzurePodIdentityException
+    singular: azurepodidentityexception
+    plural: azurepodidentityexceptions
+  scope: Namespaced
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: aad-pod-id-nmi-role
+rules:
+  - apiGroups: ["apiextensions.k8s.io"]
+    resources: ["customresourcedefinitions"]
+    verbs: ["get", "list"]
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get"]
+  - apiGroups: ["aadpodidentity.k8s.io"]
+    resources:
+      ["azureidentitybindings", "azureidentities", "azurepodidentityexceptions"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["aadpodidentity.k8s.io"]
+    resources: ["azureassignedidentities"]
+    verbs: ["get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: aad-pod-id-nmi-binding
+  labels:
+    k8s-app: aad-pod-id-nmi-binding
+subjects:
+  - kind: ServiceAccount
+    name: aad-pod-id-nmi-service-account
+    namespace: default
+roleRef:
+  kind: ClusterRole
+  name: aad-pod-id-nmi-role
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  labels:
+    kubernetes.io/cluster-service: "true"
+    component: nmi
+    tier: node
+    k8s-app: aad-pod-id
+  name: nmi
+  namespace: default
+spec:
+  updateStrategy:
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      component: nmi
+      tier: node
+  template:
+    metadata:
+      labels:
+        component: nmi
+        tier: node
+    spec:
+      serviceAccountName: aad-pod-id-nmi-service-account
+      hostNetwork: true
+      volumes:
+        - hostPath:
+            path: /run/xtables.lock
+            type: FileOrCreate
+          name: iptableslock
+      containers:
+        - name: nmi
+          image: "mcr.microsoft.com/k8s/aad-pod-identity/nmi:1.5.3"
+          imagePullPolicy: Always
+          args:
+            - "--host-ip=$(HOST_IP)"
+            - "--node=$(NODE_NAME)"
+          env:
+            - name: HOST_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          resources:
+            limits:
+              cpu: 200m
+              memory: 512Mi
+            requests:
+              cpu: 100m
+              memory: 256Mi
+          securityContext:
+            privileged: true
+            capabilities:
+              add:
+                - NET_ADMIN
+          volumeMounts:
+            - mountPath: /run/xtables.lock
+              name: iptableslock
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: 8080
+            initialDelaySeconds: 10
+            periodSeconds: 5
+      nodeSelector:
+        beta.kubernetes.io/os: linux
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: aad-pod-id-mic-service-account
+  namespace: default
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: aad-pod-id-mic-role
+rules:
+  - apiGroups: ["apiextensions.k8s.io"]
+    resources: ["customresourcedefinitions"]
+    verbs: ["*"]
+  - apiGroups: [""]
+    resources: ["pods", "nodes"]
+    verbs: ["list", "watch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["create", "patch"]
+  - apiGroups: [""]
+    resources: ["endpoints"]
+    verbs: ["create", "get", "update"]
+  - apiGroups: ["aadpodidentity.k8s.io"]
+    resources: ["azureidentitybindings", "azureidentities"]
+    verbs: ["get", "list", "watch", "post"]
+  - apiGroups: ["aadpodidentity.k8s.io"]
+    resources: ["azureassignedidentities"]
+    verbs: ["*"]
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: aad-pod-id-mic-binding
+  labels:
+    k8s-app: aad-pod-id-mic-binding
+subjects:
+  - kind: ServiceAccount
+    name: aad-pod-id-mic-service-account
+    namespace: default
+roleRef:
+  kind: ClusterRole
+  name: aad-pod-id-mic-role
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    component: mic
+    k8s-app: aad-pod-id
+  name: mic
+  namespace: default
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      component: mic
+      app: mic
+  template:
+    metadata:
+      labels:
+        component: mic
+        app: mic
+    spec:
+      serviceAccountName: aad-pod-id-mic-service-account
+      containers:
+        - name: mic
+          image: "mcr.microsoft.com/k8s/aad-pod-identity/mic:1.5.3"
+          imagePullPolicy: Always
+          args:
+            - "--cloudconfig=/etc/kubernetes/azure.json"
+            - "--logtostderr"
+          resources:
+            limits:
+              cpu: 200m
+              memory: 1024Mi
+            requests:
+              cpu: 100m
+              memory: 256Mi
+          volumeMounts:
+            - name: k8s-azure-file
+              mountPath: /etc/kubernetes/azure.json
+              readOnly: true
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: 8080
+            initialDelaySeconds: 10
+            periodSeconds: 5
+      volumes:
+        - name: k8s-azure-file
+          hostPath:
+            path: /etc/kubernetes/azure.json
+      nodeSelector:
+        beta.kubernetes.io/os: linux

deploy/azure/keyvault/kv-flex-vol-installer.yml

@@ -0,0 +1,49 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: kv
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  labels:
+    app: keyvault-flexvolume
+  name: keyvault-flexvolume
+  namespace: kv
+spec:
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: keyvault-flexvolume
+    spec:
+      tolerations:
+      containers:
+        - name: flexvol-driver-installer
+          image: "mcr.microsoft.com/k8s/flexvolume/keyvault-flexvolume:v0.0.15"
+          imagePullPolicy: Always
+          resources:
+            requests:
+              cpu: 50m
+              memory: 100Mi
+            limits:
+              cpu: 50m
+              memory: 100Mi
+          env:
+            # if you have used flex before on your cluster, use same directory
+            # set TARGET_DIR env var and mount the same directory to to the container
+            - name: TARGET_DIR
+              value: "/etc/kubernetes/volumeplugins"
+          volumeMounts:
+            - mountPath: "/etc/kubernetes/volumeplugins"
+              name: volplugins
+      volumes:
+        - hostPath:
+            # Modify this directory if your nodes are using a different one
+            # default is "/usr/libexec/kubernetes/kubelet-plugins/volume/exec"
+            # below is Azure default
+            path: "/etc/kubernetes/volumeplugins"
+          name: volplugins
+      nodeSelector:
+        beta.kubernetes.io/os: linux

deploy/azure/kustomization.yaml

@@ -10,3 +10,4 @@ resources:
   - volume-claim.yml
   - nginx-client-ca-bundle.yml
   - acme-challenges.yml
+  - aadpodidentity.yml