diff --git a/deploy/README.md b/deploy/README.md index 5e973a16..a0244977 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -30,6 +30,50 @@ envsubst < deploy/aws/storage-class.yml | kubectl apply -f - When applying configuration changes, be careful not to over-write the storage class configuration without the environment variable substituted. +#### Fluentd Configuration + +For the Fluentd/CloudWatch integration to work for logging purposes, you will need to add an additional policy to the worker nodes' role. What follows is adapted from the [EKS Workshop](https://eksworkshop.com/logging/prereqs/). + +If you used eksctl to provision the EKS cluster, there will be a CloudFormation stack associated with the cluster. The node instances within the cluster will have a role associated to define their permissions. You need the name of the role. To get it using the AWS CLI, run: + +``` +export ROLE_NAME=$(aws --profile=dds --region us-east-2 cloudformation describe-stacks --stack-name eksctl-atat-nodegroup-standard-workers | jq -r '.Stacks[].Outputs[] | select(.OutputKey=="InstanceRoleARN") | .OutputValue' | cut -f2 -d/) +``` + +(This assumes that you have [`jq`](https://stedolan.github.io/jq/) available to parse the JSON response.) + +Run `echo $ROLE_NAME` to check that the previous command worked. + +Create a file called `k8s-logs-policy.json` and add the following content: + +``` +{ + "Version": "2012-10-17", + "Statement": [ + { + "Action": [ + "logs:DescribeLogGroups", + "logs:DescribeLogStreams", + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "Resource": "*", + "Effect": "Allow" + } + ] +} +``` + +This is the new policy that allows the nodes to aggregate logs. To apply it, run the following with the AWS CLI: + +``` +aws iam put-role-policy --role-name $ROLE_NAME --policy-name Logs-Policy-For-Worker --policy-document file://./k8s-logs-policy.json +``` + +(This command assumes you are executing it in the same directory as the policy JSON file; adjust the path as needed.) + + ### Apply the config to an Azure cluster To apply the configuration to a new cluster, run: diff --git a/deploy/aws/amazon-cloudwatch-cluster-info.yml b/deploy/aws/amazon-cloudwatch-cluster-info.yml new file mode 100644 index 00000000..01e829ae --- /dev/null +++ b/deploy/aws/amazon-cloudwatch-cluster-info.yml @@ -0,0 +1,8 @@ +apiVersion: v1 +data: + cluster.name: atat + logs.region: us-east-2 +kind: ConfigMap +metadata: + name: cluster-info + namespace: amazon-cloudwatch diff --git a/deploy/aws/cloudwatch-namespace.yaml b/deploy/aws/cloudwatch-namespace.yaml new file mode 100644 index 00000000..27aa3d69 --- /dev/null +++ b/deploy/aws/cloudwatch-namespace.yaml @@ -0,0 +1,7 @@ +# create amazon-cloudwatch namespace +apiVersion: v1 +kind: Namespace +metadata: + name: amazon-cloudwatch + labels: + name: amazon-cloudwatch \ No newline at end of file diff --git a/deploy/aws/fluentd.yaml b/deploy/aws/fluentd.yaml new file mode 100644 index 00000000..d49b008e --- /dev/null +++ b/deploy/aws/fluentd.yaml @@ -0,0 +1,433 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: fluentd + namespace: amazon-cloudwatch +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: fluentd-role +rules: + - apiGroups: [""] + resources: + - namespaces + - pods + - pods/logs + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: fluentd-role-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: fluentd-role +subjects: + - kind: ServiceAccount + name: fluentd + namespace: amazon-cloudwatch +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: fluentd-config + namespace: amazon-cloudwatch + labels: + k8s-app: fluentd-cloudwatch +data: + fluent.conf: | + @include containers.conf + @include systemd.conf + @include host.conf + + + @type null + + containers.conf: | + + @type tail + @id in_tail_container_logs + @label @containers + path /var/log/containers/*.log + exclude_path ["/var/log/containers/cloudwatch-agent*", "/var/log/containers/fluentd*"] + pos_file /var/log/fluentd-containers.log.pos + tag * + read_from_head true + + @type json + time_format %Y-%m-%dT%H:%M:%S.%NZ + + + + + @type tail + @id in_tail_cwagent_logs + @label @cwagentlogs + path /var/log/containers/cloudwatch-agent* + pos_file /var/log/cloudwatch-agent.log.pos + tag * + read_from_head true + + @type json + time_format %Y-%m-%dT%H:%M:%S.%NZ + + + + + @type tail + @id in_tail_fluentd_logs + @label @fluentdlogs + path /var/log/containers/fluentd* + pos_file /var/log/fluentd.log.pos + tag * + read_from_head true + + @type json + time_format %Y-%m-%dT%H:%M:%S.%NZ + + + + + + + + + + + systemd.conf: | + + @type systemd + @id in_systemd_kubelet + @label @systemd + filters [{ "_SYSTEMD_UNIT": "kubelet.service" }] + + field_map {"MESSAGE": "message", "_HOSTNAME": "hostname", "_SYSTEMD_UNIT": "systemd_unit"} + field_map_strict true + + path /var/log/journal + + @type local + persistent true + path /var/log/fluentd-journald-kubelet-pos.json + + read_from_head true + tag kubelet.service + + + + @type systemd + @id in_systemd_kubeproxy + @label @systemd + filters [{ "_SYSTEMD_UNIT": "kubeproxy.service" }] + + field_map {"MESSAGE": "message", "_HOSTNAME": "hostname", "_SYSTEMD_UNIT": "systemd_unit"} + field_map_strict true + + path /var/log/journal + + @type local + persistent true + path /var/log/fluentd-journald-kubeproxy-pos.json + + read_from_head true + tag kubeproxy.service + + + + @type systemd + @id in_systemd_docker + @label @systemd + filters [{ "_SYSTEMD_UNIT": "docker.service" }] + + field_map {"MESSAGE": "message", "_HOSTNAME": "hostname", "_SYSTEMD_UNIT": "systemd_unit"} + field_map_strict true + + path /var/log/journal + + @type local + persistent true + path /var/log/fluentd-journald-docker-pos.json + + read_from_head true + tag docker.service + + + + host.conf: | + + @type tail + @id in_tail_dmesg + @label @hostlogs + path /var/log/dmesg + pos_file /var/log/dmesg.log.pos + tag host.dmesg + read_from_head true + + @type syslog + + + + + @type tail + @id in_tail_secure + @label @hostlogs + path /var/log/secure + pos_file /var/log/secure.log.pos + tag host.secure + read_from_head true + + @type syslog + + + + + @type tail + @id in_tail_messages + @label @hostlogs + path /var/log/messages + pos_file /var/log/messages.log.pos + tag host.messages + read_from_head true + + @type syslog + + + + +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: fluentd-cloudwatch + namespace: amazon-cloudwatch + labels: + k8s-app: fluentd-cloudwatch +spec: + template: + metadata: + labels: + k8s-app: fluentd-cloudwatch + annotations: + configHash: 8915de4cf9c3551a8dc74c0137a3e83569d28c71044b0359c2578d2e0461825 + spec: + serviceAccountName: fluentd + terminationGracePeriodSeconds: 30 + # Because the image's entrypoint requires to write on /fluentd/etc but we mount configmap there which is read-only, + # this initContainers workaround or other is needed. + # See https://github.com/fluent/fluentd-kubernetes-daemonset/issues/90 + initContainers: + - name: copy-fluentd-config + image: busybox + command: ['sh', '-c', 'cp /config-volume/..data/* /fluentd/etc'] + volumeMounts: + - name: config-volume + mountPath: /config-volume + - name: fluentdconf + mountPath: /fluentd/etc + - name: update-log-driver + image: busybox + command: ['sh','-c',''] + containers: + - name: fluentd-cloudwatch + image: fluent/fluentd-kubernetes-daemonset:v1.3.3-debian-cloudwatch-1.4 + env: + - name: REGION + valueFrom: + configMapKeyRef: + name: cluster-info + key: logs.region + - name: CLUSTER_NAME + valueFrom: + configMapKeyRef: + name: cluster-info + key: cluster.name + - name: CI_VERSION + value: "k8s/1.0.0" + resources: + limits: + memory: 200Mi + requests: + cpu: 100m + memory: 200Mi + volumeMounts: + - name: config-volume + mountPath: /config-volume + - name: fluentdconf + mountPath: /fluentd/etc + - name: varlog + mountPath: /var/log + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + readOnly: true + - name: runlogjournal + mountPath: /run/log/journal + readOnly: true + - name: dmesg + mountPath: /var/log/dmesg + readOnly: true + volumes: + - name: config-volume + configMap: + name: fluentd-config + - name: fluentdconf + emptyDir: {} + - name: varlog + hostPath: + path: /var/log + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers + - name: runlogjournal + hostPath: + path: /run/log/journal + - name: dmesg + hostPath: + path: /var/log/dmesg