eksctl create iamserviceaccount \\
--name fluentd \\
--region=ap-northeast-2 \\
--cluster gwangju-eks-cluster \\
--namespace=fluentd \\
--attach-policy-arn arn:aws:iam::aws:policy/CloudWatchFullAccess \\
--override-existing-serviceaccounts \\
--approve
kubectl create configmap cluster-info \\
--from-literal=cluster.name=gwangju-eks-cluster \\
--from-literal=logs.region=ap-northeast-2 -n fluentd
apiVersion: v1
kind: Namespace
metadata:
name: fluentd
labels:
name: amazon-cloudwatch
kubectl apply -f ns.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: fluentd-role
rules:
- apiGroups: [""]
resources:
- namespaces
- pods
- pods/logs
verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: fluentd-role-binding
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: fluentd-role
subjects:
- kind: ServiceAccount
name: fluentd
namespace: fluentd
---
apiVersion: v1
kind: ConfigMap
metadata:
name: fluentd-config
namespace: fluentd
labels:
k8s-app: fluentd-cloudwatch
data:
kubernetes.conf: |
kubernetes.conf
fluent.conf: |
@include servic-a.conf
@include servic-b.conf
@include service-c.conf
<match fluent.**>
@type null
</match>
servic-a.conf: |
<source>
@type forward
bind 0.0.0.0
port 24224
tag cloudwatch_logs.fluent-bit-a.access
</source>
<match cloudwatch_logs.fluent-bit-a.*>
@type cloudwatch_logs
log_group_name /gwangju/eks/application/logs
log_stream_name service-a-logs
auto_create_stream true
<buffer tag>
flush_mode immediate
</buffer>
</match>
servic-b.conf: |
<source>
@type forward
bind 0.0.0.0
port 24225
tag cloudwatch_logs.fluent-bit-b.access
</source>
<match cloudwatch_logs.fluent-bit-b.*>
@type cloudwatch_logs
log_group_name /gwangju/eks/application/logs
log_stream_name service-b-logs
auto_create_stream true
<buffer tag>
flush_mode immediate
</buffer>
</match>
service-c.conf: |
<source>
@type forward
bind 0.0.0.0
port 24226
tag cloudwatch_logs.fluent-bit-c.access
</source>
<match cloudwatch_logs.fluent-bit-c.*>
@type cloudwatch_logs
log_group_name /gwangju/eks/application/logs
log_stream_name service-c-logs
auto_create_stream true
<buffer tag>
flush_mode immediate
</buffer>
</match>
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: fluentd
namespace: fluentd
spec:
selector:
matchLabels:
k8s-app: fluentd-cloudwatch
template:
metadata:
labels:
k8s-app: fluentd-cloudwatch
annotations:
configHash: 8915de4cf9c3551a8dc74c0137a3e83569d28c71044b0359c2578d2e0461825
spec:
serviceAccountName: fluentd
terminationGracePeriodSeconds: 30
# Because the image's entrypoint requires to write on /fluentd/etc but we mount configmap there which is read-only,
# this initContainers workaround or other is needed.
# See <https://github.com/fluent/fluentd-kubernetes-daemonset/issues/90>
initContainers:
- name: copy-fluentd-config
image: busybox
command: ['sh', '-c', 'cp /config-volume/..data/* /fluentd/etc']
volumeMounts:
- name: config-volume
mountPath: /config-volume
- name: fluentdconf
mountPath: /fluentd/etc
- name: update-log-driver
image: busybox
command: ['sh','-c','']
containers:
- name: fluentd-cloudwatch
image: fluent/fluentd-kubernetes-daemonset:v1.10.3-debian-cloudwatch-1.0
env:
- name: AWS_REGION
valueFrom:
configMapKeyRef:
name: cluster-info
key: logs.region
- name: CLUSTER_NAME
valueFrom:
configMapKeyRef:
name: cluster-info
key: cluster.name
- name: CI_VERSION
value: "k8s/1.3.24"
- name: FLUENT_CONTAINER_TAIL_PARSER_TYPE
value: /^(?<time>.+) (?<stream>stdout|stderr) (?<logtag>[FP]) (?<log>.*)$/
resources:
limits:
memory: 400Mi
requests:
cpu: 100m
memory: 200Mi
volumeMounts:
- name: config-volume
mountPath: /config-volume
- name: fluentdconf
mountPath: /fluentd/etc
- name: fluentd-config
mountPath: /fluentd/etc/kubernetes.conf
subPath: kubernetes.conf
- name: varlog
mountPath: /var/log
- name: varlibdockercontainers
mountPath: /var/lib/docker/containers
readOnly: true
- name: runlogjournal
mountPath: /run/log/journal
readOnly: true
- name: dmesg
mountPath: /var/log/dmesg
readOnly: true
volumes:
- name: config-volume
configMap:
name: fluentd-config
- name: fluentdconf
emptyDir: {}
- name: fluentd-config
configMap:
name: fluentd-config
items:
- key: kubernetes.conf
path: kubernetes.conf
- name: varlog
hostPath:
path: /var/log
- name: varlibdockercontainers
hostPath:
path: /var/lib/docker/containers
- name: runlogjournal
hostPath:
path: /run/log/journal
- name: dmesg
hostPath:
path: /var/log/dmesg
kubectl apply -f fleuntd.yaml
apiVersion: v1
kind: Service
metadata:
name: fluentd-svc
namespace: fluentd
spec:
selector:
k8s-app: fluentd-cloudwatch
type: ClusterIP
ports:
- name : service-a
protocol: TCP
port: 24224
targetPort: 24224
- name: service-b
protocol: TCP
port: 24225
targetPort: 24225
- name: service-c
protocol: TCP
port: 24226
targetPort: 24226
kubectl apply -f service.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: fluent-bit-sidecar-a-config
namespace: app
data:
fluent-bit.conf: |
[SERVICE]
Flush 1
Log_Level info
Daemon off
[INPUT]
Name tail
Path /log/*.log
Tag service-a
Refresh_Interval 10
Mem_Buf_Limit 50MB
Skip_Long_Lines On
[OUTPUT]
Name forward
Match *
Host SVC_IP
Port 24224
Retry_Limit False
SVC_CLUSTER_IP=$(kubectl get svc -n fluentd -o json | jq -r '.items[].spec.clusterIP')