Simplifies customization by using ConfigMap instead of docker build

Author: solsson

fluent-bit-config-kafka-rest.yml

@@ -0,0 +1,86 @@
+kind: ConfigMap
+metadata:
+  name: fluent-bit-config-kafka-rest
+  namespace: kube-system
+apiVersion: v1
+data:
+  fluent-bit.conf: |-
+    [SERVICE]
+        Flush          1
+        Daemon         Off
+        Log_Level      info
+        Parsers_File   parsers.conf
+
+    [INPUT]
+        Name           tail
+        Tag            kube.*
+        Path           /var/log/containers/*.log
+        Parser         docker
+        DB             /var/log/flb_kube.db
+        Mem_Buf_Limit  5MB
+
+    [FILTER]
+        Name           kubernetes
+        Match          kube.*
+        Kube_URL       https://kubernetes.default.svc:443
+        Merge_JSON_Log On
+
+    [OUTPUT]
+        Name            kafka-rest
+        Match           *
+        Host            ${KAFKA_REST_HOST}
+        Port            ${KAFKA_REST_PORT}
+        Time_Key        @timestamp
+        Tag_Key         _fluent-tag
+        Include_Tag_Key On
+        Tag_Key         My_Tag_Key
+        Topic           ${KAFKA_TOPIC}
+        # Partition     0
+        # Message_Key abc
+
+  parsers.conf: |-
+    [PARSER]
+        Name   apache
+        Format regex
+        Regex  ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
+        Time_Key time
+        Time_Format %d/%b/%Y:%H:%M:%S %z
+
+    [PARSER]
+        Name   apache2
+        Format regex
+        Regex  ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
+        Time_Key time
+        Time_Format %d/%b/%Y:%H:%M:%S %z
+
+    [PARSER]
+        Name   apache_error
+        Format regex
+        Regex  ^\[[^ ]* (?<time>[^\]]*)\] \[(?<level>[^\]]*)\](?: \[pid (?<pid>[^\]]*)\])?( \[client (?<client>[^\]]*)\])? (?<message>.*)$
+
+    [PARSER]
+        Name   nginx
+        Format regex
+        Regex ^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
+        Time_Key time
+        Time_Format %d/%b/%Y:%H:%M:%S %z
+
+    [PARSER]
+        Name   json-test
+        Format json
+        Time_Key time
+        Time_Format %d/%b/%Y:%H:%M:%S %z
+
+    [PARSER]
+        Name        docker
+        Format      json
+        Time_Key    time
+        Time_Format %Y-%m-%dT%H:%M:%S.%L
+        Time_Keep   On
+
+    [PARSER]
+        Name        syslog
+        Format      regex
+        Regex       ^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$
+        Time_Key    time
+        Time_Format %b %d %H:%M:%S

fluent-bit-daemonset-kafka.yml

@@ -17,30 +17,37 @@ spec:
     spec:
       containers:
       - name: fluent-bit
-        image: fluent
-        # build using eval $(minikube docker-env) && docker build -t fluent docker-image/
-        imagePullPolicy: Never
+        image: fluent/fluent-bit:0.12-dev
         env:
           - name:  KAFKA_REST_HOST
             value: rest.kafka.svc.cluster.local
           - name:  KAFKA_REST_PORT
             value: "80"
           - name:  KAFKA_TOPIC
-            value: logs
+            value: logs-json
+        command:
+          - /fluent-bit/bin/fluent-bit
+          - -c
+          - /fluent-bit/etc/fluent-bit.conf
         resources:
           limits:
             memory: 100Mi
           requests:
             cpu: 100m
             memory: 100Mi
         volumeMounts:
+        - name: config
+          mountPath: /fluent-bit/etc
         - name: varlog
           mountPath: /var/log
         - name: varlibdockercontainers
           mountPath: /var/lib/docker/containers
           readOnly: true
       terminationGracePeriodSeconds: 10
       volumes:
+      - name: config
+        configMap:
+          name: fluent-bit-config-kafka-rest
       - name: varlog
         hostPath:
           path: /var/log