diff --git a/base-image/Makefile b/base-image/Makefile index 8a85ccb4..2184b19a 100644 --- a/base-image/Makefile +++ b/base-image/Makefile @@ -20,7 +20,6 @@ run: build-image -v `pwd`/failsafe.conf:/fluentd/etc/fluent.conf \ -v /var/log:/var/log \ -v /var/run:/var/run \ - -e FLUENTD_OPT="--no-supervisor" \ $(IMAGE):$(TAG) test: build-image @@ -29,9 +28,9 @@ test: build-image -v `pwd`:/workspace \ -v `pwd/plugins`:/fluentd/plugins \ -v `pwd`/test/local.conf:/fluentd/etc/fluent.conf \ - -e FLUENTD_OPT="--no-supervisor" \ - -e PAPERTRAIL_PORT=$$PAPERTRAIL_PORT \ - -e PAPERTRAIL_HOST=$$PAPERTRAIL_HOST \ + -p 0.0.0.0:24231:24231 \ + -e PAPERTRAIL_PORT=$$PAPERTRAIL_PORT \ + -e PAPERTRAIL_HOST=$$PAPERTRAIL_HOST \ $(IMAGE):$(TAG) list-gems: diff --git a/config-reloader/config/config.go b/config-reloader/config/config.go index e7717cb5..698f7e63 100644 --- a/config-reloader/config/config.go +++ b/config-reloader/config/config.go @@ -56,6 +56,7 @@ type Config struct { ParsedMetaValues map[string]string ParsedLabelSelector labels.Set ExecTimeoutSeconds int + SplitPattern string } var defaultConfig = &Config{ @@ -78,6 +79,7 @@ var defaultConfig = &Config{ MetricsPort: 9000, AdminNamespace: "kube-system", ExecTimeoutSeconds: 30, + SplitPattern: "", } var reValidID = regexp.MustCompile("([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]") @@ -250,6 +252,8 @@ func (cfg *Config) ParseFlags(args []string) error { app.Flag("admin-namespace", "Configurations defined in this namespace are copied as is, without further processing. Virtual plugins can also be defined in this namespace").Default(defaultConfig.AdminNamespace).StringVar(&cfg.AdminNamespace) + app.Flag("split-pattern", "Configurations defined special pattern for different workers").Default(defaultConfig.SplitPattern).StringVar(&cfg.SplitPattern) + app.Flag("exec-timeout", "Timeout duration (in seconds) for exec command during validation").Default(strconv.Itoa(defaultConfig.ExecTimeoutSeconds)).IntVar(&cfg.ExecTimeoutSeconds) _, err := app.Parse(args) diff --git a/config-reloader/fluentd/reloader.go b/config-reloader/fluentd/reloader.go index 843c7ffa..936203aa 100644 --- a/config-reloader/fluentd/reloader.go +++ b/config-reloader/fluentd/reloader.go @@ -33,7 +33,7 @@ func (r *Reloader) ReloadConfiguration() { logrus.Infof("Reloading fluentd configuration gracefully via POST to /api/config.gracefulReload") - resp, err := http.Get(fmt.Sprintf("http://127.0.0.1:%d/api/config.gracefulReload", r.port)) + resp, err := http.Get(fmt.Sprintf("http://127.0.0.1:%d/api/config.reload", r.port)) if err != nil { logrus.Errorf("fluentd config.gracefulReload request failed: %+v", err) } else if resp.StatusCode != 200 { diff --git a/config-reloader/fluentd/reloader_test.go b/config-reloader/fluentd/reloader_test.go index 3dcaa5f7..0a09eb9e 100644 --- a/config-reloader/fluentd/reloader_test.go +++ b/config-reloader/fluentd/reloader_test.go @@ -25,7 +25,7 @@ func TestReloaderCalls(t *testing.T) { handler := func(w http.ResponseWriter, r *http.Request) { fmt.Printf("req %+v", r) - if r.Method == "GET" && r.RequestURI == "/api/config.gracefulReload" { + if r.Method == "GET" && r.RequestURI == "/api/config.reload" { counter++ } } diff --git a/config-reloader/generator/generator.go b/config-reloader/generator/generator.go index 2d416d42..436a8f65 100644 --- a/config-reloader/generator/generator.go +++ b/config-reloader/generator/generator.go @@ -337,9 +337,11 @@ func (g *generatorInstance) renderIncludableFile(templateFile string, dest strin model := struct { ID string PrometheusEnabled bool + SplitPattern string }{ ID: util.MakeFluentdSafeName(g.cfg.ID), PrometheusEnabled: g.cfg.PrometheusEnabled, + SplitPattern: g.cfg.SplitPattern, } err = util.TemplateAndWriteFile(tmpl, model, dest) diff --git a/config-reloader/templates/fluent.conf b/config-reloader/templates/fluent.conf index 89af363d..3cb9cf1f 100644 --- a/config-reloader/templates/fluent.conf +++ b/config-reloader/templates/fluent.conf @@ -5,6 +5,7 @@ # needed to enable /api/config.reload rpc_endpoint 127.0.0.1:24444 + workers 3 # you can turn this on for debug diff --git a/config-reloader/templates/kubernetes-postprocess.conf b/config-reloader/templates/kubernetes-postprocess.conf index 2ce1b7df..aa308353 100644 --- a/config-reloader/templates/kubernetes-postprocess.conf +++ b/config-reloader/templates/kubernetes-postprocess.conf @@ -36,12 +36,14 @@ # Remove the unnecessary field as the information is already available on other fields. @type record_modifier + @id k8s_remove_unused_field remove_keys $.kubernetes.pod_id, $.kubernetes.master_url, $.kubernetes.container_image_id, $.kubernetes.namespace_id, kubernetes_namespace_container_name, $.kubernetes.labels.pod-template-generation, $.kubernetes.labels.controller-revision-hash, $.kubernetes.labels.pod-template-hash # Parse logs in the kube-system namespace using the kubernetes formatter. @type parser + @id k8s_parser_kube_system reserve_data true key_name log emit_invalid_record_to_error false diff --git a/config-reloader/templates/kubernetes.conf b/config-reloader/templates/kubernetes.conf index 60013b5b..a1cab618 100644 --- a/config-reloader/templates/kubernetes.conf +++ b/config-reloader/templates/kubernetes.conf @@ -1,27 +1,54 @@ # Sync with: # https://raw.githubusercontent.com/fluent/fluentd-kubernetes-daemonset/master/docker-image/v1.11/debian-elasticsearch7/conf/kubernetes.conf - - @type tail - @id in_tail_container_logs - path /var/log/containers/*.log - pos_file /var/log/{{.ID}}-fluentd-containers.log.pos - pos_file_compaction_interval 3m - skip_refresh_on_startup true - tag kubernetes.* - read_from_head true - read_bytes_limit_per_second 8192 - multiline_flush_interval 5s - - @type multiline - # cri-o - format1 /^(?([^\n]+ (stdout|stderr) P [^\n]+\n)*)/ - format2 /(? - + + + @type tail + @id in_tail_container_logs + path /var/log/containers/*.log + exclude_path /var/log/containers/*_{{.SplitPattern}}_*.log + pos_file /var/log/{{.ID}}-fluentd-containers.log.pos + pos_file_compaction_interval 3m + skip_refresh_on_startup true + tag kubernetes.* + read_from_head true + read_bytes_limit_per_second 8192 + multiline_flush_interval 5s + + @type multiline + # cri-o + format1 /^(?([^\n]+ (stdout|stderr) P [^\n]+\n)*)/ + format2 /(? + + + + + + @type tail + @id in_tail_container_logs + path /var/log/containers/*_{{.SplitPattern}}_*.log + pos_file /var/log/{{.ID}}-{{.SplitPattern}}-fluentd-containers.log.pos + pos_file_compaction_interval 3m + skip_refresh_on_startup true + tag kubernetes.* + read_from_head true + read_bytes_limit_per_second 8192 + multiline_flush_interval 5s + + @type multiline + # cri-o + format1 /^(?([^\n]+ (stdout|stderr) P [^\n]+\n)*)/ + format2 /(? + + # Merge cri-o partial lines @@ -48,183 +75,185 @@ - - @type tail - @id in_tail_minion - path /var/log/salt/minion - pos_file /var/log/{{.ID}}-fluentd-salt.pos - pos_file_compaction_interval 3m - skip_refresh_on_startup true - tag salt - - @type regexp - expression /^(? - - - - @type tail - @id in_tail_startupscript - path /var/log/startupscript.log - pos_file /var/log/{{.ID}}-fluentd-startupscript.log.pos - tag startupscript - pos_file_compaction_interval 3m - skip_refresh_on_startup true - - @type syslog - - - - - @type tail - @id in_tail_docker - path /var/log/docker.log - pos_file /var/log/{{.ID}}-fluentd-docker.log.pos - pos_file_compaction_interval 3m - skip_refresh_on_startup true - tag docker - - @type regexp - expression /^time="(? - - - - @type tail - @id in_tail_etcd - path /var/log/etcd.log - pos_file /var/log/{{.ID}}-fluentd-etcd.log.pos - pos_file_compaction_interval 3m - skip_refresh_on_startup true - tag k8s.etcd - - @type none - - - - - @type tail - @id in_tail_kubelet - path /var/log/kubelet.log - pos_file /var/log/{{.ID}}-fluentd-kubelet.log.pos - tag k8s.kubelet - - @type kubernetes - - - - - @type tail - @id in_tail_kube_proxy - path /var/log/kube-proxy.log - pos_file /var/log/{{.ID}}-fluentd-kube-proxy.log.pos - pos_file_compaction_interval 3m - skip_refresh_on_startup true - tag k8s.kube-proxy - - @type kubernetes - - - - - @type tail - @id in_tail_kube_apiserver - path /var/log/kube-apiserver.log - pos_file /var/log/{{.ID}}-fluentd-kube-apiserver.log.pos - pos_file_compaction_interval 3m - skip_refresh_on_startup true - tag k8s.kube-apiserver - - @type kubernetes - - - - - @type tail - @id in_tail_kube_controller_manager - path /var/log/kube-controller-manager.log - pos_file /var/log/{{.ID}}-fluentd-kube-controller-manager.log.pos - tag k8s.kube-controller-manager - - @type kubernetes - - - - - @type tail - @id in_tail_kube_scheduler - path /var/log/kube-scheduler.log - pos_file /var/log/{{.ID}}-fluentd-kube-scheduler.log.pos - pos_file_compaction_interval 3m - skip_refresh_on_startup true - tag k8s.kube-scheduler - - @type kubernetes - - - - - @type tail - @id in_tail_rescheduler - path /var/log/rescheduler.log - pos_file /var/log/{{.ID}}-fluentd-rescheduler.log.pos - pos_file_compaction_interval 3m - skip_refresh_on_startup true - tag k8s.rescheduler - - @type kubernetes - - - - - @type tail - @id in_tail_glbc - path /var/log/glbc.log - pos_file /var/log/{{.ID}}-fluentd-glbc.log.pos - pos_file_compaction_interval 3m - skip_refresh_on_startup true - tag k8s.glbc - - @type kubernetes - - - - - @type tail - @id in_tail_cluster_autoscaler - path /var/log/cluster-autoscaler.log - pos_file /var/log/{{.ID}}-fluentd-cluster-autoscaler.log.pos - pos_file_compaction_interval 3m - skip_refresh_on_startup true - tag k8s.cluster-autoscaler - - @type kubernetes - - + + + @type tail + @id in_tail_minion + path /var/log/salt/minion + pos_file /var/log/{{.ID}}-fluentd-salt.pos + pos_file_compaction_interval 3m + skip_refresh_on_startup true + tag salt + + @type regexp + expression /^(? + + + + @type tail + @id in_tail_startupscript + path /var/log/startupscript.log + pos_file /var/log/{{.ID}}-fluentd-startupscript.log.pos + tag startupscript + pos_file_compaction_interval 3m + skip_refresh_on_startup true + + @type syslog + + + + + @type tail + @id in_tail_docker + path /var/log/docker.log + pos_file /var/log/{{.ID}}-fluentd-docker.log.pos + pos_file_compaction_interval 3m + skip_refresh_on_startup true + tag docker + + @type regexp + expression /^time="(? + + + + @type tail + @id in_tail_etcd + path /var/log/etcd.log + pos_file /var/log/{{.ID}}-fluentd-etcd.log.pos + pos_file_compaction_interval 3m + skip_refresh_on_startup true + tag k8s.etcd + + @type none + + + + + @type tail + @id in_tail_kubelet + path /var/log/kubelet.log + pos_file /var/log/{{.ID}}-fluentd-kubelet.log.pos + tag k8s.kubelet + + @type kubernetes + + + + + @type tail + @id in_tail_kube_proxy + path /var/log/kube-proxy.log + pos_file /var/log/{{.ID}}-fluentd-kube-proxy.log.pos + pos_file_compaction_interval 3m + skip_refresh_on_startup true + tag k8s.kube-proxy + + @type kubernetes + + + + + @type tail + @id in_tail_kube_apiserver + path /var/log/kube-apiserver.log + pos_file /var/log/{{.ID}}-fluentd-kube-apiserver.log.pos + pos_file_compaction_interval 3m + skip_refresh_on_startup true + tag k8s.kube-apiserver + + @type kubernetes + + + + + @type tail + @id in_tail_kube_controller_manager + path /var/log/kube-controller-manager.log + pos_file /var/log/{{.ID}}-fluentd-kube-controller-manager.log.pos + tag k8s.kube-controller-manager + + @type kubernetes + + + + + @type tail + @id in_tail_kube_scheduler + path /var/log/kube-scheduler.log + pos_file /var/log/{{.ID}}-fluentd-kube-scheduler.log.pos + pos_file_compaction_interval 3m + skip_refresh_on_startup true + tag k8s.kube-scheduler + + @type kubernetes + + + + + @type tail + @id in_tail_rescheduler + path /var/log/rescheduler.log + pos_file /var/log/{{.ID}}-fluentd-rescheduler.log.pos + pos_file_compaction_interval 3m + skip_refresh_on_startup true + tag k8s.rescheduler + + @type kubernetes + + + + + @type tail + @id in_tail_glbc + path /var/log/glbc.log + pos_file /var/log/{{.ID}}-fluentd-glbc.log.pos + pos_file_compaction_interval 3m + skip_refresh_on_startup true + tag k8s.glbc + + @type kubernetes + + + + + @type tail + @id in_tail_cluster_autoscaler + path /var/log/cluster-autoscaler.log + pos_file /var/log/{{.ID}}-fluentd-cluster-autoscaler.log.pos + pos_file_compaction_interval 3m + skip_refresh_on_startup true + tag k8s.cluster-autoscaler + + @type kubernetes + + # Example: # 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="" asgroups="" namespace="default" uri="/api/v1/namespaces/default/pods" # 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200" - - @type tail - @id in_tail_kube_apiserver_audit - multiline_flush_interval 5s - path /var/log/kubernetes/kube-apiserver-audit.log - pos_file /var/log/{{.ID}}-kube-apiserver-audit.log.pos - pos_file_compaction_interval 3m - skip_refresh_on_startup true - tag k8s.kube-apiserver-audit - - @type multiline - format_firstline /^\S+\s+AUDIT:/ - # Fields must be explicitly captured by name to be parsed into the record. - # Fields may not always be present, and order may change, so this just looks - # for a list of key="\"quoted\" value" pairs separated by spaces. - # Unknown fields are ignored. - # Note: We can't separate query/response lines as format1/format2 because - # they don't always come one after the other for a given query. - format1 /^(? - + + @type tail + @id in_tail_kube_apiserver_audit + multiline_flush_interval 5s + path /var/log/kubernetes/kube-apiserver-audit.log + pos_file /var/log/{{.ID}}-kube-apiserver-audit.log.pos + pos_file_compaction_interval 3m + skip_refresh_on_startup true + tag k8s.kube-apiserver-audit + + @type multiline + format_firstline /^\S+\s+AUDIT:/ + # Fields must be explicitly captured by name to be parsed into the record. + # Fields may not always be present, and order may change, so this just looks + # for a list of key="\"quoted\" value" pairs separated by spaces. + # Unknown fields are ignored. + # Note: We can't separate query/response lines as format1/format2 because + # they don't always come one after the other for a given query. + format1 /^(? + + diff --git a/config-reloader/templates/prometheus.conf b/config-reloader/templates/prometheus.conf index 6efe122a..4a05ee10 100644 --- a/config-reloader/templates/prometheus.conf +++ b/config-reloader/templates/prometheus.conf @@ -28,7 +28,7 @@ type counter desc The total number of incoming records - host ${hostname} + tag ${tag} input_namespace $.kubernetes.namespace_name input_pod $.kubernetes.pod_name input_container $.kubernetes.container_name diff --git a/config-reloader/templates/systemd.conf b/config-reloader/templates/systemd.conf index fb8348d6..59829e19 100644 --- a/config-reloader/templates/systemd.conf +++ b/config-reloader/templates/systemd.conf @@ -2,52 +2,53 @@ # https://github.com/fluent/fluentd-kubernetes-daemonset/blob/master/docker-image/v1.11/debian-elasticsearch7/conf/systemd.conf # Logs from systemd-journal for interesting services. - - @type systemd - @id in_systemd_kubelet - matches [{ "_SYSTEMD_UNIT": "kubelet.service" }] - - @type local - persistent true - path /var/log/{{.ID}}-fluentd-journald-kubelet-cursor.json - - - fields_strip_underscores true - - read_from_head true - tag kubelet - -# Logs from docker-systemd - - @type systemd - @id in_systemd_docker - matches [{ "_SYSTEMD_UNIT": "docker.service" }] - - @type local - persistent true - path /var/log/{{.ID}}-fluentd-journald-docker-cursor.json - - - fields_strip_underscores true - - read_from_head true - tag docker.systemd - + + + @type systemd + @id in_systemd_kubelet + matches [{ "_SYSTEMD_UNIT": "kubelet.service" }] + + @type local + persistent true + path /var/log/{{.ID}}-fluentd-journald-kubelet-cursor.json + + + fields_strip_underscores true + + read_from_head true + tag kubelet + -# Logs from systemd-journal for interesting services. - - @type systemd - @id in_systemd_bootkube - matches [{ "_SYSTEMD_UNIT": "bootkube.service" }] - - @type local - persistent true - path /var/log/{{.ID}}-fluentd-journald-bootkube-cursor.json - - - fields_strip_underscores true - - read_from_head true - tag bootkube - + + @type systemd + @id in_systemd_docker + matches [{ "_SYSTEMD_UNIT": "docker.service" }] + + @type local + persistent true + path /var/log/{{.ID}}-fluentd-journald-docker-cursor.json + + + fields_strip_underscores true + + read_from_head true + tag docker.systemd + + + + @type systemd + @id in_systemd_bootkube + matches [{ "_SYSTEMD_UNIT": "bootkube.service" }] + + @type local + persistent true + path /var/log/{{.ID}}-fluentd-journald-bootkube-cursor.json + + + fields_strip_underscores true + + read_from_head true + tag bootkube + +