Skip to content

Commit a4fde17

Browse files
author
Dominik Rosiek
committed
feat: use router to detect need for trailing newline
Signed-off-by: Dominik Rosiek <[email protected]>
1 parent 69f2592 commit a4fde17

File tree

8 files changed

+65
-17
lines changed

8 files changed

+65
-17
lines changed

‎deploy/helm/sumologic/conf/logs/collector/common/filelog_receiver.yaml‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ filelog/containers:
5656
## - k8s.namespace.name
5757
## - log.file.path
5858
- type: retain
59+
id: keep-fields
5960
fields:
6061
- attributes["stream"]
6162
- attributes["k8s.pod.name"]
@@ -66,6 +67,18 @@ filelog/containers:
6667
- attributes["time"]
6768
{{ end }}
6869

70+
## Strip trailing "\n" from the log body
71+
- id: strip-trailing-newline-router
72+
{{- if .Values.sumologic.logs.multiline.enabled }}
73+
default: multiline
74+
{{- else }}
75+
default: merge-multiline-logs
76+
{{- end }}
77+
routes:
78+
- expr: body matches "^.*\n$"
79+
output: strip-trailing-newline
80+
type: router
81+
6982
## strip-trailing-newline removes the trailing "\n" from the `log` key. This is required for logs coming from Docker container runtime.
7083
## Uses attributes.log as temporary cotainer for new log
7184
## Input Body (JSON): { "log": "2001-02-03 04:05:06 very long line that was split by the logging driver\n", "stream": "stdout" }
@@ -75,14 +88,12 @@ filelog/containers:
7588
parse_to: attributes
7689
output: replace-body
7790
regex: "^(?P<log>.*)\n$"
78-
if: body matches "^.*\n$"
7991
type: regex_parser
8092

8193
- id: replace-body
8294
type: move
8395
from: attributes.log
8496
to: body
85-
if: attributes.log != nil
8697

8798
{{- if .Values.sumologic.logs.multiline.enabled }}
8899
## Perform multiline detection

‎tests/helm/logs_test.go‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -464,6 +464,7 @@ sumologic:
464464
Id string
465465
Type string
466466
Output string
467+
Fields []string
467468
}
468469
} `yaml:"filelog/containers"`
469470
}
@@ -472,10 +473,16 @@ sumologic:
472473
require.NoError(t, err)
473474

474475
keepTimeOperatorFound := false
476+
477+
operatorLoop:
475478
for _, operator := range otelConfig.Receivers.Filelog.Operators {
476-
if operator.Id == "move-time-attribute" {
477-
keepTimeOperatorFound = true
478-
break
479+
if operator.Id == "keep-fields" {
480+
for _, field := range operator.Fields {
481+
if field == "attributes[\"time\"]" {
482+
keepTimeOperatorFound = true
483+
break operatorLoop
484+
}
485+
}
479486
}
480487
}
481488
require.True(t, keepTimeOperatorFound)

‎tests/helm/testdata/goldenfile/logs_otc/basic.output.yaml‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -73,9 +73,15 @@ data:
7373
- attributes["k8s.container.name"]
7474
- attributes["k8s.namespace.name"]
7575
- attributes["log.file.path"]
76+
id: keep-fields
7677
type: retain
78+
- default: multiline
79+
id: strip-trailing-newline-router
80+
routes:
81+
- expr: body matches "^.*\n$"
82+
output: strip-trailing-newline
83+
type: router
7784
- id: strip-trailing-newline
78-
if: body matches "^.*\n$"
7985
output: replace-body
8086
parse_from: body
8187
parse_to: attributes
@@ -85,7 +91,6 @@ data:
8591
type: regex_parser
8692
- from: attributes.log
8793
id: replace-body
88-
if: attributes.log != nil
8994
to: body
9095
type: move
9196
- default: merge-multiline-logs

‎tests/helm/testdata/goldenfile/logs_otc/debug.output.yaml‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -83,9 +83,15 @@ data:
8383
- attributes["k8s.container.name"]
8484
- attributes["k8s.namespace.name"]
8585
- attributes["log.file.path"]
86+
id: keep-fields
8687
type: retain
88+
- default: multiline
89+
id: strip-trailing-newline-router
90+
routes:
91+
- expr: body matches "^.*\n$"
92+
output: strip-trailing-newline
93+
type: router
8794
- id: strip-trailing-newline
88-
if: body matches "^.*\n$"
8995
output: replace-body
9096
parse_from: body
9197
parse_to: attributes
@@ -95,7 +101,6 @@ data:
95101
type: regex_parser
96102
- from: attributes.log
97103
id: replace-body
98-
if: attributes.log != nil
99104
to: body
100105
type: move
101106
- default: merge-multiline-logs

‎tests/helm/testdata/goldenfile/logs_otc_daemonset/multiple_multiline.output.yaml‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -73,9 +73,15 @@ data:
7373
- attributes["k8s.container.name"]
7474
- attributes["k8s.namespace.name"]
7575
- attributes["log.file.path"]
76+
id: keep-fields
7677
type: retain
78+
- default: multiline
79+
id: strip-trailing-newline-router
80+
routes:
81+
- expr: body matches "^.*\n$"
82+
output: strip-trailing-newline
83+
type: router
7784
- id: strip-trailing-newline
78-
if: body matches "^.*\n$"
7985
output: replace-body
8086
parse_from: body
8187
parse_to: attributes
@@ -85,7 +91,6 @@ data:
8591
type: regex_parser
8692
- from: attributes.log
8793
id: replace-body
88-
if: attributes.log != nil
8994
to: body
9095
type: move
9196
- default: merge-multiline-logs

‎tests/helm/testdata/goldenfile/logs_otc_windows/configmap/basic.output.yaml‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,9 +52,15 @@ data:
5252
- attributes["k8s.container.name"]
5353
- attributes["k8s.namespace.name"]
5454
- attributes["log.file.path"]
55+
id: keep-fields
5556
type: retain
57+
- default: multiline
58+
id: strip-trailing-newline-router
59+
routes:
60+
- expr: body matches "^.*\n$"
61+
output: strip-trailing-newline
62+
type: router
5663
- id: strip-trailing-newline
57-
if: body matches "^.*\n$"
5864
output: replace-body
5965
parse_from: body
6066
parse_to: attributes
@@ -64,7 +70,6 @@ data:
6470
type: regex_parser
6571
- from: attributes.log
6672
id: replace-body
67-
if: attributes.log != nil
6873
to: body
6974
type: move
7075
- default: merge-multiline-logs

‎tests/helm/testdata/goldenfile/logs_otc_windows/configmap/debug.output.yaml‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,9 +63,15 @@ data:
6363
- attributes["k8s.container.name"]
6464
- attributes["k8s.namespace.name"]
6565
- attributes["log.file.path"]
66+
id: keep-fields
6667
type: retain
68+
- default: multiline
69+
id: strip-trailing-newline-router
70+
routes:
71+
- expr: body matches "^.*\n$"
72+
output: strip-trailing-newline
73+
type: router
6774
- id: strip-trailing-newline
68-
if: body matches "^.*\n$"
6975
output: replace-body
7076
parse_from: body
7177
parse_to: attributes
@@ -75,7 +81,6 @@ data:
7581
type: regex_parser
7682
- from: attributes.log
7783
id: replace-body
78-
if: attributes.log != nil
7984
to: body
8085
type: move
8186
- default: merge-multiline-logs

‎tests/helm/testdata/goldenfile/logs_otc_windows/daemonset/multiple_multiline.output.yaml‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,9 +52,15 @@ data:
5252
- attributes["k8s.container.name"]
5353
- attributes["k8s.namespace.name"]
5454
- attributes["log.file.path"]
55+
id: keep-fields
5556
type: retain
57+
- default: multiline
58+
id: strip-trailing-newline-router
59+
routes:
60+
- expr: body matches "^.*\n$"
61+
output: strip-trailing-newline
62+
type: router
5663
- id: strip-trailing-newline
57-
if: body matches "^.*\n$"
5864
output: replace-body
5965
parse_from: body
6066
parse_to: attributes
@@ -64,7 +70,6 @@ data:
6470
type: regex_parser
6571
- from: attributes.log
6672
id: replace-body
67-
if: attributes.log != nil
6873
to: body
6974
type: move
7075
- default: merge-multiline-logs

0 commit comments

Comments
 (0)