From e772d99581d667874e68ede4b2d2cb1c599ff24d Mon Sep 17 00:00:00 2001 From: hedhyw Date: Wed, 26 Mar 2025 09:31:10 +0700 Subject: [PATCH] feat: support timestamp reformatting --- assets/example.log | 4 +++- example.jlv.jsonc | 25 +++++++++++++++++++++++++ internal/pkg/config/config.go | 21 +++++++++++++++++++++ internal/pkg/config/config_test.go | 19 +++++++++++++++++++ internal/pkg/source/entry.go | 13 ++++++++++++- internal/pkg/source/entry_test.go | 6 +++++- 6 files changed, 85 insertions(+), 3 deletions(-) diff --git a/assets/example.log b/assets/example.log index 8a2eb61..1c46a6f 100644 --- a/assets/example.log +++ b/assets/example.log @@ -53,4 +53,6 @@ plain text log {"time":883612800.45,"level":"WARN","message": "If a man knows not to which port he sails, no wind is favorable","author": "Seneca"} {"time":"915148800000.45","level":"VERBOSE","message": "Begin at once to live, and count each separate day as a separate life","author": "Seneca"} {"time":946684800.45,"level":"VERBOSE","message": "Begin at once to live, and count each separate day as a separate life","author": "Seneca"} -{"@timestamp":"2025-03-06T11:55:53.723682+01:00","@version":1,"host":"bad21ee895cd","message":"Message ..","level":"INFO"} +{"@timestamp":"2025-03-06T11:55:53.723682+01:00","@version":1,"host":"bad21ee895cd","message":"Symphony format","level":"INFO"} +{"time": "02 Jan 06 15:04 MST","level":"VERBOSE","message": "RFC822 time","author": "Seneca"} +{"time": "Mon, 02 Jan 2006 15:04:05 MST","level":"VERBOSE","message": "RFC1123 time","author": "Seneca"} diff --git a/example.jlv.jsonc b/example.jlv.jsonc index 8c9786a..630ce63 100644 --- a/example.jlv.jsonc +++ b/example.jlv.jsonc @@ -65,6 +65,31 @@ "width": 0 } ], + // Time layouts to reformat. + // + // If the time field has been parsed to any of the specified layouts, + // it will be formatted according to "time_format" configuration. + // + // See: https://pkg.go.dev/time#pkg-constants. + "time_layouts": [ + "01/02 03:04:05PM '06 -0700", + "Mon Jan _2 15:04:05 2006", + "Mon Jan _2 15:04:05 MST 2006", + "Mon Jan 02 15:04:05 -0700 2006", + "02 Jan 06 15:04 MST", + "02 Jan 06 15:04 -0700", + "Monday, 02-Jan-06 15:04:05 MST", + "Mon, 02 Jan 2006 15:04:05 MST", + "Mon, 02 Jan 2006 15:04:05 -0700", + "2006-01-02T15:04:05Z07:00", + "2006-01-02T15:04:05.999999999Z07:00", + "Jan _2 15:04:05", + "Jan _2 15:04:05.000", + "Jan _2 15:04:05.000000", + "Jan _2 15:04:05.000000000", + "2006-01-02 15:04:05", + "2006-01-02" + ], // Mapping of log level. // Possible values: none, trace, debug, info, warn, error, panic, fatal. "customLevelMapping": { diff --git a/internal/pkg/config/config.go b/internal/pkg/config/config.go index ee725f6..60754e2 100644 --- a/internal/pkg/config/config.go +++ b/internal/pkg/config/config.go @@ -25,6 +25,8 @@ type Config struct { Path string `json:"-"` Fields []Field `json:"fields" validate:"min=1"` + // TimeLayouts to reformat. + TimeLayouts []string `json:"time_layouts"` CustomLevelMapping map[string]string `json:"customLevelMapping"` @@ -65,6 +67,25 @@ func GetDefaultConfig() *Config { Path: "default", CustomLevelMapping: GetDefaultCustomLevelMapping(), MaxFileSizeBytes: 2 * units.GB, + TimeLayouts: []string{ + time.Layout, + time.ANSIC, + time.UnixDate, + time.RubyDate, + time.RFC822, + time.RFC822Z, + time.RFC850, + time.RFC1123, + time.RFC1123Z, + time.RFC3339, + time.RFC3339Nano, + time.Stamp, + time.StampMilli, + time.StampMicro, + time.StampNano, + time.DateTime, + time.DateOnly, + }, Fields: []Field{{ Title: "Time", Kind: FieldKindNumericTime, diff --git a/internal/pkg/config/config_test.go b/internal/pkg/config/config_test.go index 9cf2466..d5ddd80 100644 --- a/internal/pkg/config/config_test.go +++ b/internal/pkg/config/config_test.go @@ -138,6 +138,25 @@ func ExampleGetDefaultConfig() { // "width": 0 // } // ], + // "time_layouts": [ + // "01/02 03:04:05PM '06 -0700", + // "Mon Jan _2 15:04:05 2006", + // "Mon Jan _2 15:04:05 MST 2006", + // "Mon Jan 02 15:04:05 -0700 2006", + // "02 Jan 06 15:04 MST", + // "02 Jan 06 15:04 -0700", + // "Monday, 02-Jan-06 15:04:05 MST", + // "Mon, 02 Jan 2006 15:04:05 MST", + // "Mon, 02 Jan 2006 15:04:05 -0700", + // "2006-01-02T15:04:05Z07:00", + // "2006-01-02T15:04:05.999999999Z07:00", + // "Jan _2 15:04:05", + // "Jan _2 15:04:05.000", + // "Jan _2 15:04:05.000000", + // "Jan _2 15:04:05.000000000", + // "2006-01-02 15:04:05", + // "2006-01-02" + // ], // "customLevelMapping": { // "10": "trace", // "20": "debug", diff --git a/internal/pkg/source/entry.go b/internal/pkg/source/entry.go index e610eb1..0f77a0e 100644 --- a/internal/pkg/source/entry.go +++ b/internal/pkg/source/entry.go @@ -173,7 +173,7 @@ func formatField( case config.FieldKindLevel: return string(ParseLevel(formatMessage(value), cfg.CustomLevelMapping)) case config.FieldKindTime: - return formatMessage(value) + return formatMessage(reformatTime(value, cfg.TimeLayouts, timeFormat)) case config.FieldKindSecondTime: return formatMessage(formatTimeValue(value, unitSeconds, timeFormat)) case config.FieldKindMilliTime: @@ -187,6 +187,17 @@ func formatField( } } +func reformatTime(value string, layoutsToReformat []string, timeFormat string) string { + for _, laoyout := range layoutsToReformat { + parsed, err := time.Parse(laoyout, value) + if err == nil { + return parsed.Format(timeFormat) + } + } + + return value +} + // parseLogEntry parses a single log entry from the json line. func parseLogEntry( line json.RawMessage, diff --git a/internal/pkg/source/entry_test.go b/internal/pkg/source/entry_test.go index 20e1675..7580e30 100644 --- a/internal/pkg/source/entry_test.go +++ b/internal/pkg/source/entry_test.go @@ -469,7 +469,11 @@ func TestNumericKindTimeFormatting(t *testing.T) { numericKindCases := [...]timeFormattingTestCase{{ TestName: "Date passthru", JSON: `{"timestamp":"2023-10-08 20:00:00"}`, - ExpectedOutput: "2023-10-08 20:00:00", + ExpectedOutput: "2023-10-08T20:00:00Z", + }, { + TestName: "RFC1123 passthru", + JSON: `{"@timestamp":"Mon, 02 Jan 2006 15:04:05 MST"}`, + ExpectedOutput: "2006-01-02T15:04:05Z", }, { TestName: "Non-date string", JSON: `{"timestamp":"-"}`,