From 224ea9994f538a17c71d9316bd41f496d5e3c895 Mon Sep 17 00:00:00 2001 From: Roman Suvorov Date: Mon, 23 Dec 2024 17:05:10 +0100 Subject: [PATCH 1/8] with commented code --- internal/transformations/hex_decode.go | 152 ++++++++++++++++++++ internal/transformations/transformations.go | 1 + 2 files changed, 153 insertions(+) create mode 100644 internal/transformations/hex_decode.go diff --git a/internal/transformations/hex_decode.go b/internal/transformations/hex_decode.go new file mode 100644 index 000000000..91f874632 --- /dev/null +++ b/internal/transformations/hex_decode.go @@ -0,0 +1,152 @@ +package transformations + +// import ( +// "encoding/hex" + +// "github.com/corazawaf/coraza/v3/internal/strings" +// ) + +// func hexDecode(data string) (string, bool, error) { +// src := []byte(data) + +// if len(src)%2 != 0 { +// src = src[:len(src)-1] +// } +// dst := make([]byte, hex.DecodedLen(len(src))) + +// _, err := hex.Decode(dst, src) +// if err != nil { +// return "", false, err +// } + +// return strings.WrapUnsafe(dst), true, nil +// } + +// // escapePrintableAndNonPrintable converts printable characters directly, and non-printable bytes to \xNN format. +// func escapePrintableAndNonPrintable(data []byte) string { +// var result strings.Builder +// for _, b := range data { +// if b >= 32 && b <= 126 { // Printable ASCII characters +// result.WriteByte(b) +// } else { +// // Non-printable characters, format as \xNN +// result.WriteString(fmt.Sprintf("\\x%02x", b)) +// } +// } +// return result.String() +// } + +// package transformations + +// // hexDecode decodes a hexadecimal-encoded string into a string representation. +// // Invalid or incomplete hex sequences are ignored. +// func hexDecode(data string) (string, bool, error) { +// // Filter out invalid characters +// removeNext := len(data)%2 == 0 +// cleaned := make([]byte, 0, len(data)*2) +// for i := 0; i < len(data); i++ { +// if cstrings.ValidHex(data[i]) { +// cleaned = append(cleaned, data[i]) +// continue +// } +// if removeNext { +// i++ +// removeNext = false +// } +// } + +// // Drop the last character for odd-length strings +// if len(cleaned)%2 != 0 { +// cleaned = cleaned[:len(cleaned)-1] +// } + +// // Decode cleaned input +// decoded := make([]byte, hex.DecodedLen(len(cleaned))) +// _, err := hex.Decode(decoded, cleaned) +// if err != nil { +// return "", false, fmt.Errorf("failed to decode: %w", err) +// } + +// return cstrings.WrapUnsafe(decoded), true, nil +// } + +// func hexDecode(data string) (string, bool, error) { +// src := []byte(data) +// dst := make([]byte, 0, hex.DecodedLen(len(src))) + +// var processed int + +// for { +// if processed >= len(src) { +// break +// } +// s := src[processed:] +// d := make([]byte, hex.DecodedLen(len(s))) +// n, err := hex.Decode(d, s) +// if err != nil { +// return "", false, fmt.Errorf("failed to decode: %w", err) +// } +// dst = append(dst, d...) + +// if processed < len(dst) { +// // will try to start from the next character after invalid +// processed = processed + n + 1 +// fmt.Println("here", processed) +// } +// } + +// return cstrings.WrapUnsafe(dst), true, nil +// } + +// func hexDecodeFrom() + +// const ( +// hextable = "0123456789abcdef" +// reverseHexTable = "" + +// "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + +// "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + +// "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + +// "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\xff\xff\xff\xff\xff\xff" + +// "\xff\x0a\x0b\x0c\x0d\x0e\x0f\xff\xff\xff\xff\xff\xff\xff\xff\xff" + +// "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + +// "\xff\x0a\x0b\x0c\x0d\x0e\x0f\xff\xff\xff\xff\xff\xff\xff\xff\xff" + +// "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + +// "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + +// "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + +// "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + +// "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + +// "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + +// "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + +// "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + +// "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" +// ) + +// func hDecode(dst, src []byte) { +// j := 1 +// OUT: +// for j < len(src) { +// p := src[j-1] +// q := src[j] + +// a := reverseHexTable[p] +// b := reverseHexTable[q] +// if a > 0x0f { +// // skip the symbol and continue with the next as first +// j++ +// continue +// } +// if b > 0x0f { +// if j+1 < len(src) { +// b := reverseHexTable[src[j+1]] +// if b > 0x0f { +// j += 2 +// continue +// } +// } else { +// break OUT +// } +// } +// dst = append(dst, (a<<4)|b) +// j += 2 +// } +// } diff --git a/internal/transformations/transformations.go b/internal/transformations/transformations.go index 704e04ad1..516dd8b08 100644 --- a/internal/transformations/transformations.go +++ b/internal/transformations/transformations.go @@ -34,6 +34,7 @@ func init() { Register("compressWhitespace", compressWhitespace) Register("cssDecode", cssDecode) Register("escapeSeqDecode", escapeSeqDecode) + // Register("hexDecode", hexDecode) Register("hexEncode", hexEncode) Register("htmlEntityDecode", htmlEntityDecode) Register("jsDecode", jsDecode) From 244ebc1e3dd5dd278d5e04dfde73d0250dd1282c Mon Sep 17 00:00:00 2001 From: Roman Suvorov Date: Mon, 23 Dec 2024 17:07:18 +0100 Subject: [PATCH 2/8] go sum update --- testing/coreruleset/go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/testing/coreruleset/go.sum b/testing/coreruleset/go.sum index b8b6579fe..ec4327dc9 100644 --- a/testing/coreruleset/go.sum +++ b/testing/coreruleset/go.sum @@ -101,8 +101,6 @@ github.com/valllabh/ocsf-schema-golang v1.0.3 h1:eR8k/3jP/OOqB8LRCtdJ4U+vlgd/gk5 github.com/valllabh/ocsf-schema-golang v1.0.3/go.mod h1:sZ3as9xqm1SSK5feFWIR2CuGeGRhsM7TR1MbpBctzPk= github.com/yargevad/filepathx v1.0.0 h1:SYcT+N3tYGi+NvazubCNlvgIPbzAk7i7y2dwg3I5FYc= github.com/yargevad/filepathx v1.0.0/go.mod h1:BprfX/gpYNJHJfc35GjRRpVcwWXS89gGulUIU5tK3tA= -golang.org/x/crypto v0.30.0 h1:RwoQn3GkWiMkzlX562cLB7OxWvjH1L8xutO2WoJcRoY= -golang.org/x/crypto v0.30.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= From 3cccb87d675f6307fdb6c44fdf6234882640bfee Mon Sep 17 00:00:00 2001 From: Roman Suvorov Date: Mon, 6 Jan 2025 16:25:14 +0100 Subject: [PATCH 3/8] simple implementation --- internal/transformations/hex_decode.go | 161 ++---------------- .../transformations/testdata/hexDecode.json | 77 ++++----- internal/transformations/transformations.go | 2 +- 3 files changed, 52 insertions(+), 188 deletions(-) diff --git a/internal/transformations/hex_decode.go b/internal/transformations/hex_decode.go index 91f874632..445fdc2e3 100644 --- a/internal/transformations/hex_decode.go +++ b/internal/transformations/hex_decode.go @@ -1,152 +1,23 @@ package transformations -// import ( -// "encoding/hex" +import ( + "encoding/hex" -// "github.com/corazawaf/coraza/v3/internal/strings" -// ) + "github.com/corazawaf/coraza/v3/internal/strings" +) -// func hexDecode(data string) (string, bool, error) { -// src := []byte(data) +func hexDecode(data string) (string, bool, error) { + src := []byte(data) -// if len(src)%2 != 0 { -// src = src[:len(src)-1] -// } -// dst := make([]byte, hex.DecodedLen(len(src))) + if len(src)%2 != 0 { + src = src[:len(src)-1] + } + dst := make([]byte, hex.DecodedLen(len(src))) -// _, err := hex.Decode(dst, src) -// if err != nil { -// return "", false, err -// } + _, err := hex.Decode(dst, src) + if err != nil { + return "", false, err + } -// return strings.WrapUnsafe(dst), true, nil -// } - -// // escapePrintableAndNonPrintable converts printable characters directly, and non-printable bytes to \xNN format. -// func escapePrintableAndNonPrintable(data []byte) string { -// var result strings.Builder -// for _, b := range data { -// if b >= 32 && b <= 126 { // Printable ASCII characters -// result.WriteByte(b) -// } else { -// // Non-printable characters, format as \xNN -// result.WriteString(fmt.Sprintf("\\x%02x", b)) -// } -// } -// return result.String() -// } - -// package transformations - -// // hexDecode decodes a hexadecimal-encoded string into a string representation. -// // Invalid or incomplete hex sequences are ignored. -// func hexDecode(data string) (string, bool, error) { -// // Filter out invalid characters -// removeNext := len(data)%2 == 0 -// cleaned := make([]byte, 0, len(data)*2) -// for i := 0; i < len(data); i++ { -// if cstrings.ValidHex(data[i]) { -// cleaned = append(cleaned, data[i]) -// continue -// } -// if removeNext { -// i++ -// removeNext = false -// } -// } - -// // Drop the last character for odd-length strings -// if len(cleaned)%2 != 0 { -// cleaned = cleaned[:len(cleaned)-1] -// } - -// // Decode cleaned input -// decoded := make([]byte, hex.DecodedLen(len(cleaned))) -// _, err := hex.Decode(decoded, cleaned) -// if err != nil { -// return "", false, fmt.Errorf("failed to decode: %w", err) -// } - -// return cstrings.WrapUnsafe(decoded), true, nil -// } - -// func hexDecode(data string) (string, bool, error) { -// src := []byte(data) -// dst := make([]byte, 0, hex.DecodedLen(len(src))) - -// var processed int - -// for { -// if processed >= len(src) { -// break -// } -// s := src[processed:] -// d := make([]byte, hex.DecodedLen(len(s))) -// n, err := hex.Decode(d, s) -// if err != nil { -// return "", false, fmt.Errorf("failed to decode: %w", err) -// } -// dst = append(dst, d...) - -// if processed < len(dst) { -// // will try to start from the next character after invalid -// processed = processed + n + 1 -// fmt.Println("here", processed) -// } -// } - -// return cstrings.WrapUnsafe(dst), true, nil -// } - -// func hexDecodeFrom() - -// const ( -// hextable = "0123456789abcdef" -// reverseHexTable = "" + -// "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + -// "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + -// "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + -// "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\xff\xff\xff\xff\xff\xff" + -// "\xff\x0a\x0b\x0c\x0d\x0e\x0f\xff\xff\xff\xff\xff\xff\xff\xff\xff" + -// "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + -// "\xff\x0a\x0b\x0c\x0d\x0e\x0f\xff\xff\xff\xff\xff\xff\xff\xff\xff" + -// "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + -// "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + -// "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + -// "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + -// "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + -// "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + -// "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + -// "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + -// "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" -// ) - -// func hDecode(dst, src []byte) { -// j := 1 -// OUT: -// for j < len(src) { -// p := src[j-1] -// q := src[j] - -// a := reverseHexTable[p] -// b := reverseHexTable[q] -// if a > 0x0f { -// // skip the symbol and continue with the next as first -// j++ -// continue -// } -// if b > 0x0f { -// if j+1 < len(src) { -// b := reverseHexTable[src[j+1]] -// if b > 0x0f { -// j += 2 -// continue -// } -// } else { -// break OUT -// } -// } -// dst = append(dst, (a<<4)|b) -// j += 2 -// } -// } + return strings.WrapUnsafe(dst), true, nil +} diff --git a/internal/transformations/testdata/hexDecode.json b/internal/transformations/testdata/hexDecode.json index 664fbd812..bdc7ac90e 100644 --- a/internal/transformations/testdata/hexDecode.json +++ b/internal/transformations/testdata/hexDecode.json @@ -1,44 +1,37 @@ [ - { - "ret" : 1, - "input" : "", - "type" : "tfn", - "name" : "hexDecode", - "output" : "" - }, - { - "output" : "TestCase", - "ret" : 1, - "name" : "hexDecode", - "input" : "5465737443617365", - "type" : "tfn" - }, - { - "type" : "tfn", - "input" : "546573740043617365", - "name" : "hexDecode", - "ret" : 1, - "output" : "Test\\u0000Case" - }, - { - "output" : "\\x01#Eg\\x89\\x0a#\\x01#Eg\\x89\\x0a", - "type" : "tfn", - "input" : "01234567890a0z01234567890a", - "name" : "hexDecode", - "ret" : 1 - }, - { - "type" : "tfn", - "name" : "hexDecode", - "input" : "01234567890az", - "output" : "\\x01#Eg\\x89\\x0a", - "ret" : 1 - }, - { - "type" : "tfn", - "name" : "hexDecode", - "input" : "01234567890a0", - "output" : "\\x01#Eg\\x89\\x0a", - "ret" : 1 - } + { + "ret": 1, + "input": "", + "type": "tfn", + "name": "hexDecode", + "output": "" + }, + { + "output": "TestCase", + "ret": 1, + "name": "hexDecode", + "input": "5465737443617365", + "type": "tfn" + }, + { + "type": "tfn", + "input": "546573740043617365", + "name": "hexDecode", + "ret": 1, + "output": "Test\\u0000Case" + }, + { + "type": "tfn", + "name": "hexDecode", + "input": "01234567890az", + "output": "\\x01#Eg\\x89\\x0a", + "ret": 1 + }, + { + "type": "tfn", + "name": "hexDecode", + "input": "01234567890a0", + "output": "\\x01#Eg\\x89\\x0a", + "ret": 1 + } ] diff --git a/internal/transformations/transformations.go b/internal/transformations/transformations.go index 516dd8b08..90f3cdaaa 100644 --- a/internal/transformations/transformations.go +++ b/internal/transformations/transformations.go @@ -34,7 +34,7 @@ func init() { Register("compressWhitespace", compressWhitespace) Register("cssDecode", cssDecode) Register("escapeSeqDecode", escapeSeqDecode) - // Register("hexDecode", hexDecode) + Register("hexDecode", hexDecode) Register("hexEncode", hexEncode) Register("htmlEntityDecode", htmlEntityDecode) Register("jsDecode", jsDecode) From 7d496b7ba5699c3d68a5031c78228cb8bd0043f2 Mon Sep 17 00:00:00 2001 From: Roman Suvorov Date: Mon, 6 Jan 2025 16:50:18 +0100 Subject: [PATCH 4/8] add comment --- internal/transformations/hex_decode.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/transformations/hex_decode.go b/internal/transformations/hex_decode.go index 445fdc2e3..e23210c62 100644 --- a/internal/transformations/hex_decode.go +++ b/internal/transformations/hex_decode.go @@ -9,6 +9,9 @@ import ( func hexDecode(data string) (string, bool, error) { src := []byte(data) + // According to RFC-4648 section 8, the valid lenght of src MUST be even. + // Here https://datatracker.ietf.org/doc/html/rfc4648#section-8 + // There was a decision to cut "redundant" bytes for the "best effort aproach" and proceed decoding. if len(src)%2 != 0 { src = src[:len(src)-1] } From 779f6f1769db01ce0f63afaaab9f9dd535382ced Mon Sep 17 00:00:00 2001 From: Roman Suvorov Date: Tue, 7 Jan 2025 09:49:17 +0100 Subject: [PATCH 5/8] add unit tests --- internal/transformations/hex_decode_test.go | 119 ++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 internal/transformations/hex_decode_test.go diff --git a/internal/transformations/hex_decode_test.go b/internal/transformations/hex_decode_test.go new file mode 100644 index 000000000..a61f8dc29 --- /dev/null +++ b/internal/transformations/hex_decode_test.go @@ -0,0 +1,119 @@ +package transformations + +import ( + "testing" +) + +func TestHexDecode(t *testing.T) { + tests := []struct { + name string + input string + expectedOutput string + expectedValid bool + expectError bool + }{ + { + name: "ValidHexadecimalString", + input: "48656c6c6f", + expectedOutput: "Hello", + expectedValid: true, + expectError: false, + }, + { + name: "ValidHexadecimalStringWithWords", + input: "6865786465636f6465", + expectedOutput: "hexdecode", + expectedValid: true, + expectError: false, + }, + { + name: "OddLengthHexadecimalString", + input: "48656c6c6f7", + expectedOutput: "Hello", + expectedValid: true, + expectError: false, + }, + { + name: "InvalidHexadecimalStringWithNonHexCharacters", + input: "YYY", + expectedOutput: "", + expectedValid: false, + expectError: true, + }, + { + name: "InvalidHexadecimalStringWithExtraCharacters", + input: "123G", + expectedOutput: "", + expectedValid: false, + expectError: true, + }, + { + name: "EmptyStringInput", + input: "", + expectedOutput: "", + expectedValid: true, + expectError: false, + }, + { + name: "UppercaseHexString", + input: "48454C4C4F", + expectedOutput: "HELLO", + expectedValid: true, + expectError: false, + }, + { + name: "MixedCaseHexString", + input: "48454c4C4f", + expectedOutput: "HELLO", + expectedValid: true, + expectError: false, + }, + { + name: "SpecialCharactersHexString", + input: "21402324255E262A28", + expectedOutput: "!@#$%^&*(", + expectedValid: true, + expectError: false, + }, + { + name: "LongHexString", + input: "48656C6C6F20576F726C642C20746869732069732061206C6F6E67657220737472696E67", + expectedOutput: "Hello World, this is a longer string", + expectedValid: true, + expectError: false, + }, + { + name: "MultipleConsecutiveInvalidCharacters", + input: "123XYZ789", + expectedOutput: "", + expectedValid: false, + expectError: true, + }, + { + name: "OddLengthWithInvalidCharacter", + input: "48656c6c6fZ", + expectedOutput: "Hello", + expectedValid: true, + expectError: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + output, valid, err := hexDecode(tt.input) + + if (err != nil) != tt.expectError { + t.Errorf("hexDecode(%q): expected error=%v, got error=%v", tt.input, tt.expectError, err) + } + + if output != tt.expectedOutput { + t.Errorf("hexDecode(%q): expected output=%q, got output=%q", tt.input, tt.expectedOutput, output) + } + + if valid != tt.expectedValid { + t.Errorf("hexDecode(%q): expected valid=%v, got valid=%v", tt.input, tt.expectedValid, valid) + } + }) + } +} From f7771f58bf4b8b7263364488ae0ee432a9965221 Mon Sep 17 00:00:00 2001 From: Roman Suvorov Date: Tue, 7 Jan 2025 09:54:47 +0100 Subject: [PATCH 6/8] change names in tests, remove redundant --- internal/transformations/hex_decode_test.go | 39 +++++---------------- 1 file changed, 9 insertions(+), 30 deletions(-) diff --git a/internal/transformations/hex_decode_test.go b/internal/transformations/hex_decode_test.go index a61f8dc29..78869eb0c 100644 --- a/internal/transformations/hex_decode_test.go +++ b/internal/transformations/hex_decode_test.go @@ -13,84 +13,63 @@ func TestHexDecode(t *testing.T) { expectError bool }{ { - name: "ValidHexadecimalString", + name: "valid hexadecimal string", input: "48656c6c6f", expectedOutput: "Hello", expectedValid: true, expectError: false, }, { - name: "ValidHexadecimalStringWithWords", - input: "6865786465636f6465", - expectedOutput: "hexdecode", - expectedValid: true, - expectError: false, - }, - { - name: "OddLengthHexadecimalString", + name: "odd length", input: "48656c6c6f7", expectedOutput: "Hello", expectedValid: true, expectError: false, }, { - name: "InvalidHexadecimalStringWithNonHexCharacters", + name: "invalid with non hex characters", input: "YYY", expectedOutput: "", expectedValid: false, expectError: true, }, { - name: "InvalidHexadecimalStringWithExtraCharacters", + name: "invalid with extra characters", input: "123G", expectedOutput: "", expectedValid: false, expectError: true, }, { - name: "EmptyStringInput", + name: "empty input", input: "", expectedOutput: "", expectedValid: true, expectError: false, }, { - name: "UppercaseHexString", + name: "uppercase hex string", input: "48454C4C4F", expectedOutput: "HELLO", expectedValid: true, expectError: false, }, { - name: "MixedCaseHexString", + name: "mixed case", input: "48454c4C4f", expectedOutput: "HELLO", expectedValid: true, expectError: false, }, { - name: "SpecialCharactersHexString", + name: "special characters", input: "21402324255E262A28", expectedOutput: "!@#$%^&*(", expectedValid: true, expectError: false, }, { - name: "LongHexString", - input: "48656C6C6F20576F726C642C20746869732069732061206C6F6E67657220737472696E67", - expectedOutput: "Hello World, this is a longer string", - expectedValid: true, - expectError: false, - }, - { - name: "MultipleConsecutiveInvalidCharacters", - input: "123XYZ789", - expectedOutput: "", - expectedValid: false, - expectError: true, - }, - { - name: "OddLengthWithInvalidCharacter", + name: "odd length with invalid character", input: "48656c6c6fZ", expectedOutput: "Hello", expectedValid: true, From 02b52480fc16f72c3e95b35af4e763e33ca3146e Mon Sep 17 00:00:00 2001 From: Roman Suvorov Date: Tue, 7 Jan 2025 09:56:49 +0100 Subject: [PATCH 7/8] grammar fix --- internal/transformations/hex_decode.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/transformations/hex_decode.go b/internal/transformations/hex_decode.go index e23210c62..340ab025e 100644 --- a/internal/transformations/hex_decode.go +++ b/internal/transformations/hex_decode.go @@ -9,9 +9,9 @@ import ( func hexDecode(data string) (string, bool, error) { src := []byte(data) - // According to RFC-4648 section 8, the valid lenght of src MUST be even. + // According to RFC-4648 section 8, the valid length of src MUST be even. // Here https://datatracker.ietf.org/doc/html/rfc4648#section-8 - // There was a decision to cut "redundant" bytes for the "best effort aproach" and proceed decoding. + // There was a decision to cut "redundant" bytes for the "best effort approach" and proceed decoding. if len(src)%2 != 0 { src = src[:len(src)-1] } From 81da04202110c2335346937474dc5a2bfac664a2 Mon Sep 17 00:00:00 2001 From: Roman Suvorov Date: Thu, 9 Jan 2025 16:35:14 +0100 Subject: [PATCH 8/8] updated hex method and tests --- examples/http-server/go.mod | 2 +- examples/http-server/go.sum | 8 ++++---- go.sum | 4 ---- internal/transformations/hex_decode.go | 12 +----------- internal/transformations/hex_decode_test.go | 14 +++++++------- internal/transformations/testdata/hexDecode.json | 16 ++++++++-------- testing/coreruleset/go.mod | 6 +++--- testing/coreruleset/go.sum | 12 ++++++------ 8 files changed, 30 insertions(+), 44 deletions(-) diff --git a/examples/http-server/go.mod b/examples/http-server/go.mod index 1b0fef49e..480fd7b57 100644 --- a/examples/http-server/go.mod +++ b/examples/http-server/go.mod @@ -11,7 +11,7 @@ require ( github.com/tidwall/gjson v1.18.0 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect - golang.org/x/net v0.33.0 // indirect + golang.org/x/net v0.34.0 // indirect golang.org/x/sync v0.10.0 // indirect golang.org/x/tools v0.22.0 // indirect rsc.io/binaryregexp v0.2.0 // indirect diff --git a/examples/http-server/go.sum b/examples/http-server/go.sum index 1a3cbd012..05299a64f 100644 --- a/examples/http-server/go.sum +++ b/examples/http-server/go.sum @@ -19,12 +19,12 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= -golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= +golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE= diff --git a/go.sum b/go.sum index d31bf04dc..813c8fbe4 100644 --- a/go.sum +++ b/go.sum @@ -53,8 +53,6 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= -golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= -golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -75,8 +73,6 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/internal/transformations/hex_decode.go b/internal/transformations/hex_decode.go index 340ab025e..3e0aff391 100644 --- a/internal/transformations/hex_decode.go +++ b/internal/transformations/hex_decode.go @@ -7,17 +7,7 @@ import ( ) func hexDecode(data string) (string, bool, error) { - src := []byte(data) - - // According to RFC-4648 section 8, the valid length of src MUST be even. - // Here https://datatracker.ietf.org/doc/html/rfc4648#section-8 - // There was a decision to cut "redundant" bytes for the "best effort approach" and proceed decoding. - if len(src)%2 != 0 { - src = src[:len(src)-1] - } - dst := make([]byte, hex.DecodedLen(len(src))) - - _, err := hex.Decode(dst, src) + dst, err := hex.DecodeString(data) if err != nil { return "", false, err } diff --git a/internal/transformations/hex_decode_test.go b/internal/transformations/hex_decode_test.go index 78869eb0c..9b145968b 100644 --- a/internal/transformations/hex_decode_test.go +++ b/internal/transformations/hex_decode_test.go @@ -22,13 +22,13 @@ func TestHexDecode(t *testing.T) { { name: "odd length", input: "48656c6c6f7", - expectedOutput: "Hello", - expectedValid: true, - expectError: false, + expectedOutput: "", + expectedValid: false, + expectError: true, }, { name: "invalid with non hex characters", - input: "YYY", + input: "YyYy", expectedOutput: "", expectedValid: false, expectError: true, @@ -71,9 +71,9 @@ func TestHexDecode(t *testing.T) { { name: "odd length with invalid character", input: "48656c6c6fZ", - expectedOutput: "Hello", - expectedValid: true, - expectError: false, + expectedOutput: "", + expectedValid: false, + expectError: true, }, } diff --git a/internal/transformations/testdata/hexDecode.json b/internal/transformations/testdata/hexDecode.json index bdc7ac90e..99345b222 100644 --- a/internal/transformations/testdata/hexDecode.json +++ b/internal/transformations/testdata/hexDecode.json @@ -22,16 +22,16 @@ }, { "type": "tfn", - "name": "hexDecode", - "input": "01234567890az", - "output": "\\x01#Eg\\x89\\x0a", - "ret": 1 + "name": "invalidCharacter", + "input": "01234567890z", + "output": "", + "ret": 0 }, { "type": "tfn", - "name": "hexDecode", - "input": "01234567890a0", - "output": "\\x01#Eg\\x89\\x0a", - "ret": 1 + "name": "invalidLen", + "input": "54657374004", + "output": "", + "ret": 0 } ] diff --git a/testing/coreruleset/go.mod b/testing/coreruleset/go.mod index a6904f64c..c4f091738 100644 --- a/testing/coreruleset/go.mod +++ b/testing/coreruleset/go.mod @@ -44,10 +44,10 @@ require ( github.com/tidwall/pretty v1.2.1 // indirect github.com/valllabh/ocsf-schema-golang v1.0.3 // indirect github.com/yargevad/filepathx v1.0.0 // indirect - golang.org/x/crypto v0.31.0 // indirect - golang.org/x/net v0.33.0 // indirect + golang.org/x/crypto v0.32.0 // indirect + golang.org/x/net v0.34.0 // indirect golang.org/x/sync v0.10.0 // indirect - golang.org/x/sys v0.28.0 // indirect + golang.org/x/sys v0.29.0 // indirect golang.org/x/time v0.8.0 // indirect google.golang.org/protobuf v1.34.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/testing/coreruleset/go.sum b/testing/coreruleset/go.sum index baf6b46cc..a05ccd53a 100644 --- a/testing/coreruleset/go.sum +++ b/testing/coreruleset/go.sum @@ -97,19 +97,19 @@ github.com/valllabh/ocsf-schema-golang v1.0.3 h1:eR8k/3jP/OOqB8LRCtdJ4U+vlgd/gk5 github.com/valllabh/ocsf-schema-golang v1.0.3/go.mod h1:sZ3as9xqm1SSK5feFWIR2CuGeGRhsM7TR1MbpBctzPk= github.com/yargevad/filepathx v1.0.0 h1:SYcT+N3tYGi+NvazubCNlvgIPbzAk7i7y2dwg3I5FYc= github.com/yargevad/filepathx v1.0.0/go.mod h1:BprfX/gpYNJHJfc35GjRRpVcwWXS89gGulUIU5tK3tA= -golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= -golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= +golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= -golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= +golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg= golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA=