From e4cad49780086ea2ecd79881bd6ce383b4d2a474 Mon Sep 17 00:00:00 2001 From: "vitess-bot[bot]" <108069721+vitess-bot[bot]@users.noreply.github.com> Date: Wed, 29 Jul 2026 00:31:15 +0200 Subject: [PATCH 1/2] Cherry-pick ec6fdee9830a1f7117db2e6a59f634c4f0982713 with conflicts --- go/mysql/json/LICENSE.rapidjson | 37 ++ go/mysql/json/marshal.go | 716 +++++++++++++++++++++++++++++ go/mysql/json/marshal_test.go | 155 +++++++ go/mysql/json/parser.go | 370 +++++++++++++-- go/mysql/json/parser_bench_test.go | 114 +++++ go/mysql/json/parser_test.go | 284 +++++++++++- 6 files changed, 1630 insertions(+), 46 deletions(-) create mode 100644 go/mysql/json/LICENSE.rapidjson create mode 100644 go/mysql/json/parser_bench_test.go diff --git a/go/mysql/json/LICENSE.rapidjson b/go/mysql/json/LICENSE.rapidjson new file mode 100644 index 00000000000..b1474db24d4 --- /dev/null +++ b/go/mysql/json/LICENSE.rapidjson @@ -0,0 +1,37 @@ +Tencent is pleased to support the open source community by making RapidJSON +available. + +Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. + +Parts of parser.go are derived from RapidJSON 1.1.0, the JSON parser MySQL +parses documents with, so that Vitess accepts exactly the documents MySQL does: +mysqlNumberFits follows GenericReader::ParseNumber in +include/rapidjson/reader.h, scaleByPow10 and movePoint follow +StrtodNormalPrecision and FastPath in include/rapidjson/internal/strtod.h, and +the pow10 table holds the same values as Pow10 in +include/rapidjson/internal/pow10.h. No RapidJSON source is included here; the Go +code was written to reproduce what those functions decide. + +RapidJSON is licensed under the MIT License: + +The MIT License (MIT) + +Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/go/mysql/json/marshal.go b/go/mysql/json/marshal.go index 97d14a336c8..ab60d11fab8 100644 --- a/go/mysql/json/marshal.go +++ b/go/mysql/json/marshal.go @@ -175,6 +175,722 @@ func MarshalSQLValue(buf []byte) (*sqltypes.Value, error) { return nil, err } +<<<<<<< HEAD newVal := sqltypes.MakeTrusted(querypb.Type_RAW, jsonVal.MarshalSQLTo(nil)) return &newVal, nil +||||||| parent of ec6fdee983 (mysql/json: read numbers the way MySQL does (#20722)) +// AppendMarshalSQL converts text JSON into a SQL expression using +// JSON_OBJECT/JSON_ARRAY syntax, writing directly to buf. It scans +// the raw bytes directly without building an intermediate tree. +// +// This has O(recursion depth) memory overhead versus O(total nodes * 72 +// bytes) for the tree-based Value.MarshalSQLTo, and avoids the per-token +// heap allocations of encoding/json.Decoder.Token. +// +// The output format matches the tree-based encoder so that MySQL stores +// identical binary JSON, including preservation of large integer precision +// via bare numeric literals. +func AppendMarshalSQL(buf *bytes2.Buffer, raw []byte) error { + w := sqlWriter{ + data: raw, + buf: buf, + } + if err := w.writeValue(true, 0); err != nil { + return err + } + w.skipWhitespace() + if w.pos != len(w.data) { + return errors.New("unexpected trailing data after JSON value") + } + return nil +} + +// sqlWriter converts text JSON into SQL expressions by scanning +// the raw bytes directly. It uses O(recursion depth) memory overhead +// plus a reusable scratch buffer for string unescaping. +type sqlWriter struct { + data []byte + pos int + buf *bytes2.Buffer + scratch []byte +} + +func (w *sqlWriter) skipWhitespace() { + for w.pos < len(w.data) { + switch w.data[w.pos] { + case ' ', '\t', '\n', '\r': + w.pos++ + default: + return + } + } +} + +func (w *sqlWriter) writeValue(top bool, depth int) error { + if depth >= MaxDepth { + return fmt.Errorf("too big depth for the nested JSON; it exceeds %d", MaxDepth) + } + w.skipWhitespace() + if w.pos >= len(w.data) { + return errors.New("unexpected end of JSON input") + } + switch w.data[w.pos] { + case '{': + w.pos++ + return w.writeObject(depth) + case '[': + w.pos++ + return w.writeArray(depth) + case '"': + return w.writeString(top) + case 't', 'f': + return w.writeBool(top) + case 'n': + return w.writeNull(top) + default: + if w.data[w.pos] >= '0' && w.data[w.pos] <= '9' || w.data[w.pos] == '-' { + return w.writeNumber(top) + } + return fmt.Errorf("unexpected character %q in JSON", w.data[w.pos]) + } +} + +func (w *sqlWriter) writeObject(depth int) error { + w.buf.WriteString("JSON_OBJECT(") + first := true + for { + w.skipWhitespace() + if w.pos >= len(w.data) { + return errors.New("unexpected end of JSON input in object") + } + if w.data[w.pos] == '}' { + w.pos++ + w.buf.WriteByte(')') + return nil + } + if !first { + if w.data[w.pos] != ',' { + return fmt.Errorf("expected ',' or '}' in object, got %q", w.data[w.pos]) + } + w.pos++ + w.buf.WriteString(", ") + w.skipWhitespace() + } + first = false + + // Key (always a string). + if w.pos >= len(w.data) || w.data[w.pos] != '"' { + return errors.New("expected string key in JSON object") + } + w.buf.WriteString("_utf8mb4") + if err := w.writeStringContent(); err != nil { + return fmt.Errorf("reading JSON object key: %w", err) + } + w.buf.WriteString(", ") + + // Colon separator. + w.skipWhitespace() + if w.pos >= len(w.data) || w.data[w.pos] != ':' { + return errors.New("expected ':' after object key") + } + w.pos++ + + // Value. + if err := w.writeValue(false, depth+1); err != nil { + return err + } + } +} + +func (w *sqlWriter) writeArray(depth int) error { + w.buf.WriteString("JSON_ARRAY(") + first := true + for { + w.skipWhitespace() + if w.pos >= len(w.data) { + return errors.New("unexpected end of JSON input in array") + } + if w.data[w.pos] == ']' { + w.pos++ + w.buf.WriteByte(')') + return nil + } + if !first { + if w.data[w.pos] != ',' { + return fmt.Errorf("expected ',' or ']' in array, got %q", w.data[w.pos]) + } + w.pos++ + w.buf.WriteString(", ") + } + first = false + + if err := w.writeValue(false, depth+1); err != nil { + return err + } + } +} + +func (w *sqlWriter) writeString(top bool) error { + if top { + w.buf.WriteString("CAST(JSON_QUOTE(") + } + w.buf.WriteString("_utf8mb4") + if err := w.writeStringContent(); err != nil { + return err + } + if top { + w.buf.WriteString(") as JSON)") + } + return nil +} + +// writeStringContent reads a JSON string starting at w.pos (which must point +// at the opening '"'), JSON-unescapes it, SQL-encodes it into w.buf, and +// advances w.pos past the closing '"'. +func (w *sqlWriter) writeStringContent() error { + if w.pos >= len(w.data) || w.data[w.pos] != '"' { + return errors.New("expected '\"' at start of string") + } + w.pos++ // skip opening '"' + + // Scan to find the closing quote, tracking whether escape sequences exist. + start := w.pos + hasEscape := false + for w.pos < len(w.data) { + ch := w.data[w.pos] + if ch == '\\' { + if w.pos+1 >= len(w.data) { + return errors.New("unterminated string in JSON") + } + hasEscape = true + w.pos += 2 // skip '\' and the escaped character + continue + } + if ch == '"' { + break + } + w.pos++ + } + if w.pos >= len(w.data) { + return errors.New("unterminated string in JSON") + } + + content := w.data[start:w.pos] + w.pos++ // skip closing '"' + + if !hasEscape { + // Fast path: no escape sequences, raw bytes are the decoded string. + sqltypes.MakeTrusted(querypb.Type_VARCHAR, content).EncodeSQLBytes2(w.buf) + } else { + // Slow path: unescape JSON into scratch buffer, then SQL-encode. + var err error + w.scratch, err = unescapeJSON(w.scratch[:0], content) + if err != nil { + return err + } + sqltypes.MakeTrusted(querypb.Type_VARCHAR, w.scratch).EncodeSQLBytes2(w.buf) + } + return nil +} + +// unescapeJSON appends the unescaped form of a JSON string body +// (the bytes between the quotes, not including the quotes themselves) to +// dst and returns the result. It handles all JSON escape sequences +// including \uXXXX and UTF-16 surrogate pairs. +func unescapeJSON(dst, src []byte) ([]byte, error) { + i := 0 + for i < len(src) { + if src[i] != '\\' { + dst = append(dst, src[i]) + i++ + continue + } + if i+1 >= len(src) { + return dst, errors.New("truncated escape sequence in JSON string") + } + i++ // skip '\' + switch src[i] { + case '"', '\\', '/': + dst = append(dst, src[i]) + i++ + case 'b': + dst = append(dst, '\b') + i++ + case 'f': + dst = append(dst, '\f') + i++ + case 'n': + dst = append(dst, '\n') + i++ + case 'r': + dst = append(dst, '\r') + i++ + case 't': + dst = append(dst, '\t') + i++ + case 'u': + i++ // skip 'u' + if i+4 > len(src) { + return dst, errors.New("truncated \\u escape in JSON string") + } + r := parseHex4(src[i : i+4]) + if r < 0 { + return dst, fmt.Errorf("invalid hex digit in \\u escape: %q", src[i:i+4]) + } + i += 4 + + // Handle UTF-16 surrogate pairs. + if utf16.IsSurrogate(r) { + if i+6 <= len(src) && src[i] == '\\' && src[i+1] == 'u' { + r2 := parseHex4(src[i+2 : i+6]) + if r2 >= 0 { + combined := utf16.DecodeRune(r, r2) + if combined != utf8.RuneError { + dst = utf8.AppendRune(dst, combined) + i += 6 + continue + } + } + } + // Lone surrogate: encode as replacement character. + dst = utf8.AppendRune(dst, utf8.RuneError) + continue + } + + dst = utf8.AppendRune(dst, r) + default: + return dst, fmt.Errorf("invalid escape character %q in JSON string", src[i]) + } + } + return dst, nil +} + +// parseHex4 parses exactly 4 hex digits into a rune. Returns -1 on error. +func parseHex4(s []byte) rune { + var r rune + for _, ch := range s { + r <<= 4 + switch { + case ch >= '0' && ch <= '9': + r |= rune(ch - '0') + case ch >= 'a' && ch <= 'f': + r |= rune(ch - 'a' + 10) + case ch >= 'A' && ch <= 'F': + r |= rune(ch - 'A' + 10) + default: + return -1 + } + } + return r +} + +func (w *sqlWriter) writeNumber(top bool) error { + // Use the parser's readFloat to validate number grammar, rejecting + // malformed inputs like "1+2", "1..2", or "1e+" that a simple + // character-class loop would accept. + n, ok := readFloat(w.data[w.pos:]) + if !ok || n == 0 { + return fmt.Errorf("invalid number at position %d in JSON", w.pos) + } + if top { + w.buf.WriteString("CAST(") + } + w.buf.Write(w.data[w.pos : w.pos+n]) + w.pos += n + if top { + w.buf.WriteString(" as JSON)") + } + return nil +} + +func (w *sqlWriter) writeBool(top bool) error { + if top { + w.buf.WriteString("CAST(_utf8mb4'") + } + if w.pos+4 <= len(w.data) && string(w.data[w.pos:w.pos+4]) == "true" { + w.buf.WriteString("true") + w.pos += 4 + } else if w.pos+5 <= len(w.data) && string(w.data[w.pos:w.pos+5]) == "false" { + w.buf.WriteString("false") + w.pos += 5 + } else { + return fmt.Errorf("unexpected token at position %d in JSON", w.pos) + } + if top { + w.buf.WriteString("' as JSON)") + } + return nil +} + +func (w *sqlWriter) writeNull(top bool) error { + if w.pos+4 > len(w.data) || string(w.data[w.pos:w.pos+4]) != "null" { + return fmt.Errorf("unexpected token at position %d in JSON", w.pos) + } + if top { + w.buf.WriteString("CAST(_utf8mb4'") + } + w.buf.WriteString("null") + w.pos += 4 + if top { + w.buf.WriteString("' as JSON)") + } + return nil +======= +// AppendMarshalSQL converts text JSON into a SQL expression using +// JSON_OBJECT/JSON_ARRAY syntax, writing directly to buf. It scans +// the raw bytes directly without building an intermediate tree. +// +// This has O(recursion depth) memory overhead versus O(total nodes * 72 +// bytes) for the tree-based Value.MarshalSQLTo, and avoids the per-token +// heap allocations of encoding/json.Decoder.Token. +// +// The output format matches the tree-based encoder so that MySQL stores +// identical binary JSON, including preservation of large integer precision +// via bare numeric literals. +func AppendMarshalSQL(buf *bytes2.Buffer, raw []byte) error { + w := sqlWriter{ + data: raw, + buf: buf, + } + if err := w.writeValue(true, 0); err != nil { + return err + } + w.skipWhitespace() + if w.pos != len(w.data) { + return errors.New("unexpected trailing data after JSON value") + } + return nil +} + +// sqlWriter converts text JSON into SQL expressions by scanning +// the raw bytes directly. It uses O(recursion depth) memory overhead +// plus a reusable scratch buffer for string unescaping. +type sqlWriter struct { + data []byte + pos int + buf *bytes2.Buffer + scratch []byte +} + +func (w *sqlWriter) skipWhitespace() { + for w.pos < len(w.data) { + switch w.data[w.pos] { + case ' ', '\t', '\n', '\r': + w.pos++ + default: + return + } + } +} + +func (w *sqlWriter) writeValue(top bool, depth int) error { + if depth >= MaxDepth { + return fmt.Errorf("too big depth for the nested JSON; it exceeds %d", MaxDepth) + } + w.skipWhitespace() + if w.pos >= len(w.data) { + return errors.New("unexpected end of JSON input") + } + switch w.data[w.pos] { + case '{': + w.pos++ + return w.writeObject(depth) + case '[': + w.pos++ + return w.writeArray(depth) + case '"': + return w.writeString(top) + case 't', 'f': + return w.writeBool(top) + case 'n': + return w.writeNull(top) + default: + if w.data[w.pos] >= '0' && w.data[w.pos] <= '9' || w.data[w.pos] == '-' { + return w.writeNumber(top) + } + return fmt.Errorf("unexpected character %q in JSON", w.data[w.pos]) + } +} + +func (w *sqlWriter) writeObject(depth int) error { + w.buf.WriteString("JSON_OBJECT(") + first := true + for { + w.skipWhitespace() + if w.pos >= len(w.data) { + return errors.New("unexpected end of JSON input in object") + } + if w.data[w.pos] == '}' { + w.pos++ + w.buf.WriteByte(')') + return nil + } + if !first { + if w.data[w.pos] != ',' { + return fmt.Errorf("expected ',' or '}' in object, got %q", w.data[w.pos]) + } + w.pos++ + w.buf.WriteString(", ") + w.skipWhitespace() + } + first = false + + // Key (always a string). + if w.pos >= len(w.data) || w.data[w.pos] != '"' { + return errors.New("expected string key in JSON object") + } + w.buf.WriteString("_utf8mb4") + if err := w.writeStringContent(); err != nil { + return fmt.Errorf("reading JSON object key: %w", err) + } + w.buf.WriteString(", ") + + // Colon separator. + w.skipWhitespace() + if w.pos >= len(w.data) || w.data[w.pos] != ':' { + return errors.New("expected ':' after object key") + } + w.pos++ + + // Value. + if err := w.writeValue(false, depth+1); err != nil { + return err + } + } +} + +func (w *sqlWriter) writeArray(depth int) error { + w.buf.WriteString("JSON_ARRAY(") + first := true + for { + w.skipWhitespace() + if w.pos >= len(w.data) { + return errors.New("unexpected end of JSON input in array") + } + if w.data[w.pos] == ']' { + w.pos++ + w.buf.WriteByte(')') + return nil + } + if !first { + if w.data[w.pos] != ',' { + return fmt.Errorf("expected ',' or ']' in array, got %q", w.data[w.pos]) + } + w.pos++ + w.buf.WriteString(", ") + } + first = false + + if err := w.writeValue(false, depth+1); err != nil { + return err + } + } +} + +func (w *sqlWriter) writeString(top bool) error { + if top { + w.buf.WriteString("CAST(JSON_QUOTE(") + } + w.buf.WriteString("_utf8mb4") + if err := w.writeStringContent(); err != nil { + return err + } + if top { + w.buf.WriteString(") as JSON)") + } + return nil +} + +// writeStringContent reads a JSON string starting at w.pos (which must point +// at the opening '"'), JSON-unescapes it, SQL-encodes it into w.buf, and +// advances w.pos past the closing '"'. +func (w *sqlWriter) writeStringContent() error { + if w.pos >= len(w.data) || w.data[w.pos] != '"' { + return errors.New("expected '\"' at start of string") + } + w.pos++ // skip opening '"' + + // Scan to find the closing quote, tracking whether escape sequences exist. + start := w.pos + hasEscape := false + for w.pos < len(w.data) { + ch := w.data[w.pos] + if ch == '\\' { + if w.pos+1 >= len(w.data) { + return errors.New("unterminated string in JSON") + } + hasEscape = true + w.pos += 2 // skip '\' and the escaped character + continue + } + if ch == '"' { + break + } + w.pos++ + } + if w.pos >= len(w.data) { + return errors.New("unterminated string in JSON") + } + + content := w.data[start:w.pos] + w.pos++ // skip closing '"' + + if !hasEscape { + // Fast path: no escape sequences, raw bytes are the decoded string. + sqltypes.MakeTrusted(querypb.Type_VARCHAR, content).EncodeSQLBytes2(w.buf) + } else { + // Slow path: unescape JSON into scratch buffer, then SQL-encode. + var err error + w.scratch, err = unescapeJSON(w.scratch[:0], content) + if err != nil { + return err + } + sqltypes.MakeTrusted(querypb.Type_VARCHAR, w.scratch).EncodeSQLBytes2(w.buf) + } + return nil +} + +// unescapeJSON appends the unescaped form of a JSON string body +// (the bytes between the quotes, not including the quotes themselves) to +// dst and returns the result. It handles all JSON escape sequences +// including \uXXXX and UTF-16 surrogate pairs. +func unescapeJSON(dst, src []byte) ([]byte, error) { + i := 0 + for i < len(src) { + if src[i] != '\\' { + dst = append(dst, src[i]) + i++ + continue + } + if i+1 >= len(src) { + return dst, errors.New("truncated escape sequence in JSON string") + } + i++ // skip '\' + switch src[i] { + case '"', '\\', '/': + dst = append(dst, src[i]) + i++ + case 'b': + dst = append(dst, '\b') + i++ + case 'f': + dst = append(dst, '\f') + i++ + case 'n': + dst = append(dst, '\n') + i++ + case 'r': + dst = append(dst, '\r') + i++ + case 't': + dst = append(dst, '\t') + i++ + case 'u': + i++ // skip 'u' + if i+4 > len(src) { + return dst, errors.New("truncated \\u escape in JSON string") + } + r := parseHex4(src[i : i+4]) + if r < 0 { + return dst, fmt.Errorf("invalid hex digit in \\u escape: %q", src[i:i+4]) + } + i += 4 + + // Handle UTF-16 surrogate pairs. + if utf16.IsSurrogate(r) { + if i+6 <= len(src) && src[i] == '\\' && src[i+1] == 'u' { + r2 := parseHex4(src[i+2 : i+6]) + if r2 >= 0 { + combined := utf16.DecodeRune(r, r2) + if combined != utf8.RuneError { + dst = utf8.AppendRune(dst, combined) + i += 6 + continue + } + } + } + // Lone surrogate: encode as replacement character. + dst = utf8.AppendRune(dst, utf8.RuneError) + continue + } + + dst = utf8.AppendRune(dst, r) + default: + return dst, fmt.Errorf("invalid escape character %q in JSON string", src[i]) + } + } + return dst, nil +} + +// parseHex4 parses exactly 4 hex digits into a rune. Returns -1 on error. +func parseHex4(s []byte) rune { + var r rune + for _, ch := range s { + r <<= 4 + switch { + case ch >= '0' && ch <= '9': + r |= rune(ch - '0') + case ch >= 'a' && ch <= 'f': + r |= rune(ch - 'a' + 10) + case ch >= 'A' && ch <= 'F': + r |= rune(ch - 'A' + 10) + default: + return -1 + } + } + return r +} + +func (w *sqlWriter) writeNumber(top bool) error { + // Use the parser's readFloat to validate number grammar, rejecting + // malformed inputs like "1+2", "1..2", or "1e+" that a simple + // character-class loop would accept. + n, _, ok := readFloat(w.data[w.pos:]) + if !ok || n == 0 { + return fmt.Errorf("invalid number at position %d in JSON", w.pos) + } + if top { + w.buf.WriteString("CAST(") + } + w.buf.Write(w.data[w.pos : w.pos+n]) + w.pos += n + if top { + w.buf.WriteString(" as JSON)") + } + return nil +} + +func (w *sqlWriter) writeBool(top bool) error { + if top { + w.buf.WriteString("CAST(_utf8mb4'") + } + if w.pos+4 <= len(w.data) && string(w.data[w.pos:w.pos+4]) == "true" { + w.buf.WriteString("true") + w.pos += 4 + } else if w.pos+5 <= len(w.data) && string(w.data[w.pos:w.pos+5]) == "false" { + w.buf.WriteString("false") + w.pos += 5 + } else { + return fmt.Errorf("unexpected token at position %d in JSON", w.pos) + } + if top { + w.buf.WriteString("' as JSON)") + } + return nil +} + +func (w *sqlWriter) writeNull(top bool) error { + if w.pos+4 > len(w.data) || string(w.data[w.pos:w.pos+4]) != "null" { + return fmt.Errorf("unexpected token at position %d in JSON", w.pos) + } + if top { + w.buf.WriteString("CAST(_utf8mb4'") + } + w.buf.WriteString("null") + w.pos += 4 + if top { + w.buf.WriteString("' as JSON)") + } + return nil +>>>>>>> ec6fdee983 (mysql/json: read numbers the way MySQL does (#20722)) } diff --git a/go/mysql/json/marshal_test.go b/go/mysql/json/marshal_test.go index d59f15a4892..78dd09b4de3 100644 --- a/go/mysql/json/marshal_test.go +++ b/go/mysql/json/marshal_test.go @@ -85,3 +85,158 @@ func TestMarshalSQLValueNormalizesInvalidUTF8(t *testing.T) { expected := "CAST(JSON_QUOTE(_utf8mb4" + sqltypes.EncodeStringSQL(normalized) + ") as JSON)" require.Equal(t, expected, string(got.Raw())) } +<<<<<<< HEAD +||||||| parent of ec6fdee983 (mysql/json: read numbers the way MySQL does (#20722)) + +// TestAppendMarshalSQLDepthLimit verifies that AppendMarshalSQL enforces +// the same nesting depth limit as Parser.Parse. +func TestAppendMarshalSQLDepthLimit(t *testing.T) { + for _, tc := range []struct { + depth int + wantErr bool + }{ + {depth: 299, wantErr: false}, + {depth: 300, wantErr: false}, + {depth: 301, wantErr: true}, + } { + input := strings.Repeat("[", tc.depth) + strings.Repeat("]", tc.depth) + + var p Parser + _, parseErr := p.Parse(input) + + buf := &bytes2.Buffer{} + appendErr := AppendMarshalSQL(buf, []byte(input)) + + if tc.wantErr { + require.Error(t, parseErr, "depth %d: parser should reject", tc.depth) + assert.Error(t, appendErr, "depth %d: AppendMarshalSQL should reject", tc.depth) + } else { + assert.NoError(t, parseErr, "depth %d: parser should accept", tc.depth) + assert.NoError(t, appendErr, "depth %d: AppendMarshalSQL should accept", tc.depth) + } + } +} + +// TestAppendMarshalSQLNumberGrammar verifies that AppendMarshalSQL rejects +// malformed JSON numbers that a naive character-class scanner would accept. +func TestAppendMarshalSQLNumberGrammar(t *testing.T) { + malformed := []string{ + `1+2`, + `1-2`, + `1..2`, + `1e+`, + `1e`, + `--1`, + } + for _, input := range malformed { + buf := &bytes2.Buffer{} + err := AppendMarshalSQL(buf, []byte(input)) + require.Error(t, err, "malformed number %q should be rejected", input) + } + + valid := []string{ + `0`, `42`, `-1`, `3.14`, `-0.5`, + `1e10`, `1E10`, `1e+10`, `1e-10`, `1.5e2`, + } + for _, input := range valid { + buf := &bytes2.Buffer{} + err := AppendMarshalSQL(buf, []byte(input)) + assert.NoError(t, err, "valid number %q should be accepted", input) + } +} + +// TestAppendMarshalSQLTrailingBackslash verifies that a backslash as the +// last byte of a string is rejected rather than causing an out-of-bounds read. +func TestAppendMarshalSQLTrailingBackslash(t *testing.T) { + inputs := []string{ + `"trailing\`, // backslash is last byte, no closing quote + `{"key": "val\"}`, // backslash before quote looks like escaped quote, string never closes + } + for _, input := range inputs { + buf := &bytes2.Buffer{} + err := AppendMarshalSQL(buf, []byte(input)) + require.Error(t, err, "input %q should be rejected", input) + assert.ErrorContains(t, err, "unterminated string", "input %q", input) + } +} +======= + +// TestAppendMarshalSQLDepthLimit verifies that AppendMarshalSQL enforces +// the same nesting depth limit as Parser.Parse. +func TestAppendMarshalSQLDepthLimit(t *testing.T) { + for _, tc := range []struct { + depth int + wantErr bool + }{ + {depth: 299, wantErr: false}, + {depth: 300, wantErr: false}, + {depth: 301, wantErr: true}, + } { + input := strings.Repeat("[", tc.depth) + strings.Repeat("]", tc.depth) + + var p Parser + _, parseErr := p.Parse(input) + + buf := &bytes2.Buffer{} + appendErr := AppendMarshalSQL(buf, []byte(input)) + + if tc.wantErr { + require.Error(t, parseErr, "depth %d: parser should reject", tc.depth) + assert.Error(t, appendErr, "depth %d: AppendMarshalSQL should reject", tc.depth) + } else { + assert.NoError(t, parseErr, "depth %d: parser should accept", tc.depth) + assert.NoError(t, appendErr, "depth %d: AppendMarshalSQL should accept", tc.depth) + } + } +} + +// TestAppendMarshalSQLNumberGrammar verifies that AppendMarshalSQL rejects +// malformed JSON numbers that a naive character-class scanner would accept. +func TestAppendMarshalSQLNumberGrammar(t *testing.T) { + malformed := []string{ + `1+2`, + `1-2`, + `1..2`, + `1e+`, + `1e`, + `--1`, + // Shapes JSON's grammar does not have: a written plus, an integer + // part opening with a zero, and a decimal point with nothing on + // one side of it. + `+1`, + `007`, + `.2`, + `12.`, + } + for _, input := range malformed { + buf := &bytes2.Buffer{} + err := AppendMarshalSQL(buf, []byte(input)) + require.Error(t, err, "malformed number %q should be rejected", input) + } + + valid := []string{ + `0`, `42`, `-1`, `3.14`, `-0.5`, + `1e10`, `1E10`, `1e+10`, `1e-10`, `1.5e2`, + } + for _, input := range valid { + buf := &bytes2.Buffer{} + err := AppendMarshalSQL(buf, []byte(input)) + assert.NoError(t, err, "valid number %q should be accepted", input) + } +} + +// TestAppendMarshalSQLTrailingBackslash verifies that a backslash as the +// last byte of a string is rejected rather than causing an out-of-bounds read. +func TestAppendMarshalSQLTrailingBackslash(t *testing.T) { + inputs := []string{ + `"trailing\`, // backslash is last byte, no closing quote + `{"key": "val\"}`, // backslash before quote looks like escaped quote, string never closes + } + for _, input := range inputs { + buf := &bytes2.Buffer{} + err := AppendMarshalSQL(buf, []byte(input)) + require.Error(t, err, "input %q should be rejected", input) + assert.ErrorContains(t, err, "unterminated string", "input %q", input) + } +} +>>>>>>> ec6fdee983 (mysql/json: read numbers the way MySQL does (#20722)) diff --git a/go/mysql/json/parser.go b/go/mysql/json/parser.go index 4f25e6be16e..82b3a20e08f 100644 --- a/go/mysql/json/parser.go +++ b/go/mysql/json/parser.go @@ -1,6 +1,8 @@ /* Copyright 2018 Aliaksandr Valialkin Copyright 2023 The Vitess Authors. +Portions Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. +See LICENSE.rapidjson in this directory. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -21,6 +23,7 @@ import ( "bytes" "encoding/base64" "fmt" + "math" "slices" "strconv" "strings" @@ -172,42 +175,296 @@ func parseValue(s string, c *cache, depth int) (*Value, string, error) { } if s[0] == 't' { if len(s) < len("true") || s[:len("true")] != "true" { - return nil, s, fmt.Errorf("unexpected value found: %q", s) + return nil, s, fmt.Errorf("unexpected value found: %q", startEndString(s)) } return ValueTrue, s[len("true"):], nil } if s[0] == 'f' { if len(s) < len("false") || s[:len("false")] != "false" { - return nil, s, fmt.Errorf("unexpected value found: %q", s) + return nil, s, fmt.Errorf("unexpected value found: %q", startEndString(s)) } return ValueFalse, s[len("false"):], nil } if s[0] == 'n' { if len(s) < len("null") || s[:len("null")] != "null" { - // Try parsing NaN - if len(s) >= 3 && strings.EqualFold(s[:3], "nan") { - v := c.getValue() - v.t = TypeNumber - v.s = s[:3] - return v, s[3:], nil - } - return nil, s, fmt.Errorf("unexpected value found: %q", s) + return nil, s, fmt.Errorf("unexpected value found: %q", startEndString(s)) } return ValueNull, s[len("null"):], nil } - flen, ok := readFloat(s) + flen, exponent, ok := readFloat(s) if !ok { - return nil, s[flen:], fmt.Errorf("invalid number in JSON string: %q", s) + return nil, s[flen:], fmt.Errorf("invalid number in JSON string: %q", startEndString(s)) } v := c.getValue() v.t = TypeNumber v.s = s[:flen] v.n = numberTypeRaw + if mayExceedFloat64(v.s, exponent) && !mysqlNumberFits(v.s) { + return nil, s, fmt.Errorf("number too big to be stored in double: %q", startEndString(v.s)) + } return v, s[flen:], nil } +// maxFloat64Digits is how far a decimal point can travel before a double runs +// out of room. The largest double is under 1.8e308, so a written exponent of +// 308 always leaves somewhere for the number to land and 309 need not. +const maxFloat64Digits = 308 + +// mayExceedFloat64 reports whether num is worth converting to find out whether a +// double can hold it. It errs towards yes: the job is to keep the conversion off +// the common path, not to answer the question. +// +// What can carry a number that far is the digits in front of its decimal point, +// moved by its exponent — it stays below 10^309 whenever those two together stay +// inside the places a double has, whatever it goes on to say after the point. +// Digits behind the point only ever move it the other way, and they are what buys +// the written exponent its extra room; nor can they overflow the significand on +// the way in, since it stops taking them at seventeen. +// +// Only an exponent that moves the point to the right counts against those digits. +// A negative one buys nothing back: the digits in front are read into the +// significand before the exponent is applied, so a significand that ran out of +// room on the way in has already gone infinite, and moving the point afterwards +// leaves it there. +// +// Counting those digits means walking them, so len(num) answers first. It counts +// the fraction and the exponent's own characters too, and so can only say yes too +// often — which leaves the walk where the conversion it saves is: off the path +// every ordinary number takes. +func mayExceedFloat64(num string, exponent int) bool { + room := maxFloat64Digits - max(exponent, 0) + if len(num) <= room { + return false + } + + i := 0 + if num[0] == '-' { + i++ + } + digits := 0 + for i+digits < len(num) && '0' <= num[i+digits] && num[i+digits] <= '9' { + digits++ + } + return digits > room +} + +// pow10 holds the powers of ten that a number's digits are scaled by, which is +// the table RapidJSON scales its significand with. RapidJSON writes its table +// out as decimal literals, so each entry here is read from the same spelling and +// lands on the same double. math.Pow10 would not: it multiplies two table entries +// together and comes out a bit away from the spelling for 78 of these exponents. +var pow10 [maxFloat64Digits + 1]float64 + +// The rest of the package reads numbers with fastparse, because that is a hot +// path. This table is built once at startup, so it uses strconv instead. +func init() { + for i := range pow10 { + pow10[i], _ = strconv.ParseFloat("1e"+strconv.Itoa(i), 64) + } +} + +// mysqlNumberFits reports whether MySQL can store num, which has to already be +// a grammatically valid JSON number. +// +// MySQL parses JSON with RapidJSON, and runs its number reader without +// kParseFullPrecisionFlag — which is what leaves MySQL on the approximate +// conversion below rather than a correct one, and pins this boundary to that +// reader. It decides this in two places. While reading the exponent, a written +// exponent may not move the decimal point more +// than maxFloat64Digits places past the digits already behind it. After +// converting, the result may not be larger than the largest double. Neither +// place catches what the other does: an exponent that lands on zero is refused +// only as written, and a number written within the bound can still overflow +// once converted. +// +// The conversion below is the one RapidJSON does rather than a correct one. It +// accumulates the significand into a double and scales that by a power of ten, +// which lands an ULP or two from the true value, and the boundary sits wherever +// that lands. A number a hair under the largest double therefore comes down to +// how it was spelled: 1.7976931348623158e308 does not fit, while writing the +// same value as 1.79769313486231580e308 does, because the extra digit moves +// where the significand ends and the scaling begins. Converting the same way is +// what makes a document valid here exactly when it is valid in MySQL. +// +// The float64 conversions in the digit loops keep the multiply and the add as +// two roundings. Without them the compiler is free to fuse both into one FMA +// on arm64, which rounds once and can land the accumulation one ULP away from +// where MySQL's builds put it — enough to flip which side of the largest +// double a number falls on. +func mysqlNumberFits(num string) bool { + i := 0 + minus := num[i] == '-' + if minus { + i++ + } + + var ( + u32 uint32 + u64 uint64 + use64 bool + sigDigits int + d float64 + useDouble bool + ) + digit := func() bool { return i < len(num) && num[i] >= '0' && num[i] <= '9' } + + // The integer part accumulates as an integer for as long as one can hold + // it, stepping up in width as it fills. RapidJSON leaves the leading digit + // out of sigDigits, and the seventeen-digit cut-off in the fraction below is + // measured against that count. + if num[i] == '0' { + i++ + } else { + u32 = uint32(num[i] - '0') + i++ + for digit() { + if minus { + if u32 >= 214748364 && (u32 != 214748364 || num[i] > '8') { + u64, use64 = uint64(u32), true + break + } + } else if u32 >= 429496729 && (u32 != 429496729 || num[i] > '5') { + u64, use64 = uint64(u32), true + break + } + u32 = u32*10 + uint32(num[i]-'0') + i++ + sigDigits++ + } + } + if use64 { + for digit() { + if minus { + if u64 >= 0x0CCCCCCCCCCCCCCC && (u64 != 0x0CCCCCCCCCCCCCCC || num[i] > '8') { + d, useDouble = float64(u64), true + break + } + } else if u64 >= 0x1999999999999999 && (u64 != 0x1999999999999999 || num[i] > '5') { + d, useDouble = float64(u64), true + break + } + u64 = u64*10 + uint64(num[i]-'0') + i++ + sigDigits++ + } + } + for useDouble && digit() { + d = float64(d*10) + float64(num[i]-'0') + i++ + } + + // Digits behind the decimal point move it left, which is what expFrac + // counts. Past seventeen significant digits they stop counting and stop + // arriving, so they move it no further. + expFrac := 0 + if i < len(num) && num[i] == '.' { + i++ + if !useDouble { + if !use64 { + u64 = uint64(u32) + } + for digit() { + if u64 > 0x1FFFFFFFFFFFFF { + break + } + u64 = u64*10 + uint64(num[i]-'0') + i++ + expFrac-- + if u64 != 0 { + sigDigits++ + } + } + d = float64(u64) + useDouble = true + } + for digit() { + if sigDigits < 17 { + d = float64(d*10) + float64(num[i]-'0') + expFrac-- + if d > 0 { + sigDigits++ + } + } + i++ + } + } + + exp := 0 + if i < len(num) && (num[i] == 'e' || num[i] == 'E') { + i++ + if !useDouble { + if use64 { + d = float64(u64) + } else { + d = float64(u32) + } + useDouble = true + } + negative := num[i] == '-' + if negative || num[i] == '+' { + i++ + } + exp = int(num[i] - '0') + i++ + if negative { + // A negative exponent is not bounded, only stopped before it could + // overflow the int it accumulates into. Everything out that far has + // flushed to zero long before. + maxExp := (expFrac + 2147483639) / 10 + for digit() { + exp = exp*10 + int(num[i]-'0') + i++ + if exp > maxExp { + for digit() { + i++ + } + } + } + exp = -exp + } else { + maxExp := maxFloat64Digits - expFrac + for digit() { + exp = exp*10 + int(num[i]-'0') + i++ + if exp > maxExp { + return false + } + } + } + } + + // A number that never needed a double is an integer MySQL keeps exact. + if !useDouble { + return true + } + return scaleByPow10(d, exp+expFrac) <= math.MaxFloat64 +} + +// scaleByPow10 moves d by p decimal places the way RapidJSON does, in one +// multiplication or division by a power of ten. Anything below the smallest +// power in the table is moved in two steps so that the table is never indexed +// past its end. +func scaleByPow10(d float64, p int) float64 { + if p < -maxFloat64Digits { + d = movePoint(d, -maxFloat64Digits) + return movePoint(d, p+maxFloat64Digits) + } + return movePoint(d, p) +} + +func movePoint(significand float64, exp int) float64 { + switch { + case exp < -maxFloat64Digits: + return 0 + case exp >= 0: + return significand * pow10[exp] + default: + return significand / pow10[-exp] + } +} + func parseArray(s string, c *cache, depth int) (*Value, string, error) { s = skipWS(s) if len(s) == 0 { @@ -512,63 +769,90 @@ func parseRawString(s string) (string, string, error) { } } +<<<<<<< HEAD func readFloat(s string) (i int, ok bool) { // optional sign +||||||| parent of ec6fdee983 (mysql/json: read numbers the way MySQL does (#20722)) +func readFloat[S string | []byte](s S) (i int, ok bool) { + // optional sign +======= +// readFloat reads a JSON number off the front of s, returning how much of s it +// covers and how far its exponent moves the decimal point. Whether the number +// is one a double can hold is mysqlNumberFits's job; the exponent is reported so +// that question only has to be asked of numbers whose digits could reach that +// far. +// +// That distance is bounded rather than exact. An exponent can be written to more +// digits than it takes to leave every double behind, and one that reaches +// exponentCeiling below is left there instead of read out to the end. So it +// answers how far is far enough to matter and nothing finer; mysqlNumberFits +// reads the exponent itself off the number again. +// +// What counts as a number is JSON's grammar rather than Go's: a written plus, +// a missing digit on either side of the decimal point, and an integer part +// that opens with a zero are all rejected. +func readFloat[S string | []byte](s S) (i, exponent int, ok bool) { + // optional minus. JSON numbers carry no written plus. +>>>>>>> ec6fdee983 (mysql/json: read numbers the way MySQL does (#20722)) if i >= len(s) { return } - if s[i] == '+' || s[i] == '-' { + if s[i] == '-' { i++ } - // digits - sawdot := false - sawdigits := false - nd := 0 -loop: - for ; i < len(s); i++ { - switch c := s[i]; true { - case c == '.': - if sawdot { - break loop - } - sawdot = true - continue - - case '0' <= c && c <= '9': - sawdigits = true - if c == '0' && nd == 0 { // ignore leading zeros - continue - } - nd++ - continue + // integer part: either a lone zero, or digits that do not open with one + if i >= len(s) || s[i] < '0' || s[i] > '9' { + return + } + if s[i] == '0' { + i++ + } else { + for ; i < len(s) && '0' <= s[i] && s[i] <= '9'; i++ { } - break } - if !sawdigits { - return + + // optional fraction, which has to carry a digit of its own + if i < len(s) && s[i] == '.' { + i++ + if i >= len(s) || s[i] < '0' || s[i] > '9' { + return + } + for ; i < len(s) && '0' <= s[i] && s[i] <= '9'; i++ { + } } - // optional exponent moves decimal point. - // if we read a very large, very long number, - // just be sure to move the decimal point by - // a lot (say, 100000). it doesn't matter if it's - // not the exact number. + // optional exponent moves the decimal point. An exponent larger than + // exponentCeiling stops there rather than running on: the ceiling clears + // both the largest double and every digit this number could hold after its + // decimal point, so a number stopped at it is out of reach either way. if i < len(s) && (s[i] == 'e' || s[i] == 'E') { i++ if i >= len(s) { return } + negative := false if s[i] == '+' || s[i] == '-' { + negative = s[i] == '-' i++ } if i >= len(s) || s[i] < '0' || s[i] > '9' { return } + exponentCeiling := maxFloat64Digits + len(s) for ; i < len(s) && ('0' <= s[i] && s[i] <= '9'); i++ { + if exponent <= exponentCeiling { + exponent = exponent*10 + int(s[i]-'0') + } + } + if exponent > exponentCeiling { + exponent = exponentCeiling + } + if negative { + exponent = -exponent } } - return i, true + return i, exponent, true } // Object represents JSON object. diff --git a/go/mysql/json/parser_bench_test.go b/go/mysql/json/parser_bench_test.go new file mode 100644 index 00000000000..8139b2694a4 --- /dev/null +++ b/go/mysql/json/parser_bench_test.go @@ -0,0 +1,114 @@ +/* +Copyright 2026 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package json + +import ( + "strconv" + "strings" + "testing" +) + +// numberArray builds a JSON array of n numbers, each spelled by digits. +func numberArray(n int, digits func(i int) string) string { + var sb strings.Builder + sb.WriteByte('[') + for i := range n { + if i > 0 { + sb.WriteByte(',') + } + sb.WriteString(digits(i)) + } + sb.WriteByte(']') + return sb.String() +} + +// benchDocs covers the number shapes the parser takes different paths through. +// +// The first four are the everyday ones, where the cost is the scan itself. The +// last three reach the magnitude check, which a number only pays for once it is +// written to more digits than its exponent leaves a double room for: once with +// an exponent that shrinks the room to ask the question of every element, once +// where the digits alone are past what a double has, and once for a document the +// answer rejects. reachesMagnitudeCheck holds them to that, since a case that +// falls short of the check goes on measuring the scan and reads as though the +// check were free. +var benchDocs = []struct { + name string + doc string + checked bool + rejected bool +}{ + {name: "int/1024", doc: numberArray(1024, func(i int) string { return strconv.Itoa(i * 7919) })}, + {name: "frac/1024", doc: numberArray(1024, func(i int) string { return strconv.Itoa(i) + "." + strconv.Itoa(i*7919) })}, + {name: "exp/1024", doc: numberArray(1024, func(i int) string { return strconv.Itoa(i) + "." + strconv.Itoa(i*7919) + "e" + strconv.Itoa(i%300) })}, + {name: "mixed-object", doc: `{"id":38141,"name":"a name","ok":true,"score":-12.5e3,"tags":["x","y"],"meta":null}`}, + + {name: "checked/1024", doc: numberArray(1024, func(i int) string { return "1" + strconv.Itoa(1000000000000000000+i) + "e289" }), checked: true}, + {name: "checked-long-fraction", doc: "1" + strings.Repeat("0", 308) + "." + strings.Repeat("5", 400), checked: true}, + {name: "rejected", doc: strings.Repeat("9", 400) + "e-100", checked: true, rejected: true}, +} + +// reachesMagnitudeCheck reports whether every number in doc is converted to find +// out whether a double can hold it, which is what the checked cases are for. A +// number written to fewer digits than a double has room for answers that question +// from its digits alone, and a case built out of those measures the scan it shares +// with every other case instead. +func reachesMagnitudeCheck(doc string) bool { + numbers := 0 + for i := 0; i < len(doc); { + if c := doc[i]; c != '-' && (c < '0' || c > '9') { + i++ + continue + } + flen, exponent, ok := readFloat(doc[i:]) + if !ok { + return false + } + if !mayExceedFloat64(doc[i:i+flen], exponent) { + return false + } + numbers++ + i += flen + } + return numbers > 0 +} + +func BenchmarkParse(b *testing.B) { + for _, tc := range benchDocs { + b.Run(tc.name, func(b *testing.B) { + var p Parser + + // A case that stops early measures the error path instead of the one + // it was written for, and reads as a speed-up while doing it. + if _, err := p.Parse(tc.doc); (err != nil) != tc.rejected { + b.Fatalf("document does not take the path this case measures: err=%v", err) + } + if reachesMagnitudeCheck(tc.doc) != tc.checked { + b.Fatalf("document reaches the magnitude check: %v, want %v", !tc.checked, tc.checked) + } + + b.ReportAllocs() + b.SetBytes(int64(len(tc.doc))) + for b.Loop() { + v, err := p.Parse(tc.doc) + if err == nil && v == nil { + b.Fatal("parsed to no value") + } + } + }) + } +} diff --git a/go/mysql/json/parser_test.go b/go/mysql/json/parser_test.go index 1e54007659c..af4ccdbf421 100644 --- a/go/mysql/json/parser_test.go +++ b/go/mysql/json/parser_test.go @@ -18,6 +18,7 @@ limitations under the License. package json import ( + "strings" "testing" "github.com/stretchr/testify/require" @@ -30,10 +31,18 @@ func TestParseRawNumber(t *testing.T) { f := func(s, expectedRN, expectedTail string) { t.Helper() +<<<<<<< HEAD flen, ok := readFloat(s) if !ok { t.Fatalf("unexpected error when parsing '%s'", s) } +||||||| parent of ec6fdee983 (mysql/json: read numbers the way MySQL does (#20722)) + flen, ok := readFloat(s) + require.Truef(t, ok, "unexpected error when parsing '%s'", s) +======= + flen, _, ok := readFloat(s) + require.Truef(t, ok, "unexpected error when parsing '%s'", s) +>>>>>>> ec6fdee983 (mysql/json: read numbers the way MySQL does (#20722)) rn, tail := s[:flen], s[flen:] @@ -55,15 +64,15 @@ func TestParseRawNumber(t *testing.T) { f("-12.345E+67 tail", "-12.345E+67", " tail") f("-12.345E-67,tail", "-12.345E-67", ",tail") f("-1234567.8e+90tail", "-1234567.8e+90", "tail") - f("12.tail", "12.", "tail") - f(".2tail", ".2", "tail") - f("-.2tail", "-.2", "tail") + f("0.2tail", "0.2", "tail") + f("-0.2tail", "-0.2", "tail") }) t.Run("error", func(t *testing.T) { f := func(s, expectedTail string) { t.Helper() +<<<<<<< HEAD flen, ok := readFloat(s) if ok { t.Fatalf("expecting non-nil error") @@ -71,6 +80,15 @@ func TestParseRawNumber(t *testing.T) { if s[flen:] != expectedTail { t.Fatalf("unexpected tail; got %q; want %q", s[flen:], expectedTail) } +||||||| parent of ec6fdee983 (mysql/json: read numbers the way MySQL does (#20722)) + flen, ok := readFloat(s) + require.False(t, ok, "expecting non-nil error") + require.Equalf(t, expectedTail, s[flen:], "unexpected tail; got %q; want %q", s[flen:], expectedTail) +======= + flen, _, ok := readFloat(s) + require.False(t, ok, "expecting non-nil error") + require.Equalf(t, expectedTail, s[flen:], "unexpected tail; got %q; want %q", s[flen:], expectedTail) +>>>>>>> ec6fdee983 (mysql/json: read numbers the way MySQL does (#20722)) } f("xyz", "xyz") @@ -79,7 +97,267 @@ func TestParseRawNumber(t *testing.T) { f(",", ",") f("{", "{") f("\"", "\"") + + // A decimal point needs a digit on either side of it, and a number + // opens with a minus or a digit. + f("12.tail", "tail") + f(".2tail", ".2tail") + f("-.2tail", ".2tail") + f("+1tail", "+1tail") + }) +} + +// TestParseNumberTooBigForDouble covers the boundary MySQL puts on JSON +// numbers: a number it cannot store as a double makes the whole document +// invalid, rather than being kept at the precision it was written to. +// Underflow is not rejected — it flushes to zero. +func TestParseNumberTooBigForDouble(t *testing.T) { + tooManyDigits := "1" + strings.Repeat("0", 309) + + t.Run("accepted", func(t *testing.T) { + for _, doc := range []string{ + "1e308", + "-1e308", + "1.7976931348623157e308", + "-1.7976931348623157e308", + "99999999999999999999999999999999999999999", + "1" + strings.Repeat("0", 307), + // Underflow keeps the document valid and reads as zero. + "1e-400", + "1e-1000", + "1e-1024", + "0." + strings.Repeat("0", 400) + "1", + // Digits a double has room for, moved out of the way by a negative + // exponent. + "1" + strings.Repeat("0", 307) + "e-1", + "1" + strings.Repeat("0", 307) + "e-400", + "-1" + strings.Repeat("0", 307) + "e-400", + "0." + strings.Repeat("0", 400) + "1e-400", + // A written sign and a padded exponent are spellings, not + // magnitudes, and none of these is anywhere near the limit. + "1e+0", + "1e-0", + "1e0000000000", + "1e-0000000000", + "1e+308", + "1e00000000000000000308", + // A written exponent is bounded by where it puts the decimal + // point, so digits after the point buy the same number of places + // back. Zero is subject to the bound like anything else. + "0e308", + "-0e308", + "0.0e309", + "0.00e310", + "0.1e309", + "0.01e310", + "0." + strings.Repeat("0", 400) + "1e700", + } { + t.Run(startEndString(doc), func(t *testing.T) { + var p Parser + v, err := p.Parse(doc) + require.NoError(t, err) + require.Equal(t, TypeNumber, v.Type()) + }) + } }) + + t.Run("rejected", func(t *testing.T) { + for _, doc := range []string{ + "1e309", + "-1e309", + "1e1025", + "1.7976931348623159e308", + "1e+309", + tooManyDigits, + // One place past what the digits after the point buy back. These + // all convert to zero, so only the exponent as written rules them + // out. + "0e309", + "-0e309", + "0e+309", + "0e1000", + "0.0e310", + "0.00e311", + "0.1e310", + "0.01e311", + "0." + strings.Repeat("0", 400) + "1e710", + // Within the written bound, but too big once converted. + "10e308", + "1" + strings.Repeat("0", 30) + "e279", + // More digits than a double has room for. The digits are read before + // the exponent is applied, so a negative exponent does not buy the + // room back however far it moves the decimal point afterwards. + "1" + strings.Repeat("0", 320) + "e-20", + "1" + strings.Repeat("0", 350) + "e-50", + "1" + strings.Repeat("0", 400) + "e-400", + "-1" + strings.Repeat("0", 400) + "e-400", + "1" + strings.Repeat("0", 400) + ".5e-400", + strings.Repeat("9", 400) + "e-100", + // A number anywhere in the document invalidates all of it. + "[1, 1e309]", + `{"a": 1e309}`, + "[[1e309]]", + } { + t.Run(startEndString(doc), func(t *testing.T) { + var p Parser + _, err := p.Parse(doc) + require.ErrorContains(t, err, "number too big to be stored in double") + }) + } + }) + + // A negative exponent is not bounded, only stopped before it overflows the + // int it accumulates into, and what sends a number through the conversion at + // all is being written to more digits than a double holds. These cross the + // two, so the exponent is read into an int that cannot hold it and then + // scales a significand: written past that stop, it lands wherever the + // overflow leaves it, which can be a power of ten the table does not go up + // to. Each of these stays valid and reads as zero. MySQL 8.0.46 accepts all + // three and reads them as zero too. + t.Run("a negative exponent written past what an int holds", func(t *testing.T) { + for _, doc := range []string{ + strings.Repeat("9", 400) + "e-" + strings.Repeat("2", 306), + "-" + strings.Repeat("9", 400) + "e-" + strings.Repeat("2", 306), + strings.Repeat("1", 400) + "." + strings.Repeat("5", 20) + "e-" + strings.Repeat("2", 306), + } { + t.Run(startEndString(doc), func(t *testing.T) { + var p Parser + v, err := p.Parse(doc) + require.NoError(t, err) + + f, ok := v.Float64() + require.True(t, ok) + require.Zero(t, f) + }) + } + }) + + // The significand accumulates one digit at a time, and each step rounds + // the multiplication and the addition separately, the way MySQL's builds + // run the loop. Fusing the two into one rounding — which the Go compiler + // may do on arm64 unless the conversion in mysqlNumberFits stops it — + // moves the accumulation an ULP for these documents, and that is enough + // to push them over the largest double. MySQL 8.0.45, 8.4.11 and 9.4.0 + // accept all of them. + t.Run("each accumulation step rounds on its own", func(t *testing.T) { + for _, doc := range []string{ + "17976931348623154547712857878e280", + "179769313486231559524062337652e279", + "179769313486231577704643761e282", + "1797693134862315724800793889e281", + } { + t.Run(startEndString(doc), func(t *testing.T) { + var p Parser + _, err := p.Parse(doc) + require.NoError(t, err) + }) + } + }) + + // Right at the top of the range the answer turns on how a number was + // written rather than on what it is worth. The digits are split between a + // significand and a power of ten to scale it by, and where that split falls + // decides which way the last place rounds — so writing the same value to one + // more digit moves the split and can move the answer with it. + t.Run("spelling at the largest double", func(t *testing.T) { + for _, tc := range []struct { + doc string + fits bool + }{ + {"1.7976931348623157e308", true}, + {"1.7976931348623158e308", false}, + {"1.79769313486231580e308", true}, + {"1.797693134862315800e308", true}, + {"1.79769313486231581e308", true}, + {"1.79769313486231585e308", true}, + {"1.7976931348623159e308", false}, + {"1.7976931348623157081e308", true}, + {"17976931348623157e292", true}, + {"17976931348623158e292", false}, + {"179769313486231580e291", true}, + {"1797693134862315800e290", false}, + {"17976931348623158000000e286", false}, + } { + t.Run(tc.doc, func(t *testing.T) { + var p Parser + _, err := p.Parse(tc.doc) + if tc.fits { + require.NoError(t, err) + } else { + require.ErrorContains(t, err, "number too big to be stored in double") + } + }) + } + }) +} + +// TestParseNumberGrammar covers the shapes JSON's grammar allows a number to +// take. A number opens with a minus or a digit, an integer part of more than +// one digit does not open with a zero, and a decimal point has digits on both +// sides of it. MySQL holds documents to the same grammar, and nan is not a +// number to either of them. +func TestParseNumberGrammar(t *testing.T) { + t.Run("accepted", func(t *testing.T) { + for _, doc := range []string{ + "0", "-0", "0.5", "-0.5", "1", "-1", "1.2", "0e0", "-0e0", + "1e5", "1E5", "1e007", "1e+007", "1e-5", "0.0", + "[0,1,2]", `{"a":-0.5,"b":[1e5]}`, + } { + t.Run(doc, func(t *testing.T) { + var p Parser + _, err := p.Parse(doc) + require.NoError(t, err) + }) + } + }) + + t.Run("rejected", func(t *testing.T) { + for _, doc := range []string{ + // An integer part that opens with a zero. + "007", "-003", "01", "00", "00.5", "01.5", "[007]", + // A decimal point missing a digit on one side. + ".2", "-.2", "12.", "-12.", "1.e5", `{"a": .2}`, "[12.]", + // A written plus. + "+1", "+1.5", "+0", "[+1]", + // Not a number at all. + "nan", "NaN", "NAN", "[nan]", `{"a": nan}`, "-nan", ".", "-", + } { + t.Run(doc, func(t *testing.T) { + var p Parser + _, err := p.Parse(doc) + require.Error(t, err) + }) + } + }) +} + +// TestParseErrorAbbreviatesTheDocument covers how much of a rejected document +// its error names. Nothing bounds how long a document may be, and Parse copies +// the message it wraps, so naming the text in full hands a client its own +// document back twice over. Each rejection abbreviates what it names, as the +// unparsed tail alongside it already did. +func TestParseErrorAbbreviatesTheDocument(t *testing.T) { + long := strings.Repeat("9", 100000) + + for _, tc := range []struct { + name string + doc string + }{ + {name: "a number too big for a double", doc: "1" + long}, + {name: "a written plus", doc: "+" + long}, + {name: "a decimal point with nothing before it", doc: "." + long}, + {name: "a decimal point with nothing after it", doc: long + "."}, + {name: "nan", doc: "nan" + long}, + {name: "nothing the grammar has a shape for", doc: "q" + long}, + } { + t.Run(tc.name, func(t *testing.T) { + var p Parser + _, err := p.Parse(tc.doc) + require.Error(t, err) + require.NotContains(t, err.Error(), strings.Repeat("9", 200), + "the error carries the document it is reporting on") + }) + } } func TestUnescapeStringBestEffort(t *testing.T) { From 6046cba78c35d25d245fb4c968f89cf3d733769a Mon Sep 17 00:00:00 2001 From: Arthur Schreiber Date: Tue, 28 Jul 2026 22:51:08 +0000 Subject: [PATCH 2/2] resolve conflicts for backport of #20722 marshal.go and marshal_test.go stay as release-23.0 has them: the upstream change there adapted AppendMarshalSQL's readFloat call, and that writer does not exist on this branch. parser.go takes the upstream readFloat wholesale, and parser_test.go keeps this branch's t.Fatalf style around the widened readFloat return. Co-Authored-By: Claude Fable 5 Signed-off-by: Arthur Schreiber --- go/mysql/json/marshal.go | 716 ---------------------------------- go/mysql/json/marshal_test.go | 155 -------- go/mysql/json/parser.go | 8 - go/mysql/json/parser_test.go | 22 +- 4 files changed, 2 insertions(+), 899 deletions(-) diff --git a/go/mysql/json/marshal.go b/go/mysql/json/marshal.go index ab60d11fab8..97d14a336c8 100644 --- a/go/mysql/json/marshal.go +++ b/go/mysql/json/marshal.go @@ -175,722 +175,6 @@ func MarshalSQLValue(buf []byte) (*sqltypes.Value, error) { return nil, err } -<<<<<<< HEAD newVal := sqltypes.MakeTrusted(querypb.Type_RAW, jsonVal.MarshalSQLTo(nil)) return &newVal, nil -||||||| parent of ec6fdee983 (mysql/json: read numbers the way MySQL does (#20722)) -// AppendMarshalSQL converts text JSON into a SQL expression using -// JSON_OBJECT/JSON_ARRAY syntax, writing directly to buf. It scans -// the raw bytes directly without building an intermediate tree. -// -// This has O(recursion depth) memory overhead versus O(total nodes * 72 -// bytes) for the tree-based Value.MarshalSQLTo, and avoids the per-token -// heap allocations of encoding/json.Decoder.Token. -// -// The output format matches the tree-based encoder so that MySQL stores -// identical binary JSON, including preservation of large integer precision -// via bare numeric literals. -func AppendMarshalSQL(buf *bytes2.Buffer, raw []byte) error { - w := sqlWriter{ - data: raw, - buf: buf, - } - if err := w.writeValue(true, 0); err != nil { - return err - } - w.skipWhitespace() - if w.pos != len(w.data) { - return errors.New("unexpected trailing data after JSON value") - } - return nil -} - -// sqlWriter converts text JSON into SQL expressions by scanning -// the raw bytes directly. It uses O(recursion depth) memory overhead -// plus a reusable scratch buffer for string unescaping. -type sqlWriter struct { - data []byte - pos int - buf *bytes2.Buffer - scratch []byte -} - -func (w *sqlWriter) skipWhitespace() { - for w.pos < len(w.data) { - switch w.data[w.pos] { - case ' ', '\t', '\n', '\r': - w.pos++ - default: - return - } - } -} - -func (w *sqlWriter) writeValue(top bool, depth int) error { - if depth >= MaxDepth { - return fmt.Errorf("too big depth for the nested JSON; it exceeds %d", MaxDepth) - } - w.skipWhitespace() - if w.pos >= len(w.data) { - return errors.New("unexpected end of JSON input") - } - switch w.data[w.pos] { - case '{': - w.pos++ - return w.writeObject(depth) - case '[': - w.pos++ - return w.writeArray(depth) - case '"': - return w.writeString(top) - case 't', 'f': - return w.writeBool(top) - case 'n': - return w.writeNull(top) - default: - if w.data[w.pos] >= '0' && w.data[w.pos] <= '9' || w.data[w.pos] == '-' { - return w.writeNumber(top) - } - return fmt.Errorf("unexpected character %q in JSON", w.data[w.pos]) - } -} - -func (w *sqlWriter) writeObject(depth int) error { - w.buf.WriteString("JSON_OBJECT(") - first := true - for { - w.skipWhitespace() - if w.pos >= len(w.data) { - return errors.New("unexpected end of JSON input in object") - } - if w.data[w.pos] == '}' { - w.pos++ - w.buf.WriteByte(')') - return nil - } - if !first { - if w.data[w.pos] != ',' { - return fmt.Errorf("expected ',' or '}' in object, got %q", w.data[w.pos]) - } - w.pos++ - w.buf.WriteString(", ") - w.skipWhitespace() - } - first = false - - // Key (always a string). - if w.pos >= len(w.data) || w.data[w.pos] != '"' { - return errors.New("expected string key in JSON object") - } - w.buf.WriteString("_utf8mb4") - if err := w.writeStringContent(); err != nil { - return fmt.Errorf("reading JSON object key: %w", err) - } - w.buf.WriteString(", ") - - // Colon separator. - w.skipWhitespace() - if w.pos >= len(w.data) || w.data[w.pos] != ':' { - return errors.New("expected ':' after object key") - } - w.pos++ - - // Value. - if err := w.writeValue(false, depth+1); err != nil { - return err - } - } -} - -func (w *sqlWriter) writeArray(depth int) error { - w.buf.WriteString("JSON_ARRAY(") - first := true - for { - w.skipWhitespace() - if w.pos >= len(w.data) { - return errors.New("unexpected end of JSON input in array") - } - if w.data[w.pos] == ']' { - w.pos++ - w.buf.WriteByte(')') - return nil - } - if !first { - if w.data[w.pos] != ',' { - return fmt.Errorf("expected ',' or ']' in array, got %q", w.data[w.pos]) - } - w.pos++ - w.buf.WriteString(", ") - } - first = false - - if err := w.writeValue(false, depth+1); err != nil { - return err - } - } -} - -func (w *sqlWriter) writeString(top bool) error { - if top { - w.buf.WriteString("CAST(JSON_QUOTE(") - } - w.buf.WriteString("_utf8mb4") - if err := w.writeStringContent(); err != nil { - return err - } - if top { - w.buf.WriteString(") as JSON)") - } - return nil -} - -// writeStringContent reads a JSON string starting at w.pos (which must point -// at the opening '"'), JSON-unescapes it, SQL-encodes it into w.buf, and -// advances w.pos past the closing '"'. -func (w *sqlWriter) writeStringContent() error { - if w.pos >= len(w.data) || w.data[w.pos] != '"' { - return errors.New("expected '\"' at start of string") - } - w.pos++ // skip opening '"' - - // Scan to find the closing quote, tracking whether escape sequences exist. - start := w.pos - hasEscape := false - for w.pos < len(w.data) { - ch := w.data[w.pos] - if ch == '\\' { - if w.pos+1 >= len(w.data) { - return errors.New("unterminated string in JSON") - } - hasEscape = true - w.pos += 2 // skip '\' and the escaped character - continue - } - if ch == '"' { - break - } - w.pos++ - } - if w.pos >= len(w.data) { - return errors.New("unterminated string in JSON") - } - - content := w.data[start:w.pos] - w.pos++ // skip closing '"' - - if !hasEscape { - // Fast path: no escape sequences, raw bytes are the decoded string. - sqltypes.MakeTrusted(querypb.Type_VARCHAR, content).EncodeSQLBytes2(w.buf) - } else { - // Slow path: unescape JSON into scratch buffer, then SQL-encode. - var err error - w.scratch, err = unescapeJSON(w.scratch[:0], content) - if err != nil { - return err - } - sqltypes.MakeTrusted(querypb.Type_VARCHAR, w.scratch).EncodeSQLBytes2(w.buf) - } - return nil -} - -// unescapeJSON appends the unescaped form of a JSON string body -// (the bytes between the quotes, not including the quotes themselves) to -// dst and returns the result. It handles all JSON escape sequences -// including \uXXXX and UTF-16 surrogate pairs. -func unescapeJSON(dst, src []byte) ([]byte, error) { - i := 0 - for i < len(src) { - if src[i] != '\\' { - dst = append(dst, src[i]) - i++ - continue - } - if i+1 >= len(src) { - return dst, errors.New("truncated escape sequence in JSON string") - } - i++ // skip '\' - switch src[i] { - case '"', '\\', '/': - dst = append(dst, src[i]) - i++ - case 'b': - dst = append(dst, '\b') - i++ - case 'f': - dst = append(dst, '\f') - i++ - case 'n': - dst = append(dst, '\n') - i++ - case 'r': - dst = append(dst, '\r') - i++ - case 't': - dst = append(dst, '\t') - i++ - case 'u': - i++ // skip 'u' - if i+4 > len(src) { - return dst, errors.New("truncated \\u escape in JSON string") - } - r := parseHex4(src[i : i+4]) - if r < 0 { - return dst, fmt.Errorf("invalid hex digit in \\u escape: %q", src[i:i+4]) - } - i += 4 - - // Handle UTF-16 surrogate pairs. - if utf16.IsSurrogate(r) { - if i+6 <= len(src) && src[i] == '\\' && src[i+1] == 'u' { - r2 := parseHex4(src[i+2 : i+6]) - if r2 >= 0 { - combined := utf16.DecodeRune(r, r2) - if combined != utf8.RuneError { - dst = utf8.AppendRune(dst, combined) - i += 6 - continue - } - } - } - // Lone surrogate: encode as replacement character. - dst = utf8.AppendRune(dst, utf8.RuneError) - continue - } - - dst = utf8.AppendRune(dst, r) - default: - return dst, fmt.Errorf("invalid escape character %q in JSON string", src[i]) - } - } - return dst, nil -} - -// parseHex4 parses exactly 4 hex digits into a rune. Returns -1 on error. -func parseHex4(s []byte) rune { - var r rune - for _, ch := range s { - r <<= 4 - switch { - case ch >= '0' && ch <= '9': - r |= rune(ch - '0') - case ch >= 'a' && ch <= 'f': - r |= rune(ch - 'a' + 10) - case ch >= 'A' && ch <= 'F': - r |= rune(ch - 'A' + 10) - default: - return -1 - } - } - return r -} - -func (w *sqlWriter) writeNumber(top bool) error { - // Use the parser's readFloat to validate number grammar, rejecting - // malformed inputs like "1+2", "1..2", or "1e+" that a simple - // character-class loop would accept. - n, ok := readFloat(w.data[w.pos:]) - if !ok || n == 0 { - return fmt.Errorf("invalid number at position %d in JSON", w.pos) - } - if top { - w.buf.WriteString("CAST(") - } - w.buf.Write(w.data[w.pos : w.pos+n]) - w.pos += n - if top { - w.buf.WriteString(" as JSON)") - } - return nil -} - -func (w *sqlWriter) writeBool(top bool) error { - if top { - w.buf.WriteString("CAST(_utf8mb4'") - } - if w.pos+4 <= len(w.data) && string(w.data[w.pos:w.pos+4]) == "true" { - w.buf.WriteString("true") - w.pos += 4 - } else if w.pos+5 <= len(w.data) && string(w.data[w.pos:w.pos+5]) == "false" { - w.buf.WriteString("false") - w.pos += 5 - } else { - return fmt.Errorf("unexpected token at position %d in JSON", w.pos) - } - if top { - w.buf.WriteString("' as JSON)") - } - return nil -} - -func (w *sqlWriter) writeNull(top bool) error { - if w.pos+4 > len(w.data) || string(w.data[w.pos:w.pos+4]) != "null" { - return fmt.Errorf("unexpected token at position %d in JSON", w.pos) - } - if top { - w.buf.WriteString("CAST(_utf8mb4'") - } - w.buf.WriteString("null") - w.pos += 4 - if top { - w.buf.WriteString("' as JSON)") - } - return nil -======= -// AppendMarshalSQL converts text JSON into a SQL expression using -// JSON_OBJECT/JSON_ARRAY syntax, writing directly to buf. It scans -// the raw bytes directly without building an intermediate tree. -// -// This has O(recursion depth) memory overhead versus O(total nodes * 72 -// bytes) for the tree-based Value.MarshalSQLTo, and avoids the per-token -// heap allocations of encoding/json.Decoder.Token. -// -// The output format matches the tree-based encoder so that MySQL stores -// identical binary JSON, including preservation of large integer precision -// via bare numeric literals. -func AppendMarshalSQL(buf *bytes2.Buffer, raw []byte) error { - w := sqlWriter{ - data: raw, - buf: buf, - } - if err := w.writeValue(true, 0); err != nil { - return err - } - w.skipWhitespace() - if w.pos != len(w.data) { - return errors.New("unexpected trailing data after JSON value") - } - return nil -} - -// sqlWriter converts text JSON into SQL expressions by scanning -// the raw bytes directly. It uses O(recursion depth) memory overhead -// plus a reusable scratch buffer for string unescaping. -type sqlWriter struct { - data []byte - pos int - buf *bytes2.Buffer - scratch []byte -} - -func (w *sqlWriter) skipWhitespace() { - for w.pos < len(w.data) { - switch w.data[w.pos] { - case ' ', '\t', '\n', '\r': - w.pos++ - default: - return - } - } -} - -func (w *sqlWriter) writeValue(top bool, depth int) error { - if depth >= MaxDepth { - return fmt.Errorf("too big depth for the nested JSON; it exceeds %d", MaxDepth) - } - w.skipWhitespace() - if w.pos >= len(w.data) { - return errors.New("unexpected end of JSON input") - } - switch w.data[w.pos] { - case '{': - w.pos++ - return w.writeObject(depth) - case '[': - w.pos++ - return w.writeArray(depth) - case '"': - return w.writeString(top) - case 't', 'f': - return w.writeBool(top) - case 'n': - return w.writeNull(top) - default: - if w.data[w.pos] >= '0' && w.data[w.pos] <= '9' || w.data[w.pos] == '-' { - return w.writeNumber(top) - } - return fmt.Errorf("unexpected character %q in JSON", w.data[w.pos]) - } -} - -func (w *sqlWriter) writeObject(depth int) error { - w.buf.WriteString("JSON_OBJECT(") - first := true - for { - w.skipWhitespace() - if w.pos >= len(w.data) { - return errors.New("unexpected end of JSON input in object") - } - if w.data[w.pos] == '}' { - w.pos++ - w.buf.WriteByte(')') - return nil - } - if !first { - if w.data[w.pos] != ',' { - return fmt.Errorf("expected ',' or '}' in object, got %q", w.data[w.pos]) - } - w.pos++ - w.buf.WriteString(", ") - w.skipWhitespace() - } - first = false - - // Key (always a string). - if w.pos >= len(w.data) || w.data[w.pos] != '"' { - return errors.New("expected string key in JSON object") - } - w.buf.WriteString("_utf8mb4") - if err := w.writeStringContent(); err != nil { - return fmt.Errorf("reading JSON object key: %w", err) - } - w.buf.WriteString(", ") - - // Colon separator. - w.skipWhitespace() - if w.pos >= len(w.data) || w.data[w.pos] != ':' { - return errors.New("expected ':' after object key") - } - w.pos++ - - // Value. - if err := w.writeValue(false, depth+1); err != nil { - return err - } - } -} - -func (w *sqlWriter) writeArray(depth int) error { - w.buf.WriteString("JSON_ARRAY(") - first := true - for { - w.skipWhitespace() - if w.pos >= len(w.data) { - return errors.New("unexpected end of JSON input in array") - } - if w.data[w.pos] == ']' { - w.pos++ - w.buf.WriteByte(')') - return nil - } - if !first { - if w.data[w.pos] != ',' { - return fmt.Errorf("expected ',' or ']' in array, got %q", w.data[w.pos]) - } - w.pos++ - w.buf.WriteString(", ") - } - first = false - - if err := w.writeValue(false, depth+1); err != nil { - return err - } - } -} - -func (w *sqlWriter) writeString(top bool) error { - if top { - w.buf.WriteString("CAST(JSON_QUOTE(") - } - w.buf.WriteString("_utf8mb4") - if err := w.writeStringContent(); err != nil { - return err - } - if top { - w.buf.WriteString(") as JSON)") - } - return nil -} - -// writeStringContent reads a JSON string starting at w.pos (which must point -// at the opening '"'), JSON-unescapes it, SQL-encodes it into w.buf, and -// advances w.pos past the closing '"'. -func (w *sqlWriter) writeStringContent() error { - if w.pos >= len(w.data) || w.data[w.pos] != '"' { - return errors.New("expected '\"' at start of string") - } - w.pos++ // skip opening '"' - - // Scan to find the closing quote, tracking whether escape sequences exist. - start := w.pos - hasEscape := false - for w.pos < len(w.data) { - ch := w.data[w.pos] - if ch == '\\' { - if w.pos+1 >= len(w.data) { - return errors.New("unterminated string in JSON") - } - hasEscape = true - w.pos += 2 // skip '\' and the escaped character - continue - } - if ch == '"' { - break - } - w.pos++ - } - if w.pos >= len(w.data) { - return errors.New("unterminated string in JSON") - } - - content := w.data[start:w.pos] - w.pos++ // skip closing '"' - - if !hasEscape { - // Fast path: no escape sequences, raw bytes are the decoded string. - sqltypes.MakeTrusted(querypb.Type_VARCHAR, content).EncodeSQLBytes2(w.buf) - } else { - // Slow path: unescape JSON into scratch buffer, then SQL-encode. - var err error - w.scratch, err = unescapeJSON(w.scratch[:0], content) - if err != nil { - return err - } - sqltypes.MakeTrusted(querypb.Type_VARCHAR, w.scratch).EncodeSQLBytes2(w.buf) - } - return nil -} - -// unescapeJSON appends the unescaped form of a JSON string body -// (the bytes between the quotes, not including the quotes themselves) to -// dst and returns the result. It handles all JSON escape sequences -// including \uXXXX and UTF-16 surrogate pairs. -func unescapeJSON(dst, src []byte) ([]byte, error) { - i := 0 - for i < len(src) { - if src[i] != '\\' { - dst = append(dst, src[i]) - i++ - continue - } - if i+1 >= len(src) { - return dst, errors.New("truncated escape sequence in JSON string") - } - i++ // skip '\' - switch src[i] { - case '"', '\\', '/': - dst = append(dst, src[i]) - i++ - case 'b': - dst = append(dst, '\b') - i++ - case 'f': - dst = append(dst, '\f') - i++ - case 'n': - dst = append(dst, '\n') - i++ - case 'r': - dst = append(dst, '\r') - i++ - case 't': - dst = append(dst, '\t') - i++ - case 'u': - i++ // skip 'u' - if i+4 > len(src) { - return dst, errors.New("truncated \\u escape in JSON string") - } - r := parseHex4(src[i : i+4]) - if r < 0 { - return dst, fmt.Errorf("invalid hex digit in \\u escape: %q", src[i:i+4]) - } - i += 4 - - // Handle UTF-16 surrogate pairs. - if utf16.IsSurrogate(r) { - if i+6 <= len(src) && src[i] == '\\' && src[i+1] == 'u' { - r2 := parseHex4(src[i+2 : i+6]) - if r2 >= 0 { - combined := utf16.DecodeRune(r, r2) - if combined != utf8.RuneError { - dst = utf8.AppendRune(dst, combined) - i += 6 - continue - } - } - } - // Lone surrogate: encode as replacement character. - dst = utf8.AppendRune(dst, utf8.RuneError) - continue - } - - dst = utf8.AppendRune(dst, r) - default: - return dst, fmt.Errorf("invalid escape character %q in JSON string", src[i]) - } - } - return dst, nil -} - -// parseHex4 parses exactly 4 hex digits into a rune. Returns -1 on error. -func parseHex4(s []byte) rune { - var r rune - for _, ch := range s { - r <<= 4 - switch { - case ch >= '0' && ch <= '9': - r |= rune(ch - '0') - case ch >= 'a' && ch <= 'f': - r |= rune(ch - 'a' + 10) - case ch >= 'A' && ch <= 'F': - r |= rune(ch - 'A' + 10) - default: - return -1 - } - } - return r -} - -func (w *sqlWriter) writeNumber(top bool) error { - // Use the parser's readFloat to validate number grammar, rejecting - // malformed inputs like "1+2", "1..2", or "1e+" that a simple - // character-class loop would accept. - n, _, ok := readFloat(w.data[w.pos:]) - if !ok || n == 0 { - return fmt.Errorf("invalid number at position %d in JSON", w.pos) - } - if top { - w.buf.WriteString("CAST(") - } - w.buf.Write(w.data[w.pos : w.pos+n]) - w.pos += n - if top { - w.buf.WriteString(" as JSON)") - } - return nil -} - -func (w *sqlWriter) writeBool(top bool) error { - if top { - w.buf.WriteString("CAST(_utf8mb4'") - } - if w.pos+4 <= len(w.data) && string(w.data[w.pos:w.pos+4]) == "true" { - w.buf.WriteString("true") - w.pos += 4 - } else if w.pos+5 <= len(w.data) && string(w.data[w.pos:w.pos+5]) == "false" { - w.buf.WriteString("false") - w.pos += 5 - } else { - return fmt.Errorf("unexpected token at position %d in JSON", w.pos) - } - if top { - w.buf.WriteString("' as JSON)") - } - return nil -} - -func (w *sqlWriter) writeNull(top bool) error { - if w.pos+4 > len(w.data) || string(w.data[w.pos:w.pos+4]) != "null" { - return fmt.Errorf("unexpected token at position %d in JSON", w.pos) - } - if top { - w.buf.WriteString("CAST(_utf8mb4'") - } - w.buf.WriteString("null") - w.pos += 4 - if top { - w.buf.WriteString("' as JSON)") - } - return nil ->>>>>>> ec6fdee983 (mysql/json: read numbers the way MySQL does (#20722)) } diff --git a/go/mysql/json/marshal_test.go b/go/mysql/json/marshal_test.go index 78dd09b4de3..d59f15a4892 100644 --- a/go/mysql/json/marshal_test.go +++ b/go/mysql/json/marshal_test.go @@ -85,158 +85,3 @@ func TestMarshalSQLValueNormalizesInvalidUTF8(t *testing.T) { expected := "CAST(JSON_QUOTE(_utf8mb4" + sqltypes.EncodeStringSQL(normalized) + ") as JSON)" require.Equal(t, expected, string(got.Raw())) } -<<<<<<< HEAD -||||||| parent of ec6fdee983 (mysql/json: read numbers the way MySQL does (#20722)) - -// TestAppendMarshalSQLDepthLimit verifies that AppendMarshalSQL enforces -// the same nesting depth limit as Parser.Parse. -func TestAppendMarshalSQLDepthLimit(t *testing.T) { - for _, tc := range []struct { - depth int - wantErr bool - }{ - {depth: 299, wantErr: false}, - {depth: 300, wantErr: false}, - {depth: 301, wantErr: true}, - } { - input := strings.Repeat("[", tc.depth) + strings.Repeat("]", tc.depth) - - var p Parser - _, parseErr := p.Parse(input) - - buf := &bytes2.Buffer{} - appendErr := AppendMarshalSQL(buf, []byte(input)) - - if tc.wantErr { - require.Error(t, parseErr, "depth %d: parser should reject", tc.depth) - assert.Error(t, appendErr, "depth %d: AppendMarshalSQL should reject", tc.depth) - } else { - assert.NoError(t, parseErr, "depth %d: parser should accept", tc.depth) - assert.NoError(t, appendErr, "depth %d: AppendMarshalSQL should accept", tc.depth) - } - } -} - -// TestAppendMarshalSQLNumberGrammar verifies that AppendMarshalSQL rejects -// malformed JSON numbers that a naive character-class scanner would accept. -func TestAppendMarshalSQLNumberGrammar(t *testing.T) { - malformed := []string{ - `1+2`, - `1-2`, - `1..2`, - `1e+`, - `1e`, - `--1`, - } - for _, input := range malformed { - buf := &bytes2.Buffer{} - err := AppendMarshalSQL(buf, []byte(input)) - require.Error(t, err, "malformed number %q should be rejected", input) - } - - valid := []string{ - `0`, `42`, `-1`, `3.14`, `-0.5`, - `1e10`, `1E10`, `1e+10`, `1e-10`, `1.5e2`, - } - for _, input := range valid { - buf := &bytes2.Buffer{} - err := AppendMarshalSQL(buf, []byte(input)) - assert.NoError(t, err, "valid number %q should be accepted", input) - } -} - -// TestAppendMarshalSQLTrailingBackslash verifies that a backslash as the -// last byte of a string is rejected rather than causing an out-of-bounds read. -func TestAppendMarshalSQLTrailingBackslash(t *testing.T) { - inputs := []string{ - `"trailing\`, // backslash is last byte, no closing quote - `{"key": "val\"}`, // backslash before quote looks like escaped quote, string never closes - } - for _, input := range inputs { - buf := &bytes2.Buffer{} - err := AppendMarshalSQL(buf, []byte(input)) - require.Error(t, err, "input %q should be rejected", input) - assert.ErrorContains(t, err, "unterminated string", "input %q", input) - } -} -======= - -// TestAppendMarshalSQLDepthLimit verifies that AppendMarshalSQL enforces -// the same nesting depth limit as Parser.Parse. -func TestAppendMarshalSQLDepthLimit(t *testing.T) { - for _, tc := range []struct { - depth int - wantErr bool - }{ - {depth: 299, wantErr: false}, - {depth: 300, wantErr: false}, - {depth: 301, wantErr: true}, - } { - input := strings.Repeat("[", tc.depth) + strings.Repeat("]", tc.depth) - - var p Parser - _, parseErr := p.Parse(input) - - buf := &bytes2.Buffer{} - appendErr := AppendMarshalSQL(buf, []byte(input)) - - if tc.wantErr { - require.Error(t, parseErr, "depth %d: parser should reject", tc.depth) - assert.Error(t, appendErr, "depth %d: AppendMarshalSQL should reject", tc.depth) - } else { - assert.NoError(t, parseErr, "depth %d: parser should accept", tc.depth) - assert.NoError(t, appendErr, "depth %d: AppendMarshalSQL should accept", tc.depth) - } - } -} - -// TestAppendMarshalSQLNumberGrammar verifies that AppendMarshalSQL rejects -// malformed JSON numbers that a naive character-class scanner would accept. -func TestAppendMarshalSQLNumberGrammar(t *testing.T) { - malformed := []string{ - `1+2`, - `1-2`, - `1..2`, - `1e+`, - `1e`, - `--1`, - // Shapes JSON's grammar does not have: a written plus, an integer - // part opening with a zero, and a decimal point with nothing on - // one side of it. - `+1`, - `007`, - `.2`, - `12.`, - } - for _, input := range malformed { - buf := &bytes2.Buffer{} - err := AppendMarshalSQL(buf, []byte(input)) - require.Error(t, err, "malformed number %q should be rejected", input) - } - - valid := []string{ - `0`, `42`, `-1`, `3.14`, `-0.5`, - `1e10`, `1E10`, `1e+10`, `1e-10`, `1.5e2`, - } - for _, input := range valid { - buf := &bytes2.Buffer{} - err := AppendMarshalSQL(buf, []byte(input)) - assert.NoError(t, err, "valid number %q should be accepted", input) - } -} - -// TestAppendMarshalSQLTrailingBackslash verifies that a backslash as the -// last byte of a string is rejected rather than causing an out-of-bounds read. -func TestAppendMarshalSQLTrailingBackslash(t *testing.T) { - inputs := []string{ - `"trailing\`, // backslash is last byte, no closing quote - `{"key": "val\"}`, // backslash before quote looks like escaped quote, string never closes - } - for _, input := range inputs { - buf := &bytes2.Buffer{} - err := AppendMarshalSQL(buf, []byte(input)) - require.Error(t, err, "input %q should be rejected", input) - assert.ErrorContains(t, err, "unterminated string", "input %q", input) - } -} ->>>>>>> ec6fdee983 (mysql/json: read numbers the way MySQL does (#20722)) diff --git a/go/mysql/json/parser.go b/go/mysql/json/parser.go index 82b3a20e08f..bda3673d1e5 100644 --- a/go/mysql/json/parser.go +++ b/go/mysql/json/parser.go @@ -769,13 +769,6 @@ func parseRawString(s string) (string, string, error) { } } -<<<<<<< HEAD -func readFloat(s string) (i int, ok bool) { - // optional sign -||||||| parent of ec6fdee983 (mysql/json: read numbers the way MySQL does (#20722)) -func readFloat[S string | []byte](s S) (i int, ok bool) { - // optional sign -======= // readFloat reads a JSON number off the front of s, returning how much of s it // covers and how far its exponent moves the decimal point. Whether the number // is one a double can hold is mysqlNumberFits's job; the exponent is reported so @@ -793,7 +786,6 @@ func readFloat[S string | []byte](s S) (i int, ok bool) { // that opens with a zero are all rejected. func readFloat[S string | []byte](s S) (i, exponent int, ok bool) { // optional minus. JSON numbers carry no written plus. ->>>>>>> ec6fdee983 (mysql/json: read numbers the way MySQL does (#20722)) if i >= len(s) { return } diff --git a/go/mysql/json/parser_test.go b/go/mysql/json/parser_test.go index af4ccdbf421..da50dc56b04 100644 --- a/go/mysql/json/parser_test.go +++ b/go/mysql/json/parser_test.go @@ -31,18 +31,10 @@ func TestParseRawNumber(t *testing.T) { f := func(s, expectedRN, expectedTail string) { t.Helper() -<<<<<<< HEAD - flen, ok := readFloat(s) + flen, _, ok := readFloat(s) if !ok { t.Fatalf("unexpected error when parsing '%s'", s) } -||||||| parent of ec6fdee983 (mysql/json: read numbers the way MySQL does (#20722)) - flen, ok := readFloat(s) - require.Truef(t, ok, "unexpected error when parsing '%s'", s) -======= - flen, _, ok := readFloat(s) - require.Truef(t, ok, "unexpected error when parsing '%s'", s) ->>>>>>> ec6fdee983 (mysql/json: read numbers the way MySQL does (#20722)) rn, tail := s[:flen], s[flen:] @@ -72,23 +64,13 @@ func TestParseRawNumber(t *testing.T) { f := func(s, expectedTail string) { t.Helper() -<<<<<<< HEAD - flen, ok := readFloat(s) + flen, _, ok := readFloat(s) if ok { t.Fatalf("expecting non-nil error") } if s[flen:] != expectedTail { t.Fatalf("unexpected tail; got %q; want %q", s[flen:], expectedTail) } -||||||| parent of ec6fdee983 (mysql/json: read numbers the way MySQL does (#20722)) - flen, ok := readFloat(s) - require.False(t, ok, "expecting non-nil error") - require.Equalf(t, expectedTail, s[flen:], "unexpected tail; got %q; want %q", s[flen:], expectedTail) -======= - flen, _, ok := readFloat(s) - require.False(t, ok, "expecting non-nil error") - require.Equalf(t, expectedTail, s[flen:], "unexpected tail; got %q; want %q", s[flen:], expectedTail) ->>>>>>> ec6fdee983 (mysql/json: read numbers the way MySQL does (#20722)) } f("xyz", "xyz")