From 5a1ed5bb7bfc04cc48ac5629916543c86014112c Mon Sep 17 00:00:00 2001 From: Arthur Schreiber Date: Tue, 28 Jul 2026 00:32:27 +0000 Subject: [PATCH 01/19] mysql/json: reject numbers a double cannot hold, as MySQL does MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MySQL decides a JSON number's storage when it parses the document: an integer that fits stays exact and everything else becomes a double. A number too large for a double therefore has nowhere to live, and MySQL calls the whole document invalid — JSON_VALID('1e1025') is 0, and inserting it into a JSON column is refused. Vitess parsed it happily and left every consumer to cope with a value that should not exist, which is how the same expression could error in the interpreter and answer 1 in the compiled form. Underflow is not rejected, matching MySQL: 1e-1000 is a valid document that reads as zero. The check is kept off the common path. readFloat already walks the exponent, so it reports whether one was written, and a number without one has to be longer than the largest double before it can overflow. Parsing arrays of integers and of fractional numbers both measure unchanged. Also sets the number type on the NaN branch, which left whatever the value cache last held in that field. Signed-off-by: Arthur Schreiber --- go/mysql/json/marshal.go | 2 +- go/mysql/json/parser.go | 27 ++++++++++++++++-- go/mysql/json/parser_test.go | 55 ++++++++++++++++++++++++++++++++++-- 3 files changed, 78 insertions(+), 6 deletions(-) diff --git a/go/mysql/json/marshal.go b/go/mysql/json/marshal.go index 8d527e37c4b..5d303bae5aa 100644 --- a/go/mysql/json/marshal.go +++ b/go/mysql/json/marshal.go @@ -491,7 +491,7 @@ func (w *sqlWriter) writeNumber(top bool) error { // Use the parser's readFloat to validate number grammar, rejecting // malformed inputs like "1+2", "1..2", or "1e+" that a simple // character-class loop would accept. - n, ok := readFloat(w.data[w.pos:]) + n, _, ok := readFloat(w.data[w.pos:]) if !ok || n == 0 { return fmt.Errorf("invalid number at position %d in JSON", w.pos) } diff --git a/go/mysql/json/parser.go b/go/mysql/json/parser.go index 47271e819cb..7b847615d6b 100644 --- a/go/mysql/json/parser.go +++ b/go/mysql/json/parser.go @@ -190,6 +190,7 @@ func parseValue(s string, c *cache, depth int) (*Value, string, error) { v := c.getValue() v.t = TypeNumber v.s = s[:3] + v.n = numberTypeRaw return v, s[3:], nil } return nil, s, fmt.Errorf("unexpected value found: %q", s) @@ -197,7 +198,7 @@ func parseValue(s string, c *cache, depth int) (*Value, string, error) { return ValueNull, s[len("null"):], nil } - flen, ok := readFloat(s) + flen, exponent, ok := readFloat(s) if !ok { return nil, s[flen:], fmt.Errorf("invalid number in JSON string: %q", s) } @@ -206,9 +207,28 @@ func parseValue(s string, c *cache, depth int) (*Value, string, error) { v.t = TypeNumber v.s = s[:flen] v.n = numberTypeRaw + if mayExceedFloat64(v.s, exponent) { + if _, err := fastparse.ParseFloat64(v.s); err != nil { + return nil, s, fmt.Errorf("number too big to be stored in double: %q", v.s) + } + } return v, s[flen:], nil } +// maxFloat64Digits is the number of digits it takes to write a number that a +// double cannot hold. The largest double is under 1.8e308, so 308 digits always +// fit and 309 need not. +const maxFloat64Digits = 308 + +// mayExceedFloat64 reports whether num is worth converting to find out whether +// a double can hold it. It errs towards yes: the job is to keep the conversion +// off the common path, not to answer the question. Without an exponent a number +// has to be written out to more digits than the largest double before it can be +// too big for one. +func mayExceedFloat64(num string, exponent bool) bool { + return exponent || len(num) > maxFloat64Digits +} + func parseArray(s string, c *cache, depth int) (*Value, string, error) { s = skipWS(s) if len(s) == 0 { @@ -513,7 +533,7 @@ func parseRawString(s string) (string, string, error) { } } -func readFloat[S string | []byte](s S) (i int, ok bool) { +func readFloat[S string | []byte](s S) (i int, exponent bool, ok bool) { // optional sign if i >= len(s) { return @@ -556,6 +576,7 @@ loop: // a lot (say, 100000). it doesn't matter if it's // not the exact number. if i < len(s) && (s[i] == 'e' || s[i] == 'E') { + exponent = true i++ if i >= len(s) { return @@ -569,7 +590,7 @@ loop: for ; i < len(s) && ('0' <= s[i] && s[i] <= '9'); i++ { } } - return i, true + return i, exponent, true } // Object represents JSON object. diff --git a/go/mysql/json/parser_test.go b/go/mysql/json/parser_test.go index a223b26fc40..3761b2b887a 100644 --- a/go/mysql/json/parser_test.go +++ b/go/mysql/json/parser_test.go @@ -18,6 +18,7 @@ limitations under the License. package json import ( + "strings" "testing" "github.com/stretchr/testify/require" @@ -30,7 +31,7 @@ func TestParseRawNumber(t *testing.T) { f := func(s, expectedRN, expectedTail string) { t.Helper() - flen, ok := readFloat(s) + flen, _, ok := readFloat(s) require.Truef(t, ok, "unexpected error when parsing '%s'", s) rn, tail := s[:flen], s[flen:] @@ -57,7 +58,7 @@ func TestParseRawNumber(t *testing.T) { f := func(s, expectedTail string) { t.Helper() - flen, ok := readFloat(s) + flen, _, ok := readFloat(s) require.False(t, ok, "expecting non-nil error") require.Equalf(t, expectedTail, s[flen:], "unexpected tail; got %q; want %q", s[flen:], expectedTail) } @@ -71,6 +72,56 @@ func TestParseRawNumber(t *testing.T) { }) } +// TestParseNumberTooBigForDouble covers the boundary MySQL puts on JSON +// numbers: a number it cannot store as a double makes the whole document +// invalid, rather than being kept at the precision it was written to. +// Underflow is not rejected — it flushes to zero. +func TestParseNumberTooBigForDouble(t *testing.T) { + tooManyDigits := "1" + strings.Repeat("0", 309) + + t.Run("accepted", func(t *testing.T) { + for _, doc := range []string{ + "1e308", + "-1e308", + "1.7976931348623157e308", + "-1.7976931348623157e308", + "99999999999999999999999999999999999999999", + "1" + strings.Repeat("0", 307), + // Underflow keeps the document valid and reads as zero. + "1e-400", + "1e-1000", + "0." + strings.Repeat("0", 400) + "1", + } { + t.Run(startEndString(doc), func(t *testing.T) { + var p Parser + v, err := p.Parse(doc) + require.NoError(t, err) + require.Equal(t, TypeNumber, v.Type()) + }) + } + }) + + t.Run("rejected", func(t *testing.T) { + for _, doc := range []string{ + "1e309", + "-1e309", + "1e1025", + "1.7976931348623159e308", + tooManyDigits, + // A number anywhere in the document invalidates all of it. + "[1, 1e309]", + `{"a": 1e309}`, + "[[1e309]]", + } { + t.Run(startEndString(doc), func(t *testing.T) { + var p Parser + _, err := p.Parse(doc) + require.ErrorContains(t, err, "number too big to be stored in double") + }) + } + }) +} + func TestUnescapeStringBestEffort(t *testing.T) { t.Run("success", func(t *testing.T) { testUnescapeStringBestEffort(t, ``, ``) From f4aa5ecfbd30228eb6c366666e01386af5e90feb Mon Sep 17 00:00:00 2001 From: Arthur Schreiber Date: Tue, 28 Jul 2026 00:36:26 +0000 Subject: [PATCH 02/19] mysql/json: cover the exponent spellings called out on the issue A written sign, a zero-padded exponent and an underflowing one are all spellings rather than magnitudes, and all stay valid. They go through the conversion because they carry an exponent, so they are the cases most likely to be caught by mistake. Signed-off-by: Arthur Schreiber --- go/mysql/json/parser_test.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/go/mysql/json/parser_test.go b/go/mysql/json/parser_test.go index 3761b2b887a..37ff8988420 100644 --- a/go/mysql/json/parser_test.go +++ b/go/mysql/json/parser_test.go @@ -90,7 +90,15 @@ func TestParseNumberTooBigForDouble(t *testing.T) { // Underflow keeps the document valid and reads as zero. "1e-400", "1e-1000", + "1e-1024", "0." + strings.Repeat("0", 400) + "1", + // A written sign and a padded exponent are spellings, not + // magnitudes, and none of these is anywhere near the limit. + "1e+0", + "1e-0", + "1e0000000000", + "1e-0000000000", + "1e+308", } { t.Run(startEndString(doc), func(t *testing.T) { var p Parser @@ -107,6 +115,7 @@ func TestParseNumberTooBigForDouble(t *testing.T) { "-1e309", "1e1025", "1.7976931348623159e308", + "1e+309", tooManyDigits, // A number anywhere in the document invalidates all of it. "[1, 1e309]", From e68bd7e54c81f0dd25b8a79d4708329bd63b84c2 Mon Sep 17 00:00:00 2001 From: Arthur Schreiber Date: Tue, 28 Jul 2026 07:06:08 +0000 Subject: [PATCH 03/19] mysql/json: bound the exponent as written, not just as converted MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A JSON number carries two ways of being too big, and only one of them shows up after conversion. MySQL bounds the exponent a number is written with before it converts anything: the decimal point can travel 308 places, plus one for every digit the number already has after it. 0e309 is refused on those grounds even though it is zero, while 0.00e310 is fine because its two fraction digits buy the places back. Converting caught the second way and missed the first, so every exponent that lands on zero came through: 0e309, 0.0e310, 0.00e311. Converting is still needed for the other direction — 10e308 is written within the bound and overflows anyway. readFloat already walks both the fraction digits and the exponent, so it reports them rather than reporting that an exponent was there at all. A written exponent stops at a ceiling that clears the largest double plus every digit the number could hold after its point, so a long one neither overflows nor changes the answer. Negative exponents are left alone, matching MySQL: underflow is a valid document that reads as zero. Parsing arrays of integers, of fractional numbers and of numbers with exponents all measure unchanged. Signed-off-by: Arthur Schreiber --- go/mysql/json/marshal.go | 2 +- go/mysql/json/parser.go | 55 +++++++++++++++++++++++++----------- go/mysql/json/parser_test.go | 30 ++++++++++++++++++-- 3 files changed, 67 insertions(+), 20 deletions(-) diff --git a/go/mysql/json/marshal.go b/go/mysql/json/marshal.go index 5d303bae5aa..2a3b19ea5db 100644 --- a/go/mysql/json/marshal.go +++ b/go/mysql/json/marshal.go @@ -491,7 +491,7 @@ func (w *sqlWriter) writeNumber(top bool) error { // Use the parser's readFloat to validate number grammar, rejecting // malformed inputs like "1+2", "1..2", or "1e+" that a simple // character-class loop would accept. - n, _, ok := readFloat(w.data[w.pos:]) + n, _, _, ok := readFloat(w.data[w.pos:]) if !ok || n == 0 { return fmt.Errorf("invalid number at position %d in JSON", w.pos) } diff --git a/go/mysql/json/parser.go b/go/mysql/json/parser.go index 7b847615d6b..c5e7a9330fa 100644 --- a/go/mysql/json/parser.go +++ b/go/mysql/json/parser.go @@ -198,7 +198,7 @@ func parseValue(s string, c *cache, depth int) (*Value, string, error) { return ValueNull, s[len("null"):], nil } - flen, exponent, ok := readFloat(s) + flen, exponent, fraction, ok := readFloat(s) if !ok { return nil, s[flen:], fmt.Errorf("invalid number in JSON string: %q", s) } @@ -207,6 +207,9 @@ func parseValue(s string, c *cache, depth int) (*Value, string, error) { v.t = TypeNumber v.s = s[:flen] v.n = numberTypeRaw + if exponent > maxFloat64Digits+fraction { + return nil, s, fmt.Errorf("number too big to be stored in double: %q", v.s) + } if mayExceedFloat64(v.s, exponent) { if _, err := fastparse.ParseFloat64(v.s); err != nil { return nil, s, fmt.Errorf("number too big to be stored in double: %q", v.s) @@ -215,18 +218,19 @@ func parseValue(s string, c *cache, depth int) (*Value, string, error) { return v, s[flen:], nil } -// maxFloat64Digits is the number of digits it takes to write a number that a -// double cannot hold. The largest double is under 1.8e308, so 308 digits always -// fit and 309 need not. +// maxFloat64Digits is how far a decimal point can travel before a double runs +// out of room. The largest double is under 1.8e308, so a written exponent of +// 308 always leaves somewhere for the number to land and 309 need not. const maxFloat64Digits = 308 // mayExceedFloat64 reports whether num is worth converting to find out whether // a double can hold it. It errs towards yes: the job is to keep the conversion -// off the common path, not to answer the question. Without an exponent a number -// has to be written out to more digits than the largest double before it can be -// too big for one. -func mayExceedFloat64(num string, exponent bool) bool { - return exponent || len(num) > maxFloat64Digits +// off the common path, not to answer the question. A number is below the +// largest double whenever the digits it is written to, moved by its exponent, +// stay within the ones that double has — len(num) overcounts the digits, which +// only makes the answer yes more often. +func mayExceedFloat64(num string, exponent int) bool { + return len(num)+exponent > maxFloat64Digits } func parseArray(s string, c *cache, depth int) (*Value, string, error) { @@ -533,7 +537,11 @@ func parseRawString(s string) (string, string, error) { } } -func readFloat[S string | []byte](s S) (i int, exponent bool, ok bool) { +// readFloat reads a JSON number off the front of s, returning how much of s it +// covers, the exponent it was written with, and how many digits it carries +// after its decimal point. Together those say how far the number's digits sit +// from where a double keeps them. +func readFloat[S string | []byte](s S) (i, exponent, fraction int, ok bool) { // optional sign if i >= len(s) { return @@ -558,6 +566,9 @@ loop: case '0' <= c && c <= '9': sawdigits = true + if sawdot { + fraction++ + } if c == '0' && nd == 0 { // ignore leading zeros continue } @@ -570,27 +581,37 @@ loop: return } - // optional exponent moves decimal point. - // if we read a very large, very long number, - // just be sure to move the decimal point by - // a lot (say, 100000). it doesn't matter if it's - // not the exact number. + // optional exponent moves the decimal point. An exponent larger than + // exponentCeiling stops there rather than running on: the ceiling clears + // both the largest double and every digit this number could hold after its + // decimal point, so a number stopped at it is out of reach either way. if i < len(s) && (s[i] == 'e' || s[i] == 'E') { - exponent = true i++ if i >= len(s) { return } + negative := false if s[i] == '+' || s[i] == '-' { + negative = s[i] == '-' i++ } if i >= len(s) || s[i] < '0' || s[i] > '9' { return } + exponentCeiling := maxFloat64Digits + len(s) for ; i < len(s) && ('0' <= s[i] && s[i] <= '9'); i++ { + if exponent <= exponentCeiling { + exponent = exponent*10 + int(s[i]-'0') + } + } + if exponent > exponentCeiling { + exponent = exponentCeiling + } + if negative { + exponent = -exponent } } - return i, exponent, true + return i, exponent, fraction, true } // Object represents JSON object. diff --git a/go/mysql/json/parser_test.go b/go/mysql/json/parser_test.go index 37ff8988420..67e7df8f448 100644 --- a/go/mysql/json/parser_test.go +++ b/go/mysql/json/parser_test.go @@ -31,7 +31,7 @@ func TestParseRawNumber(t *testing.T) { f := func(s, expectedRN, expectedTail string) { t.Helper() - flen, _, ok := readFloat(s) + flen, _, _, ok := readFloat(s) require.Truef(t, ok, "unexpected error when parsing '%s'", s) rn, tail := s[:flen], s[flen:] @@ -58,7 +58,7 @@ func TestParseRawNumber(t *testing.T) { f := func(s, expectedTail string) { t.Helper() - flen, _, ok := readFloat(s) + flen, _, _, ok := readFloat(s) require.False(t, ok, "expecting non-nil error") require.Equalf(t, expectedTail, s[flen:], "unexpected tail; got %q; want %q", s[flen:], expectedTail) } @@ -99,6 +99,17 @@ func TestParseNumberTooBigForDouble(t *testing.T) { "1e0000000000", "1e-0000000000", "1e+308", + "1e00000000000000000308", + // A written exponent is bounded by where it puts the decimal + // point, so digits after the point buy the same number of places + // back. Zero is subject to the bound like anything else. + "0e308", + "-0e308", + "0.0e309", + "0.00e310", + "0.1e309", + "0.01e310", + "0." + strings.Repeat("0", 400) + "1e700", } { t.Run(startEndString(doc), func(t *testing.T) { var p Parser @@ -117,6 +128,21 @@ func TestParseNumberTooBigForDouble(t *testing.T) { "1.7976931348623159e308", "1e+309", tooManyDigits, + // One place past what the digits after the point buy back. These + // all convert to zero, so only the exponent as written rules them + // out. + "0e309", + "-0e309", + "0e+309", + "0e1000", + "0.0e310", + "0.00e311", + "0.1e310", + "0.01e311", + "0." + strings.Repeat("0", 400) + "1e710", + // Within the written bound, but too big once converted. + "10e308", + "1" + strings.Repeat("0", 30) + "e279", // A number anywhere in the document invalidates all of it. "[1, 1e309]", `{"a": 1e309}`, From fa351ff9e45f4ea3ada862a53f5bca5d229f0199 Mon Sep 17 00:00:00 2001 From: Arthur Schreiber Date: Tue, 28 Jul 2026 07:25:45 +0000 Subject: [PATCH 04/19] mysql/json: read numbers by JSON's grammar, not Go's MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The number reader was a general-purpose one: it took a written plus, an integer part opening with a zero, and a decimal point with nothing on one side of it. JSON allows none of those and MySQL rejects all of them, so Vitess answered questions MySQL refuses to answer — JSON_EXTRACT('{"a": .2}', '$.a') is 0.2 here and an error there. Which one a query gets depends on whether it is evaluated in vtgate or pushed down, so the same query fails in one place and succeeds in the other. A number written 007 also kept its leading zeros all the way back out again. Reading the grammar directly turns out to be less work than the loop that read around it: the leading-zero bookkeeping and the labelled switch go away, and parsing an array of integers or of fractional numbers is around a fifth to a quarter faster. Signed-off-by: Arthur Schreiber --- go/mysql/json/parser.go | 53 ++++++++++++++++-------------------- go/mysql/json/parser_test.go | 12 ++++++-- 2 files changed, 33 insertions(+), 32 deletions(-) diff --git a/go/mysql/json/parser.go b/go/mysql/json/parser.go index c5e7a9330fa..9d290d4ef97 100644 --- a/go/mysql/json/parser.go +++ b/go/mysql/json/parser.go @@ -541,44 +541,39 @@ func parseRawString(s string) (string, string, error) { // covers, the exponent it was written with, and how many digits it carries // after its decimal point. Together those say how far the number's digits sit // from where a double keeps them. +// +// What counts as a number is JSON's grammar rather than Go's: a written plus, +// a missing digit on either side of the decimal point, and an integer part +// that opens with a zero are all rejected. func readFloat[S string | []byte](s S) (i, exponent, fraction int, ok bool) { - // optional sign + // optional minus. JSON numbers carry no written plus. if i >= len(s) { return } - if s[i] == '+' || s[i] == '-' { + if s[i] == '-' { i++ } - // digits - sawdot := false - sawdigits := false - nd := 0 -loop: - for ; i < len(s); i++ { - switch c := s[i]; true { - case c == '.': - if sawdot { - break loop - } - sawdot = true - continue - - case '0' <= c && c <= '9': - sawdigits = true - if sawdot { - fraction++ - } - if c == '0' && nd == 0 { // ignore leading zeros - continue - } - nd++ - continue + // integer part: either a lone zero, or digits that do not open with one + if i >= len(s) || s[i] < '0' || s[i] > '9' { + return + } + if s[i] == '0' { + i++ + } else { + for ; i < len(s) && '0' <= s[i] && s[i] <= '9'; i++ { } - break } - if !sawdigits { - return + + // optional fraction, which has to carry a digit of its own + if i < len(s) && s[i] == '.' { + i++ + if i >= len(s) || s[i] < '0' || s[i] > '9' { + return + } + for ; i < len(s) && '0' <= s[i] && s[i] <= '9'; i++ { + fraction++ + } } // optional exponent moves the decimal point. An exponent larger than diff --git a/go/mysql/json/parser_test.go b/go/mysql/json/parser_test.go index 67e7df8f448..23bd2bd81ce 100644 --- a/go/mysql/json/parser_test.go +++ b/go/mysql/json/parser_test.go @@ -49,9 +49,8 @@ func TestParseRawNumber(t *testing.T) { f("-12.345E+67 tail", "-12.345E+67", " tail") f("-12.345E-67,tail", "-12.345E-67", ",tail") f("-1234567.8e+90tail", "-1234567.8e+90", "tail") - f("12.tail", "12.", "tail") - f(".2tail", ".2", "tail") - f("-.2tail", "-.2", "tail") + f("0.2tail", "0.2", "tail") + f("-0.2tail", "-0.2", "tail") }) t.Run("error", func(t *testing.T) { @@ -69,6 +68,13 @@ func TestParseRawNumber(t *testing.T) { f(",", ",") f("{", "{") f("\"", "\"") + + // A decimal point needs a digit on either side of it, and a number + // opens with a minus or a digit. + f("12.tail", "tail") + f(".2tail", ".2tail") + f("-.2tail", ".2tail") + f("+1tail", "+1tail") }) } From fe181de509f4709a1375267f605389b95d5ff8e4 Mon Sep 17 00:00:00 2001 From: Arthur Schreiber Date: Tue, 28 Jul 2026 07:34:53 +0000 Subject: [PATCH 05/19] mysql/json: stop reading nan as a number MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The parser carried a branch for nan, from the JSON reader it grew out of. It is an extension to JSON rather than part of it, and MySQL does not take it: JSON_VALID('nan') is 0. A document Vitess accepted here was one no MySQL column could hold, and 'nan' as a JSON bind value reached the evalengine as a number no consumer could read — the interpreter reports a decimal range error where the compiled form compares two of them equal. Deleting the branch also removes the number type it had to set, which was whatever the value cache last left there. Signed-off-by: Arthur Schreiber --- go/mysql/json/parser.go | 8 -------- go/mysql/json/parser_test.go | 40 ++++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/go/mysql/json/parser.go b/go/mysql/json/parser.go index 9d290d4ef97..6aa4e7f9aa7 100644 --- a/go/mysql/json/parser.go +++ b/go/mysql/json/parser.go @@ -185,14 +185,6 @@ func parseValue(s string, c *cache, depth int) (*Value, string, error) { } if s[0] == 'n' { if len(s) < len("null") || s[:len("null")] != "null" { - // Try parsing NaN - if len(s) >= 3 && strings.EqualFold(s[:3], "nan") { - v := c.getValue() - v.t = TypeNumber - v.s = s[:3] - v.n = numberTypeRaw - return v, s[3:], nil - } return nil, s, fmt.Errorf("unexpected value found: %q", s) } return ValueNull, s[len("null"):], nil diff --git a/go/mysql/json/parser_test.go b/go/mysql/json/parser_test.go index 23bd2bd81ce..9a4e1c2d91b 100644 --- a/go/mysql/json/parser_test.go +++ b/go/mysql/json/parser_test.go @@ -163,6 +163,46 @@ func TestParseNumberTooBigForDouble(t *testing.T) { }) } +// TestParseNumberGrammar covers the shapes JSON's grammar allows a number to +// take. A number opens with a minus or a digit, an integer part of more than +// one digit does not open with a zero, and a decimal point has digits on both +// sides of it. MySQL holds documents to the same grammar, and nan is not a +// number to either of them. +func TestParseNumberGrammar(t *testing.T) { + t.Run("accepted", func(t *testing.T) { + for _, doc := range []string{ + "0", "-0", "0.5", "-0.5", "1", "-1", "1.2", "0e0", "-0e0", + "1e5", "1E5", "1e007", "1e+007", "1e-5", "0.0", + "[0,1,2]", `{"a":-0.5,"b":[1e5]}`, + } { + t.Run(doc, func(t *testing.T) { + var p Parser + _, err := p.Parse(doc) + require.NoError(t, err) + }) + } + }) + + t.Run("rejected", func(t *testing.T) { + for _, doc := range []string{ + // An integer part that opens with a zero. + "007", "-003", "01", "00", "00.5", "01.5", "[007]", + // A decimal point missing a digit on one side. + ".2", "-.2", "12.", "-12.", "1.e5", `{"a": .2}`, "[12.]", + // A written plus. + "+1", "+1.5", "+0", "[+1]", + // Not a number at all. + "nan", "NaN", "NAN", "[nan]", `{"a": nan}`, "-nan", ".", "-", + } { + t.Run(doc, func(t *testing.T) { + var p Parser + _, err := p.Parse(doc) + require.Error(t, err) + }) + } + }) +} + func TestUnescapeStringBestEffort(t *testing.T) { t.Run("success", func(t *testing.T) { testUnescapeStringBestEffort(t, ``, ``) From b450fe6ac27786344e5ed42589714ee7f7037f1e Mon Sep 17 00:00:00 2001 From: Arthur Schreiber Date: Tue, 28 Jul 2026 08:15:19 +0000 Subject: [PATCH 06/19] mysql/json: decide what a double can hold the way MySQL decides it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Whether a JSON number is too big is not a question about the number, it turns out — it is a question about how the number was written. MySQL parses JSON with RapidJSON, which splits the digits between a significand it accumulates into a double and a power of ten to scale that by, and the split decides which way the last place rounds. 1.7976931348623158e308 does not fit. The same value written 1.79769313486231580e308 does, because the extra digit moves the split. Comparing a correctly converted value against the largest double cannot tell those two apart, so it took the first one as well. So the conversion is now RapidJSON's rather than a correct one, deliberately landing an ULP or two off the true value so the boundary lands where MySQL's does. The written-exponent bound moves into it as well, since that is where RapidJSON applies it, leaving one place that decides whether a document holds a number instead of two that had to agree. Checked against MySQL 8.0.45, 8.4.11 and 9.4.0 over 1279 documents — both sides of the boundary at every spelling above, the exponent bound stepped one place at a time, and 900 random numbers, a fifth of them crowded up against the largest double. All three versions agree with each other and with this, and 8.4 still parses with the same flags and the same code. The check runs only for numbers whose digits could reach that far, as before, and reaching the answer in one pass rather than converting the whole number makes those about a third cheaper. Numbers that cannot reach it are untouched. Signed-off-by: Arthur Schreiber --- go/mysql/json/marshal.go | 2 +- go/mysql/json/parser.go | 224 +++++++++++++++++++++++++++++++++-- go/mysql/json/parser_test.go | 40 ++++++- 3 files changed, 250 insertions(+), 16 deletions(-) diff --git a/go/mysql/json/marshal.go b/go/mysql/json/marshal.go index 2a3b19ea5db..5d303bae5aa 100644 --- a/go/mysql/json/marshal.go +++ b/go/mysql/json/marshal.go @@ -491,7 +491,7 @@ func (w *sqlWriter) writeNumber(top bool) error { // Use the parser's readFloat to validate number grammar, rejecting // malformed inputs like "1+2", "1..2", or "1e+" that a simple // character-class loop would accept. - n, _, _, ok := readFloat(w.data[w.pos:]) + n, _, ok := readFloat(w.data[w.pos:]) if !ok || n == 0 { return fmt.Errorf("invalid number at position %d in JSON", w.pos) } diff --git a/go/mysql/json/parser.go b/go/mysql/json/parser.go index 6aa4e7f9aa7..9d78e9f8efd 100644 --- a/go/mysql/json/parser.go +++ b/go/mysql/json/parser.go @@ -22,6 +22,7 @@ import ( "encoding/base64" "errors" "fmt" + "math" "slices" "strconv" "strings" @@ -190,7 +191,7 @@ func parseValue(s string, c *cache, depth int) (*Value, string, error) { return ValueNull, s[len("null"):], nil } - flen, exponent, fraction, ok := readFloat(s) + flen, exponent, ok := readFloat(s) if !ok { return nil, s[flen:], fmt.Errorf("invalid number in JSON string: %q", s) } @@ -199,14 +200,9 @@ func parseValue(s string, c *cache, depth int) (*Value, string, error) { v.t = TypeNumber v.s = s[:flen] v.n = numberTypeRaw - if exponent > maxFloat64Digits+fraction { + if mayExceedFloat64(v.s, exponent) && !mysqlNumberFits(v.s) { return nil, s, fmt.Errorf("number too big to be stored in double: %q", v.s) } - if mayExceedFloat64(v.s, exponent) { - if _, err := fastparse.ParseFloat64(v.s); err != nil { - return nil, s, fmt.Errorf("number too big to be stored in double: %q", v.s) - } - } return v, s[flen:], nil } @@ -225,6 +221,209 @@ func mayExceedFloat64(num string, exponent int) bool { return len(num)+exponent > maxFloat64Digits } +// pow10 holds the powers of ten that a number's digits are scaled by, which is +// the table RapidJSON scales its significand with. It is built from the decimal +// spellings rather than from math.Pow10, which multiplies two table entries +// together and lands a bit away from the spelling for 78 of these exponents. +var pow10 [maxFloat64Digits + 1]float64 + +func init() { + for i := range pow10 { + pow10[i], _ = fastparse.ParseFloat64("1e" + strconv.Itoa(i)) + } +} + +// mysqlNumberFits reports whether MySQL can store num, which has to already be +// a grammatically valid JSON number. +// +// MySQL parses JSON with RapidJSON, which decides this in two places. While +// reading the exponent, a written exponent may not move the decimal point more +// than maxFloat64Digits places past the digits already behind it. After +// converting, the result may not be larger than the largest double. Neither +// place catches what the other does: an exponent that lands on zero is refused +// only as written, and a number written within the bound can still overflow +// once converted. +// +// The conversion below is the one RapidJSON does rather than a correct one. It +// accumulates the significand into a double and scales that by a power of ten, +// which lands an ULP or two from the true value, and the boundary sits wherever +// that lands. A number a hair under the largest double therefore comes down to +// how it was spelled: 1.7976931348623158e308 does not fit, while writing the +// same value as 1.79769313486231580e308 does, because the extra digit moves +// where the significand ends and the scaling begins. Converting the same way is +// what makes a document valid here exactly when it is valid in MySQL. +func mysqlNumberFits(num string) bool { + i := 0 + minus := num[i] == '-' + if minus { + i++ + } + + var ( + u32 uint32 + u64 uint64 + use64 bool + sigDigits int + d float64 + useDouble bool + ) + digit := func() bool { return i < len(num) && num[i] >= '0' && num[i] <= '9' } + + // The integer part accumulates as an integer for as long as one can hold + // it, stepping up in width as it fills. RapidJSON leaves the leading digit + // out of sigDigits, and the seventeen-digit cut-off in the fraction below is + // measured against that count. + if num[i] == '0' { + i++ + } else { + u32 = uint32(num[i] - '0') + i++ + for digit() { + if minus { + if u32 >= 214748364 && (u32 != 214748364 || num[i] > '8') { + u64, use64 = uint64(u32), true + break + } + } else if u32 >= 429496729 && (u32 != 429496729 || num[i] > '5') { + u64, use64 = uint64(u32), true + break + } + u32 = u32*10 + uint32(num[i]-'0') + i++ + sigDigits++ + } + } + if use64 { + for digit() { + if minus { + if u64 >= 0x0CCCCCCCCCCCCCCC && (u64 != 0x0CCCCCCCCCCCCCCC || num[i] > '8') { + d, useDouble = float64(u64), true + break + } + } else if u64 >= 0x1999999999999999 && (u64 != 0x1999999999999999 || num[i] > '5') { + d, useDouble = float64(u64), true + break + } + u64 = u64*10 + uint64(num[i]-'0') + i++ + sigDigits++ + } + } + for useDouble && digit() { + d = d*10 + float64(num[i]-'0') + i++ + } + + // Digits behind the decimal point move it left, which is what expFrac + // counts. Past seventeen significant digits they stop counting and stop + // arriving, so they move it no further. + expFrac := 0 + if i < len(num) && num[i] == '.' { + i++ + if !useDouble { + if !use64 { + u64 = uint64(u32) + } + for digit() { + if u64 > 0x1FFFFFFFFFFFFF { + break + } + u64 = u64*10 + uint64(num[i]-'0') + i++ + expFrac-- + if u64 != 0 { + sigDigits++ + } + } + d = float64(u64) + useDouble = true + } + for digit() { + if sigDigits < 17 { + d = d*10 + float64(num[i]-'0') + expFrac-- + if d > 0 { + sigDigits++ + } + } + i++ + } + } + + exp := 0 + if i < len(num) && (num[i] == 'e' || num[i] == 'E') { + i++ + if !useDouble { + if use64 { + d = float64(u64) + } else { + d = float64(u32) + } + useDouble = true + } + negative := num[i] == '-' + if negative || num[i] == '+' { + i++ + } + exp = int(num[i] - '0') + i++ + if negative { + // A negative exponent is not bounded, only stopped before it could + // overflow the int it accumulates into. Everything out that far has + // flushed to zero long before. + maxExp := (expFrac + 2147483639) / 10 + for digit() { + exp = exp*10 + int(num[i]-'0') + i++ + if exp > maxExp { + for digit() { + i++ + } + } + } + exp = -exp + } else { + maxExp := maxFloat64Digits - expFrac + for digit() { + exp = exp*10 + int(num[i]-'0') + i++ + if exp > maxExp { + return false + } + } + } + } + + // A number that never needed a double is an integer MySQL keeps exact. + if !useDouble { + return true + } + return scaleByPow10(d, exp+expFrac) <= math.MaxFloat64 +} + +// scaleByPow10 moves d by p decimal places the way RapidJSON does, in one +// multiplication or division by a power of ten. Anything below the smallest +// power in the table is moved in two steps so that the table is never indexed +// past its end. +func scaleByPow10(d float64, p int) float64 { + if p < -maxFloat64Digits { + d = movePoint(d, -maxFloat64Digits) + return movePoint(d, p+maxFloat64Digits) + } + return movePoint(d, p) +} + +func movePoint(significand float64, exp int) float64 { + switch { + case exp < -maxFloat64Digits: + return 0 + case exp >= 0: + return significand * pow10[exp] + default: + return significand / pow10[-exp] + } +} + func parseArray(s string, c *cache, depth int) (*Value, string, error) { s = skipWS(s) if len(s) == 0 { @@ -530,14 +729,14 @@ func parseRawString(s string) (string, string, error) { } // readFloat reads a JSON number off the front of s, returning how much of s it -// covers, the exponent it was written with, and how many digits it carries -// after its decimal point. Together those say how far the number's digits sit -// from where a double keeps them. +// covers and the exponent it was written with. Whether the number is one a +// double can hold is mysqlNumberFits's job; the exponent is reported so that +// question only has to be asked of numbers whose digits could reach that far. // // What counts as a number is JSON's grammar rather than Go's: a written plus, // a missing digit on either side of the decimal point, and an integer part // that opens with a zero are all rejected. -func readFloat[S string | []byte](s S) (i, exponent, fraction int, ok bool) { +func readFloat[S string | []byte](s S) (i, exponent int, ok bool) { // optional minus. JSON numbers carry no written plus. if i >= len(s) { return @@ -564,7 +763,6 @@ func readFloat[S string | []byte](s S) (i, exponent, fraction int, ok bool) { return } for ; i < len(s) && '0' <= s[i] && s[i] <= '9'; i++ { - fraction++ } } @@ -598,7 +796,7 @@ func readFloat[S string | []byte](s S) (i, exponent, fraction int, ok bool) { exponent = -exponent } } - return i, exponent, fraction, true + return i, exponent, true } // Object represents JSON object. diff --git a/go/mysql/json/parser_test.go b/go/mysql/json/parser_test.go index 9a4e1c2d91b..2913231cf2e 100644 --- a/go/mysql/json/parser_test.go +++ b/go/mysql/json/parser_test.go @@ -31,7 +31,7 @@ func TestParseRawNumber(t *testing.T) { f := func(s, expectedRN, expectedTail string) { t.Helper() - flen, _, _, ok := readFloat(s) + flen, _, ok := readFloat(s) require.Truef(t, ok, "unexpected error when parsing '%s'", s) rn, tail := s[:flen], s[flen:] @@ -57,7 +57,7 @@ func TestParseRawNumber(t *testing.T) { f := func(s, expectedTail string) { t.Helper() - flen, _, _, ok := readFloat(s) + flen, _, ok := readFloat(s) require.False(t, ok, "expecting non-nil error") require.Equalf(t, expectedTail, s[flen:], "unexpected tail; got %q; want %q", s[flen:], expectedTail) } @@ -161,6 +161,42 @@ func TestParseNumberTooBigForDouble(t *testing.T) { }) } }) + + // Right at the top of the range the answer turns on how a number was + // written rather than on what it is worth. The digits are split between a + // significand and a power of ten to scale it by, and where that split falls + // decides which way the last place rounds — so writing the same value to one + // more digit moves the split and can move the answer with it. + t.Run("spelling at the largest double", func(t *testing.T) { + for _, tc := range []struct { + doc string + fits bool + }{ + {"1.7976931348623157e308", true}, + {"1.7976931348623158e308", false}, + {"1.79769313486231580e308", true}, + {"1.797693134862315800e308", true}, + {"1.79769313486231581e308", true}, + {"1.79769313486231585e308", true}, + {"1.7976931348623159e308", false}, + {"1.7976931348623157081e308", true}, + {"17976931348623157e292", true}, + {"17976931348623158e292", false}, + {"179769313486231580e291", true}, + {"1797693134862315800e290", false}, + {"17976931348623158000000e286", false}, + } { + t.Run(tc.doc, func(t *testing.T) { + var p Parser + _, err := p.Parse(tc.doc) + if tc.fits { + require.NoError(t, err) + } else { + require.ErrorContains(t, err, "number too big to be stored in double") + } + }) + } + }) } // TestParseNumberGrammar covers the shapes JSON's grammar allows a number to From 139a987a147504361dac7c43cf378f466aab32d7 Mon Sep 17 00:00:00 2001 From: Arthur Schreiber Date: Tue, 28 Jul 2026 09:32:03 +0000 Subject: [PATCH 07/19] mysql/json: keep each significand digit to two roundings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The conversion accumulates the significand into a double one digit at a time, multiplying and adding. MySQL's builds round the two operations separately, but written as d*10 + digit the Go compiler is free to fuse them into one FMA on arm64, which rounds once and can land the accumulation an ULP from where MySQL puts it — enough to flip which side of the largest double a number falls on. MySQL 8.0.45, 8.4.11 and 9.4.0 all accept 17976931348623154547712857878e280; on arm64 the fused loop rejected it. The explicit float64 conversion forces the intermediate rounding that the spec otherwise lets the compiler discard. Co-Authored-By: Claude Fable 5 Signed-off-by: Arthur Schreiber --- go/mysql/json/parser.go | 10 ++++++++-- go/mysql/json/parser_test.go | 22 ++++++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/go/mysql/json/parser.go b/go/mysql/json/parser.go index 9d78e9f8efd..03bccec3f1c 100644 --- a/go/mysql/json/parser.go +++ b/go/mysql/json/parser.go @@ -252,6 +252,12 @@ func init() { // same value as 1.79769313486231580e308 does, because the extra digit moves // where the significand ends and the scaling begins. Converting the same way is // what makes a document valid here exactly when it is valid in MySQL. +// +// The float64 conversions in the digit loops keep the multiply and the add as +// two roundings. Without them the compiler is free to fuse both into one FMA +// on arm64, which rounds once and can land the accumulation one ULP away from +// where MySQL's builds put it — enough to flip which side of the largest +// double a number falls on. func mysqlNumberFits(num string) bool { i := 0 minus := num[i] == '-' @@ -310,7 +316,7 @@ func mysqlNumberFits(num string) bool { } } for useDouble && digit() { - d = d*10 + float64(num[i]-'0') + d = float64(d*10) + float64(num[i]-'0') i++ } @@ -340,7 +346,7 @@ func mysqlNumberFits(num string) bool { } for digit() { if sigDigits < 17 { - d = d*10 + float64(num[i]-'0') + d = float64(d*10) + float64(num[i]-'0') expFrac-- if d > 0 { sigDigits++ diff --git a/go/mysql/json/parser_test.go b/go/mysql/json/parser_test.go index 2913231cf2e..68671fbb747 100644 --- a/go/mysql/json/parser_test.go +++ b/go/mysql/json/parser_test.go @@ -162,6 +162,28 @@ func TestParseNumberTooBigForDouble(t *testing.T) { } }) + // The significand accumulates one digit at a time, and each step rounds + // the multiplication and the addition separately, the way MySQL's builds + // run the loop. Fusing the two into one rounding — which the Go compiler + // may do on arm64 unless the conversion in mysqlNumberFits stops it — + // moves the accumulation an ULP for these documents, and that is enough + // to push them over the largest double. MySQL 8.0.45, 8.4.11 and 9.4.0 + // accept all of them. + t.Run("each accumulation step rounds on its own", func(t *testing.T) { + for _, doc := range []string{ + "17976931348623154547712857878e280", + "179769313486231559524062337652e279", + "179769313486231577704643761e282", + "1797693134862315724800793889e281", + } { + t.Run(startEndString(doc), func(t *testing.T) { + var p Parser + _, err := p.Parse(doc) + require.NoError(t, err) + }) + } + }) + // Right at the top of the range the answer turns on how a number was // written rather than on what it is worth. The digits are split between a // significand and a power of ten to scale it by, and where that split falls From 16c9c6903fa06218e96f47447630d5700cd060f5 Mon Sep 17 00:00:00 2001 From: Arthur Schreiber Date: Tue, 28 Jul 2026 11:17:09 +0000 Subject: [PATCH 08/19] mysql/json: count only an exponent that moves the point right MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Whether a number was worth converting came from its digits and its exponent added together, which let a negative exponent pay for digits. But the digits are read into the significand before the exponent is applied, and a significand that ran out of room on the way in has already gone infinite — moving the decimal point afterwards leaves it there. So 1 followed by 400 zeros written with an e-400 is worth one, was never checked, and MySQL calls that document invalid. The bound now counts only an exponent that moves the point right, which can only send more numbers to the check and so gives up nothing it already caught. The conversion itself was right: across 5566 documents — the boundary walks, digit counts from 305 to 312, and 5000 randomly spelled numbers carrying up to 340 digits on either side of the point with exponents out to ±700 — this and MySQL 8.0.46 now agree everywhere, where before they disagreed on 130 of them. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Arthur Schreiber --- go/mysql/json/parser.go | 8 +++++++- go/mysql/json/parser_test.go | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/go/mysql/json/parser.go b/go/mysql/json/parser.go index 03bccec3f1c..a2b67fcbf61 100644 --- a/go/mysql/json/parser.go +++ b/go/mysql/json/parser.go @@ -217,8 +217,14 @@ const maxFloat64Digits = 308 // largest double whenever the digits it is written to, moved by its exponent, // stay within the ones that double has — len(num) overcounts the digits, which // only makes the answer yes more often. +// +// Only an exponent that moves the decimal point to the right counts against +// those digits. A negative one buys nothing back: the digits are read into the +// significand before the exponent is applied, so a significand that ran out of +// room on the way in has already gone infinite, and moving the point afterwards +// leaves it there. func mayExceedFloat64(num string, exponent int) bool { - return len(num)+exponent > maxFloat64Digits + return len(num)+max(exponent, 0) > maxFloat64Digits } // pow10 holds the powers of ten that a number's digits are scaled by, which is diff --git a/go/mysql/json/parser_test.go b/go/mysql/json/parser_test.go index 68671fbb747..40414cb9424 100644 --- a/go/mysql/json/parser_test.go +++ b/go/mysql/json/parser_test.go @@ -98,6 +98,12 @@ func TestParseNumberTooBigForDouble(t *testing.T) { "1e-1000", "1e-1024", "0." + strings.Repeat("0", 400) + "1", + // Digits a double has room for, moved out of the way by a negative + // exponent. + "1" + strings.Repeat("0", 307) + "e-1", + "1" + strings.Repeat("0", 307) + "e-400", + "-1" + strings.Repeat("0", 307) + "e-400", + "0." + strings.Repeat("0", 400) + "1e-400", // A written sign and a padded exponent are spellings, not // magnitudes, and none of these is anywhere near the limit. "1e+0", @@ -149,6 +155,15 @@ func TestParseNumberTooBigForDouble(t *testing.T) { // Within the written bound, but too big once converted. "10e308", "1" + strings.Repeat("0", 30) + "e279", + // More digits than a double has room for. The digits are read before + // the exponent is applied, so a negative exponent does not buy the + // room back however far it moves the decimal point afterwards. + "1" + strings.Repeat("0", 320) + "e-20", + "1" + strings.Repeat("0", 350) + "e-50", + "1" + strings.Repeat("0", 400) + "e-400", + "-1" + strings.Repeat("0", 400) + "e-400", + "1" + strings.Repeat("0", 400) + ".5e-400", + strings.Repeat("9", 400) + "e-100", // A number anywhere in the document invalidates all of it. "[1, 1e309]", `{"a": 1e309}`, From 0c64ad5f4bafa9ccf04866a03d069a7c4fabe0c9 Mon Sep 17 00:00:00 2001 From: Arthur Schreiber Date: Tue, 28 Jul 2026 11:28:53 +0000 Subject: [PATCH 09/19] mysql/json: say what readFloat's exponent is, and what it is not MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The exponent readFloat reports stops at a ceiling instead of being read out to the end: once it is further than any double reaches, the digits after it say nothing more. The comment promised the exponent the number was written with, which is a finer thing than what comes back and not something a caller should lean on — mysqlNumberFits reads the real one off the number again. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Arthur Schreiber --- go/mysql/json/parser.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/go/mysql/json/parser.go b/go/mysql/json/parser.go index a2b67fcbf61..9eaab0aa0c6 100644 --- a/go/mysql/json/parser.go +++ b/go/mysql/json/parser.go @@ -741,9 +741,16 @@ func parseRawString(s string) (string, string, error) { } // readFloat reads a JSON number off the front of s, returning how much of s it -// covers and the exponent it was written with. Whether the number is one a -// double can hold is mysqlNumberFits's job; the exponent is reported so that -// question only has to be asked of numbers whose digits could reach that far. +// covers and how far its exponent moves the decimal point. Whether the number +// is one a double can hold is mysqlNumberFits's job; the exponent is reported so +// that question only has to be asked of numbers whose digits could reach that +// far. +// +// That distance is bounded rather than exact. An exponent can be written to more +// digits than it takes to leave every double behind, and one that reaches +// exponentCeiling below is left there instead of read out to the end. So it +// answers how far is far enough to matter and nothing finer; mysqlNumberFits +// reads the exponent itself off the number again. // // What counts as a number is JSON's grammar rather than Go's: a written plus, // a missing digit on either side of the decimal point, and an integer part From 879d156933fd84ea117320b34d3dbc16c18d2498 Mon Sep 17 00:00:00 2001 From: Arthur Schreiber Date: Tue, 28 Jul 2026 11:33:58 +0000 Subject: [PATCH 10/19] mysql/json: read the power-of-ten table with strconv Reading numbers with fastparse is worth it on the parse path. This table is built once at startup, so there is nothing to gain there, and reading it with the standard library means its entries come from the same conversion they would otherwise have to be checked against. All 309 come out bit for bit where they were, so the boundary does not move. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Arthur Schreiber --- go/mysql/json/parser.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/go/mysql/json/parser.go b/go/mysql/json/parser.go index 9eaab0aa0c6..206c0ab1b69 100644 --- a/go/mysql/json/parser.go +++ b/go/mysql/json/parser.go @@ -228,14 +228,17 @@ func mayExceedFloat64(num string, exponent int) bool { } // pow10 holds the powers of ten that a number's digits are scaled by, which is -// the table RapidJSON scales its significand with. It is built from the decimal -// spellings rather than from math.Pow10, which multiplies two table entries -// together and lands a bit away from the spelling for 78 of these exponents. +// the table RapidJSON scales its significand with. RapidJSON writes its table +// out as decimal literals, so each entry here is read from the same spelling and +// lands on the same double. math.Pow10 would not: it multiplies two table entries +// together and comes out a bit away from the spelling for 78 of these exponents. var pow10 [maxFloat64Digits + 1]float64 +// The rest of the package reads numbers with fastparse, because that is a hot +// path. This table is built once at startup, so it uses strconv instead. func init() { for i := range pow10 { - pow10[i], _ = fastparse.ParseFloat64("1e" + strconv.Itoa(i)) + pow10[i], _ = strconv.ParseFloat("1e"+strconv.Itoa(i), 64) } } From 78c4bfa797a2a38584a66af7052f36e9f9fc4152 Mon Sep 17 00:00:00 2001 From: Arthur Schreiber Date: Tue, 28 Jul 2026 11:40:37 +0000 Subject: [PATCH 11/19] mysql/json: say which reader the number check follows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit mysqlNumberFits is shaped the way it is because MySQL parses JSON with RapidJSON and runs its number reader without kParseFullPrecisionFlag, which is what leaves MySQL on an approximate conversion — and so what this has to reproduce rather than improve on. That was the one fact the comment did not record, and it is the fact the whole approach rests on. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Arthur Schreiber --- go/mysql/json/parser.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/go/mysql/json/parser.go b/go/mysql/json/parser.go index 206c0ab1b69..44d22f461e0 100644 --- a/go/mysql/json/parser.go +++ b/go/mysql/json/parser.go @@ -1,6 +1,8 @@ /* Copyright 2018 Aliaksandr Valialkin Copyright 2023 The Vitess Authors. +Portions Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. +See LICENSE.rapidjson in this directory. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -245,8 +247,11 @@ func init() { // mysqlNumberFits reports whether MySQL can store num, which has to already be // a grammatically valid JSON number. // -// MySQL parses JSON with RapidJSON, which decides this in two places. While -// reading the exponent, a written exponent may not move the decimal point more +// MySQL parses JSON with RapidJSON, and runs its number reader without +// kParseFullPrecisionFlag — which is what leaves MySQL on the approximate +// conversion below rather than a correct one, and pins this boundary to that +// reader. It decides this in two places. While reading the exponent, a written +// exponent may not move the decimal point more // than maxFloat64Digits places past the digits already behind it. After // converting, the result may not be larger than the largest double. Neither // place catches what the other does: an exponent that lands on zero is refused From ed1fe05ca71795671905dddc68b31fe2a800a26f Mon Sep 17 00:00:00 2001 From: Arthur Schreiber Date: Tue, 28 Jul 2026 11:41:07 +0000 Subject: [PATCH 12/19] mysql/json: record RapidJSON's licence alongside the number check mysqlNumberFits, scaleByPow10, movePoint and the pow10 table were written to reproduce what RapidJSON's ParseNumber, StrtodNormalPrecision, FastPath and Pow10 decide, so that a document is valid here exactly when MySQL says it is. No RapidJSON source is copied, but reproducing one function's decisions step for step is close enough to it that the notice belongs here either way, and it costs nothing to carry. This follows what the package already does for fastjson, which it is derived from: the upstream notice in the directory, the copyright line in the header. The previous commit added that header line. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Arthur Schreiber --- go/mysql/json/LICENSE.rapidjson | 37 +++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 go/mysql/json/LICENSE.rapidjson diff --git a/go/mysql/json/LICENSE.rapidjson b/go/mysql/json/LICENSE.rapidjson new file mode 100644 index 00000000000..b1474db24d4 --- /dev/null +++ b/go/mysql/json/LICENSE.rapidjson @@ -0,0 +1,37 @@ +Tencent is pleased to support the open source community by making RapidJSON +available. + +Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. + +Parts of parser.go are derived from RapidJSON 1.1.0, the JSON parser MySQL +parses documents with, so that Vitess accepts exactly the documents MySQL does: +mysqlNumberFits follows GenericReader::ParseNumber in +include/rapidjson/reader.h, scaleByPow10 and movePoint follow +StrtodNormalPrecision and FastPath in include/rapidjson/internal/strtod.h, and +the pow10 table holds the same values as Pow10 in +include/rapidjson/internal/pow10.h. No RapidJSON source is included here; the Go +code was written to reproduce what those functions decide. + +RapidJSON is licensed under the MIT License: + +The MIT License (MIT) + +Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. From 2fe54bb4d7c07156bbae931bb280e004e972bcda Mon Sep 17 00:00:00 2001 From: Arthur Schreiber Date: Tue, 28 Jul 2026 11:57:31 +0000 Subject: [PATCH 13/19] mysql/json: benchmark the paths the number reader takes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The numbers this change has been quoting came from a harness that was never in the tree, so nobody reading them could run them or notice them moving. These are the same shapes, committed: three arrays of a thousand numbers written as integers, fractions and exponents, and a small mixed object. Three more cover the magnitude check, which none of the quoted numbers reached — an array where every element carries an exponent far enough out to be converted, a number written to four hundred fraction digits, and a document the check rejects. That is the path this change added, and it was the one with no measurement at all. Each case asserts up front whether the document parses, because one that stops early measures the error path instead of the one it was written for and reads as a speed-up while doing it. Two of these did, before that check went in. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Arthur Schreiber --- go/mysql/json/parser_bench_test.go | 83 ++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 go/mysql/json/parser_bench_test.go diff --git a/go/mysql/json/parser_bench_test.go b/go/mysql/json/parser_bench_test.go new file mode 100644 index 00000000000..08ac7023dbb --- /dev/null +++ b/go/mysql/json/parser_bench_test.go @@ -0,0 +1,83 @@ +/* +Copyright 2026 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package json + +import ( + "strconv" + "strings" + "testing" +) + +// numberArray builds a JSON array of n numbers, each spelled by digits. +func numberArray(n int, digits func(i int) string) string { + var sb strings.Builder + sb.WriteByte('[') + for i := range n { + if i > 0 { + sb.WriteByte(',') + } + sb.WriteString(digits(i)) + } + sb.WriteByte(']') + return sb.String() +} + +// benchDocs covers the number shapes the parser takes different paths through. +// +// The first four are the everyday ones, where the cost is the scan itself. The +// last three reach the magnitude check, which a number only pays for when its +// digits and its exponent together could carry it past the largest double: +// once with an exponent large enough to ask the question of every element, once +// where the digits alone are enough to ask it, and once for a document the +// answer rejects. +var benchDocs = []struct { + name string + doc string + rejected bool +}{ + {name: "int/1024", doc: numberArray(1024, func(i int) string { return strconv.Itoa(i * 7919) })}, + {name: "frac/1024", doc: numberArray(1024, func(i int) string { return strconv.Itoa(i) + "." + strconv.Itoa(i*7919) })}, + {name: "exp/1024", doc: numberArray(1024, func(i int) string { return strconv.Itoa(i) + "." + strconv.Itoa(i*7919) + "e" + strconv.Itoa(i%300) })}, + {name: "mixed-object", doc: `{"id":38141,"name":"a name","ok":true,"score":-12.5e3,"tags":["x","y"],"meta":null}`}, + + {name: "checked/1024", doc: numberArray(1024, func(i int) string { return "1." + strconv.Itoa(i) + "e30" + strconv.Itoa(i%8) })}, + {name: "checked-long-fraction", doc: "0." + strings.Repeat("0", 400) + "1e-400"}, + {name: "rejected", doc: strings.Repeat("9", 400) + "e-100", rejected: true}, +} + +func BenchmarkParse(b *testing.B) { + for _, tc := range benchDocs { + b.Run(tc.name, func(b *testing.B) { + var p Parser + + // A case that stops early measures the error path instead of the one + // it was written for, and reads as a speed-up while doing it. + if _, err := p.Parse(tc.doc); (err != nil) != tc.rejected { + b.Fatalf("document does not take the path this case measures: err=%v", err) + } + + b.ReportAllocs() + b.SetBytes(int64(len(tc.doc))) + for b.Loop() { + v, err := p.Parse(tc.doc) + if err == nil && v == nil { + b.Fatal("parsed to no value") + } + } + }) + } +} From e3010e5588fc8343ef0e66517f0f126a7a18a353 Mon Sep 17 00:00:00 2001 From: Arthur Schreiber Date: Tue, 28 Jul 2026 12:55:12 +0000 Subject: [PATCH 14/19] mysql/json: count the digits that can reach past a double, not the string MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Whether a number was worth converting came from the length of the whole thing, so its fraction digits, its e and its exponent's own digits all counted against the places a double has. Only the digits in front of the decimal point can carry a number that far. The ones behind it move the point the other way, they are what buys the written exponent its extra room, and they cannot overflow the significand on the way in because it stops taking them at seventeen. So an ordinary 1.5e303 was being converted to find out what its four digits and its exponent already said. Counting those digits means walking them, so the old over-count answers first and the walk only happens where it can change the answer — which keeps it off the path every short number takes. Measured on an idle machine, twenty runs a side: int/1024 8.398µs 8.658µs +3.1% (p=0.000, n=20) frac/1024 9.802µs 9.874µs ~ (p=0.213, n=20) exp/1024 12.53µs 12.06µs -3.7% (p=0.000, n=20) mixed-object 128.6n 127.5n -0.8% (p=0.004, n=20) checked/1024 18.72µs 11.32µs -39.5% (p=0.000, n=20) checked-long-fraction 564.3n 148.6n -73.7% (p=0.000, n=20) rejected 2.759µs 2.911µs +5.5% (p=0.000, n=20) The three percent on short integers is not the gate doing more work — it returns on the same first comparison — but the larger inlined body sitting in parseValue. Keeping the walk in a function of its own and out of line costs more, both there and on the numbers this is meant to help. Checked against a live MySQL 8.0.46 over 8627 documents with no mismatches, and against the check it is standing in front of: of the 5226 documents it answers without converting, every one is a number the conversion accepts. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Arthur Schreiber --- go/mysql/json/parser.go | 40 +++++++++++++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/go/mysql/json/parser.go b/go/mysql/json/parser.go index 44d22f461e0..12b71d30c0d 100644 --- a/go/mysql/json/parser.go +++ b/go/mysql/json/parser.go @@ -213,20 +213,42 @@ func parseValue(s string, c *cache, depth int) (*Value, string, error) { // 308 always leaves somewhere for the number to land and 309 need not. const maxFloat64Digits = 308 -// mayExceedFloat64 reports whether num is worth converting to find out whether -// a double can hold it. It errs towards yes: the job is to keep the conversion -// off the common path, not to answer the question. A number is below the -// largest double whenever the digits it is written to, moved by its exponent, -// stay within the ones that double has — len(num) overcounts the digits, which -// only makes the answer yes more often. +// mayExceedFloat64 reports whether num is worth converting to find out whether a +// double can hold it. It errs towards yes: the job is to keep the conversion off +// the common path, not to answer the question. // -// Only an exponent that moves the decimal point to the right counts against -// those digits. A negative one buys nothing back: the digits are read into the +// What can carry a number that far is the digits in front of its decimal point, +// moved by its exponent — it stays below 10^309 whenever those two together stay +// inside the places a double has, whatever it goes on to say after the point. +// Digits behind the point only ever move it the other way, and they are what buys +// the written exponent its extra room; nor can they overflow the significand on +// the way in, since it stops taking them at seventeen. +// +// Only an exponent that moves the point to the right counts against those digits. +// A negative one buys nothing back: the digits in front are read into the // significand before the exponent is applied, so a significand that ran out of // room on the way in has already gone infinite, and moving the point afterwards // leaves it there. +// +// Counting those digits means walking them, so len(num) answers first. It counts +// the fraction and the exponent's own characters too, and so can only say yes too +// often — which leaves the walk where the conversion it saves is: off the path +// every ordinary number takes. func mayExceedFloat64(num string, exponent int) bool { - return len(num)+max(exponent, 0) > maxFloat64Digits + room := maxFloat64Digits - max(exponent, 0) + if len(num) <= room { + return false + } + + i := 0 + if num[0] == '-' { + i++ + } + digits := 0 + for i+digits < len(num) && '0' <= num[i+digits] && num[i+digits] <= '9' { + digits++ + } + return digits > room } // pow10 holds the powers of ten that a number's digits are scaled by, which is From 97552c6f709caade6d2477adce81ab8f6419f96b Mon Sep 17 00:00:00 2001 From: Arthur Schreiber Date: Tue, 28 Jul 2026 15:42:44 +0000 Subject: [PATCH 15/19] mysql/json: cover a negative exponent written past what an int holds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The exponent accumulator is stopped before it overflows the int it reads into, and nothing reached that stop. Past it the exponent lands wherever the overflow leaves it, which can be a power of ten the scaling table does not go up to. Reaching the stop takes more than a long exponent. Counting the digits that can carry a number past a double, rather than the whole written string, means a short number with a long exponent no longer goes through the conversion at all — so these cross an exponent written past what an int holds with a digit string long enough to be converted in the first place. Each stays valid and reads as zero, as it does in MySQL 8.0.46. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Arthur Schreiber --- go/mysql/json/parser_test.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/go/mysql/json/parser_test.go b/go/mysql/json/parser_test.go index 40414cb9424..b09cf76e990 100644 --- a/go/mysql/json/parser_test.go +++ b/go/mysql/json/parser_test.go @@ -177,6 +177,32 @@ func TestParseNumberTooBigForDouble(t *testing.T) { } }) + // A negative exponent is not bounded, only stopped before it overflows the + // int it accumulates into, and what sends a number through the conversion at + // all is being written to more digits than a double holds. These cross the + // two, so the exponent is read into an int that cannot hold it and then + // scales a significand: written past that stop, it lands wherever the + // overflow leaves it, which can be a power of ten the table does not go up + // to. Each of these stays valid and reads as zero. MySQL 8.0.46 accepts all + // three and reads them as zero too. + t.Run("a negative exponent written past what an int holds", func(t *testing.T) { + for _, doc := range []string{ + strings.Repeat("9", 400) + "e-" + strings.Repeat("2", 306), + "-" + strings.Repeat("9", 400) + "e-" + strings.Repeat("2", 306), + strings.Repeat("1", 400) + "." + strings.Repeat("5", 20) + "e-" + strings.Repeat("2", 306), + } { + t.Run(startEndString(doc), func(t *testing.T) { + var p Parser + v, err := p.Parse(doc) + require.NoError(t, err) + + f, ok := v.Float64() + require.True(t, ok) + require.Zero(t, f) + }) + } + }) + // The significand accumulates one digit at a time, and each step rounds // the multiplication and the addition separately, the way MySQL's builds // run the loop. Fusing the two into one rounding — which the Go compiler From a3fb097a3c550ee806611a9438b9e47fe25e6f3b Mon Sep 17 00:00:00 2001 From: Arthur Schreiber Date: Tue, 28 Jul 2026 16:34:51 +0000 Subject: [PATCH 16/19] mysql/json: abbreviate the number an error reports on Nothing bounds how long a number may be written, and Parse copies the message it wraps, so naming the number in full carried the document into the error twice over. It goes through startEndString instead, as the unparsed tail alongside it already did. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Arthur Schreiber --- go/mysql/json/parser.go | 2 +- go/mysql/json/parser_test.go | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/go/mysql/json/parser.go b/go/mysql/json/parser.go index 12b71d30c0d..96b027fdf5d 100644 --- a/go/mysql/json/parser.go +++ b/go/mysql/json/parser.go @@ -203,7 +203,7 @@ func parseValue(s string, c *cache, depth int) (*Value, string, error) { v.s = s[:flen] v.n = numberTypeRaw if mayExceedFloat64(v.s, exponent) && !mysqlNumberFits(v.s) { - return nil, s, fmt.Errorf("number too big to be stored in double: %q", v.s) + return nil, s, fmt.Errorf("number too big to be stored in double: %q", startEndString(v.s)) } return v, s[flen:], nil } diff --git a/go/mysql/json/parser_test.go b/go/mysql/json/parser_test.go index b09cf76e990..8f020af2ac8 100644 --- a/go/mysql/json/parser_test.go +++ b/go/mysql/json/parser_test.go @@ -177,6 +177,20 @@ func TestParseNumberTooBigForDouble(t *testing.T) { } }) + // Nothing bounds how long a number may be written, and Parse copies the + // message it wraps, so naming the number in full would carry the document + // twice over into the error a client is handed. It is abbreviated instead, + // as the unparsed tail already is. + t.Run("the error abbreviates a long number", func(t *testing.T) { + doc := "1" + strings.Repeat("0", 100000) + + var p Parser + _, err := p.Parse(doc) + require.ErrorContains(t, err, "number too big to be stored in double") + require.NotContains(t, err.Error(), strings.Repeat("0", 200), + "the error carries the number it is reporting on") + }) + // A negative exponent is not bounded, only stopped before it overflows the // int it accumulates into, and what sends a number through the conversion at // all is being written to more digits than a double holds. These cross the From 94366cc04823e7be1bc6dc496456aca58c33a924 Mon Sep 17 00:00:00 2001 From: Arthur Schreiber Date: Tue, 28 Jul 2026 17:08:14 +0000 Subject: [PATCH 17/19] mysql/json: abbreviate the document every rejection names Reading numbers by JSON's grammar turned documents that used to parse into rejections, and deleting the nan branch turned another one into a token error. Both name the text they refuse in full, and Parse copies the message it wraps, so a document that arrives megabytes long comes back as its own error twice over. The magnitude check was abbreviated already; the three remaining sites follow it. One test now covers every shape that reaches them, in place of the single case that covered the magnitude error alone. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Arthur Schreiber --- go/mysql/json/parser.go | 8 +++---- go/mysql/json/parser_test.go | 43 ++++++++++++++++++++++++------------ 2 files changed, 33 insertions(+), 18 deletions(-) diff --git a/go/mysql/json/parser.go b/go/mysql/json/parser.go index 96b027fdf5d..c379c196cd9 100644 --- a/go/mysql/json/parser.go +++ b/go/mysql/json/parser.go @@ -176,26 +176,26 @@ func parseValue(s string, c *cache, depth int) (*Value, string, error) { } if s[0] == 't' { if len(s) < len("true") || s[:len("true")] != "true" { - return nil, s, fmt.Errorf("unexpected value found: %q", s) + return nil, s, fmt.Errorf("unexpected value found: %q", startEndString(s)) } return ValueTrue, s[len("true"):], nil } if s[0] == 'f' { if len(s) < len("false") || s[:len("false")] != "false" { - return nil, s, fmt.Errorf("unexpected value found: %q", s) + return nil, s, fmt.Errorf("unexpected value found: %q", startEndString(s)) } return ValueFalse, s[len("false"):], nil } if s[0] == 'n' { if len(s) < len("null") || s[:len("null")] != "null" { - return nil, s, fmt.Errorf("unexpected value found: %q", s) + return nil, s, fmt.Errorf("unexpected value found: %q", startEndString(s)) } return ValueNull, s[len("null"):], nil } flen, exponent, ok := readFloat(s) if !ok { - return nil, s[flen:], fmt.Errorf("invalid number in JSON string: %q", s) + return nil, s[flen:], fmt.Errorf("invalid number in JSON string: %q", startEndString(s)) } v := c.getValue() diff --git a/go/mysql/json/parser_test.go b/go/mysql/json/parser_test.go index 8f020af2ac8..cb144d5cc1f 100644 --- a/go/mysql/json/parser_test.go +++ b/go/mysql/json/parser_test.go @@ -177,20 +177,6 @@ func TestParseNumberTooBigForDouble(t *testing.T) { } }) - // Nothing bounds how long a number may be written, and Parse copies the - // message it wraps, so naming the number in full would carry the document - // twice over into the error a client is handed. It is abbreviated instead, - // as the unparsed tail already is. - t.Run("the error abbreviates a long number", func(t *testing.T) { - doc := "1" + strings.Repeat("0", 100000) - - var p Parser - _, err := p.Parse(doc) - require.ErrorContains(t, err, "number too big to be stored in double") - require.NotContains(t, err.Error(), strings.Repeat("0", 200), - "the error carries the number it is reporting on") - }) - // A negative exponent is not bounded, only stopped before it overflows the // int it accumulates into, and what sends a number through the conversion at // all is being written to more digits than a double holds. These cross the @@ -316,6 +302,35 @@ func TestParseNumberGrammar(t *testing.T) { }) } +// TestParseErrorAbbreviatesTheDocument covers how much of a rejected document +// its error names. Nothing bounds how long a document may be, and Parse copies +// the message it wraps, so naming the text in full hands a client its own +// document back twice over. Each rejection abbreviates what it names, as the +// unparsed tail alongside it already did. +func TestParseErrorAbbreviatesTheDocument(t *testing.T) { + long := strings.Repeat("9", 100000) + + for _, tc := range []struct { + name string + doc string + }{ + {name: "a number too big for a double", doc: "1" + long}, + {name: "a written plus", doc: "+" + long}, + {name: "a decimal point with nothing before it", doc: "." + long}, + {name: "a decimal point with nothing after it", doc: long + "."}, + {name: "nan", doc: "nan" + long}, + {name: "nothing the grammar has a shape for", doc: "q" + long}, + } { + t.Run(tc.name, func(t *testing.T) { + var p Parser + _, err := p.Parse(tc.doc) + require.Error(t, err) + require.NotContains(t, err.Error(), strings.Repeat("9", 200), + "the error carries the document it is reporting on") + }) + } +} + func TestUnescapeStringBestEffort(t *testing.T) { t.Run("success", func(t *testing.T) { testUnescapeStringBestEffort(t, ``, ``) From 97ea69c475acb402f3b99ac948a2ea91fc072c23 Mon Sep 17 00:00:00 2001 From: Arthur Schreiber Date: Tue, 28 Jul 2026 17:08:28 +0000 Subject: [PATCH 18/19] mysql/json: benchmark documents that reach the magnitude check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit None of the accepted checked cases reached it. Counting the digits that can carry a number past a double, rather than the whole written string, left a number with one integer digit and a large exponent answering the question from its digits alone — so cases built out of those went on measuring the scan they share with every other case, and read as though the check were free. The two accepted cases are now written to more digits than their exponents leave a double room for, and reachesMagnitudeCheck holds every case to what its name claims, so a document that stops short of the check fails the benchmark rather than quietly flattering it. Reaching the conversion costs what it costs: against the merge base checked/1024 goes from 21.72µs to 41.13µs and checked-long-fraction from 475.5n to 949.9n, where the same names reported a speed-up before. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Arthur Schreiber --- go/mysql/json/parser_bench_test.go | 47 +++++++++++++++++++++++++----- 1 file changed, 39 insertions(+), 8 deletions(-) diff --git a/go/mysql/json/parser_bench_test.go b/go/mysql/json/parser_bench_test.go index 08ac7023dbb..8139b2694a4 100644 --- a/go/mysql/json/parser_bench_test.go +++ b/go/mysql/json/parser_bench_test.go @@ -39,14 +39,17 @@ func numberArray(n int, digits func(i int) string) string { // benchDocs covers the number shapes the parser takes different paths through. // // The first four are the everyday ones, where the cost is the scan itself. The -// last three reach the magnitude check, which a number only pays for when its -// digits and its exponent together could carry it past the largest double: -// once with an exponent large enough to ask the question of every element, once -// where the digits alone are enough to ask it, and once for a document the -// answer rejects. +// last three reach the magnitude check, which a number only pays for once it is +// written to more digits than its exponent leaves a double room for: once with +// an exponent that shrinks the room to ask the question of every element, once +// where the digits alone are past what a double has, and once for a document the +// answer rejects. reachesMagnitudeCheck holds them to that, since a case that +// falls short of the check goes on measuring the scan and reads as though the +// check were free. var benchDocs = []struct { name string doc string + checked bool rejected bool }{ {name: "int/1024", doc: numberArray(1024, func(i int) string { return strconv.Itoa(i * 7919) })}, @@ -54,9 +57,34 @@ var benchDocs = []struct { {name: "exp/1024", doc: numberArray(1024, func(i int) string { return strconv.Itoa(i) + "." + strconv.Itoa(i*7919) + "e" + strconv.Itoa(i%300) })}, {name: "mixed-object", doc: `{"id":38141,"name":"a name","ok":true,"score":-12.5e3,"tags":["x","y"],"meta":null}`}, - {name: "checked/1024", doc: numberArray(1024, func(i int) string { return "1." + strconv.Itoa(i) + "e30" + strconv.Itoa(i%8) })}, - {name: "checked-long-fraction", doc: "0." + strings.Repeat("0", 400) + "1e-400"}, - {name: "rejected", doc: strings.Repeat("9", 400) + "e-100", rejected: true}, + {name: "checked/1024", doc: numberArray(1024, func(i int) string { return "1" + strconv.Itoa(1000000000000000000+i) + "e289" }), checked: true}, + {name: "checked-long-fraction", doc: "1" + strings.Repeat("0", 308) + "." + strings.Repeat("5", 400), checked: true}, + {name: "rejected", doc: strings.Repeat("9", 400) + "e-100", checked: true, rejected: true}, +} + +// reachesMagnitudeCheck reports whether every number in doc is converted to find +// out whether a double can hold it, which is what the checked cases are for. A +// number written to fewer digits than a double has room for answers that question +// from its digits alone, and a case built out of those measures the scan it shares +// with every other case instead. +func reachesMagnitudeCheck(doc string) bool { + numbers := 0 + for i := 0; i < len(doc); { + if c := doc[i]; c != '-' && (c < '0' || c > '9') { + i++ + continue + } + flen, exponent, ok := readFloat(doc[i:]) + if !ok { + return false + } + if !mayExceedFloat64(doc[i:i+flen], exponent) { + return false + } + numbers++ + i += flen + } + return numbers > 0 } func BenchmarkParse(b *testing.B) { @@ -69,6 +97,9 @@ func BenchmarkParse(b *testing.B) { if _, err := p.Parse(tc.doc); (err != nil) != tc.rejected { b.Fatalf("document does not take the path this case measures: err=%v", err) } + if reachesMagnitudeCheck(tc.doc) != tc.checked { + b.Fatalf("document reaches the magnitude check: %v, want %v", !tc.checked, tc.checked) + } b.ReportAllocs() b.SetBytes(int64(len(tc.doc))) From 0886b6fccc9a8139c1178e4ef39cd07d20d4342d Mon Sep 17 00:00:00 2001 From: Arthur Schreiber Date: Tue, 28 Jul 2026 18:07:22 +0000 Subject: [PATCH 19/19] mysql/json: pin the tightened grammar on the SQL-marshal path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AppendMarshalSQL scans documents with the same readFloat the parser uses, so the shapes it stopped accepting are rejected here too. Two of them reach the reader — 007 now stops after the lone zero and fails the trailing-data check, and 12. no longer reads at all — while +1 and .2 never get past writeValue's dispatch, before and after alike. All four are pinned so the two callers' expectations stay together. Co-Authored-By: Claude Fable 5 Signed-off-by: Arthur Schreiber --- go/mysql/json/marshal_test.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/go/mysql/json/marshal_test.go b/go/mysql/json/marshal_test.go index 2d278fc02be..90c796db469 100644 --- a/go/mysql/json/marshal_test.go +++ b/go/mysql/json/marshal_test.go @@ -128,6 +128,13 @@ func TestAppendMarshalSQLNumberGrammar(t *testing.T) { `1e+`, `1e`, `--1`, + // Shapes JSON's grammar does not have: a written plus, an integer + // part opening with a zero, and a decimal point with nothing on + // one side of it. + `+1`, + `007`, + `.2`, + `12.`, } for _, input := range malformed { buf := &bytes2.Buffer{}