diff --git a/go/mysql/json/LICENSE.rapidjson b/go/mysql/json/LICENSE.rapidjson new file mode 100644 index 00000000000..b1474db24d4 --- /dev/null +++ b/go/mysql/json/LICENSE.rapidjson @@ -0,0 +1,37 @@ +Tencent is pleased to support the open source community by making RapidJSON +available. + +Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. + +Parts of parser.go are derived from RapidJSON 1.1.0, the JSON parser MySQL +parses documents with, so that Vitess accepts exactly the documents MySQL does: +mysqlNumberFits follows GenericReader::ParseNumber in +include/rapidjson/reader.h, scaleByPow10 and movePoint follow +StrtodNormalPrecision and FastPath in include/rapidjson/internal/strtod.h, and +the pow10 table holds the same values as Pow10 in +include/rapidjson/internal/pow10.h. No RapidJSON source is included here; the Go +code was written to reproduce what those functions decide. + +RapidJSON is licensed under the MIT License: + +The MIT License (MIT) + +Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/go/mysql/json/parser.go b/go/mysql/json/parser.go index 4f25e6be16e..bda3673d1e5 100644 --- a/go/mysql/json/parser.go +++ b/go/mysql/json/parser.go @@ -1,6 +1,8 @@ /* Copyright 2018 Aliaksandr Valialkin Copyright 2023 The Vitess Authors. +Portions Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. +See LICENSE.rapidjson in this directory. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -21,6 +23,7 @@ import ( "bytes" "encoding/base64" "fmt" + "math" "slices" "strconv" "strings" @@ -172,42 +175,296 @@ func parseValue(s string, c *cache, depth int) (*Value, string, error) { } if s[0] == 't' { if len(s) < len("true") || s[:len("true")] != "true" { - return nil, s, fmt.Errorf("unexpected value found: %q", s) + return nil, s, fmt.Errorf("unexpected value found: %q", startEndString(s)) } return ValueTrue, s[len("true"):], nil } if s[0] == 'f' { if len(s) < len("false") || s[:len("false")] != "false" { - return nil, s, fmt.Errorf("unexpected value found: %q", s) + return nil, s, fmt.Errorf("unexpected value found: %q", startEndString(s)) } return ValueFalse, s[len("false"):], nil } if s[0] == 'n' { if len(s) < len("null") || s[:len("null")] != "null" { - // Try parsing NaN - if len(s) >= 3 && strings.EqualFold(s[:3], "nan") { - v := c.getValue() - v.t = TypeNumber - v.s = s[:3] - return v, s[3:], nil - } - return nil, s, fmt.Errorf("unexpected value found: %q", s) + return nil, s, fmt.Errorf("unexpected value found: %q", startEndString(s)) } return ValueNull, s[len("null"):], nil } - flen, ok := readFloat(s) + flen, exponent, ok := readFloat(s) if !ok { - return nil, s[flen:], fmt.Errorf("invalid number in JSON string: %q", s) + return nil, s[flen:], fmt.Errorf("invalid number in JSON string: %q", startEndString(s)) } v := c.getValue() v.t = TypeNumber v.s = s[:flen] v.n = numberTypeRaw + if mayExceedFloat64(v.s, exponent) && !mysqlNumberFits(v.s) { + return nil, s, fmt.Errorf("number too big to be stored in double: %q", startEndString(v.s)) + } return v, s[flen:], nil } +// maxFloat64Digits is how far a decimal point can travel before a double runs +// out of room. The largest double is under 1.8e308, so a written exponent of +// 308 always leaves somewhere for the number to land and 309 need not. +const maxFloat64Digits = 308 + +// mayExceedFloat64 reports whether num is worth converting to find out whether a +// double can hold it. It errs towards yes: the job is to keep the conversion off +// the common path, not to answer the question. +// +// What can carry a number that far is the digits in front of its decimal point, +// moved by its exponent — it stays below 10^309 whenever those two together stay +// inside the places a double has, whatever it goes on to say after the point. +// Digits behind the point only ever move it the other way, and they are what buys +// the written exponent its extra room; nor can they overflow the significand on +// the way in, since it stops taking them at seventeen. +// +// Only an exponent that moves the point to the right counts against those digits. +// A negative one buys nothing back: the digits in front are read into the +// significand before the exponent is applied, so a significand that ran out of +// room on the way in has already gone infinite, and moving the point afterwards +// leaves it there. +// +// Counting those digits means walking them, so len(num) answers first. It counts +// the fraction and the exponent's own characters too, and so can only say yes too +// often — which leaves the walk where the conversion it saves is: off the path +// every ordinary number takes. +func mayExceedFloat64(num string, exponent int) bool { + room := maxFloat64Digits - max(exponent, 0) + if len(num) <= room { + return false + } + + i := 0 + if num[0] == '-' { + i++ + } + digits := 0 + for i+digits < len(num) && '0' <= num[i+digits] && num[i+digits] <= '9' { + digits++ + } + return digits > room +} + +// pow10 holds the powers of ten that a number's digits are scaled by, which is +// the table RapidJSON scales its significand with. RapidJSON writes its table +// out as decimal literals, so each entry here is read from the same spelling and +// lands on the same double. math.Pow10 would not: it multiplies two table entries +// together and comes out a bit away from the spelling for 78 of these exponents. +var pow10 [maxFloat64Digits + 1]float64 + +// The rest of the package reads numbers with fastparse, because that is a hot +// path. This table is built once at startup, so it uses strconv instead. +func init() { + for i := range pow10 { + pow10[i], _ = strconv.ParseFloat("1e"+strconv.Itoa(i), 64) + } +} + +// mysqlNumberFits reports whether MySQL can store num, which has to already be +// a grammatically valid JSON number. +// +// MySQL parses JSON with RapidJSON, and runs its number reader without +// kParseFullPrecisionFlag — which is what leaves MySQL on the approximate +// conversion below rather than a correct one, and pins this boundary to that +// reader. It decides this in two places. While reading the exponent, a written +// exponent may not move the decimal point more +// than maxFloat64Digits places past the digits already behind it. After +// converting, the result may not be larger than the largest double. Neither +// place catches what the other does: an exponent that lands on zero is refused +// only as written, and a number written within the bound can still overflow +// once converted. +// +// The conversion below is the one RapidJSON does rather than a correct one. It +// accumulates the significand into a double and scales that by a power of ten, +// which lands an ULP or two from the true value, and the boundary sits wherever +// that lands. A number a hair under the largest double therefore comes down to +// how it was spelled: 1.7976931348623158e308 does not fit, while writing the +// same value as 1.79769313486231580e308 does, because the extra digit moves +// where the significand ends and the scaling begins. Converting the same way is +// what makes a document valid here exactly when it is valid in MySQL. +// +// The float64 conversions in the digit loops keep the multiply and the add as +// two roundings. Without them the compiler is free to fuse both into one FMA +// on arm64, which rounds once and can land the accumulation one ULP away from +// where MySQL's builds put it — enough to flip which side of the largest +// double a number falls on. +func mysqlNumberFits(num string) bool { + i := 0 + minus := num[i] == '-' + if minus { + i++ + } + + var ( + u32 uint32 + u64 uint64 + use64 bool + sigDigits int + d float64 + useDouble bool + ) + digit := func() bool { return i < len(num) && num[i] >= '0' && num[i] <= '9' } + + // The integer part accumulates as an integer for as long as one can hold + // it, stepping up in width as it fills. RapidJSON leaves the leading digit + // out of sigDigits, and the seventeen-digit cut-off in the fraction below is + // measured against that count. + if num[i] == '0' { + i++ + } else { + u32 = uint32(num[i] - '0') + i++ + for digit() { + if minus { + if u32 >= 214748364 && (u32 != 214748364 || num[i] > '8') { + u64, use64 = uint64(u32), true + break + } + } else if u32 >= 429496729 && (u32 != 429496729 || num[i] > '5') { + u64, use64 = uint64(u32), true + break + } + u32 = u32*10 + uint32(num[i]-'0') + i++ + sigDigits++ + } + } + if use64 { + for digit() { + if minus { + if u64 >= 0x0CCCCCCCCCCCCCCC && (u64 != 0x0CCCCCCCCCCCCCCC || num[i] > '8') { + d, useDouble = float64(u64), true + break + } + } else if u64 >= 0x1999999999999999 && (u64 != 0x1999999999999999 || num[i] > '5') { + d, useDouble = float64(u64), true + break + } + u64 = u64*10 + uint64(num[i]-'0') + i++ + sigDigits++ + } + } + for useDouble && digit() { + d = float64(d*10) + float64(num[i]-'0') + i++ + } + + // Digits behind the decimal point move it left, which is what expFrac + // counts. Past seventeen significant digits they stop counting and stop + // arriving, so they move it no further. + expFrac := 0 + if i < len(num) && num[i] == '.' { + i++ + if !useDouble { + if !use64 { + u64 = uint64(u32) + } + for digit() { + if u64 > 0x1FFFFFFFFFFFFF { + break + } + u64 = u64*10 + uint64(num[i]-'0') + i++ + expFrac-- + if u64 != 0 { + sigDigits++ + } + } + d = float64(u64) + useDouble = true + } + for digit() { + if sigDigits < 17 { + d = float64(d*10) + float64(num[i]-'0') + expFrac-- + if d > 0 { + sigDigits++ + } + } + i++ + } + } + + exp := 0 + if i < len(num) && (num[i] == 'e' || num[i] == 'E') { + i++ + if !useDouble { + if use64 { + d = float64(u64) + } else { + d = float64(u32) + } + useDouble = true + } + negative := num[i] == '-' + if negative || num[i] == '+' { + i++ + } + exp = int(num[i] - '0') + i++ + if negative { + // A negative exponent is not bounded, only stopped before it could + // overflow the int it accumulates into. Everything out that far has + // flushed to zero long before. + maxExp := (expFrac + 2147483639) / 10 + for digit() { + exp = exp*10 + int(num[i]-'0') + i++ + if exp > maxExp { + for digit() { + i++ + } + } + } + exp = -exp + } else { + maxExp := maxFloat64Digits - expFrac + for digit() { + exp = exp*10 + int(num[i]-'0') + i++ + if exp > maxExp { + return false + } + } + } + } + + // A number that never needed a double is an integer MySQL keeps exact. + if !useDouble { + return true + } + return scaleByPow10(d, exp+expFrac) <= math.MaxFloat64 +} + +// scaleByPow10 moves d by p decimal places the way RapidJSON does, in one +// multiplication or division by a power of ten. Anything below the smallest +// power in the table is moved in two steps so that the table is never indexed +// past its end. +func scaleByPow10(d float64, p int) float64 { + if p < -maxFloat64Digits { + d = movePoint(d, -maxFloat64Digits) + return movePoint(d, p+maxFloat64Digits) + } + return movePoint(d, p) +} + +func movePoint(significand float64, exp int) float64 { + switch { + case exp < -maxFloat64Digits: + return 0 + case exp >= 0: + return significand * pow10[exp] + default: + return significand / pow10[-exp] + } +} + func parseArray(s string, c *cache, depth int) (*Value, string, error) { s = skipWS(s) if len(s) == 0 { @@ -512,63 +769,82 @@ func parseRawString(s string) (string, string, error) { } } -func readFloat(s string) (i int, ok bool) { - // optional sign +// readFloat reads a JSON number off the front of s, returning how much of s it +// covers and how far its exponent moves the decimal point. Whether the number +// is one a double can hold is mysqlNumberFits's job; the exponent is reported so +// that question only has to be asked of numbers whose digits could reach that +// far. +// +// That distance is bounded rather than exact. An exponent can be written to more +// digits than it takes to leave every double behind, and one that reaches +// exponentCeiling below is left there instead of read out to the end. So it +// answers how far is far enough to matter and nothing finer; mysqlNumberFits +// reads the exponent itself off the number again. +// +// What counts as a number is JSON's grammar rather than Go's: a written plus, +// a missing digit on either side of the decimal point, and an integer part +// that opens with a zero are all rejected. +func readFloat[S string | []byte](s S) (i, exponent int, ok bool) { + // optional minus. JSON numbers carry no written plus. if i >= len(s) { return } - if s[i] == '+' || s[i] == '-' { + if s[i] == '-' { i++ } - // digits - sawdot := false - sawdigits := false - nd := 0 -loop: - for ; i < len(s); i++ { - switch c := s[i]; true { - case c == '.': - if sawdot { - break loop - } - sawdot = true - continue - - case '0' <= c && c <= '9': - sawdigits = true - if c == '0' && nd == 0 { // ignore leading zeros - continue - } - nd++ - continue + // integer part: either a lone zero, or digits that do not open with one + if i >= len(s) || s[i] < '0' || s[i] > '9' { + return + } + if s[i] == '0' { + i++ + } else { + for ; i < len(s) && '0' <= s[i] && s[i] <= '9'; i++ { } - break } - if !sawdigits { - return + + // optional fraction, which has to carry a digit of its own + if i < len(s) && s[i] == '.' { + i++ + if i >= len(s) || s[i] < '0' || s[i] > '9' { + return + } + for ; i < len(s) && '0' <= s[i] && s[i] <= '9'; i++ { + } } - // optional exponent moves decimal point. - // if we read a very large, very long number, - // just be sure to move the decimal point by - // a lot (say, 100000). it doesn't matter if it's - // not the exact number. + // optional exponent moves the decimal point. An exponent larger than + // exponentCeiling stops there rather than running on: the ceiling clears + // both the largest double and every digit this number could hold after its + // decimal point, so a number stopped at it is out of reach either way. if i < len(s) && (s[i] == 'e' || s[i] == 'E') { i++ if i >= len(s) { return } + negative := false if s[i] == '+' || s[i] == '-' { + negative = s[i] == '-' i++ } if i >= len(s) || s[i] < '0' || s[i] > '9' { return } + exponentCeiling := maxFloat64Digits + len(s) for ; i < len(s) && ('0' <= s[i] && s[i] <= '9'); i++ { + if exponent <= exponentCeiling { + exponent = exponent*10 + int(s[i]-'0') + } + } + if exponent > exponentCeiling { + exponent = exponentCeiling + } + if negative { + exponent = -exponent } } - return i, true + return i, exponent, true } // Object represents JSON object. diff --git a/go/mysql/json/parser_bench_test.go b/go/mysql/json/parser_bench_test.go new file mode 100644 index 00000000000..8139b2694a4 --- /dev/null +++ b/go/mysql/json/parser_bench_test.go @@ -0,0 +1,114 @@ +/* +Copyright 2026 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package json + +import ( + "strconv" + "strings" + "testing" +) + +// numberArray builds a JSON array of n numbers, each spelled by digits. +func numberArray(n int, digits func(i int) string) string { + var sb strings.Builder + sb.WriteByte('[') + for i := range n { + if i > 0 { + sb.WriteByte(',') + } + sb.WriteString(digits(i)) + } + sb.WriteByte(']') + return sb.String() +} + +// benchDocs covers the number shapes the parser takes different paths through. +// +// The first four are the everyday ones, where the cost is the scan itself. The +// last three reach the magnitude check, which a number only pays for once it is +// written to more digits than its exponent leaves a double room for: once with +// an exponent that shrinks the room to ask the question of every element, once +// where the digits alone are past what a double has, and once for a document the +// answer rejects. reachesMagnitudeCheck holds them to that, since a case that +// falls short of the check goes on measuring the scan and reads as though the +// check were free. +var benchDocs = []struct { + name string + doc string + checked bool + rejected bool +}{ + {name: "int/1024", doc: numberArray(1024, func(i int) string { return strconv.Itoa(i * 7919) })}, + {name: "frac/1024", doc: numberArray(1024, func(i int) string { return strconv.Itoa(i) + "." + strconv.Itoa(i*7919) })}, + {name: "exp/1024", doc: numberArray(1024, func(i int) string { return strconv.Itoa(i) + "." + strconv.Itoa(i*7919) + "e" + strconv.Itoa(i%300) })}, + {name: "mixed-object", doc: `{"id":38141,"name":"a name","ok":true,"score":-12.5e3,"tags":["x","y"],"meta":null}`}, + + {name: "checked/1024", doc: numberArray(1024, func(i int) string { return "1" + strconv.Itoa(1000000000000000000+i) + "e289" }), checked: true}, + {name: "checked-long-fraction", doc: "1" + strings.Repeat("0", 308) + "." + strings.Repeat("5", 400), checked: true}, + {name: "rejected", doc: strings.Repeat("9", 400) + "e-100", checked: true, rejected: true}, +} + +// reachesMagnitudeCheck reports whether every number in doc is converted to find +// out whether a double can hold it, which is what the checked cases are for. A +// number written to fewer digits than a double has room for answers that question +// from its digits alone, and a case built out of those measures the scan it shares +// with every other case instead. +func reachesMagnitudeCheck(doc string) bool { + numbers := 0 + for i := 0; i < len(doc); { + if c := doc[i]; c != '-' && (c < '0' || c > '9') { + i++ + continue + } + flen, exponent, ok := readFloat(doc[i:]) + if !ok { + return false + } + if !mayExceedFloat64(doc[i:i+flen], exponent) { + return false + } + numbers++ + i += flen + } + return numbers > 0 +} + +func BenchmarkParse(b *testing.B) { + for _, tc := range benchDocs { + b.Run(tc.name, func(b *testing.B) { + var p Parser + + // A case that stops early measures the error path instead of the one + // it was written for, and reads as a speed-up while doing it. + if _, err := p.Parse(tc.doc); (err != nil) != tc.rejected { + b.Fatalf("document does not take the path this case measures: err=%v", err) + } + if reachesMagnitudeCheck(tc.doc) != tc.checked { + b.Fatalf("document reaches the magnitude check: %v, want %v", !tc.checked, tc.checked) + } + + b.ReportAllocs() + b.SetBytes(int64(len(tc.doc))) + for b.Loop() { + v, err := p.Parse(tc.doc) + if err == nil && v == nil { + b.Fatal("parsed to no value") + } + } + }) + } +} diff --git a/go/mysql/json/parser_test.go b/go/mysql/json/parser_test.go index 1e54007659c..da50dc56b04 100644 --- a/go/mysql/json/parser_test.go +++ b/go/mysql/json/parser_test.go @@ -18,6 +18,7 @@ limitations under the License. package json import ( + "strings" "testing" "github.com/stretchr/testify/require" @@ -30,7 +31,7 @@ func TestParseRawNumber(t *testing.T) { f := func(s, expectedRN, expectedTail string) { t.Helper() - flen, ok := readFloat(s) + flen, _, ok := readFloat(s) if !ok { t.Fatalf("unexpected error when parsing '%s'", s) } @@ -55,16 +56,15 @@ func TestParseRawNumber(t *testing.T) { f("-12.345E+67 tail", "-12.345E+67", " tail") f("-12.345E-67,tail", "-12.345E-67", ",tail") f("-1234567.8e+90tail", "-1234567.8e+90", "tail") - f("12.tail", "12.", "tail") - f(".2tail", ".2", "tail") - f("-.2tail", "-.2", "tail") + f("0.2tail", "0.2", "tail") + f("-0.2tail", "-0.2", "tail") }) t.Run("error", func(t *testing.T) { f := func(s, expectedTail string) { t.Helper() - flen, ok := readFloat(s) + flen, _, ok := readFloat(s) if ok { t.Fatalf("expecting non-nil error") } @@ -79,7 +79,267 @@ func TestParseRawNumber(t *testing.T) { f(",", ",") f("{", "{") f("\"", "\"") + + // A decimal point needs a digit on either side of it, and a number + // opens with a minus or a digit. + f("12.tail", "tail") + f(".2tail", ".2tail") + f("-.2tail", ".2tail") + f("+1tail", "+1tail") + }) +} + +// TestParseNumberTooBigForDouble covers the boundary MySQL puts on JSON +// numbers: a number it cannot store as a double makes the whole document +// invalid, rather than being kept at the precision it was written to. +// Underflow is not rejected — it flushes to zero. +func TestParseNumberTooBigForDouble(t *testing.T) { + tooManyDigits := "1" + strings.Repeat("0", 309) + + t.Run("accepted", func(t *testing.T) { + for _, doc := range []string{ + "1e308", + "-1e308", + "1.7976931348623157e308", + "-1.7976931348623157e308", + "99999999999999999999999999999999999999999", + "1" + strings.Repeat("0", 307), + // Underflow keeps the document valid and reads as zero. + "1e-400", + "1e-1000", + "1e-1024", + "0." + strings.Repeat("0", 400) + "1", + // Digits a double has room for, moved out of the way by a negative + // exponent. + "1" + strings.Repeat("0", 307) + "e-1", + "1" + strings.Repeat("0", 307) + "e-400", + "-1" + strings.Repeat("0", 307) + "e-400", + "0." + strings.Repeat("0", 400) + "1e-400", + // A written sign and a padded exponent are spellings, not + // magnitudes, and none of these is anywhere near the limit. + "1e+0", + "1e-0", + "1e0000000000", + "1e-0000000000", + "1e+308", + "1e00000000000000000308", + // A written exponent is bounded by where it puts the decimal + // point, so digits after the point buy the same number of places + // back. Zero is subject to the bound like anything else. + "0e308", + "-0e308", + "0.0e309", + "0.00e310", + "0.1e309", + "0.01e310", + "0." + strings.Repeat("0", 400) + "1e700", + } { + t.Run(startEndString(doc), func(t *testing.T) { + var p Parser + v, err := p.Parse(doc) + require.NoError(t, err) + require.Equal(t, TypeNumber, v.Type()) + }) + } + }) + + t.Run("rejected", func(t *testing.T) { + for _, doc := range []string{ + "1e309", + "-1e309", + "1e1025", + "1.7976931348623159e308", + "1e+309", + tooManyDigits, + // One place past what the digits after the point buy back. These + // all convert to zero, so only the exponent as written rules them + // out. + "0e309", + "-0e309", + "0e+309", + "0e1000", + "0.0e310", + "0.00e311", + "0.1e310", + "0.01e311", + "0." + strings.Repeat("0", 400) + "1e710", + // Within the written bound, but too big once converted. + "10e308", + "1" + strings.Repeat("0", 30) + "e279", + // More digits than a double has room for. The digits are read before + // the exponent is applied, so a negative exponent does not buy the + // room back however far it moves the decimal point afterwards. + "1" + strings.Repeat("0", 320) + "e-20", + "1" + strings.Repeat("0", 350) + "e-50", + "1" + strings.Repeat("0", 400) + "e-400", + "-1" + strings.Repeat("0", 400) + "e-400", + "1" + strings.Repeat("0", 400) + ".5e-400", + strings.Repeat("9", 400) + "e-100", + // A number anywhere in the document invalidates all of it. + "[1, 1e309]", + `{"a": 1e309}`, + "[[1e309]]", + } { + t.Run(startEndString(doc), func(t *testing.T) { + var p Parser + _, err := p.Parse(doc) + require.ErrorContains(t, err, "number too big to be stored in double") + }) + } + }) + + // A negative exponent is not bounded, only stopped before it overflows the + // int it accumulates into, and what sends a number through the conversion at + // all is being written to more digits than a double holds. These cross the + // two, so the exponent is read into an int that cannot hold it and then + // scales a significand: written past that stop, it lands wherever the + // overflow leaves it, which can be a power of ten the table does not go up + // to. Each of these stays valid and reads as zero. MySQL 8.0.46 accepts all + // three and reads them as zero too. + t.Run("a negative exponent written past what an int holds", func(t *testing.T) { + for _, doc := range []string{ + strings.Repeat("9", 400) + "e-" + strings.Repeat("2", 306), + "-" + strings.Repeat("9", 400) + "e-" + strings.Repeat("2", 306), + strings.Repeat("1", 400) + "." + strings.Repeat("5", 20) + "e-" + strings.Repeat("2", 306), + } { + t.Run(startEndString(doc), func(t *testing.T) { + var p Parser + v, err := p.Parse(doc) + require.NoError(t, err) + + f, ok := v.Float64() + require.True(t, ok) + require.Zero(t, f) + }) + } + }) + + // The significand accumulates one digit at a time, and each step rounds + // the multiplication and the addition separately, the way MySQL's builds + // run the loop. Fusing the two into one rounding — which the Go compiler + // may do on arm64 unless the conversion in mysqlNumberFits stops it — + // moves the accumulation an ULP for these documents, and that is enough + // to push them over the largest double. MySQL 8.0.45, 8.4.11 and 9.4.0 + // accept all of them. + t.Run("each accumulation step rounds on its own", func(t *testing.T) { + for _, doc := range []string{ + "17976931348623154547712857878e280", + "179769313486231559524062337652e279", + "179769313486231577704643761e282", + "1797693134862315724800793889e281", + } { + t.Run(startEndString(doc), func(t *testing.T) { + var p Parser + _, err := p.Parse(doc) + require.NoError(t, err) + }) + } }) + + // Right at the top of the range the answer turns on how a number was + // written rather than on what it is worth. The digits are split between a + // significand and a power of ten to scale it by, and where that split falls + // decides which way the last place rounds — so writing the same value to one + // more digit moves the split and can move the answer with it. + t.Run("spelling at the largest double", func(t *testing.T) { + for _, tc := range []struct { + doc string + fits bool + }{ + {"1.7976931348623157e308", true}, + {"1.7976931348623158e308", false}, + {"1.79769313486231580e308", true}, + {"1.797693134862315800e308", true}, + {"1.79769313486231581e308", true}, + {"1.79769313486231585e308", true}, + {"1.7976931348623159e308", false}, + {"1.7976931348623157081e308", true}, + {"17976931348623157e292", true}, + {"17976931348623158e292", false}, + {"179769313486231580e291", true}, + {"1797693134862315800e290", false}, + {"17976931348623158000000e286", false}, + } { + t.Run(tc.doc, func(t *testing.T) { + var p Parser + _, err := p.Parse(tc.doc) + if tc.fits { + require.NoError(t, err) + } else { + require.ErrorContains(t, err, "number too big to be stored in double") + } + }) + } + }) +} + +// TestParseNumberGrammar covers the shapes JSON's grammar allows a number to +// take. A number opens with a minus or a digit, an integer part of more than +// one digit does not open with a zero, and a decimal point has digits on both +// sides of it. MySQL holds documents to the same grammar, and nan is not a +// number to either of them. +func TestParseNumberGrammar(t *testing.T) { + t.Run("accepted", func(t *testing.T) { + for _, doc := range []string{ + "0", "-0", "0.5", "-0.5", "1", "-1", "1.2", "0e0", "-0e0", + "1e5", "1E5", "1e007", "1e+007", "1e-5", "0.0", + "[0,1,2]", `{"a":-0.5,"b":[1e5]}`, + } { + t.Run(doc, func(t *testing.T) { + var p Parser + _, err := p.Parse(doc) + require.NoError(t, err) + }) + } + }) + + t.Run("rejected", func(t *testing.T) { + for _, doc := range []string{ + // An integer part that opens with a zero. + "007", "-003", "01", "00", "00.5", "01.5", "[007]", + // A decimal point missing a digit on one side. + ".2", "-.2", "12.", "-12.", "1.e5", `{"a": .2}`, "[12.]", + // A written plus. + "+1", "+1.5", "+0", "[+1]", + // Not a number at all. + "nan", "NaN", "NAN", "[nan]", `{"a": nan}`, "-nan", ".", "-", + } { + t.Run(doc, func(t *testing.T) { + var p Parser + _, err := p.Parse(doc) + require.Error(t, err) + }) + } + }) +} + +// TestParseErrorAbbreviatesTheDocument covers how much of a rejected document +// its error names. Nothing bounds how long a document may be, and Parse copies +// the message it wraps, so naming the text in full hands a client its own +// document back twice over. Each rejection abbreviates what it names, as the +// unparsed tail alongside it already did. +func TestParseErrorAbbreviatesTheDocument(t *testing.T) { + long := strings.Repeat("9", 100000) + + for _, tc := range []struct { + name string + doc string + }{ + {name: "a number too big for a double", doc: "1" + long}, + {name: "a written plus", doc: "+" + long}, + {name: "a decimal point with nothing before it", doc: "." + long}, + {name: "a decimal point with nothing after it", doc: long + "."}, + {name: "nan", doc: "nan" + long}, + {name: "nothing the grammar has a shape for", doc: "q" + long}, + } { + t.Run(tc.name, func(t *testing.T) { + var p Parser + _, err := p.Parse(tc.doc) + require.Error(t, err) + require.NotContains(t, err.Error(), strings.Repeat("9", 200), + "the error carries the document it is reporting on") + }) + } } func TestUnescapeStringBestEffort(t *testing.T) {