diff --git a/go/mysql/json/marshal.go b/go/mysql/json/marshal.go index 8d527e37c4b..2a3b19ea5db 100644 --- a/go/mysql/json/marshal.go +++ b/go/mysql/json/marshal.go @@ -491,7 +491,7 @@ func (w *sqlWriter) writeNumber(top bool) error { // Use the parser's readFloat to validate number grammar, rejecting // malformed inputs like "1+2", "1..2", or "1e+" that a simple // character-class loop would accept. - n, ok := readFloat(w.data[w.pos:]) + n, _, _, ok := readFloat(w.data[w.pos:]) if !ok || n == 0 { return fmt.Errorf("invalid number at position %d in JSON", w.pos) } diff --git a/go/mysql/json/parser.go b/go/mysql/json/parser.go index 47271e819cb..30b582b5ec4 100644 --- a/go/mysql/json/parser.go +++ b/go/mysql/json/parser.go @@ -22,6 +22,7 @@ import ( "encoding/base64" "errors" "fmt" + "math" "slices" "strconv" "strings" @@ -162,12 +163,15 @@ func parseValue(s string, c *cache, depth int) (*Value, string, error) { return v, tail, nil } if s[0] == '"' { - ss, tail, err := parseRawString(s[1:]) + ss, tail, unescape, err := parseRawValueString(s[1:]) if err != nil { return nil, tail, fmt.Errorf("cannot parse string: %s", err) } + if unescape { + ss = unescapeStringBestEffort(ss) + } v := c.getValue() - v.t = typeRawString + v.t = TypeString v.s = ss return v, tail, nil } @@ -185,19 +189,12 @@ func parseValue(s string, c *cache, depth int) (*Value, string, error) { } if s[0] == 'n' { if len(s) < len("null") || s[:len("null")] != "null" { - // Try parsing NaN - if len(s) >= 3 && strings.EqualFold(s[:3], "nan") { - v := c.getValue() - v.t = TypeNumber - v.s = s[:3] - return v, s[3:], nil - } return nil, s, fmt.Errorf("unexpected value found: %q", s) } return ValueNull, s[len("null"):], nil } - flen, ok := readFloat(s) + flen, exponent, fractional, ok := readFloat(s) if !ok { return nil, s[flen:], fmt.Errorf("invalid number in JSON string: %q", s) } @@ -205,10 +202,285 @@ func parseValue(s string, c *cache, depth int) (*Value, string, error) { v := c.getValue() v.t = TypeNumber v.s = s[:flen] - v.n = numberTypeRaw + // An integer that fits is nowhere near the range of a double, so only the + // kind that did not fit one can be too big for the other. + v.n = numberKind(v.s, fractional) + if v.n == NumberTypeFloat && mayExceedFloat64(v.s, exponent) && !mysqlNumberFits(v.s) { + return nil, s, fmt.Errorf("number too big to be stored in double: %q", v.s) + } return v, s[flen:], nil } +// maxFloat64Digits is how far a decimal point can travel before a double runs +// out of room. The largest double is under 1.8e308, so a written exponent of +// 308 always leaves somewhere for the number to land and 309 need not. +const maxFloat64Digits = 308 + +// mayExceedFloat64 reports whether num is worth converting to find out whether +// a double can hold it. It errs towards yes: the job is to keep the conversion +// off the common path, not to answer the question. A number is below the +// largest double whenever the digits it is written to, moved by its exponent, +// stay within the ones that double has — len(num) overcounts the digits, which +// only makes the answer yes more often. +func mayExceedFloat64(num string, exponent int) bool { + return len(num)+exponent > maxFloat64Digits +} + +// pow10 holds the powers of ten that a number's digits are scaled by, which is +// the table RapidJSON scales its significand with. It is built from the decimal +// spellings rather than from math.Pow10, which multiplies two table entries +// together and lands a bit away from the spelling for 78 of these exponents. +var pow10 [maxFloat64Digits + 1]float64 + +func init() { + for i := range pow10 { + pow10[i], _ = fastparse.ParseFloat64("1e" + strconv.Itoa(i)) + } +} + +// mysqlNumberFits reports whether MySQL can store num, which has to already be +// a grammatically valid JSON number. +// +// MySQL parses JSON with RapidJSON, which decides this in two places. While +// reading the exponent, a written exponent may not move the decimal point more +// than maxFloat64Digits places past the digits already behind it. After +// converting, the result may not be larger than the largest double. Neither +// place catches what the other does: an exponent that lands on zero is refused +// only as written, and a number written within the bound can still overflow +// once converted. +// +// The conversion below is the one RapidJSON does rather than a correct one. It +// accumulates the significand into a double and scales that by a power of ten, +// which lands an ULP or two from the true value, and the boundary sits wherever +// that lands. A number a hair under the largest double therefore comes down to +// how it was spelled: 1.7976931348623158e308 does not fit, while writing the +// same value as 1.79769313486231580e308 does, because the extra digit moves +// where the significand ends and the scaling begins. Converting the same way is +// what makes a document valid here exactly when it is valid in MySQL. +// +// The float64 conversions in the digit loops keep the multiply and the add as +// two roundings. Without them the compiler is free to fuse both into one FMA +// on arm64, which rounds once and can land the accumulation one ULP away from +// where MySQL's builds put it — enough to flip which side of the largest +// double a number falls on. +func mysqlNumberFits(num string) bool { + i := 0 + minus := num[i] == '-' + if minus { + i++ + } + + var ( + u32 uint32 + u64 uint64 + use64 bool + sigDigits int + d float64 + useDouble bool + ) + digit := func() bool { return i < len(num) && num[i] >= '0' && num[i] <= '9' } + + // The integer part accumulates as an integer for as long as one can hold + // it, stepping up in width as it fills. RapidJSON leaves the leading digit + // out of sigDigits, and the seventeen-digit cut-off in the fraction below is + // measured against that count. + if num[i] == '0' { + i++ + } else { + u32 = uint32(num[i] - '0') + i++ + for digit() { + if minus { + if u32 >= 214748364 && (u32 != 214748364 || num[i] > '8') { + u64, use64 = uint64(u32), true + break + } + } else if u32 >= 429496729 && (u32 != 429496729 || num[i] > '5') { + u64, use64 = uint64(u32), true + break + } + u32 = u32*10 + uint32(num[i]-'0') + i++ + sigDigits++ + } + } + if use64 { + for digit() { + if minus { + if u64 >= 0x0CCCCCCCCCCCCCCC && (u64 != 0x0CCCCCCCCCCCCCCC || num[i] > '8') { + d, useDouble = float64(u64), true + break + } + } else if u64 >= 0x1999999999999999 && (u64 != 0x1999999999999999 || num[i] > '5') { + d, useDouble = float64(u64), true + break + } + u64 = u64*10 + uint64(num[i]-'0') + i++ + sigDigits++ + } + } + for useDouble && digit() { + d = float64(d*10) + float64(num[i]-'0') + i++ + } + + // Digits behind the decimal point move it left, which is what expFrac + // counts. Past seventeen significant digits they stop counting and stop + // arriving, so they move it no further. + expFrac := 0 + if i < len(num) && num[i] == '.' { + i++ + if !useDouble { + if !use64 { + u64 = uint64(u32) + } + for digit() { + if u64 > 0x1FFFFFFFFFFFFF { + break + } + u64 = u64*10 + uint64(num[i]-'0') + i++ + expFrac-- + if u64 != 0 { + sigDigits++ + } + } + d = float64(u64) + useDouble = true + } + for digit() { + if sigDigits < 17 { + d = float64(d*10) + float64(num[i]-'0') + expFrac-- + if d > 0 { + sigDigits++ + } + } + i++ + } + } + + exp := 0 + if i < len(num) && (num[i] == 'e' || num[i] == 'E') { + i++ + if !useDouble { + if use64 { + d = float64(u64) + } else { + d = float64(u32) + } + useDouble = true + } + negative := num[i] == '-' + if negative || num[i] == '+' { + i++ + } + exp = int(num[i] - '0') + i++ + if negative { + // A negative exponent is not bounded, only stopped before it could + // overflow the int it accumulates into. Everything out that far has + // flushed to zero long before. + maxExp := (expFrac + 2147483639) / 10 + for digit() { + exp = exp*10 + int(num[i]-'0') + i++ + if exp > maxExp { + for digit() { + i++ + } + } + } + exp = -exp + } else { + maxExp := maxFloat64Digits - expFrac + for digit() { + exp = exp*10 + int(num[i]-'0') + i++ + if exp > maxExp { + return false + } + } + } + } + + // A number that never needed a double is an integer MySQL keeps exact. + if !useDouble { + return true + } + return scaleByPow10(d, exp+expFrac) <= math.MaxFloat64 +} + +// scaleByPow10 moves d by p decimal places the way RapidJSON does, in one +// multiplication or division by a power of ten. Anything below the smallest +// power in the table is moved in two steps so that the table is never indexed +// past its end. +func scaleByPow10(d float64, p int) float64 { + if p < -maxFloat64Digits { + d = movePoint(d, -maxFloat64Digits) + return movePoint(d, p+maxFloat64Digits) + } + return movePoint(d, p) +} + +func movePoint(significand float64, exp int) float64 { + switch { + case exp < -maxFloat64Digits: + return 0 + case exp >= 0: + return significand * pow10[exp] + default: + return significand / pow10[-exp] + } +} + +// int64Digits is the most digits an integer can be written to and still be +// certain to fit an int64. +const int64Digits = 18 + +// numberKind settles what a number is kept as, from the shape the scanner +// already read. MySQL keeps an integer that fits exactly and makes a double of +// everything else, so a fraction or an exponent decides the question outright, +// and a short run of digits decides it without converting anything. Only the +// lengths that straddle the integer limits are converted, and those at most +// once rather than once per kind. +func numberKind(num string, fractional bool) NumberType { + if fractional { + return NumberTypeFloat + } + + digits := num + var negative bool + if len(digits) > 0 && (digits[0] == '-' || digits[0] == '+') { + negative = digits[0] == '-' + digits = digits[1:] + } + if len(digits) > 1 && digits[0] == '0' { + digits = strings.TrimLeft(digits, "0") + } + if len(digits) <= int64Digits { + return NumberTypeSigned + } + + if negative { + if _, err := fastparse.ParseInt64(num, 10); err == nil { + return NumberTypeSigned + } + // Nothing negative fits an unsigned integer. + return NumberTypeFloat + } + + unsigned, err := fastparse.ParseUint64(digits, 10) + if err != nil { + return NumberTypeFloat + } + if unsigned <= math.MaxInt64 { + return NumberTypeSigned + } + return NumberTypeUnsigned +} + func parseArray(s string, c *cache, depth int) (*Value, string, error) { s = skipWS(s) if len(s) == 0 { @@ -481,6 +753,23 @@ func parseRawKey(s string) (string, string, bool, error) { return s, "", false, errors.New(`missing closing '"'`) } +// parseRawValueString reads a string value and reports whether it carries +// escapes, so that the caller can unescape it once, while the value is still +// its own. This mirrors parseRawKey, which object keys have always used. +func parseRawValueString(s string) (string, string, bool, error) { + for i := range len(s) { + if s[i] == '"' { + // Fast path. + return s[:i], s[i+1:], false, nil + } + if s[i] == '\\' { + str, tail, err := parseRawString(s) + return str, tail, true, err + } + } + return s, "", false, errors.New(`missing closing '"'`) +} + func parseRawString(s string) (string, string, error) { n := strings.IndexByte(s, '"') if n < 0 { @@ -513,63 +802,79 @@ func parseRawString(s string) (string, string, error) { } } -func readFloat[S string | []byte](s S) (i int, ok bool) { - // optional sign +// readFloat reads a JSON number off the front of s, returning how much of s it +// covers, the exponent it was written with, and whether a fraction or an +// exponent was written at all. Whether the number is one a double can hold is +// mysqlNumberFits's job; the exponent is reported so that question only has to +// be asked of numbers whose digits could reach that far, and the shape settles +// what the number is kept as without converting anything. +// +// What counts as a number is JSON's grammar rather than Go's: a written plus, +// a missing digit on either side of the decimal point, and an integer part +// that opens with a zero are all rejected. +func readFloat[S string | []byte](s S) (i, exponent int, fractional, ok bool) { + // optional minus. JSON numbers carry no written plus. if i >= len(s) { return } - if s[i] == '+' || s[i] == '-' { + if s[i] == '-' { i++ } - // digits - sawdot := false - sawdigits := false - nd := 0 -loop: - for ; i < len(s); i++ { - switch c := s[i]; true { - case c == '.': - if sawdot { - break loop - } - sawdot = true - continue - - case '0' <= c && c <= '9': - sawdigits = true - if c == '0' && nd == 0 { // ignore leading zeros - continue - } - nd++ - continue + // integer part: either a lone zero, or digits that do not open with one + if i >= len(s) || s[i] < '0' || s[i] > '9' { + return + } + if s[i] == '0' { + i++ + } else { + for ; i < len(s) && '0' <= s[i] && s[i] <= '9'; i++ { } - break } - if !sawdigits { - return + + // optional fraction, which has to carry a digit of its own + if i < len(s) && s[i] == '.' { + fractional = true + i++ + if i >= len(s) || s[i] < '0' || s[i] > '9' { + return + } + for ; i < len(s) && '0' <= s[i] && s[i] <= '9'; i++ { + } } - // optional exponent moves decimal point. - // if we read a very large, very long number, - // just be sure to move the decimal point by - // a lot (say, 100000). it doesn't matter if it's - // not the exact number. + // optional exponent moves the decimal point. An exponent larger than + // exponentCeiling stops there rather than running on: the ceiling clears + // both the largest double and every digit this number could hold after its + // decimal point, so a number stopped at it is out of reach either way. if i < len(s) && (s[i] == 'e' || s[i] == 'E') { + fractional = true i++ if i >= len(s) { return } + negative := false if s[i] == '+' || s[i] == '-' { + negative = s[i] == '-' i++ } if i >= len(s) || s[i] < '0' || s[i] > '9' { return } + exponentCeiling := maxFloat64Digits + len(s) for ; i < len(s) && ('0' <= s[i] && s[i] <= '9'); i++ { + if exponent <= exponentCeiling { + exponent = exponent*10 + int(s[i]-'0') + } + } + if exponent > exponentCeiling { + exponent = exponentCeiling + } + if negative { + exponent = -exponent } } - return i, true + return i, exponent, fractional, true } // Object represents JSON object. @@ -781,11 +1086,6 @@ func (v *Value) marshalFloat(dst []byte) []byte { // MarshalTo appends marshaled v to dst and returns the result. func (v *Value) MarshalTo(dst []byte) []byte { switch v.t { - case typeRawString: - dst = append(dst, '"') - dst = append(dst, v.s...) - dst = append(dst, '"') - return dst case TypeObject: return v.o.MarshalTo(dst) case TypeArray: @@ -907,8 +1207,6 @@ const ( // TypeBlob is JSON blob. TypeBlob - - typeRawString ) type NumberType int32 @@ -919,7 +1217,6 @@ const ( NumberTypeUnsigned NumberTypeDecimal NumberTypeFloat - numberTypeRaw ) // String returns string representation of t. @@ -950,8 +1247,6 @@ func (t Type) String() string { case TypeNull: return "null" - // typeRawString is skipped intentionally, - // since it shouldn't be visible to user. default: panic(fmt.Errorf("BUG: unknown Value type: %d", t)) } @@ -962,10 +1257,6 @@ func (v *Value) Type() Type { if v == nil { return TypeNull } - if v.t == typeRawString { - v.s = unescapeStringBestEffort(v.s) - v.t = TypeString - } return v.t } @@ -1036,28 +1327,9 @@ func (v *Value) NumberType() NumberType { if v.t != TypeNumber { return NumberTypeUnknown } - if v.n == numberTypeRaw { - v.n = parseNumberType(v.s) - } return v.n } -func parseNumberType(ns string) NumberType { - _, err := fastparse.ParseInt64(ns, 10) - if err == nil { - return NumberTypeSigned - } - _, err = fastparse.ParseUint64(ns, 10) - if err == nil { - return NumberTypeUnsigned - } - _, err = fastparse.ParseFloat64(ns) - if err == nil { - return NumberTypeFloat - } - return NumberTypeUnknown -} - func (v *Value) Int64() (int64, bool) { i, err := fastparse.ParseInt64(v.s, 10) if err != nil { diff --git a/go/mysql/json/parser_bench_test.go b/go/mysql/json/parser_bench_test.go new file mode 100644 index 00000000000..80abc2e6150 --- /dev/null +++ b/go/mysql/json/parser_bench_test.go @@ -0,0 +1,68 @@ +/* +Copyright 2026 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package json + +import ( + "fmt" + "strconv" + "strings" + "testing" +) + +func benchArray(n int, elem func(i int) string) string { + var buf strings.Builder + buf.WriteByte('[') + for i := range n { + if i > 0 { + buf.WriteByte(',') + } + buf.WriteString(elem(i)) + } + buf.WriteByte(']') + return buf.String() +} + +// BenchmarkParse covers the shapes whose cost depends on how much the parser +// settles up front: numbers of each kind, and strings with and without escapes. +func BenchmarkParse(b *testing.B) { + documents := []struct { + name string + doc string + }{ + {"integers", benchArray(1024, func(i int) string { return strconv.Itoa(i) })}, + {"fractions", benchArray(1024, func(i int) string { return strconv.Itoa(i) + ".25" })}, + {"exponents", benchArray(1024, func(i int) string { return strconv.Itoa(i) + "e3" })}, + {"big integers", benchArray(1024, func(i int) string { return fmt.Sprintf("922337203685477580%d", i%10) })}, + {"plain strings", benchArray(1024, func(i int) string { return fmt.Sprintf("%q", "value"+strconv.Itoa(i)) })}, + {"escaped strings", benchArray(1024, func(i int) string { return `"value\u0061` + strconv.Itoa(i) + `"` })}, + {"mixed object", `{"a":1,"b":2.5,"c":"str","d":[1,2,3],"e":{"f":4}}`}, + } + + for _, document := range documents { + b.Run(document.name, func(b *testing.B) { + raw := []byte(document.doc) + var p Parser + b.ReportAllocs() + b.ResetTimer() + for range b.N { + if _, err := p.ParseBytes(raw); err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/go/mysql/json/parser_test.go b/go/mysql/json/parser_test.go index a223b26fc40..154e96ad44a 100644 --- a/go/mysql/json/parser_test.go +++ b/go/mysql/json/parser_test.go @@ -18,11 +18,15 @@ limitations under the License. package json import ( + "strings" + "sync" "testing" "github.com/stretchr/testify/require" "vitess.io/vitess/go/hack" + "vitess.io/vitess/go/mysql/fastparse" + "vitess.io/vitess/go/vt/vthash" ) func TestParseRawNumber(t *testing.T) { @@ -30,7 +34,7 @@ func TestParseRawNumber(t *testing.T) { f := func(s, expectedRN, expectedTail string) { t.Helper() - flen, ok := readFloat(s) + flen, _, _, ok := readFloat(s) require.Truef(t, ok, "unexpected error when parsing '%s'", s) rn, tail := s[:flen], s[flen:] @@ -48,16 +52,15 @@ func TestParseRawNumber(t *testing.T) { f("-12.345E+67 tail", "-12.345E+67", " tail") f("-12.345E-67,tail", "-12.345E-67", ",tail") f("-1234567.8e+90tail", "-1234567.8e+90", "tail") - f("12.tail", "12.", "tail") - f(".2tail", ".2", "tail") - f("-.2tail", "-.2", "tail") + f("0.2tail", "0.2", "tail") + f("-0.2tail", "-0.2", "tail") }) t.Run("error", func(t *testing.T) { f := func(s, expectedTail string) { t.Helper() - flen, ok := readFloat(s) + flen, _, _, ok := readFloat(s) require.False(t, ok, "expecting non-nil error") require.Equalf(t, expectedTail, s[flen:], "unexpected tail; got %q; want %q", s[flen:], expectedTail) } @@ -68,7 +71,347 @@ func TestParseRawNumber(t *testing.T) { f(",", ",") f("{", "{") f("\"", "\"") + + // A decimal point needs a digit on either side of it, and a number + // opens with a minus or a digit. + f("12.tail", "tail") + f(".2tail", ".2tail") + f("-.2tail", ".2tail") + f("+1tail", "+1tail") + }) +} + +// TestParseNumberTooBigForDouble covers the boundary MySQL puts on JSON +// numbers: a number it cannot store as a double makes the whole document +// invalid, rather than being kept at the precision it was written to. +// Underflow is not rejected — it flushes to zero. +func TestParseNumberTooBigForDouble(t *testing.T) { + tooManyDigits := "1" + strings.Repeat("0", 309) + + t.Run("accepted", func(t *testing.T) { + for _, doc := range []string{ + "1e308", + "-1e308", + "1.7976931348623157e308", + "-1.7976931348623157e308", + "99999999999999999999999999999999999999999", + "1" + strings.Repeat("0", 307), + // Underflow keeps the document valid and reads as zero. + "1e-400", + "1e-1000", + "1e-1024", + "0." + strings.Repeat("0", 400) + "1", + // A written sign and a padded exponent are spellings, not + // magnitudes, and none of these is anywhere near the limit. + "1e+0", + "1e-0", + "1e0000000000", + "1e-0000000000", + "1e+308", + "1e00000000000000000308", + // A written exponent is bounded by where it puts the decimal + // point, so digits after the point buy the same number of places + // back. Zero is subject to the bound like anything else. + "0e308", + "-0e308", + "0.0e309", + "0.00e310", + "0.1e309", + "0.01e310", + "0." + strings.Repeat("0", 400) + "1e700", + } { + t.Run(startEndString(doc), func(t *testing.T) { + var p Parser + v, err := p.Parse(doc) + require.NoError(t, err) + require.Equal(t, TypeNumber, v.Type()) + }) + } + }) + + t.Run("rejected", func(t *testing.T) { + for _, doc := range []string{ + "1e309", + "-1e309", + "1e1025", + "1.7976931348623159e308", + "1e+309", + tooManyDigits, + // One place past what the digits after the point buy back. These + // all convert to zero, so only the exponent as written rules them + // out. + "0e309", + "-0e309", + "0e+309", + "0e1000", + "0.0e310", + "0.00e311", + "0.1e310", + "0.01e311", + "0." + strings.Repeat("0", 400) + "1e710", + // Within the written bound, but too big once converted. + "10e308", + "1" + strings.Repeat("0", 30) + "e279", + // A number anywhere in the document invalidates all of it. + "[1, 1e309]", + `{"a": 1e309}`, + "[[1e309]]", + } { + t.Run(startEndString(doc), func(t *testing.T) { + var p Parser + _, err := p.Parse(doc) + require.ErrorContains(t, err, "number too big to be stored in double") + }) + } + }) + + // The significand accumulates one digit at a time, and each step rounds + // the multiplication and the addition separately, the way MySQL's builds + // run the loop. Fusing the two into one rounding — which the Go compiler + // may do on arm64 unless the conversion in mysqlNumberFits stops it — + // moves the accumulation an ULP for these documents, and that is enough + // to push them over the largest double. MySQL 8.0.45, 8.4.11 and 9.4.0 + // accept all of them. + t.Run("each accumulation step rounds on its own", func(t *testing.T) { + for _, doc := range []string{ + "17976931348623154547712857878e280", + "179769313486231559524062337652e279", + "179769313486231577704643761e282", + "1797693134862315724800793889e281", + } { + t.Run(startEndString(doc), func(t *testing.T) { + var p Parser + _, err := p.Parse(doc) + require.NoError(t, err) + }) + } + }) + + // Right at the top of the range the answer turns on how a number was + // written rather than on what it is worth. The digits are split between a + // significand and a power of ten to scale it by, and where that split falls + // decides which way the last place rounds — so writing the same value to one + // more digit moves the split and can move the answer with it. + t.Run("spelling at the largest double", func(t *testing.T) { + for _, tc := range []struct { + doc string + fits bool + }{ + {"1.7976931348623157e308", true}, + {"1.7976931348623158e308", false}, + {"1.79769313486231580e308", true}, + {"1.797693134862315800e308", true}, + {"1.79769313486231581e308", true}, + {"1.79769313486231585e308", true}, + {"1.7976931348623159e308", false}, + {"1.7976931348623157081e308", true}, + {"17976931348623157e292", true}, + {"17976931348623158e292", false}, + {"179769313486231580e291", true}, + {"1797693134862315800e290", false}, + {"17976931348623158000000e286", false}, + } { + t.Run(tc.doc, func(t *testing.T) { + var p Parser + _, err := p.Parse(tc.doc) + if tc.fits { + require.NoError(t, err) + } else { + require.ErrorContains(t, err, "number too big to be stored in double") + } + }) + } + }) +} + +// TestParseNumberGrammar covers the shapes JSON's grammar allows a number to +// take. A number opens with a minus or a digit, an integer part of more than +// one digit does not open with a zero, and a decimal point has digits on both +// sides of it. MySQL holds documents to the same grammar, and nan is not a +// number to either of them. +func TestParseNumberGrammar(t *testing.T) { + t.Run("accepted", func(t *testing.T) { + for _, doc := range []string{ + "0", "-0", "0.5", "-0.5", "1", "-1", "1.2", "0e0", "-0e0", + "1e5", "1E5", "1e007", "1e+007", "1e-5", "0.0", + "[0,1,2]", `{"a":-0.5,"b":[1e5]}`, + } { + t.Run(doc, func(t *testing.T) { + var p Parser + _, err := p.Parse(doc) + require.NoError(t, err) + }) + } }) + + t.Run("rejected", func(t *testing.T) { + for _, doc := range []string{ + // An integer part that opens with a zero. + "007", "-003", "01", "00", "00.5", "01.5", "[007]", + // A decimal point missing a digit on one side. + ".2", "-.2", "12.", "-12.", "1.e5", `{"a": .2}`, "[12.]", + // A written plus. + "+1", "+1.5", "+0", "[+1]", + // Not a number at all. + "nan", "NaN", "NAN", "[nan]", `{"a": nan}`, "-nan", ".", "-", + } { + t.Run(doc, func(t *testing.T) { + var p Parser + _, err := p.Parse(doc) + require.Error(t, err) + }) + } + }) +} + +// parseNumberType is the conversion-based answer to what a number is kept as, +// which numberKind reaches by shape alone. It is the oracle for the test below +// and nothing else reads it. +func parseNumberType(ns string) NumberType { + _, err := fastparse.ParseInt64(ns, 10) + if err == nil { + return NumberTypeSigned + } + _, err = fastparse.ParseUint64(ns, 10) + if err == nil { + return NumberTypeUnsigned + } + _, err = fastparse.ParseFloat64(ns) + if err == nil { + return NumberTypeFloat + } + return NumberTypeUnknown +} + +// TestNumberKindMatchesParsing is the safety net under deciding a number's kind +// from its shape: the shape rule exists to avoid the conversions, so it has to +// reach the same answer they would. +func TestNumberKindMatchesParsing(t *testing.T) { + var spellings []string + for digits := 1; digits <= 25; digits++ { + for _, lead := range []string{"1", "9"} { + run := lead + strings.Repeat("7", digits-1) + spellings = append(spellings, run, "-"+run, "0"+run, "-0"+run, strings.Repeat("0", digits)+run) + } + } + // The values either side of every limit the rule has to respect. + spellings = append(spellings, + "0", "-0", "00", "9223372036854775806", "9223372036854775807", "9223372036854775808", + "-9223372036854775807", "-9223372036854775808", "-9223372036854775809", + "18446744073709551614", "18446744073709551615", "18446744073709551616", + ) + + for _, spelling := range spellings { + t.Run(spelling, func(t *testing.T) { + want := parseNumberType(spelling) + got := numberKind(spelling, false) + if want == NumberTypeUnknown { + // Too long for any of them to hold; the parser rejects it as a + // document, and until then it is a double like any other. + require.Equal(t, NumberTypeFloat, got) + return + } + require.Equalf(t, want, got, "%q", spelling) + }) + } +} + +// TestParseSettlesValues checks that parsing leaves nothing for a reader to +// work out later. A parsed document is shared by every goroutine running a +// cached plan, so a read that rewrites the value it read is a data race. +func TestParseSettlesValues(t *testing.T) { + var p Parser + v, err := p.Parse(`{"k": "a\u0062", "n": [1, 2.5, 3e4, 18446744073709551615], "s": "plain"}`) + require.NoError(t, err) + + var walk func(*Value) + walk = func(v *Value) { + switch v.t { + case TypeArray: + for _, elem := range v.a { + walk(elem) + } + case TypeObject: + for _, kv := range v.o.kvs { + walk(kv.v) + } + case TypeString: + require.NotContains(t, v.s, `\u`, "string still carries an escape") + case TypeNumber: + require.NotEqual(t, NumberTypeUnknown, v.n, "number kind left undecided") + } + // Reading a value must not change it. + before := *v + require.Equal(t, before.t, v.Type()) + require.Equal(t, before.n, v.NumberType()) + require.Equal(t, before.s, v.s) + } + walk(v) +} + +// TestParseUnescapesStrings pins that a string reads and renders the same +// whether or not anything has looked at it, which is what MySQL does: it +// resolves an escape when it parses the document, so \u0061 is stored and +// printed as a. +func TestParseUnescapesStrings(t *testing.T) { + for doc, want := range map[string]string{ + `"\u0061"`: `"a"`, + `["\u0061"]`: `["a"]`, + `"\u00e9"`: `"é"`, + `"a\tb"`: `"a\tb"`, + `"plain"`: `"plain"`, + } { + t.Run(doc, func(t *testing.T) { + var p Parser + v, err := p.Parse(doc) + require.NoError(t, err) + require.Equal(t, want, v.String()) + }) + } +} + +// TestParseConcurrentReads is the regression test for the races this settling +// removes: one parsed document, read by several goroutines at once, which is +// how a folded JSON literal in a cached plan is used. +func TestParseConcurrentReads(t *testing.T) { + for _, doc := range []string{ + `["a", "b", "c", "d"]`, + `["\u0061", "\u0062"]`, + `[1, 2.5, 3e4, 18446744073709551615]`, + `{"ka": "vb", "n": [1, 2.5, "x"]}`, + `2.5`, + `"a"`, + } { + t.Run(doc, func(t *testing.T) { + var p Parser + v, err := p.Parse(doc) + require.NoError(t, err) + + const readers = 8 + fingerprints := make([]vthash.Hash, readers) + renders := make([]string, readers) + + var start sync.WaitGroup + var readersDone sync.WaitGroup + start.Add(1) + for i := range readers { + readersDone.Go(func() { + start.Wait() + h := vthash.New() + v.Hash(&h) + fingerprints[i] = h.Sum128() + renders[i] = v.String() + }) + } + start.Done() + readersDone.Wait() + + for i := 1; i < readers; i++ { + require.Equal(t, fingerprints[0], fingerprints[i]) + require.Equal(t, renders[0], renders[i]) + } + }) + } } func TestUnescapeStringBestEffort(t *testing.T) {