decimal: read numeric text the way MySQL does

go/mysql/collations/charset/charset_test.go

@@ -20,6 +20,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestIsMultibyteByName(t *testing.T) {
@@ -106,3 +107,64 @@ func TestIsBackslashSafe(t *testing.T) {
 		})
 	}
 }
+
+// TestDecodeRuneAlwaysAdvances pins the DecodeRune contract that callers
+// like Convert, Expand and Length rely on to make progress: on non-empty
+// input, the reported width is at least 1 and never past the end of the
+// input, whether the leading sequence is valid or not. A decoder reporting
+// width 0 loops those callers forever on bytes a user can supply.
+func TestDecodeRuneAlwaysAdvances(t *testing.T) {
+	charsets := []Charset{
+		Charset_binary{},
+		Charset_latin1{},
+		Charset_utf8mb3{},
+		Charset_utf8mb4{},
+		Charset_utf16{},
+		Charset_utf16le{},
+		Charset_ucs2{},
+		Charset_utf32{},
+		Charset_gb18030{},
+		Charset_gb2312{},
+		Charset_ujis{},
+		Charset_sjis{},
+		Charset_cp932{},
+		Charset_eucjpms{},
+		Charset_euckr{},
+	}
+
+	var inputs [][]byte
+	for b := range 256 {
+		inputs = append(inputs, []byte{byte(b)})
+	}
+	for hi := range 256 {
+		for lo := range 256 {
+			inputs = append(inputs, []byte{byte(hi), byte(lo)})
+		}
+	}
+	// Three and four byte tails behind the byte values that lead longer
+	// sequences somewhere: UTF-16 surrogates, UTF-8 continuations and
+	// multibyte lead bytes.
+	leads := []byte{0x00, 0x31, 0x81, 0x8E, 0x8F, 0xA1, 0xC2, 0xD8, 0xDB, 0xDC, 0xDF, 0xE0, 0xED, 0xF0, 0xFF}
+	tails := []byte{0x00, 0x31, 0x80, 0xA0, 0xD8, 0xDC, 0xFF}
+	for _, l := range leads {
+		for _, m := range tails {
+			for _, e := range tails {
+				inputs = append(inputs, []byte{l, m, e})
+				inputs = append(inputs, []byte{l, m, e, 0x31})
+				inputs = append(inputs, []byte{0x31, l, m, e})
+			}
+		}
+	}
+
+	for _, cs := range charsets {
+		t.Run(cs.Name(), func(t *testing.T) {
+			for _, in := range inputs {
+				_, width := cs.DecodeRune(in)
+				if width < 1 || width > len(in) {
+					require.Failf(t, "DecodeRune width out of range",
+						"%s.DecodeRune(%#v) returned width %d, want 1..%d", cs.Name(), in, width, len(in))
+				}
+			}
+		})
+	}
+}

go/mysql/collations/charset/convert_test.go

@@ -158,6 +158,32 @@ func TestConvert(t *testing.T) {
 			dstCharset: &testCharset2{},
 			want:       []byte("😊😂🤢"),
 		},
+		// A lone surrogate is not a character; it converts to '?' and the
+		// conversion must terminate rather than stall on the invalid unit.
+		{
+			src:        []byte{0xD8, 0x00},
+			srcCharset: Charset_utf16{},
+			dst:        nil,
+			dstCharset: Charset_latin1{},
+			want:       []byte("?"),
+			err:        "Cannot convert string",
+		},
+		{
+			src:        []byte{0x00, 0xD8},
+			srcCharset: Charset_utf16le{},
+			dst:        nil,
+			dstCharset: Charset_latin1{},
+			want:       []byte("?"),
+			err:        "Cannot convert string",
+		},
+		{
+			src:        []byte{0xD8, 0x00, 0x00, 0x31},
+			srcCharset: Charset_utf16{},
+			dst:        nil,
+			dstCharset: Charset_utf8mb4{},
+			want:       []byte("?\x00?"),
+			err:        "Cannot convert string",
+		},
 	}
 
 	for _, tc := range testCases {

go/mysql/collations/charset/japanese/sjis.go

@@ -90,11 +90,15 @@ func decodeSJIS(src []byte, table *[65536]uint16) (rune, int) {
 	if c0 >= 0xA1 && c0 <= 0xDF {
 		return rune(table[c0]), 1
 	}
-	if len(src) >= 2 {
-		sj := uint16(c0)<<8 | uint16(src[1])
-		if cp := table[sj]; cp != 0 {
-			return rune(cp), 2
-		}
+	if len(src) < 2 {
+		// A lead byte with no trail byte left is invalid on its own;
+		// reporting a single-byte width keeps callers from stepping past
+		// the end of the input.
+		return utf8.RuneError, 1
+	}
+	sj := uint16(c0)<<8 | uint16(src[1])
+	if cp := table[sj]; cp != 0 {
+		return rune(cp), 2
 	}
 	return utf8.RuneError, 2
 }

go/mysql/collations/charset/unicode/utf16.go

@@ -76,7 +76,11 @@ func (Charset_utf16be) DecodeRune(b []byte) (rune, int) {
 	}
 
 	if len(b) < 4 {
-		return utf8.RuneError, 0
+		// A surrogate code unit without its pair is invalid, reported with
+		// its own two-byte width so that callers walking the input always
+		// advance. Callers that stop at invalid input still stop, as they
+		// treat any RuneError with a width below 3 as such.
+		return utf8.RuneError, 2
 	}
 
 	r2 := uint16(b[3]) | uint16(b[2])<<8
@@ -138,7 +142,11 @@ func (Charset_utf16le) DecodeRune(b []byte) (rune, int) {
 	}
 
 	if len(b) < 4 {
-		return utf8.RuneError, 0
+		// A surrogate code unit without its pair is invalid, reported with
+		// its own two-byte width so that callers walking the input always
+		// advance. Callers that stop at invalid input still stop, as they
+		// treat any RuneError with a width below 3 as such.
+		return utf8.RuneError, 2
 	}
 
 	r2 := uint16(b[2]) | uint16(b[3])<<8

go/mysql/decimal/decimal_test.go

@@ -92,6 +92,12 @@ var testTableScientificNotation = map[string]string{
 	"123.456e0":  "123.456",
 	"123.456e2":  "12345.6",
 	"123.456e10": "1234560000000",
+	// MySQL accepts a written plus on the exponent.
+	"1e+9":       "1000000000",
+	"1E+9":       "1000000000",
+	"245E+3":     "245000",
+	"123.456e+2": "12345.6",
+	"0e+5":       "0",
 }
 
 func init() {
@@ -192,6 +198,127 @@ func TestNewFromString(t *testing.T) {
 	}
 }
 
+// TestNewFromStringLeadingPlus covers a sign the negating entries in
+// testTableScientificNotation cannot: MySQL reads a leading plus as the number
+// it introduces, so CAST('+1' AS DECIMAL) is 1.
+func TestNewFromStringLeadingPlus(t *testing.T) {
+	for in, want := range map[string]string{
+		"+1":          "1",
+		"+1.5":        "1.5",
+		"+0":          "0",
+		"+1e9":        "1000000000",
+		"+1e+9":       "1000000000",
+		"+123.456e-2": "1.23456",
+		// The number starts after any leading whitespace, and so does the sign.
+		" +1":     "1",
+		" -1":     "-1",
+		"  +1.5":  "1.5",
+		" -1e+5":  "-100000",
+		"\t-2":    "-2",
+		" 1.5e+3": "1500",
+	} {
+		t.Run(in, func(t *testing.T) {
+			d, err := NewFromString(in)
+			require.NoError(t, err)
+			require.Equal(t, want, d.String())
+		})
+	}
+}
+
+// TestNewFromStringWhitespace covers the whitespace MySQL skips around numeric
+// text: a vertical tab, a form feed and a 0xA0 are leading and trailing space
+// like a blank or a tab, and a blank or a tab may sit between the exponent
+// marker and the exponent it introduces. MySQL 8.0.46 reads every spelling
+// here to the same value.
+func TestNewFromStringWhitespace(t *testing.T) {
+	for in, want := range map[string]string{
+		"\v+1":       "1",
+		"\f-1":       "-1",
+		"\v1":        "1",
+		"1\v":        "1",
+		"1\f":        "1",
+		"\v\f\n\r 1": "1",
+		// 0xA0 is a non-breaking space in latin1, and the reader reads a string
+		// through latin1's character table whatever its own charset is.
+		"\xa01":        "1",
+		"1\xa0":        "1",
+		"\xa0-1":       "-1",
+		"\xa0\t \xa01": "1",
+		"1e +5":        "100000",
+		"1e -5":        "0.00001",
+		"1e\t-5":       "0.00001",
+		"1e \t+5":      "100000",
+		"1e 5":         "100000",
+		"1e  5":        "100000",
+		"1e\t+5":       "100000",
+		"1E   -5":      "0.00001",
+		"1.5e 2":       "150",
+		".5e +3":       "500",
+		"-1.5e +3":     "-1500",
+		" 1e 2 ":       "100",
+	} {
+		t.Run(in, func(t *testing.T) {
+			d, err := NewFromString(in)
+			require.NoError(t, err)
+			require.Equal(t, want, d.String())
+		})
+	}
+}
+
+// TestNewFromStringWhitespaceBoundary pins the spellings that stop short of a
+// number MySQL would read whole. Each one keeps the mantissa parsed so far as
+// its value and reports the string as invalid, so a caller that ignores the
+// error lands on the same partial value MySQL truncates to.
+func TestNewFromStringWhitespaceBoundary(t *testing.T) {
+	for in, want := range map[string]string{
+		// Whitespace belongs before the exponent's sign, not after it.
+		"1e+ 5":  "1",
+		"1e + 5": "1",
+		"1e +":   "1",
+		"1e +x":  "1",
+		// The exponent marker itself has to follow the mantissa directly.
+		"1 e+5": "1",
+		// Only a blank or a tab is skipped after the marker.
+		"1e\v5":   "1",
+		"1e\f5":   "1",
+		"1e\n5":   "1",
+		"1e \v5":  "1",
+		"1e\xa05": "1",
+		// Whitespace after the marker still needs an exponent behind it.
+		"1e ": "1",
+		// One sign, not two.
+		"1e --5": "1",
+		"1e -+5": "1",
+		// A sign introduces a number, so whitespace cannot follow it either.
+		"+ 1": "0",
+	} {
+		t.Run(in, func(t *testing.T) {
+			d, err := NewFromString(in)
+			require.ErrorContains(t, err, "invalid decimal string")
+			require.Equal(t, want, d.String())
+		})
+	}
+}
+
+// TestNewFromStringZeroExponent pins the formatting of a zero written with a
+// positive exponent. String trims the padding away, but FormatMySQL keeps the
+// scale it is asked for, so a stale exponent surfaces as leading zeros.
+func TestNewFromStringZeroExponent(t *testing.T) {
+	for _, in := range []string{"0e5", "-0e5", "+0e5", "0E+5", "0e2", "0"} {
+		t.Run(in, func(t *testing.T) {
+			d, err := NewFromString(in)
+			require.NoError(t, err)
+			require.Equal(t, "0", d.String())
+			require.Equal(t, "0.000000", string(d.FormatMySQL(6)))
+		})
+	}
+
+	// A zero written to a scale keeps it: only a positive exponent is dropped.
+	d, err := NewFromString("0.00")
+	require.NoError(t, err)
+	require.Equal(t, int32(-2), d.Exponent())
+}
+
 func TestFloat64(t *testing.T) {
 	t.Skipf("Float64 does not check for exact")