diff --git a/json/lex_string.mbt b/json/lex_string.mbt index fbaa8d387..e4c2c9921 100644 --- a/json/lex_string.mbt +++ b/json/lex_string.mbt @@ -17,7 +17,10 @@ fn ParseContext::lex_string(ctx : ParseContext) -> String raise ParseError { let string_start = ctx.offset // Fast path for ordinary strings: scan raw UTF-16 code units and materialize // the slice directly when there are no escapes or control characters. - for i in string_start..= ctx.end_offset { + break + } let c = ctx.input.unsafe_get(i) if c == '"' { ctx.offset = i + 1 @@ -29,7 +32,23 @@ fn ParseContext::lex_string(ctx : ParseContext) -> String raise ParseError { // \t) are invalid inside a JSON string. ctx.offset = i + 1 ctx.invalid_char(shift=-1) + } else if c.is_leading_surrogate() { + if i + 1 < ctx.end_offset && + ctx.input.unsafe_get(i + 1).is_trailing_surrogate() { + continue i + 2 + } + // MoonBit strings stay Unicode well-formed, so a raw unpaired + // surrogate must be rejected rather than smuggled into the parsed + // string. (Well-formed input cannot contain one, but unsafe code can + // manufacture such a String; a clean error beats undefined behavior.) + ctx.offset = i + 1 + ctx.invalid_char(shift=-1) + } else if c.is_trailing_surrogate() { + // A bare trailing surrogate can never start a surrogate pair. + ctx.offset = i + 1 + ctx.invalid_char(shift=-1) } + continue i + 1 } raise InvalidEof } @@ -39,8 +58,14 @@ fn ParseContext::lex_string_slow(ctx : ParseContext) -> String raise ParseError let buf = StringBuilder() let mut start = ctx.offset fn flush(end : Int) { - if start > 0 && end > start { - buf.write_view(ctx.input[start:end]) + if end > start { + // `view(start_offset~, end_offset~)` only bounds-checks. The checked + // `ctx.input[start:end]` would abort on a trailing surrogate at a + // slice boundary, which a raw lone surrogate inside the string could + // place there; unpaired surrogates now raise a ParseError before any + // flush spans them, and the unchecked slice keeps this loop's + // totality independent of that validation order. + buf.write_view(ctx.input.view(start_offset=start, end_offset=end)) } } @@ -70,12 +95,20 @@ fn ParseContext::lex_string_slow(ctx : ParseContext) -> String raise ParseError } start = ctx.offset } - Some(ch) => - if ch.to_int() < 32 { + Some(ch) => { + let code = ch.to_int() + if code < 32 { + ctx.invalid_char(shift=-1) + } else if code is (0xD800..=0xDFFF) { + // `read_char` only yields a surrogate-range value when the raw + // code unit is unpaired; keep parsed strings Unicode well-formed + // by rejecting it (previously this aborted the process when a + // later flush sliced across the surrogate). ctx.invalid_char(shift=-1) } else { continue } + } None => raise InvalidEof } } diff --git a/json/lex_string_test.mbt b/json/lex_string_test.mbt index 31303069c..45b0fa510 100644 --- a/json/lex_string_test.mbt +++ b/json/lex_string_test.mbt @@ -89,3 +89,37 @@ test "lex_hex_digits accepts all hex digit ranges" { ), ) } + +///| +/// Regression for #4049. MoonBit strings must stay Unicode well-formed, so +/// the string lexer rejects raw unpaired surrogates with a ParseError. +/// Previously the escape-free fast path silently accepted them (producing +/// ill-formed strings), and combining one with an escape sequence aborted +/// the process: the slow path's `flush` sliced with the checked +/// `[start:end]`, which panics when a slice boundary lands on a trailing +/// surrogate. Well-formed input cannot contain raw lone surrogates, but +/// unsafe code can manufacture such a String; a clean error beats an abort. +test "raw unpaired surrogates are rejected with a clean parse error" { + let lone_high = String::from_array([(0xD800).unsafe_to_char()]) + let lone_low = String::from_array([(0xDC00).unsafe_to_char()]) + // Escape-free strings (fast path; used to be silently accepted). + assert_false(@json.valid("\"" + lone_high + "\"")) + assert_false(@json.valid("\"" + lone_low + "\"")) + // A reversed pair (low then high) is two unpaired surrogates. + assert_false(@json.valid("\"" + lone_low + lone_high + "\"")) + // Next to escapes (slow path; used to abort the process). + assert_false(@json.valid("\"" + lone_low + "\\n\"")) + assert_false(@json.valid("\"\\n" + lone_low + "\"")) + assert_false(@json.valid("\"" + lone_high + "\\t\"")) + // The failure is the documented ParseError, not an abort. + debug_inspect( + expect_parse_error("\"" + lone_low + "\"", "expected InvalidChar"), + content=( + #|InvalidChar({ line: 1, column: 1 }, '�') + ), + ) + // Well-formed surrogate pairs still parse, raw or next to escapes. + assert_true(@json.parse("\"\u{1F600}\"") == Json::string("\u{1F600}")) + let json = Json::string("\u{10FFFF}\\\u{1F600}\n") + assert_true(@json.parse(json.stringify()) == json) +}