diff --git a/builtin/char.mbt b/builtin/char.mbt index 1fa6dfc2f..fe75806d2 100644 --- a/builtin/char.mbt +++ b/builtin/char.mbt @@ -136,7 +136,8 @@ pub fn Char::is_ascii_uppercase(self : Self) -> Bool { /// Checks if the value is an ASCII whitespace character: /// U+0020 SPACE, U+0009 HORIZONTAL TAB, U+000A LINE FEED, U+000B VERTICAL TAB, U+000C FORM FEED, or U+000D CARRIAGE RETURN. pub fn Char::is_ascii_whitespace(self : Self) -> Bool { - self is ('\u{20}' | '\u{09}' | '\u{0A}' | '\u{0B}' | '\u{0C}' | '\u{0D}') + let code = self.to_int() + code <= 0x20 && (code == 0x20 || (code >= 0x09 && code <= 0x0D)) } ///| @@ -164,26 +165,36 @@ pub fn Char::is_digit(self : Self, radix : UInt) -> Bool { ///| /// Returns true if this char has the White_Space property. pub fn Char::is_whitespace(self : Self) -> Bool { - self - is ('\u0009'..='\u000D' - | '\u0020' - | '\u0085' - | '\u00A0' - | '\u1680' - | '\u2000'..='\u200A' - | '\u2028' - | '\u2029' - | '\u202F' - | '\u205F' - | '\u3000') + let code = self.to_int() + if code <= 0x20 { + return code == 0x20 || (code >= 0x09 && code <= 0x0D) + } + if code < 0x85 || code > 0x3000 { + return false + } + code == 0x85 || + code == 0xA0 || + code == 0x1680 || + (code >= 0x2000 && code <= 0x200A) || + code == 0x2028 || + code == 0x2029 || + code == 0x202F || + code == 0x205F || + code == 0x3000 } ///| /// Returns true if this char has one of the general categories for numbers. pub fn Char::is_numeric(self : Self) -> Bool { + let code = self.to_int() + if code <= 0x39 { + return code >= 0x30 + } + if code < 0xB2 { + return false + } self - is ('\u0030'..='\u0039' - | '\u00B2' + is ('\u00B2' | '\u00B3' | '\u00B9' | '\u00BC' diff --git a/builtin/char_predicate_bench_test.mbt b/builtin/char_predicate_bench_test.mbt new file mode 100644 index 000000000..8d7ca8eac --- /dev/null +++ b/builtin/char_predicate_bench_test.mbt @@ -0,0 +1,103 @@ +// Copyright 2026 International Digital Economy Academy +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +///| +let char_predicate_bench_rounds = 1000 + +///| +fn char_predicate_ascii_text() -> Array[Char] { + [ + 'l', 'e', 't', ' ', 'n', 'a', 'm', 'e', ' ', '=', ' ', '"', 'M', 'o', 'o', 'n', + 'B', 'i', 't', '"', '\n', 'i', 'f', ' ', 'n', 'a', 'm', 'e', '.', 'l', 'e', 'n', + 'g', 't', 'h', '(', ')', ' ', '>', ' ', '0', ' ', '{', '\n', ' ', ' ', 'p', 'r', + 'i', 'n', 't', 'l', 'n', '(', 'n', 'a', 'm', 'e', ')', '\n', '}', '\n', + ] +} + +///| +fn char_predicate_unicode_text() -> Array[Char] { + [ + '月', ' ', 'B', 'i', 't', '\n', '語', '\u{00A0}', '٣', '۶', '¾', '①', + '中', '\u{2003}', 'A', '9', '藏', '\u{2029}', '😀', '\u{3000}', 'z', '0', + ] +} + +///| +fn count_ascii_whitespace(data : Array[Char]) -> Int { + let mut count = 0 + for _ in 0.. Int { + let mut count = 0 + for _ in 0.. Int { + let mut count = 0 + for _ in 0.. BytesView { } bytes } + +///| +fn utf8_ascii8_bits(src : Bytes, offset : Int) -> Int { + src.unsafe_get(offset).to_int() | + src.unsafe_get(offset + 1).to_int() | + src.unsafe_get(offset + 2).to_int() | + src.unsafe_get(offset + 3).to_int() | + src.unsafe_get(offset + 4).to_int() | + src.unsafe_get(offset + 5).to_int() | + src.unsafe_get(offset + 6).to_int() | + src.unsafe_get(offset + 7).to_int() +} + +///| +fn utf8_find_malformed(src : Bytes, src_offset : Int, src_length : Int) -> Int { + let end = src_offset + src_length + let mut i = src_offset + while i + 8 <= end && + src.unsafe_get(i).to_int() < 0x80 && + (utf8_ascii8_bits(src, i) & 0x80) == 0 { + i += 8 + } + while i < end { + let b = src.unsafe_get(i).to_int() + if b < 0x80 { + i += 1 + } else if b < 0xC2 { + return i - src_offset + } else if b < 0xE0 { + if i + 1 >= end { + return i - src_offset + } + let c1 = src.unsafe_get(i + 1).to_int() + if c1 < 0x80 || c1 > 0xBF { + return i - src_offset + } + i += 2 + } else if b < 0xF0 { + if i + 2 >= end { + return i - src_offset + } + let c1 = src.unsafe_get(i + 1).to_int() + let c2 = src.unsafe_get(i + 2).to_int() + let lo = if b == 0xE0 { 0xA0 } else { 0x80 } + let hi = if b == 0xED { 0x9F } else { 0xBF } + if c1 < lo || c1 > hi || c2 < 0x80 || c2 > 0xBF { + return i - src_offset + } + i += 3 + } else if b < 0xF5 { + if i + 3 >= end { + return i - src_offset + } + let c1 = src.unsafe_get(i + 1).to_int() + let c2 = src.unsafe_get(i + 2).to_int() + let c3 = src.unsafe_get(i + 3).to_int() + let lo = if b == 0xF0 { 0x90 } else { 0x80 } + let hi = if b == 0xF4 { 0x8F } else { 0xBF } + if c1 < lo || c1 > hi || c2 < 0x80 || c2 > 0xBF || c3 < 0x80 || c3 > 0xBF { + return i - src_offset + } + i += 4 + } else { + return i - src_offset + } + } + -1 +} diff --git a/encoding/utf8/decode_js.mbt b/encoding/utf8/decode_js.mbt index 1be509d74..40a12daf4 100644 --- a/encoding/utf8/decode_js.mbt +++ b/encoding/utf8/decode_js.mbt @@ -50,37 +50,6 @@ extern "js" fn decode_utf8_lossy_js( #| new TextDecoder("utf-8", { ignoreBOM: false }), #| ) -///| -fn utf8_find_malformed(src : Bytes, src_offset : Int, src_length : Int) -> Int { - let view = src[src_offset:src_offset + src_length] - for bytes = view { - match bytes { - [] => break -1 - [ - _..=0x7F, - _..=0x7F, - _..=0x7F, - _..=0x7F, - _..=0x7F, - _..=0x7F, - _..=0x7F, - _..=0x7F, - .. rest, - ] => continue rest - [0..=0x7F, .. rest] => continue rest - [0xC2..=0xDF, 0x80..=0xBF, .. rest] => continue rest - [0xE0, 0xA0..=0xBF, 0x80..=0xBF, .. rest] - | [0xE1..=0xEC, 0x80..=0xBF, 0x80..=0xBF, .. rest] - | [0xED, 0x80..=0x9F, 0x80..=0xBF, .. rest] - | [0xEE..=0xEF, 0x80..=0xBF, 0x80..=0xBF, .. rest] => continue rest - [0xF0, 0x90..=0xBF, 0x80..=0xBF, 0x80..=0xBF, .. rest] - | [0xF1..=0xF3, 0x80..=0xBF, 0x80..=0xBF, 0x80..=0xBF, .. rest] - | [0xF4, 0x80..=0x8F, 0x80..=0xBF, 0x80..=0xBF, .. rest] => continue rest - malformed => break malformed.start_offset() - src_offset - } - } -} - ///| fn strict_malformed_suffix(bytes : BytesView) -> BytesView { let input = bytes.data() diff --git a/encoding/utf8/decode_test.mbt b/encoding/utf8/decode_test.mbt index b85521e7e..0c3492496 100644 --- a/encoding/utf8/decode_test.mbt +++ b/encoding/utf8/decode_test.mbt @@ -117,6 +117,32 @@ test "decode sliced views" { inspect(@utf8.decode_lossy(b".A\x80\xFFB."[1:5]), content="A��B") } +///| +test "is_valid UTF-8" { + assert_true(@utf8.is_valid(b"")) + assert_true(@utf8.is_valid(b"hello")) + assert_true(@utf8.is_valid(b"\xc2\xa9")) + assert_true(@utf8.is_valid(b"\xe4\xb8\xad")) + assert_true(@utf8.is_valid(b"\xf0\x9f\x98\x80")) + assert_true(@utf8.is_valid(b".A\xc2\xa9\xe4\xb8\xad."[1:7])) + assert_false(@utf8.is_valid(b"\x80")) + assert_false(@utf8.is_valid(b"\xc2")) + assert_false(@utf8.is_valid(b"\xc0\x80")) + assert_false(@utf8.is_valid(b"\xe0\x80\x80")) + assert_false(@utf8.is_valid(b"\xed\xa0\x80")) + assert_false(@utf8.is_valid(b"\xf4\x90\x80\x80")) + assert_false(@utf8.is_valid(b"\xff")) +} + +///| +test "is_valid UTF-8 with bom" { + let bytes = b"\xef\xbb\xbfhello" + assert_true(@utf8.is_valid(bytes)) + assert_true(@utf8.is_valid(bytes, ignore_bom=true)) + assert_false(@utf8.is_valid(b"\xef\xbb\xbf\xff")) + assert_false(@utf8.is_valid(b"\xef\xbb\xbf\xff", ignore_bom=true)) +} + ///| test "decode malformed sliced views" { try { diff --git a/encoding/utf8/is_valid_bench_test.mbt b/encoding/utf8/is_valid_bench_test.mbt new file mode 100644 index 000000000..9d1f1acda --- /dev/null +++ b/encoding/utf8/is_valid_bench_test.mbt @@ -0,0 +1,59 @@ +// Copyright 2026 International Digital Economy Academy +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +///| +let utf8_is_valid_bench_size = 4096 + +///| +fn utf8_is_valid_ascii_bytes() -> Bytes { + Bytes::makei(utf8_is_valid_bench_size, i => (0x20 + i % 0x5F).to_byte()) +} + +///| +fn utf8_is_valid_mixed_bytes() -> Bytes { + Bytes::makei(utf8_is_valid_bench_size, i => { + match i % 8 { + 0 => b'\xC3' + 1 => b'\xA9' + 2 => b'\xE4' + 3 => b'\xB8' + 4 => b'\xAD' + _ => (0x41 + i % 26).to_byte() + } + }) +} + +///| +test "bench utf8.is_valid ascii n=4096" (it : @bench.T) { + let bytes = utf8_is_valid_ascii_bytes() + it.bench(fn() { it.keep(@utf8.is_valid(bytes)) }) +} + +///| +test "bench utf8.decode ascii n=4096" (it : @bench.T) { + let bytes = utf8_is_valid_ascii_bytes() + it.bench(fn() { it.keep(try! @utf8.decode(bytes)) }) +} + +///| +test "bench utf8.is_valid mixed n=4096" (it : @bench.T) { + let bytes = utf8_is_valid_mixed_bytes() + it.bench(fn() { it.keep(@utf8.is_valid(bytes)) }) +} + +///| +test "bench utf8.decode mixed n=4096" (it : @bench.T) { + let bytes = utf8_is_valid_mixed_bytes() + it.bench(fn() { it.keep(try! @utf8.decode(bytes)) }) +} diff --git a/encoding/utf8/is_valid_js.mbt b/encoding/utf8/is_valid_js.mbt new file mode 100644 index 000000000..b335d89cc --- /dev/null +++ b/encoding/utf8/is_valid_js.mbt @@ -0,0 +1,46 @@ +// Copyright 2026 International Digital Economy Academy +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +///| +extern "js" fn is_valid_utf8_js( + bytes : Bytes, + start : Int, + len : Int, + preserve_bom : Bool, +) -> Bool = + #| ((preserveBOMDecoder, dropBOMDecoder) => function(bytes, start, len, preserveBOM) { + #| try { + #| const end = start + len; + #| const slice = bytes.subarray(start, end); + #| const decoder = preserveBOM ? preserveBOMDecoder : dropBOMDecoder; + #| decoder.decode(slice); + #| return true; + #| } catch (_) { + #| return false; + #| } + #| })( + #| new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }), + #| new TextDecoder("utf-8", { fatal: true, ignoreBOM: false }), + #| ) + +///| +/// Returns true if the input bytes are well-formed UTF-8. +pub fn is_valid(bytes : BytesView, ignore_bom? : Bool = false) -> Bool { + is_valid_utf8_js( + bytes.data(), + bytes.start_offset(), + bytes.length(), + !ignore_bom, + ) +} diff --git a/encoding/utf8/is_valid_nonjs.mbt b/encoding/utf8/is_valid_nonjs.mbt new file mode 100644 index 000000000..390586da8 --- /dev/null +++ b/encoding/utf8/is_valid_nonjs.mbt @@ -0,0 +1,20 @@ +// Copyright 2026 International Digital Economy Academy +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +///| +/// Returns true if the input bytes are well-formed UTF-8. +pub fn is_valid(bytes : BytesView, ignore_bom? : Bool = false) -> Bool { + let bytes = drop_utf8_bom(bytes, ignore_bom) + utf8_find_malformed(bytes.data(), bytes.start_offset(), bytes.length()) < 0 +} diff --git a/encoding/utf8/moon.pkg b/encoding/utf8/moon.pkg index 9278885d2..e7d18be0c 100644 --- a/encoding/utf8/moon.pkg +++ b/encoding/utf8/moon.pkg @@ -5,6 +5,7 @@ import { import { "moonbitlang/core/buffer", + "moonbitlang/core/bench", } for "test" options( @@ -13,5 +14,7 @@ options( "decode_nonjs.mbt": [ "not", "js" ], "encode_js.mbt": [ "js" ], "encode_nonjs.mbt": [ "not", "js" ], + "is_valid_js.mbt": [ "js" ], + "is_valid_nonjs.mbt": [ "not", "js" ], }, ) diff --git a/encoding/utf8/pkg.generated.mbti b/encoding/utf8/pkg.generated.mbti index cdbdb3cfa..ea977c832 100644 --- a/encoding/utf8/pkg.generated.mbti +++ b/encoding/utf8/pkg.generated.mbti @@ -12,6 +12,8 @@ pub fn decode_lossy(BytesView, ignore_bom? : Bool) -> String pub fn encode(StringView, bom? : Bool) -> Bytes +pub fn is_valid(BytesView, ignore_bom? : Bool) -> Bool + // Errors pub suberror Malformed { Malformed(BytesView)