Optimize Unicode predicate checks

builtin/char.mbt

@@ -136,7 +136,8 @@ pub fn Char::is_ascii_uppercase(self : Self) -> Bool {
 /// Checks if the value is an ASCII whitespace character:
 /// U+0020 SPACE, U+0009 HORIZONTAL TAB, U+000A LINE FEED, U+000B VERTICAL TAB, U+000C FORM FEED, or U+000D CARRIAGE RETURN.
 pub fn Char::is_ascii_whitespace(self : Self) -> Bool {
-  self is ('\u{20}' | '\u{09}' | '\u{0A}' | '\u{0B}' | '\u{0C}' | '\u{0D}')
+  let code = self.to_int()
+  code <= 0x20 && (code == 0x20 || (code >= 0x09 && code <= 0x0D))
 }
 
 ///|
@@ -164,26 +165,36 @@ pub fn Char::is_digit(self : Self, radix : UInt) -> Bool {
 ///|
 /// Returns true if this char has the White_Space property.
 pub fn Char::is_whitespace(self : Self) -> Bool {
-  self
-  is ('\u0009'..='\u000D'
-  | '\u0020'
-  | '\u0085'
-  | '\u00A0'
-  | '\u1680'
-  | '\u2000'..='\u200A'
-  | '\u2028'
-  | '\u2029'
-  | '\u202F'
-  | '\u205F'
-  | '\u3000')
+  let code = self.to_int()
+  if code <= 0x20 {
+    return code == 0x20 || (code >= 0x09 && code <= 0x0D)
+  }
+  if code < 0x85 || code > 0x3000 {
+    return false
+  }
+  code == 0x85 ||
+  code == 0xA0 ||
+  code == 0x1680 ||
+  (code >= 0x2000 && code <= 0x200A) ||
+  code == 0x2028 ||
+  code == 0x2029 ||
+  code == 0x202F ||
+  code == 0x205F ||
+  code == 0x3000
 }
 
 ///|
 /// Returns true if this char has one of the general categories for numbers.
 pub fn Char::is_numeric(self : Self) -> Bool {
+  let code = self.to_int()
+  if code <= 0x39 {
+    return code >= 0x30
+  }
+  if code < 0xB2 {
+    return false
+  }
   self
-  is ('\u0030'..='\u0039'
-  | '\u00B2'
+  is ('\u00B2'
   | '\u00B3'
   | '\u00B9'
   | '\u00BC'

builtin/char_predicate_bench_test.mbt

@@ -0,0 +1,103 @@
+// Copyright 2026 International Digital Economy Academy
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+///|
+let char_predicate_bench_rounds = 1000
+
+///|
+fn char_predicate_ascii_text() -> Array[Char] {
+  [
+    'l', 'e', 't', ' ', 'n', 'a', 'm', 'e', ' ', '=', ' ', '"', 'M', 'o', 'o', 'n',
+    'B', 'i', 't', '"', '\n', 'i', 'f', ' ', 'n', 'a', 'm', 'e', '.', 'l', 'e', 'n',
+    'g', 't', 'h', '(', ')', ' ', '>', ' ', '0', ' ', '{', '\n', ' ', ' ', 'p', 'r',
+    'i', 'n', 't', 'l', 'n', '(', 'n', 'a', 'm', 'e', ')', '\n', '}', '\n',
+  ]
+}
+
+///|
+fn char_predicate_unicode_text() -> Array[Char] {
+  [
+    '月', ' ', 'B', 'i', 't', '\n', '語', '\u{00A0}', '٣', '۶', '¾', '①',
+    '中', '\u{2003}', 'A', '9', '藏', '\u{2029}', '😀', '\u{3000}', 'z', '0',
+  ]
+}
+
+///|
+fn count_ascii_whitespace(data : Array[Char]) -> Int {
+  let mut count = 0
+  for _ in 0..<char_predicate_bench_rounds {
+    for ch in data {
+      if ch.is_ascii_whitespace() {
+        count += 1
+      }
+    }
+  }
+  count
+}
+
+///|
+fn count_whitespace(data : Array[Char]) -> Int {
+  let mut count = 0
+  for _ in 0..<char_predicate_bench_rounds {
+    for ch in data {
+      if ch.is_whitespace() {
+        count += 1
+      }
+    }
+  }
+  count
+}
+
+///|
+fn count_numeric(data : Array[Char]) -> Int {
+  let mut count = 0
+  for _ in 0..<char_predicate_bench_rounds {
+    for ch in data {
+      if ch.is_numeric() {
+        count += 1
+      }
+    }
+  }
+  count
+}
+
+///|
+test "bench Char::is_ascii_whitespace ascii n=62000" (it : @bench.T) {
+  let data = char_predicate_ascii_text()
+  it.bench(fn() { it.keep(count_ascii_whitespace(data)) })
+}
+
+///|
+test "bench Char::is_whitespace ascii n=62000" (it : @bench.T) {
+  let data = char_predicate_ascii_text()
+  it.bench(fn() { it.keep(count_whitespace(data)) })
+}
+
+///|
+test "bench Char::is_whitespace unicode n=22000" (it : @bench.T) {
+  let data = char_predicate_unicode_text()
+  it.bench(fn() { it.keep(count_whitespace(data)) })
+}
+
+///|
+test "bench Char::is_numeric ascii n=62000" (it : @bench.T) {
+  let data = char_predicate_ascii_text()
+  it.bench(fn() { it.keep(count_numeric(data)) })
+}
+
+///|
+test "bench Char::is_numeric unicode n=22000" (it : @bench.T) {
+  let data = char_predicate_unicode_text()
+  it.bench(fn() { it.keep(count_numeric(data)) })
+}

char/char_test.mbt

@@ -484,6 +484,8 @@ test "Unicode edge cases for numeric detection" {
   assert_true('\u{1D7CE}'.is_numeric()) // Mathematical monospace digit zero
 
   // Test non-numeric characters that might be confused with numbers
+  assert_false('/'.is_numeric()) // Before ASCII digit range
+  assert_false(':'.is_numeric()) // After ASCII digit range
   assert_false('O'.is_numeric()) // Letter O (not zero)
   assert_false('l'.is_numeric()) // Letter l (not one)
   assert_false('I'.is_numeric()) // Letter I (not Roman numeral)
@@ -492,12 +494,15 @@ test "Unicode edge cases for numeric detection" {
 ///|
 test "Whitespace edge cases" {
   // Test all ASCII whitespace characters
+  assert_false('\u{0008}'.is_whitespace()) // Before TAB
   assert_true('\u{0009}'.is_ascii_whitespace()) // TAB
   assert_true('\u{000A}'.is_ascii_whitespace()) // LF
   assert_true('\u{000B}'.is_ascii_whitespace()) // VT
   assert_true('\u{000C}'.is_ascii_whitespace()) // FF
   assert_true('\u{000D}'.is_ascii_whitespace()) // CR
+  assert_false('\u{000E}'.is_whitespace()) // After CR
   assert_true('\u{0020}'.is_ascii_whitespace()) // SPACE
+  assert_false('\u{0021}'.is_whitespace()) // After SPACE
 
   // Test characters that are not ASCII whitespace
   assert_false('\u{00A0}'.is_ascii_whitespace()) // NBSP (Unicode whitespace but not ASCII)

encoding/utf8/README.mbt.md

@@ -35,6 +35,18 @@ test "decode" {
 }
 ```
 
+## Validation
+
+Use `is_valid` to check whether bytes are well-formed UTF-8 without decoding them.
+
+```mbt check
+///|
+test "is_valid" {
+  inspect(@utf8.is_valid(b"hi"), content="true")
+  inspect(@utf8.is_valid(b"\x80"), content="false")
+}
+```
+
 ## Lossy Decoding
 
 Use `decode_lossy` to decode bytes that may contain invalid UTF-8, replacing invalid sequences with the Unicode replacement character (U+FFFD).

encoding/utf8/decode.mbt

@@ -27,3 +27,71 @@ fn drop_utf8_bom(bytes : BytesView, ignore_bom : Bool) -> BytesView {
   }
   bytes
 }
+
+///|
+fn utf8_ascii8_bits(src : Bytes, offset : Int) -> Int {
+  src.unsafe_get(offset).to_int() |
+  src.unsafe_get(offset + 1).to_int() |
+  src.unsafe_get(offset + 2).to_int() |
+  src.unsafe_get(offset + 3).to_int() |
+  src.unsafe_get(offset + 4).to_int() |
+  src.unsafe_get(offset + 5).to_int() |
+  src.unsafe_get(offset + 6).to_int() |
+  src.unsafe_get(offset + 7).to_int()
+}
+
+///|
+fn utf8_find_malformed(src : Bytes, src_offset : Int, src_length : Int) -> Int {
+  let end = src_offset + src_length
+  let mut i = src_offset
+  while i + 8 <= end &&
+        src.unsafe_get(i).to_int() < 0x80 &&
+        (utf8_ascii8_bits(src, i) & 0x80) == 0 {
+    i += 8
+  }
+  while i < end {
+    let b = src.unsafe_get(i).to_int()
+    if b < 0x80 {
+      i += 1
+    } else if b < 0xC2 {
+      return i - src_offset
+    } else if b < 0xE0 {
+      if i + 1 >= end {
+        return i - src_offset
+      }
+      let c1 = src.unsafe_get(i + 1).to_int()
+      if c1 < 0x80 || c1 > 0xBF {
+        return i - src_offset
+      }
+      i += 2
+    } else if b < 0xF0 {
+      if i + 2 >= end {
+        return i - src_offset
+      }
+      let c1 = src.unsafe_get(i + 1).to_int()
+      let c2 = src.unsafe_get(i + 2).to_int()
+      let lo = if b == 0xE0 { 0xA0 } else { 0x80 }
+      let hi = if b == 0xED { 0x9F } else { 0xBF }
+      if c1 < lo || c1 > hi || c2 < 0x80 || c2 > 0xBF {
+        return i - src_offset
+      }
+      i += 3
+    } else if b < 0xF5 {
+      if i + 3 >= end {
+        return i - src_offset
+      }
+      let c1 = src.unsafe_get(i + 1).to_int()
+      let c2 = src.unsafe_get(i + 2).to_int()
+      let c3 = src.unsafe_get(i + 3).to_int()
+      let lo = if b == 0xF0 { 0x90 } else { 0x80 }
+      let hi = if b == 0xF4 { 0x8F } else { 0xBF }
+      if c1 < lo || c1 > hi || c2 < 0x80 || c2 > 0xBF || c3 < 0x80 || c3 > 0xBF {
+        return i - src_offset
+      }
+      i += 4
+    } else {
+      return i - src_offset
+    }
+  }
+  -1
+}

encoding/utf8/decode_js.mbt

@@ -50,37 +50,6 @@ extern "js" fn decode_utf8_lossy_js(
   #|   new TextDecoder("utf-8", { ignoreBOM: false }),
   #| )
 
-///|
-fn utf8_find_malformed(src : Bytes, src_offset : Int, src_length : Int) -> Int {
-  let view = src[src_offset:src_offset + src_length]
-  for bytes = view {
-    match bytes {
-      [] => break -1
-      [
-        _..=0x7F,
-        _..=0x7F,
-        _..=0x7F,
-        _..=0x7F,
-        _..=0x7F,
-        _..=0x7F,
-        _..=0x7F,
-        _..=0x7F,
-        .. rest,
-      ] => continue rest
-      [0..=0x7F, .. rest] => continue rest
-      [0xC2..=0xDF, 0x80..=0xBF, .. rest] => continue rest
-      [0xE0, 0xA0..=0xBF, 0x80..=0xBF, .. rest]
-      | [0xE1..=0xEC, 0x80..=0xBF, 0x80..=0xBF, .. rest]
-      | [0xED, 0x80..=0x9F, 0x80..=0xBF, .. rest]
-      | [0xEE..=0xEF, 0x80..=0xBF, 0x80..=0xBF, .. rest] => continue rest
-      [0xF0, 0x90..=0xBF, 0x80..=0xBF, 0x80..=0xBF, .. rest]
-      | [0xF1..=0xF3, 0x80..=0xBF, 0x80..=0xBF, 0x80..=0xBF, .. rest]
-      | [0xF4, 0x80..=0x8F, 0x80..=0xBF, 0x80..=0xBF, .. rest] => continue rest
-      malformed => break malformed.start_offset() - src_offset
-    }
-  }
-}
-
 ///|
 fn strict_malformed_suffix(bytes : BytesView) -> BytesView {
   let input = bytes.data()

encoding/utf8/decode_test.mbt

@@ -117,6 +117,32 @@ test "decode sliced views" {
   inspect(@utf8.decode_lossy(b".A\x80\xFFB."[1:5]), content="A��B")
 }
 
+///|
+test "is_valid UTF-8" {
+  assert_true(@utf8.is_valid(b""))
+  assert_true(@utf8.is_valid(b"hello"))
+  assert_true(@utf8.is_valid(b"\xc2\xa9"))
+  assert_true(@utf8.is_valid(b"\xe4\xb8\xad"))
+  assert_true(@utf8.is_valid(b"\xf0\x9f\x98\x80"))
+  assert_true(@utf8.is_valid(b".A\xc2\xa9\xe4\xb8\xad."[1:7]))
+  assert_false(@utf8.is_valid(b"\x80"))
+  assert_false(@utf8.is_valid(b"\xc2"))
+  assert_false(@utf8.is_valid(b"\xc0\x80"))
+  assert_false(@utf8.is_valid(b"\xe0\x80\x80"))
+  assert_false(@utf8.is_valid(b"\xed\xa0\x80"))
+  assert_false(@utf8.is_valid(b"\xf4\x90\x80\x80"))
+  assert_false(@utf8.is_valid(b"\xff"))
+}
+
+///|
+test "is_valid UTF-8 with bom" {
+  let bytes = b"\xef\xbb\xbfhello"
+  assert_true(@utf8.is_valid(bytes))
+  assert_true(@utf8.is_valid(bytes, ignore_bom=true))
+  assert_false(@utf8.is_valid(b"\xef\xbb\xbf\xff"))
+  assert_false(@utf8.is_valid(b"\xef\xbb\xbf\xff", ignore_bom=true))
+}
+
 ///|
 test "decode malformed sliced views" {
   try {