Add UTF-8 validity check

Author: mizchi

encoding/utf8/README.mbt.md

@@ -35,6 +35,18 @@ test "decode" {
 }
 ```
 
+## Validation
+
+Use `is_valid` to check whether bytes are well-formed UTF-8 without decoding them.
+
+```mbt check
+///|
+test "is_valid" {
+  inspect(@utf8.is_valid(b"hi"), content="true")
+  inspect(@utf8.is_valid(b"\x80"), content="false")
+}
+```
+
 ## Lossy Decoding
 
 Use `decode_lossy` to decode bytes that may contain invalid UTF-8, replacing invalid sequences with the Unicode replacement character (U+FFFD).

encoding/utf8/decode.mbt

@@ -27,3 +27,71 @@ fn drop_utf8_bom(bytes : BytesView, ignore_bom : Bool) -> BytesView {
   }
   bytes
 }
+
+///|
+fn utf8_ascii8_bits(src : Bytes, offset : Int) -> Int {
+  src.unsafe_get(offset).to_int() |
+  src.unsafe_get(offset + 1).to_int() |
+  src.unsafe_get(offset + 2).to_int() |
+  src.unsafe_get(offset + 3).to_int() |
+  src.unsafe_get(offset + 4).to_int() |
+  src.unsafe_get(offset + 5).to_int() |
+  src.unsafe_get(offset + 6).to_int() |
+  src.unsafe_get(offset + 7).to_int()
+}
+
+///|
+fn utf8_find_malformed(src : Bytes, src_offset : Int, src_length : Int) -> Int {
+  let end = src_offset + src_length
+  let mut i = src_offset
+  while i + 8 <= end &&
+        src.unsafe_get(i).to_int() < 0x80 &&
+        (utf8_ascii8_bits(src, i) & 0x80) == 0 {
+    i += 8
+  }
+  while i < end {
+    let b = src.unsafe_get(i).to_int()
+    if b < 0x80 {
+      i += 1
+    } else if b < 0xC2 {
+      return i - src_offset
+    } else if b < 0xE0 {
+      if i + 1 >= end {
+        return i - src_offset
+      }
+      let c1 = src.unsafe_get(i + 1).to_int()
+      if c1 < 0x80 || c1 > 0xBF {
+        return i - src_offset
+      }
+      i += 2
+    } else if b < 0xF0 {
+      if i + 2 >= end {
+        return i - src_offset
+      }
+      let c1 = src.unsafe_get(i + 1).to_int()
+      let c2 = src.unsafe_get(i + 2).to_int()
+      let lo = if b == 0xE0 { 0xA0 } else { 0x80 }
+      let hi = if b == 0xED { 0x9F } else { 0xBF }
+      if c1 < lo || c1 > hi || c2 < 0x80 || c2 > 0xBF {
+        return i - src_offset
+      }
+      i += 3
+    } else if b < 0xF5 {
+      if i + 3 >= end {
+        return i - src_offset
+      }
+      let c1 = src.unsafe_get(i + 1).to_int()
+      let c2 = src.unsafe_get(i + 2).to_int()
+      let c3 = src.unsafe_get(i + 3).to_int()
+      let lo = if b == 0xF0 { 0x90 } else { 0x80 }
+      let hi = if b == 0xF4 { 0x8F } else { 0xBF }
+      if c1 < lo || c1 > hi || c2 < 0x80 || c2 > 0xBF || c3 < 0x80 || c3 > 0xBF {
+        return i - src_offset
+      }
+      i += 4
+    } else {
+      return i - src_offset
+    }
+  }
+  -1
+}

encoding/utf8/decode_js.mbt

@@ -50,37 +50,6 @@ extern "js" fn decode_utf8_lossy_js(
   #|   new TextDecoder("utf-8", { ignoreBOM: false }),
   #| )
 
-///|
-fn utf8_find_malformed(src : Bytes, src_offset : Int, src_length : Int) -> Int {
-  let view = src[src_offset:src_offset + src_length]
-  for bytes = view {
-    match bytes {
-      [] => break -1
-      [
-        _..=0x7F,
-        _..=0x7F,
-        _..=0x7F,
-        _..=0x7F,
-        _..=0x7F,
-        _..=0x7F,
-        _..=0x7F,
-        _..=0x7F,
-        .. rest,
-      ] => continue rest
-      [0..=0x7F, .. rest] => continue rest
-      [0xC2..=0xDF, 0x80..=0xBF, .. rest] => continue rest
-      [0xE0, 0xA0..=0xBF, 0x80..=0xBF, .. rest]
-      | [0xE1..=0xEC, 0x80..=0xBF, 0x80..=0xBF, .. rest]
-      | [0xED, 0x80..=0x9F, 0x80..=0xBF, .. rest]
-      | [0xEE..=0xEF, 0x80..=0xBF, 0x80..=0xBF, .. rest] => continue rest
-      [0xF0, 0x90..=0xBF, 0x80..=0xBF, 0x80..=0xBF, .. rest]
-      | [0xF1..=0xF3, 0x80..=0xBF, 0x80..=0xBF, 0x80..=0xBF, .. rest]
-      | [0xF4, 0x80..=0x8F, 0x80..=0xBF, 0x80..=0xBF, .. rest] => continue rest
-      malformed => break malformed.start_offset() - src_offset
-    }
-  }
-}
-
 ///|
 fn strict_malformed_suffix(bytes : BytesView) -> BytesView {
   let input = bytes.data()

encoding/utf8/decode_test.mbt

@@ -117,6 +117,32 @@ test "decode sliced views" {
   inspect(@utf8.decode_lossy(b".A\x80\xFFB."[1:5]), content="A��B")
 }
 
+///|
+test "is_valid UTF-8" {
+  assert_true(@utf8.is_valid(b""))
+  assert_true(@utf8.is_valid(b"hello"))
+  assert_true(@utf8.is_valid(b"\xc2\xa9"))
+  assert_true(@utf8.is_valid(b"\xe4\xb8\xad"))
+  assert_true(@utf8.is_valid(b"\xf0\x9f\x98\x80"))
+  assert_true(@utf8.is_valid(b".A\xc2\xa9\xe4\xb8\xad."[1:7]))
+  assert_false(@utf8.is_valid(b"\x80"))
+  assert_false(@utf8.is_valid(b"\xc2"))
+  assert_false(@utf8.is_valid(b"\xc0\x80"))
+  assert_false(@utf8.is_valid(b"\xe0\x80\x80"))
+  assert_false(@utf8.is_valid(b"\xed\xa0\x80"))
+  assert_false(@utf8.is_valid(b"\xf4\x90\x80\x80"))
+  assert_false(@utf8.is_valid(b"\xff"))
+}
+
+///|
+test "is_valid UTF-8 with bom" {
+  let bytes = b"\xef\xbb\xbfhello"
+  assert_true(@utf8.is_valid(bytes))
+  assert_true(@utf8.is_valid(bytes, ignore_bom=true))
+  assert_false(@utf8.is_valid(b"\xef\xbb\xbf\xff"))
+  assert_false(@utf8.is_valid(b"\xef\xbb\xbf\xff", ignore_bom=true))
+}
+
 ///|
 test "decode malformed sliced views" {
   try {

encoding/utf8/is_valid_bench_test.mbt

@@ -0,0 +1,59 @@
+// Copyright 2026 International Digital Economy Academy
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+///|
+let utf8_is_valid_bench_size = 4096
+
+///|
+fn utf8_is_valid_ascii_bytes() -> Bytes {
+  Bytes::makei(utf8_is_valid_bench_size, i => (0x20 + i % 0x5F).to_byte())
+}
+
+///|
+fn utf8_is_valid_mixed_bytes() -> Bytes {
+  Bytes::makei(utf8_is_valid_bench_size, i => {
+    match i % 8 {
+      0 => b'\xC3'
+      1 => b'\xA9'
+      2 => b'\xE4'
+      3 => b'\xB8'
+      4 => b'\xAD'
+      _ => (0x41 + i % 26).to_byte()
+    }
+  })
+}
+
+///|
+test "bench utf8.is_valid ascii n=4096" (it : @bench.T) {
+  let bytes = utf8_is_valid_ascii_bytes()
+  it.bench(fn() { it.keep(@utf8.is_valid(bytes)) })
+}
+
+///|
+test "bench utf8.decode ascii n=4096" (it : @bench.T) {
+  let bytes = utf8_is_valid_ascii_bytes()
+  it.bench(fn() { it.keep(try! @utf8.decode(bytes)) })
+}
+
+///|
+test "bench utf8.is_valid mixed n=4096" (it : @bench.T) {
+  let bytes = utf8_is_valid_mixed_bytes()
+  it.bench(fn() { it.keep(@utf8.is_valid(bytes)) })
+}
+
+///|
+test "bench utf8.decode mixed n=4096" (it : @bench.T) {
+  let bytes = utf8_is_valid_mixed_bytes()
+  it.bench(fn() { it.keep(try! @utf8.decode(bytes)) })
+}

encoding/utf8/is_valid_js.mbt

@@ -0,0 +1,46 @@
+// Copyright 2026 International Digital Economy Academy
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+///|
+extern "js" fn is_valid_utf8_js(
+  bytes : Bytes,
+  start : Int,
+  len : Int,
+  preserve_bom : Bool,
+) -> Bool =
+  #| ((preserveBOMDecoder, dropBOMDecoder) => function(bytes, start, len, preserveBOM) {
+  #|   try {
+  #|     const end = start + len;
+  #|     const slice = bytes.subarray(start, end);
+  #|     const decoder = preserveBOM ? preserveBOMDecoder : dropBOMDecoder;
+  #|     decoder.decode(slice);
+  #|     return true;
+  #|   } catch (_) {
+  #|     return false;
+  #|   }
+  #| })(
+  #|   new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }),
+  #|   new TextDecoder("utf-8", { fatal: true, ignoreBOM: false }),
+  #| )
+
+///|
+/// Returns true if the input bytes are well-formed UTF-8.
+pub fn is_valid(bytes : BytesView, ignore_bom? : Bool = false) -> Bool {
+  is_valid_utf8_js(
+    bytes.data(),
+    bytes.start_offset(),
+    bytes.length(),
+    !ignore_bom,
+  )
+}

encoding/utf8/is_valid_nonjs.mbt

@@ -0,0 +1,20 @@
+// Copyright 2026 International Digital Economy Academy
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+///|
+/// Returns true if the input bytes are well-formed UTF-8.
+pub fn is_valid(bytes : BytesView, ignore_bom? : Bool = false) -> Bool {
+  let bytes = drop_utf8_bom(bytes, ignore_bom)
+  utf8_find_malformed(bytes.data(), bytes.start_offset(), bytes.length()) < 0
+}

encoding/utf8/moon.pkg

@@ -5,6 +5,7 @@ import {
 
 import {
   "moonbitlang/core/buffer",
+  "moonbitlang/core/bench",
 } for "test"
 
 options(
@@ -13,5 +14,7 @@ options(
     "decode_nonjs.mbt": [ "not", "js" ],
     "encode_js.mbt": [ "js" ],
     "encode_nonjs.mbt": [ "not", "js" ],
+    "is_valid_js.mbt": [ "js" ],
+    "is_valid_nonjs.mbt": [ "not", "js" ],
   },
 )

encoding/utf8/pkg.generated.mbti

@@ -12,6 +12,8 @@ pub fn decode_lossy(BytesView, ignore_bom? : Bool) -> String
 
 pub fn encode(StringView, bom? : Bool) -> Bytes
 
+pub fn is_valid(BytesView, ignore_bom? : Bool) -> Bool
+
 // Errors
 pub suberror Malformed {
   Malformed(BytesView)