From 20a99fbfef7a829959d93e30fcc6b33bd21a2f47 Mon Sep 17 00:00:00 2001 From: mizchi Date: Mon, 20 Jul 2026 20:21:50 +0900 Subject: [PATCH 1/5] perf(ascii): vectorize decode on non-JS targets --- encoding/ascii/ascii_bench_test.mbt | 47 +++++++++++++++ encoding/ascii/decode.mbt | 87 +++++++++++++++++++++++++++ encoding/ascii/decode_v128_wbtest.mbt | 22 +++++++ encoding/ascii/moon.pkg | 9 +++ 4 files changed, 165 insertions(+) create mode 100644 encoding/ascii/ascii_bench_test.mbt create mode 100644 encoding/ascii/decode_v128_wbtest.mbt diff --git a/encoding/ascii/ascii_bench_test.mbt b/encoding/ascii/ascii_bench_test.mbt new file mode 100644 index 0000000000..44338c2a4b --- /dev/null +++ b/encoding/ascii/ascii_bench_test.mbt @@ -0,0 +1,47 @@ +///| +let ascii_bench_size = 100_000 + +///| +let ascii_bench_bytes : Bytes = Bytes::make(ascii_bench_size, b'A') + +///| +let ascii_bench_bytes_15 : Bytes = Bytes::make(15, b'A') + +///| +let ascii_bench_bytes_16 : Bytes = Bytes::make(16, b'A') + +///| +let ascii_bench_bytes_32 : Bytes = Bytes::make(32, b'A') + +///| +let ascii_bench_bytes_64 : Bytes = Bytes::make(64, b'A') + +///| +test "bench ASCII decode n=15" (it : @bench.T) { + let bytes = ascii_bench_bytes_15 + it.bench(fn() { it.keep((try! @ascii.decode(bytes)).length()) }) +} + +///| +test "bench ASCII decode n=16" (it : @bench.T) { + let bytes = ascii_bench_bytes_16 + it.bench(fn() { it.keep((try! @ascii.decode(bytes)).length()) }) +} + +///| +test "bench ASCII decode n=32" (it : @bench.T) { + let bytes = ascii_bench_bytes_32 + it.bench(fn() { it.keep((try! @ascii.decode(bytes)).length()) }) +} + +///| +test "bench ASCII decode n=64" (it : @bench.T) { + let bytes = ascii_bench_bytes_64 + it.bench(fn() { it.keep((try! @ascii.decode(bytes)).length()) }) +} + +///| +test "bench ASCII decode n=100000" (it : @bench.T) { + let bytes = ascii_bench_bytes + it.bench(fn() { it.keep((try! @ascii.decode(bytes)).length()) }) +} diff --git a/encoding/ascii/decode.mbt b/encoding/ascii/decode.mbt index c086e18ab0..12cda1887b 100644 --- a/encoding/ascii/decode.mbt +++ b/encoding/ascii/decode.mbt @@ -21,6 +21,17 @@ pub suberror Malformed { ///| fn unsafe_fixedarray_uint16_to_string(buffer : FixedArray[UInt16]) -> String = "%string.unsafe_from_uint16_fixedarray" +///| +#cfg(target="js") +#warnings("-unused_value") +fn suppress_unused_v128_import_on_js() -> Unit { + ignore(@v128.i8x16_splat(0)) +} + +///| +#cfg(not(target="js")) +fn unsafe_fixedarray_from_bytes(bytes : Bytes) -> FixedArray[Byte] = "%identity" + ///| fn finish_string(buffer : FixedArray[UInt16], len : Int) -> String { if len == buffer.length() { @@ -32,10 +43,86 @@ fn finish_string(buffer : FixedArray[UInt16], len : Int) -> String { } } +///| +#cfg(not(target="js")) +fn decode_scalar(bytes : BytesView) -> String raise Malformed { + let t : FixedArray[UInt16] = FixedArray::make(bytes.length(), 0) + let tlen = for tlen = 0, bs = bytes { + match (tlen, bs) { + (tlen, []) => break tlen + (tlen, [0..=0x7F as b, .. rest]) => { + t.unsafe_set(tlen, b.to_uint16()) + continue tlen + 1, rest + } + (_, _ as bytes) => raise Malformed(bytes) + } + } + finish_string(t, tlen) +} + +///| +#cfg(not(target="js")) +fn decode_v128(bytes : BytesView) -> String raise Malformed { + let length = bytes.length() + let result = FixedArray::make(length * 2, b'\x00') + let source = unsafe_fixedarray_from_bytes(bytes.data()) + let source_start = bytes.start_offset() + let source_end = source_start + length + let mut source_offset = source_start + let mut result_offset = 0 + while source_offset + 16 <= source_end { + let block = @v128.v128_load(source, source_offset) + let non_ascii = @v128.i8x16_bitmask(block) + if non_ascii != 0 { + let malformed_offset = source_offset - source_start + non_ascii.ctz() + raise Malformed(bytes[malformed_offset:]) + } + @v128.v128_store( + result, + result_offset, + @v128.i16x8_extend_low_i8x16_u(block), + ) + @v128.v128_store( + result, + result_offset + 16, + @v128.i16x8_extend_high_i8x16_u(block), + ) + source_offset = source_offset + 16 + result_offset = result_offset + 32 + } + while source_offset < source_end { + let byte = source.unsafe_get(source_offset) + if byte > b'\x7F' { + raise Malformed(bytes[source_offset - source_start:]) + } + result.unsafe_set(result_offset, byte) + result.unsafe_set(result_offset + 1, b'\x00') + source_offset = source_offset + 1 + result_offset = result_offset + 2 + } + result.unsafe_reinterpret_as_bytes().to_unchecked_string() +} + +///| +/// Decodes an ASCII byte array into a string. +/// +/// Raises `Malformed` if any byte is outside the ASCII range. +#cfg(not(target="js")) +#inline +pub fn decode(bytes : BytesView) -> String raise Malformed { + // The scalar loop is faster below this crossover point on supported targets. + if bytes.length() >= 64 { + decode_v128(bytes) + } else { + decode_scalar(bytes) + } +} + ///| /// Decodes an ASCII byte array into a string. /// /// Raises `Malformed` if any byte is outside the ASCII range. +#cfg(target="js") pub fn decode(bytes : BytesView) -> String raise Malformed { let t : FixedArray[UInt16] = FixedArray::make(bytes.length(), 0) let tlen = for tlen = 0, bs = bytes { diff --git a/encoding/ascii/decode_v128_wbtest.mbt b/encoding/ascii/decode_v128_wbtest.mbt new file mode 100644 index 0000000000..c6dcbb6e86 --- /dev/null +++ b/encoding/ascii/decode_v128_wbtest.mbt @@ -0,0 +1,22 @@ +///| +test "V128 decode handles a view and scalar tail" { + let bytes = b"!0123456789ABCDEF?" + inspect(decode_v128(bytes[1:]), content="0123456789ABCDEF?") +} + +///| +test "V128 decode reports the first non-ASCII byte in a block" { + let bytes = b"0123456789ABC\xffDEF" + try { + let _ = decode_v128(bytes) + panic() + } catch { + Malformed(rest) => + inspect( + rest, + content=( + #|b"\xffDEF" + ), + ) + } +} diff --git a/encoding/ascii/moon.pkg b/encoding/ascii/moon.pkg index aecf181f19..5be98ab1ac 100644 --- a/encoding/ascii/moon.pkg +++ b/encoding/ascii/moon.pkg @@ -1,4 +1,13 @@ import { "moonbitlang/core/builtin", "moonbitlang/core/debug", + "moonbitlang/core/v128", } + +import { + "moonbitlang/core/bench", +} for "test" + +options( + targets: { "decode_v128_wbtest.mbt": [ "not", "js" ] }, +) From aa4dd6e74e907d712ee1f321182aeb0e62d028d8 Mon Sep 17 00:00:00 2001 From: mizchi Date: Mon, 20 Jul 2026 20:23:26 +0900 Subject: [PATCH 2/5] perf(ascii): inline scalar decode path --- encoding/ascii/decode.mbt | 1 + 1 file changed, 1 insertion(+) diff --git a/encoding/ascii/decode.mbt b/encoding/ascii/decode.mbt index 12cda1887b..4a63003d14 100644 --- a/encoding/ascii/decode.mbt +++ b/encoding/ascii/decode.mbt @@ -45,6 +45,7 @@ fn finish_string(buffer : FixedArray[UInt16], len : Int) -> String { ///| #cfg(not(target="js")) +#inline fn decode_scalar(bytes : BytesView) -> String raise Malformed { let t : FixedArray[UInt16] = FixedArray::make(bytes.length(), 0) let tlen = for tlen = 0, bs = bytes { From 9f76d30a13dc09d77a715bc6e47ec75dc895161e Mon Sep 17 00:00:00 2001 From: mizchi Date: Mon, 20 Jul 2026 20:28:34 +0900 Subject: [PATCH 3/5] bench(ascii): cover million-byte decode --- encoding/ascii/ascii_bench_test.mbt | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/encoding/ascii/ascii_bench_test.mbt b/encoding/ascii/ascii_bench_test.mbt index 44338c2a4b..f353203b81 100644 --- a/encoding/ascii/ascii_bench_test.mbt +++ b/encoding/ascii/ascii_bench_test.mbt @@ -4,6 +4,12 @@ let ascii_bench_size = 100_000 ///| let ascii_bench_bytes : Bytes = Bytes::make(ascii_bench_size, b'A') +///| +let ascii_bench_large_size = 1_000_000 + +///| +let ascii_bench_large_bytes : Bytes = Bytes::make(ascii_bench_large_size, b'A') + ///| let ascii_bench_bytes_15 : Bytes = Bytes::make(15, b'A') @@ -45,3 +51,9 @@ test "bench ASCII decode n=100000" (it : @bench.T) { let bytes = ascii_bench_bytes it.bench(fn() { it.keep((try! @ascii.decode(bytes)).length()) }) } + +///| +test "bench ASCII decode n=1000000" (it : @bench.T) { + let bytes = ascii_bench_large_bytes + it.bench(fn() { it.keep((try! @ascii.decode(bytes)).length()) }) +} From ebf5fc8c39f406d60125a1e4db0f04c8aa9fb08b Mon Sep 17 00:00:00 2001 From: mizchi Date: Mon, 20 Jul 2026 20:33:47 +0900 Subject: [PATCH 4/5] chore(ascii): add license headers --- encoding/ascii/ascii_bench_test.mbt | 14 ++++++++++++++ encoding/ascii/decode_v128_wbtest.mbt | 14 ++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/encoding/ascii/ascii_bench_test.mbt b/encoding/ascii/ascii_bench_test.mbt index f353203b81..d0db7a6847 100644 --- a/encoding/ascii/ascii_bench_test.mbt +++ b/encoding/ascii/ascii_bench_test.mbt @@ -1,3 +1,17 @@ +// Copyright 2026 International Digital Economy Academy +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + ///| let ascii_bench_size = 100_000 diff --git a/encoding/ascii/decode_v128_wbtest.mbt b/encoding/ascii/decode_v128_wbtest.mbt index c6dcbb6e86..1fdac570dc 100644 --- a/encoding/ascii/decode_v128_wbtest.mbt +++ b/encoding/ascii/decode_v128_wbtest.mbt @@ -1,3 +1,17 @@ +// Copyright 2026 International Digital Economy Academy +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + ///| test "V128 decode handles a view and scalar tail" { let bytes = b"!0123456789ABCDEF?" From 7e84e0e603ac8b34f4b4ca91ef3867e8ac611421 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Fri, 21 Aug 2026 08:26:55 +0800 Subject: [PATCH 5/5] test(ascii): property-test the vectorized decoder Add quickcheck properties pinning decode/decode_lossy against a byte-by-byte model on poisoned views (invalid bytes just outside the view bounds), the Malformed view against the first invalid byte, the SIMD decoder against the scalar decoder on arbitrary payloads and unaligned view offsets, and finish_string truncation; plus an exhaustive invalid-byte position sweep around SIMD block and crossover boundaries. Co-Authored-By: Claude Fable 5 --- encoding/ascii/decode_v128_wbtest.mbt | 43 +++++++ encoding/ascii/finish_string_wbtest.mbt | 41 +++++++ encoding/ascii/moon.pkg | 5 + encoding/ascii/quickcheck_test.mbt | 142 ++++++++++++++++++++++++ 4 files changed, 231 insertions(+) create mode 100644 encoding/ascii/finish_string_wbtest.mbt create mode 100644 encoding/ascii/quickcheck_test.mbt diff --git a/encoding/ascii/decode_v128_wbtest.mbt b/encoding/ascii/decode_v128_wbtest.mbt index 1fdac570dc..6c2b7fa004 100644 --- a/encoding/ascii/decode_v128_wbtest.mbt +++ b/encoding/ascii/decode_v128_wbtest.mbt @@ -12,6 +12,49 @@ // See the License for the specific language governing permissions and // limitations under the License. +///| +fn v128_outcome(view : BytesView) -> (String?, Bytes?) { + (Some(decode_v128(view)), None) catch { + Malformed(rest) => (None, Some(rest.to_owned())) + } +} + +///| +fn scalar_outcome(view : BytesView) -> (String?, Bytes?) { + (Some(decode_scalar(view)), None) catch { + Malformed(rest) => (None, Some(rest.to_owned())) + } +} + +///| +/// The SIMD decoder must agree with the scalar decoder — same string on +/// success, same remaining view on failure — on arbitrary payloads seen +/// through views with arbitrary (unaligned) start offsets. +test "quickcheck: V128 decode agrees with scalar decode" { + @quickcheck.check(count=300, (input : (Array[Int], Int, Bool)) => { + let (seeds, pad_seed, all_valid) = input + let payload = seeds.map(seed => { + if all_valid || (seed & 0xF) != 0xF { + (seed & 0x7F).to_byte() + } else { + (0x80 | (seed & 0x7F)).to_byte() + } + }) + let pad = pad_seed % 17 + let pad = if pad < 0 { pad + 17 } else { pad } + let full : Array[Byte] = [] + for _ in 0.. { + let units = input.0.map(x => (x & 0x7F).to_uint16()) + let buffer = FixedArray::makei(units.length(), i => units[i]) + let take = if units.is_empty() { + 0 + } else { + let r = input.1 % (units.length() + 1) + if r < 0 { + r + units.length() + 1 + } else { + r + } + } + let expected = StringBuilder(size_hint=take) + for i in 0.. Byte { + match seed & 0x7 { + 0 => b'\x00' + 1 => b'\x7F' + 2 => b'A' + 3 => b' ' + _ => ((seed >> 3) & 0x7F).to_byte() + } +} + +///| +/// Maps an arbitrary Int into `0.. Int { + let r = value % modulus + if r < 0 { + r + modulus + } else { + r + } +} + +///| +fn model_decode_lossy(payload : Array[Byte]) -> String { + let buf = StringBuilder(size_hint=payload.length()) + for b in payload { + if b <= b'\x7F' { + buf.write_char(b.to_int().unsafe_to_char()) + } else { + buf.write_char('\u{FFFD}') + } + } + buf.to_string() +} + +///| +/// Embeds `payload` between runs of invalid `0xFF` bytes and returns the view +/// covering exactly `payload`, so any scan that reads beyond the view sees +/// bytes that must not influence the result. +fn poisoned_view(payload : Array[Byte], pad : Int) -> BytesView { + let full : Array[Byte] = [] + for _ in 0.. { + let (seeds, pad_seed) = input + let payload = seeds.map(valid_byte) + let view = poisoned_view(payload, wrap_index(pad_seed, 17)) + let expected = model_decode_lossy(payload) + try @ascii.decode(view) catch { + Malformed(_) => false + } noraise { + decoded => + decoded == expected && + // A fully valid payload decodes identically through the lossy + // decoder, and re-encoding restores the original bytes. + @ascii.decode_lossy(view) == expected && + @ascii.encode(decoded) == Bytes::from_array(payload) + } + }) +} + +///| +test "quickcheck: Malformed points at the first invalid byte" { + @quickcheck.check(count=300, (input : (Array[Int], Int, Int)) => { + let (seeds, plant_seed, pad_seed) = input + guard seeds.length() > 0 else { return true } + let payload = seeds.map(valid_byte) + let plant = wrap_index(plant_seed, payload.length()) + payload[plant] = if (plant_seed & 1) == 0 { b'\x80' } else { b'\xFF' } + let view = poisoned_view(payload, wrap_index(pad_seed, 17)) + // The remaining view starts at the first invalid byte (the planted one, + // unless the seeds already produced an earlier one). + let mut first = plant + for i, b in payload { + if b > b'\x7F' { + first = i + break + } + } + try @ascii.decode(view) catch { + Malformed(rest) => + rest.to_owned() == Bytes::from_array(payload)[first:].to_owned() && + @ascii.decode_lossy(view) == model_decode_lossy(payload) + } noraise { + _ => false + } + }) +} + +///| +/// Exhaustively plants an invalid byte at every position for lengths around +/// the 16-byte SIMD blocks and the 64-byte crossover. +test "decode malformed position sweep" { + let lengths = [1, 15, 16, 17, 31, 32, 63, 64, 65, 80, 96] + for len in lengths { + let clean = Array::make(len, b'A') + let view = poisoned_view(clean, 3) + assert_eq(try! @ascii.decode(view), "A".repeat(len)) + for pos in 0.. assert_eq(rest.length(), len - pos) + } noraise { + _ => fail("expected Malformed at \{pos} for length \{len}") + } + assert_eq(@ascii.decode_lossy(view), model_decode_lossy(payload)) + } + } +}