diff --git a/encoding/ascii/ascii_bench_test.mbt b/encoding/ascii/ascii_bench_test.mbt new file mode 100644 index 000000000..d0db7a684 --- /dev/null +++ b/encoding/ascii/ascii_bench_test.mbt @@ -0,0 +1,73 @@ +// Copyright 2026 International Digital Economy Academy +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +///| +let ascii_bench_size = 100_000 + +///| +let ascii_bench_bytes : Bytes = Bytes::make(ascii_bench_size, b'A') + +///| +let ascii_bench_large_size = 1_000_000 + +///| +let ascii_bench_large_bytes : Bytes = Bytes::make(ascii_bench_large_size, b'A') + +///| +let ascii_bench_bytes_15 : Bytes = Bytes::make(15, b'A') + +///| +let ascii_bench_bytes_16 : Bytes = Bytes::make(16, b'A') + +///| +let ascii_bench_bytes_32 : Bytes = Bytes::make(32, b'A') + +///| +let ascii_bench_bytes_64 : Bytes = Bytes::make(64, b'A') + +///| +test "bench ASCII decode n=15" (it : @bench.T) { + let bytes = ascii_bench_bytes_15 + it.bench(fn() { it.keep((try! @ascii.decode(bytes)).length()) }) +} + +///| +test "bench ASCII decode n=16" (it : @bench.T) { + let bytes = ascii_bench_bytes_16 + it.bench(fn() { it.keep((try! @ascii.decode(bytes)).length()) }) +} + +///| +test "bench ASCII decode n=32" (it : @bench.T) { + let bytes = ascii_bench_bytes_32 + it.bench(fn() { it.keep((try! @ascii.decode(bytes)).length()) }) +} + +///| +test "bench ASCII decode n=64" (it : @bench.T) { + let bytes = ascii_bench_bytes_64 + it.bench(fn() { it.keep((try! @ascii.decode(bytes)).length()) }) +} + +///| +test "bench ASCII decode n=100000" (it : @bench.T) { + let bytes = ascii_bench_bytes + it.bench(fn() { it.keep((try! @ascii.decode(bytes)).length()) }) +} + +///| +test "bench ASCII decode n=1000000" (it : @bench.T) { + let bytes = ascii_bench_large_bytes + it.bench(fn() { it.keep((try! @ascii.decode(bytes)).length()) }) +} diff --git a/encoding/ascii/decode.mbt b/encoding/ascii/decode.mbt index c086e18ab..4a63003d1 100644 --- a/encoding/ascii/decode.mbt +++ b/encoding/ascii/decode.mbt @@ -21,6 +21,17 @@ pub suberror Malformed { ///| fn unsafe_fixedarray_uint16_to_string(buffer : FixedArray[UInt16]) -> String = "%string.unsafe_from_uint16_fixedarray" +///| +#cfg(target="js") +#warnings("-unused_value") +fn suppress_unused_v128_import_on_js() -> Unit { + ignore(@v128.i8x16_splat(0)) +} + +///| +#cfg(not(target="js")) +fn unsafe_fixedarray_from_bytes(bytes : Bytes) -> FixedArray[Byte] = "%identity" + ///| fn finish_string(buffer : FixedArray[UInt16], len : Int) -> String { if len == buffer.length() { @@ -32,10 +43,87 @@ fn finish_string(buffer : FixedArray[UInt16], len : Int) -> String { } } +///| +#cfg(not(target="js")) +#inline +fn decode_scalar(bytes : BytesView) -> String raise Malformed { + let t : FixedArray[UInt16] = FixedArray::make(bytes.length(), 0) + let tlen = for tlen = 0, bs = bytes { + match (tlen, bs) { + (tlen, []) => break tlen + (tlen, [0..=0x7F as b, .. rest]) => { + t.unsafe_set(tlen, b.to_uint16()) + continue tlen + 1, rest + } + (_, _ as bytes) => raise Malformed(bytes) + } + } + finish_string(t, tlen) +} + +///| +#cfg(not(target="js")) +fn decode_v128(bytes : BytesView) -> String raise Malformed { + let length = bytes.length() + let result = FixedArray::make(length * 2, b'\x00') + let source = unsafe_fixedarray_from_bytes(bytes.data()) + let source_start = bytes.start_offset() + let source_end = source_start + length + let mut source_offset = source_start + let mut result_offset = 0 + while source_offset + 16 <= source_end { + let block = @v128.v128_load(source, source_offset) + let non_ascii = @v128.i8x16_bitmask(block) + if non_ascii != 0 { + let malformed_offset = source_offset - source_start + non_ascii.ctz() + raise Malformed(bytes[malformed_offset:]) + } + @v128.v128_store( + result, + result_offset, + @v128.i16x8_extend_low_i8x16_u(block), + ) + @v128.v128_store( + result, + result_offset + 16, + @v128.i16x8_extend_high_i8x16_u(block), + ) + source_offset = source_offset + 16 + result_offset = result_offset + 32 + } + while source_offset < source_end { + let byte = source.unsafe_get(source_offset) + if byte > b'\x7F' { + raise Malformed(bytes[source_offset - source_start:]) + } + result.unsafe_set(result_offset, byte) + result.unsafe_set(result_offset + 1, b'\x00') + source_offset = source_offset + 1 + result_offset = result_offset + 2 + } + result.unsafe_reinterpret_as_bytes().to_unchecked_string() +} + +///| +/// Decodes an ASCII byte array into a string. +/// +/// Raises `Malformed` if any byte is outside the ASCII range. +#cfg(not(target="js")) +#inline +pub fn decode(bytes : BytesView) -> String raise Malformed { + // The scalar loop is faster below this crossover point on supported targets. + if bytes.length() >= 64 { + decode_v128(bytes) + } else { + decode_scalar(bytes) + } +} + ///| /// Decodes an ASCII byte array into a string. /// /// Raises `Malformed` if any byte is outside the ASCII range. +#cfg(target="js") pub fn decode(bytes : BytesView) -> String raise Malformed { let t : FixedArray[UInt16] = FixedArray::make(bytes.length(), 0) let tlen = for tlen = 0, bs = bytes { diff --git a/encoding/ascii/decode_v128_wbtest.mbt b/encoding/ascii/decode_v128_wbtest.mbt new file mode 100644 index 000000000..6c2b7fa00 --- /dev/null +++ b/encoding/ascii/decode_v128_wbtest.mbt @@ -0,0 +1,79 @@ +// Copyright 2026 International Digital Economy Academy +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +///| +fn v128_outcome(view : BytesView) -> (String?, Bytes?) { + (Some(decode_v128(view)), None) catch { + Malformed(rest) => (None, Some(rest.to_owned())) + } +} + +///| +fn scalar_outcome(view : BytesView) -> (String?, Bytes?) { + (Some(decode_scalar(view)), None) catch { + Malformed(rest) => (None, Some(rest.to_owned())) + } +} + +///| +/// The SIMD decoder must agree with the scalar decoder — same string on +/// success, same remaining view on failure — on arbitrary payloads seen +/// through views with arbitrary (unaligned) start offsets. +test "quickcheck: V128 decode agrees with scalar decode" { + @quickcheck.check(count=300, (input : (Array[Int], Int, Bool)) => { + let (seeds, pad_seed, all_valid) = input + let payload = seeds.map(seed => { + if all_valid || (seed & 0xF) != 0xF { + (seed & 0x7F).to_byte() + } else { + (0x80 | (seed & 0x7F)).to_byte() + } + }) + let pad = pad_seed % 17 + let pad = if pad < 0 { pad + 17 } else { pad } + let full : Array[Byte] = [] + for _ in 0.. + inspect( + rest, + content=( + #|b"\xffDEF" + ), + ) + } +} diff --git a/encoding/ascii/finish_string_wbtest.mbt b/encoding/ascii/finish_string_wbtest.mbt new file mode 100644 index 000000000..4a151bdfd --- /dev/null +++ b/encoding/ascii/finish_string_wbtest.mbt @@ -0,0 +1,41 @@ +// Copyright 2026 International Digital Economy Academy +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +///| +/// Every decoder finishes through `finish_string`; it must keep exactly the +/// first `len` code units, both when the buffer is full and when it must +/// truncate. +test "quickcheck: finish_string keeps exactly the first len code units" { + @quickcheck.check(count=100, (input : (Array[Int], Int)) => { + let units = input.0.map(x => (x & 0x7F).to_uint16()) + let buffer = FixedArray::makei(units.length(), i => units[i]) + let take = if units.is_empty() { + 0 + } else { + let r = input.1 % (units.length() + 1) + if r < 0 { + r + units.length() + 1 + } else { + r + } + } + let expected = StringBuilder(size_hint=take) + for i in 0.. Byte { + match seed & 0x7 { + 0 => b'\x00' + 1 => b'\x7F' + 2 => b'A' + 3 => b' ' + _ => ((seed >> 3) & 0x7F).to_byte() + } +} + +///| +/// Maps an arbitrary Int into `0.. Int { + let r = value % modulus + if r < 0 { + r + modulus + } else { + r + } +} + +///| +fn model_decode_lossy(payload : Array[Byte]) -> String { + let buf = StringBuilder(size_hint=payload.length()) + for b in payload { + if b <= b'\x7F' { + buf.write_char(b.to_int().unsafe_to_char()) + } else { + buf.write_char('\u{FFFD}') + } + } + buf.to_string() +} + +///| +/// Embeds `payload` between runs of invalid `0xFF` bytes and returns the view +/// covering exactly `payload`, so any scan that reads beyond the view sees +/// bytes that must not influence the result. +fn poisoned_view(payload : Array[Byte], pad : Int) -> BytesView { + let full : Array[Byte] = [] + for _ in 0.. { + let (seeds, pad_seed) = input + let payload = seeds.map(valid_byte) + let view = poisoned_view(payload, wrap_index(pad_seed, 17)) + let expected = model_decode_lossy(payload) + try @ascii.decode(view) catch { + Malformed(_) => false + } noraise { + decoded => + decoded == expected && + // A fully valid payload decodes identically through the lossy + // decoder, and re-encoding restores the original bytes. + @ascii.decode_lossy(view) == expected && + @ascii.encode(decoded) == Bytes::from_array(payload) + } + }) +} + +///| +test "quickcheck: Malformed points at the first invalid byte" { + @quickcheck.check(count=300, (input : (Array[Int], Int, Int)) => { + let (seeds, plant_seed, pad_seed) = input + guard seeds.length() > 0 else { return true } + let payload = seeds.map(valid_byte) + let plant = wrap_index(plant_seed, payload.length()) + payload[plant] = if (plant_seed & 1) == 0 { b'\x80' } else { b'\xFF' } + let view = poisoned_view(payload, wrap_index(pad_seed, 17)) + // The remaining view starts at the first invalid byte (the planted one, + // unless the seeds already produced an earlier one). + let mut first = plant + for i, b in payload { + if b > b'\x7F' { + first = i + break + } + } + try @ascii.decode(view) catch { + Malformed(rest) => + rest.to_owned() == Bytes::from_array(payload)[first:].to_owned() && + @ascii.decode_lossy(view) == model_decode_lossy(payload) + } noraise { + _ => false + } + }) +} + +///| +/// Exhaustively plants an invalid byte at every position for lengths around +/// the 16-byte SIMD blocks and the 64-byte crossover. +test "decode malformed position sweep" { + let lengths = [1, 15, 16, 17, 31, 32, 63, 64, 65, 80, 96] + for len in lengths { + let clean = Array::make(len, b'A') + let view = poisoned_view(clean, 3) + assert_eq(try! @ascii.decode(view), "A".repeat(len)) + for pos in 0.. assert_eq(rest.length(), len - pos) + } noraise { + _ => fail("expected Malformed at \{pos} for length \{len}") + } + assert_eq(@ascii.decode_lossy(view), model_decode_lossy(payload)) + } + } +}