From e3f2ee0bf7728ad5d81b5eb576e743f9967b3722 Mon Sep 17 00:00:00 2001 From: mizchi Date: Mon, 20 Jul 2026 23:16:24 +0900 Subject: [PATCH 1/6] perf(string): accelerate ASCII character sets --- builtin/string_char_set_bench_test.mbt | 83 ++++++++ builtin/string_methods.mbt | 258 ++++++++++++++++++++++--- 2 files changed, 316 insertions(+), 25 deletions(-) create mode 100644 builtin/string_char_set_bench_test.mbt diff --git a/builtin/string_char_set_bench_test.mbt b/builtin/string_char_set_bench_test.mbt new file mode 100644 index 000000000..5e783abb9 --- /dev/null +++ b/builtin/string_char_set_bench_test.mbt @@ -0,0 +1,83 @@ +// Copyright 2026 International Digital Economy Academy +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +///| +let string_char_set_bench_size = 100_000 + +///| +let string_char_set_bench_ascii_set = "z \t\n\r" + +///| +let string_char_set_bench_contains_miss : String = "a".repeat( + string_char_set_bench_size, +) + +///| +let string_char_set_bench_contains_match_at_end : String = "a".repeat( + string_char_set_bench_size - 1, + ) + + "z" + +///| +let string_char_set_bench_trim_start : String = " ".repeat( + string_char_set_bench_size, + ) + + "x" + +///| +let string_char_set_bench_trim_end : String = "x" + + " ".repeat(string_char_set_bench_size) + +///| +test "bench StringView::contains_any ASCII miss n=100000" (it : @bench.T) { + it.bench(fn() { + it.keep( + string_char_set_bench_contains_miss.contains_any( + chars=string_char_set_bench_ascii_set, + ), + ) + }) +} + +///| +test "bench StringView::contains_any ASCII match at end n=100000" ( + it : @bench.T, +) { + it.bench(fn() { + it.keep( + string_char_set_bench_contains_match_at_end.contains_any( + chars=string_char_set_bench_ascii_set, + ), + ) + }) +} + +///| +test "bench StringView::trim_start ASCII n=100000" (it : @bench.T) { + it.bench(fn() { + it.keep(string_char_set_bench_trim_start.trim_start().length()) + }) +} + +///| +test "bench StringView::trim_end ASCII n=100000" (it : @bench.T) { + it.bench(fn() { it.keep(string_char_set_bench_trim_end.trim_end().length()) }) +} + +///| +test "bench StringView::trim ASCII n=100000" (it : @bench.T) { + let input = string_char_set_bench_trim_start + + " ".repeat(string_char_set_bench_size) + it.bench(fn() { it.keep(input.trim().length()) }) +} diff --git a/builtin/string_methods.mbt b/builtin/string_methods.mbt index d7de8ae2c..77532fd20 100644 --- a/builtin/string_methods.mbt +++ b/builtin/string_methods.mbt @@ -568,20 +568,168 @@ pub fn String::contains_code_unit(self : String, code : UInt16) -> Bool { string_contains_code_unit(self, 0, self.length(), code) } +///| +const ASCII_CHAR_SET_LIMIT : UInt = 128U + +///| +const ASCII_CHAR_SET_WORD_MASK : UInt = 31U + +///| +const ASCII_CHAR_SET_WORD_SHIFT = 5 + +///| +/// Tests membership in a 128-bit ASCII character set represented by four +/// scalar words, so callers do not need a temporary heap allocation. +fn ascii_char_set_contains( + bits0 : UInt, + bits1 : UInt, + bits2 : UInt, + bits3 : UInt, + c : Char, +) -> Bool { + let code = c.to_uint() + guard code < ASCII_CHAR_SET_LIMIT else { return false } + let bit = 1U << (code & ASCII_CHAR_SET_WORD_MASK).reinterpret_as_int() + match code >> ASCII_CHAR_SET_WORD_SHIFT { + 0 => (bits0 & bit) != 0U + 1 => (bits1 & bit) != 0U + 2 => (bits2 & bit) != 0U + _ => (bits3 & bit) != 0U + } +} + +///| +fn StringView::contains_any_ascii( + self : StringView, + bits0 : UInt, + bits1 : UInt, + bits2 : UInt, + bits3 : UInt, +) -> Bool { + for c in self { + if ascii_char_set_contains(bits0, bits1, bits2, bits3, c) { + return true + } + } + false +} + +///| +fn StringView::trim_start_ascii( + self : StringView, + bits0 : UInt, + bits1 : UInt, + bits2 : UInt, + bits3 : UInt, +) -> StringView { + for x = self { + match x { + [] as v => break v + [c, .. rest] as v => + if ascii_char_set_contains(bits0, bits1, bits2, bits3, c) { + continue rest + } else { + break v + } + } + } +} + +///| +fn StringView::trim_end_ascii( + self : StringView, + bits0 : UInt, + bits1 : UInt, + bits2 : UInt, + bits3 : UInt, +) -> StringView { + for x = self { + match x { + [] as v => break v + [.. rest, c] as v => + if ascii_char_set_contains(bits0, bits1, bits2, bits3, c) { + continue rest + } else { + break v + } + } + } +} + +///| +fn StringView::trim_start_with_chars( + self : StringView, + chars : StringView, +) -> StringView { + for x = self { + match x { + [] as v => break v + [c, .. rest] as v => + if chars.contains_char(c) { + continue rest + } else { + break v + } + } + } +} + +///| +fn StringView::trim_end_with_chars( + self : StringView, + chars : StringView, +) -> StringView { + for x = self { + match x { + [] as v => break v + [.. rest, c] as v => + if chars.contains_char(c) { + continue rest + } else { + break v + } + } + } +} + ///| /// Returns true if this string contains any character from the given set. pub fn StringView::contains_any(self : StringView, chars~ : StringView) -> Bool { match chars { [] => false [c] => self.contains_char(c) // specialize for single character - _ => - for c in self { - if chars.contains_char(c) { - break true + _ => { + let mut bits0 = 0U + let mut bits1 = 0U + let mut bits2 = 0U + let mut bits3 = 0U + let mut ascii_only = true + for c in chars { + let code = c.to_uint() + if code >= ASCII_CHAR_SET_LIMIT { + ascii_only = false + break + } + let bit = 1U << (code & ASCII_CHAR_SET_WORD_MASK).reinterpret_as_int() + match code >> ASCII_CHAR_SET_WORD_SHIFT { + 0 => bits0 = bits0 | bit + 1 => bits1 = bits1 | bit + 2 => bits2 = bits2 | bit + _ => bits3 = bits3 | bit + } + } + if ascii_only { + self.contains_any_ascii(bits0, bits1, bits2, bits3) + } else { + for c in self { + if chars.contains_char(c) { + break true + } + } nobreak { + false } - } nobreak { - false } + } } } @@ -649,6 +797,15 @@ test "contains_any" { inspect("hello"[:].contains_any(chars="eo"), content="true") } +///| +test "contains_any and trim ASCII character sets" { + assert_true("πŸ˜€a".contains_any(chars="az")) + assert_false("πŸ˜€".contains_any(chars="az")) + assert_true("πŸ˜€".contains_any(chars="aπŸ˜€")) + let view = "x hello \ty"[1:10] + assert_true(view.trim(chars=" \t") == "hello") +} + ///| /// Returns true if this string contains the given character. pub fn StringView::contains_char(self : StringView, c : Char) -> Bool { @@ -720,17 +877,30 @@ pub fn StringView::trim_start( self : StringView, chars? : StringView = "\t\n\r ", ) -> StringView { - for x = self { - match x { - [] as v => break v - [c, .. rest] as v => - if chars.contains_char(c) { - continue rest - } else { - break v - } + let mut bits0 = 0U + let mut bits1 = 0U + let mut bits2 = 0U + let mut bits3 = 0U + let mut ascii_only = true + for c in chars { + let code = c.to_uint() + if code >= ASCII_CHAR_SET_LIMIT { + ascii_only = false + break + } + let bit = 1U << (code & ASCII_CHAR_SET_WORD_MASK).reinterpret_as_int() + match code >> ASCII_CHAR_SET_WORD_SHIFT { + 0 => bits0 = bits0 | bit + 1 => bits1 = bits1 | bit + 2 => bits2 = bits2 | bit + _ => bits3 = bits3 | bit } } + if ascii_only { + self.trim_start_ascii(bits0, bits1, bits2, bits3) + } else { + self.trim_start_with_chars(chars) + } } ///| @@ -768,17 +938,30 @@ pub fn StringView::trim_end( self : StringView, chars? : StringView = "\t\n\r ", ) -> StringView { - for x = self { - match x { - [] as v => break v - [.. rest, c] as v => - if chars.contains_char(c) { - continue rest - } else { - break v - } + let mut bits0 = 0U + let mut bits1 = 0U + let mut bits2 = 0U + let mut bits3 = 0U + let mut ascii_only = true + for c in chars { + let code = c.to_uint() + if code >= ASCII_CHAR_SET_LIMIT { + ascii_only = false + break + } + let bit = 1U << (code & ASCII_CHAR_SET_WORD_MASK).reinterpret_as_int() + match code >> ASCII_CHAR_SET_WORD_SHIFT { + 0 => bits0 = bits0 | bit + 1 => bits1 = bits1 | bit + 2 => bits2 = bits2 | bit + _ => bits3 = bits3 | bit } } + if ascii_only { + self.trim_end_ascii(bits0, bits1, bits2, bits3) + } else { + self.trim_end_with_chars(chars) + } } ///| @@ -817,7 +1000,32 @@ pub fn StringView::trim( self : StringView, chars? : StringView = "\t\n\r ", ) -> StringView { - self.trim_start(chars~).trim_end(chars~) + let mut bits0 = 0U + let mut bits1 = 0U + let mut bits2 = 0U + let mut bits3 = 0U + let mut ascii_only = true + for c in chars { + let code = c.to_uint() + if code >= ASCII_CHAR_SET_LIMIT { + ascii_only = false + break + } + let bit = 1U << (code & ASCII_CHAR_SET_WORD_MASK).reinterpret_as_int() + match code >> ASCII_CHAR_SET_WORD_SHIFT { + 0 => bits0 = bits0 | bit + 1 => bits1 = bits1 | bit + 2 => bits2 = bits2 | bit + _ => bits3 = bits3 | bit + } + } + if ascii_only { + self + .trim_start_ascii(bits0, bits1, bits2, bits3) + .trim_end_ascii(bits0, bits1, bits2, bits3) + } else { + self.trim_start_with_chars(chars).trim_end_with_chars(chars) + } } ///| From ff5fe7c36ee4d433adf7668cdc23f4a87672d357 Mon Sep 17 00:00:00 2001 From: mizchi Date: Tue, 21 Jul 2026 01:14:14 +0900 Subject: [PATCH 2/6] refactor(string): share ASCII charset construction --- builtin/string_char_set_bench_test.mbt | 12 +- builtin/string_methods.mbt | 160 +++++++++---------------- 2 files changed, 63 insertions(+), 109 deletions(-) diff --git a/builtin/string_char_set_bench_test.mbt b/builtin/string_char_set_bench_test.mbt index 5e783abb9..f5d31340a 100644 --- a/builtin/string_char_set_bench_test.mbt +++ b/builtin/string_char_set_bench_test.mbt @@ -24,16 +24,12 @@ let string_char_set_bench_contains_miss : String = "a".repeat( ) ///| -let string_char_set_bench_contains_match_at_end : String = "a".repeat( - string_char_set_bench_size - 1, - ) + - "z" +let string_char_set_bench_contains_match_at_end : String = + "a".repeat(string_char_set_bench_size - 1) + "z" ///| -let string_char_set_bench_trim_start : String = " ".repeat( - string_char_set_bench_size, - ) + - "x" +let string_char_set_bench_trim_start : String = + " ".repeat(string_char_set_bench_size) + "x" ///| let string_char_set_bench_trim_end : String = "x" + diff --git a/builtin/string_methods.mbt b/builtin/string_methods.mbt index 77532fd20..7859fc4b1 100644 --- a/builtin/string_methods.mbt +++ b/builtin/string_methods.mbt @@ -577,6 +577,26 @@ const ASCII_CHAR_SET_WORD_MASK : UInt = 31U ///| const ASCII_CHAR_SET_WORD_SHIFT = 5 +///| +fn build_ascii_char_set(chars : StringView) -> (UInt, UInt, UInt, UInt)? { + let mut bits0 = 0U + let mut bits1 = 0U + let mut bits2 = 0U + let mut bits3 = 0U + for c in chars { + let code = c.to_uint() + guard code < ASCII_CHAR_SET_LIMIT else { return None } + let bit = 1U << (code & ASCII_CHAR_SET_WORD_MASK).reinterpret_as_int() + match code >> ASCII_CHAR_SET_WORD_SHIFT { + 0 => bits0 = bits0 | bit + 1 => bits1 = bits1 | bit + 2 => bits2 = bits2 | bit + _ => bits3 = bits3 | bit + } + } + Some((bits0, bits1, bits2, bits3)) +} + ///| /// Tests membership in a 128-bit ASCII character set represented by four /// scalar words, so callers do not need a temporary heap allocation. @@ -698,38 +718,19 @@ pub fn StringView::contains_any(self : StringView, chars~ : StringView) -> Bool match chars { [] => false [c] => self.contains_char(c) // specialize for single character - _ => { - let mut bits0 = 0U - let mut bits1 = 0U - let mut bits2 = 0U - let mut bits3 = 0U - let mut ascii_only = true - for c in chars { - let code = c.to_uint() - if code >= ASCII_CHAR_SET_LIMIT { - ascii_only = false - break - } - let bit = 1U << (code & ASCII_CHAR_SET_WORD_MASK).reinterpret_as_int() - match code >> ASCII_CHAR_SET_WORD_SHIFT { - 0 => bits0 = bits0 | bit - 1 => bits1 = bits1 | bit - 2 => bits2 = bits2 | bit - _ => bits3 = bits3 | bit - } - } - if ascii_only { - self.contains_any_ascii(bits0, bits1, bits2, bits3) - } else { - for c in self { - if chars.contains_char(c) { - break true + _ => + match build_ascii_char_set(chars) { + Some((bits0, bits1, bits2, bits3)) => + self.contains_any_ascii(bits0, bits1, bits2, bits3) + None => + for c in self { + if chars.contains_char(c) { + break true + } + } nobreak { + false } - } nobreak { - false - } } - } } } @@ -806,6 +807,20 @@ test "contains_any and trim ASCII character sets" { assert_true(view.trim(chars=" \t") == "hello") } +///| +test "build ASCII character set" { + match build_ascii_char_set("a z") { + Some((bits0, bits1, bits2, bits3)) => { + assert_true(ascii_char_set_contains(bits0, bits1, bits2, bits3, 'a')) + assert_true(ascii_char_set_contains(bits0, bits1, bits2, bits3, 'z')) + assert_true(ascii_char_set_contains(bits0, bits1, bits2, bits3, ' ')) + assert_false(ascii_char_set_contains(bits0, bits1, bits2, bits3, 'b')) + } + None => assert_false(true) + } + assert_true(build_ascii_char_set("aπŸ˜€") is None) +} + ///| /// Returns true if this string contains the given character. pub fn StringView::contains_char(self : StringView, c : Char) -> Bool { @@ -877,29 +892,10 @@ pub fn StringView::trim_start( self : StringView, chars? : StringView = "\t\n\r ", ) -> StringView { - let mut bits0 = 0U - let mut bits1 = 0U - let mut bits2 = 0U - let mut bits3 = 0U - let mut ascii_only = true - for c in chars { - let code = c.to_uint() - if code >= ASCII_CHAR_SET_LIMIT { - ascii_only = false - break - } - let bit = 1U << (code & ASCII_CHAR_SET_WORD_MASK).reinterpret_as_int() - match code >> ASCII_CHAR_SET_WORD_SHIFT { - 0 => bits0 = bits0 | bit - 1 => bits1 = bits1 | bit - 2 => bits2 = bits2 | bit - _ => bits3 = bits3 | bit - } - } - if ascii_only { - self.trim_start_ascii(bits0, bits1, bits2, bits3) - } else { - self.trim_start_with_chars(chars) + match build_ascii_char_set(chars) { + Some((bits0, bits1, bits2, bits3)) => + self.trim_start_ascii(bits0, bits1, bits2, bits3) + None => self.trim_start_with_chars(chars) } } @@ -938,29 +934,10 @@ pub fn StringView::trim_end( self : StringView, chars? : StringView = "\t\n\r ", ) -> StringView { - let mut bits0 = 0U - let mut bits1 = 0U - let mut bits2 = 0U - let mut bits3 = 0U - let mut ascii_only = true - for c in chars { - let code = c.to_uint() - if code >= ASCII_CHAR_SET_LIMIT { - ascii_only = false - break - } - let bit = 1U << (code & ASCII_CHAR_SET_WORD_MASK).reinterpret_as_int() - match code >> ASCII_CHAR_SET_WORD_SHIFT { - 0 => bits0 = bits0 | bit - 1 => bits1 = bits1 | bit - 2 => bits2 = bits2 | bit - _ => bits3 = bits3 | bit - } - } - if ascii_only { - self.trim_end_ascii(bits0, bits1, bits2, bits3) - } else { - self.trim_end_with_chars(chars) + match build_ascii_char_set(chars) { + Some((bits0, bits1, bits2, bits3)) => + self.trim_end_ascii(bits0, bits1, bits2, bits3) + None => self.trim_end_with_chars(chars) } } @@ -1000,31 +977,12 @@ pub fn StringView::trim( self : StringView, chars? : StringView = "\t\n\r ", ) -> StringView { - let mut bits0 = 0U - let mut bits1 = 0U - let mut bits2 = 0U - let mut bits3 = 0U - let mut ascii_only = true - for c in chars { - let code = c.to_uint() - if code >= ASCII_CHAR_SET_LIMIT { - ascii_only = false - break - } - let bit = 1U << (code & ASCII_CHAR_SET_WORD_MASK).reinterpret_as_int() - match code >> ASCII_CHAR_SET_WORD_SHIFT { - 0 => bits0 = bits0 | bit - 1 => bits1 = bits1 | bit - 2 => bits2 = bits2 | bit - _ => bits3 = bits3 | bit - } - } - if ascii_only { - self - .trim_start_ascii(bits0, bits1, bits2, bits3) - .trim_end_ascii(bits0, bits1, bits2, bits3) - } else { - self.trim_start_with_chars(chars).trim_end_with_chars(chars) + match build_ascii_char_set(chars) { + Some((bits0, bits1, bits2, bits3)) => + self + .trim_start_ascii(bits0, bits1, bits2, bits3) + .trim_end_ascii(bits0, bits1, bits2, bits3) + None => self.trim_start_with_chars(chars).trim_end_with_chars(chars) } } From 17c56688ab61c7aa6dfed559e0d6a0b8b24f5577 Mon Sep 17 00:00:00 2001 From: mizchi Date: Tue, 21 Jul 2026 03:08:13 +0900 Subject: [PATCH 3/6] style(bench): match stable formatter output --- builtin/string_char_set_bench_test.mbt | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/builtin/string_char_set_bench_test.mbt b/builtin/string_char_set_bench_test.mbt index f5d31340a..5e783abb9 100644 --- a/builtin/string_char_set_bench_test.mbt +++ b/builtin/string_char_set_bench_test.mbt @@ -24,12 +24,16 @@ let string_char_set_bench_contains_miss : String = "a".repeat( ) ///| -let string_char_set_bench_contains_match_at_end : String = - "a".repeat(string_char_set_bench_size - 1) + "z" +let string_char_set_bench_contains_match_at_end : String = "a".repeat( + string_char_set_bench_size - 1, + ) + + "z" ///| -let string_char_set_bench_trim_start : String = - " ".repeat(string_char_set_bench_size) + "x" +let string_char_set_bench_trim_start : String = " ".repeat( + string_char_set_bench_size, + ) + + "x" ///| let string_char_set_bench_trim_end : String = "x" + From c4ba42c4e6e28226ac52d6456c1d179dd0e820b4 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Thu, 20 Aug 2026 23:27:18 +0800 Subject: [PATCH 4/6] perf(builtin): SIMD-accelerate ASCII character set scans Scan eight UTF-16 code units at a time on native and wasm, comparing each block against every set member broadcast to a vector (at most eight members; larger sets keep the scalar bitmap scan). The scalar paths now scan raw code units instead of decoded characters, which is equivalent for ASCII-only sets since an ASCII code unit is never half of a surrogate pair; the JavaScript backend keeps the character iterator for contains_any, where it compiles to a faster loop. Inlining build_ascii_char_set removes the short-input trim overhead. Co-Authored-By: Claude Fable 5 --- builtin/simd.mbt | 7 + builtin/string_methods.mbt | 425 +++++++++++++++++++++++++++++++++---- 2 files changed, 390 insertions(+), 42 deletions(-) diff --git a/builtin/simd.mbt b/builtin/simd.mbt index 1bb82a458..a3a6d5985 100644 --- a/builtin/simd.mbt +++ b/builtin/simd.mbt @@ -126,6 +126,13 @@ fn v128_and(a : V128, b : V128) -> V128 { v128_make(v128_lo(a) & v128_lo(b), v128_hi(a) & v128_hi(b)) } +///| +#cfg(any(target="native", target="wasm")) +#intrinsic("%v128.v128_or") +fn v128_or(a : V128, b : V128) -> V128 { + v128_make(v128_lo(a) | v128_lo(b), v128_hi(a) | v128_hi(b)) +} + ///| #cfg(any(target="native", target="wasm")) #intrinsic("%v128.v128_any_true") diff --git a/builtin/string_methods.mbt b/builtin/string_methods.mbt index 7859fc4b1..01c3f744e 100644 --- a/builtin/string_methods.mbt +++ b/builtin/string_methods.mbt @@ -578,6 +578,13 @@ const ASCII_CHAR_SET_WORD_MASK : UInt = 31U const ASCII_CHAR_SET_WORD_SHIFT = 5 ///| +// At most this many set members are broadcast to vectors by the SIMD scan; +// larger sets use the scalar bitmap scan. +#cfg(any(target="native", target="wasm")) +const ASCII_CHAR_SET_SIMD_MAX_CHARS = 8 + +///| +#inline fn build_ascii_char_set(chars : StringView) -> (UInt, UInt, UInt, UInt)? { let mut bits0 = 0U let mut bits1 = 0U @@ -599,15 +606,19 @@ fn build_ascii_char_set(chars : StringView) -> (UInt, UInt, UInt, UInt)? { ///| /// Tests membership in a 128-bit ASCII character set represented by four -/// scalar words, so callers do not need a temporary heap allocation. +/// scalar words, so callers do not need a temporary heap allocation. Code +/// units outside the ASCII range are never members. +/// +/// An ASCII code unit is never half of a surrogate pair, so for ASCII-only +/// sets scanning raw code units is equivalent to scanning characters. +#inline fn ascii_char_set_contains( bits0 : UInt, bits1 : UInt, bits2 : UInt, bits3 : UInt, - c : Char, + code : UInt, ) -> Bool { - let code = c.to_uint() guard code < ASCII_CHAR_SET_LIMIT else { return false } let bit = 1U << (code & ASCII_CHAR_SET_WORD_MASK).reinterpret_as_int() match code >> ASCII_CHAR_SET_WORD_SHIFT { @@ -619,7 +630,149 @@ fn ascii_char_set_contains( } ///| -fn StringView::contains_any_ascii( +// The caller must ensure `0 <= start <= end <= str.length()`. +#cfg(not(target="js")) +#inline +fn string_contains_any_ascii_scalar( + str : String, + start : Int, + end : Int, + bits0 : UInt, + bits1 : UInt, + bits2 : UInt, + bits3 : UInt, +) -> Bool { + for i in start.. Int { + for pos = start { + if pos < end && + ascii_char_set_contains( + bits0, + bits1, + bits2, + bits3, + str.unsafe_get(pos).to_uint(), + ) { + continue pos + 1 + } else { + break pos + } + } +} + +///| +// Returns the position just past the last code unit in `start.. Int { + for pos = end { + if pos > start && + ascii_char_set_contains( + bits0, + bits1, + bits2, + bits3, + str.unsafe_get(pos - 1).to_uint(), + ) { + continue pos - 1 + } else { + break pos + } + } +} + +///| +// Broadcasts the set member at `index` (repeating the first member for unused +// slots, so the compare tree stays branchless) for the SIMD scan. +#cfg(any(target="native", target="wasm")) +#inline +fn ascii_char_set_splat(chars : StringView, count : Int, index : Int) -> V128 { + i16x8_splat(chars.unsafe_get(if index < count { index } else { 0 })) +} + +///| +// Per-lane mask of which of the eight code units in `block` are members of +// the set broadcast across `s0..s7`. +#cfg(any(target="native", target="wasm")) +#inline +fn ascii_char_set_block_mask( + block : V128, + s0 : V128, + s1 : V128, + s2 : V128, + s3 : V128, + s4 : V128, + s5 : V128, + s6 : V128, + s7 : V128, +) -> V128 { + v128_or( + v128_or( + v128_or(i16x8_eq(block, s0), i16x8_eq(block, s1)), + v128_or(i16x8_eq(block, s2), i16x8_eq(block, s3)), + ), + v128_or( + v128_or(i16x8_eq(block, s4), i16x8_eq(block, s5)), + v128_or(i16x8_eq(block, s6), i16x8_eq(block, s7)), + ), + ) +} + +///| +// On the JavaScript backend the character iterator compiles to a faster loop +// than indexed code-unit reads, and for an ASCII set the two scans agree. +#cfg(target="js") +fn string_contains_any_ascii( + str : String, + start : Int, + end : Int, + _chars : StringView, + bits0 : UInt, + bits1 : UInt, + bits2 : UInt, + bits3 : UInt, +) -> Bool { + StringView::make_view(str, start, end).contains_any_ascii_chars( + bits0, bits1, bits2, bits3, + ) +} + +///| +#cfg(target="js") +fn StringView::contains_any_ascii_chars( self : StringView, bits0 : UInt, bits1 : UInt, @@ -627,7 +780,7 @@ fn StringView::contains_any_ascii( bits3 : UInt, ) -> Bool { for c in self { - if ascii_char_set_contains(bits0, bits1, bits2, bits3, c) { + if ascii_char_set_contains(bits0, bits1, bits2, bits3, c.to_uint()) { return true } } @@ -635,45 +788,175 @@ fn StringView::contains_any_ascii( } ///| -fn StringView::trim_start_ascii( - self : StringView, +#cfg(not(any(target="native", target="wasm", target="js"))) +fn string_contains_any_ascii( + str : String, + start : Int, + end : Int, + _chars : StringView, bits0 : UInt, bits1 : UInt, bits2 : UInt, bits3 : UInt, -) -> StringView { - for x = self { - match x { - [] as v => break v - [c, .. rest] as v => - if ascii_char_set_contains(bits0, bits1, bits2, bits3, c) { - continue rest - } else { - break v - } +) -> Bool { + string_contains_any_ascii_scalar(str, start, end, bits0, bits1, bits2, bits3) +} + +///| +// Scan eight UTF-16 code units at a time on linear-memory backends, comparing +// each block against every set member at once, then scan the remaining tail +// one code unit at a time. +#cfg(any(target="native", target="wasm")) +fn string_contains_any_ascii( + str : String, + start : Int, + end : Int, + chars : StringView, + bits0 : UInt, + bits1 : UInt, + bits2 : UInt, + bits3 : UInt, +) -> Bool { + let count = chars.length() + guard count <= ASCII_CHAR_SET_SIMD_MAX_CHARS && start + 8 <= end else { + return string_contains_any_ascii_scalar( + str, start, end, bits0, bits1, bits2, bits3, + ) + } + let s0 = ascii_char_set_splat(chars, count, 0) + let s1 = ascii_char_set_splat(chars, count, 1) + let s2 = ascii_char_set_splat(chars, count, 2) + let s3 = ascii_char_set_splat(chars, count, 3) + let s4 = ascii_char_set_splat(chars, count, 4) + let s5 = ascii_char_set_splat(chars, count, 5) + let s6 = ascii_char_set_splat(chars, count, 6) + let s7 = ascii_char_set_splat(chars, count, 7) + let tail_start = for pos = start; pos + 8 <= end; { + let block = v128_load_i16x8(str, pos) + if v128_any_true( + ascii_char_set_block_mask(block, s0, s1, s2, s3, s4, s5, s6, s7), + ) { + return true } + continue pos + 8 + } nobreak { + pos } + string_contains_any_ascii_scalar( + str, tail_start, end, bits0, bits1, bits2, bits3, + ) } ///| -fn StringView::trim_end_ascii( - self : StringView, +#cfg(not(any(target="native", target="wasm"))) +fn string_trim_start_ascii( + str : String, + start : Int, + end : Int, + _chars : StringView, bits0 : UInt, bits1 : UInt, bits2 : UInt, bits3 : UInt, -) -> StringView { - for x = self { - match x { - [] as v => break v - [.. rest, c] as v => - if ascii_char_set_contains(bits0, bits1, bits2, bits3, c) { - continue rest - } else { - break v - } +) -> Int { + string_trim_start_ascii_scalar(str, start, end, bits0, bits1, bits2, bits3) +} + +///| +// Skip eight fully-trimmable code units at a time on linear-memory backends; +// the first block containing a non-member (and the sub-8 tail) is finished by +// the scalar scan. +#cfg(any(target="native", target="wasm")) +fn string_trim_start_ascii( + str : String, + start : Int, + end : Int, + chars : StringView, + bits0 : UInt, + bits1 : UInt, + bits2 : UInt, + bits3 : UInt, +) -> Int { + let count = chars.length() + guard count <= ASCII_CHAR_SET_SIMD_MAX_CHARS && start + 8 <= end else { + return string_trim_start_ascii_scalar( + str, start, end, bits0, bits1, bits2, bits3, + ) + } + let s0 = ascii_char_set_splat(chars, count, 0) + let s1 = ascii_char_set_splat(chars, count, 1) + let s2 = ascii_char_set_splat(chars, count, 2) + let s3 = ascii_char_set_splat(chars, count, 3) + let s4 = ascii_char_set_splat(chars, count, 4) + let s5 = ascii_char_set_splat(chars, count, 5) + let s6 = ascii_char_set_splat(chars, count, 6) + let s7 = ascii_char_set_splat(chars, count, 7) + let boundary = for pos = start; pos + 8 <= end; { + let block = v128_load_i16x8(str, pos) + let mask = ascii_char_set_block_mask(block, s0, s1, s2, s3, s4, s5, s6, s7) + if i16x8_bitmask(mask) != 0xFF { + break pos + } + continue pos + 8 + } nobreak { + pos + } + string_trim_start_ascii_scalar(str, boundary, end, bits0, bits1, bits2, bits3) +} + +///| +#cfg(not(any(target="native", target="wasm"))) +fn string_trim_end_ascii( + str : String, + start : Int, + end : Int, + _chars : StringView, + bits0 : UInt, + bits1 : UInt, + bits2 : UInt, + bits3 : UInt, +) -> Int { + string_trim_end_ascii_scalar(str, start, end, bits0, bits1, bits2, bits3) +} + +///| +// Mirror of `string_trim_start_ascii`, scanning blocks backward from the end. +#cfg(any(target="native", target="wasm")) +fn string_trim_end_ascii( + str : String, + start : Int, + end : Int, + chars : StringView, + bits0 : UInt, + bits1 : UInt, + bits2 : UInt, + bits3 : UInt, +) -> Int { + let count = chars.length() + guard count <= ASCII_CHAR_SET_SIMD_MAX_CHARS && start + 8 <= end else { + return string_trim_end_ascii_scalar( + str, start, end, bits0, bits1, bits2, bits3, + ) + } + let s0 = ascii_char_set_splat(chars, count, 0) + let s1 = ascii_char_set_splat(chars, count, 1) + let s2 = ascii_char_set_splat(chars, count, 2) + let s3 = ascii_char_set_splat(chars, count, 3) + let s4 = ascii_char_set_splat(chars, count, 4) + let s5 = ascii_char_set_splat(chars, count, 5) + let s6 = ascii_char_set_splat(chars, count, 6) + let s7 = ascii_char_set_splat(chars, count, 7) + let boundary = for pos = end; pos - 8 >= start; { + let block = v128_load_i16x8(str, pos - 8) + let mask = ascii_char_set_block_mask(block, s0, s1, s2, s3, s4, s5, s6, s7) + if i16x8_bitmask(mask) != 0xFF { + break pos } + continue pos - 8 + } nobreak { + pos } + string_trim_end_ascii_scalar(str, start, boundary, bits0, bits1, bits2, bits3) } ///| @@ -721,7 +1004,16 @@ pub fn StringView::contains_any(self : StringView, chars~ : StringView) -> Bool _ => match build_ascii_char_set(chars) { Some((bits0, bits1, bits2, bits3)) => - self.contains_any_ascii(bits0, bits1, bits2, bits3) + string_contains_any_ascii( + self.str(), + self.start(), + self.end(), + chars, + bits0, + bits1, + bits2, + bits3, + ) None => for c in self { if chars.contains_char(c) { @@ -811,10 +1103,18 @@ test "contains_any and trim ASCII character sets" { test "build ASCII character set" { match build_ascii_char_set("a z") { Some((bits0, bits1, bits2, bits3)) => { - assert_true(ascii_char_set_contains(bits0, bits1, bits2, bits3, 'a')) - assert_true(ascii_char_set_contains(bits0, bits1, bits2, bits3, 'z')) - assert_true(ascii_char_set_contains(bits0, bits1, bits2, bits3, ' ')) - assert_false(ascii_char_set_contains(bits0, bits1, bits2, bits3, 'b')) + assert_true( + ascii_char_set_contains(bits0, bits1, bits2, bits3, 'a'.to_uint()), + ) + assert_true( + ascii_char_set_contains(bits0, bits1, bits2, bits3, 'z'.to_uint()), + ) + assert_true( + ascii_char_set_contains(bits0, bits1, bits2, bits3, ' '.to_uint()), + ) + assert_false( + ascii_char_set_contains(bits0, bits1, bits2, bits3, 'b'.to_uint()), + ) } None => assert_false(true) } @@ -893,8 +1193,19 @@ pub fn StringView::trim_start( chars? : StringView = "\t\n\r ", ) -> StringView { match build_ascii_char_set(chars) { - Some((bits0, bits1, bits2, bits3)) => - self.trim_start_ascii(bits0, bits1, bits2, bits3) + Some((bits0, bits1, bits2, bits3)) => { + let start = string_trim_start_ascii( + self.str(), + self.start(), + self.end(), + chars, + bits0, + bits1, + bits2, + bits3, + ) + StringView::make_view(self.str(), start, self.end()) + } None => self.trim_start_with_chars(chars) } } @@ -935,8 +1246,19 @@ pub fn StringView::trim_end( chars? : StringView = "\t\n\r ", ) -> StringView { match build_ascii_char_set(chars) { - Some((bits0, bits1, bits2, bits3)) => - self.trim_end_ascii(bits0, bits1, bits2, bits3) + Some((bits0, bits1, bits2, bits3)) => { + let end = string_trim_end_ascii( + self.str(), + self.start(), + self.end(), + chars, + bits0, + bits1, + bits2, + bits3, + ) + StringView::make_view(self.str(), self.start(), end) + } None => self.trim_end_with_chars(chars) } } @@ -978,10 +1300,29 @@ pub fn StringView::trim( chars? : StringView = "\t\n\r ", ) -> StringView { match build_ascii_char_set(chars) { - Some((bits0, bits1, bits2, bits3)) => - self - .trim_start_ascii(bits0, bits1, bits2, bits3) - .trim_end_ascii(bits0, bits1, bits2, bits3) + Some((bits0, bits1, bits2, bits3)) => { + let start = string_trim_start_ascii( + self.str(), + self.start(), + self.end(), + chars, + bits0, + bits1, + bits2, + bits3, + ) + let end = string_trim_end_ascii( + self.str(), + start, + self.end(), + chars, + bits0, + bits1, + bits2, + bits3, + ) + StringView::make_view(self.str(), start, end) + } None => self.trim_start_with_chars(chars).trim_end_with_chars(chars) } } From 0593aec8035d523082de2f93d83e39122f145d8d Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Thu, 20 Aug 2026 23:27:18 +0800 Subject: [PATCH 5/6] test(builtin): property-test ASCII character set scans Add quickcheck properties pinning contains_any and the trims against a character-by-character model on adversarial strings and offset views, an exhaustive sweep across SIMD block boundaries for every set size class, lone-surrogate cases, and short-input benchmarks. Co-Authored-By: Claude Fable 5 --- builtin/string_char_set_bench_test.mbt | 34 ++++ builtin/string_char_set_quickcheck_test.mbt | 204 ++++++++++++++++++++ 2 files changed, 238 insertions(+) create mode 100644 builtin/string_char_set_quickcheck_test.mbt diff --git a/builtin/string_char_set_bench_test.mbt b/builtin/string_char_set_bench_test.mbt index 5e783abb9..93094464f 100644 --- a/builtin/string_char_set_bench_test.mbt +++ b/builtin/string_char_set_bench_test.mbt @@ -81,3 +81,37 @@ test "bench StringView::trim ASCII n=100000" (it : @bench.T) { " ".repeat(string_char_set_bench_size) it.bench(fn() { it.keep(input.trim().length()) }) } + +///| +let string_char_set_short_trim_inputs : Array[String] = [ + " hello ", "x", "", " ", "no_trim_needed", "\t indented line \n", +] + +///| +let string_char_set_short_contains_inputs : Array[String] = [ + "hello,world", "a=b&c=d", "plain", "", +] + +///| +test "bench StringView::trim short inputs" (it : @bench.T) { + it.bench(fn() { + let mut total = 0 + for s in string_char_set_short_trim_inputs { + total += s.trim().length() + } + it.keep(total) + }) +} + +///| +test "bench StringView::contains_any short inputs" (it : @bench.T) { + it.bench(fn() { + let mut hits = 0 + for s in string_char_set_short_contains_inputs { + if s.contains_any(chars=",&= ") { + hits += 1 + } + } + it.keep(hits) + }) +} diff --git a/builtin/string_char_set_quickcheck_test.mbt b/builtin/string_char_set_quickcheck_test.mbt new file mode 100644 index 000000000..4195a7d92 --- /dev/null +++ b/builtin/string_char_set_quickcheck_test.mbt @@ -0,0 +1,204 @@ +// Copyright 2026 International Digital Economy Academy +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Property-based tests for the character-set paths of `contains_any` and +// `trim`/`trim_start`/`trim_end`: the bitmap and SIMD scans must agree with a +// straightforward character-by-character model, on strings and on views with +// non-zero offsets, for ASCII sets of every size class (SIMD-broadcast, +// bitmap-only, and the non-ASCII fallback). + +///| +/// Characters a haystack is built from: mostly ASCII (including every member +/// of the generated sets and both sides of the 0x20 boundary), plus non-ASCII +/// BMP and non-BMP characters so surrogate pairs appear. +fn haystack_char(seed : Int) -> Char { + match seed & 0xF { + 0 => ' ' + 1 => '\t' + 2 => '\n' + 3 => '\r' + 4 => ',' + 5 => ';' + 6 => 'z' + 7 => 'a' + 8 => '\u{00}' + 9 => '\u{1F}' + 10 => '\u{7F}' + 11 => 'δΈ­' + 12 => 'πŸ˜€' + _ => (0x20 + ((seed >> 4) & 0x3F)).unsafe_to_char() + } +} + +///| +/// Set members: mostly ASCII delimiters; occasionally non-ASCII, which sends +/// the whole set down the fallback path. +fn set_char(seed : Int) -> Char { + match seed & 0xF { + 0 => ' ' + 1 => '\t' + 2 => '\n' + 3 => '\r' + 4 => ',' + 5 => ';' + 6 => '/' + 7 => 'z' + 8 => '0' + 9 => '\u{00}' + 10 => '\u{7F}' + 11 => 'δΈ­' + 12 => 'πŸ˜€' + _ => 'a' + } +} + +///| +fn chars_to_string(chars : Array[Char]) -> String { + let buf = StringBuilder(size_hint=chars.length()) + for c in chars { + buf.write_char(c) + } + buf.to_string() +} + +///| +fn model_member(chars : String, c : Char) -> Bool { + for d in chars { + if c == d { + return true + } + } + false +} + +///| +fn model_contains_any(s : String, chars : String) -> Bool { + for c in s { + if model_member(chars, c) { + return true + } + } + false +} + +///| +fn model_trim_start(s : String, chars : String) -> String { + let arr = s.to_array() + let mut start = 0 + while start < arr.length() && model_member(chars, arr[start]) { + start += 1 + } + chars_to_string(arr[start:].to_owned()) +} + +///| +fn model_trim_end(s : String, chars : String) -> String { + let arr = s.to_array() + let mut end = arr.length() + while end > 0 && model_member(chars, arr[end - 1]) { + end -= 1 + } + chars_to_string(arr[:end].to_owned()) +} + +///| +/// Embeds `middle` between `prefix`/`suffix` and returns the view covering +/// exactly `middle`, so the scans see non-zero view offsets. +fn embedded_view( + prefix : String, + middle : String, + suffix : String, +) -> StringView { + let full = prefix + middle + suffix + full[prefix.length():prefix.length() + middle.length()] +} + +///| +test "quickcheck: contains_any agrees with the character model" { + @quickcheck.check(count=300, (input : (Array[Int], Array[Int])) => { + let (hay_seeds, set_seeds) = input + let hay = chars_to_string(hay_seeds.map(haystack_char)) + let set = chars_to_string(set_seeds.map(set_char)) + let expected = model_contains_any(hay, set) + guard hay.contains_any(chars=set) == expected else { return false } + // The same scan through a view with non-zero offsets. + embedded_view("ab", hay, "yz").contains_any(chars=set) == expected + }) +} + +///| +test "quickcheck: trims agree with the character model" { + @quickcheck.check(count=300, (input : (Array[Int], Array[Int])) => { + let (hay_seeds, set_seeds) = input + let hay = chars_to_string(hay_seeds.map(haystack_char)) + let set = chars_to_string(set_seeds.map(set_char)) + let view = embedded_view(" πŸ˜€", hay, "πŸ˜€ ") + guard view.trim_start(chars=set).to_owned() == model_trim_start(hay, set) else { + return false + } + guard view.trim_end(chars=set).to_owned() == model_trim_end(hay, set) else { + return false + } + // The fused trim must agree with composing the two one-sided trims. + view.trim(chars=set).to_owned() == + model_trim_end(model_trim_start(hay, set), set) + }) +} + +///| +/// Exhaustively places a set member at every position of an otherwise clean +/// string for every length spanning several 8-unit SIMD blocks, for a +/// SIMD-broadcast-sized set, a bitmap-only-sized set, and the default +/// whitespace set. +test "char set boundary sweep" { + let sets = [" \t\n\r", ",;:.!?", "abcdefghij"] + for set in sets { + let probe = set[0:1].to_owned() + for len in 0..<=24 { + let clean = "x".repeat(len) + assert_false(clean.contains_any(chars=set)) + assert_eq(clean.trim(chars=set).to_owned(), clean) + for pos in 0.. Date: Thu, 20 Aug 2026 23:36:32 +0800 Subject: [PATCH 6/6] fix(builtin): keep empty character sets off the SIMD trim path An empty set reached ascii_char_set_splat, which read code unit 0 beyond the set view's bounds; an empty view over a longer backing string then broadcast the backing character as a set member and trimmed data that should have been kept. Found by adversarial review. Guard the SIMD scans on a non-empty set, cover the empty-set and empty-view cases directly, and pass generated sets through offset views in the quickcheck properties so out-of-view reads see sentinel characters. Co-Authored-By: Claude Fable 5 --- builtin/string_char_set_quickcheck_test.mbt | 34 +++++++++++++++++---- builtin/string_methods.mbt | 6 ++-- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/builtin/string_char_set_quickcheck_test.mbt b/builtin/string_char_set_quickcheck_test.mbt index 4195a7d92..1ed7f11ae 100644 --- a/builtin/string_char_set_quickcheck_test.mbt +++ b/builtin/string_char_set_quickcheck_test.mbt @@ -130,10 +130,13 @@ test "quickcheck: contains_any agrees with the character model" { let (hay_seeds, set_seeds) = input let hay = chars_to_string(hay_seeds.map(haystack_char)) let set = chars_to_string(set_seeds.map(set_char)) + // The set is passed as a view with non-zero offsets, so a scan that read + // beyond the set view would see the sentinel characters instead. + let set_view = embedded_view("Q", set, "Q") let expected = model_contains_any(hay, set) - guard hay.contains_any(chars=set) == expected else { return false } - // The same scan through a view with non-zero offsets. - embedded_view("ab", hay, "yz").contains_any(chars=set) == expected + guard hay.contains_any(chars=set_view) == expected else { return false } + // The same scan through a haystack view with non-zero offsets. + embedded_view("ab", hay, "yz").contains_any(chars=set_view) == expected }) } @@ -144,14 +147,16 @@ test "quickcheck: trims agree with the character model" { let hay = chars_to_string(hay_seeds.map(haystack_char)) let set = chars_to_string(set_seeds.map(set_char)) let view = embedded_view(" πŸ˜€", hay, "πŸ˜€ ") - guard view.trim_start(chars=set).to_owned() == model_trim_start(hay, set) else { + let set_view = embedded_view("Q", set, "Q") + guard view.trim_start(chars=set_view).to_owned() == + model_trim_start(hay, set) else { return false } - guard view.trim_end(chars=set).to_owned() == model_trim_end(hay, set) else { + guard view.trim_end(chars=set_view).to_owned() == model_trim_end(hay, set) else { return false } // The fused trim must agree with composing the two one-sided trims. - view.trim(chars=set).to_owned() == + view.trim(chars=set_view).to_owned() == model_trim_end(model_trim_start(hay, set), set) }) } @@ -190,6 +195,23 @@ test "char set boundary sweep" { } } +///| +/// An empty set trims and matches nothing β€” including an empty view whose +/// backing string continues past the view, where an out-of-view read would +/// wrongly treat the backing character as a set member. +test "char set empty set" { + let empty_view = "b"[0:0] + let bs = "b".repeat(8) + "x" + let sb = "x" + "b".repeat(8) + assert_eq(bs.trim_start(chars=empty_view).to_owned(), bs) + assert_eq(sb.trim_end(chars=empty_view).to_owned(), sb) + assert_eq(bs.trim(chars=empty_view).to_owned(), bs) + assert_false(bs.contains_any(chars=empty_view)) + assert_eq(bs.trim_start(chars="").to_owned(), bs) + assert_eq(sb.trim_end(chars="").to_owned(), sb) + assert_eq(bs.trim(chars="").to_owned(), bs) +} + ///| /// A lone surrogate is never a member of an ASCII set, and trimming stops at /// it without touching it. diff --git a/builtin/string_methods.mbt b/builtin/string_methods.mbt index 01c3f744e..d7649beea 100644 --- a/builtin/string_methods.mbt +++ b/builtin/string_methods.mbt @@ -818,7 +818,7 @@ fn string_contains_any_ascii( bits3 : UInt, ) -> Bool { let count = chars.length() - guard count <= ASCII_CHAR_SET_SIMD_MAX_CHARS && start + 8 <= end else { + guard count >= 1 && count <= ASCII_CHAR_SET_SIMD_MAX_CHARS && start + 8 <= end else { return string_contains_any_ascii_scalar( str, start, end, bits0, bits1, bits2, bits3, ) @@ -878,7 +878,7 @@ fn string_trim_start_ascii( bits3 : UInt, ) -> Int { let count = chars.length() - guard count <= ASCII_CHAR_SET_SIMD_MAX_CHARS && start + 8 <= end else { + guard count >= 1 && count <= ASCII_CHAR_SET_SIMD_MAX_CHARS && start + 8 <= end else { return string_trim_start_ascii_scalar( str, start, end, bits0, bits1, bits2, bits3, ) @@ -933,7 +933,7 @@ fn string_trim_end_ascii( bits3 : UInt, ) -> Int { let count = chars.length() - guard count <= ASCII_CHAR_SET_SIMD_MAX_CHARS && start + 8 <= end else { + guard count >= 1 && count <= ASCII_CHAR_SET_SIMD_MAX_CHARS && start + 8 <= end else { return string_trim_end_ascii_scalar( str, start, end, bits0, bits1, bits2, bits3, )