diff --git a/builtin/string_find_adversary_bench_test.mbt b/builtin/string_find_adversary_bench_test.mbt new file mode 100644 index 000000000..1d6e33b3c --- /dev/null +++ b/builtin/string_find_adversary_bench_test.mbt @@ -0,0 +1,56 @@ +// Copyright 2026 International Digital Economy Academy +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Worst-case benchmarks for substring search: needles whose anchors and +// long prefixes recur at every haystack position, which degrade the +// two-anchor filter to its fallback — plus fast-path guards that must not +// move when the fallback changes. + +///| +fn string_find_adversary_needle(half : Int) -> String { + String::make(half, 'a') + "Z" + String::make(half - 1, 'a') +} + +///| +test "bench find adversary m=64 n=4096" (it : @bench.T) { + let haystack = String::make(4096, 'a') + let needle = string_find_adversary_needle(32) + it.bench(fn() { it.keep(haystack.find(needle)) }) +} + +///| +test "bench rev_find adversary m=64 n=4096" (it : @bench.T) { + let haystack = String::make(4096, 'a') + let needle = string_find_adversary_needle(32) + it.bench(fn() { it.keep(haystack.rev_find(needle)) }) +} + +///| +test "bench find adversary m=512 n=65536" (it : @bench.T) { + let haystack = String::make(65536, 'a') + let needle = string_find_adversary_needle(256) + it.bench(fn() { it.keep(haystack.find(needle)) }) +} + +///| +test "bench find dense miss fast path n=4096" (it : @bench.T) { + let haystack = String::make(4096, 'a') + it.bench(fn() { it.keep(haystack.find("aaaaZ")) }) +} + +///| +test "bench find rare hit end fast path n=4096" (it : @bench.T) { + let haystack = String::make(4092, 'a') + "Zabc" + it.bench(fn() { it.keep(haystack.find("Zabc")) }) +} diff --git a/builtin/string_find_code_unit.mbt b/builtin/string_find_code_unit.mbt index a66252a5b..e48515dda 100644 --- a/builtin/string_find_code_unit.mbt +++ b/builtin/string_find_code_unit.mbt @@ -211,7 +211,7 @@ fn find_by_two_anchors(target : StringView, pattern : StringView) -> Int? { let failures = failures + 1 let scanned = found - target_start if two_anchor_should_fallback(failures, scanned) { - break find_pattern_scalar_from(target, pattern, scanned + 1) + break find_pattern_kmp_from(target, pattern, scanned + 1) } continue found + 1, failures } nobreak { @@ -259,7 +259,7 @@ fn rev_find_by_two_anchors(target : StringView, pattern : StringView) -> Int? { let failures = failures + 1 let scanned = last_candidate - candidate if two_anchor_should_fallback(failures, scanned) { - break rev_find_pattern_scalar_before(target, pattern, candidate) + break rev_find_pattern_kmp_before(target, pattern, candidate) } continue found, failures } nobreak { @@ -269,80 +269,133 @@ fn rev_find_by_two_anchors(target : StringView, pattern : StringView) -> Int? { ///| // Dense first/last-anchor matches make repeated SIMD candidate scans more -// expensive than direct comparison. Cut over after enough false candidates, -// following the guarded-scanner strategy used by Bytes search. +// expensive than direct comparison. Cut over to the guaranteed-linear KMP +// fallback either early, when failures pile up relative to progress +// (`failures > 4 + scanned / 8` — as soon as the 5th failure for +// candidates packed near the scan start), or at the hard cap +// (`failures > 64`, i.e. the 65th failed verification). Either way the +// failure count at cutover is at most 65, keeping pre-cutover +// verification work at O(pattern), so find/rev_find stay +// O(target + pattern) even on adversarial inputs. #inline fn two_anchor_should_fallback(failures : Int, scanned : Int) -> Bool { failures > 64 || failures > 4 + scanned / 8 } ///| -// Direct forward fallback used after the two-anchor filter encounters dense -// false positives. It checks the middle first because the cutover has already -// established that first/last matches are not selective. `start` is relative -// to `target`. -fn find_pattern_scalar_from( +// Longest-proper-border table for the KMP fallbacks: `table[i]` is the +// length of the longest proper prefix of `pattern[0..=i]` that is also a +// suffix of it. Only built after the two-anchor filter cuts over, so the +// O(pattern) allocation is paid exclusively on pathological inputs. +fn kmp_failure_table(pattern : StringView) -> FixedArray[Int] { + let m = pattern.length() + let table = FixedArray::make(m, 0) + let mut k = 0 + for i in 1.. 0 && c != pattern.unsafe_get(k) { + k = table[k - 1] + } + if c == pattern.unsafe_get(k) { + k += 1 + } + table[i] = k + } + table +} + +///| +// Guaranteed-linear forward fallback used after the two-anchor filter +// encounters dense false positives: KMP over the remaining candidates, so +// the whole search stays O(target + pattern) even when both anchors and +// long pattern prefixes recur throughout the target. Returns the first +// occurrence starting at a target-relative position >= `start`. +fn find_pattern_kmp_from( target : StringView, pattern : StringView, start : Int, ) -> Int? { - let pattern_len = pattern.length() - let last_offset = pattern_len - 1 - let first = pattern.unsafe_get(0) - let last = pattern.unsafe_get(last_offset) - let last_candidate = target.length() - pattern_len - for candidate in start..<=last_candidate { - let middle_matches = for i in 1.. 0 && c != pattern.unsafe_get(k) { + k = table[k - 1] } - if middle_matches && - target.unsafe_get(candidate) == first && - target.unsafe_get(candidate + last_offset) == last { - break Some(candidate) + if c == pattern.unsafe_get(k) { + k += 1 + } + if k == m { + return Some(i - m + 1) } - } nobreak { - None } + None } ///| -// Direct reverse fallback used after the two-anchor filter encounters dense -// false positives. It checks the middle first because the cutover has already -// established that first/last matches are not selective. `candidate_end` is -// target-relative and exclusive. -fn rev_find_pattern_scalar_before( +// Guaranteed-linear reverse fallback used after the two-anchor filter +// encounters dense false positives: forward KMP over the prefix that can +// still contain a hit, keeping the rightmost match, so the whole search +// stays O(target + pattern). Returns the last occurrence starting at a +// target-relative position strictly below `candidate_end` (exclusive). +// The caller must ensure +// `candidate_end <= target.length() - pattern.length() + 1`, so that the +// scan below stays in bounds. +fn rev_find_pattern_kmp_before( target : StringView, pattern : StringView, candidate_end : Int, ) -> Int? { guard candidate_end > 0 else { return None } - let pattern_len = pattern.length() - let last_offset = pattern_len - 1 - let first = pattern.unsafe_get(0) - let last = pattern.unsafe_get(last_offset) - for candidate = candidate_end - 1; candidate >= 0; { - let middle_matches = for i in 1.. 0 && c != pattern.unsafe_get(k) { + k = table[k - 1] } - if middle_matches && - target.unsafe_get(candidate) == first && - target.unsafe_get(candidate + last_offset) == last { - break Some(candidate) + if c == pattern.unsafe_get(k) { + k += 1 } - continue candidate - 1 - } nobreak { + if k == m { + best = i - m + 1 + // keep scanning: a later (more rightward) overlapping match wins + k = table[k - 1] + } + } + if best >= 0 { + Some(best) + } else { None } } +///| +test "kmp fallbacks handle periodic and overlapping patterns" { + // failure table borders matter for periodic patterns + let t : StringView = "aaaaaa" + assert_true(find_pattern_kmp_from(t, "aaa", 0) is Some(0)) + assert_true(find_pattern_kmp_from(t, "aaa", 2) is Some(2)) + assert_true(find_pattern_kmp_from(t, "aaa", 4) is None) + assert_true(rev_find_pattern_kmp_before(t, "aaa", 4) is Some(3)) + assert_true(rev_find_pattern_kmp_before(t, "aaa", 1) is Some(0)) + let u : StringView = "ababcababab" + assert_true(find_pattern_kmp_from(u, "abab", 0) is Some(0)) + assert_true(find_pattern_kmp_from(u, "abab", 1) is Some(5)) + assert_true(rev_find_pattern_kmp_before(u, "abab", 8) is Some(7)) + assert_true(rev_find_pattern_kmp_before(u, "ababc", 7) is Some(0)) + assert_true(find_pattern_kmp_from(u, "abcabc", 0) is None) + // start beyond any match and empty prefix bound + assert_true(rev_find_pattern_kmp_before(u, "abab", 0) is None) +} + ///| // Compares two raw UTF-16 ranges. The caller must ensure both ranges are in // bounds; using backing strings directly also permits isolated surrogate code diff --git a/builtin/string_find_fallback_test.mbt b/builtin/string_find_fallback_test.mbt new file mode 100644 index 000000000..cd879021d --- /dev/null +++ b/builtin/string_find_fallback_test.mbt @@ -0,0 +1,161 @@ +// Copyright 2026 International Digital Economy Academy +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// End-to-end coverage for the guaranteed-linear KMP fallback behind the +// SIMD two-anchor substring search. The cutover has two triggers: an +// early ratio check (`failures > 4 + scanned / 8`) and a hard cap +// (`failures > 64`). The decoy constructions here space their false +// candidates one period apart, which keeps the ratio check quiet, so +// these tests deterministically force the HARD-CAP path by providing +// more than 65 positions where both anchor code units match but +// verification fails, in the direction under test, before the +// interesting region — misses, hits beyond the cutover point, +// overlapping hits resolved through the fallback, and naive-reference +// cross-checks. + +///| +fn adversarial_needle() -> String { + // both anchors are 'a' and a 32-unit prefix of the needle recurs at every + // position of an all-'a' haystack: every candidate passes the anchor test + // and fails only deep inside verification + String::make(32, 'a') + "Z" + String::make(31, 'a') +} + +///| +test "dense-anchor miss stays correct through the kmp fallback" { + let haystack = String::make(4096, 'a') + debug_inspect(haystack.find(adversarial_needle()), content="None") + debug_inspect(haystack.rev_find(adversarial_needle()), content="None") +} + +///| +test "hit located after the cutover point is found" { + let needle = adversarial_needle() + let haystack = String::make(2000, 'a') + needle + String::make(50, 'a') + debug_inspect(haystack.find(needle), content="Some(2000)") + debug_inspect(haystack.rev_find(needle), content="Some(2000)") + // a second occurrence: find returns the first, rev_find the last + let doubled = String::make(1000, 'a') + + needle + + String::make(1000, 'a') + + needle + debug_inspect(doubled.find(needle), content="Some(1000)") + debug_inspect(doubled.rev_find(needle), content="Some(2064)") +} + +///| +// One decoy block per repetition: for the periodic needle used below +// (anchors 'a' and 'b', length 82), each block contributes exactly one +// position whose anchors match but whose interior verification fails. +fn decoy_blocks(count : Int) -> String { + (String::make(81, 'a') + "b").repeat(count) +} + +///| +// 40a b 40a b 40a b: contains the 82-unit needle 40a b 40a b at +// offsets 0 and 41 — two overlapping occurrences. +fn overlapping_hits_block() -> String { + let half = String::make(40, 'a') + "b" + half + half + half +} + +///| +test "forward cutover then kmp finds the first hit" { + let needle = String::make(40, 'a') + "b" + String::make(40, 'a') + "b" + // 70 false candidates precede the hits, exhausting the 65-failure budget; + // the first genuine occurrence straddles the last decoy block and the + // hits block (40 decoy 'a's, the decoy 'b' at 5739, 40 hit 'a's, and the + // hit 'b' at 5780), so kmp must report 5699, not the hits-block start + let haystack = decoy_blocks(70) + overlapping_hits_block() + debug_inspect(haystack.find(needle), content="Some(5699)") +} + +///| +test "reverse cutover then kmp keeps the rightmost overlapping hit" { + let needle = String::make(40, 'a') + "b" + String::make(40, 'a') + "b" + // hits sit at the start; 70 false candidates follow them, so the reverse + // scan burns its budget before reaching the hits and hands off to the + // kmp fallback, which must resolve the overlap to the rightmost start + let haystack = overlapping_hits_block() + decoy_blocks(70) + debug_inspect(haystack.rev_find(needle), content="Some(41)") + // sanity: the forward search over the same input needs no fallback + debug_inspect(haystack.find(needle), content="Some(0)") +} + +///| +fn naive_find(haystack : String, needle : String) -> Int? { + let n = haystack.length() + let m = needle.length() + for start in 0..<=(n - m) { + let matched = for i in 0.. Int? { + let n = haystack.length() + let m = needle.length() + for start = n - m; start >= 0; { + let matched = for i in 0.. String { + let sb = StringBuilder(size_hint=bits.length()) + for b in bits { + sb.write_char(if b { 'a' } else { 'b' }) + } + sb.to_string() +} + +///| +test "quickcheck: find and rev_find agree with naive references" { + @quickcheck.check( + (input : (Array[Bool], Array[Bool])) => { + let (text_bits, needle_bits) = input + let text = bools_to_ab(text_bits) + // clamp the needle into the multi-unit regime the two-anchor search + // handles; single-unit and empty needles take separate code paths + guard needle_bits.length() >= 2 else { return true } + let needle = bools_to_ab(needle_bits) + text.find(needle) == naive_find(text, needle) && + text.rev_find(needle) == naive_rev_find(text, needle) + }, + count=200, + ) +} + +///| +test "quickcheck: fallback-forcing inputs agree with naive references" { + @quickcheck.check( + (input : (UInt, UInt, UInt, Array[Bool])) => { + let (p0, q0, flip0, mix) = input + // needle 'a'*p + 'b' + 'a'*q: anchors are 'a'/'a' (or 'a'/'b' when + // q == 0), interior mostly 'a' — the adversarial family + let p = (p0 % 14).reinterpret_as_int() + 1 + let q = (q0 % 15).reinterpret_as_int() + let needle = String::make(p, 'a') + "b" + String::make(q, 'a') + let m = needle.length() + guard m >= 3 else { return true } + // corrupted block: one interior char flipped, anchors intact, so a + // block-start candidate passes the anchor test and fails verification + let flip = 1 + + (flip0 % (m - 2).reinterpret_as_uint()).reinterpret_as_int() + let sb = StringBuilder(size_hint=m) + for i in 0.. Int? { find_by_two_anchors(self, str) } } - // TODO: When the pattern string is long (>= 256), - // consider using Two-Way algorithm to ensure linear time complexity. + // Worst-case linearity: after dense false-anchor candidates the search + // cuts over to a KMP fallback (see the two-anchor search internals), so + // the total cost stays O(self + str) even on adversarial inputs. } ///| @@ -145,8 +146,9 @@ pub fn StringView::rev_find(self : StringView, str : StringView) -> Int? { rev_find_by_two_anchors(self, str) } } - // TODO: When the pattern string is long (>= 256), - // consider using Two-Way algorithm to ensure linear time complexity. + // Worst-case linearity: after dense false-anchor candidates the search + // cuts over to a KMP fallback (see the two-anchor search internals), so + // the total cost stays O(self + str) even on adversarial inputs. } ///|