Skip to content

Commit aa0a2f8

Browse files
committed
Simplify static_dict
* Convert `IsMatch(dict, ...)` into a member method. The original `IsMatch` is marked as deprecated - and we probably can delete it, but it will require a major release due to being a breaking change. So for now, lets keep it, but remove it when releasing a new breaking version * Same for `BrotliFindAllStaticDictionaryMatches` and a few more methods (they were not moved though to allow for easier review) * lots of internal simplifications - like using direct comparisons, using bools, etc
1 parent 85196be commit aa0a2f8

File tree

4 files changed

+497
-566
lines changed

4 files changed

+497
-566
lines changed

src/enc/backward_references/hq.rs

+3-6
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,7 @@ use crate::enc::command::{
1717
use crate::enc::constants::{kCopyExtra, kInsExtra};
1818
use crate::enc::encode;
1919
use crate::enc::literal_cost::BrotliEstimateBitCostsForLiterals;
20-
use crate::enc::static_dict::{
21-
BrotliDictionary, BrotliFindAllStaticDictionaryMatches, FindMatchLengthWithLimit,
22-
};
20+
use crate::enc::static_dict::{BrotliDictionary, FindMatchLengthWithLimit};
2321
use crate::enc::util::{floatX, FastLog2, FastLog2f64};
2422

2523
const BROTLI_WINDOW_GAP: usize = 16;
@@ -369,13 +367,12 @@ where
369367
{
370368
let minlen = max(4, best_len.wrapping_add(1));
371369
if dictionary.is_some()
372-
&& BrotliFindAllStaticDictionaryMatches(
373-
dictionary.unwrap(),
370+
&& dictionary.unwrap().find_all_matches(
374371
&data[cur_ix_masked..],
375372
minlen,
376373
max_length,
377374
&mut dict_matches[..],
378-
) != 0
375+
)
379376
{
380377
assert!(params.use_dictionary);
381378
let maxlen = min(37, max_length);

src/enc/backward_references/mod.rs

+69-79
Original file line numberDiff line numberDiff line change
@@ -432,8 +432,7 @@ impl<T: SliceWrapperMut<u32> + SliceWrapper<u32> + BasicHashComputer> AnyHasher
432432
}
433433
}
434434
if dictionary.is_some() && self.buckets_.USE_DICTIONARY() != 0 && !is_match_found {
435-
is_match_found = SearchInStaticDictionary(
436-
dictionary.unwrap(),
435+
is_match_found = dictionary.unwrap().search_static_item(
437436
dictionary_hash,
438437
self,
439438
&data[cur_ix_masked..],
@@ -821,8 +820,7 @@ impl<Alloc: alloc::Allocator<u16> + alloc::Allocator<u32>> AnyHasher for H9<Allo
821820
}
822821
if !is_match_found && dictionary.is_some() {
823822
let (_, cur_data) = data.split_at(cur_ix_masked);
824-
is_match_found = SearchInStaticDictionary(
825-
dictionary.unwrap(),
823+
is_match_found = dictionary.unwrap().search_static_item(
826824
dictionary_hash,
827825
self,
828826
cur_data,
@@ -1752,8 +1750,7 @@ impl<
17521750

17531751
if !is_match_found && dictionary.is_some() {
17541752
let (_, cur_data) = data.split_at(cur_ix_masked);
1755-
is_match_found = SearchInStaticDictionary(
1756-
dictionary.unwrap(),
1753+
is_match_found = dictionary.unwrap().search_static_item(
17571754
dictionary_hash,
17581755
self,
17591756
cur_data,
@@ -1849,98 +1846,91 @@ fn Hash14(data: &[u8]) -> u32 {
18491846
h >> (32i32 - 14i32)
18501847
}
18511848

1852-
fn TestStaticDictionaryItem(
1853-
dictionary: &BrotliDictionary,
1854-
item: usize,
1855-
data: &[u8],
1856-
max_length: usize,
1857-
max_backward: usize,
1858-
max_distance: usize,
1859-
h9_opts: H9Opts,
1860-
out: &mut HasherSearchResult,
1861-
) -> i32 {
1862-
let backward: usize;
1863-
1864-
let len: usize = item & 0x1fusize;
1865-
let dist: usize = item >> 5;
1866-
let offset: usize =
1867-
(dictionary.offsets_by_length[len] as usize).wrapping_add(len.wrapping_mul(dist));
1868-
if len > max_length {
1869-
return 0i32;
1870-
}
1871-
let matchlen: usize = FindMatchLengthWithLimit(data, &dictionary.data[offset..], len);
1872-
if matchlen.wrapping_add(kCutoffTransformsCount as usize) <= len || matchlen == 0usize {
1873-
return 0i32;
1874-
}
1875-
{
1876-
let cut: u64 = len.wrapping_sub(matchlen) as u64;
1877-
let transform_id: usize =
1849+
impl BrotliDictionary {
1850+
fn test_static_item(
1851+
&self,
1852+
item: usize,
1853+
data: &[u8],
1854+
max_length: usize,
1855+
max_backward: usize,
1856+
max_distance: usize,
1857+
h9_opts: H9Opts,
1858+
out: &mut HasherSearchResult,
1859+
) -> bool {
1860+
let len = item & 0x1f;
1861+
let dist = item >> 5;
1862+
let offset = (self.offsets_by_length[len] as usize).wrapping_add(len.wrapping_mul(dist));
1863+
if len > max_length {
1864+
return false;
1865+
}
1866+
let matchlen: usize = FindMatchLengthWithLimit(data, &self.data[offset..], len);
1867+
if matchlen.wrapping_add(kCutoffTransformsCount as usize) <= len || matchlen == 0 {
1868+
return false;
1869+
}
1870+
1871+
let cut = len.wrapping_sub(matchlen) as u64;
1872+
let transform_id =
18781873
(cut << 2).wrapping_add(kCutoffTransforms >> cut.wrapping_mul(6) & 0x3f) as usize;
1879-
backward = max_backward
1874+
let backward = max_backward
18801875
.wrapping_add(dist)
18811876
.wrapping_add(1)
1882-
.wrapping_add(transform_id << dictionary.size_bits_by_length[len] as i32);
1883-
}
1884-
if backward > max_distance {
1885-
return 0i32;
1886-
}
1887-
let score: u64 = BackwardReferenceScore(matchlen, backward, h9_opts);
1888-
if score < out.score {
1889-
return 0i32;
1877+
.wrapping_add(transform_id << self.size_bits_by_length[len]);
1878+
1879+
if backward > max_distance {
1880+
return false;
1881+
}
1882+
let score = BackwardReferenceScore(matchlen, backward, h9_opts);
1883+
if score < out.score {
1884+
return false;
1885+
}
1886+
out.len = matchlen;
1887+
out.len_x_code = len ^ matchlen;
1888+
out.distance = backward;
1889+
out.score = score;
1890+
true
18901891
}
1891-
out.len = matchlen;
1892-
out.len_x_code = len ^ matchlen;
1893-
out.distance = backward;
1894-
out.score = score;
1895-
1i32
1896-
}
18971892

1898-
fn SearchInStaticDictionary<HasherType: AnyHasher>(
1899-
dictionary: &BrotliDictionary,
1900-
dictionary_hash: &[u16],
1901-
handle: &mut HasherType,
1902-
data: &[u8],
1903-
max_length: usize,
1904-
max_backward: usize,
1905-
max_distance: usize,
1906-
out: &mut HasherSearchResult,
1907-
shallow: bool,
1908-
) -> bool {
1909-
let mut key: usize;
1910-
let mut i: usize;
1911-
let mut is_match_found = false;
1912-
let opts = handle.Opts();
1913-
let xself: &mut Struct1 = handle.GetHasherCommon();
1914-
if xself.dict_num_matches < xself.dict_num_lookups >> 7 {
1915-
return false;
1916-
}
1917-
key = (Hash14(data) << 1) as usize; //FIXME: works for any kind of hasher??
1918-
i = 0usize;
1919-
while i < if shallow { 1 } else { 2 } {
1920-
{
1921-
let item: usize = dictionary_hash[key] as usize;
1893+
fn search_static_item<HasherType: AnyHasher>(
1894+
&self,
1895+
dictionary_hash: &[u16],
1896+
handle: &mut HasherType,
1897+
data: &[u8],
1898+
max_length: usize,
1899+
max_backward: usize,
1900+
max_distance: usize,
1901+
out: &mut HasherSearchResult,
1902+
shallow: bool,
1903+
) -> bool {
1904+
let mut is_match_found = false;
1905+
let opts = handle.Opts();
1906+
let xself = handle.GetHasherCommon();
1907+
if xself.dict_num_matches < xself.dict_num_lookups >> 7 {
1908+
return false;
1909+
}
1910+
let mut key = (Hash14(data) << 1) as usize; //FIXME: works for any kind of hasher??
1911+
let iterations = if shallow { 1 } else { 2 };
1912+
for _ in 0..iterations {
1913+
let item = dictionary_hash[key] as usize;
19221914
xself.dict_num_lookups = xself.dict_num_lookups.wrapping_add(1);
1923-
if item != 0usize {
1924-
let item_matches: i32 = TestStaticDictionaryItem(
1925-
dictionary,
1915+
if item != 0 {
1916+
if self.test_static_item(
19261917
item,
19271918
data,
19281919
max_length,
19291920
max_backward,
19301921
max_distance,
19311922
opts,
19321923
out,
1933-
);
1934-
if item_matches != 0 {
1924+
) {
19351925
xself.dict_num_matches = xself.dict_num_matches.wrapping_add(1);
19361926
is_match_found = true;
19371927
}
19381928
}
1929+
key += 1;
19391930
}
1940-
i = i.wrapping_add(1);
1941-
key = key.wrapping_add(1);
1931+
1932+
is_match_found
19421933
}
1943-
is_match_found
19441934
}
19451935

19461936
impl<Alloc: alloc::Allocator<u16> + alloc::Allocator<u32>> CloneWithAlloc<Alloc>

src/enc/bit_cost.rs

-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
use alloc::SliceWrapperMut;
2-
32
use core::cmp::{max, min};
43

54
use super::super::alloc::SliceWrapper;
@@ -8,7 +7,6 @@ use super::util::{FastLog2, FastLog2u16};
87
use super::vectorization::Mem256i;
98
use crate::enc::floatX;
109

11-
1210
const BROTLI_REPEAT_ZERO_CODE_LENGTH: usize = 17;
1311
const BROTLI_CODE_LENGTH_CODES: usize = BROTLI_REPEAT_ZERO_CODE_LENGTH + 1;
1412

0 commit comments

Comments
 (0)