Skip to content

Commit

Permalink
feat: add native hook of real content hash plugin (#9234)
Browse files Browse the repository at this point in the history
  • Loading branch information
LingyuCoder authored Feb 11, 2025
1 parent be02514 commit f076b49
Show file tree
Hide file tree
Showing 9 changed files with 74 additions and 14 deletions.
2 changes: 2 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions crates/node_binding/binding.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1957,6 +1957,7 @@ export interface RawOptimizationOptions {
usedExports: boolean | string
providedExports: boolean
innerGraph: boolean
realContentHash: boolean
mangleExports: boolean | string
concatenateModules: boolean
avoidEntryIife: boolean
Expand Down
1 change: 1 addition & 0 deletions crates/rspack/src/builder/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3550,6 +3550,7 @@ impl OptimizationOptionsBuilder {
mangle_exports,
concatenate_modules,
avoid_entry_iife,
real_content_hash,
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1359,6 +1359,7 @@ CompilerOptions {
mangle_exports: False,
concatenate_modules: false,
avoid_entry_iife: false,
real_content_hash: false,
},
profile: false,
amd: None,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ pub struct RawOptimizationOptions {
pub used_exports: WithBool<String>,
pub provided_exports: bool,
pub inner_graph: bool,
pub real_content_hash: bool,
#[napi(ts_type = "boolean | string")]
pub mangle_exports: WithBool<String>,
pub concatenate_modules: bool,
Expand Down Expand Up @@ -50,6 +51,7 @@ impl TryFrom<RawOptimizationOptions> for Optimization {
mangle_exports: value.mangle_exports.into(),
concatenate_modules: value.concatenate_modules,
avoid_entry_iife: value.avoid_entry_iife,
real_content_hash: value.real_content_hash,
})
}
}
1 change: 1 addition & 0 deletions crates/rspack_core/src/options/optimizations.rs
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,7 @@ pub struct Optimization {
pub mangle_exports: MangleExportsOption,
pub concatenate_modules: bool,
pub avoid_entry_iife: bool,
pub real_content_hash: bool,
}

pub static DEFAULT_DELIMITER: &str = "~";
2 changes: 2 additions & 0 deletions crates/rspack_plugin_real_content_hash/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ version = "0.2.0"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

[dependencies]
dashmap = { workspace = true }
derive_more = { workspace = true, features = ["debug"] }
indexmap = { workspace = true }
once_cell = { workspace = true }
Expand All @@ -17,6 +18,7 @@ rspack_core = { workspace = true }
rspack_error = { workspace = true }
rspack_hash = { workspace = true }
rspack_hook = { workspace = true }
rspack_util = { workspace = true }
rustc-hash = { workspace = true }
tracing = { workspace = true }

Expand Down
11 changes: 11 additions & 0 deletions crates/rspack_plugin_real_content_hash/src/drive.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
use std::sync::Arc;

use rspack_core::{rspack_sources::Source, Compilation};
use rspack_hook::define_hook;

define_hook!(RealContentHashPluginUpdateHash: AsyncSeriesBail(compilation: &Compilation, assets: &[Arc<dyn Source>], old_hash: &str) -> String);

#[derive(Debug, Default)]
pub struct RealContentHashPluginHooks {
pub update_hash: RealContentHashPluginUpdateHashHook,
}
67 changes: 53 additions & 14 deletions crates/rspack_plugin_real_content_hash/src/lib.rs
Original file line number Diff line number Diff line change
@@ -1,36 +1,62 @@
#![feature(let_chains)]

mod drive;

use std::{
borrow::Cow,
hash::{BuildHasherDefault, Hasher},
sync::LazyLock,
};

use derive_more::Debug;
pub use drive::*;
use once_cell::sync::OnceCell;
use rayon::prelude::*;
use regex::{Captures, Regex};
use rspack_core::{
rspack_sources::{BoxSource, RawStringSource, SourceExt},
AssetInfo, Compilation, CompilationProcessAssets, Logger, Plugin, PluginContext,
AssetInfo, Compilation, CompilationId, CompilationProcessAssets, Logger, Plugin, PluginContext,
};
use rspack_error::Result;
use rspack_hash::RspackHash;
use rspack_hook::{plugin, plugin_hook};
use rspack_util::fx_hash::FxDashMap;
use rustc_hash::{FxHashMap as HashMap, FxHashSet as HashSet, FxHasher};

type IndexSet<T> = indexmap::IndexSet<T, BuildHasherDefault<FxHasher>>;

pub static QUOTE_META: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"[-\[\]\\/{}()*+?.^$|]").expect("Invalid regex"));

static COMPILATION_HOOKS_MAP: LazyLock<FxDashMap<CompilationId, Box<RealContentHashPluginHooks>>> =
LazyLock::new(Default::default);

#[plugin]
#[derive(Debug, Default)]
pub struct RealContentHashPlugin;

impl RealContentHashPlugin {
pub fn get_compilation_hooks(
id: CompilationId,
) -> dashmap::mapref::one::Ref<'static, CompilationId, Box<RealContentHashPluginHooks>> {
if !COMPILATION_HOOKS_MAP.contains_key(&id) {
COMPILATION_HOOKS_MAP.insert(id, Default::default());
}
COMPILATION_HOOKS_MAP
.get(&id)
.expect("should have js plugin drive")
}

pub fn get_compilation_hooks_mut(
id: CompilationId,
) -> dashmap::mapref::one::RefMut<'static, CompilationId, Box<RealContentHashPluginHooks>> {
COMPILATION_HOOKS_MAP.entry(id).or_default()
}
}

#[plugin_hook(CompilationProcessAssets for RealContentHashPlugin, stage = Compilation::PROCESS_ASSETS_STAGE_OPTIMIZE_HASH)]
async fn process_assets(&self, compilation: &mut Compilation) -> Result<()> {
inner_impl(compilation)
inner_impl(compilation).await
}

impl Plugin for RealContentHashPlugin {
Expand All @@ -52,7 +78,7 @@ impl Plugin for RealContentHashPlugin {
}
}

fn inner_impl(compilation: &mut Compilation) -> Result<()> {
async fn inner_impl(compilation: &mut Compilation) -> Result<()> {
let logger = compilation.get_logger("rspack.RealContentHashPlugin");
let start = logger.time("hash to asset names");
let mut hash_to_asset_names: HashMap<&str, Vec<&str>> = HashMap::default();
Expand Down Expand Up @@ -106,27 +132,40 @@ fn inner_impl(compilation: &mut Compilation) -> Result<()> {
let start = logger.time("old hash to new hash");
let mut hash_to_new_hash = HashMap::default();

let hooks = RealContentHashPlugin::get_compilation_hooks(compilation.id());
for old_hash in &ordered_hashes {
if let Some(asset_names) = hash_to_asset_names.get_mut(old_hash.as_str()) {
asset_names.sort();
let mut asset_contents: Vec<_> = asset_names
.par_iter()
.filter_map(|name| assets_data.get(name))
.map(|data| {
data.compute_new_source(
data.own_hashes.contains(old_hash),
&hash_to_new_hash,
&hash_regexp,
)
data
.compute_new_source(
data.own_hashes.contains(old_hash),
&hash_to_new_hash,
&hash_regexp,
)
.clone()
})
.collect();
asset_contents.dedup();
let mut hasher = RspackHash::from(&compilation.options.output);
for asset_content in asset_contents {
hasher.write(&asset_content.buffer());
}
let new_hash = hasher.digest(&compilation.options.output.hash_digest);
let new_hash = new_hash.rendered(old_hash.len()).to_string();
let updated_hash = hooks
.update_hash
.call(compilation, &asset_contents, old_hash)
.await?;

let new_hash = if let Some(new_hash) = updated_hash {
new_hash
} else {
let mut hasher = RspackHash::from(&compilation.options.output);
for asset_content in asset_contents {
hasher.write(&asset_content.buffer());
}
let new_hash = hasher.digest(&compilation.options.output.hash_digest);
let new_hash = new_hash.rendered(old_hash.len()).to_string();
new_hash
};
hash_to_new_hash.insert(old_hash, new_hash);
}
}
Expand Down

2 comments on commit f076b49

@github-actions
Copy link
Contributor

@github-actions github-actions bot commented on f076b49 Feb 11, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

📝 Benchmark detail: Open

Name Base (2025-02-11 9d82335) Current Change
10000_big_production-mode_disable-minimize + exec 37.7 s ± 637 ms 39 s ± 1.08 s +3.34 %
10000_development-mode + exec 1.88 s ± 89 ms 1.83 s ± 29 ms -2.55 %
10000_development-mode_hmr + exec 685 ms ± 7.4 ms 684 ms ± 3.4 ms -0.17 %
10000_production-mode + exec 2.28 s ± 51 ms 2.27 s ± 89 ms -0.42 %
10000_production-mode_persistent-cold + exec 2.47 s ± 72 ms 2.45 s ± 196 ms -0.60 %
10000_production-mode_persistent-hot + exec 1.65 s ± 61 ms 1.65 s ± 28 ms -0.11 %
arco-pro_development-mode + exec 1.78 s ± 145 ms 1.78 s ± 97 ms -0.11 %
arco-pro_development-mode_hmr + exec 389 ms ± 2.1 ms 387 ms ± 2.7 ms -0.29 %
arco-pro_production-mode + exec 3.59 s ± 217 ms 3.67 s ± 297 ms +2.25 %
arco-pro_production-mode_generate-package-json-webpack-plugin + exec 3.72 s ± 229 ms 3.64 s ± 90 ms -2.06 %
arco-pro_production-mode_persistent-cold + exec 3.85 s ± 164 ms 3.83 s ± 166 ms -0.42 %
arco-pro_production-mode_persistent-hot + exec 2.36 s ± 99 ms 2.42 s ± 104 ms +2.60 %
arco-pro_production-mode_traverse-chunk-modules + exec 3.63 s ± 69 ms 3.64 s ± 73 ms +0.26 %
large-dyn-imports_development-mode + exec 2.09 s ± 24 ms 2.1 s ± 72 ms +0.49 %
large-dyn-imports_production-mode + exec 2.15 s ± 71 ms 2.16 s ± 38 ms +0.50 %
threejs_development-mode_10x + exec 1.54 s ± 25 ms 1.53 s ± 34 ms -0.21 %
threejs_development-mode_10x_hmr + exec 779 ms ± 7 ms 788 ms ± 5.5 ms +1.20 %
threejs_production-mode_10x + exec 5.2 s ± 32 ms 5.28 s ± 265 ms +1.47 %
threejs_production-mode_10x_persistent-cold + exec 5.27 s ± 65 ms 5.3 s ± 164 ms +0.47 %
threejs_production-mode_10x_persistent-hot + exec 4.53 s ± 220 ms 4.54 s ± 166 ms +0.32 %
10000_big_production-mode_disable-minimize + rss memory 8706 MiB ± 32.4 MiB 8727 MiB ± 114 MiB +0.24 %
10000_development-mode + rss memory 649 MiB ± 12.6 MiB 661 MiB ± 28.2 MiB +1.96 %
10000_development-mode_hmr + rss memory 1344 MiB ± 112 MiB 1299 MiB ± 305 MiB -3.36 %
10000_production-mode + rss memory 621 MiB ± 21.8 MiB 641 MiB ± 23.3 MiB +3.33 %
10000_production-mode_persistent-cold + rss memory 746 MiB ± 24.7 MiB 743 MiB ± 26.2 MiB -0.45 %
10000_production-mode_persistent-hot + rss memory 717 MiB ± 23.3 MiB 747 MiB ± 16.6 MiB +4.16 %
arco-pro_development-mode + rss memory 569 MiB ± 17.5 MiB 572 MiB ± 31.1 MiB +0.45 %
arco-pro_development-mode_hmr + rss memory 643 MiB ± 49.1 MiB 666 MiB ± 78.3 MiB +3.56 %
arco-pro_production-mode + rss memory 714 MiB ± 28.8 MiB 725 MiB ± 35.9 MiB +1.58 %
arco-pro_production-mode_generate-package-json-webpack-plugin + rss memory 732 MiB ± 22.4 MiB 741 MiB ± 30.7 MiB +1.16 %
arco-pro_production-mode_persistent-cold + rss memory 853 MiB ± 45.5 MiB 850 MiB ± 37.7 MiB -0.39 %
arco-pro_production-mode_persistent-hot + rss memory 708 MiB ± 27.8 MiB 745 MiB ± 38.8 MiB +5.27 %
arco-pro_production-mode_traverse-chunk-modules + rss memory 727 MiB ± 48.2 MiB 731 MiB ± 31.7 MiB +0.50 %
large-dyn-imports_development-mode + rss memory 644 MiB ± 5.26 MiB 663 MiB ± 5.94 MiB +3.04 %
large-dyn-imports_production-mode + rss memory 526 MiB ± 7.84 MiB 550 MiB ± 12.8 MiB +4.46 %
threejs_development-mode_10x + rss memory 553 MiB ± 16.3 MiB 565 MiB ± 16.8 MiB +2.21 %
threejs_development-mode_10x_hmr + rss memory 1145 MiB ± 140 MiB 1142 MiB ± 166 MiB -0.21 %
threejs_production-mode_10x + rss memory 828 MiB ± 50.8 MiB 833 MiB ± 32.8 MiB +0.66 %
threejs_production-mode_10x_persistent-cold + rss memory 960 MiB ± 14.5 MiB 938 MiB ± 22.9 MiB -2.32 %
threejs_production-mode_10x_persistent-hot + rss memory 877 MiB ± 54.8 MiB 868 MiB ± 39 MiB -1.06 %

@github-actions
Copy link
Contributor

@github-actions github-actions bot commented on f076b49 Feb 11, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

📝 Ecosystem CI detail: Open

suite result
modernjs ❌ failure
rspress ✅ success
rslib ✅ success
rsbuild ❌ failure
rsdoctor ❌ failure
examples ✅ success
devserver ✅ success
nuxt ✅ success

Please sign in to comment.