Skip to content

perf: switch md5 to xxhash #262

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Apr 9, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,6 @@ alloy-json-abi = { version = "0.8", features = ["serde_json"] }
alloy-primitives = { version = "0.8", features = ["serde", "rand"] }
cfg-if = "1.0"
dunce = "1.0"
md-5 = "0.10"
memmap2 = "0.9"
path-slash = "0.2"
rayon = "1.8"
Expand Down
5 changes: 1 addition & 4 deletions crates/artifacts/solc/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,6 @@ yansi.workspace = true
tokio = { workspace = true, optional = true, features = ["fs"] }
futures-util = { workspace = true, optional = true }

# checksum
md-5 = { workspace = true, optional = true }

# walkdir
walkdir = { workspace = true, optional = true }

Expand All @@ -50,6 +47,6 @@ foundry-compilers-core = { workspace = true, features = ["test-utils"] }

[features]
async = ["dep:tokio", "dep:futures-util"]
checksum = ["dep:md-5"]
checksum = ["foundry-compilers-core/hasher"]
walkdir = ["dep:walkdir", "foundry-compilers-core/walkdir"]
rayon = ["dep:rayon"]
2 changes: 1 addition & 1 deletion crates/artifacts/solc/src/sources.rs
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,7 @@ impl Source {
/// Generate a non-cryptographically secure checksum of the given source.
#[cfg(feature = "checksum")]
pub fn content_hash_of(src: &str) -> String {
alloy_primitives::hex::encode(<md5::Md5 as md5::Digest>::digest(src))
foundry_compilers_core::utils::unique_hash(src)
}
}

Expand Down
3 changes: 1 addition & 2 deletions crates/compilers/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,14 @@ foundry-compilers-artifacts = { workspace = true, features = [
"walkdir",
"rayon",
] }
foundry-compilers-core = { workspace = true, features = ["regex"] }
foundry-compilers-core = { workspace = true, features = ["hasher", "regex"] }
serde.workspace = true
semver.workspace = true
alloy-primitives.workspace = true
serde_json.workspace = true
tracing.workspace = true
alloy-json-abi.workspace = true
rayon.workspace = true
md-5.workspace = true
thiserror.workspace = true
path-slash.workspace = true
yansi.workspace = true
Expand Down
21 changes: 5 additions & 16 deletions crates/compilers/src/buildinfo.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,7 @@
use crate::compilers::{
CompilationError, CompilerContract, CompilerInput, CompilerOutput, Language,
};
use alloy_primitives::hex;
use foundry_compilers_core::{error::Result, utils};
use md5::Digest;
use semver::Version;
use serde::{de::DeserializeOwned, Deserialize, Serialize};
use std::{
Expand Down Expand Up @@ -97,22 +95,13 @@ impl<L: Language> RawBuildInfo<L> {
let version = input.version().clone();
let build_context = BuildContext::new(input, output)?;

let mut hasher = md5::Md5::new();

hasher.update(ETHERS_FORMAT_VERSION);

let solc_short = format!("{}.{}.{}", version.major, version.minor, version.patch);
hasher.update(&solc_short);
hasher.update(version.to_string());

let input = serde_json::to_value(input)?;
hasher.update(&serde_json::to_string(&input)?);

// create the hash for `{_format,solcVersion,solcLongVersion,input}`
// N.B. this is not exactly the same as hashing the json representation of these values but
// the must efficient one
let result = hasher.finalize();
let id = hex::encode(result);
let id = utils::unique_hash_many([
ETHERS_FORMAT_VERSION,
&version.to_string(),
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

solc_short excluded since it's a subset of version

&serde_json::to_string(&input)?,
]);

let mut build_info = BTreeMap::new();

Expand Down
3 changes: 1 addition & 2 deletions crates/compilers/src/compile/output/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -607,8 +607,7 @@ impl<C: Compiler> AggregatedCompilerOutput<C> {
///
/// There can be multiple `BuildInfo`, since we support multiple versions.
///
/// The created files have the md5 hash `{_format,solcVersion,solcLongVersion,input}` as their
/// file name
/// The created files have a unique identifier as their name.
pub fn write_build_infos(&self, build_info_dir: &Path) -> Result<(), SolcError> {
if self.build_infos.is_empty() {
return Ok(());
Expand Down
6 changes: 6 additions & 0 deletions crates/core/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,11 @@ serde_json.workspace = true
serde.workspace = true
thiserror.workspace = true

# hasher
xxhash-rust = { version = "0.8", optional = true, default-features = false, features = [
"xxh3",
] }

# regex
regex = { workspace = true, optional = true }

Expand All @@ -46,6 +51,7 @@ tempfile.workspace = true

[features]
async = ["dep:tokio"]
hasher = ["dep:xxhash-rust"]
project-util = ["dep:tempfile", "dep:fs_extra"]
regex = ["dep:regex"]
svm-solc = ["dep:svm", "dep:tokio"]
Expand Down
26 changes: 26 additions & 0 deletions crates/core/src/utils/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,32 @@ pub static SUPPORTS_BASE_PATH: Lazy<VersionReq> =
pub static SUPPORTS_INCLUDE_PATH: Lazy<VersionReq> =
Lazy::new(|| VersionReq::parse(">=0.8.8").unwrap());

/// A non-cryptographic hash function for creating unique identifiers.
///
/// The exact algorithm being used shouldn't matter.
// See Hardhat: https://github.com/NomicFoundation/hardhat/blob/e9ab5332a5505a6d1fe9bfbc687f5f46bdff6dd7/packages/hardhat-core/src/internal/util/hash.ts#L1-L16
#[cfg(feature = "hasher")]
pub fn unique_hash(input: impl AsRef<[u8]>) -> String {
encode_hash(xxhash_rust::xxh3::xxh3_64(input.as_ref()))
}

/// A non-cryptographic hash function for creating unique identifiers.
///
/// See [`unique_hash`] for more details.
#[cfg(feature = "hasher")]
pub fn unique_hash_many(inputs: impl IntoIterator<Item = impl AsRef<[u8]>>) -> String {
let mut hasher = xxhash_rust::xxh3::Xxh3Default::new();
for input in inputs {
hasher.update(input.as_ref());
}
encode_hash(hasher.digest())
}

#[cfg(feature = "hasher")]
fn encode_hash(x: u64) -> String {
hex::encode(x.to_be_bytes())
}

/// Move a range by a specified offset
pub fn range_by_offset(range: &Range<usize>, offset: isize) -> Range<usize> {
Range {
Expand Down