From eb514ad992314aaca4ac5cbfa449185462b6b293 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Est=C3=A9fano=20Bargas?= Date: Mon, 13 Jan 2025 17:43:45 -0300 Subject: [PATCH] add bench crate --- Cargo.lock | 104 ++++++++++- Cargo.toml | 1 + crates/l2/prover/bench/Cargo.toml | 38 ++++ crates/l2/prover/bench/src/constants.rs | 37 ++++ crates/l2/prover/bench/src/lib.rs | 2 + crates/l2/prover/bench/src/main.rs | 161 ++++++++++++++++ crates/l2/prover/bench/src/rpc.rs | 238 ++++++++++++++++++++++++ 7 files changed, 580 insertions(+), 1 deletion(-) create mode 100644 crates/l2/prover/bench/Cargo.toml create mode 100644 crates/l2/prover/bench/src/constants.rs create mode 100644 crates/l2/prover/bench/src/lib.rs create mode 100644 crates/l2/prover/bench/src/main.rs create mode 100644 crates/l2/prover/bench/src/rpc.rs diff --git a/Cargo.lock b/Cargo.lock index c9e590a376..3d57929939 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -770,6 +770,26 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +[[package]] +name = "async-stdin" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1ff8b5d9b5ec29e0f49583ba71847b8c8888b67a8510133048a380903aa6822" +dependencies = [ + "tokio", +] + +[[package]] +name = "async-throttle" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c99532de164435a0b91279e715bff4fa0d164643b409a67761907ffc210ee8f" +dependencies = [ + "backoff", + "dashmap 5.5.3", + "tokio", +] + [[package]] name = "async-trait" version = "0.1.83" @@ -973,6 +993,20 @@ dependencies = [ "tower-service", ] +[[package]] +name = "backoff" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1" +dependencies = [ + "futures-core", + "getrandom", + "instant", + "pin-project-lite", + "rand 0.8.5", + "tokio", +] + [[package]] name = "backtrace" version = "0.3.74" @@ -1025,6 +1059,30 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d86b93f97252c47b41663388e6d155714a9d0c398b99f1005cbc5f978b29f445" +[[package]] +name = "bench" +version = "0.1.0" +dependencies = [ + "bytes", + "clap 4.5.21", + "ethrex-core", + "ethrex-l2", + "ethrex-prover", + "ethrex-rlp", + "ethrex-vm", + "futures-util", + "hex", + "reqwest 0.12.9", + "revm 14.0.3", + "revm-inspectors", + "revm-primitives 10.0.0", + "serde", + "serde_json", + "tokio", + "tokio-utils", + "zkvm_interface", +] + [[package]] name = "bincode" version = "1.3.3" @@ -1906,6 +1964,19 @@ dependencies = [ "serde", ] +[[package]] +name = "dashmap" +version = "5.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" +dependencies = [ + "cfg-if", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core 0.9.10", +] + [[package]] name = "dashu" version = "0.4.2" @@ -7669,6 +7740,15 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "shutdown-async" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2799e69bde7e68bedd86c6d94bffa783219114f1f31435ddda61f4aeba348ff" +dependencies = [ + "tokio", +] + [[package]] name = "signal-hook-registry" version = "1.4.2" @@ -8688,7 +8768,7 @@ dependencies = [ "aho-corasick 0.7.20", "clap 2.34.0", "crossbeam-channel", - "dashmap", + "dashmap 4.0.2", "dirs 3.0.2", "encoding_rs_io", "env_logger", @@ -8772,6 +8852,18 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-utils" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de75f75f464153a50fe48b9675360e3cf2ae1d7d81f9751363bd2ee4888f5ce8" +dependencies = [ + "async-stdin", + "async-throttle", + "shutdown-async", + "tub", +] + [[package]] name = "toml" version = "0.5.11" @@ -8954,6 +9046,16 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "tub" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bca43faba247bc76eb1d6c1b8b561e4a1c5bdd427cc3d7a007faabea75c683a" +dependencies = [ + "crossbeam-queue", + "tokio", +] + [[package]] name = "twirp-rs" version = "0.13.0-succinct" diff --git a/Cargo.toml b/Cargo.toml index f1f07d6bc2..effb88580e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,7 @@ members = [ "crates/vm/levm/bench/revm_comparison", "crates/l2/", "crates/l2/prover", + "crates/l2/prover/bench", "crates/l2/contracts", "crates/l2/sdk", "cmd/loc", diff --git a/crates/l2/prover/bench/Cargo.toml b/crates/l2/prover/bench/Cargo.toml new file mode 100644 index 0000000000..303c2868ca --- /dev/null +++ b/crates/l2/prover/bench/Cargo.toml @@ -0,0 +1,38 @@ +[package] +name = "bench" +version = "0.1.0" +edition = "2021" + +[dependencies] +ethrex-core.workspace = true +ethrex-rlp.workspace = true +ethrex-vm.workspace = true +ethrex-prover.workspace = true +ethrex-l2.workspace = true + +zkvm_interface = { path = "../zkvm/interface", default-features = false } + +serde.workspace = true +hex.workspace = true +serde_json.workspace = true +bytes.workspace = true + +tokio = { version = "1.21", default-features = false, features = ["full"] } +reqwest = { version = "0.12.9", features = ["json"] } +clap = { version = "4.5.21", features = ["derive"] } + +revm = { version = "14.0.3", features = [ + "serde", + "std", + "serde-json", + "optional_no_base_fee", + "optional_block_gas_limit", +], default-features = false } + +# These dependencies must be kept up to date with the corresponding revm version, otherwise errors may pop up because of trait implementation mismatches +revm-inspectors = { version = "0.8.1" } +revm-primitives = { version = "10.0.0", features = [ + "std", +], default-features = false } +futures-util = "0.3.31" +tokio-utils = "0.1.2" diff --git a/crates/l2/prover/bench/src/constants.rs b/crates/l2/prover/bench/src/constants.rs new file mode 100644 index 0000000000..9171aaf623 --- /dev/null +++ b/crates/l2/prover/bench/src/constants.rs @@ -0,0 +1,37 @@ +// Blockchain related constants + +use ethrex_core::types::ChainConfig; +use revm_primitives::SpecId; + +// Chain config for different forks as defined on https://ethereum.github.io/execution-spec-tests/v3.0.0/consuming_tests/common_types/#fork +pub static CANCUN_CONFIG: ChainConfig = ChainConfig { + chain_id: 1_u64, + homestead_block: Some(0), + dao_fork_block: Some(0), + dao_fork_support: true, + eip150_block: Some(0), + eip155_block: Some(0), + eip158_block: Some(0), + byzantium_block: Some(0), + constantinople_block: Some(0), + petersburg_block: Some(0), + istanbul_block: Some(0), + muir_glacier_block: Some(0), + berlin_block: Some(0), + london_block: Some(0), + arrow_glacier_block: Some(0), + gray_glacier_block: Some(0), + merge_netsplit_block: Some(0), + terminal_total_difficulty: Some(0), + shanghai_time: Some(0), + cancun_time: Some(0), + prague_time: None, + terminal_total_difficulty_passed: false, + verkle_time: None, +}; +pub const MAINNET_CHAIN_ID: u64 = 0x1; +pub const MAINNET_SPEC_ID: SpecId = SpecId::CANCUN; + +// RPC related constants + +pub const RPC_RATE_LIMIT: usize = 100; // requests per second diff --git a/crates/l2/prover/bench/src/lib.rs b/crates/l2/prover/bench/src/lib.rs new file mode 100644 index 0000000000..bc1bcdc5c0 --- /dev/null +++ b/crates/l2/prover/bench/src/lib.rs @@ -0,0 +1,2 @@ +pub mod constants; +pub mod rpc; diff --git a/crates/l2/prover/bench/src/main.rs b/crates/l2/prover/bench/src/main.rs new file mode 100644 index 0000000000..90746859a3 --- /dev/null +++ b/crates/l2/prover/bench/src/main.rs @@ -0,0 +1,161 @@ +use std::collections::HashMap; + +use bench::{ + constants::{CANCUN_CONFIG, MAINNET_CHAIN_ID, MAINNET_SPEC_ID, RPC_RATE_LIMIT}, + rpc::{get_account, get_block, Account, NodeRLP}, +}; +use clap::Parser; +use ethrex_core::{Address, U256}; +use ethrex_l2::utils::prover::proving_systems::ProverType; +use ethrex_prover_lib::prover::{create_prover, Prover}; +use ethrex_vm::execution_db::ExecutionDB; +use futures_util::future::join_all; +use tokio_utils::RateLimiter; +use zkvm_interface::io::ProgramInput; + +#[derive(Parser, Debug)] +struct Args { + #[arg(short, long)] + rpc_url: String, + #[arg(short, long)] + block_number: usize, +} + +#[tokio::main] +async fn main() { + let Args { + rpc_url, + block_number, + } = Args::parse(); + + println!("fetching block {block_number} and its parent header"); + let block = get_block(&rpc_url, block_number) + .await + .expect("failed to fetch block"); + let parent_block_header = get_block(&rpc_url, block_number - 1) + .await + .expect("failed to fetch block") + .header; + + println!("pre-executing transactions to get touched state"); + let touched_state: Vec<(Address, Vec)> = todo!(); + // let touched_state = get_touched_state(&block, MAINNET_CHAIN_ID, MAINNET_SPEC_ID) + //.expect("failed to get touched state"); + + println!("fetching touched state values"); + let mut accounts = HashMap::new(); + let mut storages = HashMap::new(); + let mut codes = Vec::new(); + let mut account_proofs = Vec::new(); + let mut storages_proofs: HashMap<_, Vec> = HashMap::new(); + + let rate_limiter = RateLimiter::new(std::time::Duration::from_secs(1)); + let mut fetched_accs = 0; + for request_chunk in touched_state.chunks(RPC_RATE_LIMIT) { + let account_futures = request_chunk.iter().map(|(address, storage_keys)| async { + Ok(( + *address, + get_account( + &rpc_url, + block_number - 1, + &address.clone(), + &storage_keys.clone(), + ) + .await?, + )) + }); + + let fetched_accounts = rate_limiter + .throttle(|| async { join_all(account_futures).await }) + .await + .into_iter() + .collect::, String>>() + .expect("failed to fetch accounts"); + for ( + address, + Account { + account_state, + storage, + account_proof, + storage_proofs, + code, + }, + ) in fetched_accounts + { + accounts.insert(address.to_owned(), account_state); + storages.insert(address.to_owned(), storage); + if let Some(code) = code { + codes.push(code); + } + account_proofs.extend(account_proof); + storages_proofs + .entry(address) + .or_default() + .extend(storage_proofs.into_iter().flatten()); + } + + fetched_accs += request_chunk.len(); + println!( + "fetched {} accounts of {}", + fetched_accs, + touched_state.len() + ); + } + + println!("building program input"); + let storages: HashMap> = storages + .into_iter() + .filter_map(|(address, storage)| { + if !storage.is_empty() { + Some((address, storage.into_iter().collect())) + } else { + None + } + }) + .collect(); + + let account_proofs = { + let root_node = if !account_proofs.is_empty() { + Some(account_proofs.swap_remove(0)) + } else { + None + }; + (root_node, account_proofs) + }; + + let storages_proofs: HashMap, Vec)> = storages_proofs + .into_iter() + .map(|(address, mut proofs)| { + (address, { + let root_node = if !proofs.is_empty() { + Some(proofs.swap_remove(0)) + } else { + None + }; + (root_node, proofs) + }) + }) + .collect(); + + let db = todo!(); + // let db = ExecutionDB::new( + // accounts, + // storages, + // codes, + // account_proofs, + // storages_proofs, + // CANCUN_CONFIG, + // ) + // .expect("failed to create execution db"); + + println!("proving"); + let mut prover = create_prover(ProverType::SP1); + let receipt = prover + .prove(ProgramInput { + block, + parent_block_header, + db, + }) + .expect("proving failed"); + let execution_gas = prover.get_gas().expect("failed to get execution gas"); +} diff --git a/crates/l2/prover/bench/src/rpc.rs b/crates/l2/prover/bench/src/rpc.rs new file mode 100644 index 0000000000..d4b7a19802 --- /dev/null +++ b/crates/l2/prover/bench/src/rpc.rs @@ -0,0 +1,238 @@ +use bytes::Bytes; +use ethrex_core::{ + types::{AccountState, Block, EMPTY_KECCACK_HASH}, + Address, U256, +}; +use ethrex_rlp::decode::RLPDecode; + +use serde::{de::DeserializeOwned, Deserialize}; +use serde_json::json; + +pub type NodeRLP = Vec; + +pub struct Account { + pub account_state: AccountState, + pub storage: Vec<(U256, U256)>, + pub account_proof: Vec, + pub storage_proofs: Vec>, + pub code: Option, +} + +pub async fn get_block(rpc_url: &str, block_number: usize) -> Result { + let client = reqwest::Client::new(); + + let block_number = format!("0x{block_number:x}"); + let request = &json!({ + "id": 1, + "jsonrpc": "2.0", + "method": "debug_getRawBlock", + "params": [block_number] + }); + + let response = client + .post(rpc_url) + .json(request) + .send() + .await + .map_err(|err| err.to_string())?; + + response + .json::() + .await + .map_err(|err| err.to_string()) + .and_then(get_result) + .and_then(decode_hex) + .and_then(|encoded_block| { + Block::decode_unfinished(&encoded_block) + .map_err(|err| err.to_string()) + .map(|decoded| decoded.0) + }) +} + +pub async fn get_account( + rpc_url: &str, + block_number: usize, + address: &Address, + storage_keys: &[U256], +) -> Result { + let client = reqwest::Client::new(); + + let block_number_str = format!("0x{block_number:x}"); + let address_str = format!("0x{address:x}"); + let storage_keys = storage_keys + .iter() + .map(|key| format!("0x{key:x}")) + .collect::>(); + + let request = &json!( + { + "id": 1, + "jsonrpc": "2.0", + "method": "eth_getProof", + "params":[address_str, storage_keys, block_number_str] + } + ); + let response = client + .post(rpc_url) + .json(request) + .send() + .await + .map_err(|err| err.to_string())?; + + #[derive(Deserialize)] + #[serde(rename_all = "camelCase")] + struct AccountProof { + balance: String, + code_hash: String, + nonce: String, + storage_hash: String, + storage_proof: Vec, + account_proof: Vec, + } + + #[derive(Deserialize)] + struct StorageProof { + key: String, + value: String, + proof: Vec, + } + + let AccountProof { + balance, + code_hash, + nonce, + storage_hash, + storage_proof, + account_proof, + } = response + .json::() + .await + .map_err(|err| err.to_string()) + .and_then(get_result)?; + + let (storage, storage_proofs) = storage_proof + .into_iter() + .map(|proof| -> Result<_, String> { + let key = proof + .key + .parse() + .map_err(|_| "failed to parse storage key".to_string())?; + let value = proof + .value + .parse() + .map_err(|_| "failed to parse storage value".to_string())?; + let proofs = proof + .proof + .into_iter() + .map(decode_hex) + .collect::, _>>()?; + Ok(((key, value), proofs)) + }) + .collect::, Vec<_>), _>>()?; + + let account_state = AccountState { + nonce: u64::from_str_radix(nonce.trim_start_matches("0x"), 16) + .map_err(|_| "failed to parse nonce".to_string())?, + balance: balance + .parse() + .map_err(|_| "failed to parse balance".to_string())?, + storage_root: storage_hash + .parse() + .map_err(|_| "failed to parse storage root".to_string())?, + code_hash: code_hash + .parse() + .map_err(|_| "failed to parse code hash".to_string())?, + }; + + let code = if account_state.code_hash != *EMPTY_KECCACK_HASH { + Some(get_code(rpc_url, block_number, address).await?) + } else { + None + }; + + let account_proof = account_proof + .into_iter() + .map(decode_hex) + .collect::, String>>()?; + + Ok(Account { + account_state, + storage, + account_proof, + storage_proofs, + code, + }) +} + +async fn get_code(rpc_url: &str, block_number: usize, address: &Address) -> Result { + let client = reqwest::Client::new(); + + let block_number = format!("0x{block_number:x}"); + let address = format!("0x{address:x}"); + let request = &json!({ + "id": 1, + "jsonrpc": "2.0", + "method": "eth_getCode", + "params": [address, block_number] + }); + + let response = client + .post(rpc_url) + .json(request) + .send() + .await + .map_err(|err| err.to_string())?; + + response + .json::() + .await + .map_err(|err| err.to_string()) + .and_then(get_result) + .and_then(decode_hex) + .map(Bytes::from_owner) +} + +fn get_result(response: serde_json::Value) -> Result { + match response.get("result") { + Some(result) => serde_json::from_value(result.clone()).map_err(|err| err.to_string()), + None => Err(format!("result not found, response is: {response}")), + } +} + +fn decode_hex(hex: String) -> Result, String> { + let mut trimmed = hex.trim_start_matches("0x").to_string(); + if trimmed.len() % 2 != 0 { + trimmed = "0".to_string() + &trimmed; + } + hex::decode(trimmed).map_err(|err| format!("failed to decode hex string: {err}")) +} + +#[cfg(test)] +mod test { + use ethrex_core::Address; + + use super::*; + + const BLOCK_NUMBER: usize = 21315830; + const RPC_URL: &str = ""; + const VITALIK_ADDR: &str = "d8dA6BF26964aF9D7eEd9e03E53415D37aA96045"; + + #[ignore = "needs to manually set rpc url in constant"] + #[tokio::test] + async fn get_block_works() { + get_block(RPC_URL, &BLOCK_NUMBER).await.unwrap(); + } + + #[ignore = "needs to manually set rpc url in constant"] + #[tokio::test] + async fn get_account_works() { + get_account( + RPC_URL, + BLOCK_NUMBER, + &Address::from_slice(&hex::decode(VITALIK_ADDR).unwrap()), + &[], + ) + .await + .unwrap(); + } +}