diff --git a/cmd/ef_tests/levm/deserialize.rs b/cmd/ef_tests/levm/deserialize.rs index df1d2cdbdf..37c1997b4b 100644 --- a/cmd/ef_tests/levm/deserialize.rs +++ b/cmd/ef_tests/levm/deserialize.rs @@ -1,14 +1,13 @@ use crate::types::{ EFTest, EFTestAccessListItem, EFTestAuthorizationListTuple, EFTests, - TransactionExpectedException, + TransactionExpectedException,EFTestPostValue,EFTestTransaction }; use bytes::Bytes; use ethrex_core::{H256, U256}; +use ethrex_vm::SpecId; use serde::{Deserialize, Deserializer}; use std::{collections::HashMap, str::FromStr}; -use crate::types::{EFTestRawTransaction, EFTestTransaction}; - pub fn deserialize_transaction_expected_exception<'de, D>( deserializer: D, ) -> Result>, D::Error> @@ -43,7 +42,9 @@ where "TransactionException.INSUFFICIENT_ACCOUNT_FUNDS" => { TransactionExpectedException::InsufficientAccountFunds } - "TransactionException.SENDER_NOT_EOA" => TransactionExpectedException::SenderNotEoa, + "TransactionException.SENDER_NOT_EOA" | "SenderNotEOA" => { + TransactionExpectedException::SenderNotEoa + } "TransactionException.PRIORITY_GREATER_THAN_MAX_FEE_PER_GAS" => { TransactionExpectedException::PriorityGreaterThanMaxFeePerGas } @@ -68,6 +69,21 @@ where "TransactionException.INSUFFICIENT_MAX_FEE_PER_BLOB_GAS" => { TransactionExpectedException::InsufficientMaxFeePerBlobGas } + "TR_InitCodeLimitExceeded" => TransactionExpectedException::InitCodeLimitExceeded, + "TR_NonceHasMaxValue" => TransactionExpectedException::NonceHasMaxValue, + "TR_BLOBLIST_OVERSIZE" => TransactionExpectedException::BloblistOversize, + "TR_EMPTYBLOB" => TransactionExpectedException::EmptyBlob, + "TR_BLOBCREATE" => TransactionExpectedException::BlobCreate, + "TR_BLOBVERSION_INVALID" => TransactionExpectedException::BlobVersionInvalid, + "TR_TypeNotSupported" => TransactionExpectedException::TypeNotSupported, + "TR_IntrinsicGas" | "IntrinsicGas" => TransactionExpectedException::IntrinsicGas, + "TR_NoFunds" => TransactionExpectedException::NoFunds, + "TR_TipGtFeeCap" => TransactionExpectedException::TipGtFeeCap, + "TR_GasLimitReached" => TransactionExpectedException::GasLimitReached, + "TR_FeeCapLessThanBlocks" => TransactionExpectedException::FeeCapLessThanBlocks, + "TR_NoFundsX" => TransactionExpectedException::NoFundsX, + "TR_NoFundsOrGas" => TransactionExpectedException::NoFundsOrGas, + "TR_RLP_WRONGVALUE" => TransactionExpectedException::RlpWrongValue, other => panic!("Unexpected error type: {}", other), // Should not fail, TODO is to return an error }) .collect(); @@ -78,6 +94,44 @@ where } } +pub fn deserialize_legacy_forks<'de, D>( + deserializer: D, +) -> Result>, D::Error> +where + D: Deserializer<'de>, +{ + let map: HashMap> = HashMap::deserialize(deserializer)?; + let mut result = HashMap::new(); + + for (fork_name, values) in map { + let spec_id = match fork_name.as_str() { + "Frontier" => SpecId::FRONTIER, + "Homestead" => SpecId::HOMESTEAD, + "Constantinople" => SpecId::CONSTANTINOPLE, + "ConstantinopleFix" | "Petersburg" => SpecId::CONSTANTINOPLE, + "Istanbul" => SpecId::ISTANBUL, + "Berlin" => SpecId::BERLIN, + "London" => SpecId::LONDON, + "Paris" => SpecId::MERGE, + "Merge" => SpecId::MERGE, + "Shanghai" => SpecId::SHANGHAI, + "Cancun" => SpecId::CANCUN, + "Prague" => SpecId::PRAGUE, + "Byzantium" => SpecId::BYZANTIUM, + "EIP158" => SpecId::SPURIOUS_DRAGON, + "EIP150" => SpecId::TANGERINE, + other => { + return Err(serde::de::Error::custom(format!( + "Unknown fork name: {}", + other + ))) + } + }; + result.insert(spec_id, values); + } + Ok(result) +} + pub fn deserialize_ef_post_value_indexes<'de, D>( deserializer: D, ) -> Result, D::Error> @@ -233,11 +287,20 @@ where Vec::::deserialize(deserializer)? .iter() .map(|s| { - U256::from_str(s.trim_start_matches("0x:bigint ")).map_err(|err| { + let mut s = s.trim_start_matches("0x:bigint "); + if s.len() > 64 { + s = &s[..64]; + } + U256::from_str(s).map_err(|err| { serde::de::Error::custom(format!( "error parsing U256 when deserializing U256 vector safely: {err}" )) }) + // U256::from_str(s.trim_start_matches("0x:bigint ")).map_err(|err| { + // serde::de::Error::custom(format!( + // "error parsing U256 when deserializing U256 vector safely: {err}" + // )) + // }) }) .collect() } @@ -309,6 +372,8 @@ impl<'de> Deserialize<'de> for EFTests { let mut transactions = HashMap::new(); + // Why we do this? + // We expect the same for legacy tests? // Note that in this order of iteration, in an example tx with 2 datas, 2 gasLimit and 2 values, order would be // 111, 112, 121, 122, 211, 212, 221, 222 for (data_id, data) in raw_tx.data.iter().enumerate() { diff --git a/cmd/ef_tests/levm/report.rs b/cmd/ef_tests/levm/report.rs index cef2d1b3e3..24e4b44e88 100644 --- a/cmd/ef_tests/levm/report.rs +++ b/cmd/ef_tests/levm/report.rs @@ -129,7 +129,6 @@ pub fn summary_for_slack(reports: &[EFTestReport]) -> String { "text": {{ "type": "mrkdwn", "text": "*Summary*: {total_passed}/{total_run} ({success_percentage:.2}%)\n\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n" - }} }} ] }}"#, @@ -253,10 +252,17 @@ fn fork_summary_shell(reports: &[EFTestReport], fork: SpecId) -> String { } fn fork_statistics(reports: &[EFTestReport], fork: SpecId) -> (usize, usize, f64) { - let fork_tests = reports.iter().filter(|report| report.fork == fork).count(); + // let fork_tests = reports.iter().filter(|report| report.fork == fork).count(); + let fork_tests = reports + .iter() + .filter(|report| report.fork_results.contains_key(&fork)) + .count(); let fork_passed_tests = reports .iter() - .filter(|report| report.fork == fork && report.passed()) + .filter(|report| match report.fork_results.get(&fork) { + Some(result) => result.failed_vectors.is_empty(), + None => false, + }) .count(); let fork_success_percentage = (fork_passed_tests as f64 / fork_tests as f64) * 100.0; (fork_tests, fork_passed_tests, fork_success_percentage) @@ -328,64 +334,81 @@ impl Display for EFTestsReport { writeln!(f)?; writeln!(f, "{}", test_dir_summary_for_shell(&self.0))?; for report in self.0.iter() { - if report.failed_vectors.is_empty() { + // if report.failed_vectors.is_empty() { + // continue; + // } + if report.passed() { continue; } + writeln!(f, "Test: {}", report.name)?; writeln!(f)?; - for (failed_vector, error) in &report.failed_vectors { - writeln!( - f, - "Vector: (data_index: {}, gas_limit_index: {}, value_index: {})", - failed_vector.0, failed_vector.1, failed_vector.2 - )?; - writeln!(f, "Error: {error}")?; - if let Some(re_run_report) = &report.re_run_report { - if let Some(execution_report) = - re_run_report.execution_report.get(failed_vector) - { - if let Some((levm_result, revm_result)) = - &execution_report.execution_result_mismatch - { - writeln!( - f, - "Execution result mismatch: LEVM: {levm_result:?}, REVM: {revm_result:?}", - )?; - } - if let Some((levm_gas_used, revm_gas_used)) = - &execution_report.gas_used_mismatch + for (fork, result) in &report.fork_results { + writeln!(f, "\n Fork: {:?}", fork)?; + if result.failed_vectors.is_empty() { + continue; + } + writeln!(f, " Failed Vectors:")?; + for (failed_vector, error) in &result.failed_vectors { + writeln!( + f, + "Vector: (data_index: {}, gas_limit_index: {}, value_index: {})", + failed_vector.0, failed_vector.1, failed_vector.2 + )?; + writeln!(f, "Error: {error}")?; + if let Some(re_run_report) = &report.re_run_report { + if let Some(execution_report) = + re_run_report.execution_report.get(&(*fork, *failed_vector)) { - writeln!( - f, - "Gas used mismatch: LEVM: {levm_gas_used}, REVM: {revm_gas_used} (diff: {})", - levm_gas_used.abs_diff(*revm_gas_used) - )?; + if let Some((levm_result, revm_result)) = + &execution_report.execution_result_mismatch + { + writeln!( + f, + "Execution result mismatch: LEVM: {levm_result:?}, REVM: {revm_result:?}", + )?; + } + if let Some((levm_gas_used, revm_gas_used)) = + &execution_report.gas_used_mismatch + { + writeln!( + f, + "Gas used mismatch: LEVM: {levm_gas_used}, REVM: {revm_gas_used} (diff: {})", + levm_gas_used.abs_diff(*revm_gas_used) + )?; + } + if let Some((levm_gas_refunded, revm_gas_refunded)) = + &execution_report.gas_refunded_mismatch + { + writeln!( + f, + "Gas refunded mismatch: LEVM: {levm_gas_refunded}, REVM: {revm_gas_refunded} (diff: {})", + levm_gas_refunded.abs_diff(*revm_gas_refunded) + )?; + } + if let Some((levm_result, revm_error)) = + &execution_report.re_runner_error + { + writeln!( + f, + "Re-run error: LEVM: {levm_result:?}, REVM: {revm_error}", + )?; + } } - if let Some((levm_gas_refunded, revm_gas_refunded)) = - &execution_report.gas_refunded_mismatch + + if let Some(account_update) = re_run_report + .account_updates_report + .get(&(*fork, *failed_vector)) { - writeln!( - f, - "Gas refunded mismatch: LEVM: {levm_gas_refunded}, REVM: {revm_gas_refunded} (diff: {})", - levm_gas_refunded.abs_diff(*revm_gas_refunded) - )?; - } - if let Some((levm_result, revm_error)) = &execution_report.re_runner_error { - writeln!(f, "Re-run error: LEVM: {levm_result:?}, REVM: {revm_error}",)?; + writeln!(f, "{}", &account_update.to_string())?; + } else { + writeln!(f, "No account updates report found. Account update reports are only generated for tests that failed at the post-state validation stage.")?; } - } - - if let Some(account_update) = - re_run_report.account_updates_report.get(failed_vector) - { - writeln!(f, "{}", &account_update.to_string())?; } else { - writeln!(f, "No account updates report found. Account update reports are only generated for tests that failed at the post-state validation stage.")?; + writeln!(f, "No re-run report found. Re-run reports are only generated for tests that failed at the post-state validation stage.")?; } - } else { - writeln!(f, "No re-run report found. Re-run reports are only generated for tests that failed at the post-state validation stage.")?; + writeln!(f)?; } - writeln!(f)?; } } Ok(()) @@ -397,27 +420,47 @@ pub struct EFTestReport { pub name: String, pub dir: String, pub test_hash: H256, - pub fork: SpecId, - pub skipped: bool, - pub failed_vectors: HashMap, pub re_run_report: Option, + pub fork_results: HashMap, +} + +#[derive(Debug, Default, Clone, Serialize, Deserialize)] +pub struct EFTestReportForkResult { + pub failed_vectors: HashMap, + pub skipped: bool, } impl EFTestReport { - pub fn new(name: String, dir: String, test_hash: H256, fork: SpecId) -> Self { + pub fn new(name: String, dir: String, test_hash: H256) -> Self { EFTestReport { name, dir, test_hash, - fork, - ..Default::default() + fork_results: HashMap::new(), + re_run_report: None, } } - pub fn new_skipped() -> Self { - EFTestReport { - skipped: true, - ..Default::default() + pub fn register_fork_result(&mut self, fork: SpecId, result: EFTestReportForkResult) { + self.fork_results.insert(fork, result); + } + + pub fn register_re_run_report(&mut self, re_run_report: TestReRunReport) { + self.re_run_report = Some(re_run_report); + } + + pub fn passed(&self) -> bool { + self.fork_results + .values() + .all(|result| result.failed_vectors.is_empty()) + } +} + +impl EFTestReportForkResult { + pub fn new() -> Self { + Self { + failed_vectors: HashMap::new(), + skipped: false, } } @@ -476,18 +519,6 @@ impl EFTestReport { EFTestRunnerError::ExpectedExceptionDoesNotMatchReceived(reason), ); } - - pub fn register_re_run_report(&mut self, re_run_report: TestReRunReport) { - self.re_run_report = Some(re_run_report); - } - - pub fn iter_failed(&self) -> impl Iterator { - self.failed_vectors.iter() - } - - pub fn passed(&self) -> bool { - self.failed_vectors.is_empty() - } } #[derive(Debug, Default, Clone, Serialize, Deserialize)] @@ -807,8 +838,8 @@ pub struct TestReRunExecutionReport { #[derive(Debug, Default, Clone, Serialize, Deserialize)] pub struct TestReRunReport { - pub execution_report: HashMap, - pub account_updates_report: HashMap, + pub execution_report: HashMap<(SpecId, TestVector), TestReRunExecutionReport>, + pub account_updates_report: HashMap<(SpecId, TestVector), ComparisonReport>, } impl TestReRunReport { @@ -818,13 +849,14 @@ impl TestReRunReport { pub fn register_execution_result_mismatch( &mut self, + fork: SpecId, vector: TestVector, levm_result: TxResult, revm_result: RevmExecutionResult, ) { let value = Some((levm_result, revm_result)); self.execution_report - .entry(vector) + .entry((fork, vector)) .and_modify(|report| { report.execution_result_mismatch = value.clone(); }) @@ -836,13 +868,14 @@ impl TestReRunReport { pub fn register_gas_used_mismatch( &mut self, + fork: SpecId, vector: TestVector, levm_gas_used: u64, revm_gas_used: u64, ) { let value = Some((levm_gas_used, revm_gas_used)); self.execution_report - .entry(vector) + .entry((fork, vector)) .and_modify(|report| { report.gas_used_mismatch = value; }) @@ -854,13 +887,14 @@ impl TestReRunReport { pub fn register_gas_refunded_mismatch( &mut self, + fork: SpecId, vector: TestVector, levm_gas_refunded: u64, revm_gas_refunded: u64, ) { let value = Some((levm_gas_refunded, revm_gas_refunded)); self.execution_report - .entry(vector) + .entry((fork, vector)) .and_modify(|report| { report.gas_refunded_mismatch = value; }) @@ -872,21 +906,23 @@ impl TestReRunReport { pub fn register_account_updates_report( &mut self, + fork: SpecId, vector: TestVector, report: ComparisonReport, ) { - self.account_updates_report.insert(vector, report); + self.account_updates_report.insert((fork, vector), report); } pub fn register_re_run_failure( &mut self, + fork: SpecId, vector: TestVector, levm_result: TxResult, revm_error: EVMError, ) { let value = Some((levm_result, revm_error.to_string())); self.execution_report - .entry(vector) + .entry((fork, vector)) .and_modify(|report| { report.re_runner_error = value.clone(); }) diff --git a/cmd/ef_tests/levm/runner/levm_runner.rs b/cmd/ef_tests/levm/runner/levm_runner.rs index 44ee12e12b..55ff429ba5 100644 --- a/cmd/ef_tests/levm/runner/levm_runner.rs +++ b/cmd/ef_tests/levm/runner/levm_runner.rs @@ -1,5 +1,5 @@ use crate::{ - report::{EFTestReport, TestVector}, + report::{EFTestReport, EFTestReportForkResult, TestVector}, runner::{EFTestRunnerError, InternalError}, types::{EFTest, TransactionExpectedException}, utils::{self, effective_gas_price}, @@ -15,60 +15,84 @@ use ethrex_levm::{ Environment, }; use ethrex_storage::AccountUpdate; -use ethrex_vm::{db::StoreWrapper, EvmState}; +use ethrex_vm::{db::StoreWrapper, EvmState, SpecId}; use keccak_hash::keccak; use std::{collections::HashMap, sync::Arc}; -pub fn run_ef_test(test: &EFTest) -> Result { - let hash = test._info.generated_test_hash.or(test._info.hash).unwrap(); +pub fn run_ef_test_multiple_forks(test: &EFTest) -> Result { + // The hash is used to uniquely identify the test, some legacy test may not have a hash so we default to a random one + let hash = test + ._info + .generated_test_hash + .or(test._info.hash) + .unwrap_or_default(); - let mut ef_test_report = - EFTestReport::new(test.name.clone(), test.dir.clone(), hash, test.fork()); - for (vector, _tx) in test.transactions.iter() { - match run_ef_test_tx(vector, test) { - Ok(_) => continue, - Err(EFTestRunnerError::VMInitializationFailed(reason)) => { - ef_test_report.register_vm_initialization_failure(reason, *vector); - } - Err(EFTestRunnerError::FailedToEnsurePreState(reason)) => { - ef_test_report.register_pre_state_validation_failure(reason, *vector); - } - Err(EFTestRunnerError::ExecutionFailedUnexpectedly(error)) => { - ef_test_report.register_unexpected_execution_failure(error, *vector); - } - Err(EFTestRunnerError::FailedToEnsurePostState(transaction_report, reason)) => { - ef_test_report.register_post_state_validation_failure( - transaction_report, - reason, - *vector, - ); - } - Err(EFTestRunnerError::VMExecutionMismatch(_)) => { - return Err(EFTestRunnerError::Internal(InternalError::FirstRunInternal( - "VM execution mismatch errors should only happen when running with revm. This failed during levm's execution." - .to_owned(), - ))); - } - Err(EFTestRunnerError::ExpectedExceptionDoesNotMatchReceived(reason)) => { - ef_test_report.register_post_state_validation_error_mismatch(reason, *vector); + let mut ef_test_report = EFTestReport::new(test.name.clone(), test.dir.clone(), hash); + + for fork in test.post.forks.keys() { + let mut fork_result = EFTestReportForkResult { + failed_vectors: HashMap::new(), + skipped: false, + }; + for (vector, _tx) in test.transactions.iter() { + if !test.post.has_vector_for_fork(vector, *fork) { + continue; } - Err(EFTestRunnerError::Internal(reason)) => { - return Err(EFTestRunnerError::Internal(reason)); + + match run_ef_test_tx(vector, test, *fork) { + Ok(_) => continue, + Err(EFTestRunnerError::VMInitializationFailed(reason)) => { + fork_result.register_vm_initialization_failure(reason, *vector); + } + Err(EFTestRunnerError::FailedToEnsurePreState(reason)) => { + fork_result.register_pre_state_validation_failure(reason, *vector); + } + Err(EFTestRunnerError::ExecutionFailedUnexpectedly(error)) => { + fork_result.register_unexpected_execution_failure(error, *vector); + } + Err(EFTestRunnerError::FailedToEnsurePostState(transaction_report, reason)) => { + fork_result.register_post_state_validation_failure( + transaction_report, + reason, + *vector, + ); + } + Err(EFTestRunnerError::VMExecutionMismatch(_)) => { + return Err(EFTestRunnerError::Internal(InternalError::FirstRunInternal( + "VM execution mismatch errors should only happen when running with revm. This failed during levm's execution." + .to_owned(), + ))); + } + Err(EFTestRunnerError::ExpectedExceptionDoesNotMatchReceived(reason)) => { + fork_result.register_post_state_validation_error_mismatch(reason, *vector); + } + Err(EFTestRunnerError::Internal(reason)) => { + return Err(EFTestRunnerError::Internal(reason)); + } } } + ef_test_report.register_fork_result(*fork, fork_result); } Ok(ef_test_report) } -pub fn run_ef_test_tx(vector: &TestVector, test: &EFTest) -> Result<(), EFTestRunnerError> { - let mut levm = prepare_vm_for_tx(vector, test)?; +pub fn run_ef_test_tx( + vector: &TestVector, + test: &EFTest, + fork: SpecId, +) -> Result<(), EFTestRunnerError> { + let mut levm = prepare_vm_for_tx(vector, test, fork)?; ensure_pre_state(&levm, test)?; let levm_execution_result = levm.transact(); - ensure_post_state(&levm_execution_result, vector, test)?; + ensure_post_state_multi_forks(&levm_execution_result, vector, test, fork)?; Ok(()) } -pub fn prepare_vm_for_tx(vector: &TestVector, test: &EFTest) -> Result { +pub fn prepare_vm_for_tx( + vector: &TestVector, + test: &EFTest, + fork: SpecId, +) -> Result { let (initial_state, block_hash) = utils::load_initial_state(test); let db = Arc::new(StoreWrapper { store: initial_state.database().unwrap().clone(), @@ -110,7 +134,7 @@ pub fn prepare_vm_for_tx(vector: &TestVector, test: &EFTest) -> Result, vector: &TestVector, test: &EFTest, + fork: SpecId, ) -> Result<(), EFTestRunnerError> { match levm_execution_result { Ok(execution_report) => { - match test.post.vector_post_value(vector).expect_exception { + match test + .post + .vector_post_value(vector, fork) + .unwrap() + .expect_exception + { // Execution result was successful but an exception was expected. Some(expected_exceptions) => { // Note: expected_exceptions is a vector because can only have 1 or 2 expected errors. @@ -285,7 +315,8 @@ pub fn ensure_post_state( let levm_account_updates = get_state_transitions(&initial_state, block_hash, execution_report); let pos_state_root = post_state_root(&levm_account_updates, test); - let expected_post_state_root_hash = test.post.vector_post_value(vector).hash; + let expected_post_state_root_hash = + test.post.vector_post_value(vector, fork).unwrap().hash; if expected_post_state_root_hash != pos_state_root { let error_reason = format!( "Post-state root mismatch: expected {expected_post_state_root_hash:#x}, got {pos_state_root:#x}", @@ -299,7 +330,12 @@ pub fn ensure_post_state( } } Err(err) => { - match test.post.vector_post_value(vector).expect_exception { + match test + .post + .vector_post_value(vector, fork) + .unwrap() + .expect_exception + { // Execution result was unsuccessful and an exception was expected. Some(expected_exceptions) => { // Note: expected_exceptions is a vector because can only have 1 or 2 expected errors. @@ -333,7 +369,7 @@ pub fn ensure_post_state( } } } - }; + } Ok(()) } diff --git a/cmd/ef_tests/levm/runner/mod.rs b/cmd/ef_tests/levm/runner/mod.rs index de60879e76..a499dd44bd 100644 --- a/cmd/ef_tests/levm/runner/mod.rs +++ b/cmd/ef_tests/levm/runner/mod.rs @@ -104,7 +104,7 @@ fn run_with_levm( if !opts.spinner && opts.verbose { println!("Running test: {:?}", test.name); } - let ef_test_report = match levm_runner::run_ef_test(test) { + let ef_test_report = match levm_runner::run_ef_test_multiple_forks(test) { Ok(ef_test_report) => ef_test_report, Err(EFTestRunnerError::Internal(err)) => return Err(EFTestRunnerError::Internal(err)), non_internal_errors => { @@ -174,7 +174,7 @@ fn run_with_revm( ), opts.spinner, ); - let ef_test_report = match revm_runner::_run_ef_test_revm(test) { + let ef_test_report = match revm_runner::_run_ef_test_revm_multiple_forks(test) { Ok(ef_test_report) => ef_test_report, Err(EFTestRunnerError::Internal(err)) => return Err(EFTestRunnerError::Internal(err)), non_internal_errors => { @@ -237,7 +237,7 @@ fn re_run_with_revm( opts.spinner, ); - match revm_runner::re_run_failed_ef_test( + match revm_runner::re_run_failed_ef_test_multiple_forks( ef_tests .iter() .find(|test| { @@ -245,7 +245,7 @@ fn re_run_with_revm( ._info .generated_test_hash .or(test._info.hash) - .unwrap(); + .unwrap_or_default(); let failed_hash = failed_test_report.test_hash; diff --git a/cmd/ef_tests/levm/runner/revm_runner.rs b/cmd/ef_tests/levm/runner/revm_runner.rs index 0a5ad0db46..f76b932b51 100644 --- a/cmd/ef_tests/levm/runner/revm_runner.rs +++ b/cmd/ef_tests/levm/runner/revm_runner.rs @@ -1,5 +1,5 @@ use crate::{ - report::{ComparisonReport, EFTestReport, TestReRunReport, TestVector}, + report::{ComparisonReport, EFTestReport, EFTestReportForkResult, TestReRunReport, TestVector}, runner::{ levm_runner::{self, post_state_root}, EFTestRunnerError, InternalError, @@ -14,7 +14,7 @@ use ethrex_levm::{ Account, StorageSlot, }; use ethrex_storage::{error::StoreError, AccountUpdate}; -use ethrex_vm::{db::StoreWrapper, EvmState, RevmAddress, RevmU256}; +use ethrex_vm::{db::StoreWrapper, EvmState, RevmAddress, RevmU256, SpecId}; use revm::{ db::State, inspectors::TracerEip3155 as RevmTracerEip3155, @@ -26,44 +26,54 @@ use revm::{ }; use std::collections::{HashMap, HashSet}; -pub fn re_run_failed_ef_test( +pub fn re_run_failed_ef_test_multiple_forks( test: &EFTest, failed_test_report: &EFTestReport, ) -> Result { assert_eq!(test.name, failed_test_report.name); + // let mut re_run_reports = Vec::new(); let mut re_run_report = TestReRunReport::new(); - for (vector, vector_failure) in failed_test_report.failed_vectors.iter() { - match vector_failure { - // We only want to re-run tests that failed in the post-state validation. - EFTestRunnerError::FailedToEnsurePostState(transaction_report, _) => { - match re_run_failed_ef_test_tx(vector, test, transaction_report, &mut re_run_report) { - Ok(_) => continue, - Err(EFTestRunnerError::VMInitializationFailed(reason)) => { - return Err(EFTestRunnerError::Internal(InternalError::ReRunInternal( - format!("REVM initialization failed when re-running failed test: {reason}"), re_run_report.clone() - ))); - } - Err(EFTestRunnerError::Internal(reason)) => { - return Err(EFTestRunnerError::Internal(reason)); - } - unexpected_error => { - return Err(EFTestRunnerError::Internal(InternalError::ReRunInternal(format!( - "Unexpected error when re-running failed test: {unexpected_error:?}" - ), re_run_report.clone()))); + for (fork, fork_result) in &failed_test_report.fork_results { + if fork_result.failed_vectors.is_empty() { + continue; + } + + //re-run each failed vec for this fork + for (vector, vector_failure) in &fork_result.failed_vectors { + match vector_failure { + // We only want to re-run tests that failed in the post-state validation. + EFTestRunnerError::FailedToEnsurePostState(transaction_report, _) => { + match re_run_failed_ef_test_tx(vector, test, transaction_report, &mut re_run_report,*fork) { + Ok(_) => continue, + Err(EFTestRunnerError::VMInitializationFailed(reason)) => { + return Err(EFTestRunnerError::Internal(InternalError::ReRunInternal( + format!("REVM initialization failed when re-running failed test: {reason}"), re_run_report.clone() + ))); + } + Err(EFTestRunnerError::Internal(reason)) => { + return Err(EFTestRunnerError::Internal(reason)); + } + unexpected_error => { + return Err(EFTestRunnerError::Internal(InternalError::ReRunInternal(format!( + "Unexpected error when re-running failed test: {unexpected_error:?}" + ), re_run_report.clone()))); + } } - } - }, - // Currently, we decided not to re-execute the test when the Expected exception does not match - // with the received. This can change in the future. - EFTestRunnerError::ExpectedExceptionDoesNotMatchReceived(_) => continue, - EFTestRunnerError::VMInitializationFailed(_) - | EFTestRunnerError::ExecutionFailedUnexpectedly(_) - | EFTestRunnerError::FailedToEnsurePreState(_) => continue, - EFTestRunnerError::VMExecutionMismatch(reason) => return Err(EFTestRunnerError::Internal(InternalError::ReRunInternal( - format!("VM execution mismatch errors should only happen when running with revm. This failed during levm's execution: {reason}"), re_run_report.clone()))), - EFTestRunnerError::Internal(reason) => return Err(EFTestRunnerError::Internal(reason.to_owned())), + }, + // Currently, we decided not to re-execute the test when the Expected exception does not match + // with the received. This can change in the future. + EFTestRunnerError::ExpectedExceptionDoesNotMatchReceived(_) => continue, + EFTestRunnerError::VMInitializationFailed(_) + | EFTestRunnerError::ExecutionFailedUnexpectedly(_) + | EFTestRunnerError::FailedToEnsurePreState(_) => continue, + EFTestRunnerError::VMExecutionMismatch(reason) => return Err(EFTestRunnerError::Internal(InternalError::ReRunInternal( + format!("VM execution mismatch errors should only happen when running with revm. This failed during levm's execution: {reason}"), re_run_report.clone()))), + EFTestRunnerError::Internal(reason) => return Err(EFTestRunnerError::Internal(reason.to_owned())), + } } + // re_run_reports.push(re_run_report); } + // Return just the first report for now Ok(re_run_report) } @@ -72,9 +82,14 @@ pub fn re_run_failed_ef_test_tx( test: &EFTest, levm_execution_report: &TransactionReport, re_run_report: &mut TestReRunReport, + fork: SpecId, ) -> Result<(), EFTestRunnerError> { let (mut state, _block_hash) = load_initial_state(test); - let mut revm = prepare_revm_for_tx(&mut state, vector, test)?; + let mut revm = prepare_revm_for_tx(&mut state, vector, test, fork)?; + // Check if vector is in the fork if not skip the test + if !test.post.has_vector_for_fork(vector, fork) { + return Ok(()); + } let revm_execution_result = revm.transact_commit(); drop(revm); // Need to drop the state mutable reference. compare_levm_revm_execution_results( @@ -82,6 +97,7 @@ pub fn re_run_failed_ef_test_tx( levm_execution_report, revm_execution_result, re_run_report, + fork, )?; ensure_post_state( levm_execution_report, @@ -89,6 +105,7 @@ pub fn re_run_failed_ef_test_tx( &mut state, test, re_run_report, + fork, )?; Ok(()) } @@ -97,6 +114,7 @@ pub fn prepare_revm_for_tx<'state>( initial_state: &'state mut EvmState, vector: &TestVector, test: &EFTest, + fork: SpecId, ) -> Result>, EFTestRunnerError> { let chain_spec = initial_state .chain_config() @@ -194,7 +212,8 @@ pub fn prepare_revm_for_tx<'state>( .with_block_env(block_env) .with_tx_env(tx_env) .modify_cfg_env(|cfg| cfg.chain_id = chain_spec.chain_id) - .with_spec_id(test.fork()) + // .with_spec_id(test.fork()) // here we should pass the fork + .with_spec_id(fork) // here we should pass the fork .with_external_context( RevmTracerEip3155::new(Box::new(std::io::stderr())).without_summary(), ); @@ -211,6 +230,7 @@ pub fn compare_levm_revm_execution_results( levm_execution_report: &TransactionReport, revm_execution_result: Result>, re_run_report: &mut TestReRunReport, + fork: SpecId, ) -> Result<(), EFTestRunnerError> { match (levm_execution_report, revm_execution_result) { (levm_tx_report, Ok(revm_execution_result)) => { @@ -227,6 +247,7 @@ pub fn compare_levm_revm_execution_results( ) => { if levm_tx_report.gas_used != revm_gas_used { re_run_report.register_gas_used_mismatch( + fork, *vector, levm_tx_report.gas_used, revm_gas_used, @@ -234,6 +255,7 @@ pub fn compare_levm_revm_execution_results( } if levm_tx_report.gas_refunded != revm_gas_refunded { re_run_report.register_gas_refunded_mismatch( + fork, *vector, levm_tx_report.gas_refunded, revm_gas_refunded, @@ -249,6 +271,7 @@ pub fn compare_levm_revm_execution_results( ) => { if levm_tx_report.gas_used != revm_gas_used { re_run_report.register_gas_used_mismatch( + fork, *vector, levm_tx_report.gas_used, revm_gas_used, @@ -265,6 +288,7 @@ pub fn compare_levm_revm_execution_results( // TODO: Register the revert reasons. if levm_tx_report.gas_used != revm_gas_used { re_run_report.register_gas_used_mismatch( + fork, *vector, levm_tx_report.gas_used, revm_gas_used, @@ -273,6 +297,7 @@ pub fn compare_levm_revm_execution_results( } _ => { re_run_report.register_execution_result_mismatch( + fork, *vector, levm_tx_report.result.clone(), revm_execution_result.clone(), @@ -282,6 +307,7 @@ pub fn compare_levm_revm_execution_results( } (levm_transaction_report, Err(revm_error)) => { re_run_report.register_re_run_failure( + fork, *vector, levm_transaction_report.result.clone(), revm_error, @@ -297,8 +323,15 @@ pub fn ensure_post_state( revm_state: &mut EvmState, test: &EFTest, re_run_report: &mut TestReRunReport, + fork: SpecId, ) -> Result<(), EFTestRunnerError> { - match test.post.vector_post_value(vector).expect_exception { + // unimplemented!() + match test + .post + .vector_post_value(vector, fork) + .unwrap() + .expect_exception + { Some(_expected_exception) => {} // We only want to compare account updates when no exception is expected. None => { @@ -314,7 +347,7 @@ pub fn ensure_post_state( &levm_account_updates, &revm_account_updates, ); - re_run_report.register_account_updates_report(*vector, account_updates_report); + re_run_report.register_account_updates_report(fork, *vector, account_updates_report); } } @@ -386,58 +419,72 @@ pub fn compare_levm_revm_account_updates( } } -pub fn _run_ef_test_revm(test: &EFTest) -> Result { - let hash = test._info.generated_test_hash.or(test._info.hash).unwrap(); +pub fn _run_ef_test_revm_multiple_forks(test: &EFTest) -> Result { + let hash = test + ._info + .generated_test_hash + .or(test._info.hash) + .unwrap_or_default(); - let mut ef_test_report = - EFTestReport::new(test.name.clone(), test.dir.clone(), hash, test.fork()); - for (vector, _tx) in test.transactions.iter() { - match _run_ef_test_tx_revm(vector, test) { - Ok(_) => continue, - Err(EFTestRunnerError::VMInitializationFailed(reason)) => { - ef_test_report.register_vm_initialization_failure(reason, *vector); - } - Err(EFTestRunnerError::FailedToEnsurePreState(reason)) => { - ef_test_report.register_pre_state_validation_failure(reason, *vector); - } - Err(EFTestRunnerError::ExecutionFailedUnexpectedly(error)) => { - ef_test_report.register_unexpected_execution_failure(error, *vector); - } - Err(EFTestRunnerError::FailedToEnsurePostState(transaction_report, reason)) => { - ef_test_report.register_post_state_validation_failure( - transaction_report, - reason, - *vector, - ); - } - Err(EFTestRunnerError::VMExecutionMismatch(_)) => { - return Err(EFTestRunnerError::Internal(InternalError::FirstRunInternal( - "VM execution mismatch errors should only happen when COMPARING LEVM AND REVM. This failed during revm's execution." - .to_owned(), - ))); - } - Err(EFTestRunnerError::Internal(reason)) => { - return Err(EFTestRunnerError::Internal(reason)); + let mut ef_test_report = EFTestReport::new(test.name.clone(), test.dir.clone(), hash); + for fork in test.post.forks.keys() { + let mut fork_result = EFTestReportForkResult::new(); + for (vector, _tx) in test.transactions.iter() { + if !test.post.has_vector_for_fork(vector, *fork) { + continue; } - Err(EFTestRunnerError::ExpectedExceptionDoesNotMatchReceived(_)) => { - return Err(EFTestRunnerError::Internal(InternalError::MainRunnerInternal( - "The ExpectedExceptionDoesNotMatchReceived error should only happen when executing Levm, the errors matching is not implemented in Revm" - .to_owned(), - ))); + match _run_ef_test_tx_revm(vector, test, *fork) { + Ok(_) => continue, + Err(EFTestRunnerError::VMInitializationFailed(reason)) => { + fork_result.register_vm_initialization_failure(reason, *vector); + } + Err(EFTestRunnerError::FailedToEnsurePreState(reason)) => { + fork_result.register_pre_state_validation_failure(reason, *vector); + } + Err(EFTestRunnerError::ExecutionFailedUnexpectedly(error)) => { + fork_result.register_unexpected_execution_failure(error, *vector); + } + Err(EFTestRunnerError::FailedToEnsurePostState(transaction_report, reason)) => { + fork_result.register_post_state_validation_failure( + transaction_report, + reason, + *vector, + ); + } + Err(EFTestRunnerError::VMExecutionMismatch(_)) => { + return Err(EFTestRunnerError::Internal(InternalError::FirstRunInternal( + "VM execution mismatch errors should only happen when COMPARING LEVM AND REVM. This failed during revm's execution." + .to_owned(), + ))); + } + Err(EFTestRunnerError::Internal(reason)) => { + return Err(EFTestRunnerError::Internal(reason)); + } + Err(EFTestRunnerError::ExpectedExceptionDoesNotMatchReceived(_)) => { + return Err(EFTestRunnerError::Internal(InternalError::MainRunnerInternal( + "The ExpectedExceptionDoesNotMatchReceived error should only happen when executing Levm, the errors matching is not implemented in Revm" + .to_owned(), + ))); + } } } + ef_test_report.register_fork_result(*fork, fork_result); } Ok(ef_test_report) } -pub fn _run_ef_test_tx_revm(vector: &TestVector, test: &EFTest) -> Result<(), EFTestRunnerError> { +pub fn _run_ef_test_tx_revm( + vector: &TestVector, + test: &EFTest, + fork: SpecId, +) -> Result<(), EFTestRunnerError> { // dbg!(vector); let (mut state, _block_hash) = load_initial_state(test); - let mut revm = prepare_revm_for_tx(&mut state, vector, test)?; + let mut revm = prepare_revm_for_tx(&mut state, vector, test, fork)?; let revm_execution_result = revm.transact_commit(); drop(revm); // Need to drop the state mutable reference. - _ensure_post_state_revm(revm_execution_result, vector, test, &mut state)?; + _ensure_post_state_revm(revm_execution_result, vector, test, &mut state, fork)?; Ok(()) } @@ -447,10 +494,17 @@ pub fn _ensure_post_state_revm( vector: &TestVector, test: &EFTest, revm_state: &mut EvmState, + fork: SpecId, ) -> Result<(), EFTestRunnerError> { + // unimplemented!(); match revm_execution_result { Ok(_execution_result) => { - match test.post.vector_post_value(vector).expect_exception { + match test + .post + .vector_post_value(vector, fork) + .unwrap() + .expect_exception + { // Execution result was successful but an exception was expected. Some(expected_exception) => { let error_reason = format!("Expected exception: {expected_exception:?}"); @@ -472,7 +526,8 @@ pub fn _ensure_post_state_revm( None => { let revm_account_updates = ethrex_vm::get_state_transitions(revm_state); let pos_state_root = post_state_root(&revm_account_updates, test); - let expected_post_state_root_hash = test.post.vector_post_value(vector).hash; + let expected_post_state_root_hash = + test.post.vector_post_value(vector, fork).unwrap().hash; if expected_post_state_root_hash != pos_state_root { println!( "Post-state root mismatch: expected {expected_post_state_root_hash:#x}, got {pos_state_root:#x}", @@ -498,7 +553,12 @@ pub fn _ensure_post_state_revm( } } Err(err) => { - match test.post.vector_post_value(vector).expect_exception { + match test + .post + .vector_post_value(vector, fork) + .unwrap() + .expect_exception + { // Execution result was unsuccessful and an exception was expected. // TODO: See if we want to map revm exceptions to expected exceptions, probably not. Some(_expected_exception) => {} diff --git a/cmd/ef_tests/levm/types.rs b/cmd/ef_tests/levm/types.rs index 5295f7f4da..f2d06e24a1 100644 --- a/cmd/ef_tests/levm/types.rs +++ b/cmd/ef_tests/levm/types.rs @@ -1,5 +1,6 @@ use crate::{ deserialize::{ + deserialize_legacy_forks, deserialize_access_lists, deserialize_authorization_lists, deserialize_ef_post_value_indexes, deserialize_h256_vec_optional_safe, deserialize_hex_bytes, deserialize_hex_bytes_vec, @@ -27,31 +28,11 @@ pub struct EFTest { pub dir: String, pub _info: EFTestInfo, pub env: EFTestEnv, - pub post: EFTestPost, + pub post: EFTestPostStruct, pub pre: EFTestPre, pub transactions: HashMap, } -impl EFTest { - pub fn fork(&self) -> SpecId { - match &self.post { - EFTestPost::Prague(_) => SpecId::PRAGUE, - EFTestPost::Cancun(_) => SpecId::CANCUN, - EFTestPost::Shanghai(_) => SpecId::SHANGHAI, - EFTestPost::Homestead(_) => SpecId::HOMESTEAD, - EFTestPost::Istanbul(_) => SpecId::ISTANBUL, - EFTestPost::London(_) => SpecId::LONDON, - EFTestPost::Byzantium(_) => SpecId::BYZANTIUM, - EFTestPost::Berlin(_) => SpecId::BERLIN, - EFTestPost::Constantinople(_) | EFTestPost::ConstantinopleFix(_) => { - SpecId::CONSTANTINOPLE - } - EFTestPost::Paris(_) => SpecId::MERGE, - EFTestPost::Frontier(_) => SpecId::FRONTIER, - } - } -} - impl From<&EFTest> for Genesis { fn from(test: &EFTest) -> Self { Genesis { @@ -110,6 +91,10 @@ pub struct EFTestInfo { pub reference_spec: Option, #[serde(rename = "reference-spec-version", default)] pub reference_spec_version: Option, + + // These fields are implemented in the Legacy Test + #[serde(rename = "filledwith", default)] + pub filled_with: Option, } #[derive(Debug, Deserialize)] @@ -147,42 +132,35 @@ pub enum EFTestPost { Frontier(Vec), } -impl EFTestPost { - pub fn values(self) -> Vec { - match self { - EFTestPost::Prague(v) => v, - EFTestPost::Cancun(v) => v, - EFTestPost::Shanghai(v) => v, - EFTestPost::Homestead(v) => v, - EFTestPost::Istanbul(v) => v, - EFTestPost::London(v) => v, - EFTestPost::Byzantium(v) => v, - EFTestPost::Berlin(v) => v, - EFTestPost::Constantinople(v) => v, - EFTestPost::Paris(v) => v, - EFTestPost::ConstantinopleFix(v) => v, - EFTestPost::Frontier(v) => v, - } +/// This struct is needed in order to support modernt EFTest at the same time as Legacy EFTest. +/// In legacy EFTest, the post field map to a HashMap>. +/// In modern EFTest, the post field map to a Vec. +#[derive(Debug, Deserialize)] +pub struct EFTestPostStruct { + #[serde(flatten)] + #[serde(deserialize_with = "deserialize_legacy_forks")] + pub forks: HashMap>, +} + +impl EFTestPostStruct { + // Helper method to check if a vector exists for a fork + pub fn has_vector_for_fork(&self, vector: &TestVector, fork: SpecId) -> bool { + self.forks + .get(&fork) + .and_then(|values| Self::find_vector_post_value(values, vector)) + .is_some() } - pub fn vector_post_value(&self, vector: &TestVector) -> EFTestPostValue { - match self { - EFTestPost::Prague(v) => Self::find_vector_post_value(v, vector), - EFTestPost::Cancun(v) => Self::find_vector_post_value(v, vector), - EFTestPost::Shanghai(v) => Self::find_vector_post_value(v, vector), - EFTestPost::Homestead(v) => Self::find_vector_post_value(v, vector), - EFTestPost::Istanbul(v) => Self::find_vector_post_value(v, vector), - EFTestPost::London(v) => Self::find_vector_post_value(v, vector), - EFTestPost::Byzantium(v) => Self::find_vector_post_value(v, vector), - EFTestPost::Berlin(v) => Self::find_vector_post_value(v, vector), - EFTestPost::Constantinople(v) => Self::find_vector_post_value(v, vector), - EFTestPost::Paris(v) => Self::find_vector_post_value(v, vector), - EFTestPost::ConstantinopleFix(v) => Self::find_vector_post_value(v, vector), - EFTestPost::Frontier(v) => Self::find_vector_post_value(v, vector), - } + // Get post value for a given fork + pub fn vector_post_value(&self, vector: &TestVector, fork: SpecId) -> Option { + let values = self.forks.get(&fork).unwrap(); + Self::find_vector_post_value(values, vector) } - fn find_vector_post_value(values: &[EFTestPostValue], vector: &TestVector) -> EFTestPostValue { + fn find_vector_post_value( + values: &[EFTestPostValue], + vector: &TestVector, + ) -> Option { values .iter() .find(|v| { @@ -191,25 +169,7 @@ impl EFTestPost { let value_index = v.indexes.get("value").unwrap().as_usize(); vector == &(data_index, gas_limit_index, value_index) }) - .unwrap() - .clone() - } - - pub fn iter(&self) -> impl Iterator { - match self { - EFTestPost::Prague(v) => v.iter(), - EFTestPost::Cancun(v) => v.iter(), - EFTestPost::Shanghai(v) => v.iter(), - EFTestPost::Homestead(v) => v.iter(), - EFTestPost::Istanbul(v) => v.iter(), - EFTestPost::London(v) => v.iter(), - EFTestPost::Byzantium(v) => v.iter(), - EFTestPost::Berlin(v) => v.iter(), - EFTestPost::Constantinople(v) => v.iter(), - EFTestPost::Paris(v) => v.iter(), - EFTestPost::ConstantinopleFix(v) => v.iter(), - EFTestPost::Frontier(v) => v.iter(), - } + .cloned() } } @@ -232,6 +192,22 @@ pub enum TransactionExpectedException { GasLimitPriceProductOverflow, Type3TxPreFork, InsufficientMaxFeePerBlobGas, + // LEGACY + InitCodeLimitExceeded, + NonceHasMaxValue, + BloblistOversize, + EmptyBlob, + BlobCreate, + BlobVersionInvalid, + TypeNotSupported, + IntrinsicGas, + NoFunds, + TipGtFeeCap, + GasLimitReached, + FeeCapLessThanBlocks, + NoFundsX, + NoFundsOrGas, + RlpWrongValue, } #[derive(Debug, Deserialize, Clone)] @@ -246,7 +222,7 @@ pub struct EFTestPostValue { #[serde(deserialize_with = "deserialize_ef_post_value_indexes")] pub indexes: HashMap, pub logs: H256, - #[serde(deserialize_with = "deserialize_hex_bytes")] + #[serde(default, deserialize_with = "deserialize_hex_bytes")] pub txbytes: Bytes, } diff --git a/crates/vm/levm/Makefile b/crates/vm/levm/Makefile index 02c7f2a73a..14371ebcf0 100644 --- a/crates/vm/levm/Makefile +++ b/crates/vm/levm/Makefile @@ -14,33 +14,110 @@ lint: ## ๐Ÿงน Linter check fmt: ## ๐Ÿ“„ Runs rustfmt cargo fmt --all +# ###### EF Tests ###### +# VECTORS_DIR := ../../../cmd/ef_tests/levm/vectors +# TMP_DIR := tmp +# +# SPECTEST_VERSION := v14.1 +# SPECTEST_ARTIFACT := tests_$(SPECTEST_VERSION).tar.gz +# SPECTEST_URL := https://github.com/ethereum/tests/archive/refs/tags/$(SPECTEST_VERSION).tar.gz +# +# STATETEST_VERSION := pectra-devnet-5%40v1.1.0 +# STATETEST_NET := pectra-devnet-5 +# STATETEST_ARTIFACT := fixtures_$(STATETEST_NET).tar.gz +# STATETEST_URL := https://github.com/ethereum/execution-spec-tests/releases/download/$(STATETEST_VERSION)/fixtures_$(STATETEST_NET).tar.gz +# +# download-evm-ef-tests: ## ๐Ÿ“ฅ Download EF Tests +# mkdir -p $(TMP_DIR) +# mkdir -p $(VECTORS_DIR)/GeneralStateTests +# mkdir -p $(VECTORS_DIR)/stEIP2537 +# mkdir -p $(VECTORS_DIR)/state_tests +# curl -L -o $(SPECTEST_ARTIFACT) $(SPECTEST_URL) +# tar -xzf $(SPECTEST_ARTIFACT) -C $(TMP_DIR) +# mv $(TMP_DIR)/tests-14.1/GeneralStateTests/* $(VECTORS_DIR)/GeneralStateTests/ +# mv $(TMP_DIR)/tests-14.1/EIPTests/StateTests/stEIP2537/* $(VECTORS_DIR)/stEIP2537/ +# curl -L -o $(STATETEST_ARTIFACT) $(STATETEST_URL) +# tar -xzf $(STATETEST_ARTIFACT) -C $(TMP_DIR) +# mv $(TMP_DIR)/fixtures/state_tests/* $(VECTORS_DIR)/state_tests/ +# rm -rf $(TMP_DIR) tests_*.tar.gz +# rm -rf $(TMP_DIR) fixtures_*.tar.gz +# +# run-evm-ef-tests: ## ๐Ÿƒโ€โ™‚๏ธ Run EF Tests +# if [ "$(QUIET)" = "true" ]; then \ +# cd ../../../ && \ +# time cargo test --quiet -p ef_tests-levm --test ef_tests_levm --release -- $(flags) --summary;\ +# else \ +# cd ../../../ && \ +# time cargo test -p ef_tests-levm --test ef_tests_levm --release -- $(flags);\ +# fi + +# run-evm-ef-tests-ci: ## ๐Ÿƒโ€โ™‚๏ธ Run EF Tests only with LEVM and without spinner, for CI. +# cd ../../../ && \ +# time cargo test -p ef_tests-levm --test ef_tests_levm --release -- --summary +# +# generate-evm-ef-tests-report: ## ๐Ÿ“Š Generate EF Tests Report +# cd ../../../ && \ +# cargo test -p ef_tests-levm --test ef_tests_levm --release -- --summary + +# clean-evm-ef-tests: ## ๐Ÿ—‘๏ธ Clean test vectors +# rm -rf $(VECTORS_DIR) + ###### EF Tests ###### VECTORS_DIR := ../../../cmd/ef_tests/levm/vectors TMP_DIR := tmp +TESTS_REPO := $(TMP_DIR)/ethereum-tests +LEGACY_TESTS_DIRS := Constantinople Cancun -SPECTEST_VERSION := v14.1 -SPECTEST_ARTIFACT := tests_$(SPECTEST_VERSION).tar.gz -SPECTEST_URL := https://github.com/ethereum/tests/archive/refs/tags/$(SPECTEST_VERSION).tar.gz - +# State test fixtures configuration STATETEST_VERSION := pectra-devnet-5%40v1.1.0 STATETEST_NET := pectra-devnet-5 STATETEST_ARTIFACT := fixtures_$(STATETEST_NET).tar.gz STATETEST_URL := https://github.com/ethereum/execution-spec-tests/releases/download/$(STATETEST_VERSION)/fixtures_$(STATETEST_NET).tar.gz -download-evm-ef-tests: ## ๐Ÿ“ฅ Download EF Tests - mkdir -p $(TMP_DIR) +setup-test-dirs: + mkdir -p $(VECTORS_DIR) + # Create directory structure for legacy tests + mkdir -p $(VECTORS_DIR)/legacytests/Constantinople/GeneralStateTests + mkdir -p $(VECTORS_DIR)/legacytests/Cancun/GeneralStateTests + # Create directories for other test types mkdir -p $(VECTORS_DIR)/GeneralStateTests mkdir -p $(VECTORS_DIR)/stEIP2537 mkdir -p $(VECTORS_DIR)/state_tests - curl -L -o $(SPECTEST_ARTIFACT) $(SPECTEST_URL) - tar -xzf $(SPECTEST_ARTIFACT) -C $(TMP_DIR) - mv $(TMP_DIR)/tests-14.1/GeneralStateTests/* $(VECTORS_DIR)/GeneralStateTests/ - mv $(TMP_DIR)/tests-14.1/EIPTests/StateTests/stEIP2537/* $(VECTORS_DIR)/stEIP2537/ + +clone-ef-tests: ## ๐Ÿ“ฅ Clone Ethereum Tests repository with submodules + mkdir -p $(TMP_DIR) + git clone --recurse-submodules https://github.com/ethereum/tests.git $(TESTS_REPO) + +update-ef-tests: ## ๐Ÿ”„ Update Ethereum Tests repository and submodules + cd $(TESTS_REPO) && git pull && git submodule update --init --recursive + +setup-legacy-tests: setup-test-dirs ## ๐Ÿ“ฅ Setup Legacy Tests + # Copy from main repository + cp -r $(TESTS_REPO)/GeneralStateTests/* $(VECTORS_DIR)/GeneralStateTests/ + cp -r $(TESTS_REPO)/EIPTests/StateTests/stEIP2537/* $(VECTORS_DIR)/stEIP2537/ + # Copy legacy tests + @for dir in $(LEGACY_TESTS_DIRS); do \ + if [ -d "$(TESTS_REPO)/LegacyTests/$$dir/GeneralStateTests" ]; then \ + echo "Copying tests from LegacyTests/$$dir/GeneralStateTests"; \ + cp -r $(TESTS_REPO)/LegacyTests/$$dir/GeneralStateTests/* $(VECTORS_DIR)/legacytests/$$dir/GeneralStateTests/; \ + else \ + echo "Warning: Directory $(TESTS_REPO)/LegacyTests/$$dir/GeneralStateTests not found"; \ + fi \ + done + +download-state-tests: ## ๐Ÿ“ฅ Download and setup state tests fixtures curl -L -o $(STATETEST_ARTIFACT) $(STATETEST_URL) tar -xzf $(STATETEST_ARTIFACT) -C $(TMP_DIR) mv $(TMP_DIR)/fixtures/state_tests/* $(VECTORS_DIR)/state_tests/ - rm -rf $(TMP_DIR) tests_*.tar.gz - rm -rf $(TMP_DIR) fixtures_*.tar.gz + rm -f $(STATETEST_ARTIFACT) + +download-evm-ef-tests: clone-ef-tests setup-legacy-tests download-state-tests ## ๐Ÿ“ฅ Download and setup all EF Tests + rm -rf $(TMP_DIR) + +clean-evm-ef-tests: ## ๐Ÿ—‘๏ธ Clean test vectors and temporary files + rm -rf $(VECTORS_DIR) + rm -rf $(TMP_DIR) + rm -f $(STATETEST_ARTIFACT) run-evm-ef-tests: ## ๐Ÿƒโ€โ™‚๏ธ Run EF Tests if [ "$(QUIET)" = "true" ]; then \ @@ -51,17 +128,6 @@ run-evm-ef-tests: ## ๐Ÿƒโ€โ™‚๏ธ Run EF Tests time cargo test -p ef_tests-levm --test ef_tests_levm --release -- $(flags);\ fi -run-evm-ef-tests-ci: ## ๐Ÿƒโ€โ™‚๏ธ Run EF Tests only with LEVM and without spinner, for CI. - cd ../../../ && \ - time cargo test -p ef_tests-levm --test ef_tests_levm --release -- --summary - -generate-evm-ef-tests-report: ## ๐Ÿ“Š Generate EF Tests Report - cd ../../../ && \ - cargo test -p ef_tests-levm --test ef_tests_levm --release -- --summary - -clean-evm-ef-tests: ## ๐Ÿ—‘๏ธ Clean test vectors - rm -rf $(VECTORS_DIR) - ###### Benchmarks ###### revm-comparison: ## ๐Ÿ“Š Run benchmarks of fibonacci and factorial for both REVM and LEVM