diff --git a/crates/pathfinder/src/config/integration_testing.rs b/crates/pathfinder/src/config/integration_testing.rs index 713c583cd7..6bbcccb6b1 100644 --- a/crates/pathfinder/src/config/integration_testing.rs +++ b/crates/pathfinder/src/config/integration_testing.rs @@ -100,6 +100,78 @@ mod enabled { pub trigger: super::InjectFailureTrigger, } + impl InjectFailureConfig { + pub fn proposal_init_rx(height: u64) -> Self { + Self { + height, + trigger: super::InjectFailureTrigger::ProposalInitRx, + } + } + + pub fn proposal_fin_rx(height: u64) -> Self { + Self { + height, + trigger: super::InjectFailureTrigger::ProposalFinRx, + } + } + + pub fn transaction_batch_rx(height: u64) -> Self { + Self { + height, + trigger: super::InjectFailureTrigger::TransactionBatchRx, + } + } + + pub fn proposal_finalized(height: u64) -> Self { + Self { + height, + trigger: super::InjectFailureTrigger::ProposalFinalized, + } + } + + pub fn prevote_rx(height: u64) -> Self { + Self { + height, + trigger: super::InjectFailureTrigger::PrevoteRx, + } + } + + pub fn precommit_rx(height: u64) -> Self { + Self { + height, + trigger: super::InjectFailureTrigger::PrecommitRx, + } + } + + pub fn proposal_decided(height: u64) -> Self { + Self { + height, + trigger: super::InjectFailureTrigger::ProposalDecided, + } + } + + pub fn proposal_committed(height: u64) -> Self { + Self { + height, + trigger: super::InjectFailureTrigger::ProposalCommitted, + } + } + + pub fn outdated_vote(height: u64) -> Self { + Self { + height, + trigger: super::InjectFailureTrigger::OutdatedVote, + } + } + + pub fn committed_vote_lost(height: u64) -> Self { + Self { + height, + trigger: super::InjectFailureTrigger::CommittedVoteLost, + } + } + } + fn parse_inject_failure(s: &str) -> Result { let mut items = s.split(','); let height: u64 = items diff --git a/crates/pathfinder/tests/consensus.rs b/crates/pathfinder/tests/consensus.rs index eede7e57f0..b7806c8c32 100644 --- a/crates/pathfinder/tests/consensus.rs +++ b/crates/pathfinder/tests/consensus.rs @@ -41,6 +41,38 @@ mod test { }; use crate::common::utils; + /// The different scenarios we want to test in the + /// `consensus_3_nodes_with_failures` test case. + #[derive(Copy, Clone)] + enum ConsensusScenario { + /// Start from clear genesis, without a bootstrapped devnet DB. All + /// transactions get reverted since they're random, invalid L1 + /// handlers, but we get to test starting completely from + /// scratch without relying on any pre-initialized database state. + Genesis, + /// Start with a bootstrapped devnet DB, all transactions are valid and + /// should get accepted, testing the happy path with a + /// bootstrapped DB. + BootstrapSuccess, + /// Start with a bootstrapped devnet DB but inject failures at different + /// stages of the consensus flow, testing the network's ability + /// to recover from them and make progress despite them. + BootstrapFailure(InjectFailureConfig), + } + + impl ConsensusScenario { + fn is_bootstrap(&self) -> bool { + matches!( + self, + ConsensusScenario::BootstrapSuccess | ConsensusScenario::BootstrapFailure(_) + ) + } + + fn is_failure(&self) -> bool { + matches!(self, ConsensusScenario::BootstrapFailure(_)) + } + } + // TODO Test cases that should be supported by the integration tests: // - proposals: // - [x] non-empty proposals (transactions that modify storage): @@ -61,34 +93,43 @@ mod test { // different stages), // - [ ] ??? any missing significant failure injection points ???. #[rstest] - // No bootstrap DB, all txns get reverted, testing the flow from clear genesis - #[case::happy_path(None)] - // Bootstrap DB, none of the transactions should get reverted - #[case::fail_on_proposal_init_rx(Some(InjectFailureConfig { height: 4, trigger: InjectFailureTrigger::ProposalInitRx }))] - #[case::fail_on_transaction_batch_rx(Some(InjectFailureConfig { height: 4, trigger: InjectFailureTrigger::TransactionBatchRx }))] - #[case::fail_on_proposal_fin_rx(Some(InjectFailureConfig { height: 4, trigger: InjectFailureTrigger::ProposalFinRx }))] - #[case::fail_on_proposal_finalized(Some(InjectFailureConfig { height: 4, trigger: InjectFailureTrigger::ProposalFinalized }))] - #[case::fail_on_prevote_rx(Some(InjectFailureConfig { height: 4, trigger: InjectFailureTrigger::PrevoteRx }))] - #[case::fail_on_precommit_rx(Some(InjectFailureConfig { height: 4, trigger: InjectFailureTrigger::PrecommitRx }))] - #[case::fail_on_proposal_decided(Some(InjectFailureConfig { height: 4, trigger: InjectFailureTrigger::ProposalDecided }))] - #[case::fail_on_proposal_committed(Some(InjectFailureConfig { height: 4, trigger: InjectFailureTrigger::ProposalCommitted }))] + #[case::happy_path_from_genesis_reverted_txns(ConsensusScenario::Genesis)] + #[case::happy_path_from_bootstrapped(ConsensusScenario::BootstrapSuccess)] + #[case::fail_on_proposal_init_rx(ConsensusScenario::BootstrapFailure( + InjectFailureConfig::proposal_init_rx(4) + ))] + #[case::fail_on_transaction_batch_rx(ConsensusScenario::BootstrapFailure( + InjectFailureConfig::transaction_batch_rx(4) + ))] + #[case::fail_on_proposal_fin_rx(ConsensusScenario::BootstrapFailure( + InjectFailureConfig::proposal_fin_rx(4) + ))] + #[case::fail_on_proposal_finalized(ConsensusScenario::BootstrapFailure( + InjectFailureConfig::proposal_finalized(4) + ))] + #[case::fail_on_prevote_rx(ConsensusScenario::BootstrapFailure( + InjectFailureConfig::prevote_rx(4) + ))] + #[case::fail_on_precommit_rx(ConsensusScenario::BootstrapFailure( + InjectFailureConfig::precommit_rx(4) + ))] + #[case::fail_on_proposal_decided(ConsensusScenario::BootstrapFailure( + InjectFailureConfig::proposal_decided(4) + ))] + #[case::fail_on_proposal_committed(ConsensusScenario::BootstrapFailure( + InjectFailureConfig::proposal_committed(4) + ))] #[tokio::test] - async fn consensus_3_nodes_with_failures(#[case] inject_failure: Option) { + async fn consensus_3_nodes_with_failures(#[case] scenario: ConsensusScenario) { const NUM_NODES: usize = 3; const READY_TIMEOUT: Duration = Duration::from_secs(20); const TEST_TIMEOUT: Duration = Duration::from_secs(120); const POLL_READY: Duration = Duration::from_millis(500); const POLL_HEIGHT: Duration = Duration::from_secs(1); - // IMPORTANT: - // Happy path is the only scenario which starts consensus from genesis at the - // expense of all transactions being reverted since they're random, invalid L1 - // handlers. We need this to be able to test starting completely from scratch - // without relying on any pre-initialized database state. - let disallow_reverted_txns = inject_failure.is_some(); - - let (configs, boot_height, stopwatch) = - utils::setup(NUM_NODES, disallow_reverted_txns).unwrap(); + let bootstrap_db = scenario.is_bootstrap(); + let disallow_reverted_txns = bootstrap_db; + let (configs, boot_height, stopwatch) = utils::setup(NUM_NODES, bootstrap_db).unwrap(); // System contracts start to matter after block 10 but we have a separate // regression test for that, which checks that rollback at H>10 works correctly. @@ -123,7 +164,15 @@ mod test { let boot_port = alice.consensus_p2p_port(); let mut configs = configs.map(|cfg| cfg.with_boot_port(boot_port)); - let bob_cfg = configs.next().unwrap().with_inject_failure(inject_failure); + let bob_cfg = { + let bob_cfg = configs.next().unwrap(); + match scenario { + ConsensusScenario::BootstrapFailure(inject_failure) => { + bob_cfg.with_inject_failure(Some(inject_failure)) + } + _ => bob_cfg, + } + }; let bob = PathfinderInstance::spawn(bob_cfg.clone()).unwrap(); let charlie = PathfinderInstance::spawn(configs.next().unwrap()).unwrap(); @@ -174,7 +223,7 @@ mod test { ); let maybe_bob = respawn_on_fail( - inject_failure.is_some(), + scenario.is_failure(), bob, bob_cfg, POLL_READY,