diff --git a/librustzcash/zcash_client_backend/src/proto/service.rs b/librustzcash/zcash_client_backend/src/proto/service.rs index 1d26a85c..53866170 100644 --- a/librustzcash/zcash_client_backend/src/proto/service.rs +++ b/librustzcash/zcash_client_backend/src/proto/service.rs @@ -58,9 +58,9 @@ pub struct RawTransaction { /// /// * height 0: the transaction is in the mempool /// * height 0xffffffffffffffff: the transaction has been mined on a fork that - /// is not currently the main chain + /// is not currently the main chain /// * any other height: the transaction has been mined in the main chain at the - /// given height + /// given height #[prost(uint64, tag = "2")] pub height: u64, } @@ -334,17 +334,6 @@ pub mod compact_tx_streamer_client { pub struct CompactTxStreamerClient { inner: tonic::client::Grpc, } - impl CompactTxStreamerClient { - /// Attempt to create a new client by connecting to a given endpoint. - pub async fn connect(dst: D) -> Result - where - D: TryInto, - D::Error: Into, - { - let conn = tonic::transport::Endpoint::new(dst)?.connect().await?; - Ok(Self::new(conn)) - } - } impl CompactTxStreamerClient where T: tonic::client::GrpcService, diff --git a/librustzcash/zcash_primitives/src/bft.rs b/librustzcash/zcash_primitives/src/bft.rs index 76d24069..8ef2a9cb 100644 --- a/librustzcash/zcash_primitives/src/bft.rs +++ b/librustzcash/zcash_primitives/src/bft.rs @@ -473,6 +473,10 @@ impl BftBlockAndFatPointerToIt { /// Generate a bundle of the block hash and signatures affirming its validity pub fn from_parts(block: BftBlock, height: u64, round: u32, signatures: &[FatPointerSignature]) -> Self { + assert!( + round <= 0x7fff_ffff, + "BFT round must fit the canonical 31-bit vote domain" + ); let hash = block.blake3_hash(); Self { block, @@ -685,18 +689,18 @@ impl FatPointerToBftBlock { buf } - #[allow(clippy::reversed_empty_ranges)] pub fn try_from_bytes(bytes: &[u8]) -> Option { if bytes.len() < 76 - 32 + 2 { return None; } let vote_for_block_without_finalizer_public_key = bytes[0..76 - 32].try_into().unwrap(); - let len = u16::from_le_bytes(bytes[76 - 32..2].try_into().unwrap()) as usize; + let len = u16::from_le_bytes(bytes[76 - 32..76 - 32 + 2].try_into().unwrap()) as usize; - if 76 - 32 + 2 + len * (32 + 64) > bytes.len() { + let expected_len = (76_usize - 32 + 2).checked_add(len.checked_mul(32 + 64)?)?; + if expected_len != bytes.len() { return None; } - let rem = &bytes[76 - 32 + 2..]; + let rem = &bytes[76 - 32 + 2..expected_len]; let signatures = rem .chunks_exact(32 + 64) .map(|chunk| FatPointerSignature::from_bytes(chunk.try_into().unwrap())) @@ -709,6 +713,10 @@ impl FatPointerToBftBlock { } pub fn from_parts(bft_block_hash: Blake3Hash, height: u64, round: u32, signatures: &[FatPointerSignature]) -> Self { + assert!( + round <= 0x7fff_ffff, + "BFT round must fit the canonical 31-bit vote domain" + ); let mut vote_for_block_without_finalizer_public_key = [0_u8; 76 - 32]; // 76-32 = 44 vote_for_block_without_finalizer_public_key[..32].copy_from_slice(&bft_block_hash.0); vote_for_block_without_finalizer_public_key[32..40].copy_from_slice(&height.to_le_bytes()); @@ -829,6 +837,10 @@ A signed vote will be this same layout followed by the 64 byte ed25519 signature impl Vote { pub fn to_bytes(&self) -> [u8; 76] { + assert!( + self.round >= 0, + "BFT vote round must be non-negative and canonical" + ); let mut buf = [0_u8; 76]; buf[0..32].copy_from_slice(self.validator_address.0.as_ref()); buf[32..64].copy_from_slice(&self.value.0); @@ -906,5 +918,3 @@ pub struct ScanInfo { pub max_height_seen: u32, pub total_value: u64, } - - diff --git a/tenderlink/Cargo.lock b/tenderlink/Cargo.lock index 890c220f..993bacad 100644 --- a/tenderlink/Cargo.lock +++ b/tenderlink/Cargo.lock @@ -260,6 +260,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + [[package]] name = "chacha20" version = "0.9.1" @@ -941,6 +947,18 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "nix" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "nonempty" version = "0.11.0" @@ -1614,6 +1632,7 @@ dependencies = [ "ed25519-zebra", "hex", "libc", + "nix", "rand 0.9.2", "rand_chacha 0.9.0", "rand_pcg", diff --git a/tenderlink/Cargo.toml b/tenderlink/Cargo.toml index 2d1ad9c4..8e9ac31c 100644 --- a/tenderlink/Cargo.toml +++ b/tenderlink/Cargo.toml @@ -3,6 +3,15 @@ name = "tenderlink" version = "0.1.0" edition = "2024" +[[bin]] +name = "tenderlink" +path = "src/main.rs" +required-features = ["simulation"] + +[features] +default = [] +simulation = [] + [profile.dev] opt-level = 0 # no optimizations debug = 2 # full debug info (0 = none, 1 = line tables only, 2 = full) @@ -36,6 +45,9 @@ zcash_primitives = { path = "../librustzcash/zcash_primitives" } # version = "2. #smolmix = { git = "https://github.com/ShieldedLabs/nym.git", rev = "2023a668c5ba0127d2751aad7a124e33ec47d081" } chrono = { version = "0.4.40", default-features = false, features = ["clock"] } +[target.'cfg(unix)'.dependencies] +nix = { version = "0.29.0", features = ["fs", "user"] } + [dependencies.snow] git = "https://github.com/ShieldedLabs/snow.git" rev = "54369eb3829d6563ef7e5d5f17c09b08a77bca6e" diff --git a/tenderlink/src/bandwidth_test.rs b/tenderlink/src/bandwidth_test.rs index 14f316fb..96889ef2 100644 --- a/tenderlink/src/bandwidth_test.rs +++ b/tenderlink/src/bandwidth_test.rs @@ -504,6 +504,7 @@ fn fill_packet_payload_with_unreliable_fragments(payload: &mut [u8], unreliable_ #[derive(Debug)] pub struct ConnectionTrackingData { pub creation_time_ns: u64, + pub initiated_locally: bool, pub my_ip: Ipv6Addr, // TODO: use an ipv6 type that supports Default! pub my_transport_identity_keypair: IdentityKeyPair, pub two_byte_send_prefix: u16, @@ -803,6 +804,16 @@ pub fn connect_to_endpoint( send_buffer_params: (u32, u32, u32), ) { assert!(my_connect_keypair.magic1 == endpoint.magic1); + let connection_key = ConnectionKey::from(endpoint); + if !connections_map.contains_key(&connection_key) + && !connection_capacity_allows( + connections_map.len(), + connections_map.values().filter(|connection| !connection.initiated_locally).count(), + true, + ) + { + return; + } if my_connect_keypair.magic1 == CONNECT_MAGIC1_PLAIN_TEXT { let magic2_block = build_magic2_client_block(); @@ -814,9 +825,10 @@ pub fn connect_to_endpoint( let my_ip = if endpoint.is_ipv4() { Ipv6Addr::UNSPECIFIED } else { Ipv6Addr::UNSPECIFIED }; connections_map.insert( - ConnectionKey::from(endpoint), + connection_key, ConnectionTrackingData { creation_time_ns: monotonic_clock_ns(), + initiated_locally: true, my_ip, my_transport_identity_keypair: my_connect_keypair.clone(), two_byte_send_prefix: load_u16(&my_connect_keypair.public[0..2]) << 1, @@ -827,7 +839,7 @@ pub fn connect_to_endpoint( jumbo_reassembly: Default::default(), handshake_hash: [0u8; 64], unreliable_send_buffer: UnreliableSendBuffer::new(send_buffer_params.0, send_buffer_params.1, send_buffer_params.2), - unreliable_reassembly: ReassemblySlot::new_ring(1024), + unreliable_reassembly: ReassemblySlot::new_ring(MAX_UNRELIABLE_REASSEMBLY_SLOTS), nym_sock, }, ); @@ -848,9 +860,10 @@ pub fn connect_to_endpoint( let my_ip = if endpoint.is_ipv4() { Ipv6Addr::UNSPECIFIED } else { Ipv6Addr::UNSPECIFIED }; connections_map.insert( - ConnectionKey::from(endpoint), + connection_key, ConnectionTrackingData { creation_time_ns: monotonic_clock_ns(), + initiated_locally: true, my_ip, my_transport_identity_keypair: my_connect_keypair.clone(), two_byte_send_prefix: load_u16(&my_connect_keypair.public[0..2]) << 1, @@ -861,7 +874,7 @@ pub fn connect_to_endpoint( jumbo_reassembly: Default::default(), handshake_hash: [0u8; 64], unreliable_send_buffer: UnreliableSendBuffer::new(send_buffer_params.0, send_buffer_params.1, send_buffer_params.2), - unreliable_reassembly: ReassemblySlot::new_ring(1024), + unreliable_reassembly: ReassemblySlot::new_ring(MAX_UNRELIABLE_REASSEMBLY_SLOTS), nym_sock, }, ); @@ -897,9 +910,18 @@ impl ReassemblySlot { /// returns (success, is_complete) pub fn insert(&mut self, offset: usize, data: &[u8], is_fin: bool) -> (bool, bool) { - - let start = offset as u32; - let end = (offset + data.len()) as u32; + let Some(end_usize) = offset.checked_add(data.len()) else { + return (false, false); + }; + if end_usize > MAX_JUMBOGRAM_LEN { + return (false, false); + } + let Ok(start) = u32::try_from(offset) else { + return (false, false); + }; + let Ok(end) = u32::try_from(end_usize) else { + return (false, false); + }; // If this is the final fragment, derive total_len from offset + data.len() if is_fin { @@ -970,6 +992,136 @@ pub struct JumboReassembly { pub const MAX_REASSEMBLY_SLOTS: usize = 128; // @Todo: convert max slots into max bytes instead -- which is what we really want anyway! pub const MAX_JUMBOGRAM_LEN: usize = 1 << 23; // 8 MB, matches the 23-bit field pub const MAX_JUMBOGRAM_IDS: u32 = 1 << 18; // matches 18-bit field +pub const MAX_UNRELIABLE_REASSEMBLY_BYTES: usize = MAX_JUMBOGRAM_LEN; +pub const MAX_UNRELIABLE_REASSEMBLY_SLOTS: usize = MAX_REASSEMBLY_SLOTS; +pub const MAX_NETWORK_CONNECTIONS: usize = 80; +pub const MAX_INBOUND_NETWORK_CONNECTIONS: usize = 16; +pub const MAX_NETWORK_SEND_BUFFER_BYTES: u32 = 8 * 1024 * 1024; +pub const MAX_RECEIVED_UNRELIABLE_BYTES: usize = 32 * 1024 * 1024; + +fn connection_capacity_allows( + total_connections: usize, + inbound_connections: usize, + initiated_locally: bool, +) -> bool { + total_connections < MAX_NETWORK_CONNECTIONS + && (initiated_locally || inbound_connections < MAX_INBOUND_NETWORK_CONNECTIONS) +} + +fn bounded_send_buffer_params(params: (u32, u32, u32)) -> (u32, u32, u32) { + let max_queue_size_bytes = params.2.min(MAX_NETWORK_SEND_BUFFER_BYTES); + let min_queue_size_bytes = params.1.min(max_queue_size_bytes); + (params.0, min_queue_size_bytes, max_queue_size_bytes) +} + +fn received_unreliable_budget_allows(current_bytes: usize, message_bytes: usize) -> bool { + message_bytes <= MAX_RECEIVED_UNRELIABLE_BYTES.saturating_sub(current_bytes) +} + +fn unreliable_reassembly_budget_allows( + slots: &[ReassemblySlot], + slot_idx: usize, + requested_end: usize, +) -> bool { + if requested_end > MAX_JUMBOGRAM_LEN || slot_idx >= slots.len() { + return false; + } + let target_is_empty = slots[slot_idx].buf.is_empty() && slots[slot_idx].received.is_empty(); + if target_is_empty + && slots + .iter() + .filter(|slot| !slot.buf.is_empty() || !slot.received.is_empty()) + .count() + >= MAX_UNRELIABLE_REASSEMBLY_SLOTS + { + return false; + } + let Some(current_bytes) = slots + .iter() + .try_fold(0usize, |sum, slot| sum.checked_add(slot.buf.len())) + else { + return false; + }; + let growth = requested_end.saturating_sub(slots[slot_idx].buf.len()); + current_bytes + .checked_add(growth) + .is_some_and(|total| total <= MAX_UNRELIABLE_REASSEMBLY_BYTES) +} + +#[cfg(test)] +mod bounded_reassembly_tests { + use super::*; + + #[test] + fn rejects_large_offsets_without_allocating() { + let mut slot = ReassemblySlot::new(); + assert_eq!( + slot.insert(MAX_JUMBOGRAM_LEN + 1, &[1], true), + (false, false) + ); + assert_eq!(slot.insert(usize::MAX, &[1], true), (false, false)); + assert!(slot.buf.is_empty()); + assert!(slot.received.is_empty()); + } + + #[test] + fn enforces_active_slot_and_aggregate_byte_budgets() { + let mut active_slots = ReassemblySlot::new_ring(MAX_UNRELIABLE_REASSEMBLY_SLOTS + 1); + for slot in &mut active_slots[..MAX_UNRELIABLE_REASSEMBLY_SLOTS] { + slot.buf.push(0); + } + assert!(!unreliable_reassembly_budget_allows( + &active_slots, + MAX_UNRELIABLE_REASSEMBLY_SLOTS, + 1, + )); + + let mut byte_slots = ReassemblySlot::new_ring(2); + byte_slots[0].buf.resize(MAX_JUMBOGRAM_LEN, 0); + assert!(!unreliable_reassembly_budget_allows(&byte_slots, 1, 1)); + assert!(unreliable_reassembly_budget_allows( + &byte_slots, + 0, + MAX_JUMBOGRAM_LEN, + )); + } + + #[test] + fn reserves_connection_capacity_and_caps_send_buffers() { + assert!(connection_capacity_allows(0, 0, false)); + assert!(!connection_capacity_allows( + MAX_INBOUND_NETWORK_CONNECTIONS, + MAX_INBOUND_NETWORK_CONNECTIONS, + false, + )); + assert!(connection_capacity_allows( + MAX_INBOUND_NETWORK_CONNECTIONS, + MAX_INBOUND_NETWORK_CONNECTIONS, + true, + )); + assert!(!connection_capacity_allows( + MAX_NETWORK_CONNECTIONS, + 0, + true, + )); + + let bounded = bounded_send_buffer_params(( + 1_000_000, + MAX_NETWORK_SEND_BUFFER_BYTES * 2, + u32::MAX, + )); + assert_eq!(bounded.1, MAX_NETWORK_SEND_BUFFER_BYTES); + assert_eq!(bounded.2, MAX_NETWORK_SEND_BUFFER_BYTES); + assert!(received_unreliable_budget_allows( + MAX_RECEIVED_UNRELIABLE_BYTES - 1, + 1, + )); + assert!(!received_unreliable_budget_allows( + MAX_RECEIVED_UNRELIABLE_BYTES, + 1, + )); + } +} pub const ACK_BUFFER_TIME_NS: u64 = 50_000_000; @@ -1002,6 +1154,7 @@ pub struct NetworkThreadHandle { } pub fn new_network_thread(my_keypairs: Vec, my_port: u16, max_pps: Option, send_buffer_params: (u32, u32, u32)) -> NetworkThreadHandle { + let send_buffer_params = bounded_send_buffer_params(send_buffer_params); // STP setup socket_setup(); @@ -1027,6 +1180,7 @@ pub fn new_network_thread(my_keypairs: Vec, my_port: u16, max_p let mut packet_memory_send = new_packet_memory(); // Outgoing Decrypted let mut received_unreliable_messages: Vec<(ConnectionKey, Vec)> = Vec::new(); + let mut received_unreliable_bytes = 0usize; let mut connections_map = HashMap::::new(); let mut server_nym_sockets: Vec = Vec::new(); @@ -1096,6 +1250,7 @@ pub fn new_network_thread(my_keypairs: Vec, my_port: u16, max_p } }).collect(); std::mem::swap(&mut resp.received_unreliable_messages, &mut received_unreliable_messages); + received_unreliable_bytes = 0; #[allow(unsafe_code)] unsafe { @@ -1183,7 +1338,14 @@ pub fn new_network_thread(my_keypairs: Vec, my_port: u16, max_p if OVERLY_VERBOSE { println!("Did NOT respond to connection {}: Failed condition \"&existing_connection.my_transport_identity_keypair == my_kp && existing_connection.other_transport_identity == client_key\"", connection_key.key_15_bits); } } } - else { + else if connection_capacity_allows( + connections_map.len(), + connections_map + .values() + .filter(|connection| !connection.initiated_locally) + .count(), + false, + ) { let client_key = Vec::from(client_key); store_u48(&mut packet_memory_send[0..6], 0xffff_ffff_0000 | (load_u16(&my_kp.public[0..2]) << 1) as u64); packet_memory_send[6..6+32].copy_from_slice(&my_kp.public[..]); @@ -1195,6 +1357,7 @@ pub fn new_network_thread(my_keypairs: Vec, my_port: u16, max_p connection_key, ConnectionTrackingData { creation_time_ns: monotonic_clock_ns(), + initiated_locally: false, my_ip, my_transport_identity_keypair: my_kp.clone(), two_byte_send_prefix: load_u16(&my_kp.public[0..2]) << 1, @@ -1205,7 +1368,7 @@ pub fn new_network_thread(my_keypairs: Vec, my_port: u16, max_p jumbo_reassembly: Default::default(), handshake_hash: [0u8; 64], unreliable_send_buffer: UnreliableSendBuffer::new(send_buffer_params.0, send_buffer_params.1, send_buffer_params.2), - unreliable_reassembly: ReassemblySlot::new_ring(1024), + unreliable_reassembly: ReassemblySlot::new_ring(MAX_UNRELIABLE_REASSEMBLY_SLOTS), nym_sock: None, }, ); @@ -1260,7 +1423,14 @@ pub fn new_network_thread(my_keypairs: Vec, my_port: u16, max_p if OVERLY_VERBOSE { println!("Did NOT respond to client hello from {}: @Todo explanation: Following expression was false: &existing_connection.my_transport_identity_keypair == my_kp && existing_connection.other_transport_identity == client_key", connection_key.key_15_bits); } } } - else { + else if connection_capacity_allows( + connections_map.len(), + connections_map + .values() + .filter(|connection| !connection.initiated_locally) + .count(), + false, + ) { let client_key = Vec::from(client_key); let mut chosen_magic2_bytes = [0u8; 8]; store_u64(&mut chosen_magic2_bytes, chosen_magic2); @@ -1277,6 +1447,7 @@ pub fn new_network_thread(my_keypairs: Vec, my_port: u16, max_p connection_key, ConnectionTrackingData { creation_time_ns: monotonic_clock_ns(), + initiated_locally: false, my_ip, my_transport_identity_keypair: my_kp.clone(), two_byte_send_prefix: load_u16(&my_kp.public[0..2]) << 1, @@ -1287,7 +1458,7 @@ pub fn new_network_thread(my_keypairs: Vec, my_port: u16, max_p jumbo_reassembly: Default::default(), handshake_hash, unreliable_send_buffer: UnreliableSendBuffer::new(send_buffer_params.0, send_buffer_params.1, send_buffer_params.2), - unreliable_reassembly: ReassemblySlot::new_ring(1024), + unreliable_reassembly: ReassemblySlot::new_ring(MAX_UNRELIABLE_REASSEMBLY_SLOTS), nym_sock: None, }, ); @@ -1605,8 +1776,12 @@ pub fn new_network_thread(my_keypairs: Vec, my_port: u16, max_p let frag_data = &remaining_payload[8..(8+frag_len as usize)]; remaining_payload = &remaining_payload[(8+frag_len as usize)..]; - if frag_offset + frag_len > u32::MAX as u64 { - if OVERLY_VERBOSE { println!("Error, frag_offset + frag_len overflows u32 from {connection_key:?}. Disconnecting..."); } + let Some(fragment_end) = frag_offset.checked_add(frag_len) else { + connections_map.remove(&connection_key); + break 'conn; + }; + if is_reliable || fragment_end > MAX_JUMBOGRAM_LEN as u64 { + if OVERLY_VERBOSE { println!("Error, fragment exceeds the unreliable reassembly domain from {connection_key:?}. Disconnecting..."); } connections_map.remove(&connection_key); break 'conn; } @@ -1614,15 +1789,33 @@ pub fn new_network_thread(my_keypairs: Vec, my_port: u16, max_p //if OVERLY_VERBOSE { println!("Fragment from {:?}: R:{} F:{} ID:{} L:{} O:{}", connection_key, is_reliable, is_fin, package_id, frag_len, frag_offset); } let slot_idx = package_id as usize % existing_connection.unreliable_reassembly.len(); + if !unreliable_reassembly_budget_allows( + &existing_connection.unreliable_reassembly, + slot_idx, + fragment_end as usize, + ) { + connections_map.remove(&connection_key); + break 'conn; + } let (mut success, mut complete) = existing_connection.unreliable_reassembly[slot_idx].insert(frag_offset as usize, frag_data, is_fin); if !success { existing_connection.unreliable_reassembly[slot_idx] = ReassemblySlot::new(); (success, complete) = existing_connection.unreliable_reassembly[slot_idx].insert(frag_offset as usize, frag_data, is_fin); - assert!(success); + if !success { + connections_map.remove(&connection_key); + break 'conn; + } } if complete { let completed = std::mem::replace(&mut existing_connection.unreliable_reassembly[slot_idx], ReassemblySlot::new()); - received_unreliable_messages.push((connection_key, completed.buf)); + if received_unreliable_budget_allows( + received_unreliable_bytes, + completed.buf.len(), + ) { + received_unreliable_bytes = received_unreliable_bytes + .saturating_add(completed.buf.len()); + received_unreliable_messages.push((connection_key, completed.buf)); + } } } diff --git a/tenderlink/src/condition28_tests.rs b/tenderlink/src/condition28_tests.rs new file mode 100644 index 00000000..b904458d --- /dev/null +++ b/tenderlink/src/condition28_tests.rs @@ -0,0 +1,430 @@ +use super::*; + +fn fixture() -> ([u8; 32], HashKeys, Vec) { + let namespace = [7u8; 32]; + let hash_keys = HashKeys::default(); + let signing_keys: Vec = (1u8..=4) + .map(|seed| SigningKey::from([seed; 32])) + .collect(); + + let mut cumulative_stake = 0u64; + let roster: Vec = signing_keys + .iter() + .map(|key| { + cumulative_stake += 1; + SortedRosterMember { + pub_key: PubKeyID(VerificationKeyBytes::from(key).into()), + stake: 1, + cumulative_stake, + } + }) + .collect(); + + let proposal = BlockValue(vec![42u8; 128]); + let proposal_id = proposal.id_from_value(&hash_keys); + let mut referenced = RoundData { + height: 77, + round: 1, + proposal: proposal.clone(), + proposal_valid_round: -1, + proposal_sigs: vec![TMSig([1u8; 64])], + proposal_sigs_n: 1, + proposal_id, + msg_val_sigs: vec![[(ValueId::NIL, TMSig::NIL); 2]; roster.len()], + roster: roster.clone(), + vote_namespace: namespace, + ..RoundData::EMPTY + }; + + for roster_i in 0..3 { + let signed_data = make_vote_sign_datas( + roster[roster_i].pub_key, + false, + referenced.height, + referenced.round, + proposal_id, + )[1]; + referenced.msg_val_sigs[roster_i][0] = ( + proposal_id, + TMSig(sign_with_namespace( + &signing_keys[roster_i], + &signed_data, + &namespace, + )), + ); + } + + let current = RoundData { + height: 77, + round: 2, + proposal, + proposal_valid_round: 1, + proposal_sigs: vec![TMSig([2u8; 64])], + proposal_sigs_n: 1, + proposal_id, + msg_val_sigs: vec![[(ValueId::NIL, TMSig::NIL); 2]; roster.len()], + roster, + vote_namespace: namespace, + ..RoundData::EMPTY + }; + + (namespace, hash_keys, vec![referenced, current]) +} + +#[test] +fn accepts_verified_referenced_round_quorum_when_current_yes_is_zero() { + let (namespace, hash_keys, rounds) = fixture(); + assert_eq!( + verified_referenced_prevote_certificate(&rounds, 1, &namespace, &hash_keys), + Some((1, 3, 3)), + ); + assert_eq!(rounds[1].counts.yes_prevotes, 0); +} + +#[test] +fn rejects_referenced_certificate_outside_canonical_round_domain() { + let (namespace, hash_keys, rounds) = fixture(); + + let mut high_current = rounds.clone(); + high_current[1].round = MAX_CONSENSUS_ROUND + 1; + assert_eq!( + verified_referenced_prevote_certificate(&high_current, 1, &namespace, &hash_keys), + None, + ); + + let mut high_referenced = rounds; + high_referenced[0].round = MAX_CONSENSUS_ROUND + 1; + high_referenced[1].proposal_valid_round = i64::from(MAX_CONSENSUS_ROUND) + 1; + assert_eq!( + verified_referenced_prevote_certificate(&high_referenced, 1, &namespace, &hash_keys), + None, + ); +} + +#[test] +fn rejects_subquorum_forgery_and_cross_domain_evidence() { + let (namespace, hash_keys, rounds) = fixture(); + + let mut subquorum = rounds.clone(); + subquorum[0].msg_val_sigs[2][0] = (ValueId::NIL, TMSig::NIL); + assert_eq!( + verified_referenced_prevote_certificate(&subquorum, 1, &namespace, &hash_keys), + None, + ); + + let mut forged = rounds.clone(); + forged[0].msg_val_sigs[0][0].1 = TMSig([9u8; 64]); + assert_eq!( + verified_referenced_prevote_certificate(&forged, 1, &namespace, &hash_keys), + None, + ); + + let mut wrong_roster = rounds.clone(); + wrong_roster[0].roster[0].stake = 2; + assert_eq!( + verified_referenced_prevote_certificate(&wrong_roster, 1, &namespace, &hash_keys), + None, + ); + + let mut wrong_value = rounds.clone(); + wrong_value[1].proposal.0[0] ^= 1; + assert_eq!( + verified_referenced_prevote_certificate(&wrong_value, 1, &namespace, &hash_keys), + None, + ); + + let mut wrong_namespace = rounds.clone(); + wrong_namespace[0].vote_namespace = [8u8; 32]; + assert_eq!( + verified_referenced_prevote_certificate( + &wrong_namespace, + 1, + &namespace, + &hash_keys, + ), + None, + ); +} + +#[test] +fn accepts_quorum_despite_one_correctly_signed_conflicting_minority_vote() { + let (namespace, hash_keys, mut rounds) = fixture(); + let conflicting = ValueId([99u8; 32]); + let key = SigningKey::from([4u8; 32]); + let signed = make_vote_sign_datas( + rounds[0].roster[3].pub_key, + false, + rounds[0].height, + rounds[0].round, + conflicting, + )[1]; + rounds[0].msg_val_sigs[3][0] = ( + conflicting, + TMSig(sign_with_namespace(&key, &signed, &namespace)), + ); + assert_eq!( + verified_referenced_prevote_certificate(&rounds, 1, &namespace, &hash_keys), + Some((1, 3, 3)), + ); +} + +#[test] +fn referenced_quorum_uses_only_the_active_first_hundred_members() { + let namespace = [31u8; 32]; + let hash_keys = HashKeys::default(); + let signing_keys: Vec = (1u8..=101) + .map(|seed| SigningKey::from([seed; 32])) + .collect(); + let mut cumulative_stake = 0u64; + let roster: Vec = signing_keys.iter().map(|key| { + cumulative_stake += 1; + SortedRosterMember { + pub_key: PubKeyID(VerificationKeyBytes::from(key).into()), + stake: 1, + cumulative_stake, + } + }).collect(); + assert_eq!(active_roster_len(&roster), 100); + + let proposal = BlockValue(vec![32u8; 128]); + let proposal_id = proposal.id_from_value(&hash_keys); + let mut referenced = RoundData { + height: 88, + round: 3, + proposal: proposal.clone(), + proposal_sigs: vec![TMSig([1u8; 64])], + proposal_sigs_n: 1, + proposal_id, + msg_val_sigs: vec![[(ValueId::NIL, TMSig::NIL); 2]; 100], + roster: roster.clone(), + vote_namespace: namespace, + ..RoundData::EMPTY + }; + for roster_i in 0..67 { + let signed = make_vote_sign_datas( + roster[roster_i].pub_key, + false, + referenced.height, + referenced.round, + proposal_id, + )[1]; + referenced.msg_val_sigs[roster_i][0] = ( + proposal_id, + TMSig(sign_with_namespace(&signing_keys[roster_i], &signed, &namespace)), + ); + } + let current = RoundData { + height: 88, + round: 4, + proposal, + proposal_valid_round: 3, + proposal_sigs: vec![TMSig([2u8; 64])], + proposal_sigs_n: 1, + proposal_id, + msg_val_sigs: vec![[(ValueId::NIL, TMSig::NIL); 2]; 100], + roster, + vote_namespace: namespace, + ..RoundData::EMPTY + }; + + assert_eq!( + verified_referenced_prevote_certificate( + &[referenced, current], + 1, + &namespace, + &hash_keys, + ), + Some((3, 67, 67)), + ); +} + +fn state_for_condition_28(mut rounds: Vec, namespace: [u8; 32]) -> (TMState, Vec) { + let original_roster = rounds[1].roster.clone(); + let mut order: Vec = (0..original_roster.len()).collect(); + order.sort_by(|left, right| { + (original_roster[*right].stake, original_roster[*right].pub_key) + .cmp(&(original_roster[*left].stake, original_roster[*left].pub_key)) + }); + let mut cumulative_stake = 0u64; + let canonical_roster: Vec = order + .iter() + .map(|index| { + let mut member = original_roster[*index].clone(); + cumulative_stake += member.stake; + member.cumulative_stake = cumulative_stake; + member + }) + .collect(); + for round in &mut rounds { + let previous_signatures = round.msg_val_sigs.clone(); + round.roster = canonical_roster.clone(); + round.msg_val_sigs = order + .iter() + .map(|index| previous_signatures[*index]) + .collect(); + } + let roster = rounds[1].roster.clone(); + let signing_key = (1u8..=4) + .map(|seed| SigningKey::from([seed; 32])) + .find(|key| PubKeyID(VerificationKeyBytes::from(key).into()) == roster[0].pub_key) + .expect("canonical first roster member must be in the fixture key set"); + let public_key = PubKeyID(VerificationKeyBytes::from(&signing_key).into()); + assert_eq!(public_key, roster[0].pub_key); + let signer = DurableSigner::ephemeral_for_simulation( + signing_key, + SignerEpochBinding { + public_key, + chain_id: [1u8; 32], + height: 77, + parent_commit: [2u8; 32], + vote_namespace: namespace, + consensus_config_hash: [3u8; 32], + roster_hash: canonical_roster_hash(&roster).unwrap(), + roster_index: 0, + active_roster_len: roster.len() as u32, + }, + ); + let mut state = TMState::init( + signer, + public_key, + 3032, + ClosureToProposeNewBlock(Arc::new(|| Box::pin(async { None }))), + ClosureToValidateProposedBlock(Arc::new(|_| { + Box::pin(async { (TMStatus::Pass, TMStatusReason::None) }) + })), + ClosureToPushDecidedBlock(Arc::new(|_, _, _, _| { + Box::pin(async { Err("decision closure is unused in condition-28 test".into()) }) + })), + ClosureToUpdatePeers(Arc::new(|_| Box::pin(async {}))), + ClosureToAllowBftAccess(Arc::new(|_, _| Box::pin(async {}))), + ); + state.height = 77; + state.round = 2; + state.step = TMStep::Propose; + state.vote_namespace = namespace; + state.rounds_data = rounds; + (state, roster) +} + +#[test] +fn condition_28_state_machine_prevotes_reproposal_with_historical_qc_and_current_yv_zero() { + tokio::runtime::Runtime::new().unwrap().block_on(async { + let (namespace, hash_keys, rounds) = fixture(); + assert_eq!(rounds[1].counts.yes_prevotes, 0); + let proposal_id = rounds[1].proposal_id; + let (mut state, mut roster) = state_for_condition_28(rounds, namespace); + state.hash_keys = hash_keys; + + state.bft_update(&mut roster).await; + + assert_eq!(state.step, TMStep::Prevote); + let current = state + .rounds_data + .iter() + .find(|round| (round.height, round.round) == (77, 2)) + .unwrap(); + assert_eq!(current.msg_val_sigs[0][0].0, proposal_id); + assert_ne!(current.msg_val_sigs[0][0].1, TMSig::NIL); + }); +} + +#[test] +fn condition_28_state_machine_does_not_vote_on_forged_historical_qc() { + tokio::runtime::Runtime::new().unwrap().block_on(async { + let (namespace, hash_keys, mut rounds) = fixture(); + rounds[0].msg_val_sigs[0][0].1 = TMSig([0x99; 64]); + let (mut state, mut roster) = state_for_condition_28(rounds, namespace); + state.hash_keys = hash_keys; + + state.bft_update(&mut roster).await; + + assert_eq!(state.step, TMStep::Propose); + let current = state + .rounds_data + .iter() + .find(|round| (round.height, round.round) == (77, 2)) + .unwrap(); + assert_eq!(current.msg_val_sigs[0][0], (ValueId::NIL, TMSig::NIL)); + }); +} + +#[test] +fn condition_28_indeterminate_validation_does_not_suppress_timeout_or_round_advance() { + tokio::runtime::Runtime::new().unwrap().block_on(async { + let (namespace, hash_keys, rounds) = fixture(); + let (mut state, mut roster) = state_for_condition_28(rounds, namespace); + state.hash_keys = hash_keys; + state.validate_closure = ClosureToValidateProposedBlock(Arc::new(|_| { + Box::pin(async { + ( + TMStatus::Indeterminate, + TMStatusReason::NeedsBlock { hash: [9u8; 32] }, + ) + }) + })); + let current = state + .rounds_data + .iter_mut() + .find(|round| (round.height, round.round) == (77, 2)) + .unwrap(); + current.active_timeout = Some(Timeout { + time: Instant::now() - std::time::Duration::from_secs(1), + height: 77, + round: 2, + step: TMStep::Propose, + }); + + state.bft_update(&mut roster).await; + + assert_eq!(state.step, TMStep::Prevote); + let current = state + .rounds_data + .iter() + .find(|round| (round.height, round.round) == (77, 2)) + .unwrap(); + assert_eq!(current.msg_val_sigs[0][0].0, ValueId::NIL); + assert_ne!(current.msg_val_sigs[0][0].1, TMSig::NIL); + + // Supply a NIL-prevote quorum so the node reaches precommit, then prove the + // precommit timeout still advances to the next round while the proposal's + // PoW dependency remains unavailable. + let signing_keys: Vec = (1u8..=4) + .map(|seed| SigningKey::from([seed; 32])) + .collect(); + for roster_i in 0..3 { + let member = roster[roster_i].pub_key; + let key = signing_keys + .iter() + .find(|key| PubKeyID(VerificationKeyBytes::from(*key).into()) == member) + .unwrap(); + let signable = make_vote_sign_datas(member, false, 77, 2, ValueId::NIL)[0]; + state.check_and_incorporate_msg( + 77, + 2, + 0, + ValueId::NIL, + -2, + &roster, + roster_i, + PACKET_TYPE_PREVOTE_SIGNATURES, + &signable, + TMSig(sign_with_namespace(key, &signable, &namespace)), + ); + } + state.bft_update(&mut roster).await; + assert_eq!(state.step, TMStep::Precommit); + let current = state + .rounds_data + .iter_mut() + .find(|round| (round.height, round.round) == (77, 2)) + .unwrap(); + current.active_timeout = Some(Timeout { + time: Instant::now() - std::time::Duration::from_secs(1), + height: 77, + round: 2, + step: TMStep::Precommit, + }); + state.bft_update(&mut roster).await; + assert_eq!(state.round, 3); + }); +} diff --git a/tenderlink/src/gossip_tests.rs b/tenderlink/src/gossip_tests.rs new file mode 100644 index 00000000..b7c782dd --- /dev/null +++ b/tenderlink/src/gossip_tests.rs @@ -0,0 +1,85 @@ +use super::*; + +fn round(height: u64, round: u32, valid_round: i64) -> RoundData { + RoundData { + height, + round, + proposal_valid_round: valid_round, + ..RoundData::EMPTY + } +} + +#[test] +fn gossip_always_includes_current_and_referenced_round_and_is_bounded() { + let rounds = vec![ + round(9, 0, -1), + round(9, 1, -1), + round(9, 2, -1), + round(9, 3, -1), + round(9, 4, 1), + ]; + + for cursor in 0..20 { + let (selected, _) = round_indices_to_gossip(&rounds, 9, 4, cursor); + assert!(selected.contains(&4)); + assert!(selected.contains(&1)); + assert!(selected.len() <= 3); + } +} + +#[test] +fn gossip_rotation_eventually_covers_every_other_round() { + let rounds = vec![ + round(8, 0, -1), + round(9, 0, -1), + round(9, 1, -1), + round(9, 2, -1), + round(9, 3, -1), + round(9, 4, 1), + round(10, 0, -1), + ]; + + let mut cursor = 0; + let mut seen = std::collections::BTreeSet::new(); + for _ in 0..6 { + let (selected, next_cursor) = round_indices_to_gossip(&rounds, 9, 4, cursor); + cursor = next_cursor; + seen.extend(selected); + } + + assert_eq!( + seen, + std::collections::BTreeSet::from([1usize, 2, 3, 4, 5]), + ); +} + +#[test] +fn gossip_missing_current_round_fails_closed() { + let rounds = vec![round(9, 0, -1), round(9, 1, -1)]; + assert_eq!(round_indices_to_gossip(&rounds, 9, 2, 99), (Vec::new(), 0)); +} + +#[test] +fn historical_cache_metadata_lookup_is_checked_base_relative_and_exact() { + let cache = vec![round(40, 7, -1), round(41, 3, -1)]; + assert_eq!(commit_round_cache_entry_at_height(&cache, 40).unwrap().height, 40); + assert_eq!(commit_round_cache_entry_at_height(&cache, 41).unwrap().height, 41); + assert!(commit_round_cache_entry_at_height(&cache, 0).is_none()); + assert!(commit_round_cache_entry_at_height(&cache, 39).is_none()); + assert!(commit_round_cache_entry_at_height(&cache, 42).is_none()); + assert!(commit_round_cache_entry_at_height(&cache, u64::MAX).is_none()); + + let gapped = vec![round(40, 7, -1), round(42, 3, -1)]; + assert!(commit_round_cache_entry_at_height(&gapped, 41).is_none()); + + let mut relayable = round(50, 2, -1); + relayable.proposal = BlockValue(vec![5; 32]); + relayable.proposal_id = relayable.proposal.id_from_value(&HashKeys::default()); + relayable.proposal_sigs = vec![TMSig([5; 64])]; + relayable.proposal_sigs_n = 1; + let mut cache = vec![relayable]; + assert!(cached_commit_round_at_height(&cache, 50).is_some()); + compact_round_proposal_payload(&mut cache[0]); + assert!(commit_round_cache_entry_at_height(&cache, 50).is_some()); + assert!(cached_commit_round_at_height(&cache, 50).is_none()); +} diff --git a/tenderlink/src/lib.rs b/tenderlink/src/lib.rs index 834ed631..1fa3828e 100644 --- a/tenderlink/src/lib.rs +++ b/tenderlink/src/lib.rs @@ -4,6 +4,9 @@ #![allow(clippy::never_loop)] #![allow(clippy::eq_op)] +mod signer_wal; +use signer_wal::*; + const PRINT_PROTOCOL: bool = 1 == 1; const PRINT_PROTOCOL_TAG: bool = 0 == 1; const PRINT_ROSTER: bool = 0 == 1; @@ -102,6 +105,29 @@ fn is_timeout(e: std::io::ErrorKind) -> bool{ e == std::io::ErrorKind::WouldBlock || e == std::io::ErrorKind::TimedOut } +fn attestation_window_is_valid(issued: u64, expiry: u64, now: u64) -> bool { + let Some(lifetime) = expiry.checked_sub(issued) else { + return false; + }; + let Some(minimum_expiry) = now.checked_add(60) else { + return false; + }; + let Some(maximum_issued) = now.checked_add(MAX_ATTESTATION_CLOCK_SKEW_SECONDS) else { + return false; + }; + let Some(maximum_expiry) = now + .checked_add(MAX_ATTESTATION_LIFETIME_SECONDS) + .and_then(|value| value.checked_add(MAX_ATTESTATION_CLOCK_SKEW_SECONDS)) + else { + return false; + }; + lifetime >= 60 + && lifetime <= MAX_ATTESTATION_LIFETIME_SECONDS + && expiry >= minimum_expiry + && issued <= maximum_issued + && expiry <= maximum_expiry +} + #[derive(Default)] pub struct NetworkStats { bytes_sent: usize, @@ -156,14 +182,26 @@ pub struct ClosureToValidateProposedBlock(pub Arc Fn(&'a BlockValue) impl std::fmt::Debug for ClosureToValidateProposedBlock { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str("ClosureToValidateProposedBlock(..)") } } -#[derive(Clone)] -// Returns the roster for the next height together with that height's 32-byte vote-namespacing -// domain separator (the cumulative hash of hardforks in effect; `[0; 32]` when there are none). -pub struct ClosureToPushDecidedBlock(pub Arc)-> core::pin::Pin, [u8; 32])> + Send>> + Send + Sync + 'static>); +#[derive(Clone, Debug)] +pub struct DurableDecisionOutcome { + pub next_roster: Vec, + pub next_vote_namespace: [u8; 32], + /// Exact decided value hash reread from the durably synced committed store. + /// `None` keeps production signing disabled when no durable store exists. + pub durable_parent_commit: Option<[u8; 32]>, +} + +// Returns the durably reread parent commit plus the roster and namespace for the next height. +pub struct ClosureToPushDecidedBlock(pub Arc)-> core::pin::Pin> + Send>> + Send + Sync + 'static>); impl std::fmt::Debug for ClosureToPushDecidedBlock { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str("ClosureToPushDecidedBlock(..)") } } #[derive(Clone)] +pub struct ClosureToLoadCommittedRound(pub Arc core::pin::Pin, String>> + Send>> + Send + Sync + 'static>); +impl std::fmt::Debug for ClosureToLoadCommittedRound { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str("ClosureToLoadCommittedRound(..)") } +} +#[derive(Clone)] pub struct ClosureToUpdatePeers(pub Arc) -> core::pin::Pin + Send>> + Send + Sync + 'static>); impl std::fmt::Debug for ClosureToUpdatePeers { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str("ClosureToUpdatePeers(..)") } @@ -174,13 +212,40 @@ impl std::fmt::Debug for ClosureToAllowBftAccess { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str("ClosureToAllowBftAccess(..)") } } +#[derive(Clone, Debug)] +pub enum SignerStartup { + #[cfg(any(test, feature = "simulation"))] + EphemeralSimulation { + chain_id: [u8; 32], + parent_commit: [u8; 32], + consensus_config_hash: [u8; 32], + }, + ObserverOnly { + reason: String, + chain_id: [u8; 32], + parent_commit: [u8; 32], + consensus_config_hash: [u8; 32], + }, + Durable { + wal_path: std::path::PathBuf, + anchor_path: std::path::PathBuf, + independent_anchor_authorized: bool, + non_genesis_bootstrap_receipt_hash: Option<[u8; 32]>, + chain_id: [u8; 32], + parent_commit: [u8; 32], + consensus_config_hash: [u8; 32], + }, +} + fn round_data_to_fat_pointer(round_data: &RoundData, roster: &[SortedRosterMember]) -> FatPointerToBftBlock { let vote_for_block_without_finalizer_public_key: [u8; 76 - 32]; { let mut sign_data = [0; 76 - 32]; round_data.proposal_id.0.write_to(&mut sign_data[0..32]); round_data.height.write_to(&mut sign_data[32..]); - (round_data.round + 0x8000_0000).write_to(&mut sign_data[40..]); + canonical_vote_round(round_data.round, true) + .expect("decided round must be in the canonical 31-bit domain") + .write_to(&mut sign_data[40..]); vote_for_block_without_finalizer_public_key = sign_data; } @@ -202,6 +267,85 @@ fn round_data_to_fat_pointer(round_data: &RoundData, roster: &[SortedRosterMembe } } +/// Verify a reconstructed decided round against its exact active roster, signatures, +/// namespace, canonical round domain, and weighted n-f quorum. This is the storage/network +/// boundary verifier; it does not authorize signing or advance a signer epoch. +pub fn verify_reconstructed_precommit_quorum( + round_data: &RoundData, + roster: &[SortedRosterMember], +) -> Result<(), String> { + let active_len = active_roster_len(roster); + let Some(first_member) = roster.first().filter(|_| active_len > 0) else { + return Err("precommit roster is empty".into()); + }; + let certificate = canonical_precommit_certificate(round_data, roster) + .map_err(|error| error.to_string())?; + let epoch = SignerEpochBinding { + public_key: first_member.pub_key, + chain_id: [0u8; 32], + height: round_data.height, + parent_commit: [0u8; 32], + vote_namespace: round_data.vote_namespace, + consensus_config_hash: [0u8; 32], + roster_hash: canonical_roster_hash(roster).map_err(|error| error.to_string())?, + roster_index: 0, + active_roster_len: active_len + .try_into() + .map_err(|_| "active roster length does not fit u32")?, + }; + verify_precommit_certificate( + &certificate, + round_data.round, + round_data.proposal_id, + &epoch, + roster, + ) + .map_err(|error| error.to_string()) +} + +/// Validate the exact consensus roster representation without binding it to a local signer. +/// Raw public keys are identities: callers must not normalize byte-reversed twins. +pub fn validate_consensus_roster(roster: &[SortedRosterMember]) -> Result<(), String> { + canonical_roster_hash(roster) + .map(|_| ()) + .map_err(|error| error.to_string()) +} + +/// Return the canonical hash of the exact raw consensus roster. +pub fn consensus_roster_hash(roster: &[SortedRosterMember]) -> Result<[u8; 32], String> { + canonical_roster_hash(roster).map_err(|error| error.to_string()) +} + +/// Return the exact roster fields bound into a durable signer epoch. +pub fn signer_epoch_roster_binding( + roster: &[SortedRosterMember], + public_key: PubKeyID, +) -> Result<([u8; 32], u32, u32), String> { + let active_len = active_roster_len(roster); + let roster_hash = canonical_roster_hash(roster).map_err(|error| error.to_string())?; + let roster_index = roster[..active_len] + .iter() + .position(|member| member.pub_key == public_key) + .ok_or_else(|| "signing key is absent from the active consensus roster".to_owned())?; + Ok(( + roster_hash, + roster_index + .try_into() + .map_err(|_| "signer roster index does not fit u32")?, + active_len + .try_into() + .map_err(|_| "active roster length does not fit u32")?, + )) +} + +/// Bind configured consensus rules to the exact Tenderlink hash-key suite. +pub fn signer_consensus_config_binding(configured: [u8; 32]) -> [u8; 32] { + let mut hasher = blake3::Hasher::new(); + hasher.update(&configured); + hasher.update(&consensus_hash_keys_fingerprint(&HashKeys::default())); + hasher.finalize().into() +} + #[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum TMStatus { Indeterminate, @@ -279,7 +423,10 @@ impl RoundData { self.proposal_sigs_n > 0 && self.proposal_sigs_n == self.proposal_sigs.len() } - fn flush_for_amnesiac_proposer(&mut self, value_id: ValueId, valid_round: i64, roster: &[SortedRosterMember], proposal_size: u32) { + fn flush_for_amnesiac_proposer(&mut self, value_id: ValueId, valid_round: i64, roster: &[SortedRosterMember], proposal_size: u32) -> bool { + let Some(proposal_chunks_n) = proposal_chunk_count(proposal_size) else { + return false; + }; let roster_n = active_roster_len(roster); *self = RoundData { height: self.height, @@ -294,7 +441,8 @@ impl RoundData { ..RoundData::EMPTY }; self.proposal.0 = vec![0; proposal_size as usize]; - self.proposal_sigs = vec![TMSig::NIL; self.proposal.chunks_n()]; + self.proposal_sigs = vec![TMSig::NIL; proposal_chunks_n]; + true } fn has_enough_info_to_determine_validity(&self) -> bool { self.proposal_is_faulty || self.has_full_proposal() @@ -313,10 +461,273 @@ impl RoundData { } } +fn rosters_match_exact(left: &[SortedRosterMember], right: &[SortedRosterMember]) -> bool { + left.len() == right.len() && + left.iter().zip(right).all(|(left, right)| { + left.pub_key == right.pub_key && + left.stake == right.stake && + left.cumulative_stake == right.cumulative_stake + }) +} + +fn quorum_threshold(total_power: u64) -> u64 { + let max_faulty_power = total_power.saturating_sub(1) / 3; + total_power.saturating_sub(max_faulty_power) +} + +fn verified_referenced_prevote_certificate( + rounds_data: &[RoundData], + current_round_i: usize, + current_namespace: &[u8; 32], + hash_keys: &HashKeys, +) -> Option<(u32, u64, u64)> { + let current = rounds_data.get(current_round_i)?; + let valid_round: u32 = current.proposal_valid_round.try_into().ok()?; + if current.round > MAX_CONSENSUS_ROUND || + valid_round > MAX_CONSENSUS_ROUND || + valid_round >= current.round || + current.proposal_is_faulty || + !current.has_full_proposal() || + current.proposal_id == ValueId::NIL || + current.proposal_id != current.proposal.id_from_value(hash_keys) || + current.vote_namespace != *current_namespace + { + return None; + } + + let referenced_i = rounds_data + .binary_search_by_key(&(current.height, valid_round), |round| (round.height, round.round)) + .ok()?; + let referenced = &rounds_data[referenced_i]; + let current_active_len = active_roster_len(¤t.roster); + let referenced_active_len = active_roster_len(&referenced.roster); + // The referenced proposal body is deliberately allowed to be compacted. + // The current proposer binds the complete current body to `proposal_id`, + // while the verified referenced prevotes bind the same id to `valid_round`. + if referenced.round > MAX_CONSENSUS_ROUND || + referenced.proposal_id != current.proposal_id || + referenced.vote_namespace != *current_namespace || + current_active_len != referenced_active_len || + !rosters_match_exact( + &referenced.roster[..referenced_active_len], + ¤t.roster[..current_active_len], + ) || + referenced.msg_val_sigs.len() != referenced_active_len + { + return None; + } + + let mut total_power = 0u64; + let mut yes_power = 0u64; + for (roster_i, member) in referenced.roster[..referenced_active_len].iter().enumerate() { + total_power = total_power.checked_add(member.stake)?; + if member.cumulative_stake != total_power { + return None; + } + + let (value_id, sig) = referenced.msg_val_sigs[roster_i][0]; + if value_id == ValueId::NIL { + if sig != TMSig::NIL { + let signed_data = + make_vote_sign_datas(member.pub_key, false, referenced.height, valid_round, value_id)[0]; + sig.verify_with_namespace(member.pub_key, &signed_data, current_namespace).ok()?; + } + continue; + } + if sig == TMSig::NIL { + return None; + } + let signed_data = + make_vote_sign_datas(member.pub_key, false, referenced.height, valid_round, value_id)[1]; + sig.verify_with_namespace(member.pub_key, &signed_data, current_namespace).ok()?; + if value_id == referenced.proposal_id { + yes_power = yes_power.checked_add(member.stake)?; + } + } + + if total_power == 0 { + return None; + } + let quorum = quorum_threshold(total_power); + (yes_power >= quorum).then_some((valid_round, yes_power, quorum)) +} + +fn round_indices_to_gossip( + rounds_data: &[RoundData], + height: u64, + current_round: u32, + historical_cursor: usize, +) -> (Vec, usize) { + let Ok(current_i) = rounds_data + .binary_search_by_key(&(height, current_round), |round| (round.height, round.round)) + else { + return (Vec::new(), 0); + }; + + let mut selected = vec![current_i]; + let current = &rounds_data[current_i]; + if let Ok(valid_round) = u32::try_from(current.proposal_valid_round) { + if valid_round < current_round { + if let Ok(valid_i) = rounds_data + .binary_search_by_key(&(height, valid_round), |round| (round.height, round.round)) + { + selected.push(valid_i); + } + } + } + + let historical: Vec = rounds_data + .iter() + .enumerate() + .filter_map(|(round_i, round)| { + (round.height == height && + round_i != current_i && + !selected.contains(&round_i)) + .then_some(round_i) + }) + .collect(); + + if historical.is_empty() { + return (selected, 0); + } + + let cursor = historical_cursor % historical.len(); + selected.push(historical[cursor]); + (selected, (cursor + 1) % historical.len()) +} + +fn commit_round_cache_entry_at_height(cache: &[RoundData], height: u64) -> Option<&RoundData> { + let base_height = cache.first()?.height; + let relative: usize = height.checked_sub(base_height)?.try_into().ok()?; + let round = cache.get(relative)?; + (round.height == height).then_some(round) +} + +fn cached_commit_round_at_height(cache: &[RoundData], height: u64) -> Option<&RoundData> { + commit_round_cache_entry_at_height(cache, height) + .filter(|round| round.has_full_proposal()) +} + +fn commit_round_for_relay<'a>( + cache: &'a [RoundData], + loaded_historical_round: Option<&'a RoundData>, + height: u64, +) -> Option<&'a RoundData> { + cached_commit_round_at_height(cache, height).or_else(|| { + loaded_historical_round.filter(|round| round.height == height) + }) +} + +fn defer_historical_round_retry( + retries: &mut std::collections::VecDeque<(u64, tokio::time::Instant)>, + height: u64, + retry_after: tokio::time::Instant, +) { + retries.retain(|(retry_height, _)| *retry_height != height); + if retries.len() == MAX_RECENT_COMMIT_ROUNDS_IN_MEMORY { + retries.pop_front(); + } + retries.push_back((height, retry_after)); +} + +fn validate_commit_round_cache( + cache: &[RoundData], + expected_height: u64, +) -> Result<(), String> { + let cache_height = u64::try_from(cache.len()) + .map_err(|_| "commit-round cache length does not fit u64".to_string())?; + if cache.len() > MAX_RECENT_COMMIT_ROUNDS_IN_MEMORY || cache_height > expected_height { + return Err(format!( + "commit-round cache length {cache_height} is invalid for BFT height {expected_height}" + )); + } + let base_height = expected_height - cache_height; + for (index, round) in cache.iter().enumerate() { + let expected_round_height = base_height + .checked_add( + u64::try_from(index) + .map_err(|_| "commit-round cache index does not fit u64".to_string())?, + ) + .ok_or("commit-round cache height overflows u64")?; + if round.height != expected_round_height { + return Err(format!( + "commit-round cache entry {index} carries height {} instead of {expected_round_height}", + round.height + )); + } + } + Ok(()) +} + +fn compact_round_proposal_payload(round: &mut RoundData) { + // Replacing the vectors drops their backing allocations. `clear()` alone + // would retain the capacity and preserve the stall-era heap footprint. + round.proposal = BlockValue(Vec::new()); + round.proposal_sigs = Vec::new(); + round.proposal_sigs_n = 0; + round.proposal_checked_validity = (TMStatus::Indeterminate, TMStatusReason::None); +} + +fn compact_recent_commit_payloads(cache: &mut [RoundData]) { + let compact_before = cache + .len() + .saturating_sub(MAX_RECENT_COMMIT_PAYLOADS_IN_MEMORY); + for round in &mut cache[..compact_before] { + compact_round_proposal_payload(round); + } +} + +fn append_recent_commit_round(cache: &mut Vec, round: RoundData) { + cache.push(round); + let overflow = cache + .len() + .saturating_sub(MAX_RECENT_COMMIT_ROUNDS_IN_MEMORY); + if overflow != 0 { + cache.drain(..overflow); + } + compact_recent_commit_payloads(cache); +} + +/// Verify that reconstructed durable proposal context is the exact canonical +/// chunk manifest signed by the deterministic proposer for this height/round. +/// Callers must run this before admitting persisted signatures into gossip. +pub fn verify_reconstructed_proposal_manifest( + hash_keys: &HashKeys, + round_data: &RoundData, +) -> Result<(), String> { + validate_consensus_roster(&round_data.roster)?; + let active_len = active_roster_len(&round_data.roster); + let epoch = SignerEpochBinding { + public_key: PubKeyID::NIL, + chain_id: [0u8; 32], + height: round_data.height, + parent_commit: [0u8; 32], + vote_namespace: round_data.vote_namespace, + consensus_config_hash: [0u8; 32], + roster_hash: canonical_roster_hash(&round_data.roster) + .map_err(|error| error.to_string())?, + roster_index: u32::MAX, + active_roster_len: active_len + .try_into() + .map_err(|_| "active roster length does not fit u32".to_string())?, + }; + verify_proposal_signature_manifest( + hash_keys, + &epoch, + &round_data.roster, + round_data.round, + round_data.proposal_valid_round, + &round_data.proposal, + round_data.proposal_id, + &round_data.proposal_sigs, + ) + .map_err(|error| error.to_string()) +} + enum TMMsgData { Proposal(BlockValue, i64), Prevote(ValueId), - Precommit(ValueId), + Precommit(ValueId, Option), } pub struct TMMsg { height: u64, @@ -427,16 +838,38 @@ impl Timeout { const ROSTER_MAX_N: usize = 100; +const NORMAL_FUTURE_ROUND_WINDOW: u32 = 32; +const RETAIN_PAST_ROUND_WINDOW: u32 = 64; +pub const MAX_RECENT_COMMIT_ROUNDS_IN_MEMORY: usize = 64; +const MAX_RECENT_COMMIT_PAYLOADS_IN_MEMORY: usize = 1; +const MAX_INFLIGHT_PROPOSAL_BYTES: usize = 32 * 1024 * 1024; +const MAX_PROPOSAL_CHUNKS_PER_ROUND_PER_TICK: usize = 2; +const MAX_ROUTED_BFT_KEYS: usize = 256; +const MAX_ENDPOINTS_PER_BFT_KEY: usize = 4; +const MAX_DYNAMIC_ATTESTATIONS: usize = 256; +const MAX_ATTESTATIONS_PER_PACKET: usize = 8; +const MAX_ATTESTATIONS_PER_PEER_PER_MINUTE: usize = 32; +const MAX_ATTESTATION_LIFETIME_SECONDS: u64 = 2 * 24 * 60 * 60; +const MAX_ATTESTATION_CLOCK_SKEW_SECONDS: u64 = 5 * 60; fn active_roster_len(roster: &[SortedRosterMember]) -> usize { usize::min(ROSTER_MAX_N, roster.len()) } fn total_roster_len(roster: &[SortedRosterMember]) -> usize { roster.len() } +#[derive(Clone, Copy, Debug)] +struct FutureRoundVote { + round: u32, + packet_type: u8, + value_id: ValueId, + sig: TMSig, +} + + #[derive(Debug)] pub struct TMState { pub hash_keys: HashKeys, pub my_port: u16, - pub my_signing_key: SigningKey, + pub durable_signer: DurableSigner, pub my_pub_key: PubKeyID, pub round: u32, pub step: TMStep, @@ -456,7 +889,19 @@ pub struct TMState { pub rounds_data: Vec, - pub recent_commit_round_cache: Vec, // for now will hold all completed heights + /// Compact, one-slot-per-validator evidence used to authorize a jump beyond + /// [`NORMAL_FUTURE_ROUND_WINDOW`]. A Byzantine minority can churn its own + /// slots but cannot allocate roster-sized `RoundData` or proposal buffers. + future_round_votes: Vec>, + admitted_far_round: Option, + + /// Bounded recent decided-round window. Historical state is never retained + /// for every height in the consensus process. + pub recent_commit_round_cache: Vec, + + /// Latches an ambiguous decision-application boundary. No retry or height + /// advancement is permitted until restart reconciles the durable journals. + reconciliation_required: bool, propose_closure: ClosureToProposeNewBlock, validate_closure: ClosureToValidateProposedBlock, @@ -467,7 +912,7 @@ pub struct TMState { } impl TMState { fn init( - my_signing_key: SigningKey, my_pub_key: PubKeyID, my_port: u16, + durable_signer: DurableSigner, my_pub_key: PubKeyID, my_port: u16, propose_closure: ClosureToProposeNewBlock, validate_closure: ClosureToValidateProposedBlock, push_block_closure: ClosureToPushDecidedBlock, @@ -477,7 +922,7 @@ impl TMState { Self { hash_keys: HashKeys::default(), my_port, - my_signing_key, + durable_signer, my_pub_key, round: 0, step: TMStep::Propose, @@ -487,7 +932,10 @@ impl TMState { locked_value_round: (None, -1), rounds_data: Vec::new(), + future_round_votes: Vec::new(), + admitted_far_round: None, recent_commit_round_cache: Vec::new(), + reconciliation_required: false, propose_closure, validate_closure, @@ -518,7 +966,7 @@ impl TMState { proposal_id: /*if proposal.0[1] % 5 == 0 { ValueId([6;32]) } else*/ { proposal.id_from_value(&self.hash_keys) }, valid_round, }; - + let mut signable_parts = Vec::with_capacity(proposal.chunks_n()); for chunk_i in 0..proposal.chunks_n() { // NOTE: excluding packet_type // TODO: check this hdr.chunk_i = chunk_i as u32; let mut o = 0; @@ -526,28 +974,58 @@ impl TMState { let (chunk_o, chunk_size) = proposal.chunk_o_size(chunk_i); o += proposal.0[chunk_o..chunk_o + chunk_size].write_to(&mut buf[o..]); - - // NOTE: we *DON'T* want to write it immediately to our proper store because it - // will confuse check_and_incorporate_msg - let sig = TMSig(sign_with_namespace(&self.my_signing_key, &buf[..o], &self.vote_namespace)); + signable_parts.push(buf[..o].to_vec()); + } + let signatures = match self.durable_signer.sign_proposal( + &self.hash_keys, + roster, + round, + valid_round, + hdr.proposal_id, + &proposal.0, + &signable_parts, + ) { + Ok(signatures) => signatures, + Err(error) => { + eprintln!("{ctx_str} {ANSI_RED}SIGNING BLOCKED{ANSI_RST}: {error}"); + return self.step; + } + }; + for (chunk_i, (signed_data, sig)) in signable_parts.iter().zip(signatures).enumerate() { if PRINT_SIGN { println!("{ctx_str} {ANSI_GRY}SIGN{ANSI_RST}: signed proposal with {:?}", sig) }; - - // NOTE: we're faulty if we give our pub key for this if it's not our proposal self.check_and_incorporate_msg( - height, round, chunk_i, hdr.proposal_id, hdr.valid_round, - roster, roster_i, PACKET_TYPE_PROPOSAL_CHUNK, &buf[..o], sig + height, round, chunk_i, hdr.proposal_id, valid_round, + roster, roster_i, PACKET_TYPE_PROPOSAL_CHUNK, signed_data, sig ); } TMStep::Propose } - TMMsgData::Prevote(value_id) | TMMsgData::Precommit(value_id) => { - let is_precommit: u8 = if let TMMsgData::Precommit(..) = msg { 1 } else { 0 }; + vote_msg @ (TMMsgData::Prevote(..) | TMMsgData::Precommit(..)) => { + let (is_precommit, value_id, transition) = match vote_msg { + TMMsgData::Prevote(value_id) => (0u8, value_id, None), + TMMsgData::Precommit(value_id, transition) => (1u8, value_id, transition), + TMMsgData::Proposal(..) => unreachable!(), + }; if PRINT_BFT_VOTE { println!("{ctx_str} {ANSI_GRY}BFT_VOTE{ANSI_RST}: {} on {}", ["prevoting", "precommitting"][is_precommit as usize], value_id); } let packet_type = PACKET_TYPE_PREVOTE_SIGNATURES + is_precommit; let signed_data = make_vote_sign_datas(roster[roster_i].pub_key, is_precommit != 0, height, round, value_id)[1]; - let sig = TMSig(sign_with_namespace(&self.my_signing_key, &signed_data, &self.vote_namespace)); + let sig = match self.durable_signer.sign_vote( + &self.hash_keys, + roster, + round, + is_precommit != 0, + value_id, + &signed_data, + transition, + ) { + Ok(sig) => sig, + Err(error) => { + eprintln!("{ctx_str} {ANSI_RED}SIGNING BLOCKED{ANSI_RST}: {error}"); + return self.step; + } + }; if PRINT_SIGN { println!("{ctx_str} {ANSI_GRY}SIGN{ANSI_RST}: signed {} with {:?}", ["prevote", "precommit"][is_precommit as usize], sig) }; self.check_and_incorporate_msg( @@ -589,6 +1067,95 @@ impl TMState { (Some(roster_i), roster[roster_i].pub_key) } + fn prune_rounds_for_current_height(&mut self) { + let floor = self.round.saturating_sub(RETAIN_PAST_ROUND_WINDOW); + let ceiling = self.round.saturating_add(NORMAL_FUTURE_ROUND_WINDOW); + let mut protected = Vec::new(); + for data in &self.rounds_data { + if data.height == self.height + && data.round >= floor + && data.round <= ceiling + && data.proposal_valid_round >= 0 + { + if let Ok(referenced) = u32::try_from(data.proposal_valid_round) { + protected.push(referenced); + } + } + } + for round in [self.locked_value_round.1, self.valid_value_round.1] { + if let Ok(round) = u32::try_from(round) { + protected.push(round); + } + } + self.rounds_data.retain(|data| { + data.height == self.height + && ((data.round >= floor && data.round <= ceiling) + || protected.contains(&data.round)) + }); + + // Old rounds retain ids, vote signatures, counts, rosters and timeout + // metadata, but not a gap-sized BftBlock body. Only the current, + // protocol-locked and protocol-valid rounds may keep proposal bytes. + let mut payload_protected = vec![self.round]; + for round in [self.locked_value_round.1, self.valid_value_round.1] { + if let Ok(round) = u32::try_from(round) { + if !payload_protected.contains(&round) { + payload_protected.push(round); + } + } + } + for data in &mut self.rounds_data { + if data.height == self.height + && data.round < self.round + && !payload_protected.contains(&data.round) + { + compact_round_proposal_payload(data); + } + } + } + + fn clear_proposal_storage(data: &mut RoundData) { + compact_round_proposal_payload(data); + data.proposal_valid_round = -1; + data.proposal_is_faulty = false; + } + + fn reserve_proposal_storage(&mut self, target_i: usize, requested: usize) -> bool { + let existing = self.rounds_data[target_i].proposal.0.len(); + let projected = self + .rounds_data + .iter() + .try_fold(0usize, |total, data| { + total.checked_add(data.proposal.0.len()) + }) + .and_then(|total| total.checked_sub(existing)) + .and_then(|total| total.checked_add(requested)); + if projected.is_some_and(|bytes| bytes <= MAX_INFLIGHT_PROPOSAL_BYTES) { + return true; + } + + // Current-round consensus traffic outranks speculative future proposals. + // Drop only their payload/signature buffers; compact vote evidence stays. + if self.rounds_data[target_i].round == self.round { + for (index, data) in self.rounds_data.iter_mut().enumerate() { + if index != target_i && data.height == self.height && data.round > self.round { + Self::clear_proposal_storage(data); + } + } + let after_eviction = self + .rounds_data + .iter() + .try_fold(0usize, |total, data| { + total.checked_add(data.proposal.0.len()) + }) + .and_then(|total| total.checked_sub(existing)) + .and_then(|total| total.checked_add(requested)); + return after_eviction + .is_some_and(|bytes| bytes <= MAX_INFLIGHT_PROPOSAL_BYTES); + } + false + } + fn insert_round(&mut self, insert_i: usize, round: u32, roster: &[SortedRosterMember]) -> usize { let roster_n = active_roster_len(roster); self.rounds_data.insert(insert_i, RoundData { @@ -604,14 +1171,34 @@ impl TMState { async fn start_round(&mut self, roster: &[SortedRosterMember], now: Instant, round: u32) { self.round = round; + self.prune_rounds_for_current_height(); + if self.admitted_far_round.is_some_and(|admitted| admitted <= round) { + self.admitted_far_round = None; + } + for slot in &mut self.future_round_votes { + if slot.is_some_and(|evidence| evidence.round <= round) { + *slot = None; + } + } // self.active_proposal_value_round = (None, -1); let round_i = match self.rounds_data.binary_search_by_key(&(self.height, round), |el| (el.height, el.round)) { Ok(round_i) => round_i, Err(round_i) => self.insert_round(round_i, round, roster) }; - - if Self::proposer_from_height_round(&self.hash_keys, roster, self.height, round).1 == self.my_pub_key { + // Arm the propose timeout before invoking external proposal construction. A + // slow or temporarily unavailable state service must consume the round's + // bounded proposal budget, not postpone the timeout indefinitely. + self.rounds_data[round_i].active_timeout = Some(Timeout::new( + now, + self.height, + self.round, + TMStep::Propose, + )); + + if self.durable_signer.is_active() && + Self::proposer_from_height_round(&self.hash_keys, roster, self.height, round).1 == self.my_pub_key + { let ctx_str = self.ctx_str(roster); let proposal = if let Some(valid_value) = self.valid_value_round.0.clone() { Some(valid_value) @@ -632,22 +1219,357 @@ impl TMState { } else { self.step = TMStep::Propose; } - self.rounds_data[round_i].active_timeout = Some(Timeout::new(now, self.height, self.round, TMStep::Propose)); + } + + async fn reconcile_pending_commit( + &mut self, + roster: &mut Vec, + ) -> Result { + let Some(recovery) = self + .durable_signer + .pending_commit_recovery(&self.hash_keys, roster) + .map_err(|error| error.to_string())? + else { + return Ok(false); + }; + let digest = recovery.digest; + let recovered_round = recovery.round_data.clone(); + let next_height = self + .height + .checked_add(1) + .ok_or("BFT height overflow during pending commit recovery")?; + if let Err(error) = validate_commit_round_cache( + &self.recent_commit_round_cache, + self.height, + ) + .and_then(|()| { + (recovered_round.height == self.height) + .then_some(()) + .ok_or_else(|| { + format!( + "recovered commit carries height {} while BFT expects {}", + recovered_round.height, self.height + ) + }) + }) { + let reason = format!("pending commit cache reconciliation failed: {error}"); + self.durable_signer + .require_reconciliation(digest, reason.clone()) + .map_err(|latch_error| { + format!("{reason}; could not preserve reconciliation latch: {latch_error}") + })?; + self.reconciliation_required = true; + return Err(reason); + } + let push = self.push_block_closure.0.clone(); + let outcome = match push( + recovery.proposal, + recovery.fat_pointer, + recovery.proposal_valid_round, + recovery.proposal_sigs, + ) + .await { + Ok(outcome) => outcome, + Err(error) => { + let reason = format!("pending commit application failed: {error}"); + self.durable_signer + .require_reconciliation(digest, reason.clone()) + .map_err(|latch_error| { + format!("{reason}; could not preserve reconciliation latch: {latch_error}") + })?; + self.reconciliation_required = true; + return Err(reason); + } + }; + let durable_parent_commit = match outcome.durable_parent_commit { + Some(commit) => commit, + None => { + let reason = + "pending commit recovery was not reread from the durable PoS store".to_string(); + self.durable_signer + .require_reconciliation(digest, reason.clone()) + .map_err(|latch_error| { + format!("{reason}; could not preserve reconciliation latch: {latch_error}") + })?; + self.reconciliation_required = true; + return Err(reason); + } + }; + if let Err(error) = self.durable_signer.complete_commit( + digest, + durable_parent_commit, + outcome.next_vote_namespace, + &outcome.next_roster, + ) { + self.reconciliation_required = true; + return Err(format!("pending commit completion failed: {error}")); + } + append_recent_commit_round(&mut self.recent_commit_round_cache, recovered_round); + *roster = outcome.next_roster; + self.height = next_height; + self.vote_namespace = outcome.next_vote_namespace; + self.future_round_votes.clear(); + self.admitted_far_round = None; + self.locked_value_round = (None, -1); + self.valid_value_round = (None, -1); + Ok(true) + } + + fn restore_durable_signer_state(&mut self, roster: &[SortedRosterMember], now: Instant) -> Result { + if !self.durable_signer.is_active() { return Ok(false); } + if let Some(transition) = self.durable_signer.durable_transition().cloned() { + verify_transition_certificate( + &transition, + self.durable_signer.epoch(), + &self.hash_keys, + roster, + )?; + self.locked_value_round = if transition.locked_round >= 0 { + (Some(BlockValue(transition.locked_value.clone())), transition.locked_round) + } else { + (None, -1) + }; + self.valid_value_round = if transition.valid_round >= 0 { + (Some(BlockValue(transition.valid_value.clone())), transition.valid_round) + } else { + (None, -1) + }; + } + + let intents = self.durable_signer.replay_intents(); + if intents.is_empty() { return Ok(false); } + let mut latest_round = 0u32; + for intent in intents { + let round = match &intent { + SignedIntent::Proposal { round, .. } | SignedIntent::Vote { round, .. } => *round, + }; + latest_round = latest_round.max(round); + let round_i = match self.rounds_data.binary_search_by_key(&(self.height, round), |value| (value.height, value.round)) { + Ok(round_i) => round_i, + Err(round_i) => self.insert_round(round_i, round, roster), + }; + match intent { + SignedIntent::Proposal { valid_round, proposal, .. } => { + self.step = self.broadcast(roster, round_i, TMMsgData::Proposal(BlockValue(proposal), valid_round)); + } + SignedIntent::Vote { kind, value_id, transition, .. } => { + self.step = match kind { + SlotKind::Prevote => self.broadcast(roster, round_i, TMMsgData::Prevote(value_id)), + SlotKind::Precommit => self.broadcast(roster, round_i, TMMsgData::Precommit(value_id, transition)), + SlotKind::Proposal => unreachable!(), + }; + } + } + if !self.durable_signer.is_active() { + return Err(SignerError::Conflict("exact WAL replay diverged and disabled signing".into())); + } + } + self.round = latest_round; + self.prune_rounds_for_current_height(); + let current_i = self.rounds_data.binary_search_by_key(&(self.height, latest_round), |value| (value.height, value.round)) + .map_err(|_| SignerError::Integrity("replayed current round is missing".into()))?; + self.rounds_data[current_i].active_timeout = Some(Timeout::new(now, self.height, latest_round, self.step)); + Ok(true) } fn f_from_n(n: u64) -> u64 { - (n - 1) / 3 + n.saturating_sub(1) / 3 } fn check_and_incorporate_msg(&mut self, height: u64, round: u32, chunk_i: usize, value_id: ValueId, valid_round: i64, roster: &[SortedRosterMember], roster_i: usize, packet_type: u8, signed_data: &[u8], sig: TMSig) -> TMStatus { + self.check_and_incorporate_msg_inner( + height, + round, + chunk_i, + value_id, + valid_round, + roster, + roster_i, + packet_type, + signed_data, + sig, + false, + ) + } + + /// Admit ordinary near-future votes directly. A vote farther ahead is kept + /// only in a compact per-validator slot until f+1 independently signed + /// evidence names the same round. Proposal chunks can never create that + /// certificate, so a lone Byzantine proposer cannot allocate 8 MiB at + /// arbitrarily many rounds. + fn check_and_incorporate_network_vote( + &mut self, + height: u64, + round: u32, + value_id: ValueId, + roster: &[SortedRosterMember], + roster_i: usize, + packet_type: u8, + signed_data: &[u8], + sig: TMSig, + ) -> TMStatus { + if height != self.height + || round > MAX_CONSENSUS_ROUND + || roster_i >= active_roster_len(roster) + || !matches!( + packet_type, + PACKET_TYPE_PREVOTE_SIGNATURES | PACKET_TYPE_PRECOMMIT_SIGNATURES + ) + { + return TMStatus::Fail; + } + let normal_limit = self.round.saturating_add(NORMAL_FUTURE_ROUND_WINDOW); + if round <= normal_limit || self.admitted_far_round == Some(round) { + return self.check_and_incorporate_msg_inner( + height, + round, + 0, + value_id, + -2, + roster, + roster_i, + packet_type, + signed_data, + sig, + self.admitted_far_round == Some(round), + ); + } + + let is_precommit = packet_type == PACKET_TYPE_PRECOMMIT_SIGNATURES; + let expected = make_vote_sign_datas( + roster[roster_i].pub_key, + is_precommit, + height, + round, + value_id, + )[(value_id != ValueId::NIL) as usize]; + if signed_data != expected + || sig + .verify_with_namespace( + roster[roster_i].pub_key, + &expected, + &self.vote_namespace, + ) + .is_err() + { + return TMStatus::Fail; + } + + self.future_round_votes + .resize(active_roster_len(roster), None); + self.future_round_votes[roster_i] = Some(FutureRoundVote { + round, + packet_type, + value_id, + sig, + }); + let evidence_power = self + .future_round_votes + .iter() + .enumerate() + .filter_map(|(index, evidence)| { + evidence + .as_ref() + .filter(|evidence| evidence.round == round) + .map(|_| roster[index].stake) + }) + .try_fold(0u64, |total, stake| total.checked_add(stake)); + let Some(evidence_power) = evidence_power else { + return TMStatus::Fail; + }; + let active_len = active_roster_len(roster); + let total_power = roster + .get(active_len.saturating_sub(1)) + .map_or(0, |member| member.cumulative_stake); + if total_power == 0 || evidence_power < Self::f_from_n(total_power).saturating_add(1) { + return TMStatus::Pass; + } + + if let Some(previous) = self.admitted_far_round { + if previous != round && previous > normal_limit { + self.rounds_data.retain(|data| { + !(data.height == self.height && data.round == previous) + }); + } + } + self.admitted_far_round = Some(round); + let certified_votes: Vec<(usize, FutureRoundVote)> = self + .future_round_votes + .iter_mut() + .enumerate() + .filter_map(|(index, slot)| { + let evidence = slot + .as_ref() + .copied() + .filter(|evidence| evidence.round == round)?; + *slot = None; + Some((index, evidence)) + }) + .collect(); + for (index, evidence) in certified_votes { + let is_precommit = evidence.packet_type == PACKET_TYPE_PRECOMMIT_SIGNATURES; + let signable = make_vote_sign_datas( + roster[index].pub_key, + is_precommit, + height, + round, + evidence.value_id, + )[(evidence.value_id != ValueId::NIL) as usize]; + self.check_and_incorporate_msg_inner( + height, + round, + 0, + evidence.value_id, + -2, + roster, + index, + evidence.packet_type, + &signable, + evidence.sig, + true, + ); + } + TMStatus::Pass + } + + fn check_and_incorporate_msg_inner(&mut self, height: u64, round: u32, chunk_i: usize, value_id: ValueId, valid_round: i64, roster: &[SortedRosterMember], roster_i: usize, packet_type: u8, signed_data: &[u8], sig: TMSig, allow_certified_far_round: bool) -> TMStatus { let ctx_str = self.ctx_str(roster); let pkt_str = format!("{:20} {}.{}.{}", packet_name_from_tag(packet_type), height, round, chunk_i); - if height != self.height { + if height != self.height + || round > MAX_CONSENSUS_ROUND + || (!allow_certified_far_round + && round > self.round.saturating_add(NORMAL_FUTURE_ROUND_WINDOW)) + { // eprintln!("{ctx_str} {ANSI_GRY}BFT{ANSI_RST}: received [{}] when we're at height {}", pkt_str, self.height); return TMStatus::Fail; } + if packet_type == PACKET_TYPE_PROPOSAL_CHUNK { + let Some(hdr) = PacketProposalChunkHeader::read_from(&mut &signed_data[..]) else { + return TMStatus::Fail; + }; + let canonical_valid_round = valid_round == -1 + || (valid_round >= 0 + && valid_round <= i64::from(MAX_CONSENSUS_ROUND) + && valid_round < i64::from(round)); + let Some((_, chunk_size, _)) = proposal_chunk_layout(hdr.proposal_size, hdr.chunk_i) + else { + return TMStatus::Fail; + }; + if hdr.height != height + || hdr.round != round + || hdr.chunk_i as usize != chunk_i + || hdr.proposal_id != value_id + || hdr.valid_round != valid_round + || !canonical_valid_round + || signed_data.len() + != PacketProposalChunkHeader::SERIALIZED_SIZE + chunk_size + { + return TMStatus::Fail; + } + } + // check if in (active) roster if roster_i >= active_roster_len(roster) { eprintln!("{ctx_str} ({}): {ANSI_RED}BFT FAULT{ANSI_RST}: {} is not in the active roster.", pkt_str, roster_i); @@ -685,6 +1607,14 @@ impl TMState { Ok(round_i) => (true, round_i), Err(round_i) => (false, self.insert_round(round_i, round, roster)), }; + if packet_type == PACKET_TYPE_PROPOSAL_CHUNK { + let Some(header) = PacketProposalChunkHeader::read_from(&mut &signed_data[..]) else { + return TMStatus::Fail; + }; + if !self.reserve_proposal_storage(round_i, header.proposal_size as usize) { + return TMStatus::Fail; + } + } let round_data = &mut self.rounds_data[round_i]; // TODO: Keep a dynamic array to solve the "Amnesiac Proposer's Dilemma". @@ -723,7 +1653,9 @@ impl TMState { (round_data.proposal.0.len() != hdr.proposal_size as usize || round_data.proposal_id != value_id || round_data.proposal_valid_round != valid_round) { // Amnesiac Proposer's Dilemma - round_data.flush_for_amnesiac_proposer(value_id, valid_round, roster, hdr.proposal_size); + if !round_data.flush_for_amnesiac_proposer(value_id, valid_round, roster, hdr.proposal_size) { + return TMStatus::Fail; + } eprintln!("{ctx_str} {ANSI_YLW}AMNESIAC PROPOSER{ANSI_RST} at {}.{}.{}: Flushing proposal...", height, round, chunk_i); } else { if round_data.proposal.0.len() != hdr.proposal_size as usize { @@ -745,8 +1677,11 @@ impl TMState { } } } else { - round_data.proposal.0 = vec![0; hdr.proposal_size as usize]; - round_data.proposal_sigs = vec![TMSig::NIL; round_data.proposal.chunks_n()]; + let Some(proposal_chunks_n) = proposal_chunk_count(hdr.proposal_size) else { + return TMStatus::Fail; + }; + round_data.proposal.0 = vec![0; hdr.proposal_size as usize]; + round_data.proposal_sigs = vec![TMSig::NIL; proposal_chunks_n]; } // Preliminary checks now finished (although not infallible from here) ////////////////////////// @@ -907,6 +1842,9 @@ impl TMState { } async fn bft_update(&mut self, roster: &mut Vec) { + if self.reconciliation_required { + return; + } debug_assert!(self.rounds_data.iter().all(|r| r.height >= self.height)); let now = Instant::now(); @@ -916,15 +1854,10 @@ impl TMState { } let total_active_stake = total_active_stake; let f = Self::f_from_n(total_active_stake); - let big_threshold; - let small_threshold; - if f == 0 { - big_threshold = total_active_stake; - small_threshold = total_active_stake; - } else { - big_threshold = 2*f+1; - small_threshold = f+1; - } + // For arbitrary weighted totals, 2f+1 is safe only when total = 3f+1. + // Use n-f so any two quorums intersect in more than f voting power. + let big_threshold = quorum_threshold(total_active_stake); + let small_threshold = if total_active_stake == 0 { 0 } else { f.saturating_add(1) }; let ctx_str = self.ctx_str(roster); // NOTE: binary search to {current height, round 0} to avoid looping through data for unneeded decided heights @@ -937,6 +1870,20 @@ impl TMState { // TODO: don't spam "while" messages repeatedly let is_current_height_and_round = (self.height, self.round) == (self.rounds_data[i].height, self.rounds_data[i].round); + let referenced_prevote_certificate = if on_roster && + is_current_height_and_round && + has_enough_info_to_determine_validity && + self.step == TMStep::Propose + { + verified_referenced_prevote_certificate( + &self.rounds_data, + i, + &self.vote_namespace, + &self.hash_keys, + ) + } else { + None + }; // println!("{:#?}", self); if PRINT_BFT_STATE { println!("{ctx_str} {ANSI_GRY}BFT_STATE{ANSI_RST}: {}={}.{}, {}/{}, {}", ["!","="][is_current_height_and_round as usize], @@ -978,19 +1925,27 @@ impl TMState { if (on_roster && is_current_height_and_round && has_enough_info_to_determine_validity && - big_threshold <= counts.yes_prevotes && self.step == TMStep::Propose && - 0 <= self.rounds_data[i].proposal_valid_round && self.rounds_data[i].proposal_valid_round < self.round as i64) // we have received the proposal value + referenced_prevote_certificate.is_some()) { - if self.rounds_data[i].proposal_is_valid(self.validate_closure.clone()).await == TMStatus::Pass && ( - self.locked_value_round.1 <= self.rounds_data[i].proposal_valid_round || - self.locked_value_round.0 == Some(self.rounds_data[i].proposal.clone())) - { - if PRINT_BFT_CONDITIONS { println!("{ctx_str} {ANSI_GRY}BFT_CONDITIONS{ANSI_RST}: in condition 28-0: received 2f+1 prevotes"); } - self.step = self.broadcast(roster, i, TMMsgData::Prevote(self.rounds_data[i].proposal_id)); - } else { - if PRINT_BFT_CONDITIONS { println!("{ctx_str} {ANSI_GRY}BFT_CONDITIONS{ANSI_RST}: in condition 28-1: received 2f+1 prevotes"); } - self.step = self.broadcast(roster, i, TMMsgData::Prevote(ValueId::NIL)); + let proposal_status = self.rounds_data[i] + .proposal_is_valid(self.validate_closure.clone()) + .await; + // Indeterminate means "do not cast a positive vote yet", not "skip the + // rest of this consensus tick". In particular, the propose timeout below + // must remain reachable so a missing PoW dependency cannot pin this node + // in Propose forever. + if proposal_status != TMStatus::Indeterminate { + if proposal_status == TMStatus::Pass && ( + self.locked_value_round.1 <= self.rounds_data[i].proposal_valid_round || + self.locked_value_round.0 == Some(self.rounds_data[i].proposal.clone())) + { + if PRINT_BFT_CONDITIONS { println!("{ctx_str} {ANSI_GRY}BFT_CONDITIONS{ANSI_RST}: in condition 28-0: received 2f+1 prevotes"); } + self.step = self.broadcast(roster, i, TMMsgData::Prevote(self.rounds_data[i].proposal_id)); + } else { + if PRINT_BFT_CONDITIONS { println!("{ctx_str} {ANSI_GRY}BFT_CONDITIONS{ANSI_RST}: in condition 28-1: received 2f+1 prevotes"); } + self.step = self.broadcast(roster, i, TMMsgData::Prevote(ValueId::NIL)); + } } } @@ -1019,12 +1974,49 @@ impl TMState { (self.step == TMStep::Prevote || self.step == TMStep::Precommit)) // TODO: "for the first time" { if PRINT_BFT_CONDITIONS { println!("{ctx_str} {ANSI_GRY}BFT_CONDITIONS{ANSI_RST}: in condition 36: seen 2f+1 valid prevotes"); } + let proposal = self.rounds_data[i].proposal.clone(); + let proposal_id = self.rounds_data[i].proposal_id; + let certificate = match canonical_prevote_certificate(&self.rounds_data[i], roster) { + Ok(certificate) => certificate, + Err(error) => { + self.durable_signer.fail_closed(format!("failed to encode lock certificate: {error}")); + continue; + } + }; + let (locked_value, locked_round) = if self.step == TMStep::Prevote { + (Some(proposal.clone()), self.round as i64) + } else { + self.locked_value_round.clone() + }; + let locked_value_id = locked_value.as_ref() + .map(|value| value.id_from_value(&self.hash_keys)) + .unwrap_or(ValueId::NIL); + let transition = LockValidTransition { + locked_round, + locked_value_id, + locked_value: locked_value.as_ref().map(|value| value.0.clone()).unwrap_or_default(), + valid_round: self.round as i64, + valid_value_id: proposal_id, + valid_value: proposal.0.clone(), + certificate, + }; + if let Err(error) = verify_transition_certificate( + &transition, + self.durable_signer.epoch(), + &self.hash_keys, + roster, + ) { + self.durable_signer.fail_closed(format!("lock certificate self-check failed: {error}")); + continue; + } + self.valid_value_round = (Some(proposal.clone()), self.round as i64); if self.step == TMStep::Prevote { if PRINT_BFT_CONDITIONS { println!("{ctx_str} {ANSI_GRY}BFT_CONDITIONS{ANSI_RST}: in condition 36-0: seen 2f+1 valid prevotes"); } - self.locked_value_round = (Some(self.rounds_data[i].proposal.clone()), self.round as i64); - self.step = self.broadcast(roster, i, TMMsgData::Precommit(self.rounds_data[i].proposal_id)); + self.locked_value_round = (Some(proposal), self.round as i64); + self.step = self.broadcast(roster, i, TMMsgData::Precommit(proposal_id, Some(transition))); + } else if let Err(error) = self.durable_signer.persist_transition(transition, &self.hash_keys, roster) { + eprintln!("{ctx_str} {ANSI_RED}SIGNING BLOCKED{ANSI_RST}: {error}"); } - self.valid_value_round = (Some(self.rounds_data[i].proposal.clone()), self.round as i64); } // line 44: seen 2f+1 nil prevotes: precommit nil @@ -1036,7 +2028,7 @@ impl TMState { self.step == TMStep::Prevote) { if PRINT_BFT_CONDITIONS { println!("{ctx_str} {ANSI_GRY}BFT_CONDITIONS{ANSI_RST}: in condition 44: seen 2f+1 nil prevotes"); } - self.step = self.broadcast(roster, i, TMMsgData::Precommit(ValueId::NIL)); + self.step = self.broadcast(roster, i, TMMsgData::Precommit(ValueId::NIL, None)); } // line 47: last orders on precommit period @@ -1061,14 +2053,130 @@ impl TMState { self.rounds_data[i].proposal_is_valid(self.validate_closure.clone()).await == TMStatus::Pass) { if PRINT_BFT_CONDITIONS { println!("{ctx_str} {ANSI_GRY}BFT_CONDITIONS{ANSI_RST}: in condition 49: value decided"); } - let (new_roster, new_vote_namespace) = self.push_block_closure.0(self.rounds_data[i].proposal.clone(), round_data_to_fat_pointer(&self.rounds_data[i], roster), self.rounds_data[i].proposal_sigs.clone()).await; - if PRINT_ROSTER { println!("{ctx_str} {ANSI_GRY}ROSTER{ANSI_RST}: new roster: {:?}", new_roster); } - *roster = new_roster; - self.height += 1; + let commit_certificate = match canonical_precommit_certificate(&self.rounds_data[i], roster) { + Ok(certificate) => certificate, + Err(error) => { + self.durable_signer.fail_closed(format!("could not encode decided precommit certificate: {error}")); + continue; + } + }; + // Decision/QC verification is a network rule, not a local-signer rule. + // Off-roster and fail-closed observers must still be able to follow a valid + // decision, while local epoch membership remains mandatory on signing paths. + if let Err(error) = verify_reconstructed_precommit_quorum( + &self.rounds_data[i], + roster, + ) { + self.durable_signer.fail_closed(format!("decided precommit certificate self-check failed: {error}")); + continue; + } + let decided_proposal = self.rounds_data[i].proposal.clone(); + let decided_fat_pointer = round_data_to_fat_pointer(&self.rounds_data[i], roster); + let decided_proposal_sigs = self.rounds_data[i].proposal_sigs.clone(); + let decided_round = self.rounds_data[i].clone(); + let next_height = match self.height.checked_add(1) { + Some(next_height) => next_height, + None => { + eprintln!("{ctx_str} {ANSI_RED}DECISION APPLY BLOCKED{ANSI_RST}: BFT height overflow"); + self.reconciliation_required = true; + return; + } + }; + if let Err(error) = validate_commit_round_cache( + &self.recent_commit_round_cache, + self.height, + ) + .and_then(|()| { + (decided_round.height == self.height) + .then_some(()) + .ok_or_else(|| { + format!( + "decided round carries height {} while BFT expects {}", + decided_round.height, self.height + ) + }) + }) { + eprintln!("{ctx_str} {ANSI_RED}DECISION APPLY BLOCKED{ANSI_RST}: {error}"); + self.reconciliation_required = true; + return; + } + let commit_intent_digest = match self.durable_signer.begin_or_resume_commit( + &self.hash_keys, + self.rounds_data[i].round, + self.rounds_data[i].proposal_id, + &decided_proposal, + self.rounds_data[i].proposal_valid_round, + &decided_proposal_sigs, + &commit_certificate, + roster, + ) { + Ok(digest) => digest, + Err(error) => { + eprintln!("{ctx_str} {ANSI_RED}SIGNING BLOCKED{ANSI_RST}: could not persist or reconcile commit intent: {error}"); + self.reconciliation_required = true; + return; + } + }; + let outcome = match self.push_block_closure.0( + decided_proposal, + decided_fat_pointer, + self.rounds_data[i].proposal_valid_round, + decided_proposal_sigs, + ).await { + Ok(outcome) => outcome, + Err(error) => { + if let Some(digest) = commit_intent_digest { + let reason = format!("decided block application failed: {error}"); + if let Err(latch_error) = self + .durable_signer + .require_reconciliation(digest, reason) + { + eprintln!("{ctx_str} {ANSI_RED}SIGNING BLOCKED{ANSI_RST}: could not preserve reconciliation latch: {latch_error}"); + } + } + eprintln!("{ctx_str} {ANSI_RED}DECISION APPLY BLOCKED{ANSI_RST}: {error}"); + self.reconciliation_required = true; + return; + } + }; + if PRINT_ROSTER { println!("{ctx_str} {ANSI_GRY}ROSTER{ANSI_RST}: new roster: {:?}", outcome.next_roster); } + + if let Some(digest) = commit_intent_digest { + match outcome.durable_parent_commit { + Some(durable_parent_commit) => { + if let Err(error) = self.durable_signer.complete_commit( + digest, + durable_parent_commit, + outcome.next_vote_namespace, + &outcome.next_roster, + ) { + eprintln!("{ctx_str} {ANSI_RED}SIGNING BLOCKED{ANSI_RST}: commit completion failed: {error}"); + self.reconciliation_required = true; + return; + } + } + None => { + let reason = "decided block was not reread from a durable store"; + if let Err(error) = self + .durable_signer + .require_reconciliation(digest, reason) + { + eprintln!("{ctx_str} {ANSI_RED}SIGNING BLOCKED{ANSI_RST}: could not preserve reconciliation latch: {error}"); + } + self.reconciliation_required = true; + return; + } + } + } + + append_recent_commit_round(&mut self.recent_commit_round_cache, decided_round); + *roster = outcome.next_roster; + self.height = next_height; // Vote namespacing: adopt the namespace for the new height (supplied alongside the // new roster by the decided-block closure). - self.vote_namespace = new_vote_namespace; - self.recent_commit_round_cache.push(self.rounds_data[i].clone()); + self.vote_namespace = outcome.next_vote_namespace; + self.future_round_votes.clear(); + self.admitted_far_round = None; self.rounds_data.retain(|r| r.height >= self.height); self.locked_value_round = (None, -1); self.valid_value_round = (None, -1); @@ -1102,7 +2210,7 @@ impl TMState { }, TMStep::Prevote => if self.step == TMStep::Prevote { if PRINT_BFT_TIMEOUTS { println!("{ctx_str} {ANSI_GRY}BFT_TIMEOUTS{ANSI_RST}: hit timeout prevote"); } - self.step = self.broadcast(roster, i, TMMsgData::Precommit(ValueId::NIL)); + self.step = self.broadcast(roster, i, TMMsgData::Precommit(ValueId::NIL, None)); }, TMStep::Precommit => { if PRINT_BFT_TIMEOUTS { println!("{ctx_str} {ANSI_GRY}BFT_TIMEOUTS{ANSI_RST}: hit timeout precommit"); } @@ -1136,8 +2244,16 @@ pub struct Peer { pub latest_status: Option, pub index_counter: u64, // for some peer randomness pub bft_pk: PubKeyID, // for convenience // @Todo: @Remove! @@@!!! Don't replicate state like this; look it up from canonical unique sources. + /// Set only after the connection's handshake hash is verified by this exact + /// raw consensus key. Status, telemetry, catch-up, and relay authorization + /// must never infer identity from a mutable address map. + pub authenticated_bft_pk: Option, pub stp_address: STPAddress, // for convenience // @Todo: @Remove! @@@!!! Don't replicate state like this; look it up from canonical unique sources. pub stp_handshake_hash: [u8; 64], // for convenience // @Todo: @Remove! @@@!!! Don't replicate state like this; look it up from canonical unique sources. + pub historical_round_cursor: usize, + pub proposal_chunk_cursor: usize, + pub attestation_window_started: Option, + pub attestations_in_window: usize, } impl Default for Peer { fn default() -> Self { Self { @@ -1145,8 +2261,13 @@ impl Default for Peer { latest_status: Default::default(), index_counter: Default::default(), bft_pk: Default::default(), + authenticated_bft_pk: None, stp_address: Default::default(), stp_handshake_hash: [0u8; 64], + historical_round_cursor: 0, + proposal_chunk_cursor: 0, + attestation_window_started: None, + attestations_in_window: 0, } } } impl Peer { @@ -1243,18 +2364,59 @@ fn sign_with_namespace(key: &SigningKey, data: &[u8], namespace: &[u8; 32]) -> [ } } +fn canonical_vote_round(round: u32, is_precommit: bool) -> Option { + if round > MAX_CONSENSUS_ROUND { + return None; + } + if is_precommit { + round.checked_add(0x8000_0000) + } else { + Some(round) + } +} + +fn proposal_chunk_count(proposal_size: u32) -> Option { + let proposal_size = proposal_size as usize; + if proposal_size == 0 || proposal_size > MAX_PROPOSAL_BYTES { + return None; + } + proposal_size + .checked_add(PROPOSAL_CHUNK_DATA_SIZE - 1) + .map(|size| size / PROPOSAL_CHUNK_DATA_SIZE) +} + +fn proposal_chunk_layout(proposal_size: u32, chunk_i: u32) -> Option<(usize, usize, usize)> { + let chunks_n = proposal_chunk_count(proposal_size)?; + let chunk_i = chunk_i as usize; + if chunk_i >= chunks_n { + return None; + } + let chunk_o = chunk_i.checked_mul(PROPOSAL_CHUNK_DATA_SIZE)?; + let remaining = (proposal_size as usize).checked_sub(chunk_o)?; + Some(( + chunk_o, + usize::min(PROPOSAL_CHUNK_DATA_SIZE, remaining), + chunks_n, + )) +} + fn make_vote_sign_datas(pub_key: PubKeyID, is_precommit: bool, height: u64, round: u32, value_id: ValueId) -> [[u8; 76]; 2] { let mut sign_data_no = [0; 76]; sign_data_no[0..32].copy_from_slice(&pub_key.0[..]); height.write_to(&mut sign_data_no[64..]); - (round + 0x8000_0000 * (is_precommit as u32)).write_to(&mut sign_data_no[72..]); + canonical_vote_round(round, is_precommit) + .expect("vote round must be in the canonical 31-bit domain") + .write_to(&mut sign_data_no[72..]); let mut sign_data_yes = sign_data_no; value_id.0.write_to(&mut sign_data_yes[32..64]); [sign_data_no, sign_data_yes] } -pub fn gen_mostly_empty_rngs bool>(n: usize, f: F) -> Vec<[usize; 2]> { - let mut rngs: Vec<[usize;2]> = Vec::with_capacity(n); +fn visit_mostly_empty_rngs bool, V: FnMut([usize; 2])>( + n: usize, + f: &F, + mut visit: V, +) { let mut filled_c = 0; // consecutive fills let mut rng = [0, 0]; // TODO(perf): these can be split arbitrarily & merged if we wanted to go wide @@ -1268,7 +2430,7 @@ pub fn gen_mostly_empty_rngs bool>(n: usize, f: F) -> Vec<[usize } else { filled_c += 1; if filled_c > 1 { // 2 in a row - rngs.push(rng); + visit(rng); filled_c = 0; rng[0] = i+1; rng[1] = i+1; @@ -1276,12 +2438,40 @@ pub fn gen_mostly_empty_rngs bool>(n: usize, f: F) -> Vec<[usize } } if rng[0] != rng[1] { - rngs.push(rng); + visit(rng); } +} +pub fn gen_mostly_empty_rngs bool>(n: usize, f: F) -> Vec<[usize; 2]> { + let mut rngs = Vec::new(); + visit_mostly_empty_rngs(n, &f, |rng| rngs.push(rng)); rngs } +fn select_mostly_empty_rng bool>( + n: usize, + f: F, + selector: u64, +) -> Option<[usize; 2]> { + let mut range_count = 0usize; + visit_mostly_empty_rngs(n, &f, |_| range_count += 1); + if range_count == 0 { + return None; + } + + let wanted = (selector % range_count as u64) as usize; + let mut range_index = 0usize; + let mut selected = None; + visit_mostly_empty_rngs(n, &f, |rng| { + if range_index == wanted { + selected = Some(rng); + } + range_index += 1; + }); + selected +} + +#[cfg(any(test, feature = "simulation"))] async fn instance( my_root_private_key: SigningKey, my_stp_keypair: Option, @@ -1307,6 +2497,11 @@ async fn instance( let pub_key = PubKeyID(VerificationKeyBytes::from(&my_root_private_key.clone()).into()); entry_point(my_root_private_key, my_stp_keypair, my_endpoint, roster, finalizer_peer_addresses, maybe_seed, + SignerStartup::EphemeralSimulation { + chain_id: [0u8; 32], + parent_commit: [0u8; 32], + consensus_config_hash: consensus_hash_keys_fingerprint(&HashKeys::default()), + }, ClosureToProposeNewBlock(Arc::new(move || { let block_rng = Arc::clone(&block_rng); Box::pin(async move { @@ -1327,17 +2522,25 @@ async fn instance( // else { (TMStatus::Fail, TMStatusReason::None) } }) })), - ClosureToPushDecidedBlock(Arc::new(move |block, fat_pointer, _tender_proposal_sigs| { + ClosureToPushDecidedBlock(Arc::new(move |block, fat_pointer, _proposal_valid_round, _tender_proposal_sigs| { let decisions = Arc::clone(&decisions); let roster2 = roster2.clone(); Box::pin(async move { + let durable_parent_commit = fat_pointer.points_at_block_hash().0; decisions.lock().unwrap().push((block, fat_pointer)); let mut ret = roster2.clone(); ret.truncate(3 + decisions.lock().unwrap().len() % 2); // Sim/test has no hardforks → nil namespace (backwards-compatible no-op). - (ret, [0u8; 32]) + Ok(DurableDecisionOutcome { + next_roster: ret, + next_vote_namespace: [0u8; 32], + durable_parent_commit: Some(durable_parent_commit), + }) }) })), + ClosureToLoadCommittedRound(Arc::new(move |_height| { + Box::pin(async move { Ok(None) }) + })), ClosureToUpdatePeers(Arc::new(move |_all_peers| { Box::pin(async move { })})), @@ -1398,9 +2601,36 @@ pub struct BftAddressMap { } impl BftAddressMap { pub fn new() -> Self { Self::default() } - pub fn insert(&mut self, key: &PubKeyID, addr: &STPAddress, attestation: Option) { - self.by_key.entry(*key).or_default().insert(addr.clone(), attestation); + pub fn insert(&mut self, key: &PubKeyID, addr: &STPAddress, attestation: Option) -> bool { + if self.by_addr.get(addr).is_some_and(|existing| existing != key) { + return false; + } + if !self.by_key.contains_key(key) && self.by_key.len() >= MAX_ROUTED_BFT_KEYS { + return false; + } + let dynamic_count = self + .by_key + .values() + .flat_map(|routes| routes.values()) + .filter(|entry| entry.is_some()) + .count(); + let routes = self.by_key.entry(*key).or_default(); + if !routes.contains_key(addr) && routes.len() >= MAX_ENDPOINTS_PER_BFT_KEY { + return false; + } + if attestation.is_some() { + // A configured route (`None`) is immutable and cannot be replaced by + // network gossip. Dynamic refreshes remain bounded and key-stable. + if routes.get(addr).is_some_and(Option::is_none) { + return false; + } + if !routes.contains_key(addr) && dynamic_count >= MAX_DYNAMIC_ATTESTATIONS { + return false; + } + } + routes.insert(addr.clone(), attestation); self.by_addr.insert(addr.clone(), *key); + true } pub fn get_key(&self, addr: &STPAddress) -> Option<&PubKeyID> { self.by_addr.get(addr) } pub fn get_addrs(&self, key: &PubKeyID) -> impl Iterator)> { self.by_key.get(key).map(|v| v.iter()).unwrap_or_default() } @@ -1415,9 +2645,11 @@ pub async fn entry_point(my_root_private_key: SigningKey, roster: Vec, finalizer_peer_addresses: Vec, maybe_seed: Option, + signer_startup: SignerStartup, propose_closure: ClosureToProposeNewBlock, validate_closure: ClosureToValidateProposedBlock, push_block_closure: ClosureToPushDecidedBlock, + load_committed_round_closure: ClosureToLoadCommittedRound, peer_cmd_closure: ClosureToUpdatePeers, bft_access_closure: ClosureToAllowBftAccess, ingest_startup_data: Vec, @@ -1445,14 +2677,107 @@ pub async fn entry_point(my_root_private_key: SigningKey, println!("\""); } + let my_pub_key = PubKeyID(my_root_public_bft_key.into()); + let active_len = active_roster_len(&roster); + let roster_index: u32 = roster_i_from_pub_key(&roster[..active_len], my_pub_key) + .map(|index| index.try_into().unwrap()) + .unwrap_or(u32::MAX); + let roster_hash = canonical_roster_hash(&roster) + .map_err(|_| std::io::Error::from(std::io::ErrorKind::InvalidData))?; + let startup_height = match ingest_startup_data.last() { + Some(round) => round.height.checked_add(1).ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + "startup BFT height overflows u64", + ) + })?, + None => 0, + }; + validate_commit_round_cache(&ingest_startup_data, startup_height) + .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?; + let (epoch, durable_signer) = match signer_startup { + #[cfg(any(test, feature = "simulation"))] + SignerStartup::EphemeralSimulation { chain_id, parent_commit, consensus_config_hash } => { + let epoch = SignerEpochBinding { + public_key: my_pub_key, + chain_id, + height: startup_height, + parent_commit, + vote_namespace: initial_vote_namespace, + consensus_config_hash: signer_consensus_config_binding(consensus_config_hash), + roster_hash, + roster_index, + active_roster_len: active_len.try_into().unwrap(), + }; + let signer = DurableSigner::ephemeral_for_simulation(my_root_private_key, epoch.clone()); + (epoch, signer) + } + SignerStartup::ObserverOnly { + reason, + chain_id, + parent_commit, + consensus_config_hash, + } => { + let epoch = SignerEpochBinding { + public_key: my_pub_key, + chain_id, + height: startup_height, + parent_commit, + vote_namespace: initial_vote_namespace, + consensus_config_hash: signer_consensus_config_binding(consensus_config_hash), + roster_hash, + roster_index, + active_roster_len: active_len.try_into().unwrap(), + }; + let signer = DurableSigner::observer_only( + my_root_private_key, + epoch.clone(), + reason, + ); + (epoch, signer) + } + SignerStartup::Durable { + wal_path, + anchor_path, + independent_anchor_authorized, + non_genesis_bootstrap_receipt_hash, + chain_id, + parent_commit, + consensus_config_hash, + } => { + let epoch = SignerEpochBinding { + public_key: my_pub_key, + chain_id, + height: startup_height, + parent_commit, + vote_namespace: initial_vote_namespace, + consensus_config_hash: signer_consensus_config_binding(consensus_config_hash), + roster_hash, + roster_index, + active_roster_len: active_len.try_into().unwrap(), + }; + let signer = DurableSigner::open_or_observer( + my_root_private_key, + DurableSignerConfig { + wal_path, + anchor_path, + independent_anchor_authorized, + non_genesis_bootstrap_receipt_hash, + }, + epoch.clone(), + ); + (epoch, signer) + } + }; + let my_stp_keypair = my_stp_keypair.unwrap_or(new_keypair_from_connect_magic1(CRYPTO_MAGIC).unwrap()); use crate::bandwidth_test::*; use crate::native_sockets::*; let my_port = my_endpoint.map(|e| e.port).unwrap_or(23485); // @Dev: .unwrap_or(0); // @Todo! Get local port after sock creation! @@@ - // small min keeps the send buffer rate-adaptive (clamp(1s * rate, 512KiB, 256MiB)) instead of a flat 256MiB/conn - let network_thread_handle = new_network_thread(vec![my_stp_keypair.clone()], my_port, None, (1_000_000, 512 * 1024, 256 * 1024 * 1024)); + // Keep the one-second rate-adaptive queue, but make its per-connection memory ceiling explicit. + let network_thread_handle = new_network_thread(vec![my_stp_keypair.clone()], my_port, None, (1_000_000, 512 * 1024, 8 * 1024 * 1024)); let mut current_connections = Vec::<(STPAddress, [u8; 64])>::new(); let mut initiate_connections = Vec::::new(); let mut messages_to_send = Vec::new(); @@ -1460,10 +2785,13 @@ pub async fn entry_point(my_root_private_key: SigningKey, let mut peers = HashMap::::new(); let mut bft_address_map = BftAddressMap::new(); - let mut my_address_attestations = Vec::new(); - for FinalizerPeerAddress { bft_pk, address } in &finalizer_peer_addresses { - bft_address_map.insert(bft_pk, address, None); + if !bft_address_map.insert(bft_pk, address, None) { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "configured BFT endpoint map exceeds bounds or contains an address/key collision", + )); + } if address.magic1 != CRYPTO_MAGIC { // @Dev @@ -1478,10 +2806,9 @@ pub async fn entry_point(my_root_private_key: SigningKey, if PRINT_PROTOCOL { println!("socket port={:05}, peers endpoints={:?}", my_port, bft_address_map.by_key); } - // TODO: only convert private to public in 1 location let mut bft_state = TMState::init( - my_root_private_key, - PubKeyID(my_root_public_bft_key.into()), + durable_signer, + my_pub_key, my_port, propose_closure, validate_closure, @@ -1490,11 +2817,32 @@ pub async fn entry_point(my_root_private_key: SigningKey, bft_access_closure, ); // TODO: double-check this is the right key - bft_state.height = ingest_startup_data.len() as u64; + bft_state.height = startup_height; bft_state.vote_namespace = initial_vote_namespace; bft_state.recent_commit_round_cache = ingest_startup_data; - - bft_state.start_round(&roster, Instant::now(), 0).await; + compact_recent_commit_payloads(&mut bft_state.recent_commit_round_cache); + + // A crash can leave the certified commit intent ahead of the PoS store. + // Recover from the exact proposal bytes and QC sealed in the signer WAL; + // do not wait for peers to gossip a historical round that they may no + // longer retain. The signer remains observer-only until the closure has + // durably applied/reread the value and `complete_commit` seals the successor. + bft_state + .reconcile_pending_commit(&mut roster) + .await + .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?; + + let startup_now = Instant::now(); + let restored = match bft_state.restore_durable_signer_state(&roster, startup_now) { + Ok(restored) => restored, + Err(error) => { + bft_state.durable_signer.fail_closed(error.to_string()); + false + } + }; + if !restored { + bft_state.start_round(&roster, startup_now, 0).await; + } const ONE_SECOND: tokio::time::Duration = tokio::time::Duration::from_secs(1); let mut net_stats_window_start = tokio::time::Instant::now(); @@ -1505,22 +2853,29 @@ pub async fn entry_point(my_root_private_key: SigningKey, let mut send_buf1 = [0u8; 2048]; let mut next_tick_time = tokio::time::Instant::now(); + const HISTORICAL_ROUND_RELAY_BURST_TICKS: usize = 8; + const HISTORICAL_ROUND_RETRY_DELAY: tokio::time::Duration = + tokio::time::Duration::from_secs(30); + struct LoadedHistoricalRound { + round: RoundData, + relay_ticks: usize, + } + let mut historical_round_load: + Option<(u64, tokio::task::JoinHandle, String>>)> = None; + let mut loaded_historical_round: Option = None; + let mut last_historical_load_height: Option = None; + let mut historical_round_retries = std::collections::VecDeque::new(); loop { let ctx_str = bft_state.ctx_str(&roster); { - let mut peers = peers.iter().map(|(ck, p)| { - let mut bft_key = PubKeyID::NIL; - for (c, _) in ¤t_connections { - if c.connection_key() == *ck { - if let Some(k) = bft_address_map.get_key(c) { - bft_key = *k; - } - break; - } - } - p.info(true, bft_key) - }).collect::>(); + let peers = peers + .values() + .filter_map(|peer| { + peer.authenticated_bft_pk + .map(|key| peer.info(true, key)) + }) + .collect::>(); bft_state.update_peers_cmd_closure.0(peers).await; } @@ -1570,30 +2925,34 @@ pub async fn entry_point(my_root_private_key: SigningKey, let round_data = &bft_state.rounds_data[current_round_i]; - let proposal_chunk_rngs = gen_mostly_empty_rngs(round_data.proposal_sigs.len(), |i| round_data.proposal_sigs[i] == TMSig::NIL); - if proposal_chunk_rngs.len() > 0 { - let mut random_i = peer_random; - for dst_rng in &mut status.need_proposal_chunk_rngs { - let rng = proposal_chunk_rngs[random_i as usize % proposal_chunk_rngs.len()]; + for (selection_i, dst_rng) in status.need_proposal_chunk_rngs.iter_mut().enumerate() { + let selector = peer_random.wrapping_add( + (selection_i as u64).wrapping_mul(1610612741), + ); + if let Some(rng) = select_mostly_empty_rng( + round_data.proposal_sigs.len(), + |i| round_data.proposal_sigs[i] == TMSig::NIL, + selector, + ) { *dst_rng = [rng[0].try_into().unwrap(), rng[1].try_into().unwrap()]; - random_i = random_i.wrapping_add(1610612741); // large prime - // TODO: "with removal" } } - if PRINT_RNGS { println!("{ctx_str} {ANSI_GRY}RNGS{ANSI_RST}: request proposal chunks {:?} from {:?}", status.need_proposal_chunk_rngs, proposal_chunk_rngs); } + if PRINT_RNGS { println!("{ctx_str} {ANSI_GRY}RNGS{ANSI_RST}: request proposal chunks {:?}", status.need_proposal_chunk_rngs); } for is_precommit in 0..2 { - let vote_rngs = gen_mostly_empty_rngs(active_roster_len(roster), |i| round_data.msg_val_sigs[i][is_precommit].1 == TMSig::NIL); - if vote_rngs.len() > 0 { - let mut random_i = peer_random; - for dst_rng in &mut status.need_vote_rngs[is_precommit] { - let rng = vote_rngs[random_i as usize % vote_rngs.len()]; + for (selection_i, dst_rng) in status.need_vote_rngs[is_precommit].iter_mut().enumerate() { + let selector = peer_random.wrapping_add( + (selection_i as u64).wrapping_mul(1610612741), + ); + if let Some(rng) = select_mostly_empty_rng( + active_roster_len(roster), + |i| round_data.msg_val_sigs[i][is_precommit].1 == TMSig::NIL, + selector, + ) { *dst_rng = [rng[0].try_into().unwrap(), rng[1].try_into().unwrap()]; - random_i = random_i.wrapping_add(1610612741); // large prime - // TODO: "with removal" } } - if PRINT_RNGS { println!("{ctx_str} {ANSI_GRY}RNGS{ANSI_RST}: request {:9} chunks {:?} from {:?}", ["prevote", "precommit"][is_precommit], status.need_vote_rngs[is_precommit], vote_rngs); } + if PRINT_RNGS { println!("{ctx_str} {ANSI_GRY}RNGS{ANSI_RST}: request {:9} chunks {:?}", ["prevote", "precommit"][is_precommit], status.need_vote_rngs[is_precommit]); } } } @@ -1642,6 +3001,64 @@ pub async fn entry_point(my_root_private_key: SigningKey, // account for the state updates we've accumulated bft_state.bft_update(&mut roster).await; + if historical_round_load + .as_ref() + .is_some_and(|(_, task)| task.is_finished()) + { + let (requested_height, task) = historical_round_load + .take() + .expect("finished historical-round task exists"); + match task.await { + Ok(Ok(Some(round))) if round.height == requested_height => { + loaded_historical_round = Some(LoadedHistoricalRound { + round, + relay_ticks: 0, + }); + } + Ok(Ok(Some(round))) => { + eprintln!( + "{ctx_str} {ANSI_RED}BFT ERROR{ANSI_RST}: historical loader returned height {} for request {requested_height}", + round.height, + ); + defer_historical_round_retry( + &mut historical_round_retries, + requested_height, + tokio::time::Instant::now() + HISTORICAL_ROUND_RETRY_DELAY, + ); + } + Ok(Ok(None)) => { + defer_historical_round_retry( + &mut historical_round_retries, + requested_height, + tokio::time::Instant::now() + HISTORICAL_ROUND_RETRY_DELAY, + ); + } + Ok(Err(error)) => { + eprintln!( + "{ctx_str} {ANSI_RED}BFT ERROR{ANSI_RST}: failed to load authenticated historical round {requested_height}: {error}", + ); + defer_historical_round_retry( + &mut historical_round_retries, + requested_height, + tokio::time::Instant::now() + HISTORICAL_ROUND_RETRY_DELAY, + ); + } + Err(error) => { + eprintln!( + "{ctx_str} {ANSI_RED}BFT ERROR{ANSI_RST}: historical round loader task failed for height {requested_height}: {error}", + ); + defer_historical_round_retry( + &mut historical_round_retries, + requested_height, + tokio::time::Instant::now() + HISTORICAL_ROUND_RETRY_DELAY, + ); + } + } + } + let retry_now = tokio::time::Instant::now(); + historical_round_retries + .retain(|(_, retry_after)| retry_now < *retry_after); + fn send_round_data_to_peer(bft_state: &TMState, should_send_prevotes: bool, round_data: &RoundData, @@ -1676,7 +3093,15 @@ pub async fn entry_point(my_root_private_key: SigningKey, let mut sent_c: [usize; 2] = [0; 2]; if round_data.proposal_sigs_n > 0 { - for chunk_i in 0..round_data.proposal_sigs.len() { + let chunks_len = round_data.proposal_sigs.len(); + let start = peer.proposal_chunk_cursor % chunks_len; + let mut scanned = 0usize; + for offset in 0..chunks_len { + if sent_chunk_cs >= MAX_PROPOSAL_CHUNKS_PER_ROUND_PER_TICK { + break; + } + scanned = offset + 1; + let chunk_i = (start + offset) % chunks_len; // send all of the proposal chunks we've seen if round_data.proposal_sigs[chunk_i] != TMSig::NIL { chunk_hdr.chunk_i = chunk_i as u32; @@ -1712,6 +3137,7 @@ pub async fn entry_point(my_root_private_key: SigningKey, send_stp_msg(messages_to_send, connection_key, &send_buf1[..o], stats); } } + peer.proposal_chunk_cursor = (start + scanned) % chunks_len; } let vote_start: u8 = if should_send_prevotes { 0 } else { 1 }; @@ -1834,35 +3260,112 @@ pub async fn entry_point(my_root_private_key: SigningKey, (bft_key, p.latest_status.clone()) }).collect::>()); } + if historical_round_load.is_none() && loaded_historical_round.is_none() { + let mut requested_heights = peers + .values() + .filter_map(|peer| { + let peer_bft_key = peer.authenticated_bft_pk?; + let height = peer.latest_status_request_height?; + (height < bft_state.height + && cached_commit_round_at_height( + &bft_state.recent_commit_round_cache, + height, + ) + .is_none() + && roster_i_from_pub_key( + &roster[..active_roster_len(&roster)], + peer_bft_key, + ) + .is_some() + && !historical_round_retries + .iter() + .any(|(retry_height, _)| *retry_height == height)) + .then_some(height) + }) + .collect::>(); + requested_heights.sort_unstable(); + requested_heights.dedup(); + let selected_height = last_historical_load_height + .and_then(|last_height| { + requested_heights + .iter() + .copied() + .find(|height| *height > last_height) + }) + .or_else(|| requested_heights.first().copied()); + if let Some(height) = selected_height { + last_historical_load_height = Some(height); + let loader = load_committed_round_closure.clone(); + historical_round_load = Some(( + height, + tokio::spawn(async move { (loader.0)(height).await }), + )); + } + } + + let mut relayed_bft_keys = std::collections::HashSet::new(); + let mut relayed_loaded_historical_round = false; for (connection_key, peer) in &mut peers { - let mut peer_bft_key = PubKeyID::NIL; - for (c, _) in ¤t_connections { - if c.connection_key() == *connection_key { - if let Some(k) = bft_address_map.get_key(&c) { - peer_bft_key = *k; - } - break; - } + let Some(peer_bft_key) = peer.authenticated_bft_pk else { + continue; + }; + // Multiple transport connections for one validator share one + // relay allowance; otherwise a single key can multiply the + // node's outbound work by reconnecting repeatedly. + if !relayed_bft_keys.insert(peer_bft_key) { + continue; } if let Some(height) = peer.latest_status_request_height && height < bft_state.height { - peer.latest_status_request_height = None; - send_round_data_to_peer(&bft_state, - false, - &bft_state.recent_commit_round_cache[height as usize], - &ctx_str, - &roster, - &mut messages_to_send, - &mut send_buf1, - peer, - connection_key, - peer_bft_key, - &mut net_stats); + let cached_round = cached_commit_round_at_height( + &bft_state.recent_commit_round_cache, + height, + ); + let loaded_round = loaded_historical_round + .as_ref() + .map(|loaded| &loaded.round); + if let Some(committed_round) = commit_round_for_relay( + &bft_state.recent_commit_round_cache, + loaded_round, + height, + ) { + let requester_is_authorized = roster_i_from_pub_key( + &roster[..active_roster_len(&roster)], + peer_bft_key, + ) + .is_some() + || roster_i_from_pub_key( + &committed_round.roster[..active_roster_len(&committed_round.roster)], + peer_bft_key, + ) + .is_some(); + if requester_is_authorized { + if cached_round.is_none() { + relayed_loaded_historical_round = true; + } + send_round_data_to_peer(&bft_state, + false, + committed_round, + &ctx_str, + &roster, + &mut messages_to_send, + &mut send_buf1, + peer, + connection_key, + peer_bft_key, + &mut net_stats); + } + } } else if roster_i_from_pub_key(&roster[..active_roster_len(&roster)], peer_bft_key).is_some() { - if let Ok(current_height_start_i) = bft_state.rounds_data.binary_search_by_key(&(bft_state.height, 0), |el| (el.height, el.round)) - { - for round_i in (current_height_start_i..bft_state.rounds_data.len()).rev() - { + let (round_indices, next_cursor) = round_indices_to_gossip( + &bft_state.rounds_data, + bft_state.height, + bft_state.round, + peer.historical_round_cursor, + ); + peer.historical_round_cursor = next_cursor; + if !round_indices.is_empty() { + for round_i in round_indices { let round_data = &bft_state.rounds_data[round_i]; send_round_data_to_peer(&bft_state, true, @@ -1881,6 +3384,18 @@ pub async fn entry_point(my_root_private_key: SigningKey, } } } + let clear_loaded_historical_round = if let Some(loaded) = loaded_historical_round.as_mut() { + if relayed_loaded_historical_round { + loaded.relay_ticks = loaded.relay_ticks.saturating_add(1); + } + !relayed_loaded_historical_round + || loaded.relay_ticks >= HISTORICAL_ROUND_RELAY_BURST_TICKS + } else { + false + }; + if clear_loaded_historical_round { + loaded_historical_round = None; + } // Prune attestations that expire in <60s let now: u64 = chrono::Utc::now().timestamp().try_into().expect("should fit in a u64"); @@ -1890,13 +3405,17 @@ pub async fn entry_point(my_root_private_key: SigningKey, return true; // keep forever if None. // @Todo: @Incomplete? }; - if peer_attestation.expiry + 120 <= now { + if peer_attestation.expiry <= now.saturating_sub(120) { return false; // prune } if peer_attestation.issued >= peer_attestation.expiry { return false; // prune } - if peer_attestation.issued + 60 > peer_attestation.expiry { + if peer_attestation + .expiry + .checked_sub(peer_attestation.issued) + .map_or(true, |lifetime| lifetime < 60) + { return false; // prune } @@ -2015,7 +3534,7 @@ pub async fn entry_point(my_root_private_key: SigningKey, let resp = service_connections(&network_thread_handle, NetworkThreadPush { initiate_connections, wanted_connections: current_connections.clone(), send_unreliable: messages_to_send, }); current_connections = resp.current_connections; initiate_connections = Vec::new(); - let mut messages_received = resp.received_unreliable_messages; + let messages_received = resp.received_unreliable_messages; messages_to_send = Vec::new(); // Ensure a Peer entry exists for every active connection @@ -2043,7 +3562,7 @@ pub async fn entry_point(my_root_private_key: SigningKey, assert!(peer.stp_handshake_hash.len() == 64); let keyed_hash_of_stp_handshake_hash = hash_key_for_stp_handshake_hash.hash(&peer.stp_handshake_hash[..]); - TMSig(my_root_private_key.sign(&keyed_hash_of_stp_handshake_hash[..]).to_bytes()) + bft_state.durable_signer.sign_auxiliary_digest(&keyed_hash_of_stp_handshake_hash) }; PacketIdVerification { pk, sig } @@ -2066,14 +3585,16 @@ pub async fn entry_point(my_root_private_key: SigningKey, let mut connection_keys_to_disconnect = Vec::new(); // READ - 'process_packets: while messages_received.len() > 0 { - let (connection_key, mut peer, msg) = { - let (key, packet) = messages_received.remove(0); - let Some(peer) = peers.get_mut(&key) + // Preserve network arrival order while consuming the batch in linear time. + // Vec::remove(0) shifted the remaining packet vector once per packet, + // making a large batch quadratic and delaying the next consensus tick. + 'process_packets: for (connection_key, msg) in messages_received { + let (connection_key, peer, msg) = { + let Some(peer) = peers.get_mut(&connection_key) else { continue; }; - (key, peer, packet) + (connection_key, peer, msg) }; let msg: &[u8] = &msg[..]; @@ -2088,8 +3609,16 @@ pub async fn entry_point(my_root_private_key: SigningKey, if let Some(status) = status { - peer.latest_status_request_height = Some(status.height); - peer.latest_status = Some(status); + if let Some(peer_key) = peer.authenticated_bft_pk + && roster_i_from_pub_key( + &roster[..active_roster_len(&roster)], + peer_key, + ) + .is_some() + { + peer.latest_status_request_height = Some(status.height); + peer.latest_status = Some(status); + } } const_assert!(PACKET_TYPE_PREVOTE_SIGNATURES + 1 == PACKET_TYPE_PRECOMMIT_SIGNATURES); @@ -2098,9 +3627,13 @@ pub async fn entry_point(my_root_private_key: SigningKey, eprintln!("{:05}: couldn't read proposal header", my_port); continue; }; - let proposal_size = hdr.proposal_size as usize; - let chunk_i = hdr.chunk_i as usize; - let chunk_size = usize::min(PROPOSAL_CHUNK_DATA_SIZE, proposal_size - chunk_i * PROPOSAL_CHUNK_DATA_SIZE); + let Some((_, chunk_size, _)) = proposal_chunk_layout(hdr.proposal_size, hdr.chunk_i) + else { + continue; + }; + if hdr.round > MAX_CONSENSUS_ROUND { + continue; + } let packet_size = chunk_size + PROPOSAL_PACKET_EXTRA; // NOTE: assume for the moment that this is the valid height, we'll check in the subsequent call @@ -2118,16 +3651,34 @@ pub async fn entry_point(my_root_private_key: SigningKey, else if packet_type == PACKET_TYPE_PREVOTE_SIGNATURES || packet_type == PACKET_TYPE_PRECOMMIT_SIGNATURES { if let Some(packet) = PacketVotes::read_from(&mut &msg[read_o..]) { + if packet.round > MAX_CONSENSUS_ROUND { + continue; + } let is_precommit = packet_type - PACKET_TYPE_PREVOTE_SIGNATURES; let value_ids = [ ValueId::NIL, packet.value_id ]; - for vote_i in 0..(packet.no_votes_n + packet.yes_votes_n) as usize { + let Some(votes_n) = packet.no_votes_n.checked_add(packet.yes_votes_n) + else { + continue; + }; + if votes_n == 0 || votes_n as usize > packet.votes.len() { + continue; + } + for vote_i in 0..votes_n as usize { // Note(Sam): We can change the format of votes to be cool and branchless after the workshop. if let Some(roster_member) = roster.get(packet.votes[vote_i].roster_i as usize) { let sign_datas = make_vote_sign_datas(roster_member.pub_key, is_precommit != 0, packet.height, packet.round, packet.value_id); let no_yes_i = (vote_i >= packet.no_votes_n as usize) as usize; - bft_state.check_and_incorporate_msg(packet.height, packet.round, 0, value_ids[no_yes_i], -2, - &roster, packet.votes[vote_i].roster_i as usize, packet_type, &sign_datas[no_yes_i], TMSig(packet.votes[vote_i].sig.0)); + bft_state.check_and_incorporate_network_vote( + packet.height, + packet.round, + value_ids[no_yes_i], + &roster, + packet.votes[vote_i].roster_i as usize, + packet_type, + &sign_datas[no_yes_i], + TMSig(packet.votes[vote_i].sig.0), + ); } } } else { @@ -2136,7 +3687,55 @@ pub async fn entry_point(my_root_private_key: SigningKey, } else if packet_type == PACKET_TYPE_PEER_ATTESTATIONS { - let chunks = msg[read_o..].chunks(PEER_ATTESTATION_SERIALIZED_SIZE); + let Some(sender_key) = peer.authenticated_bft_pk else { + connection_keys_to_disconnect.push(connection_key); + continue; + }; + let key_is_routed = |key: PubKeyID| { + roster_i_from_pub_key(&roster[..active_roster_len(&roster)], key).is_some() + || finalizer_peer_addresses + .iter() + .any(|configured| configured.bft_pk == key) + }; + if !key_is_routed(sender_key) { + connection_keys_to_disconnect.push(connection_key); + continue; + } + let payload = &msg[read_o..]; + if payload.is_empty() + || payload.len() % PEER_ATTESTATION_SERIALIZED_SIZE != 0 + || payload.len() / PEER_ATTESTATION_SERIALIZED_SIZE + > MAX_ATTESTATIONS_PER_PACKET + { + connection_keys_to_disconnect.push(connection_key); + continue; + } + let packet_attestations = payload.len() / PEER_ATTESTATION_SERIALIZED_SIZE; + let now_instant = Instant::now(); + if peer.attestation_window_started.is_none() + || peer + .attestation_window_started + .is_some_and(|started| { + now_instant.duration_since(started) + >= std::time::Duration::from_secs(60) + }) + { + peer.attestation_window_started = Some(now_instant); + peer.attestations_in_window = 0; + } + let Some(next_attestation_count) = peer + .attestations_in_window + .checked_add(packet_attestations) + else { + connection_keys_to_disconnect.push(connection_key); + continue; + }; + if next_attestation_count > MAX_ATTESTATIONS_PER_PEER_PER_MINUTE { + connection_keys_to_disconnect.push(connection_key); + continue; + } + peer.attestations_in_window = next_attestation_count; + let chunks = payload.chunks_exact(PEER_ATTESTATION_SERIALIZED_SIZE); for chunk in chunks { let Some(peer_attestation) = PeerAttestation::read_from(&mut &chunk[..]) else { if PRINT_PROTOCOL { println!("{ctx_str} {ANSI_RED}PROTOCOL{ANSI_RST}: Peer sent invalid peer attestation: Failed to read peer attestation"); } @@ -2144,22 +3743,47 @@ pub async fn entry_point(my_root_private_key: SigningKey, continue 'process_packets; }; - // @Todo: prune attestees to only BFT PKs on a current or imminently upcoming roster - // @Todo: prune attesters to only BFT PKs on a current or imminently upcoming roster - - let now: u64 = chrono::Utc::now().timestamp().try_into().expect("should fit in a u64"); - if peer_attestation.expiry + 60 <= now { - if PRINT_PROTOCOL { println!("{ctx_str} {ANSI_RED}PROTOCOL{ANSI_RST}: Peer sent peer attestation that will expire too soon (<60s)"); } + if !key_is_routed(peer_attestation.attester_bft_pk) + || !key_is_routed(peer_attestation.attestee_bft_pk) + { connection_keys_to_disconnect.push(connection_key); - continue; + continue 'process_packets; } - if peer_attestation.issued >= peer_attestation.expiry { - if PRINT_PROTOCOL { println!("{ctx_str} {ANSI_RED}PROTOCOL{ANSI_RST}: Peer sent invalid peer attestation: Issued after expired"); } + + let Ok(now) = u64::try_from(chrono::Utc::now().timestamp()) else { + continue 'process_packets; + }; + let Some(lifetime) = peer_attestation + .expiry + .checked_sub(peer_attestation.issued) + else { + connection_keys_to_disconnect.push(connection_key); + continue 'process_packets; + }; + let Some(minimum_expiry) = now.checked_add(60) else { + continue 'process_packets; + }; + let Some(maximum_issued) = now.checked_add(MAX_ATTESTATION_CLOCK_SKEW_SECONDS) + else { + continue 'process_packets; + }; + let Some(maximum_expiry) = now + .checked_add(MAX_ATTESTATION_LIFETIME_SECONDS) + .and_then(|value| value.checked_add(MAX_ATTESTATION_CLOCK_SKEW_SECONDS)) + else { + continue 'process_packets; + }; + if peer_attestation.expiry < minimum_expiry { + if PRINT_PROTOCOL { println!("{ctx_str} {ANSI_RED}PROTOCOL{ANSI_RST}: Peer sent peer attestation that will expire too soon (<60s)"); } connection_keys_to_disconnect.push(connection_key); continue; } - if peer_attestation.issued + 60 > peer_attestation.expiry { - if PRINT_PROTOCOL { println!("{ctx_str} {ANSI_RED}PROTOCOL{ANSI_RST}: Peer sent invalid peer attestation: Expires less than 60 seconds after issued"); } + if lifetime < 60 + || lifetime > MAX_ATTESTATION_LIFETIME_SECONDS + || peer_attestation.issued > maximum_issued + || peer_attestation.expiry > maximum_expiry + { + if PRINT_PROTOCOL { println!("{ctx_str} {ANSI_RED}PROTOCOL{ANSI_RST}: Peer sent invalid peer attestation lifetime or timestamp"); } connection_keys_to_disconnect.push(connection_key); continue; } @@ -2199,7 +3823,16 @@ pub async fn entry_point(my_root_private_key: SigningKey, } }; } - bft_address_map.insert(&peer_attestation.attestee_bft_pk.clone(), &peer_attestation.stp_address.clone(), Some(peer_attestation)); + let attestee_bft_pk = peer_attestation.attestee_bft_pk; + let attested_address = peer_attestation.stp_address.clone(); + if !bft_address_map.insert( + &attestee_bft_pk, + &attested_address, + Some(peer_attestation), + ) { + connection_keys_to_disconnect.push(connection_key); + continue 'process_packets; + } } } @@ -2224,8 +3857,12 @@ pub async fn entry_point(my_root_private_key: SigningKey, } }; } - bft_address_map.insert(&their_verification.pk, &peer.stp_address, None); + if !bft_address_map.insert(&their_verification.pk, &peer.stp_address, None) { + connection_keys_to_disconnect.push(connection_key); + continue; + } peer.bft_pk = their_verification.pk; + peer.authenticated_bft_pk = Some(their_verification.pk); let my_verification = { // almost @Duplicate let pk_bytes = my_root_public_bft_key.as_ref(); @@ -2237,7 +3874,7 @@ pub async fn entry_point(my_root_private_key: SigningKey, assert!(peer.stp_handshake_hash.len() == 64); let keyed_hash_of_stp_handshake_hash = hash_key_for_stp_handshake_hash.hash(&peer.stp_handshake_hash[..]); - TMSig(my_root_private_key.sign(&keyed_hash_of_stp_handshake_hash[..]).to_bytes()) + bft_state.durable_signer.sign_auxiliary_digest(&keyed_hash_of_stp_handshake_hash) }; PacketIdVerification { pk, sig } @@ -2246,13 +3883,8 @@ pub async fn entry_point(my_root_private_key: SigningKey, let attestation = { // @Duplicate let addr = peer.stp_address.clone(); let issued: u64 = chrono::Utc::now().timestamp().try_into().expect("should fit in a u64"); - let expiry = { - let seconds_per_minute = 60; - let minutes_per_hour = 60; - let hours_per_day = 24; - let days_expiry = 1; - - issued + (seconds_per_minute * minutes_per_hour * hours_per_day * days_expiry) + let Some(expiry) = issued.checked_add(24 * 60 * 60) else { + continue; }; let sig = { let hash_key_for_attestation = HashKey(blake3::Hasher::new_derive_key("Tenderlink One Party Signed Peer Attestation").finalize().into()); @@ -2268,7 +3900,7 @@ pub async fn entry_point(my_root_private_key: SigningKey, hasher.update(&my_root_public_bft_key.as_ref()[..]); let keyed_hash_of_one_party_signed_attestation = hasher.finalize(); - TMSig(my_root_private_key.sign(&keyed_hash_of_one_party_signed_attestation.as_bytes()[..]).to_bytes()) + bft_state.durable_signer.sign_auxiliary_digest(keyed_hash_of_one_party_signed_attestation.as_bytes()) }; PacketIdAttestation { issued, expiry, addr, sig } @@ -2307,8 +3939,12 @@ pub async fn entry_point(my_root_private_key: SigningKey, } }; } - bft_address_map.insert(&their_verification.pk, &peer.stp_address, None); + if !bft_address_map.insert(&their_verification.pk, &peer.stp_address, None) { + connection_keys_to_disconnect.push(connection_key); + continue; + } peer.bft_pk = their_verification.pk; + peer.authenticated_bft_pk = Some(their_verification.pk); let Some(attestation) = PacketIdAttestation::read_from(msg) else { if PRINT_PROTOCOL { println!("{ctx_str} {ANSI_RED}PROTOCOL{ANSI_RST}: Peer failed ID verification: Failed to read ID Hello Ack packet: Failed to read attestation"); } @@ -2334,19 +3970,11 @@ pub async fn entry_point(my_root_private_key: SigningKey, connection_keys_to_disconnect.push(connection_key); continue; } - let now: u64 = chrono::Utc::now().timestamp().try_into().expect("should fit in a u64"); - if attestation.expiry <= now { - if PRINT_PROTOCOL { println!("{ctx_str} {ANSI_RED}PROTOCOL{ANSI_RST}: Peer failed ID attestation: Attestation has already expired"); } - connection_keys_to_disconnect.push(connection_key); - continue; - } - if attestation.issued >= attestation.expiry { - if PRINT_PROTOCOL { println!("{ctx_str} {ANSI_RED}PROTOCOL{ANSI_RST}: Peer failed ID attestation: Issued after expired"); } - connection_keys_to_disconnect.push(connection_key); + let Ok(now) = u64::try_from(chrono::Utc::now().timestamp()) else { continue; - } - if attestation.issued + 60 > attestation.expiry { - if PRINT_PROTOCOL { println!("{ctx_str} {ANSI_RED}PROTOCOL{ANSI_RST}: Peer failed ID attestation: Expires less than 60 seconds after issued"); } + }; + if !attestation_window_is_valid(attestation.issued, attestation.expiry, now) { + if PRINT_PROTOCOL { println!("{ctx_str} {ANSI_RED}PROTOCOL{ANSI_RST}: Peer failed ID attestation: invalid lifetime or timestamp"); } connection_keys_to_disconnect.push(connection_key); continue; } @@ -2384,7 +4012,7 @@ pub async fn entry_point(my_root_private_key: SigningKey, let hash_key_for_attestation = HashKey(blake3::Hasher::new_derive_key("Tenderlink Two Party Signed Peer Attestation").finalize().into()); assert!(attestation.sig.0.len() == 64); let keyed_hash_of_two_party_signed_attestation = hash_key_for_attestation.hash(&attestation.sig.0[..]); - TMSig(my_root_private_key.sign(&keyed_hash_of_two_party_signed_attestation[..]).to_bytes()) + bft_state.durable_signer.sign_auxiliary_digest(&keyed_hash_of_two_party_signed_attestation) }; let peer_attestation = PeerAttestation { @@ -2396,8 +4024,14 @@ pub async fn entry_point(my_root_private_key: SigningKey, attester_sig: attestation.sig, attestee_sig: sig, }; - my_address_attestations.push(peer_attestation.clone()); - bft_address_map.insert(&peer_attestation.attestee_bft_pk, &peer_attestation.stp_address, Some(peer_attestation.clone())); + if !bft_address_map.insert( + &peer_attestation.attestee_bft_pk, + &peer_attestation.stp_address, + Some(peer_attestation.clone()), + ) { + connection_keys_to_disconnect.push(connection_key); + continue; + } // @Todo: Decide if @Temporary? // if false @@ -2455,8 +4089,10 @@ pub async fn entry_point(my_root_private_key: SigningKey, else { } - if let Some(pk) = bft_address_map.get_key(&peer.stp_address) { - bft_address_map.last_packet_utcs.insert(*pk, chrono::Utc::now().timestamp()); + if let Some(pk) = peer.authenticated_bft_pk { + bft_address_map + .last_packet_utcs + .insert(pk, chrono::Utc::now().timestamp()); } } @@ -2660,7 +4296,15 @@ impl PacketVotes { o += self.height .write_to(&mut buf[o..]); o += self.value_id.0 .write_to(&mut buf[o..]); // NOTE(azmr): slight saving of bytes-on-wire if unused? i.e. initial few times each - for i in 0..(self.no_votes_n + self.yes_votes_n) as usize { + let votes_n = self + .no_votes_n + .checked_add(self.yes_votes_n) + .expect("local vote packet count must not overflow"); + assert!( + votes_n as usize <= self.votes.len(), + "local vote packet exceeds wire capacity" + ); + for i in 0..votes_n as usize { o += &self.votes[i].roster_i.write_to(&mut buf[o..]); o += &self.votes[i].sig .0.write_to(&mut buf[o..]); } @@ -2676,10 +4320,20 @@ impl PacketVotes { value_id: ValueId(SliceRead::read_from(buf)?), ..Default::default() }; - for i in 0..(packet.no_votes_n + packet.yes_votes_n) as usize { + if packet.round > MAX_CONSENSUS_ROUND { + return None; + } + let votes_n = packet.no_votes_n.checked_add(packet.yes_votes_n)?; + if votes_n == 0 || votes_n as usize > packet.votes.len() { + return None; + } + for i in 0..votes_n as usize { packet.votes[i].roster_i = u16::read_from(buf)?; packet.votes[i].sig.0 = SliceRead::read_from(buf)?; } + if !buf.is_empty() { + return None; + } Some(packet) } } @@ -2928,6 +4582,7 @@ fn hook_fail_on_panic() { })) } +#[cfg(any(test, feature = "simulation"))] pub fn run_instances(i: usize) { let rt = tokio::runtime::Runtime::new().unwrap(); @@ -3007,10 +4662,79 @@ pub mod helpers; use helpers::*; +#[cfg(test)] +mod condition28_tests; +#[cfg(test)] +mod gossip_tests; +#[cfg(test)] +mod signer_wal_tests; + #[cfg(test)] mod tests { use super::*; + fn full_round(height: u64, byte: u8) -> RoundData { + let proposal = BlockValue(vec![byte; 32]); + RoundData { + height, + proposal_id: proposal.id_from_value(&HashKeys::default()), + proposal, + proposal_sigs: vec![TMSig([byte; 64])], + proposal_sigs_n: 1, + proposal_checked_validity: (TMStatus::Pass, TMStatusReason::None), + ..RoundData::EMPTY + } + } + + fn encoded_vote_packet(no_votes_n: u8, yes_votes_n: u8, round: u32, votes_n: usize) -> Vec { + let mut bytes = Vec::with_capacity(46 + votes_n * 66); + bytes.push(no_votes_n); + bytes.push(yes_votes_n); + bytes.extend_from_slice(&round.to_le_bytes()); + bytes.extend_from_slice(&0u64.to_le_bytes()); + bytes.extend_from_slice(&[0u8; 32]); + bytes.resize(46 + votes_n * 66, 0); + bytes + } + + #[test] + fn vote_round_step_encoding_has_one_canonical_domain() { + assert_eq!(canonical_vote_round(MAX_CONSENSUS_ROUND, false), Some(MAX_CONSENSUS_ROUND)); + assert_eq!(canonical_vote_round(MAX_CONSENSUS_ROUND, true), Some(u32::MAX)); + assert_eq!(canonical_vote_round(MAX_CONSENSUS_ROUND + 1, false), None); + assert_eq!(canonical_vote_round(MAX_CONSENSUS_ROUND + 1, true), None); + } + + #[test] + fn proposal_chunk_layout_rejects_zero_huge_and_out_of_range_headers() { + assert_eq!(proposal_chunk_layout(0, 0), None); + assert_eq!(proposal_chunk_layout(u32::MAX, 0), None); + assert_eq!(proposal_chunk_layout(1, 1), None); + assert_eq!(proposal_chunk_layout(MAX_PROPOSAL_BYTES as u32, u32::MAX), None); + assert_eq!(proposal_chunk_layout(1, 0), Some((0, 1, 1))); + let last_chunk = proposal_chunk_count(MAX_PROPOSAL_BYTES as u32).unwrap() - 1; + assert!(proposal_chunk_layout(MAX_PROPOSAL_BYTES as u32, last_chunk as u32).is_some()); + } + + #[test] + fn vote_packet_decoder_rejects_count_overflow_capacity_and_trailing_bytes() { + let valid = encoded_vote_packet(9, 9, MAX_CONSENSUS_ROUND, 18); + assert!(PacketVotes::read_from(&mut &valid[..]).is_some()); + + for malformed in [ + encoded_vote_packet(19, 0, 0, 0), + encoded_vote_packet(255, 255, 0, 0), + encoded_vote_packet(0, 0, 0, 0), + encoded_vote_packet(1, 0, MAX_CONSENSUS_ROUND + 1, 1), + ] { + assert!(PacketVotes::read_from(&mut &malformed[..]).is_none()); + } + + let mut trailing = encoded_vote_packet(1, 0, 0, 1); + trailing.push(0); + assert!(PacketVotes::read_from(&mut &trailing[..]).is_none()); + } + // #[ignore] // #[test] // fn multi_rt() { @@ -3031,6 +4755,7 @@ mod tests { // } #[test] + #[ignore = "manual multi-node simulator binds fixed ports and runs indefinitely"] fn single_rt() { run_instances(usize::MAX); } @@ -3090,6 +4815,204 @@ mod tests { for (test_i, test) in tests.iter().enumerate() { let rngs = gen_mostly_empty_rngs(test.arr.len(), |i| test.arr[i] == b'0'); assert_eq!(test.rngs, &rngs, "index {}", test_i); + for selector in 0..rngs.len().saturating_mul(3) { + assert_eq!( + select_mostly_empty_rng( + test.arr.len(), + |i| test.arr[i] == b'0', + selector as u64, + ), + Some(rngs[selector % rngs.len()]), + "selection index {selector} in test {test_i}", + ); + } } + assert_eq!(select_mostly_empty_rng(8, |_| false, 0), None); + } + + #[test] + fn recent_commit_cache_is_a_bounded_contiguous_suffix() { + let total = MAX_RECENT_COMMIT_ROUNDS_IN_MEMORY + 7; + let mut cache = Vec::new(); + for height in 0..total { + append_recent_commit_round(&mut cache, full_round(height as u64, height as u8)); + } + + assert_eq!(cache.len(), MAX_RECENT_COMMIT_ROUNDS_IN_MEMORY); + assert_eq!(cache.first().unwrap().height, 7); + assert_eq!(cache.last().unwrap().height, (total - 1) as u64); + validate_commit_round_cache(&cache, total as u64).unwrap(); + assert_eq!(cache.iter().filter(|round| round.has_full_proposal()).count(), 1); + assert!(cached_commit_round_at_height(&cache, 6).is_none()); + assert_eq!(commit_round_cache_entry_at_height(&cache, 7).unwrap().height, 7); + assert!(cached_commit_round_at_height(&cache, 7).is_none()); + assert_eq!( + cached_commit_round_at_height(&cache, (total - 1) as u64) + .unwrap() + .height, + (total - 1) as u64, + ); + } + + #[test] + fn historical_relay_source_covers_the_64_65_and_far_behind_boundaries() { + let mut first_64 = (0..MAX_RECENT_COMMIT_ROUNDS_IN_MEMORY) + .map(|height| full_round(height as u64, height as u8)) + .collect::>(); + compact_recent_commit_payloads(&mut first_64); + assert!(commit_round_for_relay(&first_64, None, 0).is_none()); + let loaded_height_zero = full_round(0, 200); + assert_eq!( + commit_round_for_relay(&first_64, Some(&loaded_height_zero), 0) + .unwrap() + .height, + 0 + ); + assert_eq!( + commit_round_for_relay( + &first_64, + None, + (MAX_RECENT_COMMIT_ROUNDS_IN_MEMORY - 1) as u64, + ) + .unwrap() + .height, + (MAX_RECENT_COMMIT_ROUNDS_IN_MEMORY - 1) as u64, + ); + + let mut after_65 = (1..=MAX_RECENT_COMMIT_ROUNDS_IN_MEMORY) + .map(|height| full_round(height as u64, height as u8)) + .collect::>(); + compact_recent_commit_payloads(&mut after_65); + assert!(commit_round_for_relay(&after_65, None, 0).is_none()); + assert_eq!( + commit_round_for_relay(&after_65, Some(&loaded_height_zero), 0) + .unwrap() + .height, + 0 + ); + + let mut far_cache = (100..100 + MAX_RECENT_COMMIT_ROUNDS_IN_MEMORY) + .map(|height| full_round(height as u64, height as u8)) + .collect::>(); + compact_recent_commit_payloads(&mut far_cache); + let loaded_far_behind = full_round(7, 201); + assert_eq!( + commit_round_for_relay(&far_cache, Some(&loaded_far_behind), 7) + .unwrap() + .height, + 7 + ); + assert!(commit_round_for_relay(&far_cache, Some(&loaded_far_behind), 8).is_none()); + } + + #[test] + fn stale_round_payload_compaction_preserves_identity_and_vote_evidence() { + let proposal_id = ValueId([7; 32]); + let vote = (proposal_id, TMSig([9; 64])); + let mut rounds = vec![ + RoundData { + height: 4, + round: 3, + proposal: BlockValue(vec![1; 4096]), + proposal_valid_round: 1, + proposal_sigs: vec![TMSig([2; 64]); 4], + proposal_sigs_n: 4, + proposal_id, + proposal_checked_validity: (TMStatus::Pass, TMStatusReason::None), + msg_val_sigs: vec![[vote, vote]], + counts: ConsensusCounts { + anys: 1, + prevotes: 1, + nil_prevotes: 0, + yes_prevotes: 1, + precommits: 1, + yes_precommits: 1, + }, + ..RoundData::EMPTY + }, + full_round(4, 8), + ]; + rounds[1].round = 4; + + compact_round_proposal_payload(&mut rounds[0]); + + assert!(rounds[0].proposal.0.is_empty()); + assert!(rounds[0].proposal_sigs.is_empty()); + assert_eq!(rounds[0].proposal_sigs_n, 0); + assert_eq!(rounds[0].proposal_id, proposal_id); + assert_eq!(rounds[0].proposal_valid_round, 1); + assert_eq!(rounds[0].msg_val_sigs, vec![[vote, vote]]); + assert_eq!(rounds[0].counts.yes_precommits, 1); + assert!(rounds[1].has_full_proposal()); + } + + #[test] + fn referenced_prevote_qc_survives_referenced_payload_compaction() { + let keys = (1u8..=4) + .map(|byte| SigningKey::from([byte; 32])) + .collect::>(); + let mut cumulative_stake = 0u64; + let roster = keys + .iter() + .map(|key| { + cumulative_stake += 1; + SortedRosterMember { + pub_key: PubKeyID(key.verification_key().into()), + stake: 1, + cumulative_stake, + } + }) + .collect::>(); + let namespace = [0u8; 32]; + let proposal = BlockValue(vec![42; 128]); + let proposal_id = proposal.id_from_value(&HashKeys::default()); + let mut referenced_votes = vec![[(ValueId::NIL, TMSig::NIL); 2]; roster.len()]; + for (index, key) in keys.iter().take(3).enumerate() { + let signable = make_vote_sign_datas( + roster[index].pub_key, + false, + 9, + 1, + proposal_id, + )[1]; + referenced_votes[index][0] = + (proposal_id, TMSig(sign_with_namespace(key, &signable, &namespace))); + } + let mut referenced = RoundData { + height: 9, + round: 1, + proposal: proposal.clone(), + proposal_id, + proposal_sigs: vec![TMSig([1; 64])], + proposal_sigs_n: 1, + msg_val_sigs: referenced_votes, + roster: roster.clone(), + vote_namespace: namespace, + ..RoundData::EMPTY + }; + compact_round_proposal_payload(&mut referenced); + let current = RoundData { + height: 9, + round: 3, + proposal, + proposal_valid_round: 1, + proposal_id, + proposal_sigs: vec![TMSig([2; 64])], + proposal_sigs_n: 1, + roster, + vote_namespace: namespace, + ..RoundData::EMPTY + }; + let rounds = vec![referenced, current]; + + assert_eq!( + verified_referenced_prevote_certificate( + &rounds, + 1, + &namespace, + &HashKeys::default(), + ), + Some((1, 3, 3)), + ); } } diff --git a/tenderlink/src/signer_wal.rs b/tenderlink/src/signer_wal.rs new file mode 100644 index 00000000..129df661 --- /dev/null +++ b/tenderlink/src/signer_wal.rs @@ -0,0 +1,2878 @@ +use super::*; + +use std::{ + collections::{BTreeMap, BTreeSet}, + fs::{File, OpenOptions}, + io::{Read, Seek, SeekFrom, Write}, + path::{Path, PathBuf}, +}; + +const WAL_MAGIC: [u8; 8] = *b"TLWAL002"; +const ANCHOR_MAGIC: [u8; 8] = *b"TLANCH02"; +const WAL_VERSION: u16 = 2; +const MAX_RECORD_BYTES: usize = 16 * 1024 * 1024; +pub(crate) const MAX_PROPOSAL_BYTES: usize = 8 * 1024 * 1024; +const MAX_SIGNABLE_PARTS: usize = 16 * 1024; +const MAX_SIGNABLE_BYTES: usize = 16 * 1024; +pub(crate) const MAX_CONSENSUS_ROUND: u32 = 0x7fff_ffff; +const CONSENSUS_RULES_VERSION: &[u8] = + b"ctaz-tenderlink-consensus-v2:referenced-qc:n-minus-f:durable-intent:atomic-commit"; +const PENDING_COMMIT_RECOVERY_REASON: &str = + "commit recovery is incomplete; durable store reconciliation is required"; +const LEGACY_PENDING_COMMIT_REASON: &str = + "legacy pending commit lacks exact proposal valid-round/signature evidence; automatic recovery is disabled"; + +#[cfg(test)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum WalFailpoint { + AfterWalWrite, + AfterWalSync, + AfterAnchorWrite, + AfterAnchorSync, + AfterCommitApplied, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SignerEpochBinding { + pub public_key: PubKeyID, + pub chain_id: [u8; 32], + pub height: u64, + pub parent_commit: [u8; 32], + pub vote_namespace: [u8; 32], + pub consensus_config_hash: [u8; 32], + pub roster_hash: [u8; 32], + pub roster_index: u32, + pub active_roster_len: u32, +} + +#[derive(Clone, Debug)] +pub struct DurableSignerConfig { + pub wal_path: PathBuf, + pub anchor_path: PathBuf, + /// This is an action gate, not a self-attestation. The caller may set it only after proving + /// the anchor is outside the WAL/store rollback domain and globally fences this key. + pub independent_anchor_authorized: bool, + /// Hash of an operator-sealed, one-time bootstrap receipt for an already-live + /// non-genesis key. The caller must verify the receipt and global key fence + /// outside this rollback domain before supplying it. Genesis uses no receipt. + pub non_genesis_bootstrap_receipt_hash: Option<[u8; 32]>, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum SignerStatus { + Active, + ObserverOnly(String), + /// A certified commit intent is durable, but applying it to the PoS store or + /// sealing its successor did not complete. This state blocks every signing + /// path while permitting only exact recovery/completion of `digest`. + ReconciliationRequired([u8; 32], String), + Poisoned(String), +} + +#[derive(Debug)] +pub enum SignerError { + ObserverOnly(String), + ReconciliationRequired([u8; 32], String), + Conflict(String), + Integrity(String), + Io(std::io::Error), +} + +impl From for SignerError { + fn from(error: std::io::Error) -> Self { + Self::Io(error) + } +} + +impl std::fmt::Display for SignerError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::ObserverOnly(reason) => write!(f, "observer only: {reason}"), + Self::ReconciliationRequired(_, reason) => { + write!(f, "commit reconciliation required: {reason}") + } + Self::Conflict(reason) => write!(f, "signing conflict: {reason}"), + Self::Integrity(reason) => write!(f, "WAL integrity failure: {reason}"), + Self::Io(error) => write!(f, "WAL I/O failure: {error}"), + } + } +} + +impl std::error::Error for SignerError {} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub(super) enum SlotKind { + Proposal = 1, + Prevote = 2, + Precommit = 3, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +struct SignerSlot { + round: u32, + kind: SlotKind, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LockValidTransition { + pub locked_round: i64, + pub locked_value_id: ValueId, + pub locked_value: Vec, + pub valid_round: i64, + pub valid_value_id: ValueId, + pub valid_value: Vec, + /// Canonical raw certificate evidence. Replay must reverify it before restoring state. + pub certificate: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) enum SignedIntent { + Proposal { + round: u32, + valid_round: i64, + proposal_id: ValueId, + proposal: Vec, + signable_parts: Vec>, + }, + Vote { + round: u32, + kind: SlotKind, + value_id: ValueId, + signable: Vec, + transition: Option, + }, +} + +impl SignedIntent { + fn slot(&self) -> SignerSlot { + match self { + Self::Proposal { round, .. } => SignerSlot { + round: *round, + kind: SlotKind::Proposal, + }, + Self::Vote { round, kind, .. } => SignerSlot { + round: *round, + kind: *kind, + }, + } + } +} + +#[derive(Debug)] +struct LoadedWal { + epoch: Option, + authorized: bool, + intents: BTreeMap, + transition: Option, + poisoned: Option, + pending_commit: Option, + commit_applied_epoch: Option, + bootstrap_origin: Option<[u8; 32]>, + next_sequence: u64, + last_hash: [u8; 32], + clean_tail: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct PendingCommit { + digest: [u8; 32], + decided_value_id: ValueId, + proposal: Vec, + certificate: Vec, + proposal_evidence: PendingProposalEvidence, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +enum PendingProposalEvidence { + /// RECORD_COMMIT_INTENT (v1) deliberately remains readable so completed + /// historical WALs replay, but an unfinished legacy intent cannot safely + /// reconstruct authenticated proposal gossip. + LegacyUnavailable, + Exact { + round: u32, + valid_round: i64, + proposal_sigs: Vec, + }, +} + +#[derive(Clone, Debug)] +pub(super) struct PendingCommitRecovery { + pub digest: [u8; 32], + pub proposal: BlockValue, + pub proposal_valid_round: i64, + pub proposal_sigs: Vec, + /// Exact certified round reconstructed solely from the durable WAL, epoch, + /// and roster. This includes proposal bytes, valid-round, ordered proposal + /// chunk signatures, precommit evidence, counts, and vote namespace. + pub round_data: RoundData, + pub fat_pointer: FatPointerToBftBlock, +} + +fn validate_commit_successor( + current: &SignerEpochBinding, + pending: &PendingCommit, + next: &SignerEpochBinding, +) -> Result<(), SignerError> { + let expected_height = current + .height + .checked_add(1) + .ok_or_else(|| SignerError::Integrity("signer height overflow".into()))?; + if next.public_key != current.public_key + || next.chain_id != current.chain_id + || next.height != expected_height + || next.parent_commit != pending.decided_value_id.0 + || next.consensus_config_hash != current.consensus_config_hash + { + return Err(SignerError::Integrity( + "commit successor does not match the certified current epoch".into(), + )); + } + if next.active_roster_len == 0 + || (next.roster_index != u32::MAX && next.roster_index >= next.active_roster_len) + { + return Err(SignerError::Integrity( + "commit successor roster binding is invalid".into(), + )); + } + Ok(()) +} + +impl Default for LoadedWal { + fn default() -> Self { + Self { + epoch: None, + authorized: false, + intents: BTreeMap::new(), + transition: None, + poisoned: None, + pending_commit: None, + commit_applied_epoch: None, + bootstrap_origin: None, + next_sequence: 0, + last_hash: [0u8; 32], + clean_tail: true, + } + } +} + +fn put_u16(out: &mut Vec, value: u16) { + out.extend_from_slice(&value.to_le_bytes()); +} +fn put_u32(out: &mut Vec, value: u32) { + out.extend_from_slice(&value.to_le_bytes()); +} +fn put_u64(out: &mut Vec, value: u64) { + out.extend_from_slice(&value.to_le_bytes()); +} +fn put_i64(out: &mut Vec, value: i64) { + out.extend_from_slice(&value.to_le_bytes()); +} +fn put_bytes(out: &mut Vec, value: &[u8]) -> Result<(), SignerError> { + let len: u32 = value + .len() + .try_into() + .map_err(|_| SignerError::Integrity("field too large".into()))?; + put_u32(out, len); + out.extend_from_slice(value); + Ok(()) +} + +#[cfg(test)] +pub(super) fn encode_legacy_commit_intent( + decided_value_id: ValueId, + proposal: &[u8], + certificate: &[u8], +) -> Result, SignerError> { + if decided_value_id == ValueId::NIL { + return Err(SignerError::Integrity( + "commit intent cannot decide NIL".into(), + )); + } + if proposal.is_empty() || proposal.len() > MAX_PROPOSAL_BYTES { + return Err(SignerError::Integrity( + "commit recovery proposal exceeds its bound".into(), + )); + } + if certificate.len() > MAX_RECORD_BYTES { + return Err(SignerError::Integrity( + "commit certificate exceeds bound".into(), + )); + } + let mut payload = Vec::new(); + payload.extend_from_slice(&decided_value_id.0); + put_bytes(&mut payload, proposal)?; + put_bytes(&mut payload, certificate)?; + if payload.len() > MAX_RECORD_BYTES { + return Err(SignerError::Integrity( + "commit intent record exceeds bound".into(), + )); + } + Ok(payload) +} + +fn encode_commit_intent( + round: u32, + decided_value_id: ValueId, + proposal: &[u8], + proposal_valid_round: i64, + proposal_sigs: &[TMSig], + certificate: &[u8], +) -> Result, SignerError> { + if round > MAX_CONSENSUS_ROUND { + return Err(SignerError::Integrity( + "commit round exceeds the canonical 31-bit domain".into(), + )); + } + if proposal_valid_round < -1 || proposal_valid_round >= i64::from(round) { + return Err(SignerError::Integrity( + "commit proposal valid-round is outside the canonical range".into(), + )); + } + if decided_value_id == ValueId::NIL { + return Err(SignerError::Integrity( + "commit intent cannot decide NIL".into(), + )); + } + if proposal.is_empty() || proposal.len() > MAX_PROPOSAL_BYTES { + return Err(SignerError::Integrity( + "commit recovery proposal exceeds its bound".into(), + )); + } + if proposal_sigs.is_empty() || proposal_sigs.len() > MAX_SIGNABLE_PARTS { + return Err(SignerError::Integrity( + "commit proposal-signature manifest exceeds its bound".into(), + )); + } + if proposal_sigs.iter().any(|signature| *signature == TMSig::NIL) { + return Err(SignerError::Integrity( + "commit proposal-signature manifest is incomplete".into(), + )); + } + if certificate.len() > MAX_RECORD_BYTES { + return Err(SignerError::Integrity( + "commit certificate exceeds bound".into(), + )); + } + + let mut payload = Vec::new(); + put_u32(&mut payload, round); + put_i64(&mut payload, proposal_valid_round); + payload.extend_from_slice(&decided_value_id.0); + put_bytes(&mut payload, proposal)?; + put_u32( + &mut payload, + proposal_sigs + .len() + .try_into() + .map_err(|_| SignerError::Integrity("too many proposal signatures".into()))?, + ); + for signature in proposal_sigs { + payload.extend_from_slice(&signature.0); + } + put_bytes(&mut payload, certificate)?; + if payload.len() > MAX_RECORD_BYTES { + return Err(SignerError::Integrity( + "commit intent record exceeds bound".into(), + )); + } + Ok(payload) +} + +struct Decoder<'a> { + bytes: &'a [u8], + offset: usize, +} + +impl<'a> Decoder<'a> { + fn new(bytes: &'a [u8]) -> Self { + Self { bytes, offset: 0 } + } + fn take(&mut self, len: usize) -> Result<&'a [u8], SignerError> { + let end = self + .offset + .checked_add(len) + .ok_or_else(|| SignerError::Integrity("length overflow".into()))?; + let value = self + .bytes + .get(self.offset..end) + .ok_or_else(|| SignerError::Integrity("truncated record".into()))?; + self.offset = end; + Ok(value) + } + fn u8(&mut self) -> Result { + Ok(self.take(1)?[0]) + } + fn u16(&mut self) -> Result { + Ok(u16::from_le_bytes(self.take(2)?.try_into().unwrap())) + } + fn u32(&mut self) -> Result { + Ok(u32::from_le_bytes(self.take(4)?.try_into().unwrap())) + } + fn u64(&mut self) -> Result { + Ok(u64::from_le_bytes(self.take(8)?.try_into().unwrap())) + } + fn i64(&mut self) -> Result { + Ok(i64::from_le_bytes(self.take(8)?.try_into().unwrap())) + } + fn array32(&mut self) -> Result<[u8; 32], SignerError> { + Ok(self.take(32)?.try_into().unwrap()) + } + fn bytes(&mut self, max: usize) -> Result, SignerError> { + let len: usize = self.u32()?.try_into().unwrap(); + if len > max { + return Err(SignerError::Integrity("bounded field exceeds limit".into())); + } + Ok(self.take(len)?.to_vec()) + } + fn finish(self) -> Result<(), SignerError> { + if self.offset == self.bytes.len() { + Ok(()) + } else { + Err(SignerError::Integrity("trailing record bytes".into())) + } + } +} + +pub fn canonical_roster_hash(roster: &[SortedRosterMember]) -> Result<[u8; 32], SignerError> { + let active_len = active_roster_len(roster); + if active_len == 0 { + return Err(SignerError::Integrity("consensus roster is empty".into())); + } + let mut cumulative_stake = 0u64; + let mut identities = BTreeSet::new(); + for (index, member) in roster.iter().enumerate() { + if !identities.insert(member.pub_key) { + return Err(SignerError::Integrity( + "consensus roster contains a duplicate key".into(), + )); + } + cumulative_stake = cumulative_stake + .checked_add(member.stake) + .ok_or_else(|| SignerError::Integrity("consensus roster stake overflow".into()))?; + if member.cumulative_stake != cumulative_stake { + return Err(SignerError::Integrity( + "consensus roster cumulative stake is noncanonical".into(), + )); + } + if index > 0 { + let previous = &roster[index - 1]; + if (previous.stake, previous.pub_key) < (member.stake, member.pub_key) { + return Err(SignerError::Integrity( + "consensus roster ordering is noncanonical".into(), + )); + } + } + } + let mut encoded = Vec::with_capacity(32 + roster.len() * 48); + encoded.extend_from_slice(b"tenderlink-roster-v1"); + put_u32( + &mut encoded, + roster + .len() + .try_into() + .map_err(|_| SignerError::Integrity("roster too large".into()))?, + ); + put_u32( + &mut encoded, + active_len + .try_into() + .map_err(|_| SignerError::Integrity("roster too large".into()))?, + ); + for member in roster { + encoded.extend_from_slice(&member.pub_key.0); + put_u64(&mut encoded, member.stake); + put_u64(&mut encoded, member.cumulative_stake); + } + Ok(blake3::hash(&encoded).into()) +} + +fn validate_epoch_consensus_roster( + epoch: &SignerEpochBinding, + roster: &[SortedRosterMember], +) -> Result<(), SignerError> { + let active_len = active_roster_len(roster); + if active_len != epoch.active_roster_len as usize { + return Err(SignerError::Integrity( + "epoch active-roster length mismatch".into(), + )); + } + if canonical_roster_hash(roster)? != epoch.roster_hash { + return Err(SignerError::Integrity( + "epoch roster fingerprint mismatch".into(), + )); + } + Ok(()) +} + +fn validate_epoch_roster( + epoch: &SignerEpochBinding, + roster: &[SortedRosterMember], +) -> Result<(), SignerError> { + validate_epoch_consensus_roster(epoch, roster)?; + let active_len = active_roster_len(roster); + let roster_index: usize = epoch.roster_index.try_into().map_err(|_| { + SignerError::Integrity("epoch roster index does not fit this platform".into()) + })?; + if roster_index >= active_len || roster[roster_index].pub_key != epoch.public_key { + return Err(SignerError::Integrity( + "epoch signer index does not resolve to its public key".into(), + )); + } + Ok(()) +} + +pub fn consensus_hash_keys_fingerprint(hash_keys: &HashKeys) -> [u8; 32] { + let mut encoded = Vec::with_capacity(128 + 28); + encoded.extend_from_slice(CONSENSUS_RULES_VERSION); + encoded.extend_from_slice(&WAL_VERSION.to_le_bytes()); + encoded.extend_from_slice(&(ROSTER_MAX_N as u64).to_le_bytes()); + encoded.extend_from_slice(&(PROPOSAL_CHUNK_DATA_SIZE as u64).to_le_bytes()); + encoded.extend_from_slice(&hash_keys.proposer.0); + encoded.extend_from_slice(&hash_keys.value_id.0); + encoded.extend_from_slice(&hash_keys.connect_contention.0); + encoded.extend_from_slice(&hash_keys.proposal_sig.0); + blake3::hash(&encoded).into() +} + +pub fn canonical_prevote_certificate( + round_data: &RoundData, + roster: &[SortedRosterMember], +) -> Result, SignerError> { + if round_data.round > MAX_CONSENSUS_ROUND { + return Err(SignerError::Integrity( + "prevote certificate round exceeds the canonical 31-bit domain".into(), + )); + } + let active_len = active_roster_len(roster); + if round_data.msg_val_sigs.len() < active_len || round_data.roster.len() < active_len { + return Err(SignerError::Integrity( + "round evidence is shorter than the active roster".into(), + )); + } + let mut out = Vec::with_capacity(128 + active_len * 144); + out.extend_from_slice(b"tenderlink-prevote-qc-v1"); + put_u64(&mut out, round_data.height); + put_u32(&mut out, round_data.round); + out.extend_from_slice(&round_data.proposal_id.0); + out.extend_from_slice(&round_data.vote_namespace); + put_u32( + &mut out, + active_len + .try_into() + .map_err(|_| SignerError::Integrity("active roster too large".into()))?, + ); + for (index, member) in roster[..active_len].iter().enumerate() { + if round_data.roster[index].pub_key != member.pub_key + || round_data.roster[index].stake != member.stake + || round_data.roster[index].cumulative_stake != member.cumulative_stake + { + return Err(SignerError::Integrity( + "round roster differs from epoch roster".into(), + )); + } + out.extend_from_slice(&member.pub_key.0); + put_u64(&mut out, member.stake); + put_u64(&mut out, member.cumulative_stake); + out.extend_from_slice(&round_data.msg_val_sigs[index][0].0 .0); + out.extend_from_slice(&round_data.msg_val_sigs[index][0].1 .0); + } + Ok(out) +} + +pub fn canonical_precommit_certificate( + round_data: &RoundData, + roster: &[SortedRosterMember], +) -> Result, SignerError> { + if round_data.round > MAX_CONSENSUS_ROUND { + return Err(SignerError::Integrity( + "precommit certificate round exceeds the canonical 31-bit domain".into(), + )); + } + let active_len = active_roster_len(roster); + if round_data.msg_val_sigs.len() < active_len || round_data.roster.len() < active_len { + return Err(SignerError::Integrity( + "round evidence is shorter than the active roster".into(), + )); + } + let mut out = Vec::with_capacity(128 + active_len * 144); + out.extend_from_slice(b"tenderlink-precommit-qc-v1"); + put_u64(&mut out, round_data.height); + put_u32(&mut out, round_data.round); + out.extend_from_slice(&round_data.proposal_id.0); + out.extend_from_slice(&round_data.vote_namespace); + put_u32( + &mut out, + active_len + .try_into() + .map_err(|_| SignerError::Integrity("active roster too large".into()))?, + ); + for (index, member) in roster[..active_len].iter().enumerate() { + if round_data.roster[index].pub_key != member.pub_key + || round_data.roster[index].stake != member.stake + || round_data.roster[index].cumulative_stake != member.cumulative_stake + { + return Err(SignerError::Integrity( + "round roster differs from epoch roster".into(), + )); + } + out.extend_from_slice(&member.pub_key.0); + put_u64(&mut out, member.stake); + put_u64(&mut out, member.cumulative_stake); + out.extend_from_slice(&round_data.msg_val_sigs[index][1].0 .0); + out.extend_from_slice(&round_data.msg_val_sigs[index][1].1 .0); + } + Ok(out) +} + +pub fn verify_precommit_certificate( + certificate: &[u8], + expected_round: u32, + decided_value_id: ValueId, + epoch: &SignerEpochBinding, + roster: &[SortedRosterMember], +) -> Result<(), SignerError> { + // Decision verification is roster/quorum work and must also succeed for an intentional + // off-roster observer. Local signer membership is enforced separately on signing paths. + validate_epoch_consensus_roster(epoch, roster)?; + let mut decoder = Decoder::new(certificate); + if decoder.take(b"tenderlink-precommit-qc-v1".len())? != b"tenderlink-precommit-qc-v1" { + return Err(SignerError::Integrity( + "precommit certificate domain mismatch".into(), + )); + } + if decoder.u64()? != epoch.height { + return Err(SignerError::Integrity( + "precommit certificate height mismatch".into(), + )); + } + let round = decoder.u32()?; + if round != expected_round || round > MAX_CONSENSUS_ROUND { + return Err(SignerError::Integrity( + "precommit certificate round mismatch".into(), + )); + } + let proposal_id = ValueId(decoder.array32()?); + if proposal_id != decided_value_id || proposal_id == ValueId::NIL { + return Err(SignerError::Integrity( + "precommit certificate proposal ID mismatch".into(), + )); + } + if decoder.array32()? != epoch.vote_namespace { + return Err(SignerError::Integrity( + "precommit certificate namespace mismatch".into(), + )); + } + let active_len: usize = decoder.u32()?.try_into().unwrap(); + if active_len != epoch.active_roster_len as usize || active_len != active_roster_len(roster) { + return Err(SignerError::Integrity( + "precommit certificate active-roster length mismatch".into(), + )); + } + + let mut total_power = 0u64; + let mut yes_power = 0u64; + for member in &roster[..active_len] { + if PubKeyID(decoder.array32()?) != member.pub_key + || decoder.u64()? != member.stake + || decoder.u64()? != member.cumulative_stake + { + return Err(SignerError::Integrity( + "precommit certificate roster mismatch".into(), + )); + } + total_power = total_power + .checked_add(member.stake) + .ok_or_else(|| SignerError::Integrity("precommit total-power overflow".into()))?; + if total_power != member.cumulative_stake { + return Err(SignerError::Integrity( + "precommit cumulative stake mismatch".into(), + )); + } + let value_id = ValueId(decoder.array32()?); + let sig = TMSig(decoder.take(64)?.try_into().unwrap()); + if value_id == ValueId::NIL { + if sig != TMSig::NIL { + let signed = + make_vote_sign_datas(member.pub_key, true, epoch.height, round, value_id)[0]; + sig.verify_with_namespace(member.pub_key, &signed, &epoch.vote_namespace) + .map_err(|_| { + SignerError::Integrity( + "invalid NIL precommit signature in certificate".into(), + ) + })?; + } + continue; + } + if sig == TMSig::NIL { + return Err(SignerError::Integrity( + "precommit certificate contains an unsigned non-NIL vote".into(), + )); + } + let signed = make_vote_sign_datas(member.pub_key, true, epoch.height, round, value_id)[1]; + sig.verify_with_namespace(member.pub_key, &signed, &epoch.vote_namespace) + .map_err(|_| { + SignerError::Integrity("invalid YES precommit signature in certificate".into()) + })?; + if value_id == proposal_id { + yes_power = yes_power + .checked_add(member.stake) + .ok_or_else(|| SignerError::Integrity("precommit YES-power overflow".into()))?; + } + } + decoder.finish()?; + if total_power == 0 { + return Err(SignerError::Integrity( + "zero-power precommit certificate".into(), + )); + } + let quorum = quorum_threshold(total_power); + if yes_power < quorum { + return Err(SignerError::Integrity( + "precommit certificate is below quorum".into(), + )); + } + Ok(()) +} + +fn recover_fat_pointer_from_commit_certificate( + pending: &PendingCommit, + epoch: &SignerEpochBinding, + roster: &[SortedRosterMember], +) -> Result< + ( + u32, + FatPointerToBftBlock, + Vec<(ValueId, TMSig)>, + ), + SignerError, +> { + let mut header = Decoder::new(&pending.certificate); + if header.take(b"tenderlink-precommit-qc-v1".len())? + != b"tenderlink-precommit-qc-v1" + { + return Err(SignerError::Integrity( + "precommit recovery certificate domain mismatch".into(), + )); + } + if header.u64()? != epoch.height { + return Err(SignerError::Integrity( + "precommit recovery certificate height mismatch".into(), + )); + } + let round = header.u32()?; + verify_precommit_certificate( + &pending.certificate, + round, + pending.decided_value_id, + epoch, + roster, + )?; + + let proposal_id = ValueId(header.array32()?); + if proposal_id != pending.decided_value_id || header.array32()? != epoch.vote_namespace { + return Err(SignerError::Integrity( + "precommit recovery certificate value or namespace mismatch".into(), + )); + } + let active_len: usize = header.u32()?.try_into().unwrap(); + if active_len != active_roster_len(roster) { + return Err(SignerError::Integrity( + "precommit recovery active-roster length mismatch".into(), + )); + } + let mut signatures = Vec::new(); + let mut precommits = Vec::with_capacity(active_len); + for member in &roster[..active_len] { + if PubKeyID(header.array32()?) != member.pub_key + || header.u64()? != member.stake + || header.u64()? != member.cumulative_stake + { + return Err(SignerError::Integrity( + "precommit recovery roster mismatch".into(), + )); + } + let value_id = ValueId(header.array32()?); + let signature = TMSig(header.take(64)?.try_into().unwrap()); + precommits.push((value_id, signature)); + if value_id == pending.decided_value_id && signature != TMSig::NIL { + signatures.push(FatPointerSignature { + pub_key: member.pub_key, + vote_signature: signature.0, + }); + } + } + header.finish()?; + + let mut vote_for_block_without_finalizer_public_key = [0u8; 76 - 32]; + pending + .decided_value_id + .0 + .write_to(&mut vote_for_block_without_finalizer_public_key[0..32]); + epoch + .height + .write_to(&mut vote_for_block_without_finalizer_public_key[32..]); + canonical_vote_round(round, true) + .ok_or_else(|| SignerError::Integrity("commit round is outside the canonical domain".into()))? + .write_to(&mut vote_for_block_without_finalizer_public_key[40..]); + Ok(( + round, + FatPointerToBftBlock { + vote_for_block_without_finalizer_public_key, + signatures, + }, + precommits, + )) +} + +pub(super) fn verify_proposal_signature_manifest( + hash_keys: &HashKeys, + epoch: &SignerEpochBinding, + roster: &[SortedRosterMember], + round: u32, + proposal_valid_round: i64, + proposal: &BlockValue, + proposal_id: ValueId, + proposal_sigs: &[TMSig], +) -> Result<(), SignerError> { + validate_epoch_consensus_roster(epoch, roster)?; + if round > MAX_CONSENSUS_ROUND { + return Err(SignerError::Integrity( + "proposal manifest round exceeds the canonical domain".into(), + )); + } + if proposal_valid_round < -1 || proposal_valid_round >= i64::from(round) { + return Err(SignerError::Integrity( + "proposal manifest valid-round is outside the canonical range".into(), + )); + } + if proposal.0.is_empty() || proposal.0.len() > MAX_PROPOSAL_BYTES { + return Err(SignerError::Integrity( + "proposal manifest value exceeds its bound".into(), + )); + } + if proposal.id_from_value(hash_keys) != proposal_id || proposal_id == ValueId::NIL { + return Err(SignerError::Integrity( + "proposal manifest bytes do not match the decided value ID".into(), + )); + } + let chunks_n = proposal.chunks_n(); + if chunks_n == 0 + || chunks_n > MAX_SIGNABLE_PARTS + || proposal_sigs.len() != chunks_n + || proposal_sigs.iter().any(|signature| *signature == TMSig::NIL) + { + return Err(SignerError::Integrity( + "proposal signature manifest is incomplete or noncanonical".into(), + )); + } + let (proposer_i, proposer_pub_key) = + TMState::proposer_from_height_round(hash_keys, roster, epoch.height, round); + if proposer_i.is_none() || proposer_pub_key == PubKeyID::NIL { + return Err(SignerError::Integrity( + "proposal manifest has no canonical proposer".into(), + )); + } + + let mut header = PacketProposalChunkHeader { + height: epoch.height, + round, + chunk_i: 0, + proposal_size: proposal + .0 + .len() + .try_into() + .map_err(|_| SignerError::Integrity("proposal size does not fit u32".into()))?, + proposal_id, + valid_round: proposal_valid_round, + }; + for (chunk_i, signature) in proposal_sigs.iter().enumerate() { + header.chunk_i = chunk_i + .try_into() + .map_err(|_| SignerError::Integrity("proposal chunk index does not fit u32".into()))?; + let (chunk_offset, chunk_size) = proposal.chunk_o_size(chunk_i); + let mut signable = vec![0u8; PacketProposalChunkHeader::SERIALIZED_SIZE + chunk_size]; + let header_len = header.write_to(&mut signable); + signable[header_len..] + .copy_from_slice(&proposal.0[chunk_offset..chunk_offset + chunk_size]); + signature + .verify_with_namespace(proposer_pub_key, &signable, &epoch.vote_namespace) + .map_err(|_| { + SignerError::Integrity( + "proposal signature manifest contains an invalid chunk signature".into(), + ) + })?; + } + Ok(()) +} + +pub fn verify_transition_certificate( + transition: &LockValidTransition, + epoch: &SignerEpochBinding, + hash_keys: &HashKeys, + roster: &[SortedRosterMember], +) -> Result<(), SignerError> { + validate_epoch_consensus_roster(epoch, roster)?; + validate_transition_shape(transition)?; + let valid_value = BlockValue(transition.valid_value.clone()); + if valid_value.id_from_value(hash_keys) != transition.valid_value_id { + return Err(SignerError::Integrity( + "valid value bytes do not match the persisted ID".into(), + )); + } + if transition.locked_round >= 0 { + let locked_value = BlockValue(transition.locked_value.clone()); + if locked_value.id_from_value(hash_keys) != transition.locked_value_id { + return Err(SignerError::Integrity( + "locked value bytes do not match the persisted ID".into(), + )); + } + } + + let mut decoder = Decoder::new(&transition.certificate); + if decoder.take(b"tenderlink-prevote-qc-v1".len())? != b"tenderlink-prevote-qc-v1" { + return Err(SignerError::Integrity("certificate domain mismatch".into())); + } + if decoder.u64()? != epoch.height { + return Err(SignerError::Integrity("certificate height mismatch".into())); + } + let round = decoder.u32()?; + if round > MAX_CONSENSUS_ROUND || i64::from(round) != transition.valid_round { + return Err(SignerError::Integrity( + "certificate round does not establish valid state".into(), + )); + } + let proposal_id = ValueId(decoder.array32()?); + if proposal_id != transition.valid_value_id { + return Err(SignerError::Integrity( + "certificate proposal ID mismatch".into(), + )); + } + if decoder.array32()? != epoch.vote_namespace { + return Err(SignerError::Integrity( + "certificate namespace mismatch".into(), + )); + } + let active_len: usize = decoder.u32()?.try_into().unwrap(); + if active_len != epoch.active_roster_len as usize || active_len != active_roster_len(roster) { + return Err(SignerError::Integrity( + "certificate active-roster length mismatch".into(), + )); + } + + let mut total_power = 0u64; + let mut yes_power = 0u64; + for member in &roster[..active_len] { + if PubKeyID(decoder.array32()?) != member.pub_key + || decoder.u64()? != member.stake + || decoder.u64()? != member.cumulative_stake + { + return Err(SignerError::Integrity("certificate roster mismatch".into())); + } + total_power = total_power + .checked_add(member.stake) + .ok_or_else(|| SignerError::Integrity("certificate total-power overflow".into()))?; + if total_power != member.cumulative_stake { + return Err(SignerError::Integrity( + "certificate cumulative stake mismatch".into(), + )); + } + let value_id = ValueId(decoder.array32()?); + let sig = TMSig(decoder.take(64)?.try_into().unwrap()); + if value_id == ValueId::NIL { + if sig != TMSig::NIL { + let signed = + make_vote_sign_datas(member.pub_key, false, epoch.height, round, value_id)[0]; + sig.verify_with_namespace(member.pub_key, &signed, &epoch.vote_namespace) + .map_err(|_| { + SignerError::Integrity( + "invalid NIL prevote signature in certificate".into(), + ) + })?; + } + continue; + } + if sig == TMSig::NIL { + return Err(SignerError::Integrity( + "certificate contains an unsigned non-NIL prevote".into(), + )); + } + let signed = make_vote_sign_datas(member.pub_key, false, epoch.height, round, value_id)[1]; + sig.verify_with_namespace(member.pub_key, &signed, &epoch.vote_namespace) + .map_err(|_| { + SignerError::Integrity("invalid YES prevote signature in certificate".into()) + })?; + if value_id == proposal_id { + yes_power = yes_power + .checked_add(member.stake) + .ok_or_else(|| SignerError::Integrity("certificate YES-power overflow".into()))?; + } + } + decoder.finish()?; + if total_power == 0 { + return Err(SignerError::Integrity("zero-power certificate".into())); + } + let quorum = quorum_threshold(total_power); + if yes_power < quorum { + return Err(SignerError::Integrity("certificate is below quorum".into())); + } + if transition.locked_round == transition.valid_round + && (transition.locked_value_id != transition.valid_value_id + || transition.locked_value != transition.valid_value) + { + return Err(SignerError::Integrity( + "same-round lock and valid values differ".into(), + )); + } + Ok(()) +} + +const RECORD_EPOCH: u8 = 1; +const RECORD_INTENT: u8 = 2; +const RECORD_STATE: u8 = 3; +const RECORD_POISON: u8 = 4; +const RECORD_COMMIT_INTENT: u8 = 5; +const RECORD_COMMIT_APPLIED: u8 = 6; +const RECORD_BOOTSTRAP_ORIGIN: u8 = 7; +/// Versioned commit intent that carries the exact proposer valid-round and the +/// ordered, complete proposal-chunk signature manifest. Tag 5 remains readable +/// only for compatibility with already-completed historical WAL epochs. +const RECORD_COMMIT_INTENT_V2: u8 = 8; + +fn encode_epoch(epoch: &SignerEpochBinding, authorized: bool) -> Vec { + let mut out = Vec::with_capacity(32 * 6 + 24); + out.extend_from_slice(&epoch.public_key.0); + out.extend_from_slice(&epoch.chain_id); + put_u64(&mut out, epoch.height); + out.extend_from_slice(&epoch.parent_commit); + out.extend_from_slice(&epoch.vote_namespace); + out.extend_from_slice(&epoch.consensus_config_hash); + out.extend_from_slice(&epoch.roster_hash); + put_u32(&mut out, epoch.roster_index); + put_u32(&mut out, epoch.active_roster_len); + out.push(authorized as u8); + out +} + +fn decode_epoch(payload: &[u8]) -> Result<(SignerEpochBinding, bool), SignerError> { + let mut decoder = Decoder::new(payload); + let epoch = SignerEpochBinding { + public_key: PubKeyID(decoder.array32()?), + chain_id: decoder.array32()?, + height: decoder.u64()?, + parent_commit: decoder.array32()?, + vote_namespace: decoder.array32()?, + consensus_config_hash: decoder.array32()?, + roster_hash: decoder.array32()?, + roster_index: decoder.u32()?, + active_roster_len: decoder.u32()?, + }; + let authorized = match decoder.u8()? { + 0 => false, + 1 => true, + _ => { + return Err(SignerError::Integrity( + "invalid epoch authorization flag".into(), + )) + } + }; + decoder.finish()?; + Ok((epoch, authorized)) +} + +fn encode_transition( + out: &mut Vec, + transition: &LockValidTransition, +) -> Result<(), SignerError> { + put_i64(out, transition.locked_round); + out.extend_from_slice(&transition.locked_value_id.0); + put_bytes(out, &transition.locked_value)?; + put_i64(out, transition.valid_round); + out.extend_from_slice(&transition.valid_value_id.0); + put_bytes(out, &transition.valid_value)?; + put_bytes(out, &transition.certificate) +} + +fn decode_transition(decoder: &mut Decoder<'_>) -> Result { + Ok(LockValidTransition { + locked_round: decoder.i64()?, + locked_value_id: ValueId(decoder.array32()?), + locked_value: decoder.bytes(MAX_PROPOSAL_BYTES)?, + valid_round: decoder.i64()?, + valid_value_id: ValueId(decoder.array32()?), + valid_value: decoder.bytes(MAX_PROPOSAL_BYTES)?, + certificate: decoder.bytes(MAX_RECORD_BYTES)?, + }) +} + +fn validate_transition_shape(next: &LockValidTransition) -> Result<(), SignerError> { + for (label, round, value_id, value) in [ + ( + "locked", + next.locked_round, + next.locked_value_id, + &next.locked_value, + ), + ( + "valid", + next.valid_round, + next.valid_value_id, + &next.valid_value, + ), + ] { + if round < -1 { + return Err(SignerError::Conflict(format!("invalid {label} round"))); + } + if round > i64::from(MAX_CONSENSUS_ROUND) { + return Err(SignerError::Conflict(format!( + "{label} round exceeds the canonical 31-bit domain" + ))); + } + if round == -1 && (value_id != ValueId::NIL || !value.is_empty()) { + return Err(SignerError::Conflict(format!( + "{label} value without a round" + ))); + } + if round >= 0 && (value_id == ValueId::NIL || value.is_empty()) { + return Err(SignerError::Conflict(format!( + "{label} round without a value" + ))); + } + } + if next.certificate.len() > MAX_RECORD_BYTES { + return Err(SignerError::Conflict("certificate exceeds bound".into())); + } + if next.locked_round > next.valid_round { + return Err(SignerError::Conflict( + "locked round is newer than valid round".into(), + )); + } + Ok(()) +} + +pub(super) fn validate_transition( + previous: Option<&LockValidTransition>, + next: &LockValidTransition, +) -> Result<(), SignerError> { + validate_transition_shape(next)?; + if let Some(previous) = previous { + for (label, old_round, old_value, old_bytes, new_round, new_value, new_bytes) in [ + ( + "locked", + previous.locked_round, + previous.locked_value_id, + &previous.locked_value, + next.locked_round, + next.locked_value_id, + &next.locked_value, + ), + ( + "valid", + previous.valid_round, + previous.valid_value_id, + &previous.valid_value, + next.valid_round, + next.valid_value_id, + &next.valid_value, + ), + ] { + if new_round < old_round { + return Err(SignerError::Conflict(format!( + "non-monotonic {label} round" + ))); + } + if new_round == old_round + && new_round >= 0 + && (new_value != old_value || new_bytes != old_bytes) + { + return Err(SignerError::Conflict(format!( + "same-round {label} value changed" + ))); + } + } + if next.locked_round > previous.locked_round + && (next.locked_round != next.valid_round + || next.locked_value_id != next.valid_value_id + || next.locked_value != next.valid_value) + { + return Err(SignerError::Conflict( + "a new lock is not established by the current valid certificate".into(), + )); + } + } else if next.locked_round >= 0 + && (next.locked_round != next.valid_round + || next.locked_value_id != next.valid_value_id + || next.locked_value != next.valid_value) + { + return Err(SignerError::Conflict( + "initial lock lacks its own exact quorum certificate".into(), + )); + } + Ok(()) +} + +fn encode_intent(intent: &SignedIntent) -> Result, SignerError> { + let mut out = Vec::new(); + match intent { + SignedIntent::Proposal { + round, + valid_round, + proposal_id, + proposal, + signable_parts, + } => { + out.push(SlotKind::Proposal as u8); + put_u32(&mut out, *round); + put_i64(&mut out, *valid_round); + out.extend_from_slice(&proposal_id.0); + put_bytes(&mut out, proposal)?; + let count: u32 = signable_parts + .len() + .try_into() + .map_err(|_| SignerError::Integrity("too many proposal parts".into()))?; + put_u32(&mut out, count); + for part in signable_parts { + put_bytes(&mut out, part)?; + } + } + SignedIntent::Vote { + round, + kind, + value_id, + signable, + transition, + } => { + out.push(*kind as u8); + put_u32(&mut out, *round); + out.extend_from_slice(&value_id.0); + put_bytes(&mut out, signable)?; + out.push(transition.is_some() as u8); + if let Some(transition) = transition { + encode_transition(&mut out, transition)?; + } + } + } + Ok(out) +} + +fn decode_intent(payload: &[u8]) -> Result { + let mut decoder = Decoder::new(payload); + let kind = decoder.u8()?; + let round = decoder.u32()?; + if round > MAX_CONSENSUS_ROUND { + return Err(SignerError::Integrity( + "round collides with vote-step high bit".into(), + )); + } + let intent = match kind { + 1 => { + let valid_round = decoder.i64()?; + let proposal_id = ValueId(decoder.array32()?); + let proposal = decoder.bytes(MAX_PROPOSAL_BYTES)?; + let count: usize = decoder.u32()?.try_into().unwrap(); + if count > MAX_SIGNABLE_PARTS { + return Err(SignerError::Integrity("too many proposal parts".into())); + } + let mut signable_parts = Vec::with_capacity(count); + for _ in 0..count { + signable_parts.push(decoder.bytes(MAX_SIGNABLE_BYTES)?); + } + SignedIntent::Proposal { + round, + valid_round, + proposal_id, + proposal, + signable_parts, + } + } + 2 | 3 => { + let value_id = ValueId(decoder.array32()?); + let signable = decoder.bytes(MAX_SIGNABLE_BYTES)?; + let transition = match decoder.u8()? { + 0 => None, + 1 => Some(decode_transition(&mut decoder)?), + _ => return Err(SignerError::Integrity("invalid transition flag".into())), + }; + SignedIntent::Vote { + round, + kind: if kind == 2 { + SlotKind::Prevote + } else { + SlotKind::Precommit + }, + value_id, + signable, + transition, + } + } + _ => return Err(SignerError::Integrity("unknown signer slot kind".into())), + }; + decoder.finish()?; + Ok(intent) +} + +fn frame_bytes( + kind: u8, + sequence: u64, + previous_hash: [u8; 32], + payload: &[u8], +) -> Result<(Vec, [u8; 32]), SignerError> { + if payload.len() > MAX_RECORD_BYTES { + return Err(SignerError::Integrity("record exceeds size limit".into())); + } + let payload_len: u32 = payload + .len() + .try_into() + .map_err(|_| SignerError::Integrity("record length overflow".into()))?; + let mut frame = Vec::with_capacity(88 + payload.len()); + frame.extend_from_slice(&WAL_MAGIC); + put_u16(&mut frame, WAL_VERSION); + frame.push(kind); + frame.push(0); + put_u32(&mut frame, payload_len); + put_u64(&mut frame, sequence); + frame.extend_from_slice(&previous_hash); + frame.extend_from_slice(payload); + let digest: [u8; 32] = blake3::hash(&frame).into(); + frame.extend_from_slice(&digest); + Ok((frame, digest)) +} + +fn apply_record(loaded: &mut LoadedWal, kind: u8, payload: &[u8]) -> Result<(), SignerError> { + match kind { + RECORD_BOOTSTRAP_ORIGIN => { + if loaded.epoch.is_some() || loaded.bootstrap_origin.is_some() { + return Err(SignerError::Integrity( + "bootstrap origin must be the first and only origin record".into(), + )); + } + if payload.len() != 32 { + return Err(SignerError::Integrity( + "bootstrap origin hash must be exactly 32 bytes".into(), + )); + } + let mut hash = [0u8; 32]; + hash.copy_from_slice(payload); + loaded.bootstrap_origin = Some(hash); + } + RECORD_EPOCH => { + let (epoch, authorized) = decode_epoch(payload)?; + if let Some(current) = loaded.epoch.as_ref() { + if !loaded.authorized { + return Err(SignerError::Integrity( + "unauthorized epoch cannot advance".into(), + )); + } + let expected = loaded.commit_applied_epoch.as_ref().ok_or_else(|| { + SignerError::Integrity( + "epoch advance without a durable commit-applied successor".into(), + ) + })?; + if &epoch != expected || !authorized { + return Err(SignerError::Integrity( + "next epoch differs from the durable commit-applied successor".into(), + )); + } + let pending = loaded.pending_commit.as_ref().ok_or_else(|| { + SignerError::Integrity("next epoch has no pending certified commit".into()) + })?; + validate_commit_successor(current, pending, &epoch)?; + } + loaded.epoch = Some(epoch); + loaded.authorized = authorized; + loaded.intents.clear(); + loaded.transition = None; + loaded.poisoned = None; + loaded.pending_commit = None; + loaded.commit_applied_epoch = None; + } + RECORD_INTENT => { + if loaded.epoch.is_none() { + return Err(SignerError::Integrity("intent before epoch".into())); + } + let intent = decode_intent(payload)?; + let slot = intent.slot(); + if let Some(existing) = loaded.intents.get(&slot) { + if existing != &intent { + return Err(SignerError::Integrity( + "conflicting durable intents in one slot".into(), + )); + } + } else { + if let SignedIntent::Vote { + transition: Some(transition), + .. + } = &intent + { + validate_transition(loaded.transition.as_ref(), transition)?; + loaded.transition = Some(transition.clone()); + } + loaded.intents.insert(slot, intent); + } + } + RECORD_STATE => { + if loaded.epoch.is_none() { + return Err(SignerError::Integrity("state before epoch".into())); + } + let mut decoder = Decoder::new(payload); + let transition = decode_transition(&mut decoder)?; + decoder.finish()?; + validate_transition(loaded.transition.as_ref(), &transition)?; + loaded.transition = Some(transition); + } + RECORD_POISON => { + let mut decoder = Decoder::new(payload); + let reason = String::from_utf8(decoder.bytes(4096)?) + .map_err(|_| SignerError::Integrity("poison reason is not UTF-8".into()))?; + decoder.finish()?; + loaded.poisoned = Some(reason); + loaded.authorized = false; + } + RECORD_COMMIT_INTENT => { + if loaded.epoch.is_none() { + return Err(SignerError::Integrity("commit record before epoch".into())); + } + if payload.len() > MAX_RECORD_BYTES { + return Err(SignerError::Integrity("commit record exceeds limit".into())); + } + if loaded.commit_applied_epoch.is_some() { + return Err(SignerError::Integrity( + "commit intent follows an unapplied epoch successor".into(), + )); + } + let mut decoder = Decoder::new(payload); + let decided_value_id = ValueId(decoder.array32()?); + if decided_value_id == ValueId::NIL { + return Err(SignerError::Integrity( + "commit intent cannot decide NIL".into(), + )); + } + let proposal = decoder.bytes(MAX_PROPOSAL_BYTES)?; + if proposal.is_empty() { + return Err(SignerError::Integrity( + "commit intent has an empty recovery proposal".into(), + )); + } + let certificate = decoder.bytes(MAX_RECORD_BYTES)?; + decoder.finish()?; + let digest: [u8; 32] = blake3::hash(payload).into(); + let pending = PendingCommit { + digest, + decided_value_id, + proposal, + certificate, + proposal_evidence: PendingProposalEvidence::LegacyUnavailable, + }; + if let Some(existing) = &loaded.pending_commit { + if existing != &pending { + return Err(SignerError::Integrity( + "conflicting commit intents in one epoch".into(), + )); + } + } + loaded.pending_commit = Some(pending); + } + RECORD_COMMIT_INTENT_V2 => { + if loaded.epoch.is_none() { + return Err(SignerError::Integrity("commit record before epoch".into())); + } + if payload.len() > MAX_RECORD_BYTES { + return Err(SignerError::Integrity("commit record exceeds limit".into())); + } + if loaded.commit_applied_epoch.is_some() { + return Err(SignerError::Integrity( + "commit intent follows an unapplied epoch successor".into(), + )); + } + let mut decoder = Decoder::new(payload); + let round = decoder.u32()?; + if round > MAX_CONSENSUS_ROUND { + return Err(SignerError::Integrity( + "commit round exceeds the canonical domain".into(), + )); + } + let valid_round = decoder.i64()?; + if valid_round < -1 || valid_round >= i64::from(round) { + return Err(SignerError::Integrity( + "commit proposal valid-round is outside the canonical range".into(), + )); + } + let decided_value_id = ValueId(decoder.array32()?); + if decided_value_id == ValueId::NIL { + return Err(SignerError::Integrity( + "commit intent cannot decide NIL".into(), + )); + } + let proposal = decoder.bytes(MAX_PROPOSAL_BYTES)?; + if proposal.is_empty() { + return Err(SignerError::Integrity( + "commit intent has an empty recovery proposal".into(), + )); + } + let signature_count: usize = decoder.u32()?.try_into().unwrap(); + if signature_count == 0 || signature_count > MAX_SIGNABLE_PARTS { + return Err(SignerError::Integrity( + "commit proposal-signature count exceeds its bound".into(), + )); + } + let mut proposal_sigs = Vec::with_capacity(signature_count); + for _ in 0..signature_count { + let signature = TMSig(decoder.take(64)?.try_into().unwrap()); + if signature == TMSig::NIL { + return Err(SignerError::Integrity( + "commit proposal-signature manifest is incomplete".into(), + )); + } + proposal_sigs.push(signature); + } + let certificate = decoder.bytes(MAX_RECORD_BYTES)?; + decoder.finish()?; + let digest: [u8; 32] = blake3::hash(payload).into(); + let pending = PendingCommit { + digest, + decided_value_id, + proposal, + certificate, + proposal_evidence: PendingProposalEvidence::Exact { + round, + valid_round, + proposal_sigs, + }, + }; + if let Some(existing) = &loaded.pending_commit { + if existing != &pending { + return Err(SignerError::Integrity( + "conflicting commit intents in one epoch".into(), + )); + } + } + loaded.pending_commit = Some(pending); + } + RECORD_COMMIT_APPLIED => { + let current = loaded.epoch.as_ref().ok_or_else(|| { + SignerError::Integrity("commit-applied marker before epoch".into()) + })?; + let pending = loaded.pending_commit.as_ref().ok_or_else(|| { + SignerError::Integrity("commit-applied marker without intent".into()) + })?; + if loaded.commit_applied_epoch.is_some() { + return Err(SignerError::Integrity( + "duplicate commit-applied successor".into(), + )); + } + if payload.len() < 32 || payload[..32] != pending.digest { + return Err(SignerError::Integrity( + "commit-applied digest mismatch".into(), + )); + } + let (next_epoch, authorized) = decode_epoch(&payload[32..])?; + if !authorized { + return Err(SignerError::Integrity( + "commit-applied successor is unauthorized".into(), + )); + } + validate_commit_successor(current, pending, &next_epoch)?; + loaded.commit_applied_epoch = Some(next_epoch); + } + _ => return Err(SignerError::Integrity("unknown WAL record kind".into())), + } + Ok(()) +} + +fn load_wal_bytes(bytes: &[u8]) -> Result { + const HEADER: usize = 56; + const DIGEST: usize = 32; + let mut loaded = LoadedWal::default(); + let mut offset = 0usize; + while offset < bytes.len() { + if bytes.len() - offset < HEADER { + loaded.clean_tail = false; + break; + } + let header = &bytes[offset..offset + HEADER]; + if header[..8] != WAL_MAGIC { + return Err(SignerError::Integrity("WAL magic mismatch".into())); + } + let mut decoder = Decoder::new(&header[8..]); + if decoder.u16()? != WAL_VERSION { + return Err(SignerError::Integrity("unsupported WAL version".into())); + } + let kind = decoder.u8()?; + if decoder.u8()? != 0 { + return Err(SignerError::Integrity("nonzero WAL reserved byte".into())); + } + let payload_len: usize = decoder.u32()?.try_into().unwrap(); + let sequence = decoder.u64()?; + let previous_hash = decoder.array32()?; + decoder.finish()?; + if payload_len > MAX_RECORD_BYTES { + return Err(SignerError::Integrity("WAL payload exceeds limit".into())); + } + if sequence != loaded.next_sequence { + return Err(SignerError::Integrity("WAL sequence gap or reorder".into())); + } + if previous_hash != loaded.last_hash { + return Err(SignerError::Integrity("WAL hash-chain mismatch".into())); + } + let frame_len = HEADER + .checked_add(payload_len) + .and_then(|n| n.checked_add(DIGEST)) + .ok_or_else(|| SignerError::Integrity("WAL frame length overflow".into()))?; + if bytes.len() - offset < frame_len { + loaded.clean_tail = false; + break; + } + let payload_start = offset + HEADER; + let payload_end = payload_start + payload_len; + let expected_digest: [u8; 32] = + bytes[payload_end..payload_end + DIGEST].try_into().unwrap(); + let actual_digest: [u8; 32] = blake3::hash(&bytes[offset..payload_end]).into(); + if expected_digest != actual_digest { + return Err(SignerError::Integrity("WAL record digest mismatch".into())); + } + apply_record(&mut loaded, kind, &bytes[payload_start..payload_end])?; + loaded.last_hash = expected_digest; + loaded.next_sequence += 1; + offset += frame_len; + } + Ok(loaded) +} + +#[derive(Debug, Default)] +struct LoadedAnchor { + next_sequence: u64, + last_wal_hash: [u8; 32], + last_hash: [u8; 32], + clean_tail: bool, +} + +fn anchor_frame_bytes( + sequence: u64, + wal_hash: [u8; 32], + previous_hash: [u8; 32], +) -> (Vec, [u8; 32]) { + let mut frame = Vec::with_capacity(116); + frame.extend_from_slice(&ANCHOR_MAGIC); + put_u16(&mut frame, WAL_VERSION); + put_u16(&mut frame, 0); + put_u64(&mut frame, sequence); + frame.extend_from_slice(&wal_hash); + frame.extend_from_slice(&previous_hash); + let digest: [u8; 32] = blake3::hash(&frame).into(); + frame.extend_from_slice(&digest); + (frame, digest) +} + +fn load_anchor_bytes(bytes: &[u8]) -> Result { + const FRAME: usize = 116; + let mut loaded = LoadedAnchor { + clean_tail: true, + ..LoadedAnchor::default() + }; + let mut offset = 0usize; + while offset < bytes.len() { + if bytes.len() - offset < FRAME { + loaded.clean_tail = false; + break; + } + let frame = &bytes[offset..offset + FRAME]; + if frame[..8] != ANCHOR_MAGIC { + return Err(SignerError::Integrity("anchor magic mismatch".into())); + } + let mut decoder = Decoder::new(&frame[8..84]); + if decoder.u16()? != WAL_VERSION { + return Err(SignerError::Integrity("anchor version mismatch".into())); + } + if decoder.u16()? != 0 { + return Err(SignerError::Integrity( + "anchor reserved field is nonzero".into(), + )); + } + let sequence = decoder.u64()?; + let wal_hash = decoder.array32()?; + let previous_hash = decoder.array32()?; + decoder.finish()?; + if sequence != loaded.next_sequence { + return Err(SignerError::Integrity( + "anchor sequence gap or reorder".into(), + )); + } + if previous_hash != loaded.last_hash { + return Err(SignerError::Integrity("anchor hash-chain mismatch".into())); + } + let expected: [u8; 32] = frame[84..116].try_into().unwrap(); + let actual: [u8; 32] = blake3::hash(&frame[..84]).into(); + if expected != actual { + return Err(SignerError::Integrity("anchor digest mismatch".into())); + } + loaded.next_sequence += 1; + loaded.last_wal_hash = wal_hash; + loaded.last_hash = expected; + offset += FRAME; + } + Ok(loaded) +} + +fn sync_parent(path: &Path) -> Result<(), SignerError> { + let parent = path + .parent() + .ok_or_else(|| SignerError::Integrity("WAL path has no parent".into()))?; + #[cfg(unix)] + { + File::open(parent)?.sync_all()?; + } + #[cfg(not(unix))] + { + let _ = parent; + } + Ok(()) +} + +fn open_locked_file(path: &Path) -> Result<(File, bool), SignerError> { + let existed = path.exists(); + let mut options = OpenOptions::new(); + options.read(true).append(true).create(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options + .mode(0o600) + .custom_flags(libc::O_CLOEXEC | libc::O_NOFOLLOW); + } + #[cfg(windows)] + { + use std::os::windows::fs::OpenOptionsExt; + options.share_mode(0); + } + let file = options.open(path)?; + let metadata = file.metadata()?; + if !metadata.file_type().is_file() { + return Err(SignerError::Integrity( + "WAL path is not a regular file".into(), + )); + } + #[cfg(unix)] + { + use nix::{fcntl::FlockArg, unistd::geteuid}; + use std::os::{fd::AsRawFd, unix::fs::MetadataExt}; + if metadata.uid() != geteuid().as_raw() { + return Err(SignerError::Integrity("WAL owner mismatch".into())); + } + if metadata.mode() & 0o077 != 0 { + return Err(SignerError::Integrity( + "WAL permissions are broader than 0600".into(), + )); + } + #[allow(deprecated)] + let lock_result = + nix::fcntl::flock(file.as_raw_fd(), FlockArg::LockExclusiveNonblock); + if lock_result.is_err() { + return Err(SignerError::Integrity( + "another signer process holds this WAL".into(), + )); + } + } + if !existed { + sync_parent(path)?; + } + Ok((file, !existed)) +} + +fn read_all(file: &mut File) -> Result, SignerError> { + file.seek(SeekFrom::Start(0))?; + let mut bytes = Vec::new(); + file.read_to_end(&mut bytes)?; + file.seek(SeekFrom::End(0))?; + Ok(bytes) +} + +#[derive(Debug)] +struct WalFiles { + wal: File, + anchor: File, + loaded: LoadedWal, + loaded_anchor: LoadedAnchor, + #[cfg(test)] + failpoint: Option, +} + +impl WalFiles { + #[cfg(test)] + fn fail_if(&mut self, point: WalFailpoint) -> Result<(), SignerError> { + if self.failpoint == Some(point) { + self.failpoint = None; + return Err(SignerError::Io(std::io::Error::new( + std::io::ErrorKind::Other, + format!("injected WAL fault at {point:?}"), + ))); + } + Ok(()) + } + + fn append(&mut self, kind: u8, payload: &[u8]) -> Result<[u8; 32], SignerError> { + if self.loaded.next_sequence != self.loaded_anchor.next_sequence { + return Err(SignerError::Integrity( + "WAL/anchor sequence mismatch".into(), + )); + } + let sequence = self.loaded.next_sequence; + let (frame, wal_hash) = frame_bytes(kind, sequence, self.loaded.last_hash, payload)?; + self.wal.write_all(&frame)?; + #[cfg(test)] + self.fail_if(WalFailpoint::AfterWalWrite)?; + self.wal.sync_all()?; + #[cfg(test)] + self.fail_if(WalFailpoint::AfterWalSync)?; + + let (anchor_frame, anchor_hash) = + anchor_frame_bytes(sequence, wal_hash, self.loaded_anchor.last_hash); + self.anchor.write_all(&anchor_frame)?; + #[cfg(test)] + self.fail_if(WalFailpoint::AfterAnchorWrite)?; + self.anchor.sync_all()?; + #[cfg(test)] + self.fail_if(WalFailpoint::AfterAnchorSync)?; + + apply_record(&mut self.loaded, kind, payload)?; + self.loaded.next_sequence += 1; + self.loaded.last_hash = wal_hash; + self.loaded_anchor.next_sequence += 1; + self.loaded_anchor.last_wal_hash = wal_hash; + self.loaded_anchor.last_hash = anchor_hash; + Ok(wal_hash) + } +} + +pub struct DurableSigner { + signing_key: SigningKey, + epoch: SignerEpochBinding, + status: SignerStatus, + files: Option, + pending_commit: Option, + intents: BTreeMap, + transition: Option, +} + +impl std::fmt::Debug for DurableSigner { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DurableSigner") + .field("public_key", &self.epoch.public_key) + .field("height", &self.epoch.height) + .field("status", &self.status) + .field("intent_count", &self.intents.len()) + .finish() + } +} + +impl DurableSigner { + pub fn observer_only( + signing_key: SigningKey, + epoch: SignerEpochBinding, + reason: impl Into, + ) -> Self { + Self { + signing_key, + epoch, + status: SignerStatus::ObserverOnly(reason.into()), + files: None, + pending_commit: None, + intents: BTreeMap::new(), + transition: None, + } + } + + #[cfg(any(test, feature = "simulation"))] + pub fn ephemeral_for_simulation(signing_key: SigningKey, epoch: SignerEpochBinding) -> Self { + Self { + signing_key, + epoch, + status: SignerStatus::Active, + files: None, + pending_commit: None, + intents: BTreeMap::new(), + transition: None, + } + } + + pub fn open( + signing_key: SigningKey, + config: DurableSignerConfig, + epoch: SignerEpochBinding, + ) -> Result { + let public_key = PubKeyID(VerificationKeyBytes::from(&signing_key).into()); + if public_key != epoch.public_key { + return Err(SignerError::Integrity( + "signing key does not match epoch public key".into(), + )); + } + if config.wal_path == config.anchor_path { + return Err(SignerError::Integrity( + "WAL and anchor paths must differ".into(), + )); + } + // Authority is a prerequisite to opening either journal. In particular, + // OpenOptions::create must never materialize an empty WAL/anchor pair that + // can later be mistaken for initialized signer history. + if !config.independent_anchor_authorized { + return Ok(Self::observer_only( + signing_key, + epoch, + "independent anti-rollback/key-fencing authority is absent", + )); + } + if epoch.height > 0 + && config + .non_genesis_bootstrap_receipt_hash + .filter(|hash| *hash != [0u8; 32]) + .is_none() + { + return Ok(Self::observer_only( + signing_key, + epoch, + "exact non-genesis bootstrap receipt is absent", + )); + } + + let (mut wal, _) = open_locked_file(&config.wal_path)?; + let (mut anchor, _) = open_locked_file(&config.anchor_path)?; + let wal_bytes = read_all(&mut wal)?; + let anchor_bytes = read_all(&mut anchor)?; + let loaded = load_wal_bytes(&wal_bytes)?; + let loaded_anchor = load_anchor_bytes(&anchor_bytes)?; + + let mut files = WalFiles { + wal, + anchor, + loaded, + loaded_anchor, + #[cfg(test)] + failpoint: None, + }; + let empty = files.loaded.next_sequence == 0 && files.loaded_anchor.next_sequence == 0; + if empty { + let bootstrap_origin = if epoch.height == 0 { + Some([0u8; 32]) + } else { + config + .non_genesis_bootstrap_receipt_hash + .filter(|hash| *hash != [0u8; 32]) + }; + if let Some(origin) = bootstrap_origin { + files.append(RECORD_BOOTSTRAP_ORIGIN, &origin)?; + } + let authorized = config.independent_anchor_authorized && bootstrap_origin.is_some(); + files.append(RECORD_EPOCH, &encode_epoch(&epoch, authorized))?; + } + + let pre_recovery_anchor_matches = files.loaded.next_sequence + == files.loaded_anchor.next_sequence + && files.loaded.last_hash == files.loaded_anchor.last_wal_hash; + let recovery_pair = files + .loaded + .epoch + .clone() + .zip(files.loaded.pending_commit.clone()); + let bootstrap_matches = match files.loaded.bootstrap_origin { + Some(origin) if origin == [0u8; 32] => true, + Some(origin) => config.non_genesis_bootstrap_receipt_hash == Some(origin), + None => false, + }; + if files.loaded.clean_tail + && files.loaded_anchor.clean_tail + && pre_recovery_anchor_matches + && files.loaded.poisoned.is_none() + && files.loaded.authorized + && config.independent_anchor_authorized + && bootstrap_matches + { + if let Some((current, pending)) = recovery_pair { + if validate_commit_successor(¤t, &pending, &epoch).is_ok() { + let may_finish = match files.loaded.commit_applied_epoch.as_ref() { + Some(expected) => expected == &epoch, + None => { + let mut applied = + Vec::with_capacity(32 + encode_epoch(&epoch, true).len()); + applied.extend_from_slice(&pending.digest); + applied.extend_from_slice(&encode_epoch(&epoch, true)); + files.append(RECORD_COMMIT_APPLIED, &applied)?; + true + } + }; + if may_finish { + files.append(RECORD_EPOCH, &encode_epoch(&epoch, true))?; + } + } + } + } + + let anchor_matches = files.loaded.next_sequence == files.loaded_anchor.next_sequence + && files.loaded.last_hash == files.loaded_anchor.last_wal_hash; + let epoch_matches = files.loaded.epoch.as_ref() == Some(&epoch); + let status = if !files.loaded.clean_tail || !files.loaded_anchor.clean_tail { + SignerStatus::ObserverOnly("torn WAL or anchor tail; unknown signing history".into()) + } else if !anchor_matches { + SignerStatus::ObserverOnly("WAL and independent high-water anchor disagree".into()) + } else if let Some(reason) = &files.loaded.poisoned { + SignerStatus::Poisoned(reason.clone()) + } else if !epoch_matches { + SignerStatus::ObserverOnly("durable signer epoch does not match committed state".into()) + } else if epoch.roster_index >= epoch.active_roster_len { + SignerStatus::ObserverOnly("signing key is not in the active epoch roster".into()) + } else if let Some(pending) = files.loaded.pending_commit.as_ref() { + match &pending.proposal_evidence { + PendingProposalEvidence::Exact { .. } => SignerStatus::ReconciliationRequired( + pending.digest, + PENDING_COMMIT_RECOVERY_REASON.into(), + ), + PendingProposalEvidence::LegacyUnavailable => { + SignerStatus::ObserverOnly(LEGACY_PENDING_COMMIT_REASON.into()) + } + } + } else if !bootstrap_matches { + SignerStatus::ObserverOnly( + "genesis origin or exact non-genesis bootstrap receipt is absent".into(), + ) + } else if !config.independent_anchor_authorized { + SignerStatus::ObserverOnly( + "independent anti-rollback/key-fencing authority is absent".into(), + ) + } else if !files.loaded.authorized { + SignerStatus::ObserverOnly("unknown pre-WAL key history at a non-genesis height".into()) + } else { + SignerStatus::Active + }; + + let intents = files.loaded.intents.clone(); + let transition = files.loaded.transition.clone(); + let pending_commit = files.loaded.pending_commit.clone(); + Ok(Self { + signing_key, + epoch, + status, + files: Some(files), + pending_commit, + intents, + transition, + }) + } + + pub fn open_or_observer( + signing_key: SigningKey, + config: DurableSignerConfig, + epoch: SignerEpochBinding, + ) -> Self { + let observer_key = signing_key.clone(); + match Self::open(signing_key, config, epoch.clone()) { + Ok(signer) => signer, + Err(error) => Self::observer_only( + observer_key, + epoch, + format!("durable signer open failed; consensus signing disabled: {error}"), + ), + } + } + + pub fn status(&self) -> &SignerStatus { + &self.status + } + pub fn is_active(&self) -> bool { + self.status == SignerStatus::Active + } + pub fn epoch(&self) -> &SignerEpochBinding { + &self.epoch + } + + // Observer nodes still need to authenticate their transport identity to collect + // evidence. Restrict that escape hatch to an already domain-separated 32-byte + // digest, so no caller can feed raw proposal or vote bytes through it. + pub(super) fn sign_auxiliary_digest(&self, digest: &[u8; 32]) -> TMSig { + TMSig(self.signing_key.sign(digest).to_bytes()) + } + + pub fn fail_closed(&mut self, reason: impl Into) { + let reason = reason.into(); + self.poison(reason); + } + + /// Latch an ambiguous/transient decision-application boundary without + /// writing a poison record. The supplied digest must name the one exact + /// durable pending commit. Restart derives the same state from that WAL + /// record, so this cannot grant authority or select a different value. + pub fn require_reconciliation( + &mut self, + commit_intent_digest: [u8; 32], + reason: impl Into, + ) -> Result<(), SignerError> { + let reason = reason.into(); + let Some(pending) = self.pending_commit.as_ref() else { + self.status = SignerStatus::ObserverOnly(format!( + "{reason}; no durable pending commit exists" + )); + return Err(SignerError::Integrity( + "cannot enter reconciliation without a pending commit".into(), + )); + }; + if pending.digest != commit_intent_digest { + let conflict = + "reconciliation digest does not match the durable pending commit".to_string(); + self.poison(conflict.clone()); + return Err(SignerError::Conflict(conflict)); + } + if !matches!( + &pending.proposal_evidence, + PendingProposalEvidence::Exact { .. } + ) { + self.status = SignerStatus::ObserverOnly(LEGACY_PENDING_COMMIT_REASON.into()); + return Err(SignerError::Integrity(LEGACY_PENDING_COMMIT_REASON.into())); + } + self.status = SignerStatus::ReconciliationRequired(commit_intent_digest, reason); + Ok(()) + } + + fn require_active(&self) -> Result<(), SignerError> { + match &self.status { + SignerStatus::Active => Ok(()), + SignerStatus::ObserverOnly(reason) => Err(SignerError::ObserverOnly(reason.clone())), + SignerStatus::ReconciliationRequired(digest, reason) => Err( + SignerError::ReconciliationRequired(*digest, reason.clone()), + ), + SignerStatus::Poisoned(reason) => Err(SignerError::Conflict(reason.clone())), + } + } + + fn poison(&mut self, reason: String) { + let mut payload = Vec::new(); + let durable_reason = if put_bytes(&mut payload, reason.as_bytes()).is_ok() { + self.files + .as_mut() + .and_then(|files| files.append(RECORD_POISON, &payload).ok()) + .is_some() + } else { + false + }; + self.status = SignerStatus::Poisoned(if durable_reason { + reason + } else { + format!("{reason}; poison persistence failed") + }); + } + + fn prepare_intent(&mut self, intent: SignedIntent) -> Result<(), SignerError> { + self.require_active()?; + let slot = intent.slot(); + if slot.round > MAX_CONSENSUS_ROUND { + self.poison("round collides with the vote-step high bit".into()); + return Err(SignerError::Conflict( + "round collides with the vote-step high bit".into(), + )); + } + if let Some(existing) = self.intents.get(&slot) { + if existing == &intent { + return Ok(()); + } + let reason = format!("different exact bytes requested for slot {:?}", slot); + self.poison(reason.clone()); + return Err(SignerError::Conflict(reason)); + } + if let SignedIntent::Vote { + transition: Some(transition), + .. + } = &intent + { + if let Err(error) = validate_transition(self.transition.as_ref(), transition) { + let reason = error.to_string(); + self.poison(reason.clone()); + return Err(SignerError::Conflict(reason)); + } + } + let payload = encode_intent(&intent)?; + if let Some(files) = &mut self.files { + if let Err(error) = files.append(RECORD_INTENT, &payload) { + self.status = SignerStatus::ObserverOnly(format!( + "durability failure before signing: {error}" + )); + return Err(error); + } + } + if let SignedIntent::Vote { + transition: Some(transition), + .. + } = &intent + { + self.transition = Some(transition.clone()); + } + self.intents.insert(slot, intent); + Ok(()) + } + + pub fn sign_proposal( + &mut self, + hash_keys: &HashKeys, + roster: &[SortedRosterMember], + round: u32, + valid_round: i64, + proposal_id: ValueId, + proposal: &[u8], + signable_parts: &[Vec], + ) -> Result, SignerError> { + self.require_active()?; + if let Err(error) = validate_epoch_roster(&self.epoch, roster) { + let reason = format!("proposal roster failed epoch binding: {error}"); + self.poison(reason.clone()); + return Err(SignerError::Conflict(reason)); + } + if round > MAX_CONSENSUS_ROUND { + self.poison("proposal round collides with the vote-step high bit".into()); + return Err(SignerError::Conflict( + "proposal round collides with the vote-step high bit".into(), + )); + } + if valid_round < -1 || valid_round >= i64::from(round) { + self.poison("proposal valid-round is outside the safe range".into()); + return Err(SignerError::Conflict( + "proposal valid-round is outside the safe range".into(), + )); + } + if proposal.is_empty() + || proposal.len() > MAX_PROPOSAL_BYTES + || signable_parts.len() > MAX_SIGNABLE_PARTS + { + return Err(SignerError::Integrity( + "proposal manifest exceeds bound".into(), + )); + } + if signable_parts + .iter() + .any(|part| part.len() > MAX_SIGNABLE_BYTES) + { + return Err(SignerError::Integrity( + "proposal signable part exceeds bound".into(), + )); + } + let computed_id = BlockValue(proposal.to_vec()).id_from_value(hash_keys); + if computed_id != proposal_id { + self.poison("proposal bytes do not match the requested value ID".into()); + return Err(SignerError::Conflict( + "proposal bytes do not match the requested value ID".into(), + )); + } + let mut expected_parts = Vec::with_capacity(BlockValue(proposal.to_vec()).chunks_n()); + let proposal_value = BlockValue(proposal.to_vec()); + let mut header = PacketProposalChunkHeader { + height: self.epoch.height, + round, + chunk_i: 0, + proposal_size: proposal.len().try_into().unwrap(), + proposal_id, + valid_round, + }; + let mut buffer = [0u8; 2048]; + for chunk_i in 0..proposal_value.chunks_n() { + header.chunk_i = chunk_i.try_into().unwrap(); + let mut offset = header.write_to(&mut buffer); + let (chunk_offset, chunk_size) = proposal_value.chunk_o_size(chunk_i); + offset += + proposal[chunk_offset..chunk_offset + chunk_size].write_to(&mut buffer[offset..]); + expected_parts.push(buffer[..offset].to_vec()); + } + if signable_parts != expected_parts { + self.poison("proposal signable chunks are not the canonical manifest".into()); + return Err(SignerError::Conflict( + "proposal signable chunks are not the canonical manifest".into(), + )); + } + let intent = SignedIntent::Proposal { + round, + valid_round, + proposal_id, + proposal: proposal.to_vec(), + signable_parts: signable_parts.to_vec(), + }; + self.prepare_intent(intent)?; + Ok(signable_parts + .iter() + .map(|part| { + TMSig(sign_with_namespace( + &self.signing_key, + part, + &self.epoch.vote_namespace, + )) + }) + .collect()) + } + + pub fn sign_vote( + &mut self, + hash_keys: &HashKeys, + roster: &[SortedRosterMember], + round: u32, + is_precommit: bool, + value_id: ValueId, + signable: &[u8], + transition: Option, + ) -> Result { + self.require_active()?; + if let Err(error) = validate_epoch_roster(&self.epoch, roster) { + let reason = format!("vote roster failed epoch binding: {error}"); + self.poison(reason.clone()); + return Err(SignerError::Conflict(reason)); + } + if round > MAX_CONSENSUS_ROUND { + self.poison("vote round collides with the vote-step high bit".into()); + return Err(SignerError::Conflict( + "vote round collides with the vote-step high bit".into(), + )); + } + if signable.len() > MAX_SIGNABLE_BYTES { + return Err(SignerError::Integrity( + "vote signable bytes exceed bound".into(), + )); + } + let expected = make_vote_sign_datas( + self.epoch.public_key, + is_precommit, + self.epoch.height, + round, + value_id, + )[1]; + if signable != expected { + self.poison("vote signable bytes do not match the epoch/round/step/value".into()); + return Err(SignerError::Conflict( + "vote signable bytes do not match the epoch/round/step/value".into(), + )); + } + match (is_precommit, value_id == ValueId::NIL, transition.as_ref()) { + (true, false, Some(transition)) => { + if transition.locked_round != i64::from(round) + || transition.locked_value_id != value_id + || transition.valid_round != i64::from(round) + || transition.valid_value_id != value_id + { + self.poison("precommit transition does not exactly bind this vote".into()); + return Err(SignerError::Conflict( + "precommit transition does not exactly bind this vote".into(), + )); + } + if let Err(error) = + verify_transition_certificate(transition, &self.epoch, hash_keys, roster) + { + let reason = format!( + "precommit transition certificate failed signer verification: {error}" + ); + self.poison(reason.clone()); + return Err(SignerError::Conflict(reason)); + } + } + (true, false, None) => { + self.poison("non-NIL precommit is missing its durable quorum transition".into()); + return Err(SignerError::Conflict( + "non-NIL precommit is missing its durable quorum transition".into(), + )); + } + (_, _, Some(_)) => { + self.poison("precommit transition does not exactly bind this vote".into()); + return Err(SignerError::Conflict( + "precommit transition does not exactly bind this vote".into(), + )); + } + (_, _, None) => {} + } + let intent = SignedIntent::Vote { + round, + kind: if is_precommit { + SlotKind::Precommit + } else { + SlotKind::Prevote + }, + value_id, + signable: signable.to_vec(), + transition, + }; + self.prepare_intent(intent)?; + Ok(TMSig(sign_with_namespace( + &self.signing_key, + signable, + &self.epoch.vote_namespace, + ))) + } + + pub fn persist_transition( + &mut self, + transition: LockValidTransition, + hash_keys: &HashKeys, + roster: &[SortedRosterMember], + ) -> Result<(), SignerError> { + self.require_active()?; + if let Err(error) = + verify_transition_certificate(&transition, &self.epoch, hash_keys, roster) + { + let reason = + format!("state transition certificate failed signer verification: {error}"); + self.poison(reason.clone()); + return Err(SignerError::Conflict(reason)); + } + if let Err(error) = validate_transition(self.transition.as_ref(), &transition) { + let reason = error.to_string(); + self.poison(reason.clone()); + return Err(SignerError::Conflict(reason)); + } + if self.transition.as_ref() == Some(&transition) { + return Ok(()); + } + let mut payload = Vec::new(); + encode_transition(&mut payload, &transition)?; + if let Some(files) = &mut self.files { + if let Err(error) = files.append(RECORD_STATE, &payload) { + self.status = SignerStatus::ObserverOnly(format!( + "durability failure before state use: {error}" + )); + return Err(error); + } + } + self.transition = Some(transition); + Ok(()) + } + + pub fn begin_commit( + &mut self, + hash_keys: &HashKeys, + round: u32, + decided_value_id: ValueId, + proposal: &BlockValue, + proposal_valid_round: i64, + proposal_sigs: &[TMSig], + commit_certificate: &[u8], + roster: &[SortedRosterMember], + ) -> Result<[u8; 32], SignerError> { + self.require_active()?; + if proposal.id_from_value(hash_keys) != decided_value_id { + let reason = "commit proposal bytes do not match the decided value ID".to_string(); + self.poison(reason.clone()); + return Err(SignerError::Conflict(reason)); + } + if let Err(error) = verify_proposal_signature_manifest( + hash_keys, + &self.epoch, + roster, + round, + proposal_valid_round, + proposal, + decided_value_id, + proposal_sigs, + ) { + let reason = format!("commit proposal manifest failed signer verification: {error}"); + self.poison(reason.clone()); + return Err(SignerError::Conflict(reason)); + } + if let Err(error) = verify_precommit_certificate( + commit_certificate, + round, + decided_value_id, + &self.epoch, + roster, + ) { + let reason = format!("commit certificate failed signer verification: {error}"); + self.poison(reason.clone()); + return Err(SignerError::Conflict(reason)); + } + let payload = encode_commit_intent( + round, + decided_value_id, + &proposal.0, + proposal_valid_round, + proposal_sigs, + commit_certificate, + )?; + let digest: [u8; 32] = blake3::hash(&payload).into(); + if let Some(files) = &mut self.files { + if let Err(error) = files.append(RECORD_COMMIT_INTENT_V2, &payload) { + self.status = SignerStatus::ObserverOnly(format!( + "commit intent durability failure: {error}" + )); + return Err(error); + } + } + self.pending_commit = Some(PendingCommit { + digest, + decided_value_id, + proposal: proposal.0.clone(), + certificate: commit_certificate.to_vec(), + proposal_evidence: PendingProposalEvidence::Exact { + round, + valid_round: proposal_valid_round, + proposal_sigs: proposal_sigs.to_vec(), + }, + }); + Ok(digest) + } + + /// Persist a new commit intent for an active signer, or resume the one exact + /// pending intent after the durable PoS store has been idempotently reconciled. + /// Ordinary observers return `None` and never gain signing authority here. + pub fn begin_or_resume_commit( + &mut self, + hash_keys: &HashKeys, + round: u32, + decided_value_id: ValueId, + proposal: &BlockValue, + proposal_valid_round: i64, + proposal_sigs: &[TMSig], + commit_certificate: &[u8], + roster: &[SortedRosterMember], + ) -> Result, SignerError> { + if self.is_active() { + return self + .begin_commit( + hash_keys, + round, + decided_value_id, + proposal, + proposal_valid_round, + proposal_sigs, + commit_certificate, + roster, + ) + .map(Some); + } + let recovery_digest = match &self.status { + SignerStatus::ReconciliationRequired(digest, _) => Some(*digest), + _ => None, + }; + let Some(recovery_digest) = recovery_digest else { + return Ok(None); + }; + let pending = self + .pending_commit + .clone() + .ok_or_else(|| { + SignerError::Integrity("recovery observer has no durable commit intent".into()) + })?; + verify_precommit_certificate( + commit_certificate, + round, + decided_value_id, + &self.epoch, + roster, + )?; + verify_proposal_signature_manifest( + hash_keys, + &self.epoch, + roster, + round, + proposal_valid_round, + proposal, + decided_value_id, + proposal_sigs, + )?; + if proposal.id_from_value(hash_keys) != decided_value_id { + let reason = "observed recovery proposal does not match the pending value ID".to_string(); + self.poison(reason.clone()); + return Err(SignerError::Conflict(reason)); + } + let payload = encode_commit_intent( + round, + decided_value_id, + &proposal.0, + proposal_valid_round, + proposal_sigs, + commit_certificate, + )?; + let digest: [u8; 32] = blake3::hash(&payload).into(); + if digest != recovery_digest { + let reason = "observed decision digest conflicts with reconciliation state".to_string(); + self.poison(reason.clone()); + return Err(SignerError::Conflict(reason)); + } + if pending.decided_value_id != decided_value_id + || pending.proposal != proposal.0 + || pending.certificate != commit_certificate + || pending.proposal_evidence + != (PendingProposalEvidence::Exact { + round, + valid_round: proposal_valid_round, + proposal_sigs: proposal_sigs.to_vec(), + }) + || pending.digest != digest + { + let reason = "observed decision conflicts with the pending durable commit".to_string(); + self.poison(reason.clone()); + return Err(SignerError::Conflict(reason)); + } + Ok(Some(digest)) + } + + /// Return the one exact certified decision needed to reconcile a crash that + /// happened after commit-intent durability but before PoS-store durability. + /// This never authorizes signing; the caller must apply and durably reread the + /// decision, then call `complete_commit` before the signer can become active. + pub(super) fn pending_commit_recovery( + &self, + hash_keys: &HashKeys, + roster: &[SortedRosterMember], + ) -> Result, SignerError> { + let Some(pending) = self.pending_commit.as_ref() else { + return Ok(None); + }; + let status_digest = match &self.status { + SignerStatus::ReconciliationRequired(digest, _) => *digest, + _ => { + return Err(SignerError::Integrity( + "pending commit exists outside exact reconciliation state".into(), + )) + } + }; + if status_digest != pending.digest { + return Err(SignerError::Integrity( + "reconciliation status digest differs from the pending commit".into(), + )); + } + let (recorded_round, proposal_valid_round, proposal_sigs) = + match &pending.proposal_evidence { + PendingProposalEvidence::Exact { + round, + valid_round, + proposal_sigs, + } => (*round, *valid_round, proposal_sigs.clone()), + PendingProposalEvidence::LegacyUnavailable => { + return Err(SignerError::Integrity(LEGACY_PENDING_COMMIT_REASON.into())) + } + }; + let proposal = BlockValue(pending.proposal.clone()); + if proposal.id_from_value(hash_keys) != pending.decided_value_id { + return Err(SignerError::Integrity( + "durable recovery proposal bytes do not match the certified value ID".into(), + )); + } + let (certificate_round, fat_pointer, precommits) = + recover_fat_pointer_from_commit_certificate(pending, &self.epoch, roster)?; + if certificate_round != recorded_round { + return Err(SignerError::Integrity( + "durable proposal manifest round differs from its commit certificate".into(), + )); + } + verify_proposal_signature_manifest( + hash_keys, + &self.epoch, + roster, + recorded_round, + proposal_valid_round, + &proposal, + pending.decided_value_id, + &proposal_sigs, + )?; + + let active_len = active_roster_len(roster); + if precommits.len() != active_len { + return Err(SignerError::Integrity( + "recovered precommit evidence length differs from the active roster".into(), + )); + } + let mut msg_val_sigs = vec![[(ValueId::NIL, TMSig::NIL); 2]; roster.len()]; + let mut precommit_power = 0u64; + let mut yes_precommit_power = 0u64; + for (index, (value_id, signature)) in precommits.into_iter().enumerate() { + msg_val_sigs[index][1] = (value_id, signature); + if signature != TMSig::NIL { + precommit_power = precommit_power + .checked_add(roster[index].stake) + .ok_or_else(|| SignerError::Integrity("precommit power overflow".into()))?; + if value_id == pending.decided_value_id { + yes_precommit_power = yes_precommit_power + .checked_add(roster[index].stake) + .ok_or_else(|| { + SignerError::Integrity("YES precommit power overflow".into()) + })?; + } + } + } + let round_data = RoundData { + height: self.epoch.height, + round: recorded_round, + proposal, + proposal_valid_round, + proposal_sigs_n: proposal_sigs.len(), + proposal_sigs, + proposal_id: pending.decided_value_id, + msg_val_sigs, + roster: roster.to_vec(), + counts: ConsensusCounts { + anys: precommit_power, + prevotes: 0, + nil_prevotes: 0, + yes_prevotes: 0, + precommits: precommit_power, + yes_precommits: yes_precommit_power, + }, + vote_namespace: self.epoch.vote_namespace, + ..RoundData::EMPTY + }; + verify_reconstructed_precommit_quorum(&round_data, roster) + .map_err(SignerError::Integrity)?; + Ok(Some(PendingCommitRecovery { + digest: pending.digest, + proposal: round_data.proposal.clone(), + proposal_valid_round: round_data.proposal_valid_round, + proposal_sigs: round_data.proposal_sigs.clone(), + round_data, + fat_pointer, + })) + } + + pub fn complete_commit( + &mut self, + commit_intent_digest: [u8; 32], + durable_store_readback: [u8; 32], + next_vote_namespace: [u8; 32], + next_roster: &[SortedRosterMember], + ) -> Result<(), SignerError> { + match self.status.clone() { + SignerStatus::Active => {} + SignerStatus::ReconciliationRequired(digest, _) + if digest == commit_intent_digest => {} + SignerStatus::ReconciliationRequired(_, _) => { + let reason = + "commit completion digest conflicts with reconciliation state".to_string(); + self.poison(reason.clone()); + return Err(SignerError::Conflict(reason)); + } + _ => self.require_active()?, + } + if let Some(files) = &self.files { + let Some(pending) = files.loaded.pending_commit.as_ref() else { + let reason = "commit completion has no durable pending intent".to_string(); + self.poison(reason.clone()); + return Err(SignerError::Conflict(reason)); + }; + if pending.digest != commit_intent_digest { + let reason = + "commit completion does not match the durable pending intent".to_string(); + self.poison(reason.clone()); + return Err(SignerError::Conflict(reason)); + } + if pending.decided_value_id.0 != durable_store_readback { + let reason = + "durable store readback is not the value authorized by the commit certificate" + .to_string(); + self.poison(reason.clone()); + return Err(SignerError::Conflict(reason)); + } + } + let next_epoch = (|| -> Result { + let active_len = active_roster_len(next_roster); + let roster_hash = canonical_roster_hash(next_roster)?; + let roster_index = + roster_i_from_pub_key(&next_roster[..active_len], self.epoch.public_key) + .map(|index| index.try_into().unwrap()) + .unwrap_or(u32::MAX); + Ok(SignerEpochBinding { + public_key: self.epoch.public_key, + chain_id: self.epoch.chain_id, + height: self + .epoch + .height + .checked_add(1) + .ok_or_else(|| SignerError::Integrity("signer height overflow".into()))?, + parent_commit: durable_store_readback, + vote_namespace: next_vote_namespace, + consensus_config_hash: self.epoch.consensus_config_hash, + roster_hash, + roster_index, + active_roster_len: active_len + .try_into() + .map_err(|_| SignerError::Integrity("active roster too large".into()))?, + }) + })(); + let next_epoch = match next_epoch { + Ok(next_epoch) => next_epoch, + Err(error) => { + self.status = SignerStatus::ReconciliationRequired( + commit_intent_digest, + format!("could not derive the exact successor epoch: {error}"), + ); + return Err(error); + } + }; + let mut applied = Vec::with_capacity(32 + encode_epoch(&next_epoch, true).len()); + applied.extend_from_slice(&commit_intent_digest); + applied.extend_from_slice(&encode_epoch(&next_epoch, true)); + if let Some(files) = &mut self.files { + if let Err(error) = files.append(RECORD_COMMIT_APPLIED, &applied) { + self.status = SignerStatus::ReconciliationRequired( + commit_intent_digest, + format!("commit-applied durability failure: {error}"), + ); + return Err(error); + } + #[cfg(test)] + if let Err(error) = files.fail_if(WalFailpoint::AfterCommitApplied) { + self.status = SignerStatus::ReconciliationRequired( + commit_intent_digest, + format!("injected crash after commit-applied durability: {error}"), + ); + return Err(error); + } + if let Err(error) = files.append(RECORD_EPOCH, &encode_epoch(&next_epoch, true)) { + self.status = SignerStatus::ReconciliationRequired( + commit_intent_digest, + format!("next-epoch durability failure: {error}"), + ); + return Err(error); + } + } + self.epoch = next_epoch; + self.pending_commit = None; + self.intents.clear(); + self.transition = None; + self.status = if self.epoch.roster_index < self.epoch.active_roster_len { + SignerStatus::Active + } else { + SignerStatus::ObserverOnly( + "signing key is not in the active epoch roster".into(), + ) + }; + Ok(()) + } + + pub(super) fn replay_intents(&self) -> Vec { + self.intents.values().cloned().collect() + } + + pub fn durable_transition(&self) -> Option<&LockValidTransition> { + self.transition.as_ref() + } + + #[cfg(test)] + pub(super) fn append_legacy_pending_commit_for_test( + &mut self, + decided_value_id: ValueId, + proposal: &BlockValue, + certificate: &[u8], + ) -> Result<[u8; 32], SignerError> { + self.require_active()?; + let payload = encode_legacy_commit_intent( + decided_value_id, + &proposal.0, + certificate, + )?; + let digest: [u8; 32] = blake3::hash(&payload).into(); + self.files + .as_mut() + .ok_or_else(|| SignerError::Integrity("test requires a durable WAL".into()))? + .append(RECORD_COMMIT_INTENT, &payload)?; + self.pending_commit = Some(PendingCommit { + digest, + decided_value_id, + proposal: proposal.0.clone(), + certificate: certificate.to_vec(), + proposal_evidence: PendingProposalEvidence::LegacyUnavailable, + }); + Ok(digest) + } + + #[cfg(test)] + pub(super) fn set_failpoint(&mut self, point: WalFailpoint) { + self.files + .as_mut() + .expect("durable signer required for WAL failpoint") + .failpoint = Some(point); + } +} diff --git a/tenderlink/src/signer_wal_tests.rs b/tenderlink/src/signer_wal_tests.rs new file mode 100644 index 00000000..6031bf82 --- /dev/null +++ b/tenderlink/src/signer_wal_tests.rs @@ -0,0 +1,1464 @@ +use super::*; + +use std::{ + fs::OpenOptions, + io::{Read, Seek, SeekFrom, Write}, + path::PathBuf, + sync::atomic::{AtomicU64, Ordering}, +}; + +static NEXT_TEST_ID: AtomicU64 = AtomicU64::new(0); + +#[test] +fn weighted_quorum_is_n_minus_max_faulty_for_every_remainder() { + assert_eq!( + (0u64..=10).map(|n| quorum_threshold(n)).collect::>(), + vec![0, 1, 2, 3, 3, 4, 5, 5, 6, 7, 7], + ); + assert_eq!(quorum_threshold(6), 5); + assert!(2 * quorum_threshold(6) > 6 + ((6 - 1) / 3)); +} + +#[test] +fn canonical_roster_rejects_ambiguity_and_hashes_the_inactive_tail() { + let mut keys: Vec = (1u8..=102) + .map(|seed| SigningKey::from([seed; 32])) + .collect(); + keys.sort_by_key(|key| std::cmp::Reverse(PubKeyID(VerificationKeyBytes::from(key).into()))); + let mut cumulative_stake = 0u64; + let roster: Vec = keys + .iter() + .take(101) + .enumerate() + .map(|(index, key)| { + let stake = 101 - index as u64; + cumulative_stake += stake; + SortedRosterMember { + pub_key: PubKeyID(VerificationKeyBytes::from(key).into()), + stake, + cumulative_stake, + } + }) + .collect(); + let original_hash = canonical_roster_hash(&roster).unwrap(); + assert_eq!(active_roster_len(&roster), 100); + + let mut duplicate = roster.clone(); + duplicate[1].pub_key = duplicate[0].pub_key; + assert!(canonical_roster_hash(&duplicate).is_err()); + + let mut bad_cumulative = roster.clone(); + bad_cumulative[50].cumulative_stake += 1; + assert!(canonical_roster_hash(&bad_cumulative).is_err()); + + let mut wrong_order = roster.clone(); + wrong_order.swap(0, 1); + let mut cumulative = 0; + for member in &mut wrong_order { + cumulative += member.stake; + member.cumulative_stake = cumulative; + } + assert!(canonical_roster_hash(&wrong_order).is_err()); + + let mut changed_tail = roster; + changed_tail[100].pub_key = PubKeyID(VerificationKeyBytes::from(&keys[101]).into()); + assert_ne!(canonical_roster_hash(&changed_tail).unwrap(), original_hash); +} + +#[test] +fn a_new_lock_must_be_the_value_certified_at_its_round() { + let old = LockValidTransition { + locked_round: 6, + locked_value_id: ValueId([1u8; 32]), + locked_value: vec![1u8; 32], + valid_round: 6, + valid_value_id: ValueId([1u8; 32]), + valid_value: vec![1u8; 32], + certificate: vec![1u8; 32], + }; + let uncertified_new_lock = LockValidTransition { + locked_round: 7, + locked_value_id: ValueId([2u8; 32]), + locked_value: vec![2u8; 32], + valid_round: 8, + valid_value_id: ValueId([3u8; 32]), + valid_value: vec![3u8; 32], + certificate: vec![3u8; 32], + }; + assert!(matches!( + validate_transition(Some(&old), &uncertified_new_lock), + Err(SignerError::Conflict(reason)) if reason.contains("not established") + )); +} + +#[test] +fn six_unit_stake_certificate_requires_five_yes_votes() { + let mut keys: Vec = (111u8..=116) + .map(|seed| SigningKey::from([seed; 32])) + .collect(); + keys.sort_by_key(|key| std::cmp::Reverse(PubKeyID(VerificationKeyBytes::from(key).into()))); + let mut cumulative_stake = 0; + let roster: Vec = keys + .iter() + .map(|key| { + cumulative_stake += 1; + SortedRosterMember { + pub_key: PubKeyID(VerificationKeyBytes::from(key).into()), + stake: 1, + cumulative_stake, + } + }) + .collect(); + let namespace = [117u8; 32]; + let proposal = BlockValue(vec![118u8; 128]); + let proposal_id = proposal.id_from_value(&HashKeys::default()); + let mut round_data = RoundData { + height: 0, + round: 9, + proposal, + proposal_id, + msg_val_sigs: vec![[(ValueId::NIL, TMSig::NIL); 2]; roster.len()], + roster: roster.clone(), + vote_namespace: namespace, + ..RoundData::EMPTY + }; + let epoch = SignerEpochBinding { + public_key: roster[0].pub_key, + chain_id: [119u8; 32], + height: 0, + parent_commit: [120u8; 32], + vote_namespace: namespace, + consensus_config_hash: [121u8; 32], + roster_hash: canonical_roster_hash(&roster).unwrap(), + roster_index: 0, + active_roster_len: 6, + }; + for roster_i in 0..4 { + let signed = make_vote_sign_datas( + roster[roster_i].pub_key, + true, + round_data.height, + round_data.round, + proposal_id, + )[1]; + round_data.msg_val_sigs[roster_i][1] = ( + proposal_id, + TMSig(sign_with_namespace(&keys[roster_i], &signed, &namespace)), + ); + } + let four_yes = canonical_precommit_certificate(&round_data, &roster).unwrap(); + assert!(verify_precommit_certificate( + &four_yes, + round_data.round, + proposal_id, + &epoch, + &roster, + ) + .is_err()); + + let signed = make_vote_sign_datas( + roster[4].pub_key, + true, + round_data.height, + round_data.round, + proposal_id, + )[1]; + round_data.msg_val_sigs[4][1] = ( + proposal_id, + TMSig(sign_with_namespace(&keys[4], &signed, &namespace)), + ); + let five_yes = canonical_precommit_certificate(&round_data, &roster).unwrap(); + verify_precommit_certificate(&five_yes, round_data.round, proposal_id, &epoch, &roster) + .unwrap(); +} + +struct TestPaths { + dir: PathBuf, + wal: PathBuf, + anchor: PathBuf, +} + +impl TestPaths { + fn new(label: &str) -> Self { + let id = NEXT_TEST_ID.fetch_add(1, Ordering::Relaxed); + let dir = std::env::temp_dir().join(format!( + "tenderlink-signer-wal-{label}-{}-{id}", + std::process::id(), + )); + std::fs::create_dir(&dir).unwrap(); + Self { + wal: dir.join("signer.wal"), + anchor: dir.join("signer.anchor"), + dir, + } + } + + fn config(&self, authorized: bool) -> DurableSignerConfig { + DurableSignerConfig { + wal_path: self.wal.clone(), + anchor_path: self.anchor.clone(), + independent_anchor_authorized: authorized, + non_genesis_bootstrap_receipt_hash: Some([0x42; 32]), + } + } +} + +impl Drop for TestPaths { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.dir); + } +} + +fn one_key_fixture( + height: u64, +) -> ( + SigningKey, + Vec, + SignerEpochBinding, + HashKeys, +) { + let key = SigningKey::from([41u8; 32]); + let pub_key = PubKeyID(VerificationKeyBytes::from(&key).into()); + let roster = vec![SortedRosterMember { + pub_key, + stake: 10, + cumulative_stake: 10, + }]; + let hash_keys = HashKeys::default(); + let epoch = SignerEpochBinding { + public_key: pub_key, + chain_id: [1u8; 32], + height, + parent_commit: [2u8; 32], + vote_namespace: [3u8; 32], + consensus_config_hash: [4u8; 32], + roster_hash: canonical_roster_hash(&roster).unwrap(), + roster_index: 0, + active_roster_len: 1, + }; + (key, roster, epoch, hash_keys) +} + +fn vote_bytes(epoch: &SignerEpochBinding, round: u32, precommit: bool, value: ValueId) -> Vec { + make_vote_sign_datas(epoch.public_key, precommit, epoch.height, round, value)[1].to_vec() +} + +fn proposal_parts( + epoch: &SignerEpochBinding, + hash_keys: &HashKeys, + round: u32, + valid_round: i64, + proposal: &[u8], +) -> (ValueId, Vec>) { + let value = BlockValue(proposal.to_vec()); + let proposal_id = value.id_from_value(hash_keys); + let mut header = PacketProposalChunkHeader { + height: epoch.height, + round, + chunk_i: 0, + proposal_size: proposal.len().try_into().unwrap(), + proposal_id, + valid_round, + }; + let mut buffer = [0u8; 2048]; + let mut parts = Vec::new(); + for chunk_i in 0..value.chunks_n() { + header.chunk_i = chunk_i.try_into().unwrap(); + let mut offset = header.write_to(&mut buffer); + let (chunk_offset, chunk_size) = value.chunk_o_size(chunk_i); + offset += proposal[chunk_offset..chunk_offset + chunk_size].write_to(&mut buffer[offset..]); + parts.push(buffer[..offset].to_vec()); + } + (proposal_id, parts) +} + +#[test] +fn exact_vote_replay_is_stable_across_restart() { + let paths = TestPaths::new("vote-replay"); + let (key, roster, epoch, hash_keys) = one_key_fixture(0); + let value = ValueId([9u8; 32]); + let bytes = vote_bytes(&epoch, 7, false, value); + let expected = { + let mut signer = + DurableSigner::open(key.clone(), paths.config(true), epoch.clone()).unwrap(); + assert!(signer.is_active()); + let first = signer + .sign_vote(&hash_keys, &roster, 7, false, value, &bytes, None) + .unwrap(); + let replay = signer + .sign_vote(&hash_keys, &roster, 7, false, value, &bytes, None) + .unwrap(); + assert_eq!(first, replay); + first + }; + let mut reopened = DurableSigner::open(key, paths.config(true), epoch).unwrap(); + assert!(reopened.is_active()); + assert_eq!( + reopened + .sign_vote(&hash_keys, &roster, 7, false, value, &bytes, None) + .unwrap(), + expected, + ); +} + +#[test] +fn conflicting_or_malformed_vote_poison_signing() { + for malformed in [false, true] { + let paths = TestPaths::new(if malformed { + "vote-malformed" + } else { + "vote-conflict" + }); + let (key, roster, epoch, hash_keys) = one_key_fixture(0); + let first_value = ValueId([11u8; 32]); + let first_bytes = vote_bytes(&epoch, 3, false, first_value); + let mut signer = DurableSigner::open(key, paths.config(true), epoch.clone()).unwrap(); + signer + .sign_vote( + &hash_keys, + &roster, + 3, + false, + first_value, + &first_bytes, + None, + ) + .unwrap(); + + let second_value = if malformed { + first_value + } else { + ValueId([12u8; 32]) + }; + let mut second_bytes = vote_bytes(&epoch, 3, false, second_value); + if malformed { + second_bytes[0] ^= 1; + } + assert!(signer + .sign_vote( + &hash_keys, + &roster, + 3, + false, + second_value, + &second_bytes, + None + ) + .is_err()); + assert!(matches!(signer.status(), SignerStatus::Poisoned(_))); + } +} + +#[test] +fn proposal_manifest_is_exact_and_single_value_per_round() { + let paths = TestPaths::new("proposal"); + let (key, _roster, epoch, hash_keys) = one_key_fixture(0); + let proposal = vec![21u8; PROPOSAL_CHUNK_DATA_SIZE + 37]; + let (proposal_id, parts) = proposal_parts(&epoch, &hash_keys, 4, -1, &proposal); + let mut signer = DurableSigner::open(key, paths.config(true), epoch.clone()).unwrap(); + let first = signer + .sign_proposal(&hash_keys, &_roster, 4, -1, proposal_id, &proposal, &parts) + .unwrap(); + assert_eq!( + signer + .sign_proposal(&hash_keys, &_roster, 4, -1, proposal_id, &proposal, &parts) + .unwrap(), + first, + ); + + let different = vec![22u8; PROPOSAL_CHUNK_DATA_SIZE + 37]; + let (different_id, different_parts) = proposal_parts(&epoch, &hash_keys, 4, -1, &different); + assert!(signer + .sign_proposal( + &hash_keys, + &_roster, + 4, + -1, + different_id, + &different, + &different_parts + ) + .is_err()); + assert!(matches!(signer.status(), SignerStatus::Poisoned(_))); +} + +#[test] +fn proposal_and_vote_reject_any_roster_outside_the_epoch_fingerprint() { + let (key, roster, epoch, hash_keys) = one_key_fixture(0); + let mut alternate = roster.clone(); + alternate[0].stake += 1; + alternate[0].cumulative_stake += 1; + assert_ne!( + canonical_roster_hash(&alternate).unwrap(), + epoch.roster_hash + ); + + let proposal_paths = TestPaths::new("proposal-roster-mismatch"); + let proposal = vec![23u8; 512]; + let (proposal_id, parts) = proposal_parts(&epoch, &hash_keys, 1, -1, &proposal); + let mut proposal_signer = + DurableSigner::open(key.clone(), proposal_paths.config(true), epoch.clone()).unwrap(); + assert!(proposal_signer + .sign_proposal( + &hash_keys, + &alternate, + 1, + -1, + proposal_id, + &proposal, + &parts, + ) + .is_err()); + assert!(matches!( + proposal_signer.status(), + SignerStatus::Poisoned(_) + )); + + let vote_paths = TestPaths::new("vote-roster-mismatch"); + let value = ValueId([24u8; 32]); + let bytes = vote_bytes(&epoch, 1, false, value); + let mut vote_signer = DurableSigner::open(key, vote_paths.config(true), epoch).unwrap(); + assert!(vote_signer + .sign_vote(&hash_keys, &alternate, 1, false, value, &bytes, None,) + .is_err()); + assert!(matches!(vote_signer.status(), SignerStatus::Poisoned(_))); +} + +#[test] +fn signer_index_must_resolve_to_the_epoch_public_key() { + let paths = TestPaths::new("signer-index-mismatch"); + let (key, roster, mut epoch, hash_keys) = one_key_fixture(0); + epoch.roster_index = 1; + let value = ValueId([25u8; 32]); + let bytes = vote_bytes(&epoch, 1, false, value); + let mut signer = DurableSigner::open(key, paths.config(true), epoch).unwrap(); + assert!(signer + .sign_vote(&hash_keys, &roster, 1, false, value, &bytes, None,) + .is_err()); + assert!(!signer.is_active()); +} + +#[test] +fn non_genesis_without_receipt_or_unfenced_empty_history_is_observer_only() { + for (height, authorized) in [(8, true), (0, false)] { + let paths = TestPaths::new("unknown-history"); + let (key, _roster, epoch, _hash_keys) = one_key_fixture(height); + let mut config = paths.config(authorized); + if height > 0 { + config.non_genesis_bootstrap_receipt_hash = None; + } + let signer = DurableSigner::open(key, config, epoch).unwrap(); + assert!(matches!(signer.status(), SignerStatus::ObserverOnly(_))); + assert!(!paths.wal.exists(), "observer-only startup created a WAL"); + assert!( + !paths.anchor.exists(), + "observer-only startup created an anchor" + ); + } +} + +#[test] +fn non_genesis_bootstrap_requires_the_same_sealed_receipt_on_restart() { + let paths = TestPaths::new("non-genesis-receipt"); + let (key, _roster, epoch, _hash_keys) = one_key_fixture(8); + let receipt = [0x42; 32]; + let mut config = paths.config(true); + config.non_genesis_bootstrap_receipt_hash = Some(receipt); + let signer = DurableSigner::open(key.clone(), config, epoch.clone()).unwrap(); + assert!(signer.is_active()); + drop(signer); + + let mut wrong = paths.config(true); + wrong.non_genesis_bootstrap_receipt_hash = Some([0x43; 32]); + let signer = DurableSigner::open(key, wrong, epoch).unwrap(); + assert!(matches!(signer.status(), SignerStatus::ObserverOnly(_))); +} + +#[test] +fn high_bit_round_is_rejected_before_encoding() { + let paths = TestPaths::new("high-round"); + let (key, roster, epoch, hash_keys) = one_key_fixture(0); + let mut signer = DurableSigner::open(key, paths.config(true), epoch).unwrap(); + assert!(signer + .sign_vote( + &hash_keys, + &roster, + 0x8000_0000, + false, + ValueId([1u8; 32]), + &[], + None, + ) + .is_err()); + assert!(matches!(signer.status(), SignerStatus::Poisoned(_))); +} + +#[test] +fn a_second_process_cannot_open_the_same_signer_files() { + let paths = TestPaths::new("exclusive-lock"); + let (key, _roster, epoch, _hash_keys) = one_key_fixture(0); + let first = DurableSigner::open(key.clone(), paths.config(true), epoch.clone()).unwrap(); + assert!(first.is_active()); + assert!(DurableSigner::open(key, paths.config(true), epoch).is_err()); +} + +fn seed_vote( + paths: &TestPaths, +) -> ( + SigningKey, + Vec, + SignerEpochBinding, + HashKeys, +) { + let (key, roster, epoch, hash_keys) = one_key_fixture(0); + let value = ValueId([44u8; 32]); + let bytes = vote_bytes(&epoch, 1, false, value); + let mut signer = DurableSigner::open(key.clone(), paths.config(true), epoch.clone()).unwrap(); + signer + .sign_vote(&hash_keys, &roster, 1, false, value, &bytes, None) + .unwrap(); + drop(signer); + (key, roster, epoch, hash_keys) +} + +#[test] +fn torn_or_rolled_back_anchor_fails_closed() { + for clean_frame_rollback in [false, true] { + let paths = TestPaths::new("anchor-damage"); + let (key, _roster, epoch, _hash_keys) = seed_vote(&paths); + let anchor_len = std::fs::metadata(&paths.anchor).unwrap().len(); + let new_len = if clean_frame_rollback { + anchor_len - 116 + } else { + anchor_len - 1 + }; + OpenOptions::new() + .write(true) + .open(&paths.anchor) + .unwrap() + .set_len(new_len) + .unwrap(); + let reopened = DurableSigner::open(key, paths.config(true), epoch).unwrap(); + assert!(matches!(reopened.status(), SignerStatus::ObserverOnly(_))); + } +} + +#[test] +fn torn_wal_fails_closed_and_corruption_is_rejected() { + for corrupt in [false, true] { + let paths = TestPaths::new("wal-damage"); + let (key, _roster, epoch, _hash_keys) = seed_vote(&paths); + let wal_len = std::fs::metadata(&paths.wal).unwrap().len(); + if corrupt { + let mut file = OpenOptions::new() + .read(true) + .write(true) + .open(&paths.wal) + .unwrap(); + file.seek(SeekFrom::End(-1)).unwrap(); + let mut byte = [0u8; 1]; + file.read_exact(&mut byte).unwrap(); + byte[0] ^= 1; + file.seek(SeekFrom::End(-1)).unwrap(); + file.write_all(&byte).unwrap(); + file.sync_all().unwrap(); + assert!(DurableSigner::open(key.clone(), paths.config(true), epoch.clone()).is_err()); + let observer = DurableSigner::open_or_observer(key, paths.config(true), epoch); + assert!( + matches!(observer.status(), SignerStatus::ObserverOnly(reason) + if reason.contains("durable signer open failed")) + ); + } else { + OpenOptions::new() + .write(true) + .open(&paths.wal) + .unwrap() + .set_len(wal_len - 1) + .unwrap(); + let reopened = DurableSigner::open(key, paths.config(true), epoch).unwrap(); + assert!(matches!(reopened.status(), SignerStatus::ObserverOnly(_))); + } + } +} + +#[test] +fn injected_wal_or_anchor_failure_never_returns_a_signature() { + for point in [ + WalFailpoint::AfterWalWrite, + WalFailpoint::AfterWalSync, + WalFailpoint::AfterAnchorWrite, + WalFailpoint::AfterAnchorSync, + ] { + let paths = TestPaths::new("failpoint"); + let (key, roster, epoch, hash_keys) = one_key_fixture(0); + let value = ValueId([55u8; 32]); + let bytes = vote_bytes(&epoch, 2, false, value); + let mut signer = + DurableSigner::open(key.clone(), paths.config(true), epoch.clone()).unwrap(); + signer.set_failpoint(point); + assert!(signer + .sign_vote(&hash_keys, &roster, 2, false, value, &bytes, None) + .is_err()); + assert!(matches!(signer.status(), SignerStatus::ObserverOnly(_))); + drop(signer); + + let reopened = DurableSigner::open(key, paths.config(true), epoch).unwrap(); + match point { + WalFailpoint::AfterWalWrite | WalFailpoint::AfterWalSync => { + assert!(matches!(reopened.status(), SignerStatus::ObserverOnly(_))); + } + WalFailpoint::AfterAnchorWrite | WalFailpoint::AfterAnchorSync => { + assert!(reopened.is_active()); + assert_eq!(reopened.replay_intents().len(), 1); + } + WalFailpoint::AfterCommitApplied => { + unreachable!("commit-only failpoint is tested separately") + } + } + } +} + +fn quorum_fixture( + precommit: bool, +) -> ( + Vec, + Vec, + RoundData, + SignerEpochBinding, + HashKeys, +) { + let mut keys: Vec = (61u8..=64) + .map(|seed| SigningKey::from([seed; 32])) + .collect(); + keys.sort_by_key(|key| std::cmp::Reverse(PubKeyID(VerificationKeyBytes::from(key).into()))); + let mut cumulative_stake = 0; + let roster: Vec = keys + .iter() + .map(|key| { + cumulative_stake += 1; + SortedRosterMember { + pub_key: PubKeyID(VerificationKeyBytes::from(key).into()), + stake: 1, + cumulative_stake, + } + }) + .collect(); + let hash_keys = HashKeys::default(); + // Span multiple chunks so WAL recovery proves ordered-manifest survival, + // not merely preservation of a single signature. + let proposal = BlockValue(vec![88u8; PROPOSAL_CHUNK_DATA_SIZE * 2 + 17]); + let proposal_id = proposal.id_from_value(&hash_keys); + let namespace = [71u8; 32]; + let mut round_data = RoundData { + height: 0, + round: 6, + proposal, + proposal_id, + proposal_valid_round: -1, + msg_val_sigs: vec![[(ValueId::NIL, TMSig::NIL); 2]; roster.len()], + roster: roster.clone(), + vote_namespace: namespace, + ..RoundData::EMPTY + }; + let vote_i = usize::from(precommit); + for roster_i in 0..3 { + let signed = make_vote_sign_datas( + roster[roster_i].pub_key, + precommit, + round_data.height, + round_data.round, + proposal_id, + )[1]; + round_data.msg_val_sigs[roster_i][vote_i] = ( + proposal_id, + TMSig(sign_with_namespace(&keys[roster_i], &signed, &namespace)), + ); + } + populate_proposal_manifest(&keys, &roster, &hash_keys, &mut round_data, -1); + let epoch = SignerEpochBinding { + public_key: roster[0].pub_key, + chain_id: [72u8; 32], + height: 0, + parent_commit: [73u8; 32], + vote_namespace: namespace, + consensus_config_hash: [74u8; 32], + roster_hash: canonical_roster_hash(&roster).unwrap(), + roster_index: 0, + active_roster_len: roster.len().try_into().unwrap(), + }; + (keys, roster, round_data, epoch, hash_keys) +} + +fn populate_proposal_manifest( + keys: &[SigningKey], + roster: &[SortedRosterMember], + hash_keys: &HashKeys, + round_data: &mut RoundData, + valid_round: i64, +) { + round_data.proposal_valid_round = valid_round; + let (_, proposer) = TMState::proposer_from_height_round( + hash_keys, + roster, + round_data.height, + round_data.round, + ); + let proposer_key = keys + .iter() + .find(|key| PubKeyID(VerificationKeyBytes::from(*key).into()) == proposer) + .expect("fixture contains the selected proposer"); + let mut header = PacketProposalChunkHeader { + height: round_data.height, + round: round_data.round, + chunk_i: 0, + proposal_size: round_data.proposal.0.len().try_into().unwrap(), + proposal_id: round_data.proposal_id, + valid_round, + }; + round_data.proposal_sigs.clear(); + for chunk_i in 0..round_data.proposal.chunks_n() { + header.chunk_i = chunk_i.try_into().unwrap(); + let (chunk_offset, chunk_size) = round_data.proposal.chunk_o_size(chunk_i); + let mut signable = vec![0u8; PacketProposalChunkHeader::SERIALIZED_SIZE + chunk_size]; + let header_len = header.write_to(&mut signable); + signable[header_len..].copy_from_slice( + &round_data.proposal.0[chunk_offset..chunk_offset + chunk_size], + ); + round_data.proposal_sigs.push(TMSig(sign_with_namespace( + proposer_key, + &signable, + &round_data.vote_namespace, + ))); + } + round_data.proposal_sigs_n = round_data.proposal_sigs.len(); +} + +#[test] +fn lock_and_valid_state_requires_a_fresh_exact_quorum_certificate() { + let (keys, roster, round_data, epoch, hash_keys) = quorum_fixture(false); + let certificate = canonical_prevote_certificate(&round_data, &roster).unwrap(); + let transition = LockValidTransition { + locked_round: i64::from(round_data.round), + locked_value_id: round_data.proposal_id, + locked_value: round_data.proposal.0.clone(), + valid_round: i64::from(round_data.round), + valid_value_id: round_data.proposal_id, + valid_value: round_data.proposal.0.clone(), + certificate, + }; + let paths = TestPaths::new("transition-ok"); + let bytes = vote_bytes(&epoch, round_data.round, true, round_data.proposal_id); + let mut signer = + DurableSigner::open(keys[0].clone(), paths.config(true), epoch.clone()).unwrap(); + signer + .sign_vote( + &hash_keys, + &roster, + round_data.round, + true, + round_data.proposal_id, + &bytes, + Some(transition.clone()), + ) + .unwrap(); + assert_eq!(signer.durable_transition(), Some(&transition)); + drop(signer); + let reopened = DurableSigner::open(keys[0].clone(), paths.config(true), epoch).unwrap(); + assert!(reopened.is_active()); + assert_eq!(reopened.durable_transition(), Some(&transition)); + + let bad_paths = TestPaths::new("transition-bad"); + let mut bad = transition; + *bad.certificate.last_mut().unwrap() ^= 1; + let mut signer = DurableSigner::open(keys[0].clone(), bad_paths.config(true), { + let (_, _, _, e, _) = quorum_fixture(false); + e + }) + .unwrap(); + assert!(signer + .sign_vote( + &hash_keys, + &roster, + round_data.round, + true, + round_data.proposal_id, + &bytes, + Some(bad), + ) + .is_err()); + assert!(matches!(signer.status(), SignerStatus::Poisoned(_))); +} + +#[test] +fn transition_certificate_rejects_high_bit_round_alias() { + let (_, roster, round_data, epoch, hash_keys) = quorum_fixture(false); + let mut certificate = canonical_prevote_certificate(&round_data, &roster).unwrap(); + let round_offset = b"tenderlink-prevote-qc-v1".len() + 8; + certificate[round_offset..round_offset + 4] + .copy_from_slice(&(MAX_CONSENSUS_ROUND + 1).to_le_bytes()); + let transition = LockValidTransition { + locked_round: i64::from(MAX_CONSENSUS_ROUND) + 1, + locked_value_id: round_data.proposal_id, + locked_value: round_data.proposal.0.clone(), + valid_round: i64::from(MAX_CONSENSUS_ROUND) + 1, + valid_value_id: round_data.proposal_id, + valid_value: round_data.proposal.0, + certificate, + }; + assert!(verify_transition_certificate(&transition, &epoch, &hash_keys, &roster).is_err()); +} + +#[test] +fn off_roster_observer_can_verify_a_valid_decision_certificate() { + let (_, roster, round_data, mut epoch, _) = quorum_fixture(true); + let certificate = canonical_precommit_certificate(&round_data, &roster).unwrap(); + epoch.public_key = PubKeyID([199u8; 32]); + epoch.roster_index = u32::MAX; + verify_precommit_certificate( + &certificate, + round_data.round, + round_data.proposal_id, + &epoch, + &roster, + ) + .unwrap(); +} + +#[test] +fn vote_step_and_value_require_the_exact_transition_shape() { + let (keys, roster, round_data, epoch, hash_keys) = quorum_fixture(false); + let certificate = canonical_prevote_certificate(&round_data, &roster).unwrap(); + let transition = LockValidTransition { + locked_round: i64::from(round_data.round), + locked_value_id: round_data.proposal_id, + locked_value: round_data.proposal.0.clone(), + valid_round: i64::from(round_data.round), + valid_value_id: round_data.proposal_id, + valid_value: round_data.proposal.0.clone(), + certificate, + }; + + let missing_paths = TestPaths::new("precommit-missing-transition"); + let precommit = vote_bytes(&epoch, round_data.round, true, round_data.proposal_id); + let mut missing = + DurableSigner::open(keys[0].clone(), missing_paths.config(true), epoch.clone()).unwrap(); + assert!(missing + .sign_vote( + &hash_keys, + &roster, + round_data.round, + true, + round_data.proposal_id, + &precommit, + None, + ) + .is_err()); + assert!(matches!(missing.status(), SignerStatus::Poisoned(_))); + + let prevote_paths = TestPaths::new("prevote-with-transition"); + let prevote = vote_bytes(&epoch, round_data.round, false, round_data.proposal_id); + let mut prevote_signer = + DurableSigner::open(keys[0].clone(), prevote_paths.config(true), epoch.clone()).unwrap(); + assert!(prevote_signer + .sign_vote( + &hash_keys, + &roster, + round_data.round, + false, + round_data.proposal_id, + &prevote, + Some(transition.clone()), + ) + .is_err()); + assert!(matches!(prevote_signer.status(), SignerStatus::Poisoned(_))); + + let nil_paths = TestPaths::new("nil-precommit-with-transition"); + let nil_precommit = vote_bytes(&epoch, round_data.round, true, ValueId::NIL); + let mut nil_signer = + DurableSigner::open(keys[0].clone(), nil_paths.config(true), epoch).unwrap(); + assert!(nil_signer + .sign_vote( + &hash_keys, + &roster, + round_data.round, + true, + ValueId::NIL, + &nil_precommit, + Some(transition), + ) + .is_err()); + assert!(matches!(nil_signer.status(), SignerStatus::Poisoned(_))); +} + +#[test] +fn commit_intent_requires_quorum_and_incomplete_commit_resumes_only_exact_decision() { + let (keys, roster, round_data, epoch, hash_keys) = quorum_fixture(true); + let certificate = canonical_precommit_certificate(&round_data, &roster).unwrap(); + verify_precommit_certificate( + &certificate, + round_data.round, + round_data.proposal_id, + &epoch, + &roster, + ) + .unwrap(); + + let pending_paths = TestPaths::new("commit-pending"); + let mut pending = + DurableSigner::open(keys[0].clone(), pending_paths.config(true), epoch.clone()).unwrap(); + pending + .begin_commit( + &hash_keys, + round_data.round, + round_data.proposal_id, + &round_data.proposal, + round_data.proposal_valid_round, + &round_data.proposal_sigs, + &certificate, + &roster, + ) + .unwrap(); + drop(pending); + let mut reopened = + DurableSigner::open(keys[0].clone(), pending_paths.config(true), epoch.clone()).unwrap(); + assert!(matches!( + reopened.status(), + SignerStatus::ReconciliationRequired(_, _) + )); + let recovery = reopened + .pending_commit_recovery(&hash_keys, &roster) + .unwrap() + .expect("pending commit must carry an exact local recovery value"); + assert_eq!(recovery.round_data.proposal, round_data.proposal); + assert_eq!( + recovery.round_data.proposal_valid_round, + round_data.proposal_valid_round + ); + assert_eq!(recovery.round_data.proposal_sigs, round_data.proposal_sigs); + assert_eq!( + recovery.fat_pointer, + round_data_to_fat_pointer(&round_data, &roster) + ); + let resumed_digest = reopened + .begin_or_resume_commit( + &hash_keys, + round_data.round, + round_data.proposal_id, + &round_data.proposal, + round_data.proposal_valid_round, + &round_data.proposal_sigs, + &certificate, + &roster, + ) + .unwrap() + .expect("the exact pending commit must be resumable"); + reopened + .complete_commit( + resumed_digest, + round_data.proposal_id.0, + [93u8; 32], + &roster, + ) + .unwrap(); + assert!(reopened.is_active()); + assert_eq!(reopened.epoch().height, 1); + + let complete_paths = TestPaths::new("commit-complete"); + let mut complete = + DurableSigner::open(keys[0].clone(), complete_paths.config(true), epoch).unwrap(); + let digest = complete + .begin_commit( + &hash_keys, + round_data.round, + round_data.proposal_id, + &round_data.proposal, + round_data.proposal_valid_round, + &round_data.proposal_sigs, + &certificate, + &roster, + ) + .unwrap(); + let durable_parent = round_data.proposal_id.0; + let next_namespace = [92u8; 32]; + complete + .complete_commit(digest, durable_parent, next_namespace, &roster) + .unwrap(); + assert!(complete.is_active()); + assert_eq!(complete.epoch().height, 1); + assert_eq!(complete.epoch().parent_commit, durable_parent); + assert_eq!(complete.epoch().vote_namespace, next_namespace); + let next_epoch = complete.epoch().clone(); + drop(complete); + let reopened = + DurableSigner::open(keys[0].clone(), complete_paths.config(true), next_epoch).unwrap(); + assert!(reopened.is_active()); + + let unrelated_paths = TestPaths::new("commit-unrelated-readback"); + let (_, _, _, unrelated_epoch, _) = quorum_fixture(true); + let mut unrelated = DurableSigner::open( + keys[0].clone(), + unrelated_paths.config(true), + unrelated_epoch, + ) + .unwrap(); + let unrelated_digest = unrelated + .begin_commit( + &hash_keys, + round_data.round, + round_data.proposal_id, + &round_data.proposal, + round_data.proposal_valid_round, + &round_data.proposal_sigs, + &certificate, + &roster, + ) + .unwrap(); + assert!(unrelated + .complete_commit(unrelated_digest, [91u8; 32], next_namespace, &roster,) + .is_err()); + assert!(matches!(unrelated.status(), SignerStatus::Poisoned(_))); + + let forged_paths = TestPaths::new("commit-forged"); + let (_, _, _, forged_epoch, _) = quorum_fixture(true); + let mut forged = + DurableSigner::open(keys[0].clone(), forged_paths.config(true), forged_epoch).unwrap(); + let mut bad_certificate = certificate; + *bad_certificate.last_mut().unwrap() ^= 1; + assert!(forged + .begin_commit( + &hash_keys, + round_data.round, + round_data.proposal_id, + &round_data.proposal, + round_data.proposal_valid_round, + &round_data.proposal_sigs, + &bad_certificate, + &roster + ) + .is_err()); + assert!(matches!(forged.status(), SignerStatus::Poisoned(_))); +} + +#[test] +fn pending_commit_reconciles_from_local_wal_without_network_redelivery() { + let (keys, roster, round_data, epoch, hash_keys) = quorum_fixture(true); + let certificate = canonical_precommit_certificate(&round_data, &roster).unwrap(); + let paths = TestPaths::new("commit-local-recovery"); + let mut signer = + DurableSigner::open(keys[0].clone(), paths.config(true), epoch.clone()).unwrap(); + signer + .begin_commit( + &hash_keys, + round_data.round, + round_data.proposal_id, + &round_data.proposal, + round_data.proposal_valid_round, + &round_data.proposal_sigs, + &certificate, + &roster, + ) + .unwrap(); + drop(signer); + + let signer = DurableSigner::open(keys[0].clone(), paths.config(true), epoch.clone()).unwrap(); + assert!(matches!( + signer.status(), + SignerStatus::ReconciliationRequired(_, _) + )); + let expected_proposal = round_data.proposal.clone(); + let expected_pointer = round_data_to_fat_pointer(&round_data, &roster); + let expected_valid_round = round_data.proposal_valid_round; + let expected_proposal_sigs = round_data.proposal_sigs.clone(); + let expected_parent = round_data.proposal_id.0; + let next_namespace = [96u8; 32]; + let next_roster = roster.clone(); + let calls = Arc::new(AtomicU64::new(0)); + let calls_for_push = calls.clone(); + let push = ClosureToPushDecidedBlock(Arc::new(move |proposal, pointer, valid_round, proposal_sigs| { + assert_eq!(proposal, expected_proposal); + assert_eq!(pointer, expected_pointer); + assert_eq!(valid_round, expected_valid_round); + assert_eq!(proposal_sigs, expected_proposal_sigs); + calls_for_push.fetch_add(1, Ordering::SeqCst); + let next_roster = next_roster.clone(); + Box::pin(async move { + Ok(DurableDecisionOutcome { + next_roster, + next_vote_namespace: next_namespace, + durable_parent_commit: Some(expected_parent), + }) + }) + })); + let public_key = PubKeyID(VerificationKeyBytes::from(&keys[0]).into()); + let mut state = TMState::init( + signer, + public_key, + 3032, + ClosureToProposeNewBlock(Arc::new(|| Box::pin(async { None }))), + ClosureToValidateProposedBlock(Arc::new(|_| { + Box::pin(async { (TMStatus::Pass, TMStatusReason::None) }) + })), + push, + ClosureToUpdatePeers(Arc::new(|_| Box::pin(async {}))), + ClosureToAllowBftAccess(Arc::new(|_, _| Box::pin(async {}))), + ); + state.hash_keys = hash_keys; + state.height = epoch.height; + state.vote_namespace = epoch.vote_namespace; + let mut recovered_roster = roster; + + tokio::runtime::Runtime::new() + .unwrap() + .block_on(state.reconcile_pending_commit(&mut recovered_roster)) + .unwrap(); + + assert_eq!(calls.load(Ordering::SeqCst), 1); + assert!(state.durable_signer.is_active()); + assert_eq!(state.height, epoch.height + 1); + assert_eq!(state.vote_namespace, next_namespace); + assert_eq!(state.recent_commit_round_cache.len(), 1); + let cached = &state.recent_commit_round_cache[0]; + assert_eq!(cached.height, round_data.height); + assert_eq!(cached.round, round_data.round); + assert_eq!(cached.proposal.0, round_data.proposal.0); + assert_eq!(cached.proposal_id, round_data.proposal_id); + assert_eq!(cached.proposal_valid_round, round_data.proposal_valid_round); + assert_eq!(cached.proposal_sigs, round_data.proposal_sigs); + assert_eq!(cached.msg_val_sigs, round_data.msg_val_sigs); + verify_reconstructed_precommit_quorum(cached, &recovered_roster).unwrap(); +} + +#[test] +fn transient_decision_apply_latches_exact_reconciliation_without_poison_or_advance() { + let (keys, roster, mut round_data, epoch, hash_keys) = quorum_fixture(true); + round_data.counts = round_data + .msg_val_sigs + .iter() + .zip(&roster) + .fold(ConsensusCounts::ZERO, |counts, (signatures, member)| { + counts + ConsensusCounts::from(&(*signatures, member.stake)) + }); + let paths = TestPaths::new("decision-transient-reconciliation"); + let signer = + DurableSigner::open(keys[0].clone(), paths.config(true), epoch.clone()).unwrap(); + let public_key = PubKeyID(VerificationKeyBytes::from(&keys[0]).into()); + let calls = Arc::new(AtomicU64::new(0)); + let calls_for_push = calls.clone(); + let mut state = TMState::init( + signer, + public_key, + 3032, + ClosureToProposeNewBlock(Arc::new(|| Box::pin(async { None }))), + ClosureToValidateProposedBlock(Arc::new(|_| { + Box::pin(async { (TMStatus::Pass, TMStatusReason::None) }) + })), + ClosureToPushDecidedBlock(Arc::new(move |_, _, _, _| { + calls_for_push.fetch_add(1, Ordering::SeqCst); + Box::pin(async { Err("injected transient PoS apply timeout".into()) }) + })), + ClosureToUpdatePeers(Arc::new(|_| Box::pin(async {}))), + ClosureToAllowBftAccess(Arc::new(|_, _| Box::pin(async {}))), + ); + state.hash_keys = hash_keys; + state.height = epoch.height; + state.vote_namespace = epoch.vote_namespace; + state.round = round_data.round; + state.step = TMStep::Precommit; + state.rounds_data = vec![round_data]; + let mut live_roster = roster; + + tokio::runtime::Runtime::new() + .unwrap() + .block_on(state.bft_update(&mut live_roster)); + + assert_eq!(calls.load(Ordering::SeqCst), 1); + assert!(matches!( + state.durable_signer.status(), + SignerStatus::ReconciliationRequired(_, reason) + if reason.contains("injected transient PoS apply timeout") + )); + assert_eq!(state.height, epoch.height); + assert!(state.recent_commit_round_cache.is_empty()); + assert!(state.reconciliation_required); + + drop(state); + let reopened = DurableSigner::open(keys[0].clone(), paths.config(true), epoch).unwrap(); + assert!(matches!( + reopened.status(), + SignerStatus::ReconciliationRequired(_, _) + )); +} + +#[test] +fn reproposal_manifest_survives_wal_restart_and_exact_recovery() { + let (keys, roster, mut round_data, epoch, hash_keys) = quorum_fixture(true); + populate_proposal_manifest(&keys, &roster, &hash_keys, &mut round_data, 4); + let certificate = canonical_precommit_certificate(&round_data, &roster).unwrap(); + let paths = TestPaths::new("commit-reproposal-manifest"); + let mut signer = + DurableSigner::open(keys[0].clone(), paths.config(true), epoch.clone()).unwrap(); + let digest = signer + .begin_commit( + &hash_keys, + round_data.round, + round_data.proposal_id, + &round_data.proposal, + round_data.proposal_valid_round, + &round_data.proposal_sigs, + &certificate, + &roster, + ) + .unwrap(); + signer + .require_reconciliation(digest, "injected transient PoS apply failure") + .unwrap(); + assert!(matches!( + signer.status(), + SignerStatus::ReconciliationRequired(found, _) if *found == digest + )); + drop(signer); + + let mut reopened = + DurableSigner::open(keys[0].clone(), paths.config(true), epoch).unwrap(); + assert!(matches!( + reopened.status(), + SignerStatus::ReconciliationRequired(found, _) if *found == digest + )); + let recovery = reopened + .pending_commit_recovery(&hash_keys, &roster) + .unwrap() + .unwrap(); + assert_eq!(recovery.digest, digest); + assert_eq!(recovery.round_data.round, round_data.round); + assert_eq!(recovery.round_data.proposal, round_data.proposal); + assert_eq!(recovery.round_data.proposal_valid_round, 4); + assert_eq!(recovery.round_data.proposal_sigs, round_data.proposal_sigs); + assert_eq!(recovery.round_data.proposal_sigs_n, round_data.proposal_sigs_n); + verify_reconstructed_precommit_quorum(&recovery.round_data, &roster).unwrap(); + reopened + .complete_commit(digest, round_data.proposal_id.0, [97u8; 32], &roster) + .unwrap(); + assert!(reopened.is_active()); +} + +#[test] +fn reconciliation_digest_conflict_is_poison_but_transient_latch_is_not() { + let (keys, roster, round_data, epoch, hash_keys) = quorum_fixture(true); + let certificate = canonical_precommit_certificate(&round_data, &roster).unwrap(); + let paths = TestPaths::new("commit-reconciliation-conflict"); + let mut signer = + DurableSigner::open(keys[0].clone(), paths.config(true), epoch.clone()).unwrap(); + let digest = signer + .begin_commit( + &hash_keys, + round_data.round, + round_data.proposal_id, + &round_data.proposal, + round_data.proposal_valid_round, + &round_data.proposal_sigs, + &certificate, + &roster, + ) + .unwrap(); + signer + .require_reconciliation(digest, "transient apply timeout") + .unwrap(); + assert!(!matches!(signer.status(), SignerStatus::Poisoned(_))); + drop(signer); + let mut reopened = + DurableSigner::open(keys[0].clone(), paths.config(true), epoch).unwrap(); + assert!(matches!( + reopened.status(), + SignerStatus::ReconciliationRequired(_, _) + )); + let mut wrong = digest; + wrong[0] ^= 1; + assert!(matches!( + reopened.require_reconciliation(wrong, "wrong digest"), + Err(SignerError::Conflict(_)) + )); + assert!(matches!(reopened.status(), SignerStatus::Poisoned(_))); +} + +#[test] +fn unfinished_legacy_commit_intent_is_readable_but_not_auto_recoverable() { + let (keys, roster, round_data, epoch, _) = quorum_fixture(true); + let certificate = canonical_precommit_certificate(&round_data, &roster).unwrap(); + let paths = TestPaths::new("commit-legacy-pending"); + let mut signer = + DurableSigner::open(keys[0].clone(), paths.config(true), epoch.clone()).unwrap(); + signer + .append_legacy_pending_commit_for_test( + round_data.proposal_id, + &round_data.proposal, + &certificate, + ) + .unwrap(); + drop(signer); + + let legacy = DurableSigner::open(keys[0].clone(), paths.config(true), epoch).unwrap(); + assert!(matches!( + legacy.status(), + SignerStatus::ObserverOnly(reason) if reason.contains("legacy pending commit") + )); +} + +#[test] +fn crash_after_commit_applied_rejects_old_epoch_then_recovers_exact_successor() { + let (keys, roster, round_data, epoch, hash_keys) = quorum_fixture(true); + let certificate = canonical_precommit_certificate(&round_data, &roster).unwrap(); + let paths = TestPaths::new("commit-applied-crash"); + let mut signer = + DurableSigner::open(keys[0].clone(), paths.config(true), epoch.clone()).unwrap(); + let digest = signer + .begin_commit( + &hash_keys, + round_data.round, + round_data.proposal_id, + &round_data.proposal, + round_data.proposal_valid_round, + &round_data.proposal_sigs, + &certificate, + &roster, + ) + .unwrap(); + signer.set_failpoint(WalFailpoint::AfterCommitApplied); + assert!(signer + .complete_commit(digest, round_data.proposal_id.0, [92u8; 32], &roster) + .is_err()); + drop(signer); + + let old_epoch = + DurableSigner::open(keys[0].clone(), paths.config(true), epoch.clone()).unwrap(); + assert!( + matches!(old_epoch.status(), SignerStatus::ReconciliationRequired(_, reason) + if reason.contains("commit recovery is incomplete")) + ); + drop(old_epoch); + + let mut successor = epoch; + successor.height += 1; + successor.parent_commit = round_data.proposal_id.0; + successor.vote_namespace = [92u8; 32]; + let recovered = + DurableSigner::open(keys[0].clone(), paths.config(true), successor.clone()).unwrap(); + assert!(recovered.is_active()); + assert_eq!(recovered.epoch(), &successor); +} + +#[test] +fn store_ahead_crash_after_commit_intent_recovers_exact_successor_automatically() { + let (keys, roster, round_data, epoch, hash_keys) = quorum_fixture(true); + let certificate = canonical_precommit_certificate(&round_data, &roster).unwrap(); + let paths = TestPaths::new("commit-intent-store-ahead"); + let mut signer = + DurableSigner::open(keys[0].clone(), paths.config(true), epoch.clone()).unwrap(); + signer + .begin_commit( + &hash_keys, + round_data.round, + round_data.proposal_id, + &round_data.proposal, + round_data.proposal_valid_round, + &round_data.proposal_sigs, + &certificate, + &roster, + ) + .unwrap(); + drop(signer); + + let mut successor = epoch; + successor.height += 1; + successor.parent_commit = round_data.proposal_id.0; + successor.vote_namespace = [93u8; 32]; + let recovered = + DurableSigner::open(keys[0].clone(), paths.config(true), successor.clone()).unwrap(); + assert!(recovered.is_active()); + assert_eq!(recovered.epoch(), &successor); + + drop(recovered); + let replayed = DurableSigner::open(keys[0].clone(), paths.config(true), successor).unwrap(); + assert!(replayed.is_active()); +} + +#[test] +fn store_ahead_recovery_rejects_an_unrelated_successor_without_mutating_history() { + let (keys, roster, round_data, epoch, hash_keys) = quorum_fixture(true); + let certificate = canonical_precommit_certificate(&round_data, &roster).unwrap(); + let paths = TestPaths::new("commit-intent-unrelated-successor"); + let mut signer = + DurableSigner::open(keys[0].clone(), paths.config(true), epoch.clone()).unwrap(); + signer + .begin_commit( + &hash_keys, + round_data.round, + round_data.proposal_id, + &round_data.proposal, + round_data.proposal_valid_round, + &round_data.proposal_sigs, + &certificate, + &roster, + ) + .unwrap(); + drop(signer); + + let mut unrelated = epoch.clone(); + unrelated.height += 1; + unrelated.parent_commit = [94u8; 32]; + unrelated.vote_namespace = [95u8; 32]; + let rejected = DurableSigner::open(keys[0].clone(), paths.config(true), unrelated).unwrap(); + assert!(matches!(rejected.status(), SignerStatus::ObserverOnly(_))); + drop(rejected); + + let mut exact = epoch; + exact.height += 1; + exact.parent_commit = round_data.proposal_id.0; + exact.vote_namespace = [95u8; 32]; + let recovered = DurableSigner::open(keys[0].clone(), paths.config(true), exact).unwrap(); + assert!(recovered.is_active()); +} + +#[test] +fn quorum_certificates_allow_a_signed_conflicting_minority() { + for precommit in [false, true] { + let (keys, roster, mut round_data, epoch, hash_keys) = quorum_fixture(precommit); + let conflicting = ValueId([101u8; 32]); + let signed = make_vote_sign_datas( + roster[3].pub_key, + precommit, + round_data.height, + round_data.round, + conflicting, + )[1]; + round_data.msg_val_sigs[3][usize::from(precommit)] = ( + conflicting, + TMSig(sign_with_namespace( + &keys[3], + &signed, + &epoch.vote_namespace, + )), + ); + if precommit { + let certificate = canonical_precommit_certificate(&round_data, &roster).unwrap(); + verify_precommit_certificate( + &certificate, + round_data.round, + round_data.proposal_id, + &epoch, + &roster, + ) + .unwrap(); + } else { + let transition = LockValidTransition { + locked_round: i64::from(round_data.round), + locked_value_id: round_data.proposal_id, + locked_value: round_data.proposal.0.clone(), + valid_round: i64::from(round_data.round), + valid_value_id: round_data.proposal_id, + valid_value: round_data.proposal.0.clone(), + certificate: canonical_prevote_certificate(&round_data, &roster).unwrap(), + }; + verify_transition_certificate(&transition, &epoch, &hash_keys, &roster).unwrap(); + } + } +} diff --git a/tenderlink/tools/drain_regression.rs b/tenderlink/tools/drain_regression.rs new file mode 100644 index 00000000..1b0ea7c8 --- /dev/null +++ b/tenderlink/tools/drain_regression.rs @@ -0,0 +1,49 @@ +use std::time::Instant; + +fn batch(count: usize) -> Vec<(usize, Vec)> { + (0..count) + .map(|sequence| (sequence, vec![sequence as u8; 64])) + .collect() +} + +fn front_remove(mut packets: Vec<(usize, Vec)>) -> u64 { + let mut digest = 0u64; + for expected in 0..packets.len() { + let (sequence, payload) = packets.remove(0); + assert_eq!(sequence, expected); + digest = digest + .wrapping_mul(31) + .wrapping_add(sequence as u64 ^ payload[0] as u64); + } + digest +} + +fn linear_consume(packets: Vec<(usize, Vec)>) -> u64 { + let mut digest = 0u64; + for (expected, (sequence, payload)) in packets.into_iter().enumerate() { + assert_eq!(sequence, expected); + digest = digest + .wrapping_mul(31) + .wrapping_add(sequence as u64 ^ payload[0] as u64); + } + digest +} + +fn main() { + let count = 50_000; + + let started = Instant::now(); + let old_digest = front_remove(batch(count)); + let old_ms = started.elapsed().as_secs_f64() * 1_000.0; + + let started = Instant::now(); + let new_digest = linear_consume(batch(count)); + let linear_ms = started.elapsed().as_secs_f64() * 1_000.0; + + assert_eq!(old_digest, new_digest); + println!( + "DRAIN_REGRESSION_PASS packets={count} digest={new_digest} \ + old_ms={old_ms:.3} linear_ms={linear_ms:.3} speedup={:.1}x", + old_ms / linear_ms, + ); +} diff --git a/zebra-crosslink/Cargo.lock b/zebra-crosslink/Cargo.lock index 3382c881..c6ad73ff 100644 --- a/zebra-crosslink/Cargo.lock +++ b/zebra-crosslink/Cargo.lock @@ -7326,6 +7326,7 @@ dependencies = [ "ed25519-zebra", "hex", "libc", + "nix 0.29.0", "rand 0.9.2", "rand_chacha 0.9.0", "rand_pcg", @@ -10223,6 +10224,7 @@ dependencies = [ "macroquad-profiler", "miniquad", "multiaddr", + "nix 0.29.0", "pin-project", "prost 0.14.3", "prost-build 0.13.5", diff --git a/zebra-crosslink/zebra-crosslink/Cargo.toml b/zebra-crosslink/zebra-crosslink/Cargo.toml index 2e3442bf..61108af5 100644 --- a/zebra-crosslink/zebra-crosslink/Cargo.toml +++ b/zebra-crosslink/zebra-crosslink/Cargo.toml @@ -72,6 +72,9 @@ lazy_static = "1.5.0" tenderlink = { workspace = true } visualizer_zcash = { workspace = true, optional = true } +[target.'cfg(unix)'.dependencies] +nix = { workspace = true, features = ["fs", "user"] } + [dev-dependencies] zebra-test = { path = "../zebra-test", version = "1.0.0-beta.45" } diff --git a/zebra-crosslink/zebra-crosslink/src/lib.rs b/zebra-crosslink/zebra-crosslink/src/lib.rs index e136053d..a84ba3a6 100644 --- a/zebra-crosslink/zebra-crosslink/src/lib.rs +++ b/zebra-crosslink/zebra-crosslink/src/lib.rs @@ -23,14 +23,13 @@ use zebra_state::crosslink::*; use multiaddr::Multiaddr; use rand::{CryptoRng, RngCore}; -use rand::{Rng, SeedableRng}; -use std::collections::{HashMap, HashSet}; -use std::fs::OpenOptions; -use std::hash::{DefaultHasher, Hasher}; +use rand::Rng; +use std::collections::{HashMap, HashSet, VecDeque}; +use std::fs::{File, OpenOptions}; use std::io::Cursor; -use std::io::Read; +use std::io::{Read, Seek, SeekFrom}; use std::io::Write; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::str::FromStr; use std::sync::Arc; use std::time::Duration; @@ -58,6 +57,85 @@ use tokio::sync::Mutex as TokioMutex; pub static TEST_INSTR_C: Mutex = Mutex::new(0); pub static TEST_MODE: Mutex = Mutex::new(false); +const MAX_POS_STORE_ROSTER_MEMBERS: u64 = 100_000; +const MAX_POS_STORE_PROPOSAL_SIGNATURES: u64 = 100_000; +const POS_STORE_V2_MAGIC: [u8; 8] = *b"CTAZPSV2"; +const POS_STORE_V2_HASH_DOMAIN: &[u8] = b"ctaz-pos-store-v2-frame"; +const POS_STORE_V2_HEADER_LEN: u64 = 8 + 8 + 32; +const MAX_POS_STORE_V2_PAYLOAD_BYTES: u64 = 64 * 1024 * 1024; +const MAX_CANONICAL_CONSENSUS_ROUND: i64 = 0x7fff_ffff; +const SIGNER_MIGRATION_RECEIPT_SCHEMA: &str = "ctaz.signer-migration-receipt.v1"; +const SIGNER_MIGRATION_RECEIPT_ACTION: &str = "authorize_non_genesis_signer_bootstrap"; +const MAX_SIGNER_MIGRATION_RECEIPT_BYTES: u64 = 64 * 1024; +pub(crate) const SERVICE_HEALTH_STARTING: u8 = 0; +pub(crate) const SERVICE_HEALTH_READY: u8 = 1; +pub(crate) const SERVICE_HEALTH_OBSERVER_ONLY: u8 = 2; +pub(crate) const SERVICE_HEALTH_FAILED: u8 = 3; + +fn open_exclusive_pos_store(path: &Path) -> Result<(File, bool), String> { + let existed = match std::fs::symlink_metadata(path) { + Ok(metadata) => { + if metadata.file_type().is_symlink() || !metadata.file_type().is_file() { + return Err(format!("PoS store must be a regular non-symlink file: {}", path.display())); + } + true + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => false, + Err(error) => return Err(format!("failed to inspect PoS store {}: {error}", path.display())), + }; + + let mut options = OpenOptions::new(); + options.read(true).write(true).create(true); + #[cfg(unix)] + { + use nix::fcntl::OFlag; + use std::os::unix::fs::OpenOptionsExt; + options + .mode(0o600) + .custom_flags((OFlag::O_NOFOLLOW | OFlag::O_CLOEXEC).bits()); + } + #[cfg(windows)] + { + use std::os::windows::fs::OpenOptionsExt; + options.share_mode(0); + } + let file = options.open(path) + .map_err(|error| format!("failed to open PoS store {}: {error}", path.display()))?; + let metadata = file.metadata() + .map_err(|error| format!("failed to stat opened PoS store {}: {error}", path.display()))?; + if !metadata.file_type().is_file() { + return Err(format!("opened PoS store is not a regular file: {}", path.display())); + } + #[cfg(unix)] + { + use nix::unistd::geteuid; + use std::os::unix::fs::MetadataExt; + if metadata.uid() != geteuid().as_raw() { + return Err(format!("PoS store is not owned by the service user: {}", path.display())); + } + if metadata.mode() & 0o077 != 0 { + return Err(format!("PoS store permissions are broader than 0600: {}", path.display())); + } + if metadata.nlink() != 1 { + return Err(format!("PoS store has unexpected hard links: {}", path.display())); + } + } + file.try_lock() + .map_err(|error| format!("failed to acquire exclusive PoS-store ownership {}: {error}", path.display()))?; + + if !existed { + file.sync_all() + .map_err(|error| format!("failed to sync new PoS store {}: {error}", path.display()))?; + #[cfg(unix)] + { + let parent = path.parent().unwrap_or_else(|| Path::new(".")); + File::open(parent) + .and_then(|directory| directory.sync_all()) + .map_err(|error| format!("failed to sync PoS-store parent {}: {error}", parent.display()))?; + } + } + Ok((file, !existed)) +} pub static TEST_FAILED: Mutex = Mutex::new(0); pub static TEST_FAILED_INSTR_IDXS: Mutex> = Mutex::new(Vec::new()); pub static TEST_CHECK_ASSERT: Mutex = Mutex::new(1); @@ -113,6 +191,7 @@ pub mod service; /// Configuration for the state service. pub mod config { use serde::{Deserialize, Serialize}; + use std::fmt; // The canonical hardfork types live in `zebra-chain` so that zebra-state and // zebra-consensus — which cannot depend on zebra-crosslink — can share them. @@ -121,6 +200,57 @@ pub mod config { shipped_hardforks, HardForkConfig, HardForkSchedule, }; + /// An exact 32-byte lowercase-hex secret. Its debug representation is always redacted. + #[derive(Clone, Eq, PartialEq, Deserialize, Serialize)] + #[serde(transparent)] + pub struct SecretHex32(String); + + impl SecretHex32 { + pub(crate) fn expose_secret(&self) -> &str { + &self.0 + } + } + + impl fmt::Debug for SecretHex32 { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("SecretHex32(REDACTED)") + } + } + + /// A legacy opaque secret whose serialized form remains compatible but whose debug output + /// is always redacted. It is never admitted as validator identity. + #[derive(Clone, Eq, PartialEq, Deserialize, Serialize)] + #[serde(transparent)] + pub struct RedactedLegacySecret(String); + + impl fmt::Debug for RedactedLegacySecret { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("RedactedLegacySecret(REDACTED)") + } + } + + /// Canonical binding between a consensus identity and one explicit Noise endpoint. + #[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize)] + #[serde(deny_unknown_fields)] + pub struct BftPeerIdentity { + /// Raw roster key, exactly 32 bytes of lowercase hex. + pub consensus_public_key: String, + /// Network endpoint in `IP:port` or `[IPv6]:port` form. + pub address: String, + /// Noise static public key, exactly 32 bytes of lowercase hex. + pub noise_public_key: String, + } + + /// Canonical bootstrap voting identity. Transport routes are deliberately separate. + #[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize)] + #[serde(deny_unknown_fields)] + pub struct BftBootstrapRosterMember { + /// Raw roster key, exactly 32 bytes of lowercase hex. + pub consensus_public_key: String, + /// Explicit nonzero genesis/bootstrap voting power. + pub voting_power: u64, + } + /// Configuration for the state service. #[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize)] #[serde(deny_unknown_fields, default)] @@ -129,9 +259,24 @@ pub mod config { /// internally, or the public IP address if using externally. pub public_address: Option, /// Use the public IP instead of the generated seed - pub explicit_bft_key_seed: Option, + /// + /// Legacy only. It is never used for validator identity; a configuration that + /// supplies only this field starts unready and observer-only. + pub explicit_bft_key_seed: Option, /// List of public IP addresses for peers, in the same format as `public_address`. + /// + /// Legacy only. Endpoints are never converted into consensus or Noise keys. pub bft_peers: Vec, + /// Explicit validator Ed25519 signing seed, exactly 32 bytes of lowercase hex. + pub validator_signing_key_seed: Option, + /// Exact raw consensus public key expected from `validator_signing_key_seed`. + pub validator_consensus_public_key: Option, + /// Explicit local Noise static-key seed, exactly 32 bytes of lowercase hex. + pub validator_noise_static_key_seed: Option, + /// Explicit key-bound peer endpoints. These are transport routes, never roster grants. + pub bft_peer_identities: Vec, + /// Explicit initial roster used only before a durable PoS history supplies its next roster. + pub bootstrap_bft_roster: Vec, /// Disable the headless wallet. pub disable_the_headless_wallet: bool, /// Disable zaino. @@ -146,6 +291,22 @@ pub mod config { /// operator specify the entire hardfork schedule manually instead of /// inheriting the built-in (mainnet) assumed past. Defaults to `false`. pub disable_shipped_hardforks: bool, + /// Key-scoped append-only Tenderlink signing WAL. Both this and an independent anchor + /// are required before the process may sign; absence is observer-only. + pub signer_wal_path: Option, + /// Monotonic anchor outside the WAL/state rollback domain and shared by every holder of + /// this consensus key. Merely placing a second file on the same disk is not sufficient. + pub signer_anchor_path: Option, + /// Explicit action gate proving the configured anchor also globally fences this key. + /// False is always observer-only. + pub signer_independent_anchor_authorized: bool, + /// BLAKE3 hash (64 lowercase hex characters) of the operator-sealed one-time + /// non-genesis bootstrap receipt. The exact receipt file and this hash are both + /// required at a non-genesis startup; neither value self-authorizes a key. + pub signer_non_genesis_bootstrap_receipt_blake3: Option, + /// Local structured receipt whose exact bytes are pinned by + /// `signer_non_genesis_bootstrap_receipt_blake3`. It contains public bindings only. + pub signer_non_genesis_bootstrap_receipt_path: Option, } impl Default for Config { fn default() -> Self { @@ -153,10 +314,20 @@ pub mod config { public_address: None, bft_peers: Vec::new(), explicit_bft_key_seed: None, + validator_signing_key_seed: None, + validator_consensus_public_key: None, + validator_noise_static_key_seed: None, + bft_peer_identities: Vec::new(), + bootstrap_bft_roster: Vec::new(), disable_the_headless_wallet: false, disable_zaino: false, hardforks: Vec::new(), disable_shipped_hardforks: false, + signer_wal_path: None, + signer_anchor_path: None, + signer_independent_anchor_authorized: false, + signer_non_genesis_bootstrap_receipt_blake3: None, + signer_non_genesis_bootstrap_receipt_path: None, } } } @@ -259,6 +430,7 @@ pub(crate) struct TFLServiceInternal { bft_msg_flags: u64, // ALT: Vec of messages/combine flags bft_err_flags: u64, bft_blocks: Vec, + bft_height_by_hash: HashMap<[u8; 32], usize>, fat_pointer_to_tip: FatPointerToBftBlock, our_set_bft_string: Option, active_bft_string: Option, @@ -273,6 +445,42 @@ pub(crate) struct TFLServiceInternal { current_bc_final: Option<(ZebBlockHeight, ZebBlockHash)>, path_to_pos_store_file: PathBuf, + // Held for the lifetime of the service. This removes pathname re-open races and + // keeps exclusive ownership of the committed PoS history while signing is possible. + pos_store_file: Option, + // A duplicated handle to the same exclusively held file. Historical relays use + // positional reads, so they never move the append cursor. + pos_store_read_file: Option>, + // One fixed-size authenticated replay receipt per committed height. The block + // and proposal bytes remain on disk rather than growing another in-memory chain. + pos_store_records: Vec, + // A post-install CrosslinkFinalizeBlock reflush remains pending until state confirms it. + // This is deliberately in memory: replay reconstructs it from the installed durable tip. + pending_reflush: Option, + // A final v2 frame that ended at EOF before completion. It is never discarded on replay. + // Only an exact certified decision whose complete frame has these bytes as a strict prefix + // may replace it. + pos_store_unverified_tail: Option, +} + +#[derive(Debug, Clone)] +struct PosStoreTornTail { + offset: u64, + bytes: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct PosStoreRecordIndex { + offset: u64, + len: u64, + finalized_bc_height: u32, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct PosStoreAppendReceipt { + durable_parent_commit: [u8; 32], + offset: u64, + len: u64, } fn call_from_state_to_crosslink_to_ask_about_fat_pointers(internal_handle: &TFLServiceHandle, parent_fat_pointer: FatPointerToBftBlock, child_fat_pointer: FatPointerToBftBlock, pow_block_height: ZebBlockHeight) -> Option { @@ -311,7 +519,11 @@ fn call_from_state_to_crosslink_to_ask_about_fat_pointers(internal_handle: &TFLS let child_index = if child_is_null { None } else { - match internal.bft_blocks.iter().position(|b| b.blake3_hash() == child_fat_pointer.points_at_block_hash()) { + match internal + .bft_height_by_hash + .get(&child_fat_pointer.points_at_block_hash().0) + .copied() + { Some(h) => Some(h), None => return None, // unresolved child -> defer (reversible) } @@ -332,7 +544,11 @@ fn call_from_state_to_crosslink_to_ask_about_fat_pointers(internal_handle: &TFLS let parent_index = if parent_is_null { None } else { - match internal.bft_blocks.iter().position(|b| b.blake3_hash() == parent_fat_pointer.points_at_block_hash()) { + match internal + .bft_height_by_hash + .get(&parent_fat_pointer.points_at_block_hash().0) + .copied() + { Some(h) => Some(h), None => return None, // unresolved parent -> defer (reversible) } @@ -526,21 +742,6 @@ async fn tfl_final_block_height_hash_pre_locked( } } -// NAME: rng_sk_pk_from_addr -pub fn rng_private_public_key_from_address( - addr: &[u8], -) -> (rand::rngs::StdRng, ed25519_zebra::SigningKey, PubKeyID) { -// ) -> (rand::rngs::StdRng, ed25519_zebra::SigningKey, ed25519_zebra::VerificationKeyBytes) { - let mut hasher = DefaultHasher::new(); - hasher.write(addr); - let seed = hasher.finish(); - let mut rng = rand::rngs::StdRng::seed_from_u64(seed); - let private_key = ed25519_zebra::SigningKey::new(&mut rng); - let public_key = ed25519_zebra::VerificationKeyBytes::from(&private_key); - let pub_key = PubKeyID(<[u8; 32]>::from(public_key)); - (rng, private_key, pub_key) -} - async fn push_new_bft_msg_flags( tfl_handle: &TFLServiceHandle, bft_msg_flags: u64, @@ -561,15 +762,24 @@ async fn propose_new_bft_block(tfl_handle: &TFLServiceHandle) -> Option value, + Ok(StateResponse::Tip(None)) => return None, + Ok(_) => { + warn!("BFT proposer PoW-tip lookup returned the wrong response type"); return None; - }; + } + Err(error) => { + warn!(%error, "BFT proposer could not read the PoW tip"); + return None; + } + }; use std::ops::Sub; use zebra_chain::block::HeightDiff as BlockHeightDiff; @@ -611,31 +821,47 @@ async fn propose_new_bft_block(tfl_handle: &TFLServiceHandle) -> Option hash, + Ok(_) => { + warn!("BFT proposer finality-candidate lookup returned the wrong response type"); + return None; + } + Err(error) => { + warn!(%error, "BFT proposer could not read the finality candidate"); + return None; + } }; // NOTE: probably faster to request 2x as many blocks as we need rather than have another async call - let resp = (call.state)(StateRequest::FindBlockHeaders { - known_blocks: vec![candidate_hash], - stop: None, - }) - .await; - - let mut headers: Vec = if let Ok(StateResponse::BlockHeaders(hdrs)) = resp { - // TODO: do we want these in chain order or "walk-back order" - hdrs.into_iter() - .map(|ch| bc_hdr_to_lrz(&ch.header)) - .collect() - } else { - // Error or unexpected response type: - panic!("TODO: improve error handling."); + let mut headers: Vec = match bounded_proposer_state_call( + &call, + StateRequest::FindBlockHeaders { + known_blocks: vec![candidate_hash], + stop: None, + }, + "BFT proposer confirmation-header lookup", + ) + .await + { + Ok(StateResponse::BlockHeaders(headers)) => headers + .into_iter() + .map(|counted| bc_hdr_to_lrz(&counted.header)) + .collect(), + Ok(_) => { + warn!("BFT proposer confirmation-header lookup returned the wrong response type"); + return None; + } + Err(error) => { + warn!(%error, "BFT proposer could not read confirmation headers"); + return None; + } }; headers.truncate(params.bc_confirmation_depth_sigma as usize); @@ -685,12 +911,841 @@ async fn propose_new_bft_block(tfl_handle: &TFLServiceHandle) -> Option Result { + match tokio::time::timeout(PROPOSER_STATE_CALL_TIMEOUT, (call.state)(request)).await { + Err(_) => Err(format!("{operation} timed out after 2 seconds")), + Ok(Err(error)) => Err(format!("{operation} failed: {error}")), + Ok(Ok(response)) => Ok(response), + } +} + +async fn bounded_state_call( + call: &TFLServiceCalls, + request: StateRequest, + operation: &'static str, +) -> Result { + match tokio::time::timeout(CONSENSUS_STATE_CALL_TIMEOUT, (call.state)(request)).await { + Err(_) => Err(format!("{operation} timed out after 8 seconds")), + Ok(Err(error)) => Err(format!("{operation} failed: {error}")), + Ok(Ok(response)) => Ok(response), + } +} + +async fn bounded_crosslink_reflush( + call: &TFLServiceCalls, + final_hash: ZebBlockHash, + operation: &'static str, +) -> Result<(), String> { + let response = bounded_state_call( + call, + StateRequest::CrosslinkFinalizeBlock(final_hash), + operation, + ) + .await?; + let StateResponse::CrosslinkFinalized(reflushed_hash, _) = response else { + return Err(format!("{operation} returned the wrong response type")); + }; + if reflushed_hash != final_hash { + return Err(format!("{operation} finalized a different PoW hash")); + } + Ok(()) +} + +fn read_stored_roster_member(reader: &mut R) -> Result { + let mut pub_key = [0u8; 32]; + reader + .read_exact(&mut pub_key) + .map_err(|error| format!("stored roster key is truncated: {error}"))?; + let mut u64_bytes = [0u8; 8]; + reader + .read_exact(&mut u64_bytes) + .map_err(|error| format!("stored roster stake is truncated: {error}"))?; + let voting_power = u64::from_le_bytes(u64_bytes); + reader + .read_exact(&mut u64_bytes) + .map_err(|error| format!("stored roster txid count is truncated: {error}"))?; + let txids_len = u64::from_le_bytes(u64_bytes); + if txids_len != 0 { + return Err("PoS-store rosters must not contain transaction-detail vectors".into()); + } + Ok(RosterMember { + pub_key, + voting_power, + txids: Vec::new(), + }) +} + +fn reconstructed_decided_round( + block: &BftBlock, + fat_pointer: &FatPointerToBftBlock, + roster: &[SortedRosterMember], + vote_namespace: [u8; 32], + proposal_sigs: Vec, +) -> Result { + tenderlink::validate_consensus_roster(roster)?; + let active_len = usize::min(100, roster.len()); + let block_hash = block.blake3_hash(); + if fat_pointer.points_at_block_hash() != block_hash { + return Err("fat pointer does not identify the decided BFT block".into()); + } + let vote = fat_pointer.get_vote_template(); + if !vote.typ || vote.height != block.height as u64 || vote.value != block_hash { + return Err("fat-pointer vote template has the wrong step, height, or value".into()); + } + let round = u32::try_from(vote.round) + .map_err(|_| "fat-pointer vote round is outside the canonical domain")?; + if round > 0x7fff_ffff { + return Err("fat-pointer vote round exceeds the canonical 31-bit domain".into()); + } + + let mut signatures = HashMap::with_capacity(fat_pointer.signatures.len()); + for signature in &fat_pointer.signatures { + if !roster[..active_len] + .iter() + .any(|member| member.pub_key == signature.pub_key) + { + return Err("fat pointer contains a signer outside the active roster".into()); + } + if signatures + .insert(signature.pub_key, signature.vote_signature) + .is_some() + { + return Err("fat pointer contains a duplicate signer key".into()); + } + } + + let proposal = tenderlink::BlockValue( + block + .zcash_serialize_to_vec() + .map_err(|error| format!("failed to serialize decided BFT block: {error}"))?, + ); + let proposal_id = tenderlink::ValueId(block_hash.0); + let msg_val_sigs = roster + .iter() + .map(|member| { + let signature = signatures + .get(&member.pub_key) + .copied() + .map(TMSig) + .unwrap_or(TMSig::NIL); + [ + (tenderlink::ValueId::NIL, TMSig::NIL), + (if signature == TMSig::NIL { + tenderlink::ValueId::NIL + } else { + proposal_id + }, signature), + ] + }) + .collect(); + let proposal_sigs_n = proposal_sigs.len(); + Ok(tenderlink::RoundData { + height: block.height as u64, + round, + proposal, + proposal_id, + proposal_sigs, + proposal_sigs_n, + msg_val_sigs, + roster: roster.to_vec(), + vote_namespace, + ..tenderlink::RoundData::EMPTY + }) +} + +fn verify_decided_fat_pointer_quorum( + block: &BftBlock, + fat_pointer: &FatPointerToBftBlock, + roster: &[SortedRosterMember], + vote_namespace: [u8; 32], + proposal_sigs: Vec, +) -> Result { + let round_data = reconstructed_decided_round( + block, + fat_pointer, + roster, + vote_namespace, + proposal_sigs, + )?; + tenderlink::verify_reconstructed_precommit_quorum(&round_data, roster)?; + Ok(round_data) +} + +async fn validated_pow_header_chain( + call: &TFLServiceCalls, + block: &BftBlock, + previous_final_height: Option, +) -> Result<(ZebBlockHeight, ZebBlockHash), String> { + let expected = PROTOTYPE_PARAMETERS.bc_confirmation_depth_sigma as usize; + if block.headers.len() != expected { + return Err(format!( + "BFT block carries {} PoW headers, expected {expected}", + block.headers.len() + )); + } + + let first_header = block + .headers + .first() + .ok_or("BFT block has no finalization candidate")?; + let first_hash = ZebBlockHash(BlockHash::from_header_data(first_header).0); + let mut previous_hash = BlockHash::from_header_data(first_header); + for carried in block.headers.iter().skip(1) { + if carried.prev_block != previous_hash { + return Err("carried PoW headers are not a contiguous hash-linked chain".into()); + } + previous_hash = BlockHash::from_header_data(carried); + } + + // One exact best-chain lookup of the last carried header proves the complete + // locally hash-linked prefix is its canonical ancestry. Avoid one state/RPC + // round trip per confirmation header during live validation and replay. + let last_header = block.headers.last().expect("non-empty checked above"); + let last_hash = ZebBlockHash(BlockHash::from_header_data(last_header).0); + let response = bounded_state_call( + call, + StateRequest::BlockHeader(last_hash.into()), + "canonical PoW-header lookup", + ) + .await?; + let StateResponse::BlockHeader { + header, + height: last_height, + hash, + .. + } = response + else { + return Err("canonical PoW-header lookup returned the wrong response type".into()); + }; + if hash != last_hash || header.hash() != hash || bc_hdr_to_lrz(&header) != *last_header { + return Err("last carried PoW header is not byte-identical to the canonical best chain".into()); + } + let preceding = u32::try_from(expected - 1) + .map_err(|_| "PoW confirmation depth does not fit u32")?; + let first_height = ZebBlockHeight( + last_height + .0 + .checked_sub(preceding) + .ok_or("canonical PoW-header height underflows the carried chain")?, + ); + if previous_final_height.is_some_and(|height| first_height <= height) { + return Err("BFT decision does not strictly advance the PoW finality target".into()); + } + let response = bounded_state_call(call, StateRequest::Tip, "PoW tip lookup").await?; + let StateResponse::Tip(Some((tip_height, _))) = response else { + return Err("PoW tip is unavailable while validating BFT depth".into()); + }; + let confirmed_span = tip_height + .0 + .checked_sub(first_height.0) + .and_then(|depth| depth.checked_add(1)) + .ok_or("BFT finalization candidate is above the PoW tip")?; + if confirmed_span < expected as u32 { + return Err("BFT finalization candidate lacks the required canonical depth".into()); + } + Ok((first_height, first_hash)) +} + +fn validate_proposal_valid_round(valid_round: i64, decision_round: i32) -> Result<(), String> { + if !(0..=MAX_CANONICAL_CONSENSUS_ROUND).contains(&i64::from(decision_round)) { + return Err("decision round is outside the canonical domain".into()); + } + if valid_round == -1 { + return Ok(()); + } + if !(0..=MAX_CANONICAL_CONSENSUS_ROUND).contains(&valid_round) { + return Err("proposal valid_round is outside the canonical domain".into()); + } + if valid_round >= i64::from(decision_round) { + return Err("proposal valid_round must precede the decision round".into()); + } + Ok(()) +} + +fn pos_store_v2_payload_hash(payload: &[u8]) -> [u8; 32] { + let mut hasher = blake3::Hasher::new(); + hasher.update(POS_STORE_V2_HASH_DOMAIN); + hasher.update(&(payload.len() as u64).to_le_bytes()); + hasher.update(payload); + hasher.finalize().into() +} + +fn encode_pos_store_v2_frame( + new_block: &BftBlock, + fat_pointer: &FatPointerToBftBlock, + next_finalizers: &[RosterMember], + proposal_valid_round: i64, + tender_proposal_sigs: &[TMSig], +) -> Result, String> { + if next_finalizers.len() as u64 > MAX_POS_STORE_ROSTER_MEMBERS { + return Err("next PoS roster exceeds the durable-store bound".into()); + } + if tender_proposal_sigs.len() as u64 > MAX_POS_STORE_PROPOSAL_SIGNATURES { + return Err("proposal-signature set exceeds the durable-store bound".into()); + } + validate_proposal_valid_round( + proposal_valid_round, + fat_pointer.get_vote_template().round, + )?; + if tender_proposal_sigs.is_empty() && proposal_valid_round != -1 { + return Err("proposal valid_round requires a non-empty proposal manifest".into()); + } + + let mut payload = Vec::new(); + new_block + .zcash_serialize(&mut payload) + .map_err(|error| format!("failed to encode BFT decision: {error}"))?; + fat_pointer + .zcash_serialize(&mut payload) + .map_err(|error| format!("failed to encode BFT certificate: {error}"))?; + payload.extend_from_slice(&(next_finalizers.len() as u64).to_le_bytes()); + for member in next_finalizers { + member.write_to_vec(&mut payload); + } + payload.extend_from_slice(&proposal_valid_round.to_le_bytes()); + payload.extend_from_slice(&(tender_proposal_sigs.len() as u64).to_le_bytes()); + for signature in tender_proposal_sigs { + payload.extend_from_slice(&signature.0); + } + if payload.len() as u64 > MAX_POS_STORE_V2_PAYLOAD_BYTES { + return Err("PoS v2 decision payload exceeds the durable-store bound".into()); + } + + let mut frame = Vec::with_capacity(POS_STORE_V2_HEADER_LEN as usize + payload.len()); + frame.extend_from_slice(&POS_STORE_V2_MAGIC); + frame.extend_from_slice(&(payload.len() as u64).to_le_bytes()); + frame.extend_from_slice(&pos_store_v2_payload_hash(&payload)); + frame.extend_from_slice(&payload); + Ok(frame) +} + +fn decode_complete_pos_store_v2_frame(bytes: &[u8]) -> Result { + if bytes.len() < POS_STORE_V2_HEADER_LEN as usize || bytes[..8] != POS_STORE_V2_MAGIC { + return Err("PoS v2 frame header is missing".into()); + } + let payload_len = u64::from_le_bytes(bytes[8..16].try_into().unwrap()); + if payload_len > MAX_POS_STORE_V2_PAYLOAD_BYTES { + return Err("PoS v2 payload length exceeds the durable-store bound".into()); + } + let expected_len = POS_STORE_V2_HEADER_LEN + .checked_add(payload_len) + .ok_or("PoS v2 frame length overflows")?; + if bytes.len() as u64 != expected_len { + return Err("PoS v2 frame length does not match its header".into()); + } + let payload = &bytes[POS_STORE_V2_HEADER_LEN as usize..]; + let expected_hash: [u8; 32] = bytes[16..48].try_into().unwrap(); + if pos_store_v2_payload_hash(payload) != expected_hash { + return Err("PoS v2 frame payload hash mismatch".into()); + } + let mut cursor = Cursor::new(payload); + let record = read_stored_pos_decision_payload(&mut cursor, true)?; + if cursor.position() != payload.len() as u64 { + return Err("PoS v2 payload contains trailing bytes".into()); + } + Ok(record) +} + +fn decode_complete_pos_store_record(bytes: &[u8]) -> Result { + if bytes.starts_with(&POS_STORE_V2_MAGIC) { + return decode_complete_pos_store_v2_frame(bytes); + } + let mut cursor = Cursor::new(bytes); + let record = read_stored_pos_decision_payload(&mut cursor, false)?; + if cursor.position() != bytes.len() as u64 { + return Err("legacy PoS record contains trailing bytes".into()); + } + Ok(record) +} + +fn read_exact_pos_store_at(file: &File, offset: u64, bytes: &mut [u8]) -> Result<(), String> { + let mut filled = 0usize; + while filled < bytes.len() { + let read_offset = offset + .checked_add( + u64::try_from(filled) + .map_err(|_| "PoS positional-read offset does not fit u64")?, + ) + .ok_or("PoS positional-read offset overflows u64")?; + #[cfg(unix)] + let read = { + use std::os::unix::fs::FileExt; + file.read_at(&mut bytes[filled..], read_offset) + }; + #[cfg(windows)] + let read = { + use std::os::windows::fs::FileExt; + file.seek_read(&mut bytes[filled..], read_offset) + }; + #[cfg(not(any(unix, windows)))] + let read: std::io::Result = Err(std::io::Error::new( + std::io::ErrorKind::Unsupported, + "positional PoS-store reads are unsupported on this platform", + )); + let read = read.map_err(|error| { + format!("failed positional PoS-store read at byte {read_offset}: {error}") + })?; + if read == 0 { + return Err(format!( + "PoS-store record is truncated at byte {read_offset}" + )); + } + filled = filled + .checked_add(read) + .ok_or("PoS positional-read length overflows usize")?; + } + Ok(()) +} + +fn read_indexed_pos_store_record( + file: &File, + index: PosStoreRecordIndex, +) -> Result { + let maximum_len = POS_STORE_V2_HEADER_LEN + .checked_add(MAX_POS_STORE_V2_PAYLOAD_BYTES) + .ok_or("maximum PoS record length overflows u64")?; + if index.len == 0 || index.len > maximum_len { + return Err(format!( + "indexed PoS record length {} is outside the bounded loader domain", + index.len, + )); + } + let len = usize::try_from(index.len) + .map_err(|_| "indexed PoS record length does not fit usize")?; + let mut bytes = vec![0u8; len]; + read_exact_pos_store_at(file, index.offset, &mut bytes)?; + decode_complete_pos_store_record(&bytes) +} + +fn populate_reconstructed_round_counts( + round: &mut tenderlink::RoundData, + roster: &[SortedRosterMember], + fat_pointer: &FatPointerToBftBlock, +) -> Result<(), String> { + let active_len = usize::min(100, roster.len()); + let signed_stake = roster[..active_len] + .iter() + .filter(|member| { + fat_pointer + .signatures + .iter() + .any(|signature| signature.pub_key == member.pub_key) + }) + .try_fold(0u64, |total, member| total.checked_add(member.stake)) + .ok_or("reconstructed signing stake overflows u64")?; + round.counts = tenderlink::ConsensusCounts { + anys: signed_stake, + prevotes: 0, + nil_prevotes: 0, + yes_prevotes: 0, + precommits: signed_stake, + yes_precommits: signed_stake, + }; + Ok(()) +} + +async fn load_historical_committed_round( + tfl_handle: &TFLServiceHandle, + bootstrap_roster: Arc>, + height: u64, +) -> Result, String> { + let requested_index = usize::try_from(height) + .map_err(|_| "requested historical BFT height does not fit usize")?; + let (file, target_index, previous_index) = { + let internal = tfl_handle.internal.lock().await; + if internal.path_to_pos_store_file.as_os_str().is_empty() { + return Ok(None); + } + if internal.pos_store_records.len() != internal.bft_blocks.len() { + return Err("PoS record index is not aligned with the committed BFT chain".into()); + } + if requested_index >= internal.bft_blocks.len() { + return Ok(None); + } + let target_index = *internal + .pos_store_records + .get(requested_index) + .ok_or("authenticated PoS record index is missing the requested height")?; + let previous_index = requested_index + .checked_sub(1) + .map(|index| internal.pos_store_records[index]); + let file = internal + .pos_store_read_file + .as_ref() + .cloned() + .ok_or("configured PoS store has no held positional-read handle")?; + (file, target_index, previous_index) + }; + let config = tfl_handle.config.clone(); + + tokio::task::spawn_blocking(move || { + let target = read_indexed_pos_store_record(&file, target_index)?; + if !target.is_v2 || target.proposal_sigs.is_empty() { + // Legacy records and force-fed records do not carry an authenticated + // proposal manifest, so proposal chunks must never be fabricated. + return Ok(None); + } + let previous = previous_index + .map(|index| read_indexed_pos_store_record(&file, index)) + .transpose()?; + let null_parent = FatPointerToBftBlock::null(); + let expected_parent = previous + .as_ref() + .map(|record| &record.fat_pointer) + .unwrap_or(&null_parent); + validate_stored_bft_semantics( + &config, + &target.block, + previous.as_ref().map(|record| &record.block), + height, + expected_parent, + )?; + + let previous_final_height = previous_index + .map(|index| u64::from(index.finalized_bc_height)) + .unwrap_or(0); + let current_finalizers = previous + .as_ref() + .map(|record| record.next_roster.clone()) + .unwrap_or_else(|| bootstrap_roster.as_ref().clone()); + let terminated = terminated_finalizers_at( + &config.hardforks, + height, + previous_final_height, + ); + let current_roster = tenderlink_roster_from_internal( + ¤t_finalizers, + &terminated, + ); + let vote_namespace = namespace_for_bft_height(&config.hardforks, height); + let mut round = verify_decided_fat_pointer_quorum( + &target.block, + &target.fat_pointer, + ¤t_roster, + vote_namespace, + target.proposal_sigs.clone(), + )?; + round.proposal_valid_round = target.proposal_valid_round; + tenderlink::verify_reconstructed_proposal_manifest( + &HashKeys::default(), + &round, + )?; + populate_reconstructed_round_counts( + &mut round, + ¤t_roster, + &target.fat_pointer, + )?; + if round.height != height { + return Err("authenticated historical round has the wrong height".into()); + } + Ok(Some(round)) + }) + .await + .map_err(|error| format!("historical PoS loader task failed: {error}"))? +} + +fn is_exact_strict_frame_prefix(tail: &[u8], complete_frame: &[u8]) -> bool { + tail.len() < complete_frame.len() && complete_frame.starts_with(tail) +} + +fn append_pos_store_decision( + internal: &mut TFLServiceInternal, + new_block: &BftBlock, + fat_pointer: &FatPointerToBftBlock, + next_finalizers: &[RosterMember], + proposal_valid_round: i64, + tender_proposal_sigs: &[TMSig], +) -> Result, String> { + if internal.path_to_pos_store_file.as_os_str().is_empty() { + return Ok(None); + } + let append_bytes = encode_pos_store_v2_frame( + new_block, + fat_pointer, + next_finalizers, + proposal_valid_round, + tender_proposal_sigs, + )?; + + let torn_tail = internal.pos_store_unverified_tail.clone(); + + let file = internal + .pos_store_file + .as_mut() + .ok_or("configured PoS store is not held exclusively")?; + let durable_end = file + .seek(SeekFrom::End(0)) + .map_err(|error| format!("failed to seek PoS store: {error}"))?; + let (start_offset, original_tail) = if let Some(torn) = &torn_tail { + if torn.offset.checked_add(torn.bytes.len() as u64) != Some(durable_end) { + return Err("quarantined PoS tail does not end at the durable EOF".into()); + } + if !is_exact_strict_frame_prefix(&torn.bytes, &append_bytes) { + return Err( + "quarantined PoS tail is not an exact strict prefix of this certified decision" + .into(), + ); + } + file.seek(SeekFrom::Start(torn.offset)) + .map_err(|error| format!("failed to seek exact PoS-tail repair: {error}"))?; + (torn.offset, Some(torn.bytes.clone())) + } else { + (durable_end, None) + }; + let write_result = (|| -> Result<[u8; 32], String> { + file.write_all(&append_bytes) + .map_err(|error| format!("failed to append PoS decision: {error}"))?; + file.sync_all() + .map_err(|error| format!("failed to sync PoS decision: {error}"))?; + file.seek(SeekFrom::Start(start_offset)) + .map_err(|error| format!("failed to seek PoS decision readback: {error}"))?; + let mut readback = vec![0u8; append_bytes.len()]; + file.read_exact(&mut readback) + .map_err(|error| format!("failed to reread PoS decision: {error}"))?; + if readback != append_bytes { + return Err("durable PoS decision differs from its append bytes".into()); + } + let reread = decode_complete_pos_store_v2_frame(&readback)?; + if reread.block != *new_block || reread.fat_pointer != *fat_pointer { + return Err("durably reread BFT decision differs from the certified value".into()); + } + if reread.next_roster != next_finalizers { + return Err("durably reread next roster differs from finalized state".into()); + } + if reread.proposal_valid_round != proposal_valid_round + || reread.proposal_sigs != tender_proposal_sigs + { + return Err("durable proposal context changed during readback".into()); + } + Ok(reread.block.blake3_hash().0) + })(); + + match write_result { + Ok(hash) => { + internal.pos_store_unverified_tail = None; + file.seek(SeekFrom::End(0)) + .map_err(|error| format!("failed to restore PoS append position: {error}"))?; + Ok(Some(PosStoreAppendReceipt { + durable_parent_commit: hash, + offset: start_offset, + len: u64::try_from(append_bytes.len()) + .map_err(|_| "PoS decision frame length does not fit u64")?, + })) + } + Err(error) => { + let rollback = (|| -> std::io::Result<()> { + file.set_len(start_offset)?; + file.seek(SeekFrom::Start(start_offset))?; + if let Some(bytes) = &original_tail { + file.write_all(bytes)?; + } + file.sync_all()?; + file.seek(SeekFrom::End(0))?; + Ok(()) + })(); + match rollback { + Ok(()) => Err(error), + Err(rollback_error) => Err(format!( + "{error}; failed to roll back the torn PoS-store append: {rollback_error}" + )), + } + } + } +} + +async fn apply_verified_decided_bft_block( + tfl_handle: &TFLServiceHandle, + new_block: &BftBlock, + fat_pointer: &FatPointerToBftBlock, + proposal_valid_round: i64, + tender_proposal_sigs: Vec, +) -> Result<(Vec, [u8; 32], Option<[u8; 32]>), String> { + let _decision_guard = tfl_handle.decision_apply_gate.lock().await; + let call = tfl_handle.call.clone(); + let (current_finalizers, previous_final, expected_tip, expected_height) = { + let internal = tfl_handle.internal.lock().await; + ( + internal.finalizers_at_current_height.clone(), + internal.latest_final_block, + internal.fat_pointer_to_tip.clone(), + internal.bft_blocks.len(), + ) + }; + if new_block.height as usize != expected_height { + return Err("decided BFT block height is not the next chain index".into()); + } + if new_block.previous_block_fat_ptr.points_at_block_hash() + != expected_tip.points_at_block_hash() + { + return Err("decided BFT block does not extend the current certified tip".into()); + } + let previous_final_height = previous_final.map(|(height, _)| height); + let terminated = terminated_finalizers_at( + &tfl_handle.config.hardforks, + new_block.height as u64, + previous_final_height.map_or(0, |height| height.0 as u64), + ); + let current_roster = tenderlink_roster_from_internal(¤t_finalizers, &terminated); + let vote_namespace = namespace_for_bft_height( + &tfl_handle.config.hardforks, + new_block.height as u64, + ); + let mut decided_round = verify_decided_fat_pointer_quorum( + new_block, + fat_pointer, + ¤t_roster, + vote_namespace, + tender_proposal_sigs.clone(), + )?; + decided_round.proposal_valid_round = proposal_valid_round; + if tender_proposal_sigs.is_empty() { + if proposal_valid_round != -1 { + return Err("proposal valid_round requires a complete proposal manifest".into()); + } + } else { + tenderlink::verify_reconstructed_proposal_manifest( + &HashKeys::default(), + &decided_round, + )?; + } + + if validate_bft_block(tfl_handle, new_block).await + != (tenderlink::TMStatus::Pass, tenderlink::TMStatusReason::None) + { + return Err("decided BFT block failed commit-time semantic validation".into()); + } + let (new_final_height, new_final_hash) = + validated_pow_header_chain(&call, new_block, previous_final_height).await?; + + let response = bounded_state_call( + &call, + StateRequest::CrosslinkFinalizeBlock(new_final_hash), + "crosslink finalization", + ) + .await?; + let StateResponse::CrosslinkFinalized(finalized_hash, aggregated_stakes) = response else { + return Err("crosslink finalization returned the wrong response type".into()); + }; + if finalized_hash != new_final_hash { + return Err("state finalized a different PoW hash".into()); + } + let next_finalizers = if aggregated_stakes.is_empty() { + if expected_height == 0 { + current_finalizers + } else { + return Err("state returned an empty bonded roster after non-genesis finalization".into()); + } + } else { + aggregated_stakes + .into_iter() + .map(|(pub_key, voting_power)| RosterMember { + pub_key, + voting_power, + txids: Vec::new(), + }) + .collect() + }; + let next_bft_height = (new_block.height as u64) + .checked_add(1) + .ok_or("BFT height overflow")?; + let next_terminated = terminated_finalizers_at( + &tfl_handle.config.hardforks, + next_bft_height, + new_final_height.0 as u64, + ); + let next_roster = tenderlink_roster_from_internal(&next_finalizers, &next_terminated); + tenderlink::validate_consensus_roster(&next_roster)?; + let next_vote_namespace = + namespace_for_bft_height(&tfl_handle.config.hardforks, next_bft_height); + + let mut internal = tfl_handle.internal.lock().await; + if internal.bft_blocks.len() != expected_height + || internal.fat_pointer_to_tip != expected_tip + || internal.latest_final_block != previous_final + || internal.bft_height_by_hash.len() != expected_height + { + return Err("BFT tip changed while the decided block was being applied".into()); + } + let new_block_hash = new_block.blake3_hash().0; + if internal.bft_height_by_hash.contains_key(&new_block_hash) { + return Err("decided BFT block hash already exists at another height".into()); + } + if !internal.path_to_pos_store_file.as_os_str().is_empty() + && internal.pos_store_records.len() != expected_height + { + return Err("PoS record index is not aligned with the decided BFT height".into()); + } + if !internal.path_to_pos_store_file.as_os_str().is_empty() + && internal.pos_store_read_file.is_none() + { + return Err("configured PoS store has no held positional-read handle".into()); + } + let append_receipt = append_pos_store_decision( + &mut internal, + new_block, + fat_pointer, + &next_finalizers, + proposal_valid_round, + &tender_proposal_sigs, + )?; + let durable_parent_commit = append_receipt.map(|receipt| receipt.durable_parent_commit); + if let Some(receipt) = append_receipt { + internal.pos_store_records.push(PosStoreRecordIndex { + offset: receipt.offset, + len: receipt.len, + finalized_bc_height: new_final_height.0, + }); + } + internal.bft_blocks.push(new_block.clone()); + let replaced = internal + .bft_height_by_hash + .insert(new_block_hash, expected_height); + debug_assert!(replaced.is_none()); + internal.fat_pointer_to_tip = fat_pointer.clone(); + internal.latest_final_block = Some((new_final_height, new_final_hash)); + internal.current_bc_final = Some((new_final_height, new_final_hash)); + internal.finalizers_at_current_height = next_finalizers; + internal.pending_reflush = Some(new_final_hash); + drop(internal); + + match bounded_crosslink_reflush( + &call, + new_final_hash, + "post-install crosslink reflush", + ) + .await + { + Ok(()) => { + let mut internal = tfl_handle.internal.lock().await; + if internal.pending_reflush == Some(new_final_hash) { + internal.pending_reflush = None; + } + } + Err(error) => { + warn!(%error, "post-install crosslink reflush remains pending for bounded retry"); + } + } + info!( + "Applied certified BFT block {} and crosslink-finalized {}", + new_block.height, new_final_hash + ); + Ok((next_roster, next_vote_namespace, durable_parent_commit)) +} + +#[cfg(any())] async fn handle_new_decided_bft_block( tfl_handle: &TFLServiceHandle, new_block: &BftBlock, fat_pointer: &FatPointerToBftBlock, tender_proposal_sigs: Vec, -) -> Vec { +) -> (Vec, [u8; 32], Option<[u8; 32]>) { // CHECK PRECONDITIONS { if fat_pointer.points_at_block_hash() != new_block.blake3_hash() { @@ -716,6 +1771,8 @@ async fn handle_new_decided_bft_block( } let call = tfl_handle.call.clone(); + #[cfg(any())] + { let new_final_hash = ZebBlockHash(BlockHash::from_header_data(new_block.headers.first().expect("at least 1 header")).0); let new_final_height = block_height_from_hash(&call, new_final_hash).await.unwrap(); // `height` is now the 0-based canonical height, i.e. the chain index directly. @@ -757,11 +1814,7 @@ async fn handle_new_decided_bft_block( internal.fat_pointer_to_tip = fat_pointer.clone(); internal.latest_final_block = Some((new_final_height, new_final_hash)); - // Note(Sam): IT IS VERY IMPORTANT THAT WE DROP THE LOCK BECAUSE ZEBRA_STATE MAY CALL US BACK. - // @Todo: once new_network syncs the BFT chain itself, this finalize becomes a message to - // new_network carrying the whole decision, and the reentrancy hazard goes with it -- the - // call graph stops being circular. - drop(internal); + drop(internal); // Note(Sam): IT IS VERY IMPORTANT THAT WE DROP THE LOCK BECAUSE ZEBRA_STATE MAY CALL US BACK let got_stakes = loop { match (call.state)(zebra_state::Request::CrosslinkFinalizeBlock(new_final_hash)).await { Ok(zebra_state::Response::CrosslinkFinalized(hash, aggregated_stakes)) => { @@ -798,7 +1851,11 @@ async fn handle_new_decided_bft_block( } //println!("Storing pow ({:?}, {:?}) with roster: {:?}", new_final_height, new_final_hash, internal.finalizers_at_current_height); - if internal.path_to_pos_store_file.to_str() != Some("") { + let (durable_parent_commit, durable_next_finalizers) = if internal.path_to_pos_store_file.to_str() != Some("") { + assert!((internal.finalizers_at_current_height.len() as u64) <= MAX_POS_STORE_ROSTER_MEMBERS, + "next PoS roster exceeds the durable store bound"); + assert!((tender_proposal_sigs.len() as u64) <= MAX_POS_STORE_PROPOSAL_SIGNATURES, + "proposal-signature set exceeds the durable store bound"); let mut append_bytes: Vec = Vec::new(); new_block.zcash_serialize(&mut append_bytes).unwrap(); fat_pointer.zcash_serialize(&mut append_bytes).unwrap(); @@ -807,30 +1864,88 @@ async fn handle_new_decided_bft_block( v.write_to_vec(&mut append_bytes); } append_bytes.extend_from_slice(&(tender_proposal_sigs.len() as u64).to_le_bytes()); - for sig in tender_proposal_sigs { + for sig in &tender_proposal_sigs { append_bytes.extend_from_slice(&sig.0); } - let mut file = OpenOptions::new().append(true).open(&internal.path_to_pos_store_file).unwrap(); - file.write_all(&append_bytes).unwrap(); - file.flush().unwrap(); - } + let (reread_hash, reread_roster) = { + let file = internal.pos_store_file.as_mut() + .expect("configured PoS store must remain exclusively open for the service lifetime"); + let start_offset = file.seek(SeekFrom::End(0)).unwrap(); + file.write_all(&append_bytes).unwrap(); + file.sync_all().unwrap(); + file.seek(SeekFrom::Start(start_offset)).unwrap(); + let mut readback = vec![0u8; append_bytes.len()]; + file.read_exact(&mut readback).unwrap(); + assert_eq!(readback, append_bytes, "durable PoS-store readback differs from appended decision"); + let mut cursor = Cursor::new(&readback); + let reread_block = BftBlock::zcash_deserialize(&mut cursor).unwrap(); + assert_eq!(reread_block, *new_block, "durably reread BFT block differs from decided block"); + let reread_fat_pointer = FatPointerToBftBlock::zcash_deserialize(&mut cursor).unwrap(); + assert_eq!(reread_fat_pointer, *fat_pointer, "durably reread fat pointer differs from decided certificate"); + + let mut count_bytes = [0u8; 8]; + cursor.read_exact(&mut count_bytes).unwrap(); + let reread_roster_count = u64::from_le_bytes(count_bytes); + assert!(reread_roster_count <= MAX_POS_STORE_ROSTER_MEMBERS, + "durably reread roster exceeds the PoS-store bound"); + let mut reread_roster = Vec::with_capacity(reread_roster_count as usize); + for _ in 0..reread_roster_count { + reread_roster.push(RosterMember::read_from(&mut cursor).unwrap()); + } + assert_eq!(reread_roster, internal.finalizers_at_current_height, + "durably reread next roster differs from the finalized state roster"); + + cursor.read_exact(&mut count_bytes).unwrap(); + let reread_signature_count = u64::from_le_bytes(count_bytes); + assert!(reread_signature_count <= MAX_POS_STORE_PROPOSAL_SIGNATURES, + "durably reread proposal-signature set exceeds the PoS-store bound"); + let mut reread_signatures = Vec::with_capacity(reread_signature_count as usize); + for _ in 0..reread_signature_count { + let mut signature = TMSig::NIL; + cursor.read_exact(&mut signature.0).unwrap(); + reread_signatures.push(signature); + } + assert_eq!(reread_signatures, tender_proposal_sigs, + "durably reread proposal signatures differ from the decided record"); + assert_eq!(cursor.position(), readback.len() as u64, + "durable PoS-store decision record has trailing or unparsed bytes"); + (reread_block.blake3_hash().0, reread_roster) + }; + (Some(reread_hash), reread_roster) + } else { + (None, internal.finalizers_at_current_height.clone()) + }; // The returned roster is for the NEXT height (tenderlink advances to it after this decision): // its index is the new chain length. Exclude finalizers terminated at that height, inclusive, // so they are already out of the roster that will vote on a hardfork block scheduled there. let next_bft_height = internal.bft_blocks.len() as u64; let terminated = terminated_finalizers_at(&tfl_handle.config.hardforks, next_bft_height, new_final_height.0 as u64); - tenderlink_roster_from_internal(&internal.finalizers_at_current_height, &terminated) + let next_vote_namespace = namespace_for_bft_height(&tfl_handle.config.hardforks, next_bft_height); + ( + tenderlink_roster_from_internal( + &durable_next_finalizers, + &terminated, + ), + next_vote_namespace, + durable_parent_commit, + ) +} } /// Build the tenderlink consensus roster from the internal roster, excluding any finalizer in /// `terminated` (terminated by a user-led hardfork; see [`terminated_finalizers_at`]). The /// filtering is a pure membership test, mirroring how the viz excludes terminated finalizers. /// Pass an empty set to build the roster unfiltered. -fn tenderlink_roster_from_internal(vals: &[RosterMember], terminated: &HashSet) -> Vec { +fn tenderlink_roster_from_internal( + vals: &[RosterMember], + terminated: &HashSet, +) -> Vec { let mut ret: Vec = vals .iter() .map(|v| SortedRosterMember { + // Consensus keys are raw bond identities. Byte-reversed twins are distinct + // identities and must never be normalized relative to this node. pub_key: PubKeyID(v.pub_key.into()), stake: v.voting_power, cumulative_stake: 0, @@ -860,6 +1975,75 @@ fn tenderlink_roster_from_internal(vals: &[RosterMember], terminated: &HashSet

Result, String> { + let mut seen = HashSet::new(); + let mut roster = Vec::with_capacity(config.bootstrap_bft_roster.len()); + for member in &config.bootstrap_bft_roster { + let public_key = decode_consensus_public_key_hex(&member.consensus_public_key)?; + if member.voting_power == 0 { + return Err(format!("bootstrap roster member {public_key} has zero voting power")); + } + if !seen.insert(public_key) { + return Err(format!("bootstrap roster contains duplicate key {public_key}")); + } + roster.push(RosterMember { + pub_key: public_key.0, + voting_power: member.voting_power, + txids: Vec::new(), + }); + } + Ok(roster) +} + +fn finalizer_peer_addresses_from_explicit_config( + configured_peers: &[crate::config::BftPeerIdentity], + public_address: &str, + my_public_key: PubKeyID, + my_noise_keypair: &tenderlink::bandwidth_test::IdentityKeyPair, +) -> Result, String> { + use tenderlink::bandwidth_test::STPAddress; + + let mut configured_by_key = std::collections::BTreeMap::new(); + for peer in configured_peers { + let public_key = decode_consensus_public_key_hex(&peer.consensus_public_key)?; + let noise_public_key = decode_exact_lower_hex_32_named( + &peer.noise_public_key, + "peer Noise public key", + true, + )?; + let (ip, port) = tenderlink::parse_to_ipv6_bytes(&peer.address) + .map_err(|error| format!("invalid peer endpoint {}: {error}", peer.address))?; + let address = STPAddress { + ip, + port, + magic1: tenderlink::CRYPTO_MAGIC, + key: noise_public_key.to_vec(), + }; + if configured_by_key.insert(public_key, address).is_some() { + return Err(format!("duplicate peer consensus key {public_key}")); + } + } + let (local_ip, local_port) = tenderlink::parse_to_ipv6_bytes(public_address) + .map_err(|error| format!("invalid local validator endpoint {public_address}: {error}"))?; + let local_address = STPAddress::from(local_ip, local_port, my_noise_keypair); + if let Some(configured_local) = configured_by_key.insert(my_public_key, local_address.clone()) { + if configured_local != local_address { + return Err( + "local validator peer binding conflicts with its asserted endpoint or Noise key" + .into(), + ); + } + } + Ok(configured_by_key + .into_iter() + .map(|(bft_pk, address)| { + tenderlink::FinalizerPeerAddress { bft_pk, address } + }) + .collect()) +} + async fn validate_bft_block( tfl_handle: &TFLServiceHandle, new_block: &BftBlock, @@ -867,6 +2051,11 @@ async fn validate_bft_block( let mut internal = tfl_handle.internal.lock().await; let call = tfl_handle.call.clone(); + if new_block.headers.is_empty() { + warn!("BFT block has no PoW finalization-candidate header"); + return (tenderlink::TMStatus::Fail, tenderlink::TMStatusReason::None); + } + if new_block.previous_block_fat_ptr.points_at_block_hash() != internal.fat_pointer_to_tip.points_at_block_hash() { @@ -975,32 +2164,29 @@ async fn validate_bft_block( // Captured before dropping the lock: an already-finalized hash we can safely use to kick // the state's non-finalized queue below without risking a premature finalization. - let already_finalized_hash = internal.latest_final_block.map(|(_, hash)| hash); + let previous_final_height = internal.latest_final_block.map(|(height, _)| height); drop(internal); - let new_final_hash = ZebBlockHash(BlockHash::from_header_data(new_block.headers.first().expect("at least 1 header")).0); - let new_final_pow_height = - if let Some(new_final_height) = block_height_from_hash(&call, new_final_hash).await { - new_final_height.0 - } else { - warn!( - "Didn't have hash available for confirmation: {}", - new_final_hash - ); - // The PoW block we need is most likely sitting deferred in the state's non-finalized - // queue (held back by the crosslink commit gate — e.g. a transient try_lock miss, or - // waiting on a BFT block that has since arrived). Kick the queue so it is re-evaluated - // and committed, letting tenderlink's retried validation find it. We trigger the - // re-flush via a finalize of the *already-finalized* tip: that finalize is a guaranteed - // no-op (the tip is no longer in the non-finalized state, so nothing is prematurely - // finalized), but the request handler re-flushes the non-finalized queue regardless. - // (Removed) This used to issue a no-op finalize of the already-finalized tip purely - // to make the state re-flush its deferred non-finalized queue. new_network owns that - // queue now and re-evaluates it every tick, so there is nothing to kick. - let _ = already_finalized_hash; - return (tenderlink::TMStatus::Indeterminate, tenderlink::TMStatusReason::NeedsBlock { hash: new_final_hash.0 }); - }; - return (tenderlink::TMStatus::Pass, tenderlink::TMStatusReason::None); + match validated_pow_header_chain(&call, new_block, previous_final_height).await { + Ok(_) => (tenderlink::TMStatus::Pass, tenderlink::TMStatusReason::None), + Err(error) => { + warn!(%error, "BFT proposal failed canonical PoW ancestry validation"); + let is_transient = error.contains("timed out") + || error.contains(" lookup failed:") + || error.contains("is unavailable"); + if is_transient { + let needed_hash = BlockHash::from_header_data( + new_block.headers.first().expect("non-empty checked above"), + ); + ( + tenderlink::TMStatus::Indeterminate, + tenderlink::TMStatusReason::NeedsBlock { hash: needed_hash.0 }, + ) + } else { + (tenderlink::TMStatus::Fail, tenderlink::TMStatusReason::None) + } + } + } } fn fat_pointer_to_block_at_height( @@ -1117,36 +2303,853 @@ pub fn run_tfl_test(internal_handle: TFLServiceHandle) { eprintln!("..."); } - eprintln!("\n\nInstruction sequence:"); - dump_test_instrs(); + eprintln!("\n\nInstruction sequence:"); + dump_test_instrs(); + + #[cfg(not(feature = "viz_gui"))] + std::process::abort(); + } + })); + + tokio::task::spawn(test_format::instr_reader(internal_handle)); +} + +/// Vote-namespacing domain separator for a BFT height: a flat blake3 hash of the prefix of +/// scheduled hardforks whose `bft_certificate_height <= bft_height` (inclusive of a hardfork at +/// `bft_height` itself), concatenated in canonical schedule order. An empty prefix yields +/// `[0; 32]` (nil), so the no-hardfork case is a backwards-compatible no-op in tenderlink's +/// signing. The schedule is sorted with non-decreasing `bft_certificate_height` (several rules +/// may share one certificate height), so the filtered set is exactly the prefix. +fn namespace_for_bft_height(hardforks: &[crate::config::HardForkConfig], bft_height: u64) -> [u8; 32] { + let mut hasher = blake3::Hasher::new(); + let mut any = false; + for hf in hardforks.iter().filter(|hf| hf.bft_certificate_height <= bft_height) { + let mut bytes = Vec::new(); + hf.zcash_serialize(&mut bytes).expect("serializing to a Vec is infallible"); + hasher.update(&bytes); + any = true; + } + if any { hasher.finalize().into() } else { [0u8; 32] } +} + +fn deserialize_bft_block_exact(bytes: &[u8]) -> Result { + let mut cursor = Cursor::new(bytes); + let block = BftBlock::zcash_deserialize(&mut cursor) + .map_err(|error| format!("failed to deserialize BFT block: {error}"))?; + if cursor.position() != bytes.len() as u64 { + return Err("BFT block payload contains trailing bytes".into()); + } + Ok(block) +} + +fn decode_exact_lower_hex_32_named( + value: &str, + field: &'static str, + reject_zero: bool, +) -> Result<[u8; 32], String> { + if value.len() != 64 || !value.bytes().all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) { + return Err(format!( + "{field} must be exactly 64 lowercase hex characters" + )); + } + let nibble = |byte: u8| -> u8 { + match byte { + b'0'..=b'9' => byte - b'0', + b'a'..=b'f' => byte - b'a' + 10, + _ => unreachable!("hex alphabet checked above"), + } + }; + let bytes = value.as_bytes(); + let mut decoded = [0u8; 32]; + for (index, output) in decoded.iter_mut().enumerate() { + *output = (nibble(bytes[index * 2]) << 4) | nibble(bytes[index * 2 + 1]); + } + if reject_zero && decoded == [0u8; 32] { + return Err(format!("{field} must not be zero")); + } + Ok(decoded) +} + +fn decode_exact_lower_hex_32(value: &str) -> Result<[u8; 32], String> { + decode_exact_lower_hex_32_named(value, "bootstrap receipt BLAKE3", true) +} + +pub(crate) fn decode_consensus_public_key_hex(value: &str) -> Result { + decode_exact_lower_hex_32_named(value, "consensus public key", true).map(PubKeyID) +} + +#[derive(Debug, serde::Deserialize)] +#[serde(deny_unknown_fields)] +struct SignerMigrationReceiptV1 { + schema: String, + action: String, + operator_authorized: bool, + independent_anchor_authorized: bool, + global_single_signer_fence_confirmed: bool, + frozen_legacy_binary_sha256: String, + frozen_legacy_config_sha256: String, + composite_checkpoint_manifest_sha256: String, + pos_store_sha256: String, + pos_store_size_bytes: u64, + pos_store_complete_eof: bool, + pos_store_record_count: u64, + pos_store_first_bft_height: u64, + validator_consensus_public_key: String, + chain_id: String, + replayed_next_bft_height: u64, + bootstrap_parent_commit: String, + bootstrap_vote_namespace: String, + bootstrap_consensus_config_hash: String, + authenticated_bootstrap_roster_hash: String, + active_roster_hash: String, + active_roster_index: u32, + active_roster_len: u32, + finalized_pow_height: u32, + finalized_pow_hash: String, + peer_route_map_blake3: String, + peer_route_voting_power: u64, + required_route_voting_power: u64, + legacy_signer_fence_receipt_sha256: String, + wal_path: PathBuf, + anchor_path: PathBuf, + pos_store_path: PathBuf, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum SignerJournalState { + Uninitialized, + Existing, +} + +#[derive(Clone, Copy)] +struct SignerMigrationContext<'a> { + validator_consensus_public_key: PubKeyID, + chain_id: [u8; 32], + startup_bft_height: u64, + parent_commit: [u8; 32], + vote_namespace: [u8; 32], + consensus_config_hash: [u8; 32], + authenticated_bootstrap_roster_hash: [u8; 32], + active_roster_hash: [u8; 32], + active_roster_index: u32, + active_roster_len: u32, + finalized_pow_height: u32, + finalized_pow_hash: [u8; 32], + pos_store_size_bytes: u64, + pos_store_record_count: u64, + pos_store_complete_eof: bool, + wal_path: &'a Path, + anchor_path: &'a Path, + pos_store_path: &'a Path, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct SignerStartupAuthority { + non_genesis_receipt_hash: Option<[u8; 32]>, +} + +fn signer_journal_file_len(path: &Path) -> Result { + match std::fs::symlink_metadata(path) { + Ok(metadata) => { + if metadata.file_type().is_symlink() || !metadata.file_type().is_file() { + return Err("signer journal path must be a regular non-symlink file".into()); + } + Ok(metadata.len()) + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(0), + Err(error) => Err(format!("failed to inspect signer journal path: {error}")), + } +} + +fn signer_journal_state(wal_path: &Path, anchor_path: &Path) -> Result { + let wal_len = signer_journal_file_len(wal_path)?; + let anchor_len = signer_journal_file_len(anchor_path)?; + match (wal_len, anchor_len) { + (0, 0) => Ok(SignerJournalState::Uninitialized), + (wal, anchor) if wal > 0 && anchor > 0 => Ok(SignerJournalState::Existing), + _ => Err("signer WAL and anchor initialization states differ".into()), + } +} + +fn read_sealed_signer_migration_receipt(path: &Path) -> Result, String> { + let metadata = std::fs::symlink_metadata(path) + .map_err(|error| format!("failed to inspect signer migration receipt: {error}"))?; + if metadata.file_type().is_symlink() || !metadata.file_type().is_file() { + return Err("signer migration receipt must be a regular non-symlink file".into()); + } + if metadata.len() == 0 || metadata.len() > MAX_SIGNER_MIGRATION_RECEIPT_BYTES { + return Err("signer migration receipt size is outside the accepted bound".into()); + } + + let mut options = OpenOptions::new(); + options.read(true); + #[cfg(unix)] + { + use nix::fcntl::OFlag; + use std::os::unix::fs::OpenOptionsExt; + options.custom_flags((OFlag::O_NOFOLLOW | OFlag::O_CLOEXEC).bits()); + } + #[cfg(windows)] + { + use std::os::windows::fs::OpenOptionsExt; + options.share_mode(0); + } + let mut file = options + .open(path) + .map_err(|error| format!("failed to open signer migration receipt: {error}"))?; + let opened_metadata = file + .metadata() + .map_err(|error| format!("failed to stat signer migration receipt: {error}"))?; + if !opened_metadata.file_type().is_file() || opened_metadata.len() != metadata.len() { + return Err("signer migration receipt changed while it was opened".into()); + } + #[cfg(unix)] + { + use nix::unistd::geteuid; + use std::os::unix::fs::MetadataExt; + if opened_metadata.uid() != geteuid().as_raw() { + return Err("signer migration receipt owner mismatch".into()); + } + if opened_metadata.mode() & 0o077 != 0 { + return Err("signer migration receipt permissions are broader than 0600".into()); + } + if opened_metadata.nlink() != 1 { + return Err("signer migration receipt has unexpected hard links".into()); + } + } + + let mut bytes = Vec::with_capacity(opened_metadata.len() as usize); + std::io::Read::by_ref(&mut file) + .take(MAX_SIGNER_MIGRATION_RECEIPT_BYTES + 1) + .read_to_end(&mut bytes) + .map_err(|error| format!("failed to read signer migration receipt: {error}"))?; + if bytes.len() as u64 != opened_metadata.len() + || bytes.len() as u64 > MAX_SIGNER_MIGRATION_RECEIPT_BYTES + { + return Err("signer migration receipt changed or exceeded its bound while read".into()); + } + Ok(bytes) +} + +fn receipt_hash_field(value: &str, field: &'static str) -> Result<[u8; 32], String> { + decode_exact_lower_hex_32_named(value, field, true) +} + +fn require_receipt_hash( + value: &str, + field: &'static str, + expected: [u8; 32], +) -> Result<(), String> { + if receipt_hash_field(value, field)? != expected { + return Err(format!("signer migration receipt {field} mismatch")); + } + Ok(()) +} + +fn verify_signer_migration_receipt( + bytes: &[u8], + pinned_hash: [u8; 32], + journal_state: SignerJournalState, + context: &SignerMigrationContext<'_>, +) -> Result { + let actual_hash: [u8; 32] = blake3::hash(bytes).into(); + if actual_hash != pinned_hash { + return Err("signer migration receipt bytes do not match the configured BLAKE3".into()); + } + let receipt: SignerMigrationReceiptV1 = serde_json::from_slice(bytes) + .map_err(|error| format!("signer migration receipt JSON is invalid: {error}"))?; + if receipt.schema != SIGNER_MIGRATION_RECEIPT_SCHEMA + || receipt.action != SIGNER_MIGRATION_RECEIPT_ACTION + { + return Err("signer migration receipt schema or action mismatch".into()); + } + if !receipt.operator_authorized + || !receipt.independent_anchor_authorized + || !receipt.global_single_signer_fence_confirmed + { + return Err("signer migration receipt lacks explicit operator/fence authority".into()); + } + for (value, field) in [ + (receipt.frozen_legacy_binary_sha256.as_str(), "frozen legacy binary SHA256"), + (receipt.frozen_legacy_config_sha256.as_str(), "frozen legacy config SHA256"), + ( + receipt.composite_checkpoint_manifest_sha256.as_str(), + "composite checkpoint manifest SHA256", + ), + (receipt.pos_store_sha256.as_str(), "PoS store SHA256"), + ( + receipt.legacy_signer_fence_receipt_sha256.as_str(), + "legacy signer fence receipt SHA256", + ), + (receipt.peer_route_map_blake3.as_str(), "peer route map BLAKE3"), + ] { + receipt_hash_field(value, field)?; + } + if receipt.pos_store_size_bytes == 0 + || !receipt.pos_store_complete_eof + || receipt.pos_store_record_count == 0 + || receipt.pos_store_first_bft_height != 0 + || receipt.replayed_next_bft_height != receipt.pos_store_record_count + { + return Err("signer migration receipt does not bind a complete height-zero PoS history".into()); + } + if !context.pos_store_complete_eof + || receipt.pos_store_size_bytes > context.pos_store_size_bytes + || receipt.pos_store_record_count > context.pos_store_record_count + || receipt.replayed_next_bft_height > context.startup_bft_height + { + return Err("signer migration receipt PoS checkpoint is not an ancestor of loaded history".into()); + } + if receipt.required_route_voting_power == 0 + || receipt.peer_route_voting_power < receipt.required_route_voting_power + { + return Err("signer migration receipt lacks required authenticated peer-route stake".into()); + } + if decode_consensus_public_key_hex(&receipt.validator_consensus_public_key)? + != context.validator_consensus_public_key + { + return Err("signer migration receipt validator consensus key mismatch".into()); + } + require_receipt_hash(&receipt.chain_id, "chain ID", context.chain_id)?; + require_receipt_hash( + &receipt.bootstrap_consensus_config_hash, + "consensus config hash", + context.consensus_config_hash, + )?; + require_receipt_hash( + &receipt.authenticated_bootstrap_roster_hash, + "authenticated bootstrap roster hash", + context.authenticated_bootstrap_roster_hash, + )?; + if receipt.wal_path != context.wal_path + || receipt.anchor_path != context.anchor_path + || receipt.pos_store_path != context.pos_store_path + { + return Err("signer migration receipt journal or PoS-store path mismatch".into()); + } + + match journal_state { + SignerJournalState::Uninitialized => { + if receipt.replayed_next_bft_height != context.startup_bft_height + || receipt.pos_store_size_bytes != context.pos_store_size_bytes + || receipt.pos_store_record_count != context.pos_store_record_count + || receipt.active_roster_index != context.active_roster_index + || receipt.active_roster_len != context.active_roster_len + || receipt.finalized_pow_height != context.finalized_pow_height + { + return Err("signer migration receipt bootstrap counters mismatch".into()); + } + require_receipt_hash( + &receipt.bootstrap_parent_commit, + "bootstrap parent commit", + context.parent_commit, + )?; + require_receipt_hash( + &receipt.bootstrap_vote_namespace, + "bootstrap vote namespace", + context.vote_namespace, + )?; + require_receipt_hash( + &receipt.active_roster_hash, + "active roster hash", + context.active_roster_hash, + )?; + require_receipt_hash( + &receipt.finalized_pow_hash, + "finalized PoW hash", + context.finalized_pow_hash, + )?; + } + SignerJournalState::Existing => { + if receipt.active_roster_len == 0 + || receipt.active_roster_index >= receipt.active_roster_len + || receipt.finalized_pow_height > context.finalized_pow_height + { + return Err("signer migration receipt origin counters are invalid".into()); + } + receipt_hash_field(&receipt.bootstrap_parent_commit, "bootstrap parent commit")?; + receipt_hash_field(&receipt.bootstrap_vote_namespace, "bootstrap vote namespace")?; + receipt_hash_field(&receipt.active_roster_hash, "active roster hash")?; + receipt_hash_field(&receipt.finalized_pow_hash, "finalized PoW hash")?; + } + } + + Ok(SignerStartupAuthority { + non_genesis_receipt_hash: Some(pinned_hash), + }) +} + +fn signer_startup_authority( + config: &crate::config::Config, + context: &SignerMigrationContext<'_>, +) -> Result { + if context.wal_path == context.anchor_path { + return Err("signer WAL and independent anchor paths must differ".into()); + } + if !config.signer_independent_anchor_authorized { + return Err("independent anti-rollback/key-fencing authority is absent".into()); + } + if !context.pos_store_complete_eof { + return Err("PoS history has an unverified torn tail".into()); + } + let journal_state = signer_journal_state(context.wal_path, context.anchor_path)?; + if context.startup_bft_height == 0 { + if config.signer_non_genesis_bootstrap_receipt_blake3.is_some() + || config.signer_non_genesis_bootstrap_receipt_path.is_some() + { + return Err("genesis signer startup must not supply a non-genesis receipt".into()); + } + return Ok(SignerStartupAuthority { + non_genesis_receipt_hash: None, + }); + } + + let pinned_hash_text = config + .signer_non_genesis_bootstrap_receipt_blake3 + .as_deref() + .ok_or_else(|| "non-genesis signer startup receipt hash is absent".to_owned())?; + let pinned_hash = decode_exact_lower_hex_32(pinned_hash_text)?; + let receipt_path = config + .signer_non_genesis_bootstrap_receipt_path + .as_deref() + .ok_or("non-genesis signer startup receipt path is absent")?; + if receipt_path == context.wal_path + || receipt_path == context.anchor_path + || receipt_path == context.pos_store_path + { + return Err("signer migration receipt must be outside the WAL, anchor, and PoS store".into()); + } + let bytes = read_sealed_signer_migration_receipt(receipt_path)?; + verify_signer_migration_receipt(&bytes, pinned_hash, journal_state, context) +} + +fn canonical_validator_identity_configured( + config: &crate::config::Config, +) -> Result { + let fields = [ + config.validator_signing_key_seed.is_some(), + config.validator_consensus_public_key.is_some(), + config.validator_noise_static_key_seed.is_some(), + ]; + let configured = fields.iter().filter(|value| **value).count(); + if configured != 0 && configured != fields.len() { + return Err("canonical validator identity is partially configured".into()); + } + let complete = configured == fields.len(); + if complete && (config.explicit_bft_key_seed.is_some() || !config.bft_peers.is_empty()) { + return Err( + "legacy endpoint-derived identity fields cannot be mixed with canonical validator identity" + .into(), + ); + } + Ok(complete) +} + +struct StoredPosDecision { + block: BftBlock, + fat_pointer: FatPointerToBftBlock, + next_roster: Vec, + proposal_valid_round: i64, + proposal_sigs: Vec, + is_v2: bool, +} + +fn read_stored_pos_decision_payload( + reader: &mut R, + is_v2: bool, +) -> Result { + let block = BftBlock::zcash_deserialize(&mut *reader) + .map_err(|error| format!("stored BFT block is invalid or truncated: {error}"))?; + let fat_pointer = FatPointerToBftBlock::zcash_deserialize(&mut *reader) + .map_err(|error| format!("stored BFT certificate is invalid or truncated: {error}"))?; + + let mut count_bytes = [0u8; 8]; + reader + .read_exact(&mut count_bytes) + .map_err(|error| format!("stored next-roster count is truncated: {error}"))?; + let roster_count = u64::from_le_bytes(count_bytes); + if roster_count > MAX_POS_STORE_ROSTER_MEMBERS { + return Err(format!( + "stored next-roster count {roster_count} exceeds {MAX_POS_STORE_ROSTER_MEMBERS}" + )); + } + let mut next_roster = Vec::with_capacity(roster_count as usize); + for _ in 0..roster_count { + next_roster.push(read_stored_roster_member(reader)?); + } + + let proposal_valid_round = if is_v2 { + reader + .read_exact(&mut count_bytes) + .map_err(|error| format!("stored proposal valid_round is truncated: {error}"))?; + i64::from_le_bytes(count_bytes) + } else { + -1 + }; + validate_proposal_valid_round( + proposal_valid_round, + fat_pointer.get_vote_template().round, + )?; + + reader + .read_exact(&mut count_bytes) + .map_err(|error| format!("stored proposal-signature count is truncated: {error}"))?; + let proposal_sig_count = u64::from_le_bytes(count_bytes); + if proposal_sig_count > MAX_POS_STORE_PROPOSAL_SIGNATURES { + return Err(format!( + "stored proposal-signature count {proposal_sig_count} exceeds {MAX_POS_STORE_PROPOSAL_SIGNATURES}" + )); + } + let mut proposal_sigs = Vec::with_capacity(proposal_sig_count as usize); + for _ in 0..proposal_sig_count { + let mut signature = TMSig::NIL; + reader + .read_exact(&mut signature.0) + .map_err(|error| format!("stored proposal signature is truncated: {error}"))?; + proposal_sigs.push(signature); + } + + Ok(StoredPosDecision { + block, + fat_pointer, + next_roster, + proposal_valid_round, + proposal_sigs, + is_v2, + }) +} + +fn validate_stored_bft_semantics( + config: &crate::config::Config, + block: &BftBlock, + parent: Option<&BftBlock>, + expected_height: u64, + expected_parent: &FatPointerToBftBlock, +) -> Result<(), String> { + if block.headers.is_empty() { + return Err("stored BFT block has no PoW finalization-candidate header".into()); + } + if block.height as u64 != expected_height { + return Err(format!( + "stored BFT block height {} does not match index {expected_height}", + block.height + )); + } + if block.previous_block_fat_ptr.points_at_block_hash() + != expected_parent.points_at_block_hash() + { + return Err("stored BFT block does not extend the preceding certified tip".into()); + } + + let parent_version = parent.map_or(0, |value| value.version); + let parent_minimum = parent.map_or(0, |value| value.do_not_include_until_bc_height); + if block.version < parent_version { + return Err("stored BFT block version regresses its parent".into()); + } + if block.do_not_include_until_bc_height < parent_minimum { + return Err("stored BFT inclusion floor regresses its parent".into()); + } + + let scheduled: Vec<&crate::config::HardForkConfig> = config + .hardforks + .iter() + .filter(|rule| rule.bft_certificate_height == expected_height) + .collect(); + let serialize = |rule: &crate::config::HardForkConfig| -> Result, String> { + let mut bytes = Vec::new(); + rule.zcash_serialize(&mut bytes) + .map_err(|error| format!("failed to encode configured hardfork rule: {error}"))?; + Ok(bytes) + }; + if block.hardforks.len() != scheduled.len() { + return Err("stored BFT block carries the wrong scheduled-hardfork count".into()); + } + for (carried, expected) in block.hardforks.iter().zip(scheduled.iter()) { + if serialize(carried)? != serialize(expected)? { + return Err("stored BFT block carries a non-canonical hardfork rule".into()); + } + } + if let Some(last) = scheduled.last() { + if block.do_not_include_until_bc_height != last.pow_activation_height { + return Err("stored hardfork block carries the wrong inclusion floor".into()); + } + if last.pow_activation_height < parent_minimum { + return Err("stored hardfork activation regresses its parent inclusion floor".into()); + } + } + Ok(()) +} + +struct VerifiedPosReplay { + file: File, + rounds: Vec, + records: Vec, + blocks: Vec, + tip: FatPointerToBftBlock, + next_roster: Vec, + final_block: Option<(ZebBlockHeight, ZebBlockHash)>, + torn_tail: Option, +} + +async fn replay_verified_pos_store( + path: &Path, + call: &TFLServiceCalls, + config: &crate::config::Config, + initial_roster: Vec, +) -> Result { + let (mut file, _) = open_exclusive_pos_store(path)?; + let file_len = file + .metadata() + .map_err(|error| format!("failed to stat PoS store before replay: {error}"))? + .len(); + let mut rounds = VecDeque::with_capacity( + tenderlink::MAX_RECENT_COMMIT_ROUNDS_IN_MEMORY, + ); + let mut records = Vec::new(); + let mut blocks: Vec = Vec::new(); + let mut tip = FatPointerToBftBlock::null(); + let mut next_roster = initial_roster; + let mut final_block: Option<(ZebBlockHeight, ZebBlockHash)> = None; + let mut torn_tail = None; + + while file + .stream_position() + .map_err(|error| format!("failed reading PoS-store position: {error}"))? + < file_len + { + let record_offset = file + .stream_position() + .map_err(|error| format!("failed reading PoS-store record position: {error}"))?; + let remaining = file_len - record_offset; + let prefix_len = usize::try_from(remaining.min(POS_STORE_V2_MAGIC.len() as u64)) + .map_err(|_| "PoS-store prefix length does not fit usize")?; + let mut prefix = vec![0u8; prefix_len]; + file.read_exact(&mut prefix) + .map_err(|error| format!("failed to inspect PoS-store record at byte {record_offset}: {error}"))?; + file.seek(SeekFrom::Start(record_offset)) + .map_err(|error| format!("failed to rewind PoS-store record at byte {record_offset}: {error}"))?; + + let is_v2_prefix = POS_STORE_V2_MAGIC[..prefix_len] == prefix; + let record = if remaining < POS_STORE_V2_MAGIC.len() as u64 && is_v2_prefix { + let mut bytes = vec![0u8; remaining as usize]; + file.read_exact(&mut bytes) + .map_err(|error| format!("failed to quarantine short PoS v2 tail: {error}"))?; + torn_tail = Some(PosStoreTornTail { offset: record_offset, bytes }); + break; + } else if prefix.as_slice() == POS_STORE_V2_MAGIC { + if remaining < POS_STORE_V2_HEADER_LEN { + let mut bytes = vec![0u8; remaining as usize]; + file.read_exact(&mut bytes) + .map_err(|error| format!("failed to quarantine PoS v2 header tail: {error}"))?; + torn_tail = Some(PosStoreTornTail { offset: record_offset, bytes }); + break; + } + let mut header = [0u8; POS_STORE_V2_HEADER_LEN as usize]; + file.read_exact(&mut header) + .map_err(|error| format!("failed to read PoS v2 header at byte {record_offset}: {error}"))?; + let payload_len = u64::from_le_bytes(header[8..16].try_into().unwrap()); + if payload_len > MAX_POS_STORE_V2_PAYLOAD_BYTES { + return Err(format!( + "PoS v2 record at byte {record_offset} declares an oversized payload" + )); + } + let frame_len = POS_STORE_V2_HEADER_LEN + .checked_add(payload_len) + .ok_or("PoS v2 frame length overflows")?; + file.seek(SeekFrom::Start(record_offset)) + .map_err(|error| format!("failed to rewind PoS v2 record: {error}"))?; + if remaining < frame_len { + let mut bytes = vec![0u8; remaining as usize]; + file.read_exact(&mut bytes) + .map_err(|error| format!("failed to quarantine PoS v2 payload tail: {error}"))?; + torn_tail = Some(PosStoreTornTail { offset: record_offset, bytes }); + break; + } + let mut frame = vec![0u8; frame_len as usize]; + file.read_exact(&mut frame) + .map_err(|error| format!("failed to read complete PoS v2 frame: {error}"))?; + decode_complete_pos_store_v2_frame(&frame) + .map_err(|error| format!("PoS v2 record at byte {record_offset} is rejected: {error}"))? + } else { + read_stored_pos_decision_payload(&mut file, false) + .map_err(|error| format!("legacy PoS-store record at byte {record_offset} is rejected: {error}"))? + }; + let record_end = file + .stream_position() + .map_err(|error| format!("failed reading PoS-store record end: {error}"))?; + if record_end > file_len { + return Err(format!( + "PoS-store record at byte {record_offset} extends beyond the durable file" + )); + } + + let expected_height = blocks.len() as u64; + validate_stored_bft_semantics( + config, + &record.block, + blocks.last(), + expected_height, + &tip, + )?; + let previous_final_height = final_block.map(|(height, _)| height); + let terminated = terminated_finalizers_at( + &config.hardforks, + expected_height, + previous_final_height.map_or(0, |height| height.0 as u64), + ); + let current_roster = tenderlink_roster_from_internal(&next_roster, &terminated); + let vote_namespace = namespace_for_bft_height(&config.hardforks, expected_height); + let mut round = verify_decided_fat_pointer_quorum( + &record.block, + &record.fat_pointer, + ¤t_roster, + vote_namespace, + record.proposal_sigs.clone(), + ) + .map_err(|error| format!("PoS-store certificate at byte {record_offset} is rejected: {error}"))?; + round.proposal_valid_round = record.proposal_valid_round; + let has_authenticated_proposal_context = record.is_v2 && !record.proposal_sigs.is_empty(); + if record.is_v2 && record.proposal_sigs.is_empty() && record.proposal_valid_round != -1 { + return Err(format!( + "PoS v2 record at byte {record_offset} has valid_round without a proposal manifest" + )); + } + if has_authenticated_proposal_context { + tenderlink::verify_reconstructed_proposal_manifest( + &HashKeys::default(), + &round, + ) + .map_err(|error| { + format!("PoS v2 proposal manifest at byte {record_offset} is rejected: {error}") + })?; + } + let (new_final_height, new_final_hash) = validated_pow_header_chain( + call, + &record.block, + previous_final_height, + ) + .await + .map_err(|error| format!("PoS-store PoW ancestry at byte {record_offset} is rejected: {error}"))?; + + let response = bounded_state_call( + call, + StateRequest::CrosslinkFinalizeBlock(new_final_hash), + "PoS-store replay finalization", + ) + .await?; + let StateResponse::CrosslinkFinalized(finalized_hash, aggregated_stakes) = response else { + return Err("PoS-store replay finalization returned the wrong response type".into()); + }; + if finalized_hash != new_final_hash { + return Err("PoS-store replay finalized a different PoW hash".into()); + } + let expected_next_roster = if aggregated_stakes.is_empty() { + if expected_height == 0 { + next_roster.clone() + } else { + return Err("PoS-store replay found an empty non-genesis bonded roster".into()); + } + } else { + aggregated_stakes + .into_iter() + .map(|(pub_key, voting_power)| RosterMember { + pub_key, + voting_power, + txids: Vec::new(), + }) + .collect() + }; + if record.next_roster != expected_next_roster { + return Err(format!( + "PoS-store next roster at byte {record_offset} differs from finalized state" + )); + } + let next_height = expected_height + .checked_add(1) + .ok_or("BFT replay height overflow")?; + let next_terminated = terminated_finalizers_at( + &config.hardforks, + next_height, + new_final_height.0 as u64, + ); + tenderlink::validate_consensus_roster(&tenderlink_roster_from_internal( + &expected_next_roster, + &next_terminated, + ))?; - #[cfg(not(feature = "viz_gui"))] - std::process::abort(); + let active_len = usize::min(100, current_roster.len()); + let signed_stake = current_roster[..active_len] + .iter() + .filter(|member| { + record + .fat_pointer + .signatures + .iter() + .any(|signature| signature.pub_key == member.pub_key) + }) + .try_fold(0u64, |total, member| total.checked_add(member.stake)) + .ok_or("replayed signing stake overflows u64")?; + round.counts = tenderlink::ConsensusCounts { + anys: signed_stake, + prevotes: 0, + nil_prevotes: 0, + yes_prevotes: 0, + precommits: signed_stake, + yes_precommits: signed_stake, + }; + if !has_authenticated_proposal_context { + // Legacy records do not carry valid_round, and explicit force-fed v2 decisions can + // lack a proposal manifest. Retain the precommit QC but never advertise proposal + // chunks without complete verified signing context. + round.proposal_valid_round = -1; + round.proposal_sigs.clear(); + round.proposal_sigs_n = 0; } - })); - tokio::task::spawn(test_format::instr_reader(internal_handle)); -} + records.push(PosStoreRecordIndex { + offset: record_offset, + len: record_end + .checked_sub(record_offset) + .ok_or("PoS-store record end precedes its start")?, + finalized_bc_height: new_final_height.0, + }); + if rounds.len() == tenderlink::MAX_RECENT_COMMIT_ROUNDS_IN_MEMORY { + rounds.pop_front(); + } + rounds.push_back(round); + blocks.push(record.block); + tip = record.fat_pointer; + next_roster = expected_next_roster; + final_block = Some((new_final_height, new_final_hash)); + } -/// Vote-namespacing domain separator for a BFT height: a flat blake3 hash of the prefix of -/// scheduled hardforks whose `bft_certificate_height <= bft_height` (inclusive of a hardfork at -/// `bft_height` itself), concatenated in canonical schedule order. An empty prefix yields -/// `[0; 32]` (nil), so the no-hardfork case is a backwards-compatible no-op in tenderlink's -/// signing. The schedule is sorted with non-decreasing `bft_certificate_height` (several rules -/// may share one certificate height), so the filtered set is exactly the prefix. -fn namespace_for_bft_height(hardforks: &[crate::config::HardForkConfig], bft_height: u64) -> [u8; 32] { - let mut hasher = blake3::Hasher::new(); - let mut any = false; - for hf in hardforks.iter().filter(|hf| hf.bft_certificate_height <= bft_height) { - let mut bytes = Vec::new(); - hf.zcash_serialize(&mut bytes).expect("serializing to a Vec is infallible"); - hasher.update(&bytes); - any = true; + if file + .stream_position() + .map_err(|error| format!("failed reading final PoS-store position: {error}"))? + != file_len + { + return Err("PoS-store replay did not end exactly at the durable EOF".into()); } - if any { hasher.finalize().into() } else { [0u8; 32] } + file.seek(SeekFrom::End(0)) + .map_err(|error| format!("failed to position verified PoS store for append: {error}"))?; + Ok(VerifiedPosReplay { + file, + rounds: rounds.into_iter().collect(), + records, + blocks, + tip, + next_roster, + final_block, + torn_tail, + }) } -async fn tfl_service_main_loop(internal_handle: TFLServiceHandle, global_seed: [u8; 32], path_to_pos_store_file: PathBuf) -> Result<(), String> { +async fn tfl_service_main_loop( + internal_handle: TFLServiceHandle, + global_seed: [u8; 32], + path_to_pos_store_file: PathBuf, + is_regtest: bool, +) -> Result<(), String> { let call = internal_handle.call.clone(); let config = internal_handle.config.clone(); let params = &PROTOTYPE_PARAMETERS; @@ -1178,30 +3181,90 @@ async fn tfl_service_main_loop(internal_handle: TFLServiceHandle, global_seed: [ let public_ip_string = config .public_address - .unwrap_or(format!("127.0.0.1:{}", rand::thread_rng().next_u32() % 45869 + 2000)); - info!("public IP: {}", public_ip_string); - - let bft_key_seed = if let Some(explicit_bft_key_seed) = config.explicit_bft_key_seed { - explicit_bft_key_seed.clone() - } else { - format!("adrheardhed{:?}", global_seed) - }; - - let (_, my_private_key, my_public_key) = - rng_private_public_key_from_address(&bft_key_seed.as_bytes()); + .clone() + .unwrap_or_else(|| "127.0.0.1:23485".to_owned()); + info!(endpoint = %public_ip_string, "configured BFT endpoint"); + + let canonical_validator_identity = canonical_validator_identity_configured(&config)?; + + let (my_private_key, my_public_key, static_keypair, local_endpoint) = + if canonical_validator_identity { + let signing_seed = decode_exact_lower_hex_32_named( + config + .validator_signing_key_seed + .as_ref() + .expect("complete identity checked above") + .expose_secret(), + "validator signing-key seed", + true, + )?; + let private_key = ed25519_zebra::SigningKey::from(signing_seed); + let derived_public_key = PubKeyID( + <[u8; 32]>::from(ed25519_zebra::VerificationKeyBytes::from(&private_key)), + ); + let asserted_public_key = decode_consensus_public_key_hex( + config + .validator_consensus_public_key + .as_ref() + .expect("complete identity checked above"), + )?; + if derived_public_key != asserted_public_key { + return Err( + "validator signing seed does not match validator_consensus_public_key".into(), + ); + } + let noise_seed = decode_exact_lower_hex_32_named( + config + .validator_noise_static_key_seed + .as_ref() + .expect("complete identity checked above") + .expose_secret(), + "validator Noise static-key seed", + true, + )?; + let noise_keypair = tenderlink::bandwidth_test::new_keypair_from_connect_magic1_with_seed( + tenderlink::CRYPTO_MAGIC, + noise_seed, + ) + .ok_or("failed to derive the configured Noise static key")?; + let (ip, port) = tenderlink::parse_to_ipv6_bytes(&public_ip_string) + .map_err(|error| format!("invalid local validator endpoint: {error}"))?; + let endpoint = tenderlink::bandwidth_test::STPAddress::from(ip, port, &noise_keypair); + (private_key, derived_public_key, noise_keypair, endpoint) + } else { + // Legacy configuration never enters validator mode. Use separate domain-derived, + // process-scoped observer identities; neither secret comes from an endpoint and the + // durable signer is forced observer-only below. + let observer_secret = |domain: &[u8]| -> [u8; 32] { + let mut hasher = blake3::Hasher::new(); + hasher.update(domain); + hasher.update(&global_seed); + hasher.finalize().into() + }; + let private_key = ed25519_zebra::SigningKey::from(observer_secret( + b"ctaz-observer-consensus-key-v1", + )); + let public_key = PubKeyID( + <[u8; 32]>::from(ed25519_zebra::VerificationKeyBytes::from(&private_key)), + ); + let noise_keypair = tenderlink::bandwidth_test::new_keypair_from_connect_magic1_with_seed( + tenderlink::CRYPTO_MAGIC, + observer_secret(b"ctaz-observer-noise-key-v1"), + ) + .ok_or("failed to derive observer Noise static key")?; + let (ip, port) = tenderlink::parse_to_ipv6_bytes(&public_ip_string) + .map_err(|error| format!("invalid observer endpoint: {error}"))?; + let endpoint = tenderlink::bandwidth_test::STPAddress::from(ip, port, &noise_keypair); + warn!("legacy BFT identity configuration is observer-only and fails readiness"); + (private_key, public_key, noise_keypair, endpoint) + }; internal_handle.internal.lock().await.my_public_key = my_public_key; + let mut tenderlink_task: tokio::task::JoinHandle>; + let validator_readiness_configured: bool; { - use tenderlink::bandwidth_test::IdentityKeyPair; - use tenderlink::{parse_to_ipv6_bytes, addr_string_to_stuff}; - - use std::net::{Ipv6Addr, SocketAddr}; - - let mut static_keypair_maybe = None; - let mut endpoint_maybe = None; - let (a, b) = addr_string_to_stuff(&public_ip_string); - static_keypair_maybe = Some(a); - endpoint_maybe = Some(b); + let static_keypair_maybe = Some(static_keypair.clone()); + let endpoint_maybe = Some(local_endpoint.clone()); let tfl_handle1 = internal_handle.clone(); let tfl_handle2 = internal_handle.clone(); @@ -1212,6 +3275,7 @@ async fn tfl_service_main_loop(internal_handle: TFLServiceHandle, global_seed: [ let tfl_handle7 = internal_handle.clone(); let tfl_handle8 = internal_handle.clone(); let tfl_handle9 = internal_handle.clone(); + let tfl_handle10 = internal_handle.clone(); *wallet::TENDERLINK_PUBLIC_KEY.lock().unwrap() = my_public_key; @@ -1220,62 +3284,76 @@ async fn tfl_service_main_loop(internal_handle: TFLServiceHandle, global_seed: [ let mut i_bft_blocks: Vec = Vec::new(); let mut fat_pointer_to_tip: FatPointerToBftBlock = FatPointerToBftBlock::null(); - let mut unsorted_roster = internal_handle - .internal - .lock() - .await - .finalizers_at_current_height - .clone(); - - use tenderlink::FinalizerPeerAddress; - // Note(Sam): We do not support human names in the start config for now. - let finalizer_peer_addresses: Vec = unsorted_roster - .iter() - .enumerate() - .map(|(i, m)| { - let string = format!("{:?}", m); - let mut hasher = DefaultHasher::new(); - hasher.write(string.as_bytes()); - let seed = hasher.finish(); - let string = format!("127.0.0.1:{}", seed % 4000); - let (a, b) = - addr_string_to_stuff(&config.bft_peers.get(i).unwrap_or_else(|| &string)); - FinalizerPeerAddress { - bft_pk: PubKeyID(m.pub_key.into()), - address: b, - } + let mut unsorted_roster = bootstrap_roster_from_config(&config)?; + let bootstrap_roster_for_history = Arc::new(unsorted_roster.clone()); + + let mut held_pos_store_file = None; + let mut pos_store_read_file = None; + let mut pos_store_records = Vec::new(); + let mut replay_final_block = None; + let mut replay_torn_tail = None; + if path_to_pos_store_file.to_str() != Some("") { + let replay = replay_verified_pos_store( + &path_to_pos_store_file, + &call, + &config, + unsorted_roster.clone(), + ) + .await?; + let replay_read_file = Arc::new(replay.file.try_clone().map_err(|error| { + format!("failed to duplicate exclusive PoS-store handle: {error}") + })?); + ingest_data_for_tenderlink = replay.rounds; + pos_store_records = replay.records; + i_bft_blocks = replay.blocks; + fat_pointer_to_tip = replay.tip; + unsorted_roster = replay.next_roster; + replay_final_block = replay.final_block; + replay_torn_tail = replay.torn_tail; + held_pos_store_file = Some(replay.file); + pos_store_read_file = Some(replay_read_file); + } + let pos_store_size_bytes = held_pos_store_file + .as_ref() + .map(|file| { + file.metadata() + .map(|metadata| metadata.len()) + .map_err(|error| format!("failed to stat replayed PoS store: {error}")) }) - .collect(); - + .transpose()? + .unwrap_or(0); + let pos_store_record_count = u64::try_from(pos_store_records.len()) + .map_err(|_| "PoS record count does not fit u64")?; + let pos_store_complete_eof = replay_torn_tail.is_none(); + #[cfg(any())] if path_to_pos_store_file.to_str() != Some("") { - let mut pos_file = OpenOptions::new().read(true).write(true).create(true).open(&path_to_pos_store_file).unwrap(); - let mut pos_file_bytes = Vec::new(); - pos_file.read_to_end(&mut pos_file_bytes).unwrap(); - - let mut cursor = Cursor::new(pos_file_bytes); + let (mut pos_file, _) = open_exclusive_pos_store(&path_to_pos_store_file)?; let mut valid_byte_count = 0; 'big_loop: loop { - valid_byte_count = cursor.position(); - let block = if let Ok(block) = BftBlock::zcash_deserialize(&mut cursor) { block } else { break; }; - let fat_pointer = if let Ok(fat_pointer) = FatPointerToBftBlock::zcash_deserialize(&mut cursor) { fat_pointer } else { break; }; + valid_byte_count = pos_file.stream_position() + .map_err(|error| format!("failed reading PoS-store position: {error}"))?; + let block = if let Ok(block) = BftBlock::zcash_deserialize(&mut pos_file) { block } else { break; }; + let fat_pointer = if let Ok(fat_pointer) = FatPointerToBftBlock::zcash_deserialize(&mut pos_file) { fat_pointer } else { break; }; let mut buf = [0u8; 8]; - if cursor.read_exact(&mut buf).is_err() { break; } + if pos_file.read_exact(&mut buf).is_err() { break; } let new_roster_count = u64::from_le_bytes(buf); + if new_roster_count > MAX_POS_STORE_ROSTER_MEMBERS { break; } let mut new_roster = Vec::new(); for _ in 0..new_roster_count { - if let Ok(v) = RosterMember::read_from(&mut cursor) { + if let Ok(v) = RosterMember::read_from(&mut pos_file) { new_roster.push(v); - } else { break; } + } else { break 'big_loop; } } let mut buf = [0u8; 8]; - if cursor.read_exact(&mut buf).is_err() { break; } + if pos_file.read_exact(&mut buf).is_err() { break; } let proposal_sigs_n = u64::from_le_bytes(buf); + if proposal_sigs_n > MAX_POS_STORE_PROPOSAL_SIGNATURES { break; } let mut proposal_sigs = Vec::new(); for _ in 0..proposal_sigs_n { let mut sig = TMSig::NIL; - if cursor.read_exact(&mut sig.0).is_err() { break 'big_loop; } + if pos_file.read_exact(&mut sig.0).is_err() { break 'big_loop; } proposal_sigs.push(sig); } @@ -1293,7 +3371,10 @@ async fn tfl_service_main_loop(internal_handle: TFLServiceHandle, global_seed: [ block_height_from_hash(&call, candidate_hash).await.map(|h| h.0 as u64).unwrap_or(0) } else { 0 }; let this_terminated = terminated_finalizers_at(&config.hardforks, this_bft_height, this_finalized_bc_height); - round_data.roster = tenderlink_roster_from_internal(&unsorted_roster, &this_terminated); + round_data.roster = tenderlink_roster_from_internal( + &unsorted_roster, + &this_terminated, + ); round_data.msg_val_sigs = round_data.roster.iter().map(|v| fat_pointer.signatures.iter().find(|s| s.pub_key == v.pub_key).map(|s| s.vote_signature).unwrap_or([0u8; 64])).map(|s| [(tenderlink::ValueId::NIL, TMSig::NIL), (tenderlink::ValueId(fat_pointer.points_at_block_hash().0), TMSig(s))]).collect(); round_data.counts.precommits = fat_pointer.signatures.len() as u64; round_data.counts.yes_precommits = fat_pointer.signatures.len() as u64; @@ -1312,30 +3393,72 @@ async fn tfl_service_main_loop(internal_handle: TFLServiceHandle, global_seed: [ fat_pointer_to_tip = fat_pointer; unsorted_roster = new_roster; } - pos_file.set_len(valid_byte_count).unwrap(); + if pos_file.metadata() + .map_err(|error| format!("failed to stat PoS store after replay: {error}"))? + .len() != valid_byte_count + { + pos_file.set_len(valid_byte_count) + .map_err(|error| format!("failed to truncate torn PoS-store tail: {error}"))?; + pos_file.sync_all() + .map_err(|error| format!("failed to sync PoS-store truncation: {error}"))?; + } + pos_file.seek(SeekFrom::End(0)) + .map_err(|error| format!("failed to position PoS store for append: {error}"))?; + held_pos_store_file = Some(pos_file); } - let mut new_final_hash = ZebBlockHash([0; 32]); - let mut new_final_height = ZebBlockHeight(0); - - if let Some(new_block) = i_bft_blocks.last() { - new_final_hash.0 = BlockHash::from_header_data(new_block.headers.first().expect("at least 1 header")).0; - new_final_height = block_height_from_hash(&call, new_final_hash).await.unwrap(); -//println!("Loaded at pow ({:?}, {:?}) with roster: {:?}", new_final_height, new_final_hash, unsorted_roster); + // Peer routes are explicit key bindings. Neither consensus nor Noise identity is ever + // derived from the endpoint string. + let finalizer_peer_addresses = finalizer_peer_addresses_from_explicit_config( + &config.bft_peer_identities, + &public_ip_string, + my_public_key, + &static_keypair, + )?; + + let (new_final_height, new_final_hash) = replay_final_block + .unwrap_or((ZebBlockHeight(0), ZebBlockHash([0; 32]))); + + let signer_parent_commit = fat_pointer_to_tip.points_at_block_hash().0; + let startup_bft_height = u64::try_from(i_bft_blocks.len()) + .map_err(|_| "BFT history length does not fit u64")?; + let bft_height_by_hash: HashMap<[u8; 32], usize> = i_bft_blocks + .iter() + .enumerate() + .map(|(height, block)| (block.blake3_hash().0, height)) + .collect(); + if bft_height_by_hash.len() != i_bft_blocks.len() { + return Err("PoS store replays duplicate BFT block hashes".into()); } - let roster = { let mut internal = internal_handle.internal.lock().await; // Startup roster is for the next height to decide (the loaded chain length), with the // terminated finalizers excluded inclusively at that height — derived purely from the // schedule (no stored blacklist; see `terminated_finalizers_at`). - let startup_bft_height = i_bft_blocks.len() as u64; let terminated = terminated_finalizers_at(&config.hardforks, startup_bft_height, new_final_height.0 as u64); - let roster = tenderlink_roster_from_internal(&unsorted_roster, &terminated); + let roster = tenderlink_roster_from_internal( + &unsorted_roster, + &terminated, + ); + if canonical_validator_identity + && !roster[..usize::min(100, roster.len())] + .iter() + .any(|member| member.pub_key == my_public_key) + { + return Err( + "asserted validator consensus key is absent from the active startup roster" + .into(), + ); + } internal.finalizers_at_current_height = unsorted_roster; internal.bft_blocks = i_bft_blocks; + internal.bft_height_by_hash = bft_height_by_hash; internal.fat_pointer_to_tip = fat_pointer_to_tip; + internal.pos_store_file = held_pos_store_file; + internal.pos_store_read_file = pos_store_read_file; + internal.pos_store_records = pos_store_records; + internal.pos_store_unverified_tail = replay_torn_tail; if new_final_hash != ZebBlockHash([0; 32]) { internal.current_bc_final = Some((new_final_height, new_final_hash)); internal.latest_final_block = Some((new_final_height, new_final_hash)); @@ -1343,29 +3466,120 @@ async fn tfl_service_main_loop(internal_handle: TFLServiceHandle, global_seed: [ roster }; - // CROSSLINK: the BFT chain is now loaded, which may make previously-deferred - // non-finalized PoW blocks' fat pointers resolvable. Trigger a re-flush of the - // non-finalized queue by issuing a finalize for the loaded tip. The finalize itself - // is a no-op once that tip is already finalized, but the state request handler - // re-flushes the queue regardless — so deferred blocks are re-evaluated now rather - // than waiting for the first new BFT decision (which may never come on an idle chain). - // The internal lock is released above, so the handler's callback into the fat-pointer - // closure will not deadlock. - // (Removed) This used to issue a no-op finalize of the loaded tip to re-flush the - // state's deferred queue at startup. new_network re-evaluates its own queue every tick. - - // Vote namespacing: the startup height is the number of ingested (decided) rounds; its - // domain separator is computed before the call since `ingest_data_for_tenderlink` is - // moved into it below. - let initial_vote_namespace = namespace_for_bft_height(&config.hardforks, ingest_data_for_tenderlink.len() as u64); - - tokio::spawn(tenderlink::entry_point( + // new_network owns the deferred PoW queue and re-evaluates it every tick. + // Replaying the durable BFT tip must not synthesize a finalize request. + + // Vote namespacing uses the durable chain height, not the length of the + // bounded in-memory recent-round window. + let initial_vote_namespace = + namespace_for_bft_height(&config.hardforks, startup_bft_height); + + let signer_chain_id: [u8; 32] = { + let mut hasher = blake3::Hasher::new(); + hasher.update(b"ctaz-tenderlink-canonical-network-v2"); + hasher.update(if is_regtest { + b"ctaz-regtest".as_slice() + } else { + b"ctaz-public-network".as_slice() + }); + hasher.finalize().into() + }; + let signer_consensus_config_hash: [u8; 32] = { + let mut bytes = vec![config.disable_shipped_hardforks as u8]; + bytes.extend_from_slice(&(config.hardforks.len() as u64).to_le_bytes()); + for hardfork in &config.hardforks { + hardfork.zcash_serialize(&mut bytes).expect("serializing hardfork config to Vec is infallible"); + } + blake3::hash(&bytes).into() + }; + let observer_only = |reason: String| tenderlink::SignerStartup::ObserverOnly { + reason, + chain_id: signer_chain_id, + parent_commit: signer_parent_commit, + consensus_config_hash: signer_consensus_config_hash, + }; + let (readiness_configured, signer_startup) = if !canonical_validator_identity { + ( + false, + observer_only("canonical validator identity is absent".into()), + ) + } else if path_to_pos_store_file.as_os_str().is_empty() { + ( + false, + observer_only("durable PoS history path is absent".into()), + ) + } else { + match ( + config.signer_wal_path.clone(), + config.signer_anchor_path.clone(), + ) { + (Some(wal_path), Some(anchor_path)) => { + let bootstrap_tenderlink_roster = tenderlink_roster_from_internal( + bootstrap_roster_for_history.as_ref().as_slice(), + &HashSet::new(), + ); + let authenticated_bootstrap_roster_hash = + tenderlink::consensus_roster_hash(&bootstrap_tenderlink_roster)?; + let (active_roster_hash, active_roster_index, active_roster_len) = + tenderlink::signer_epoch_roster_binding(&roster, my_public_key)?; + let context = SignerMigrationContext { + validator_consensus_public_key: my_public_key, + chain_id: signer_chain_id, + startup_bft_height, + parent_commit: signer_parent_commit, + vote_namespace: initial_vote_namespace, + consensus_config_hash: tenderlink::signer_consensus_config_binding( + signer_consensus_config_hash, + ), + authenticated_bootstrap_roster_hash, + active_roster_hash, + active_roster_index, + active_roster_len, + finalized_pow_height: new_final_height.0, + finalized_pow_hash: new_final_hash.0, + pos_store_size_bytes, + pos_store_record_count, + pos_store_complete_eof, + wal_path: &wal_path, + anchor_path: &anchor_path, + pos_store_path: &path_to_pos_store_file, + }; + match signer_startup_authority(&config, &context) { + Ok(authority) => ( + true, + tenderlink::SignerStartup::Durable { + wal_path, + anchor_path, + independent_anchor_authorized: true, + non_genesis_bootstrap_receipt_hash: authority + .non_genesis_receipt_hash, + chain_id: signer_chain_id, + parent_commit: signer_parent_commit, + consensus_config_hash: signer_consensus_config_hash, + }, + ), + Err(reason) => { + warn!(%reason, "validator signer remains observer-only"); + (false, observer_only(reason)) + } + } + } + _ => ( + false, + observer_only("signer WAL or independent anchor path is absent".into()), + ), + } + }; + validator_readiness_configured = readiness_configured; + + tenderlink_task = tokio::spawn(tenderlink::entry_point( my_private_key, static_keypair_maybe, endpoint_maybe, roster, finalizer_peer_addresses, None, + signer_startup, tenderlink::ClosureToProposeNewBlock(Arc::new(move || { let tfl_handle1 = tfl_handle1.clone(); Box::pin(async move { @@ -1377,37 +3591,44 @@ async fn tfl_service_main_loop(internal_handle: TFLServiceHandle, global_seed: [ tenderlink::ClosureToValidateProposedBlock(Arc::new(move |block| { let tfl_handle2 = tfl_handle2.clone(); Box::pin(async move { - use bytes::Buf; - use zebra_chain::serialization::ZcashDeserialize; - - if let Ok(bft_block) = BftBlock::zcash_deserialize(block.0.reader()) { - validate_bft_block(&tfl_handle2, &bft_block).await - } else { - error!("Failed to deserialize Tenderlink payload."); - (tenderlink::TMStatus::Fail, tenderlink::TMStatusReason::None) + match deserialize_bft_block_exact(&block.0) { + Ok(bft_block) => validate_bft_block(&tfl_handle2, &bft_block).await, + Err(error) => { + error!(%error, "Failed to deserialize exact Tenderlink payload"); + (tenderlink::TMStatus::Fail, tenderlink::TMStatusReason::None) + } } }) })), - tenderlink::ClosureToPushDecidedBlock(Arc::new(move |block, fat_pointer, tender_proposal_sigs| { + tenderlink::ClosureToPushDecidedBlock(Arc::new(move |block, fat_pointer, proposal_valid_round, tender_proposal_sigs| { let tfl_handle3 = tfl_handle3.clone(); Box::pin(async move { - use bytes::Buf; - use zebra_chain::serialization::ZcashDeserialize; - - let decided_block = BftBlock::zcash_deserialize(block.0.reader()).unwrap(); - let roster = handle_new_decided_bft_block( + let decided_block = deserialize_bft_block_exact(&block.0)?; + let (roster, namespace, durable_parent_commit) = apply_verified_decided_bft_block( &tfl_handle3, &decided_block, &fat_pointer.into(), + proposal_valid_round, tender_proposal_sigs, ) - .await; - // Vote namespacing: the next height is the decided block's height + 1; its - // namespace is the cumulative hardfork hash inclusive of any hardfork scheduled - // at that next height. - let next_height = decided_block.height as u64 + 1; - let namespace = namespace_for_bft_height(&tfl_handle3.config.hardforks, next_height); - (roster, namespace) + .await?; + Ok(tenderlink::DurableDecisionOutcome { + next_roster: roster, + next_vote_namespace: namespace, + durable_parent_commit, + }) + }) + })), + tenderlink::ClosureToLoadCommittedRound(Arc::new(move |height| { + let tfl_handle = tfl_handle10.clone(); + let bootstrap_roster = Arc::clone(&bootstrap_roster_for_history); + Box::pin(async move { + load_historical_committed_round( + &tfl_handle, + bootstrap_roster, + height, + ) + .await }) })), tenderlink::ClosureToUpdatePeers(Arc::new(move |all_peers| { @@ -1429,6 +3650,7 @@ async fn tfl_service_main_loop(internal_handle: TFLServiceHandle, global_seed: [ let tfl_handle = tfl_handle9.clone(); Box::pin(async move { let now_utc = chrono::Utc::now().timestamp(); + let signer_is_active = bft_state.durable_signer.is_active(); let mut finalizer_statuses = Vec::<(PubKeyID, FinalizerRecencyStatus)>::new(); // ~current height @@ -1486,6 +3708,17 @@ async fn tfl_service_main_loop(internal_handle: TFLServiceHandle, global_seed: [ my_valid_round: bft_state.valid_value_round.1, finalizer_statuses, }; + let has_torn_tail = internal.pos_store_unverified_tail.is_some(); + drop(internal); + tfl_handle.set_service_health(if validator_readiness_configured { + if signer_is_active && !has_torn_tail { + SERVICE_HEALTH_READY + } else { + SERVICE_HEALTH_STARTING + } + } else { + SERVICE_HEALTH_OBSERVER_ONLY + }); }) })), ingest_data_for_tenderlink, @@ -1493,22 +3726,80 @@ async fn tfl_service_main_loop(internal_handle: TFLServiceHandle, global_seed: [ )); } + tokio::task::yield_now().await; + if tenderlink_task.is_finished() { + internal_handle.set_service_health(SERVICE_HEALTH_FAILED); + return match tenderlink_task.await { + Ok(Ok(())) => Err("Tenderlink terminated unexpectedly".into()), + Ok(Err(error)) => Err(format!("Tenderlink terminated with an I/O error: {error}")), + Err(error) => Err(format!("Tenderlink task panicked or was cancelled: {error}")), + }; + } + internal_handle.set_service_health( + if validator_readiness_configured { + SERVICE_HEALTH_STARTING + } else { + SERVICE_HEALTH_OBSERVER_ONLY + }, + ); + let mut run_instant = Instant::now(); let mut last_diagnostic_print = Instant::now(); let mut current_bc_tip: Option<(ZebBlockHeight, ZebBlockHash)> = None; loop { - // Calculate this prior to message handling so that handlers can use it: - let new_bc_tip = if let Ok(StateResponse::Tip(val)) = (call.state)(StateRequest::Tip).await + // Calculate this prior to message handling so that handlers can use it. The state + // service cannot freeze consensus indefinitely. + let new_bc_tip = match bounded_state_call( + &call, + StateRequest::Tip, + "crosslink main-loop PoW-tip lookup", + ) + .await { - val - } else { - None + Ok(StateResponse::Tip(value)) => value, + Ok(_) => { + warn!("crosslink main-loop PoW-tip lookup returned the wrong response type"); + None + } + Err(error) => { + warn!(%error, "crosslink main-loop PoW-tip lookup failed"); + None + } }; - tokio::time::sleep_until(run_instant).await; + tokio::select! { + tenderlink_result = &mut tenderlink_task => { + internal_handle.set_service_health(SERVICE_HEALTH_FAILED); + return match tenderlink_result { + Ok(Ok(())) => Err("Tenderlink terminated unexpectedly".into()), + Ok(Err(error)) => Err(format!("Tenderlink terminated with an I/O error: {error}")), + Err(error) => Err(format!("Tenderlink task panicked or was cancelled: {error}")), + }; + } + _ = tokio::time::sleep_until(run_instant) => {} + } run_instant += MAIN_LOOP_SLEEP_INTERVAL; + let pending_reflush = internal_handle.internal.lock().await.pending_reflush; + if let Some(final_hash) = pending_reflush { + match bounded_crosslink_reflush( + &call, + final_hash, + "pending crosslink reflush retry", + ) + .await + { + Ok(()) => { + let mut internal = internal_handle.internal.lock().await; + if internal.pending_reflush == Some(final_hash) { + internal.pending_reflush = None; + } + } + Err(error) => warn!(%error, "bounded crosslink reflush retry remains pending"), + } + } + // from this point onwards we must race to completion in order to avoid stalling incoming requests // NOTE: split to avoid deadlock from non-recursive mutex - can we reasonably change type? #[allow(unused_mut)] @@ -2140,3 +4431,570 @@ async fn _tfl_dump_block_sequence( .await; tfl_dump_blocks(&blocks[..], &infos[..]); } + +#[cfg(test)] +mod liveness_regression_tests { + use super::*; + + fn roster_member(key: PubKeyID, voting_power: u64) -> RosterMember { + RosterMember { + pub_key: key.0, + voting_power, + txids: Vec::new(), + } + } + + #[test] + fn raw_roster_identity_preserves_reversed_twins() { + let mut bytes = [0u8; 32]; + for (index, byte) in bytes.iter_mut().enumerate() { + *byte = index as u8; + } + let raw = PubKeyID(bytes); + bytes.reverse(); + let reversed_twin = PubKeyID(bytes); + let roster = tenderlink_roster_from_internal( + &[roster_member(raw, 2), roster_member(reversed_twin, 1)], + &HashSet::new(), + ); + assert_eq!(roster.len(), 2); + assert!(roster.iter().any(|member| member.pub_key == raw)); + assert!(roster.iter().any(|member| member.pub_key == reversed_twin)); + } + + #[test] + fn peer_addresses_are_key_bound_and_survive_roster_changes() { + let peer_a = "127.0.0.1:30111".to_owned(); + let peer_b = "127.0.0.1:30112".to_owned(); + let public_address = "127.0.0.1:30113"; + let key_a = PubKeyID([0x11; 32]); + let key_b = PubKeyID([0x22; 32]); + let noise_a = [0x31; 32]; + let noise_b = [0x32; 32]; + let mut local_bytes = [0u8; 32]; + for (index, byte) in local_bytes.iter_mut().enumerate() { + *byte = (index as u8).wrapping_add(1); + } + let local = PubKeyID(local_bytes); + let local_noise = tenderlink::bandwidth_test::new_keypair_from_connect_magic1_with_seed( + tenderlink::CRYPTO_MAGIC, + [0x33; 32], + ) + .unwrap(); + let configured = vec![ + crate::config::BftPeerIdentity { + consensus_public_key: "11".repeat(32), + address: peer_a, + noise_public_key: "31".repeat(32), + }, + crate::config::BftPeerIdentity { + consensus_public_key: "22".repeat(32), + address: peer_b, + noise_public_key: "32".repeat(32), + }, + ]; + + let as_map = || { + finalizer_peer_addresses_from_explicit_config( + &configured, + public_address, + local, + &local_noise, + ) + .unwrap() + .into_iter() + .map(|entry| (entry.bft_pk, entry.address)) + .collect::>() + }; + let first = as_map(); + assert_eq!(first.len(), 3); + assert_eq!(first.get(&key_a).unwrap().key, noise_a); + assert_eq!(first.get(&key_b).unwrap().key, noise_b); + assert_eq!(first.get(&local).unwrap().key, local_noise.public); + // Configured endpoints are transport seeds even before their key enters a + // post-recovery roster. Consensus messages remain authorized by the live + // roster, not by presence in this address map. + assert!(first.contains_key(&key_a)); + } + + #[test] + fn transport_routes_never_grant_bootstrap_voting_power() { + let mut config = crate::config::Config::default(); + config.bft_peer_identities.push(crate::config::BftPeerIdentity { + consensus_public_key: "11".repeat(32), + address: "127.0.0.1:30111".to_owned(), + noise_public_key: "31".repeat(32), + }); + assert!(bootstrap_roster_from_config(&config).unwrap().is_empty()); + + config.bootstrap_bft_roster.push(crate::config::BftBootstrapRosterMember { + consensus_public_key: "11".repeat(32), + voting_power: 7, + }); + let roster = bootstrap_roster_from_config(&config).unwrap(); + assert_eq!(roster, vec![roster_member(PubKeyID([0x11; 32]), 7)]); + + config.bootstrap_bft_roster.push(crate::config::BftBootstrapRosterMember { + consensus_public_key: "11".repeat(32), + voting_power: 8, + }); + assert!(bootstrap_roster_from_config(&config).is_err()); + } + + #[test] + fn pos_v2_frame_is_exact_hashed_and_preserves_context_marker() { + let block = BftBlock { + version: 2, + height: 0, + previous_block_fat_ptr: FatPointerToBftBlock::null(), + headers: Vec::new(), + hardforks: Vec::new(), + do_not_include_until_bc_height: 0, + }; + let pointer = FatPointerToBftBlock::from_parts(block.blake3_hash(), 0, 1, &[]); + let next_roster = vec![roster_member(PubKeyID([7u8; 32]), 42)]; + let proposal_sigs = vec![TMSig([9u8; 64])]; + let frame = encode_pos_store_v2_frame( + &block, + &pointer, + &next_roster, + 0, + &proposal_sigs, + ) + .unwrap(); + let decoded = decode_complete_pos_store_v2_frame(&frame).unwrap(); + assert!(decoded.is_v2); + assert_eq!(decoded.block, block); + assert_eq!(decoded.fat_pointer, pointer); + assert_eq!(decoded.next_roster, next_roster); + assert_eq!(decoded.proposal_valid_round, 0); + assert_eq!(decoded.proposal_sigs, proposal_sigs); + + let mut corrupt = frame.clone(); + *corrupt.last_mut().unwrap() ^= 1; + assert!(decode_complete_pos_store_v2_frame(&corrupt).is_err()); + assert!(decode_complete_pos_store_v2_frame(&frame[..frame.len() - 1]).is_err()); + for prefix_len in 1..frame.len() { + assert!(is_exact_strict_frame_prefix(&frame[..prefix_len], &frame)); + } + assert!(!is_exact_strict_frame_prefix(&frame, &frame)); + let mut mismatched_prefix = frame[..frame.len() - 1].to_vec(); + *mismatched_prefix.last_mut().unwrap() ^= 1; + assert!(!is_exact_strict_frame_prefix(&mismatched_prefix, &frame)); + } + + #[test] + fn indexed_pos_store_read_reauthenticates_without_moving_append_cursor() { + let block = BftBlock { + version: 1, + height: 0, + previous_block_fat_ptr: FatPointerToBftBlock::null(), + headers: Vec::new(), + hardforks: Vec::new(), + do_not_include_until_bc_height: 0, + }; + let pointer = FatPointerToBftBlock::from_parts(block.blake3_hash(), 0, 1, &[]); + let next_roster = vec![roster_member(PubKeyID([7u8; 32]), 42)]; + let proposal_sigs = vec![TMSig([9u8; 64])]; + let frame = encode_pos_store_v2_frame( + &block, + &pointer, + &next_roster, + 0, + &proposal_sigs, + ) + .unwrap(); + + let mut append_file = tempfile::tempfile().unwrap(); + let prefix = b"held-prefix"; + append_file.write_all(prefix).unwrap(); + append_file.write_all(&frame).unwrap(); + let append_cursor = append_file.stream_position().unwrap(); + let read_file = append_file.try_clone().unwrap(); + let decoded = read_indexed_pos_store_record( + &read_file, + PosStoreRecordIndex { + offset: prefix.len() as u64, + len: frame.len() as u64, + finalized_bc_height: 123, + }, + ) + .unwrap(); + + assert!(decoded.is_v2); + assert_eq!(decoded.block, block); + assert_eq!(decoded.fat_pointer, pointer); + assert_eq!(decoded.next_roster, next_roster); + assert_eq!(append_file.stream_position().unwrap(), append_cursor); + } + + #[test] + fn legacy_pos_payload_is_explicitly_contextless() { + let block = BftBlock { + version: 1, + height: 0, + previous_block_fat_ptr: FatPointerToBftBlock::null(), + headers: Vec::new(), + hardforks: Vec::new(), + do_not_include_until_bc_height: 0, + }; + let pointer = FatPointerToBftBlock::from_parts(block.blake3_hash(), 0, 1, &[]); + let mut legacy = block.zcash_serialize_to_vec().unwrap(); + pointer.zcash_serialize(&mut legacy).unwrap(); + legacy.extend_from_slice(&1u64.to_le_bytes()); + roster_member(PubKeyID([7u8; 32]), 42).write_to_vec(&mut legacy); + legacy.extend_from_slice(&1u64.to_le_bytes()); + legacy.extend_from_slice(&[9u8; 64]); + + let mut cursor = Cursor::new(&legacy); + let decoded = read_stored_pos_decision_payload(&mut cursor, false).unwrap(); + assert!(!decoded.is_v2); + assert_eq!(decoded.proposal_valid_round, -1); + assert_eq!(decoded.proposal_sigs, vec![TMSig([9u8; 64])]); + assert_eq!(cursor.position(), legacy.len() as u64); + } + + #[test] + fn torn_v2_tail_is_repaired_only_by_its_exact_certified_frame() { + fn test_block() -> BftBlock { + BftBlock { + version: 2, + height: 0, + previous_block_fat_ptr: FatPointerToBftBlock::null(), + headers: Vec::new(), + hardforks: Vec::new(), + do_not_include_until_bc_height: 0, + } + } + fn internal_with_tail( + path: PathBuf, + file: File, + prefix: Vec, + ) -> TFLServiceInternal { + TFLServiceInternal { + my_public_key: PubKeyID::NIL, + latest_final_block: None, + tfl_is_activated: false, + final_change_tx: broadcast::channel(1).0, + bft_msg_flags: 0, + bft_err_flags: 0, + bft_blocks: Vec::new(), + bft_height_by_hash: HashMap::new(), + fat_pointer_to_tip: FatPointerToBftBlock::null(), + our_set_bft_string: None, + active_bft_string: None, + peer_strings: Vec::new(), + finalizers_keys_to_names: HashMap::new(), + finalizers_at_current_height: Vec::new(), + recency_status: TFLRecencyStatus::default(), + current_bc_final: None, + path_to_pos_store_file: path, + pos_store_file: Some(file), + pos_store_read_file: None, + pos_store_records: Vec::new(), + pending_reflush: None, + pos_store_unverified_tail: Some(PosStoreTornTail { + offset: 0, + bytes: prefix, + }), + } + } + fn held_bytes(internal: &mut TFLServiceInternal) -> Vec { + let file = internal.pos_store_file.as_mut().unwrap(); + file.seek(SeekFrom::Start(0)).unwrap(); + let mut bytes = Vec::new(); + file.read_to_end(&mut bytes).unwrap(); + bytes + } + + let block = test_block(); + let pointer = FatPointerToBftBlock::from_parts(block.blake3_hash(), 0, 1, &[]); + let roster = vec![roster_member(PubKeyID([7u8; 32]), 42)]; + let sigs = vec![TMSig([9u8; 64])]; + let expected = encode_pos_store_v2_frame(&block, &pointer, &roster, 0, &sigs).unwrap(); + let prefix = expected[..POS_STORE_V2_HEADER_LEN as usize].to_vec(); + + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("repair.pos"); + let (mut file, _) = open_exclusive_pos_store(&path).unwrap(); + file.write_all(&prefix).unwrap(); + file.sync_all().unwrap(); + let mut internal = internal_with_tail(path, file, prefix.clone()); + append_pos_store_decision(&mut internal, &block, &pointer, &roster, 0, &sigs) + .unwrap(); + assert!(internal.pos_store_unverified_tail.is_none()); + assert_eq!(held_bytes(&mut internal), expected); + + let path = temp.path().join("mismatch.pos"); + let (mut file, _) = open_exclusive_pos_store(&path).unwrap(); + file.write_all(&prefix).unwrap(); + file.sync_all().unwrap(); + let mut internal = internal_with_tail(path, file, prefix.clone()); + let mismatched_sigs = vec![TMSig([8u8; 64])]; + assert!(append_pos_store_decision( + &mut internal, + &block, + &pointer, + &roster, + 0, + &mismatched_sigs, + ) + .is_err()); + assert!(internal.pos_store_unverified_tail.is_some()); + assert_eq!(held_bytes(&mut internal), prefix); + } + + #[test] + fn secret_config_debug_is_redacted() { + /* + let secret: crate::config::SecretHex32 = + serde_json::from_str(&format!("\{}\", "ab".repeat(32))).unwrap(); + */ + let secret: crate::config::SecretHex32 = + serde_json::from_value(serde_json::Value::String("ab".repeat(32))).unwrap(); + let rendered = format!("{secret:?}"); + assert!(rendered.contains("REDACTED")); + assert!(!rendered.contains(&"ab".repeat(32))); + + let legacy: crate::config::RedactedLegacySecret = serde_json::from_value( + serde_json::Value::String("legacy-secret-material".to_owned()), + ) + .unwrap(); + let mut config = crate::config::Config::default(); + config.explicit_bft_key_seed = Some(legacy); + let rendered_config = format!("{config:?}"); + assert!(rendered_config.contains("REDACTED")); + assert!(!rendered_config.contains("legacy-secret-material")); + } + + #[test] + fn legacy_or_partial_identity_never_enters_validator_mode() { + let secret = || -> crate::config::SecretHex32 { + serde_json::from_value(serde_json::Value::String("ab".repeat(32))).unwrap() + }; + let mut config = crate::config::Config::default(); + assert!(!canonical_validator_identity_configured(&config).unwrap()); + + config.validator_signing_key_seed = Some(secret()); + assert!(canonical_validator_identity_configured(&config).is_err()); + config.validator_consensus_public_key = Some("11".repeat(32)); + config.validator_noise_static_key_seed = Some(secret()); + assert!(canonical_validator_identity_configured(&config).unwrap()); + + config.bft_peers.push("127.0.0.1:30111".to_owned()); + assert!(canonical_validator_identity_configured(&config).is_err()); + } + + #[test] + fn exact_bft_decoder_rejects_a_valid_prefix_with_trailing_bytes() { + let block = BftBlock { + version: 0, + height: 0, + previous_block_fat_ptr: FatPointerToBftBlock::null(), + headers: Vec::new(), + hardforks: Vec::new(), + do_not_include_until_bc_height: 0, + }; + let mut bytes = block.zcash_serialize_to_vec().unwrap(); + assert_eq!(deserialize_bft_block_exact(&bytes).unwrap(), block); + bytes.push(0); + assert!(deserialize_bft_block_exact(&bytes).is_err()); + } + + #[test] + fn bootstrap_receipt_hash_is_exact_lowercase_nonzero_hex() { + assert_eq!( + decode_exact_lower_hex_32(&"01".repeat(32)).unwrap(), + [1u8; 32] + ); + assert!(decode_exact_lower_hex_32(&"00".repeat(32)).is_err()); + assert!(decode_exact_lower_hex_32(&"AA".repeat(32)).is_err()); + assert!(decode_exact_lower_hex_32(&"01".repeat(31)).is_err()); + assert!(decode_exact_lower_hex_32(&format!("{}g1", "01".repeat(31))).is_err()); + } + + fn migration_context<'a>( + wal_path: &'a Path, + anchor_path: &'a Path, + pos_store_path: &'a Path, + ) -> SignerMigrationContext<'a> { + SignerMigrationContext { + validator_consensus_public_key: PubKeyID([0x11; 32]), + chain_id: [0x12; 32], + startup_bft_height: 7, + parent_commit: [0x13; 32], + vote_namespace: [0x14; 32], + consensus_config_hash: [0x15; 32], + authenticated_bootstrap_roster_hash: [0x16; 32], + active_roster_hash: [0x17; 32], + active_roster_index: 1, + active_roster_len: 3, + finalized_pow_height: 42, + finalized_pow_hash: [0x18; 32], + pos_store_size_bytes: 12_345, + pos_store_record_count: 7, + pos_store_complete_eof: true, + wal_path, + anchor_path, + pos_store_path, + } + } + + fn valid_migration_receipt(context: &SignerMigrationContext<'_>) -> serde_json::Value { + serde_json::json!({ + "schema": SIGNER_MIGRATION_RECEIPT_SCHEMA, + "action": SIGNER_MIGRATION_RECEIPT_ACTION, + "operator_authorized": true, + "independent_anchor_authorized": true, + "global_single_signer_fence_confirmed": true, + "frozen_legacy_binary_sha256": "21".repeat(32), + "frozen_legacy_config_sha256": "22".repeat(32), + "composite_checkpoint_manifest_sha256": "23".repeat(32), + "pos_store_sha256": "24".repeat(32), + "pos_store_size_bytes": context.pos_store_size_bytes, + "pos_store_complete_eof": true, + "pos_store_record_count": context.pos_store_record_count, + "pos_store_first_bft_height": 0, + "validator_consensus_public_key": "11".repeat(32), + "chain_id": "12".repeat(32), + "replayed_next_bft_height": context.startup_bft_height, + "bootstrap_parent_commit": "13".repeat(32), + "bootstrap_vote_namespace": "14".repeat(32), + "bootstrap_consensus_config_hash": "15".repeat(32), + "authenticated_bootstrap_roster_hash": "16".repeat(32), + "active_roster_hash": "17".repeat(32), + "active_roster_index": context.active_roster_index, + "active_roster_len": context.active_roster_len, + "finalized_pow_height": context.finalized_pow_height, + "finalized_pow_hash": "18".repeat(32), + "peer_route_map_blake3": "25".repeat(32), + "peer_route_voting_power": 5, + "required_route_voting_power": 5, + "legacy_signer_fence_receipt_sha256": "26".repeat(32), + "wal_path": context.wal_path, + "anchor_path": context.anchor_path, + "pos_store_path": context.pos_store_path, + }) + } + + #[test] + fn structured_migration_receipt_is_exact_and_context_bound() { + let temp = tempfile::tempdir().unwrap(); + let wal = temp.path().join("signer.wal"); + let anchor = temp.path().join("signer.anchor"); + let pos = temp.path().join("pos.chain"); + let context = migration_context(&wal, &anchor, &pos); + let value = valid_migration_receipt(&context); + let bytes = serde_json::to_vec(&value).unwrap(); + let pinned: [u8; 32] = blake3::hash(&bytes).into(); + assert_eq!( + verify_signer_migration_receipt( + &bytes, + pinned, + SignerJournalState::Uninitialized, + &context, + ) + .unwrap() + .non_genesis_receipt_hash, + Some(pinned), + ); + + let mut wrong_context = value.clone(); + wrong_context["active_roster_hash"] = serde_json::Value::String("31".repeat(32)); + let wrong_bytes = serde_json::to_vec(&wrong_context).unwrap(); + let wrong_pin: [u8; 32] = blake3::hash(&wrong_bytes).into(); + assert!(verify_signer_migration_receipt( + &wrong_bytes, + wrong_pin, + SignerJournalState::Uninitialized, + &context, + ) + .is_err()); + + let mut unknown = value; + unknown["unsealed_extra_authority"] = serde_json::Value::Bool(true); + let unknown_bytes = serde_json::to_vec(&unknown).unwrap(); + let unknown_pin: [u8; 32] = blake3::hash(&unknown_bytes).into(); + assert!(verify_signer_migration_receipt( + &unknown_bytes, + unknown_pin, + SignerJournalState::Uninitialized, + &context, + ) + .is_err()); + } + + #[test] + fn complete_authority_gate_is_read_only_until_every_binding_passes() { + let temp = tempfile::tempdir().unwrap(); + let wal = temp.path().join("signer.wal"); + let anchor = temp.path().join("signer.anchor"); + let pos = temp.path().join("pos.chain"); + let receipt_path = temp.path().join("migration-receipt.json"); + let context = migration_context(&wal, &anchor, &pos); + let bytes = serde_json::to_vec(&valid_migration_receipt(&context)).unwrap(); + let pinned: [u8; 32] = blake3::hash(&bytes).into(); + + let mut config = crate::config::Config::default(); + config.signer_independent_anchor_authorized = true; + assert!(signer_startup_authority(&config, &context).is_err()); + assert!(!wal.exists()); + assert!(!anchor.exists()); + + std::fs::write(&receipt_path, &bytes).unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&receipt_path, std::fs::Permissions::from_mode(0o600)) + .unwrap(); + } + config.signer_non_genesis_bootstrap_receipt_blake3 = Some(hex::encode(pinned)); + config.signer_non_genesis_bootstrap_receipt_path = Some(receipt_path); + assert_eq!( + signer_startup_authority(&config, &context) + .unwrap() + .non_genesis_receipt_hash, + Some(pinned), + ); + assert!(!wal.exists(), "authority check created a WAL"); + assert!(!anchor.exists(), "authority check created an anchor"); + } + + #[test] + fn stored_roster_rejects_transaction_detail_vectors() { + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[7u8; 32]); + bytes.extend_from_slice(&42u64.to_le_bytes()); + bytes.extend_from_slice(&1u64.to_le_bytes()); + assert!(read_stored_roster_member(&mut Cursor::new(bytes)).is_err()); + + let mut canonical = Vec::new(); + canonical.extend_from_slice(&[7u8; 32]); + canonical.extend_from_slice(&42u64.to_le_bytes()); + canonical.extend_from_slice(&0u64.to_le_bytes()); + assert_eq!( + read_stored_roster_member(&mut Cursor::new(canonical)).unwrap(), + roster_member(PubKeyID([7u8; 32]), 42) + ); + } + + #[test] + fn strict_store_semantics_rejects_headerless_history() { + let block = BftBlock { + version: 0, + height: 0, + previous_block_fat_ptr: FatPointerToBftBlock::null(), + headers: Vec::new(), + hardforks: Vec::new(), + do_not_include_until_bc_height: 0, + }; + assert!(validate_stored_bft_semantics( + &crate::config::Config::default(), + &block, + None, + 0, + &FatPointerToBftBlock::null(), + ) + .is_err()); + } +} diff --git a/zebra-crosslink/zebra-crosslink/src/service.rs b/zebra-crosslink/zebra-crosslink/src/service.rs index 606967ff..9366d48f 100644 --- a/zebra-crosslink/zebra-crosslink/src/service.rs +++ b/zebra-crosslink/zebra-crosslink/src/service.rs @@ -10,8 +10,10 @@ use std::path::PathBuf; use std::pin::Pin; use std::str::FromStr; use std::sync::Arc; +use std::sync::atomic::{AtomicU8, Ordering}; use std::task::{Context, Poll}; +use futures::task::AtomicWaker; use tokio::sync::{broadcast, Mutex}; use tokio::task::JoinHandle; @@ -25,8 +27,10 @@ use zebra_state::{crosslink::*, Request as StateRequest, Response as StateRespon use zcash_primitives::transaction::RosterMember; use zcash_primitives::bft::*; use crate::{ - rng_private_public_key_from_address, tfl_service_incoming_request, TFLBlockFinality, - TFLServiceInternal, + bootstrap_roster_from_config, decode_consensus_public_key_hex, + tfl_service_incoming_request, TFLBlockFinality, TFLServiceInternal, + SERVICE_HEALTH_FAILED, SERVICE_HEALTH_READY, + SERVICE_HEALTH_STARTING, }; use tower::Service; @@ -36,7 +40,27 @@ impl Service for TFLServiceHandle { type Future = Pin> + Send>>; fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll> { - Poll::Ready(Ok(())) + loop { + match self.service_health.load(Ordering::Acquire) { + SERVICE_HEALTH_READY => return Poll::Ready(Ok(())), + SERVICE_HEALTH_STARTING => { + self.service_health_waker.register(_cx.waker()); + if self.service_health.load(Ordering::Acquire) == SERVICE_HEALTH_STARTING { + return Poll::Pending; + } + } + SERVICE_HEALTH_FAILED => { + return Poll::Ready(Err(TFLServiceError::Misc( + "crosslink consensus service has terminated".to_owned(), + ))) + } + _ => { + return Poll::Ready(Err(TFLServiceError::Misc( + "crosslink is observer-only and is not validator-ready".to_owned(), + ))) + } + } + } } fn call(&mut self, request: TFLServiceRequest) -> Self::Future { @@ -129,30 +153,15 @@ pub fn spawn_new_tfl_service( closure_from_state_to_here_mutex: Arc>>, ) -> (TFLServiceHandle, JoinHandle>) { let (finalizers_at_current_height, finalizers_keys_to_names) = { - let mut array = Vec::with_capacity(config.bft_peers.len()); - let mut map = std::collections::HashMap::with_capacity(config.bft_peers.len()); - - for (i, peer) in config.bft_peers.iter().enumerate() { - let (_, _, public_key) = rng_private_public_key_from_address(peer.as_bytes()); - array.push(RosterMember { pub_key:public_key.0, voting_power: 1, txids: Vec::new() }); - // array.push(crate::MalValidator::new(public_key, vec![StakeTxId{ txid: [0;32], zats:((i as u64) * 5) + 1 }])); // @Phillip @Testing - map.insert(public_key, peer.to_string()); - } + let array = bootstrap_roster_from_config(&config).unwrap_or_default(); + let mut map = + std::collections::HashMap::with_capacity(config.bft_peer_identities.len()); - if array.is_empty() { - let public_ip_string = config - .public_address - .clone() - .unwrap_or(String::from_str("/ip4/127.0.0.1/udp/45869/quic-v1").unwrap()); - let bft_key_seed = config - .explicit_bft_key_seed - .clone() - .unwrap_or(public_ip_string); - // .unwrap_or(String::from_str("tester").unwrap()); - info!("bft_key_seed: {}", bft_key_seed); - let (_, _, public_key) = rng_private_public_key_from_address(&bft_key_seed.as_bytes()); - array.push(RosterMember { pub_key:public_key.0, voting_power: 1, txids: Vec::new() }); - map.insert(public_key, bft_key_seed); + for peer in &config.bft_peer_identities { + if let Ok(public_key) = decode_consensus_public_key_hex(&peer.consensus_public_key) { + // Transport configuration never grants roster membership. + map.insert(public_key, peer.address.clone()); + } } (array, map) @@ -166,6 +175,7 @@ pub fn spawn_new_tfl_service( bft_msg_flags: 0, bft_err_flags: 0, bft_blocks: Vec::new(), + bft_height_by_hash: std::collections::HashMap::new(), fat_pointer_to_tip: FatPointerToBftBlock::null(), peer_strings: Vec::new(), our_set_bft_string: None, @@ -174,9 +184,17 @@ pub fn spawn_new_tfl_service( finalizers_keys_to_names, current_bc_final: None, path_to_pos_store_file: path_to_pos_store_file.clone(), + pos_store_file: None, + pos_store_read_file: None, + pos_store_records: Vec::new(), + pending_reflush: None, + pos_store_unverified_tail: None, recency_status: TFLRecencyStatus::default(), })); + let service_health = Arc::new(AtomicU8::new(SERVICE_HEALTH_STARTING)); + let service_health_waker = Arc::new(AtomicWaker::new()); + let handle_mtx = Arc::new(std::sync::Mutex::new(None)); let handle_mtx2 = handle_mtx.clone(); @@ -192,9 +210,15 @@ pub fn spawn_new_tfl_service( let (status, reason) = crate::validate_bft_block(&handle, block.as_ref()).await; match status { tenderlink::TMStatus::Pass => { - info!("Successfully force-fed BFT block"); - crate::handle_new_decided_bft_block(&handle, block.as_ref(), &fat_pointer, Vec::new()) - .await; + crate::apply_verified_decided_bft_block( + &handle, + block.as_ref(), + &fat_pointer, + -1, + Vec::new(), + ) + .await?; + info!("Successfully force-fed and durably applied certified BFT block"); Ok(()) }, @@ -209,6 +233,7 @@ pub fn spawn_new_tfl_service( let handle1 = TFLServiceHandle { internal, + decision_apply_gate: Arc::new(Mutex::new(())), call: TFLServiceCalls { state: state_service_call, read_state: read_state_service_call, @@ -216,6 +241,8 @@ pub fn spawn_new_tfl_service( force_feed_pos, }, config, + service_health: service_health.clone(), + service_health_waker: service_health_waker.clone(), }; *handle_mtx.lock().unwrap() = Some(handle1.clone()); @@ -224,10 +251,23 @@ pub fn spawn_new_tfl_service( *closure_from_state_to_here_mutex.lock().unwrap() = Some(Arc::new(move |fpa, fpb, height| crate::call_from_state_to_crosslink_to_ask_about_fat_pointers(&handle3, fpa, fpb, height))); let handle2 = handle1.clone(); - ( - handle1, - tokio::spawn(async move { crate::tfl_service_main_loop(handle2, global_seed, path_to_pos_store_file).await }), - ) + let service_health_for_task = service_health; + let service_health_waker_for_task = service_health_waker; + let task = tokio::spawn(async move { + let result = crate::tfl_service_main_loop( + handle2, + global_seed, + path_to_pos_store_file, + is_regtest, + ) + .await; + if result.is_err() { + service_health_for_task.store(SERVICE_HEALTH_FAILED, Ordering::Release); + service_health_waker_for_task.wake(); + } + result + }); + (handle1, task) } /// A wrapper around the `TFLServiceInternal` and `TFLServiceCalls` types, used to manage @@ -236,8 +276,22 @@ pub fn spawn_new_tfl_service( pub struct TFLServiceHandle { /// A threadsafe wrapper around the stored internal data pub(crate) internal: Arc>, + /// Serializes decided-value application without holding `internal` across + /// the state callback, which must re-enter the crosslink service. + pub(crate) decision_apply_gate: Arc>, /// The collection of service calls available pub(crate) call: TFLServiceCalls, /// The file-generated config data pub config: crate::config::Config, + /// Readiness is separate from task existence: legacy/observer configurations remain unready. + pub(crate) service_health: Arc, + pub(crate) service_health_waker: Arc, +} + +impl TFLServiceHandle { + pub(crate) fn set_service_health(&self, status: u8) { + if self.service_health.swap(status, Ordering::AcqRel) != status { + self.service_health_waker.wake(); + } + } } diff --git a/zebra-crosslink/zebra-state/src/error.rs b/zebra-crosslink/zebra-state/src/error.rs index 2751c63a..aafd3e89 100644 --- a/zebra-crosslink/zebra-state/src/error.rs +++ b/zebra-crosslink/zebra-state/src/error.rs @@ -298,6 +298,10 @@ pub enum ValidateContextError { #[non_exhaustive] CrosslinkNotReady { block_height: block::Height }, + #[error("block height {block_height:?} was rejected because the bounded non-finalized queue is full")] + #[non_exhaustive] + QueuedBlockMemoryLimit { block_height: block::Height }, + #[error("block height {block_height:?} references a BFT block that must not be included until BC height {do_not_include_until}")] #[non_exhaustive] CrosslinkFatPointerTooEarly { block_height: block::Height, do_not_include_until: u64 },