From 994081edb886d16ef5a3500d27abf8395b617676 Mon Sep 17 00:00:00 2001 From: roman Date: Sun, 5 Jul 2026 20:49:21 -0600 Subject: [PATCH 01/27] fix(zakura): recover header sync from a frontier stranded on an abandoned branch On repeated first_header_does_not_link rejections from independent peers, walk the request anchor back below the verified block tip (exponentially deeper per round, floored at the finalized height) until responses link, commit the higher-work range through the fork point, and invalidate the stranded committed body suffix so block sync re-downloads the new branch. Non-linking forward responses no longer score peers. --- .../src/zakura/header_sync/events.rs | 14 + .../src/zakura/header_sync/reactor.rs | 149 +++++++++- .../src/zakura/header_sync/service.rs | 7 + zebra-network/src/zakura/header_sync/state.rs | 50 ++++ zebra-network/src/zakura/header_sync/tests.rs | 277 +++++++++++++++++- zebra-network/src/zakura/testkit/cluster.rs | 29 +- zebra-state/src/lib.rs | 4 +- zebra-state/src/response.rs | 15 + zebra-state/src/service.rs | 7 +- .../service/finalized_state/zebra_db/block.rs | 12 +- .../zebra_db/block/tests/vectors.rs | 17 +- zebra-state/src/service/tests.rs | 17 +- zebra-state/src/service/write.rs | 10 +- .../start/zakura/header_sync_driver.rs | 171 ++++++++++- 14 files changed, 746 insertions(+), 33 deletions(-) diff --git a/zebra-network/src/zakura/header_sync/events.rs b/zebra-network/src/zakura/header_sync/events.rs index 86db8022269..25b2d2a7b13 100644 --- a/zebra-network/src/zakura/header_sync/events.rs +++ b/zebra-network/src/zakura/header_sync/events.rs @@ -217,6 +217,13 @@ pub enum HeaderSyncEvent { }, /// State finalized or verified-body frontiers changed. StateFrontiersChanged(HeaderSyncFrontiers), + /// State answered a [`HeaderSyncAction::QueryReanchorTarget`] walk-back query. + ReanchorTargetLoaded { + /// The queried height. + height: block::Height, + /// The locally stored header hash at `height`, if one exists. + hash: Option, + }, /// State successfully committed a header range. HeaderRangeCommitted { /// First committed height. @@ -282,6 +289,7 @@ impl HeaderSyncEvent { Self::WireDecodeFailed { .. } => "wire_decode_failed", Self::WireProtocolFailure { .. } => "wire_protocol_failure", Self::StateFrontiersChanged(_) => "state_frontiers_changed", + Self::ReanchorTargetLoaded { .. } => "reanchor_target_loaded", Self::HeaderRangeCommitted { .. } => "header_range_committed", Self::HeaderRangeCommitFailed { .. } => "header_range_commit_failed", Self::HeaderRangeResponseFinished { .. } => "header_range_response_finished", @@ -320,6 +328,12 @@ pub enum HeaderSyncAction { }, /// Ask state for the durable best header tip. QueryBestHeaderTip, + /// Ask state for the locally stored header hash at `height`, so the + /// stranded-frontier walk-back can re-anchor below the verified block tip. + QueryReanchorTarget { + /// The height to re-anchor to if a local header exists there. + height: block::Height, + }, /// Ask state for a bounded contiguous range of headers. QueryHeadersByHeightRange { /// Peer that requested the range. diff --git a/zebra-network/src/zakura/header_sync/reactor.rs b/zebra-network/src/zakura/header_sync/reactor.rs index d1f46f86fd9..85b240908e8 100644 --- a/zebra-network/src/zakura/header_sync/reactor.rs +++ b/zebra-network/src/zakura/header_sync/reactor.rs @@ -190,6 +190,9 @@ impl HeaderSyncReactor { HeaderSyncEvent::StateFrontiersChanged(frontiers) => { self.handle_state_frontiers_changed(frontiers).await; } + HeaderSyncEvent::ReanchorTargetLoaded { height, hash } => { + self.handle_reanchor_target_loaded(height, hash).await; + } HeaderSyncEvent::HeaderRangeCommitted { start_height, tip_height, @@ -597,7 +600,12 @@ impl HeaderSyncReactor { self.state.finalized_height = frontiers.finalized_height; self.state.verified_block_tip = frontiers.verified_block_tip; self.state.verified_block_hash = frontiers.verified_block_hash; - if self.state.best_header_tip <= self.state.verified_block_tip { + // During a walk-back the frontier is deliberately at or below the + // verified block tip, so this reset would wipe the recovery evidence + // on every frontier update and stall the walk-back forever. + if self.state.best_header_tip <= self.state.verified_block_tip + && !self.state.fork_recovery.is_active() + { self.state.stale_anchor.reset(); } self.schedule().await; @@ -629,6 +637,15 @@ impl HeaderSyncReactor { if tip_height > self.state.best_header_tip { self.publish_best_tip(tip_height, tip_hash).await; } + // A commit strictly above the verified block tip means peer responses + // link again: the fork (if any) has been crossed, so walk-back state + // and stale-anchor evidence are obsolete. A commit at or below the + // verified tip can be a stranded peer re-serving our own chain, which + // must not reset an in-progress walk-back. + if tip_height > self.state.verified_block_tip { + self.state.stale_anchor.reset(); + self.state.fork_recovery.reset(); + } self.notify_body_gaps().await; self.schedule().await; } @@ -1207,6 +1224,16 @@ impl HeaderSyncReactor { }); } + /// Handles a `FirstHeaderDoesNotLink` rejection as possible evidence that + /// the local frontier — not the peer — is stale. + /// + /// A non-linking response to a forward request is what an honest peer + /// sends when the network abandoned the branch our frontier sits on, so it + /// must never be treated as misbehavior. Returns `true` when the failure + /// was absorbed here (recorded, range retried, and possibly a re-anchor + /// started); `false` when the caller should score the peer (backward + /// checkpoint-bracket ranges anchor on checkpoint hashes, which cannot be + /// stale). async fn handle_possible_stale_anchor_link_failure( &mut self, peer: &ZakuraPeerId, @@ -1216,9 +1243,10 @@ impl HeaderSyncReactor { if !matches!(error, HeaderSyncWireError::FirstHeaderDoesNotLink) || range.priority != RangePriority::Forward || range.finalized - || self.state.best_header_tip <= self.state.verified_block_tip { - self.state.stale_anchor.reset(); + if !self.state.fork_recovery.is_active() { + self.state.stale_anchor.reset(); + } return false; } @@ -1231,7 +1259,18 @@ impl HeaderSyncReactor { return true; } - self.reanchor_to_verified_block_tip().await; + if self.state.best_header_tip > self.state.verified_block_tip { + // The frontier is a header-only extension of the verified chain; + // dropping back to the verified block tip is free and sufficient + // when only the header suffix is stale. + self.reanchor_to_verified_block_tip().await; + } else { + // The frontier sits at (or below) the verified block tip and + // independent peers still cannot link to it: the committed chain + // suffix itself is on an abandoned branch. Walk the anchor back + // below the verified tip to find the fork point. + self.begin_fork_recovery_walk_back(); + } true } @@ -1249,6 +1288,90 @@ impl HeaderSyncReactor { self.publish_best_tip_reanchored(height, hash).await; } + /// Starts (or deepens) a walk-back round: asks state for the local header + /// hash at an exponentially deeper ancestor of the verified block tip, so + /// [`Self::handle_reanchor_target_loaded`] can re-anchor the frontier there. + fn begin_fork_recovery_walk_back(&mut self) { + self.state.stale_anchor.reset(); + if self.state.fork_recovery.awaiting_target.is_some() { + return; + } + + let depth = self.state.fork_recovery.next_depth(); + // The fork point cannot be below the finalized height (the state + // cannot reorg finalized blocks), and the startup anchor is the + // deepest hash this reactor trusts. + let floor = self.state.finalized_height.max(self.state.anchor.0); + let target = block::Height( + self.state + .verified_block_tip + .0 + .saturating_sub(depth) + .max(floor.0), + ); + + metrics::counter!("sync.header.fork_recovery.walk_back").increment(1); + tracing::warn!( + ?depth, + ?target, + verified_block_tip = ?self.state.verified_block_tip, + best_header_tip = ?self.state.best_header_tip, + "Zakura header-sync frontier appears stranded on an abandoned \ + branch; walking the request anchor back to find the fork point" + ); + + self.state.fork_recovery.awaiting_target = Some(target); + if !self.dispatch_action(HeaderSyncAction::QueryReanchorTarget { height: target }) { + // Try again when the next non-linking responses accumulate. + self.state.fork_recovery.awaiting_target = None; + } + } + + /// Applies a walk-back re-anchor once state supplies the local hash at the + /// queried target height. + async fn handle_reanchor_target_loaded( + &mut self, + height: block::Height, + hash: Option, + ) { + if self.state.fork_recovery.awaiting_target != Some(height) { + return; + } + self.state.fork_recovery.awaiting_target = None; + + let Some(hash) = hash else { + // No local header at the target height; nothing to anchor on. + // Later rounds retry (deeper) when non-linking responses recur. + tracing::warn!( + ?height, + "Zakura header-sync walk-back target has no local header; cannot re-anchor" + ); + return; + }; + if height >= self.state.best_header_tip { + // The frontier moved at or below the target while the query was in + // flight; re-anchoring would move it the wrong way. + return; + } + + metrics::counter!("sync.header.fork_recovery.reanchored").increment(1); + tracing::info!( + ?height, + ?hash, + depth = ?self.state.fork_recovery.depth, + "Zakura header-sync re-anchoring below the verified block tip for fork recovery" + ); + + self.state.stale_anchor.reset(); + self.state.schedule.clear_forward(); + self.state + .pending_commits + .retain(|_, range| range.priority != RangePriority::Forward); + self.cancel_forward_outstanding(); + self.publish_best_tip_reanchored(height, hash).await; + self.schedule().await; + } + async fn handle_timeouts(&mut self) { let now = Instant::now(); let mut timed_out = Vec::new(); @@ -1482,7 +1605,12 @@ impl HeaderSyncReactor { self.state.verified_block_tip = height; self.state.verified_block_hash = hash; } - if self.state.best_header_tip <= self.state.verified_block_tip { + // See handle_state_frontiers_changed: a walk-back holds the frontier + // at or below the verified tip on purpose, so this reset must not fire + // while recovery is active. + if self.state.best_header_tip <= self.state.verified_block_tip + && !self.state.fork_recovery.is_active() + { self.state.stale_anchor.reset(); } } @@ -1746,6 +1874,13 @@ impl HeaderSyncReactor { insert_u64(row, hs_trace::RANGE_COUNT, headers.len() as u64); insert_u64(row, hs_trace::EXPECTED_COUNT, u64::from(*requested_count)); } + HeaderSyncEvent::ReanchorTargetLoaded { height, hash } => { + insert_optional_str(row, hs_trace::KIND, Some("reanchor_target_loaded")); + insert_height(row, hs_trace::HEIGHT, *height); + if let Some(hash) = hash { + insert_hash(row, hs_trace::HASH, *hash); + } + } }); } @@ -1813,6 +1948,10 @@ impl HeaderSyncReactor { HeaderSyncAction::QueryBestHeaderTip => { insert_optional_str(row, hs_trace::KIND, Some("query_best_header_tip")); } + HeaderSyncAction::QueryReanchorTarget { height } => { + insert_optional_str(row, hs_trace::KIND, Some("query_reanchor_target")); + insert_height(row, hs_trace::HEIGHT, *height); + } HeaderSyncAction::QueryMissingBlockBodies { from, limit } => { insert_optional_str(row, hs_trace::KIND, Some("query_missing_block_bodies")); insert_height(row, hs_trace::RANGE_START, *from); diff --git a/zebra-network/src/zakura/header_sync/service.rs b/zebra-network/src/zakura/header_sync/service.rs index 1262bade3a1..b54543ed287 100644 --- a/zebra-network/src/zakura/header_sync/service.rs +++ b/zebra-network/src/zakura/header_sync/service.rs @@ -283,6 +283,13 @@ pub(crate) async fn drive_header_sync_actions( "suppressing Zakura header range commit until state driver is wired" ); } + HeaderSyncAction::QueryReanchorTarget { height } => { + // No state driver is wired here, so answer "no local header" + // to keep the reactor's walk-back state machine live. + let _ = handle + .send(HeaderSyncEvent::ReanchorTargetLoaded { height, hash: None }) + .await; + } HeaderSyncAction::QueryBestHeaderTip | HeaderSyncAction::QueryMissingBlockBodies { .. } | HeaderSyncAction::BodyGaps { .. } diff --git a/zebra-network/src/zakura/header_sync/state.rs b/zebra-network/src/zakura/header_sync/state.rs index 7b7826a1c37..98f31ff8b8c 100644 --- a/zebra-network/src/zakura/header_sync/state.rs +++ b/zebra-network/src/zakura/header_sync/state.rs @@ -8,6 +8,12 @@ pub(super) const HEADER_SYNC_ADVISORY_BACKOFF: Duration = Duration::from_secs(60 pub(super) const HEADER_SYNC_ADVISORY_TTL: Duration = DEFAULT_LIVE_SERVICE_SUMMARY_TTL; pub(super) const HEADER_SYNC_STALE_ANCHOR_LINK_FAILURES: u32 = 3; pub(super) const HEADER_SYNC_STALE_ANCHOR_DISTINCT_PEERS: usize = 2; +/// Deepest walk-back re-anchor below the verified block tip. +/// +/// Mirrors `zebra_state::MAX_BLOCK_REORG_HEIGHT` (which this crate cannot +/// depend on): the state rejects header-chain reorgs deeper than that window, +/// so anchoring below it can never lead to a committable range. +pub(super) const HEADER_SYNC_REANCHOR_MAX_DEPTH: u32 = 1_000; #[derive(Clone, Debug)] pub(super) struct HeaderSyncCore { @@ -25,6 +31,7 @@ pub(super) struct HeaderSyncCore { pub(super) pending_commits: HashMap, pub(super) advisory: HashMap, pub(super) stale_anchor: StaleAnchorFailures, + pub(super) fork_recovery: ForkRecovery, } impl HeaderSyncCore { @@ -47,6 +54,7 @@ impl HeaderSyncCore { pending_commits: HashMap::new(), advisory: HashMap::new(), stale_anchor: StaleAnchorFailures::default(), + fork_recovery: ForkRecovery::default(), }) } @@ -123,6 +131,48 @@ impl HeaderSyncCore { } } +/// Walk-back state for recovering a frontier stranded on an abandoned branch. +/// +/// When repeated non-linking responses arrive while the header frontier sits +/// at (or below) the verified block tip, the frontier itself — including the +/// committed block suffix — may be on a branch the network reorged away from. +/// Recovery re-anchors the frontier to a local ancestor at an exponentially +/// growing depth below the verified block tip until peer responses link, at +/// which point the higher-work range commits through the fork point. +#[derive(Clone, Debug, Default)] +pub(super) struct ForkRecovery { + /// Depth below the verified block tip of the most recent walk-back round. + /// Zero when no walk-back is active; doubles every round that still fails + /// to link. + pub(super) depth: u32, + /// The target height of an outstanding [`HeaderSyncAction::QueryReanchorTarget`], + /// if one is in flight. + pub(super) awaiting_target: Option, +} + +impl ForkRecovery { + /// Whether a walk-back is in progress, so stale-anchor evidence and the + /// walk-back depth must survive frontier updates. + pub(super) fn is_active(&self) -> bool { + self.depth > 0 || self.awaiting_target.is_some() + } + + /// Returns the next walk-back depth: 1 on the first round, then doubling, + /// capped at [`HEADER_SYNC_REANCHOR_MAX_DEPTH`]. + pub(super) fn next_depth(&mut self) -> u32 { + self.depth = self + .depth + .saturating_mul(2) + .clamp(1, HEADER_SYNC_REANCHOR_MAX_DEPTH); + self.depth + } + + pub(super) fn reset(&mut self) { + self.depth = 0; + self.awaiting_target = None; + } +} + #[derive(Clone, Debug, Default)] pub(super) struct StaleAnchorFailures { pub(super) count: u32, diff --git a/zebra-network/src/zakura/header_sync/tests.rs b/zebra-network/src/zakura/header_sync/tests.rs index 5cddd0d4dc4..ba56cb0c20a 100644 --- a/zebra-network/src/zakura/header_sync/tests.rs +++ b/zebra-network/src/zakura/header_sync/tests.rs @@ -3711,10 +3711,15 @@ async fn rejected_non_linking_range_traces_link_stage_and_error_kind() { .await .unwrap(); + // A non-linking forward response is stale-frontier evidence, not peer + // misbehavior: the range is retried and the peer is never scored. match next_non_query_action(&mut fixture.actions).await { - HeaderSyncAction::Misbehavior { peer, reason } => { + HeaderSyncAction::SendMessage { + peer, + msg: HeaderSyncMessage::GetHeaders { start_height, .. }, + } => { assert_eq!(peer, peer_id); - assert_eq!(reason, HeaderSyncMisbehavior::InvalidRange); + assert_eq!(start_height, block::Height(1)); } action => panic!("unexpected action: {action:?}"), } @@ -4216,6 +4221,274 @@ async fn single_peer_forward_link_failures_do_not_reanchor_globally() { assert_no_commit_or_misbehavior(&mut fixture.actions).await; } +/// Responds to every outbound `GetHeaders` with a non-linking header until the +/// reactor asks for a walk-back re-anchor target, and returns that height. +/// Panics if any peer is scored or no walk-back query arrives. +async fn drive_non_linking_until_reanchor_query( + fixture: &mut ReactorFixture, + non_linking: Arc, +) -> block::Height { + for _ in 0..16 { + match next_non_query_action(&mut fixture.actions).await { + HeaderSyncAction::SendMessage { + peer, + msg: HeaderSyncMessage::GetHeaders { start_height, .. }, + } => { + fixture + .handle + .send(HeaderSyncEvent::WireMessage { + peer, + msg: headers_message_from(start_height, vec![non_linking.clone()]), + }) + .await + .unwrap(); + } + HeaderSyncAction::QueryReanchorTarget { height } => return height, + HeaderSyncAction::Misbehavior { peer, reason } => { + panic!("unexpected misbehavior from {peer:?}: {reason:?}"); + } + _ => {} + } + } + panic!("walk-back reanchor query did not arrive"); +} + +#[tokio::test(flavor = "current_thread")] +async fn stranded_frontier_walks_back_and_recovers_through_fork_point() { + let network = regtest_network(); + let anchor = (block::Height(0), network.genesis_hash()); + // The committed chain suffix sits on an abandoned branch: header and body + // frontiers are equal, and no honest response can link to the branch hash. + let stranded = (block::Height(2), block::Hash([0xBB; 32])); + let header1 = mainnet_header(&BLOCK_MAINNET_1_BYTES); + let header2 = mainnet_header(&BLOCK_MAINNET_2_BYTES); + let header3 = mainnet_header(&BLOCK_MAINNET_3_BYTES); + let header4 = mainnet_header(&BLOCK_MAINNET_4_BYTES); + let fork_point_hash = block::Hash::from(header1.as_ref()); + + let mut startup = HeaderSyncStartup::new( + network, + anchor, + HeaderSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: stranded.0, + verified_block_hash: stranded.1, + }, + Some(stranded), + ZakuraHeaderSyncConfig::default(), + LOCAL_MAX_MESSAGE_BYTES, + ); + startup.range_state_actions_enabled = true; + let mut fixture = spawn_test_reactor(startup); + let mut tip = fixture.handle.subscribe_tip(); + let peers = [peer(71), peer(72)]; + + for peer_id in peers.iter().cloned() { + connect_peer(&fixture, peer_id.clone()).await; + advertise_tip( + &fixture, + peer_id, + block::Height(0), + block::Height(4), + DEFAULT_HS_RANGE, + 1, + ) + .await; + } + + // Repeated non-linking responses from two independent peers start the + // walk-back one block below the verified tip. + let target = drive_non_linking_until_reanchor_query(&mut fixture, header3.clone()).await; + assert_eq!(target, block::Height(1)); + + fixture + .handle + .send(HeaderSyncEvent::ReanchorTargetLoaded { + height: target, + hash: Some(fork_point_hash), + }) + .await + .unwrap(); + + // The frontier re-anchors below the verified block tip. + tip.changed().await.unwrap(); + assert_eq!(*tip.borrow(), (block::Height(1), fork_point_hash)); + + // The next forward request anchors at the fork point, so a linking + // response commits straight through the abandoned suffix. + let (serving_peer, start_height, count) = loop { + let candidate = next_outbound_get_headers(&mut fixture.actions).await; + if candidate.1 == block::Height(2) { + break candidate; + } + }; + assert_eq!(count, 3); + fixture + .handle + .send(HeaderSyncEvent::WireMessage { + peer: serving_peer.clone(), + msg: headers_message_from( + start_height, + vec![header2.clone(), header3.clone(), header4.clone()], + ), + }) + .await + .unwrap(); + + loop { + match next_non_query_action(&mut fixture.actions).await { + HeaderSyncAction::CommitHeaderRange { + peer, + anchor, + start_height, + headers, + .. + } => { + assert_eq!(peer, serving_peer); + assert_eq!(anchor, fork_point_hash); + assert_eq!(start_height, block::Height(2)); + assert_eq!(headers.len(), 3); + break; + } + HeaderSyncAction::Misbehavior { peer, reason } => { + panic!("unexpected misbehavior from {peer:?}: {reason:?}"); + } + _ => {} + } + } + + // A commit above the verified tip completes the recovery. + let tip_hash = block::Hash::from(header4.as_ref()); + fixture + .handle + .send(HeaderSyncEvent::HeaderRangeCommitted { + start_height: block::Height(2), + tip_height: block::Height(4), + tip_hash, + }) + .await + .unwrap(); + tip.changed().await.unwrap(); + assert_eq!(*tip.borrow(), (block::Height(4), tip_hash)); +} + +#[tokio::test(flavor = "current_thread")] +async fn stranded_walk_back_deepens_exponentially_and_respects_finalized_floor() { + let network = regtest_network(); + let anchor = (block::Height(0), network.genesis_hash()); + let stranded = (block::Height(10), block::Hash([0xCC; 32])); + let non_linking = mainnet_header(&BLOCK_MAINNET_2_BYTES); + + let mut startup = HeaderSyncStartup::new( + network, + anchor, + HeaderSyncFrontiers { + finalized_height: block::Height(6), + verified_block_tip: stranded.0, + verified_block_hash: stranded.1, + }, + Some(stranded), + ZakuraHeaderSyncConfig::default(), + LOCAL_MAX_MESSAGE_BYTES, + ); + startup.range_state_actions_enabled = true; + let mut fixture = spawn_test_reactor(startup); + let peers = [peer(73), peer(74)]; + + for peer_id in peers.iter().cloned() { + connect_peer(&fixture, peer_id.clone()).await; + advertise_tip( + &fixture, + peer_id, + block::Height(0), + block::Height(12), + DEFAULT_HS_RANGE, + 1, + ) + .await; + } + + // Depths double per failed round (1, 2, 4, 8, ...) and the target never + // walks below the finalized height: 9, 8, 6, then floored at 6. + for (round, expected_target) in [9_u32, 8, 6, 6].into_iter().enumerate() { + let target = + drive_non_linking_until_reanchor_query(&mut fixture, non_linking.clone()).await; + assert_eq!( + target, + block::Height(expected_target), + "unexpected walk-back target in round {round}" + ); + fixture + .handle + .send(HeaderSyncEvent::ReanchorTargetLoaded { + height: target, + hash: Some(block::Hash([0x90 + round as u8; 32])), + }) + .await + .unwrap(); + } +} + +#[tokio::test(flavor = "current_thread")] +async fn stranded_single_peer_link_failures_do_not_walk_back() { + let network = regtest_network(); + let anchor = (block::Height(0), network.genesis_hash()); + let stranded = (block::Height(2), block::Hash([0xDD; 32])); + let non_linking = mainnet_header(&BLOCK_MAINNET_3_BYTES); + + let mut startup = HeaderSyncStartup::new( + network, + anchor, + HeaderSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: stranded.0, + verified_block_hash: stranded.1, + }, + Some(stranded), + ZakuraHeaderSyncConfig::default(), + LOCAL_MAX_MESSAGE_BYTES, + ); + startup.range_state_actions_enabled = true; + let mut fixture = spawn_test_reactor(startup); + let mut tip = fixture.handle.subscribe_tip(); + let peer_id = peer(75); + + connect_peer(&fixture, peer_id.clone()).await; + advertise_tip( + &fixture, + peer_id.clone(), + block::Height(0), + block::Height(4), + DEFAULT_HS_RANGE, + 1, + ) + .await; + + for _ in 0..5 { + let (served_peer, start_height, _count) = + next_outbound_get_headers(&mut fixture.actions).await; + assert_eq!(served_peer, peer_id); + assert_eq!(start_height, block::Height(3)); + fixture + .handle + .send(HeaderSyncEvent::WireMessage { + peer: served_peer, + msg: headers_message_from(start_height, vec![non_linking.clone()]), + }) + .await + .unwrap(); + } + + assert!( + tokio::time::timeout(std::time::Duration::from_millis(50), tip.changed()) + .await + .is_err(), + "one peer alone must not walk the stranded frontier back" + ); + assert_eq!(fixture.handle.best_header_tip(), stranded); + assert_no_commit_or_misbehavior(&mut fixture.actions).await; +} + #[tokio::test(flavor = "current_thread")] async fn forward_genesis_backfill_reaches_checkpoint_before_finalized_commit() { let headers = [ diff --git a/zebra-network/src/zakura/testkit/cluster.rs b/zebra-network/src/zakura/testkit/cluster.rs index 3220d44357d..c781b6ad5fe 100644 --- a/zebra-network/src/zakura/testkit/cluster.rs +++ b/zebra-network/src/zakura/testkit/cluster.rs @@ -997,6 +997,19 @@ mod tests { }) .await; } + HeaderSyncAction::QueryReanchorTarget { height } => { + let hash = local + .store + .lock() + .expect("test store mutex is not poisoned") + .headers_by_range(height, 1) + .first() + .map(|header| block::Hash::from(header.as_ref())); + let _ = local + .handle + .send(HeaderSyncEvent::ReanchorTargetLoaded { height, hash }) + .await; + } HeaderSyncAction::QueryMissingBlockBodies { from, limit } => { let heights = local .store @@ -1351,6 +1364,14 @@ mod tests { .send(HeaderSyncEvent::NewBlockDuplicate { peer, height, hash }) .await; } + HeaderSyncAction::QueryReanchorTarget { height } => { + let Some(handle) = endpoint.header_sync() else { + continue; + }; + let _ = handle + .send(HeaderSyncEvent::ReanchorTargetLoaded { height, hash: None }) + .await; + } HeaderSyncAction::QueryBestHeaderTip | HeaderSyncAction::QueryMissingBlockBodies { .. } | HeaderSyncAction::BodyGaps { .. } @@ -3100,6 +3121,9 @@ mod tests { ) .await?; + // A non-contiguous response is provably malformed and still scores the + // peer. (A merely non-linking response no longer does: it is stale- + // frontier evidence for fork recovery, not misbehavior.) let out_of_range = e2e_peer(95); cluster.connect_peer(victim, out_of_range.clone()).await; cluster @@ -3112,7 +3136,10 @@ mod tests { .inject( victim, out_of_range, - headers_message(vec![mainnet_block(&BLOCK_MAINNET_2_BYTES).header.clone()]), + headers_message(vec![ + mainnet_block(&BLOCK_MAINNET_1_BYTES).header.clone(), + mainnet_block(&BLOCK_MAINNET_3_BYTES).header.clone(), + ]), ) .await; cluster diff --git a/zebra-state/src/lib.rs b/zebra-state/src/lib.rs index 015f190c428..7b9bd04b08c 100644 --- a/zebra-state/src/lib.rs +++ b/zebra-state/src/lib.rs @@ -54,8 +54,8 @@ pub use request::{ pub use request::Spend; pub use response::{ - AnyTx, GetBlockTemplateChainInfo, KnownBlock, MinedTx, NonFinalizedBlocksListener, - ReadResponse, Response, + AnyTx, GetBlockTemplateChainInfo, HeaderRangeCommitOutcome, KnownBlock, MinedTx, + NonFinalizedBlocksListener, ReadResponse, Response, }; pub use service::{ chain_tip::{ChainTipBlock, ChainTipChange, ChainTipSender, LatestChainTip, TipAction}, diff --git a/zebra-state/src/response.rs b/zebra-state/src/response.rs index 80991e95d54..e1f617fb95b 100644 --- a/zebra-state/src/response.rs +++ b/zebra-state/src/response.rs @@ -27,6 +27,17 @@ use crate::{ReadRequest, Request}; use crate::{service::read::AddressUtxos, NonFinalizedState, TransactionLocation, WatchReceiver}; +/// The outcome of a successful [`Request::CommitHeaderRange`]. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub struct HeaderRangeCommitOutcome { + /// Hash of the last header in the committed range. + pub tip_hash: block::Hash, + /// First height where the committed range replaced a previously stored + /// conflicting header suffix, when the commit reorged the stored header + /// chain. `None` when the range only extended or duplicated stored headers. + pub reorged_at: Option, +} + #[derive(Clone, Debug, PartialEq, Eq)] /// A response to a [`StateService`](crate::service::StateService) [`Request`]. pub enum Response { @@ -34,6 +45,10 @@ pub enum Response { /// indicating that a block was successfully committed to the state. Committed(block::Hash), + /// Response to [`Request::CommitHeaderRange`] indicating that a header + /// range was successfully committed to the state. + CommittedHeaderRange(HeaderRangeCommitOutcome), + /// Response to [`Request::InvalidateBlock`] indicating that a block was found and /// invalidated in the state. Invalidated(block::Hash), diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs index 4641208fc79..9b4b4279f84 100644 --- a/zebra-state/src/service.rs +++ b/zebra-state/src/service.rs @@ -59,7 +59,8 @@ use crate::{ watch_receiver::WatchReceiver, }, BoxError, CheckpointVerifiedBlock, CommitHeaderRangeError, CommitSemanticallyVerifiedError, - Config, KnownBlock, ReadRequest, ReadResponse, Request, Response, SemanticallyVerifiedBlock, + Config, HeaderRangeCommitOutcome, KnownBlock, ReadRequest, ReadResponse, Request, Response, + SemanticallyVerifiedBlock, }; pub mod block_iter; @@ -988,7 +989,7 @@ impl StateService { headers: Vec>, body_sizes: Vec, tree_aux_roots: Vec, - ) -> oneshot::Receiver> { + ) -> oneshot::Receiver> { let (rsp_tx, rsp_rx) = oneshot::channel(); let Some(sender) = &self.block_write_sender.non_finalized else { @@ -1255,7 +1256,7 @@ impl Service for StateService { .map_err(|_recv_error| CommitHeaderRangeError::CommitResponseDropped) .and_then(|result| result) .map_err(BoxError::from) - .map(Response::Committed) + .map(Response::CommittedHeaderRange) } .instrument(span) .boxed() diff --git a/zebra-state/src/service/finalized_state/zebra_db/block.rs b/zebra-state/src/service/finalized_state/zebra_db/block.rs index 8feead51841..53fb8ae2c99 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block.rs @@ -38,6 +38,7 @@ use crate::{ }, error::{CommitCheckpointVerifiedError, CommitHeaderRangeError}, request::FinalizedBlock, + response::HeaderRangeCommitOutcome, service::check, service::finalized_state::{ disk_db::{DiskDb, DiskWriteBatch, ReadDisk, WriteDisk}, @@ -1956,7 +1957,7 @@ impl DiskWriteBatch { anchor: block::Hash, headers: &[Arc], body_sizes: &[u32], - ) -> Result { + ) -> Result { let roots = inferred_header_range_roots(zebra_db, anchor, headers.len())?; self.prepare_header_range_batch_with_roots(zebra_db, anchor, headers, body_sizes, &roots) } @@ -1971,7 +1972,7 @@ impl DiskWriteBatch { headers: &[Arc], body_sizes: &[u32], tree_aux_roots: &[BlockCommitmentRoots], - ) -> Result { + ) -> Result { if headers.is_empty() { return Err(CommitHeaderRangeError::EmptyRange); } @@ -2205,9 +2206,10 @@ impl DiskWriteBatch { } } - Ok(block::Hash::from( - &**headers.last().expect("headers is non-empty"), - )) + Ok(HeaderRangeCommitOutcome { + tip_hash: block::Hash::from(&**headers.last().expect("headers is non-empty")), + reorged_at: first_conflicting_height, + }) } /// Deletes the block header at `height`. diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs index 92230f2e8d1..bd6f1108d42 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs @@ -114,7 +114,8 @@ fn header_range_commit_keeps_body_availability_separate() { .write_batch(batch) .expect("header range batch writes successfully"); - assert_eq!(committed_hash, block1.hash()); + assert_eq!(committed_hash.tip_hash, block1.hash()); + assert_eq!(committed_hash.reorged_at, None); assert_eq!(state.best_header_tip(), Some((Height(1), block1.hash()))); assert_eq!(state.finalized_tip_height(), Some(Height(0))); assert_eq!(state.tip(), Some((Height(0), genesis.hash()))); @@ -217,18 +218,24 @@ fn header_range_reorg_resets_advertised_body_sizes() { let original = synthetic_headers_from_state(&state, Height(0), genesis.hash(), 2, 1); let mut batch = DiskWriteBatch::new(); - batch + let outcome = batch .prepare_header_range_batch(&state, genesis.hash(), &original, &[111, 222]) .expect("original synthetic headers are valid"); + assert_eq!(outcome.reorged_at, None); state.write_batch(batch).expect("header batch writes"); assert_eq!(state.advertised_body_size(Height(1)), Some(111)); assert_eq!(state.advertised_body_size(Height(2)), Some(222)); let replacement = synthetic_headers_from_state(&state, Height(0), genesis.hash(), 3, 9); let mut batch = DiskWriteBatch::new(); - batch + let outcome = batch .prepare_header_range_batch(&state, genesis.hash(), &replacement, &[0, 0, 333]) .expect("higher-work replacement synthetic headers are valid"); + assert_eq!( + outcome.reorged_at, + Some(Height(1)), + "replacing a conflicting suffix reports the first replaced height" + ); state.write_batch(batch).expect("header batch writes"); assert_eq!(state.advertised_body_size(Height(1)), None); @@ -1401,13 +1408,13 @@ fn commit_header_range( ) -> block::Hash { let mut batch = DiskWriteBatch::new(); let body_sizes = vec![0; headers.len()]; - let committed_hash = batch + let outcome = batch .prepare_header_range_batch(state, anchor, headers, &body_sizes) .expect("header range is valid"); state .write_batch(batch) .expect("header range batch writes successfully"); - committed_hash + outcome.tip_hash } fn write_full_block_header_and_transactions(state: &ZebraDb, block: Arc) { diff --git a/zebra-state/src/service/tests.rs b/zebra-state/src/service/tests.rs index 8f363c28cdb..675e4ac4001 100644 --- a/zebra-state/src/service/tests.rs +++ b/zebra-state/src/service/tests.rs @@ -35,8 +35,8 @@ use crate::{ setup::{partial_nu5_chain_strategy, transaction_v4_from_coinbase}, FakeChainHelper, }, - BoxError, CheckpointVerifiedBlock, Config, ReadRequest, ReadResponse, Request, Response, - SemanticallyVerifiedBlock, + BoxError, CheckpointVerifiedBlock, Config, HeaderRangeCommitOutcome, ReadRequest, ReadResponse, + Request, Response, SemanticallyVerifiedBlock, }; const LAST_BLOCK_HEIGHT: u32 = 10; @@ -538,7 +538,10 @@ async fn header_only_service_requests_preserve_body_boundary() -> std::result::R tree_aux_roots: roots_from_height(Height(1), 2), }) .await?, - Response::Committed(block2_hash), + Response::CommittedHeaderRange(HeaderRangeCommitOutcome { + tip_hash: block2_hash, + reorged_at: None, + }), ); assert_eq!( @@ -793,7 +796,13 @@ async fn commit_header_range_completes_while_in_finalized_write_phase( .await .expect("CommitHeaderRange must not deadlock while in the finalized write phase")?; - assert_eq!(committed, Response::Committed(block2_hash)); + assert_eq!( + committed, + Response::CommittedHeaderRange(HeaderRangeCommitOutcome { + tip_hash: block2_hash, + reorged_at: None, + }) + ); assert_eq!( state diff --git a/zebra-state/src/service/write.rs b/zebra-state/src/service/write.rs index 21c0ac0d974..588078056f1 100644 --- a/zebra-state/src/service/write.rs +++ b/zebra-state/src/service/write.rs @@ -29,7 +29,7 @@ use crate::{ queued_blocks::{QueuedCheckpointVerified, QueuedSemanticallyVerified}, ChainTipBlock, ChainTipSender, InvalidateError, ReconsiderError, }, - SemanticallyVerifiedBlock, ValidateContextError, + HeaderRangeCommitOutcome, SemanticallyVerifiedBlock, ValidateContextError, }; // These types are used in doc links @@ -134,7 +134,7 @@ fn commit_header_range( headers: Vec>, body_sizes: Vec, tree_aux_roots: Vec, - rsp_tx: oneshot::Sender>, + rsp_tx: oneshot::Sender>, ) { let mut batch = crate::service::finalized_state::DiskWriteBatch::new(); let result = batch @@ -145,11 +145,11 @@ fn commit_header_range( &body_sizes, &tree_aux_roots, ) - .and_then(|hash| { + .and_then(|outcome| { finalized_state .db .write_batch(batch) - .map(|()| hash) + .map(|()| outcome) .map_err(|error| { tracing::error!(?error, "failed to write validated header range"); @@ -198,7 +198,7 @@ pub enum NonFinalizedWriteMessage { headers: Vec>, body_sizes: Vec, tree_aux_roots: Vec, - rsp_tx: oneshot::Sender>, + rsp_tx: oneshot::Sender>, }, /// The hash of a block that should be invalidated and removed from /// the non-finalized state, if present. diff --git a/zebrad/src/commands/start/zakura/header_sync_driver.rs b/zebrad/src/commands/start/zakura/header_sync_driver.rs index a1d9ca0bec7..d5af4e3c580 100644 --- a/zebrad/src/commands/start/zakura/header_sync_driver.rs +++ b/zebrad/src/commands/start/zakura/header_sync_driver.rs @@ -636,6 +636,9 @@ pub(crate) async fn drive_zakura_header_sync_actions { + Ok(zebra_state::Response::CommittedHeaderRange(outcome)) => { + let tip_hash = outcome.tip_hash; + if let Some(reorged_at) = outcome.reorged_at { + // The offset fits usize: it is bounded by the + // committed range length, which is far below + // u32::MAX and platform usize on all targets. + let new_hash = committed_headers + .get(reorged_at.0.saturating_sub(start_height.0) as usize) + .map(|header| block::Hash::from(header.as_ref())); + invalidate_reorged_body_suffix( + state.clone(), + read_state.clone(), + reorged_at, + new_hash, + &trace, + ) + .await; + } emit_commit_state( &trace, cs_trace::COMMIT_FINISH, @@ -864,6 +884,36 @@ pub(crate) async fn drive_zakura_header_sync_actions { + let hash = match read_state + .clone() + .oneshot(zebra_state::ReadRequest::HeadersByHeightRange { + start: height, + count: 1, + }) + .await + { + Ok(zebra_state::ReadResponse::Headers(headers)) => { + headers.first().map(|(_height, hash, _header)| *hash) + } + Ok(response) => { + warn!(?response, "unexpected reanchor-target headers response"); + None + } + Err(error) => { + warn!( + ?height, + ?error, + "failed to read Zakura walk-back reanchor target from state" + ); + None + } + }; + let _ = handles + .header_sync + .send(HeaderSyncEvent::ReanchorTargetLoaded { height, hash }) + .await; + } HeaderSyncAction::QueryMissingBlockBodies { from, limit } => { log_missing_block_bodies(read_state.clone(), from, limit, &trace).await; } @@ -1096,6 +1146,121 @@ async fn log_missing_block_bodies( } } +/// Drops the committed body suffix stranded by a header-chain reorg. +/// +/// `reorged_at` is the first height where a committed header range replaced a +/// conflicting stored header; `new_hash` is the new branch's hash there. If +/// the best body chain still holds a different block at that height, that +/// block and its descendants are invalidated, resetting the chain tip to the +/// fork point. The chain-tip mirror then publishes the reset frontier +/// (`VerifiedReset`), and block sync re-downloads the heights on the new +/// branch. Without this, body-gap discovery starts above the stale tip and +/// the new branch's bodies below it are never fetched. +async fn invalidate_reorged_body_suffix( + state: State, + read_state: ReadState, + reorged_at: block::Height, + new_hash: Option, + trace: &ZakuraTrace, +) where + State: Service< + zebra_state::Request, + Response = zebra_state::Response, + Error = zebra_state::BoxError, + > + Clone + + Send + + 'static, + State::Future: Send + 'static, + ReadState: Service< + zebra_state::ReadRequest, + Response = zebra_state::ReadResponse, + Error = zebra_state::BoxError, + > + Clone + + Send + + 'static, + ReadState::Future: Send + 'static, +{ + let old_hash = match read_state + .oneshot(zebra_state::ReadRequest::BestChainBlockHash(reorged_at)) + .await + { + Ok(zebra_state::ReadResponse::BlockHash(hash)) => hash, + Ok(response) => { + warn!(?response, "unexpected BestChainBlockHash response"); + None + } + Err(error) => { + warn!( + ?reorged_at, + ?error, + "failed to read the stranded body hash after a Zakura header reorg" + ); + None + } + }; + let Some(old_hash) = old_hash else { + // No committed body at the reorged height; nothing to invalidate. + return; + }; + if new_hash == Some(old_hash) { + return; + } + + warn!( + ?reorged_at, + ?old_hash, + ?new_hash, + "Zakura header reorg crossed the committed body suffix; \ + invalidating the stranded branch so block sync can re-download it" + ); + emit_commit_state(trace, cs_trace::COMMIT_START, "header_sync_driver", |row| { + insert_cs_str(row, cs_trace::ACTION, "invalidate_reorged_body_suffix"); + insert_cs_height(row, cs_trace::HEIGHT, reorged_at); + insert_cs_hash(row, cs_trace::HASH, old_hash); + }); + let started = Instant::now(); + let result = state + .oneshot(zebra_state::Request::InvalidateBlock(old_hash)) + .await; + emit_commit_state( + trace, + cs_trace::COMMIT_FINISH, + "header_sync_driver", + |row| { + insert_cs_str(row, cs_trace::ACTION, "invalidate_reorged_body_suffix"); + insert_cs_height(row, cs_trace::HEIGHT, reorged_at); + insert_cs_hash(row, cs_trace::HASH, old_hash); + insert_cs_str( + row, + cs_trace::RESULT, + if result.is_ok() { + "invalidated" + } else { + "failed" + }, + ); + insert_cs_u64(row, cs_trace::ELAPSED_MS, elapsed_ms(started)); + }, + ); + match result { + Ok(zebra_state::Response::Invalidated(hash)) => { + metrics::counter!("sync.header.fork_recovery.body_suffix_invalidated").increment(1); + info!( + ?reorged_at, + ?hash, + "invalidated the stranded body suffix after a Zakura header reorg" + ); + } + Ok(response) => warn!(?response, "unexpected InvalidateBlock response"), + Err(error) => warn!( + ?reorged_at, + ?old_hash, + ?error, + "failed to invalidate the stranded body suffix after a Zakura header reorg" + ), + } +} + pub(crate) fn header_range_commit_failure_kind( error: &(dyn std::error::Error + Send + Sync + 'static), ) -> HeaderSyncCommitFailureKind { @@ -1385,6 +1550,10 @@ fn trace_header_driver_action(trace: &ZakuraTrace, action: &HeaderSyncAction) { HeaderSyncAction::QueryBestHeaderTip => { insert_cs_str(row, cs_trace::ACTION, "query_best_header_tip"); } + HeaderSyncAction::QueryReanchorTarget { height } => { + insert_cs_str(row, cs_trace::ACTION, "query_reanchor_target"); + insert_cs_height(row, cs_trace::HEIGHT, *height); + } HeaderSyncAction::QueryHeadersByHeightRange { peer, start, count, .. } => { From d5f04567fcca261f34929d93b577f0ebf318fc99 Mon Sep 17 00:00:00 2001 From: roman Date: Sun, 5 Jul 2026 21:47:14 -0600 Subject: [PATCH 02/27] docs: changelog entry for zakura header-sync fork recovery --- CHANGELOG.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 41acab50ed4..1db7ffeaac0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,19 @@ and this project adheres to [Semantic Versioning](https://semver.org). - Fixed a near-tip sync restart loop when a timed-out `AwaitUtxo` lookup in the transaction verifier was converted to `InternalDowncastError` instead of a missing transparent input. +- Fixed Zakura header sync permanently stranding a node whose committed chain + suffix ends up on a branch the network abandoned. Header sync only requested + headers forward from its own frontier hash and treated every non-linking + response as peer misbehavior, so after a reorg a zakura-only node rejected + all honest headers forever (`first_header_does_not_link`), flagged the + honest peers, and never caught up — restarts did not help. Non-linking + forward responses are no longer scored; after repeated failures from + independent peers the request anchor now walks back below the verified block + tip (exponentially deeper per round, at most `MAX_BLOCK_REORG_HEIGHT`, never + below the finalized height) until responses link, the higher-work header + range commits through the fork point, and the stranded committed body suffix + is invalidated so block sync re-downloads the new branch. Recovery progress + is logged and counted (`sync.header.fork_recovery.*`). - Fixed dual-stack nodes (`v2_p2p` and `legacy_p2p` both enabled) permanently shutting down their own Zakura header- and block-sync drivers when a legacy peer on a foreign fork answered the body-sync stall watchdog's cross-check From 1aa8ad63734f24b4bd0c850a2f0cc7ef5c379c37 Mon Sep 17 00:00:00 2001 From: roman Date: Sun, 5 Jul 2026 22:07:50 -0600 Subject: [PATCH 03/27] fix(zakura): only advance and gossip NewBlocks that land on the best chain An accepted NewBlock could have committed to a side chain; advancing the header/verified frontiers and forwarding it made the whole Zakura layer follow and propagate losing branches (e.g. testnet min-difficulty forks) while each node's own chain stayed honest, stranding zakura-only peers. The driver now checks the committed hash against the best chain (ReadRequest::Depth) and routes side-chain accepts to a dedup-only event. --- CHANGELOG.md | 9 ++ .../src/zakura/header_sync/events.rs | 15 ++ .../src/zakura/header_sync/reactor.rs | 24 ++++ zebra-network/src/zakura/header_sync/tests.rs | 88 ++++++++++++ zebrad/src/commands/start.rs | 135 ++++++++++++++++++ .../start/zakura/header_sync_driver.rs | 78 +++++++++- 6 files changed, 342 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1db7ffeaac0..b38ae2c84c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,15 @@ and this project adheres to [Semantic Versioning](https://semver.org). - Fixed a near-tip sync restart loop when a timed-out `AwaitUtxo` lookup in the transaction verifier was converted to `InternalDowncastError` instead of a missing transparent input. +- Fixed Zakura nodes gossiping and following side-chain blocks. An inbound + `NewBlock` that committed as a side chain (for example a testnet + min-difficulty branch) still advanced the node's Zakura header and verified + frontiers and was forwarded to peers, so a whole fleet could advertise and + propagate a losing branch over Zakura while each node's own chain stayed on + the best one — stranding zakura-only peers that followed the gossip. An + accepted `NewBlock` is now checked against the best chain before it advances + any frontier or is forwarded; side-chain commits are remembered for dedup + only and counted in `sync.header.tip.new_block.side_chain`. - Fixed Zakura header sync permanently stranding a node whose committed chain suffix ends up on a branch the network abandoned. Header sync only requested headers forward from its own frontier hash and treated every non-linking diff --git a/zebra-network/src/zakura/header_sync/events.rs b/zebra-network/src/zakura/header_sync/events.rs index 25b2d2a7b13..78b9b866d84 100644 --- a/zebra-network/src/zakura/header_sync/events.rs +++ b/zebra-network/src/zakura/header_sync/events.rs @@ -185,6 +185,20 @@ pub enum HeaderSyncEvent { /// Duplicate block hash. hash: block::Hash, }, + /// The node's block pipeline accepted an inbound `NewBlock` body, but it + /// committed to a side chain instead of the best chain. The block is + /// remembered for dedup only: a side-chain block must not advance the + /// header or verified frontiers and must not be forwarded to peers, or + /// the whole Zakura layer gossips a losing branch while the node's own + /// chain stays on the best one. + NewBlockAcceptedSideChain { + /// Source peer. + peer: ZakuraPeerId, + /// Accepted side-chain block height. + height: block::Height, + /// Accepted side-chain block hash. + hash: block::Hash, + }, /// The node's block pipeline rejected an inbound `NewBlock` body. NewBlockRejected { /// Source peer. @@ -284,6 +298,7 @@ impl HeaderSyncEvent { Self::FullBlockCommitted { .. } => "full_block_committed", Self::NewBlockAccepted { .. } => "new_block_accepted", Self::NewBlockDuplicate { .. } => "new_block_duplicate", + Self::NewBlockAcceptedSideChain { .. } => "new_block_accepted_side_chain", Self::NewBlockRejected { .. } => "new_block_rejected", Self::WireMessage { .. } => "wire_message", Self::WireDecodeFailed { .. } => "wire_decode_failed", diff --git a/zebra-network/src/zakura/header_sync/reactor.rs b/zebra-network/src/zakura/header_sync/reactor.rs index 85b240908e8..0219ed1f429 100644 --- a/zebra-network/src/zakura/header_sync/reactor.rs +++ b/zebra-network/src/zakura/header_sync/reactor.rs @@ -171,6 +171,9 @@ impl HeaderSyncReactor { HeaderSyncEvent::NewBlockDuplicate { peer, height, hash } => { self.handle_new_block_duplicate(peer, height, hash) } + HeaderSyncEvent::NewBlockAcceptedSideChain { peer, height, hash } => { + self.handle_new_block_accepted_side_chain(peer, height, hash) + } HeaderSyncEvent::NewBlockRejected { peer, hash } => { self.handle_new_block_rejected(peer, hash).await } @@ -556,6 +559,21 @@ impl HeaderSyncReactor { self.trace_new_block_deduped(&peer, height, hash, "already_in_chain"); } + /// Remembers an accepted side-chain `NewBlock` for dedup without advancing + /// any frontier or forwarding it. See + /// [`HeaderSyncEvent::NewBlockAcceptedSideChain`]. + fn handle_new_block_accepted_side_chain( + &mut self, + peer: ZakuraPeerId, + height: block::Height, + hash: block::Hash, + ) { + self.state.pending_new_blocks.remove(&hash); + let _ = self.state.seen.insert(hash); + metrics::counter!("sync.header.tip.new_block.side_chain").increment(1); + self.trace_new_block_deduped(&peer, height, hash, "side_chain"); + } + async fn handle_new_block_rejected(&mut self, peer: ZakuraPeerId, hash: block::Hash) { self.state.pending_new_blocks.remove(&hash); metrics::counter!("sync.header.tip.new_block.rejected").increment(1); @@ -1775,6 +1793,12 @@ impl HeaderSyncReactor { insert_height(row, hs_trace::HEIGHT, *height); insert_hash(row, hs_trace::HASH, *hash); } + HeaderSyncEvent::NewBlockAcceptedSideChain { peer, height, hash } => { + insert_optional_str(row, hs_trace::KIND, Some("new_block_accepted_side_chain")); + insert_peer(row, hs_trace::PEER, peer); + insert_height(row, hs_trace::HEIGHT, *height); + insert_hash(row, hs_trace::HASH, *hash); + } HeaderSyncEvent::NewBlockRejected { peer, hash } => { insert_optional_str(row, hs_trace::KIND, Some("new_block_rejected")); insert_peer(row, hs_trace::PEER, peer); diff --git a/zebra-network/src/zakura/header_sync/tests.rs b/zebra-network/src/zakura/header_sync/tests.rs index ba56cb0c20a..b40079532a2 100644 --- a/zebra-network/src/zakura/header_sync/tests.rs +++ b/zebra-network/src/zakura/header_sync/tests.rs @@ -2903,6 +2903,94 @@ async fn inbound_unseen_valid_new_block_is_seen_and_forwarded_to_eligible_peers( } } +#[tokio::test(flavor = "current_thread")] +async fn accepted_side_chain_new_block_is_deduped_without_advancing_or_forwarding() { + let network = Network::Mainnet; + let block = mainnet_block(&BLOCK_MAINNET_1_BYTES); + let hash = block.hash(); + let height = block.coinbase_height().expect("test block has height"); + let anchor = (block::Height(0), network.genesis_hash()); + let mut fixture = spawn_test_reactor(startup_for(network.clone(), anchor, None)); + let mut tip = fixture.handle.subscribe_tip(); + let source = peer(55); + let would_be_destination = peer(56); + + // The destination's advertised tip is below the block height, so a + // best-chain accept at this height WOULD forward to it. + for peer_id in [source.clone(), would_be_destination.clone()] { + connect_peer(&fixture, peer_id.clone()).await; + advertise_tip( + &fixture, + peer_id, + block::Height(0), + block::Height(0), + DEFAULT_HS_RANGE, + 1, + ) + .await; + } + + fixture + .handle + .send(HeaderSyncEvent::NewBlockAcceptedSideChain { + peer: source.clone(), + height, + hash, + }) + .await + .unwrap(); + + // A side-chain accept advances no frontier and forwards nothing. + while let Ok(Some(action)) = tokio::time::timeout( + std::time::Duration::from_millis(200), + fixture.actions.recv(), + ) + .await + { + if matches!( + action, + HeaderSyncAction::ForwardNewBlock { .. } + | HeaderSyncAction::HeaderAdvanced { .. } + | HeaderSyncAction::HeaderReanchored { .. } + ) { + panic!("side-chain accept must not advance frontiers or forward: {action:?}"); + } + } + assert_eq!(fixture.handle.best_header_tip(), anchor); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(50), tip.changed()) + .await + .is_err(), + "side-chain accept must not publish a new best header tip" + ); + + // The hash is remembered: a later wire NewBlock for it dedups without + // re-entering the block pipeline or scoring the sender. + fixture + .handle + .send(HeaderSyncEvent::WireMessage { + peer: source, + msg: HeaderSyncMessage::NewBlock(block), + }) + .await + .unwrap(); + while let Ok(Some(action)) = tokio::time::timeout( + std::time::Duration::from_millis(200), + fixture.actions.recv(), + ) + .await + { + if matches!( + action, + HeaderSyncAction::NewBlockReceived { .. } + | HeaderSyncAction::ForwardNewBlock { .. } + | HeaderSyncAction::Misbehavior { .. } + ) { + panic!("seen side-chain block must be cheap-deduped without scoring: {action:?}"); + } + } +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn concurrent_duplicate_new_block_dedups_pending_acceptance_without_scoring() { let network = Network::Mainnet; diff --git a/zebrad/src/commands/start.rs b/zebrad/src/commands/start.rs index d3a6799870d..6e20001c166 100644 --- a/zebrad/src/commands/start.rs +++ b/zebrad/src/commands/start.rs @@ -2775,6 +2775,141 @@ mod zakura_header_sync_driver_tests { endpoint.shutdown().await; } + /// End-to-end driver + reactor: an accepted `NewBlock` that committed to a + /// side chain (state `Depth` = `None`) must not advance the header + /// frontier, while a best-chain accept (`Depth` = `Some`) must. + #[tokio::test] + async fn new_block_side_chain_commit_does_not_advance_header_frontier() { + let network = zebra_chain::parameters::Network::Mainnet; + let genesis_hash = network.genesis_hash(); + let mut config = zebra_network::Config { + network: network.clone(), + ..zebra_network::Config::default() + }; + config.zakura.listen_addr = None; + let endpoint = zebra_network::zakura::spawn_zakura_endpoint_with_header_sync_driver( + &config, + |_supervisor, _trace| Arc::new(NoopZakuraService) as Arc, + Some(ZakuraHeaderSyncDriverStartup { + frontiers: HeaderSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(0), + verified_block_hash: genesis_hash, + }, + best_header_tip: Some((block::Height(0), genesis_hash)), + verified_block_tip_hash: genesis_hash, + }), + ) + .await + .expect("Zakura endpoint starts") + .expect("v2_p2p starts an endpoint"); + let header_sync = endpoint + .header_sync() + .expect("driver startup starts header sync"); + + // Block 2 plays the side-chain commit; block 1 plays the best-chain one. + let side_chain_block = mainnet_block(&BLOCK_MAINNET_2_BYTES); + let side_chain_hash = side_chain_block.hash(); + let best_chain_block = mainnet_block(&BLOCK_MAINNET_1_BYTES); + let best_chain_hash = best_chain_block.hash(); + + let (action_tx, action_rx) = mpsc::channel(4); + let (shutdown_tx, shutdown_rx) = oneshot::channel(); + let handles = ZakuraHeaderSyncDriverHandles { + endpoint: endpoint.clone(), + header_sync: header_sync.clone(), + }; + let state = service_fn(|request: zebra_state::Request| async move { + panic!("unexpected state request from NewBlockReceived: {request:?}"); + #[allow(unreachable_code)] + Ok::<_, zebra_state::BoxError>(zebra_state::Response::Committed(block::Hash([0; 32]))) + }); + let read_state = service_fn(move |request: zebra_state::ReadRequest| async move { + match request { + zebra_state::ReadRequest::Depth(hash) if hash == side_chain_hash => { + Ok::<_, zebra_state::BoxError>(zebra_state::ReadResponse::Depth(None)) + } + zebra_state::ReadRequest::Depth(hash) if hash == best_chain_hash => { + Ok(zebra_state::ReadResponse::Depth(Some(0))) + } + request => panic!("unexpected read request: {request:?}"), + } + }); + // The verifier accepts both blocks; only the state decides which chain + // they landed on. + let verifier = service_fn(|request: zebra_consensus::Request| async move { + match request { + zebra_consensus::Request::Commit(block) => { + Ok::<_, zebra_consensus::BoxError>(block.hash()) + } + request => panic!("unexpected verifier request: {request:?}"), + } + }); + let driver = tokio::spawn(drive_zakura_header_sync_actions( + action_rx, + handles, + state, + read_state, + verifier, + zebra_network::zakura::ZakuraTrace::noop(), + async move { + let _ = shutdown_rx.await; + }, + )); + + let source = + zebra_network::zakura::ZakuraPeerId::new(vec![9; 32]).expect("test peer id is valid"); + action_tx + .send(zebra_network::zakura::HeaderSyncAction::NewBlockReceived { + peer: source.clone(), + height: side_chain_block + .coinbase_height() + .expect("test block has height"), + hash: side_chain_hash, + block: side_chain_block, + }) + .await + .expect("driver action channel stays open"); + + // The side-chain accept must not move the reactor's best header tip. + // Give the driver + reactor time to (incorrectly) advance before + // checking; the follow-up best-chain accept below proves liveness. + tokio::time::sleep(Duration::from_millis(200)).await; + assert_eq!( + header_sync.best_header_tip(), + (block::Height(0), genesis_hash), + "a side-chain NewBlock commit must not advance the header frontier" + ); + + let best_height = best_chain_block + .coinbase_height() + .expect("test block has height"); + action_tx + .send(zebra_network::zakura::HeaderSyncAction::NewBlockReceived { + peer: source, + height: best_height, + hash: best_chain_hash, + block: best_chain_block, + }) + .await + .expect("driver action channel stays open"); + + tokio::time::timeout(Duration::from_secs(2), async { + loop { + if header_sync.best_header_tip() == (best_height, best_chain_hash) { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("a best-chain NewBlock commit advances the header frontier"); + + let _ = shutdown_tx.send(()); + driver.await.expect("driver task exits cleanly"); + endpoint.shutdown().await; + } + #[tokio::test] async fn block_sync_driver_coalesces_stale_needed_queries() { let (action_tx, mut action_rx) = mpsc::channel(8); diff --git a/zebrad/src/commands/start/zakura/header_sync_driver.rs b/zebrad/src/commands/start/zakura/header_sync_driver.rs index d5af4e3c580..3f69c5f641a 100644 --- a/zebrad/src/commands/start/zakura/header_sync_driver.rs +++ b/zebrad/src/commands/start/zakura/header_sync_driver.rs @@ -252,32 +252,58 @@ pub(crate) async fn drive_zakura_header_sync_actions { + // A contextually valid block also commits when it lands + // on a side chain, but only a best-chain block may + // advance the header/verified frontiers or be forwarded + // to peers: gossiping side-chain blocks makes the whole + // Zakura layer follow a losing branch while the node's + // own chain stays honest, stranding zakura-only peers. + let on_best_chain = + new_block_is_on_best_chain(read_state.clone(), hash).await; + let result_label = if on_best_chain { + "accepted" + } else { + "accepted_side_chain" + }; trace_header_commit_finish( &trace, "new_block", &peer, height, hash, - "accepted", + result_label, started, ); trace_header_reactor_event( &trace, - "new_block_accepted", + if on_best_chain { + "new_block_accepted" + } else { + "new_block_accepted_side_chain" + }, Some(&peer), height, hash, 1, ); - let _ = handles - .header_sync - .send(HeaderSyncEvent::NewBlockAccepted { + let event = if on_best_chain { + HeaderSyncEvent::NewBlockAccepted { peer, height, hash, block, - }) - .await; + } + } else { + debug!( + ?peer, + ?height, + ?hash, + "Zakura NewBlock committed to a side chain; \ + not advancing frontiers or forwarding" + ); + HeaderSyncEvent::NewBlockAcceptedSideChain { peer, height, hash } + }; + let _ = handles.header_sync.send(event).await; } Ok(committed_hash) => { trace_header_commit_finish( @@ -1146,6 +1172,44 @@ async fn log_missing_block_bodies( } } +/// Returns whether a just-committed `NewBlock` landed on the best chain. +/// +/// `ReadRequest::Depth` returns `Some` only for best-chain blocks, so it +/// distinguishes a best-chain extension (or a reorg the block just won) from a +/// side-chain commit. Read failures are treated as *not* best-chain: the +/// node's own frontier still advances through the chain-tip mirror, so the +/// only cost of a false negative is skipping one gossip forward, while a +/// false positive would gossip a possibly losing branch. +async fn new_block_is_on_best_chain(read_state: ReadState, hash: block::Hash) -> bool +where + ReadState: Service< + zebra_state::ReadRequest, + Response = zebra_state::ReadResponse, + Error = zebra_state::BoxError, + > + Send + + 'static, + ReadState::Future: Send + 'static, +{ + match read_state + .oneshot(zebra_state::ReadRequest::Depth(hash)) + .await + { + Ok(zebra_state::ReadResponse::Depth(depth)) => depth.is_some(), + Ok(response) => { + warn!(?response, "unexpected Depth response for Zakura NewBlock"); + false + } + Err(error) => { + warn!( + ?hash, + ?error, + "failed to read Zakura NewBlock depth from state" + ); + false + } + } +} + /// Drops the committed body suffix stranded by a header-chain reorg. /// /// `reorged_at` is the first height where a committed header range replaced a From cf43cb840c80ba96a844abfbb9f0a8f6b058567c Mon Sep 17 00:00:00 2001 From: roman Date: Sun, 5 Jul 2026 23:25:39 -0600 Subject: [PATCH 04/27] fix(zakura): keep Zakura reactors alive when the stall watchdog falls back to legacy sync Killing the reactors turned every fallback into a fleet-wide Zakura outage: the fallback node is often the only peer with working block ingest, and with its reactors dead it stops advertising, serving, and forwarding, pinning every Zakura peer's frontier. Legacy ChainSync now resumes as the body-sync driver while Zakura quiesces into a serving/advertising bridge. Also makes inbound block-gossip ingest failures and queue drops visible at info with counters, so tip-ingest freezes are attributable from logs. --- CHANGELOG.md | 13 +++ zebrad/src/commands/start.rs | 17 ++-- zebrad/src/components/inbound/downloads.rs | 31 +++++-- zebrad/src/components/sync.rs | 66 +++++++------- zebrad/src/components/sync/tests/fallback.rs | 91 +++++++------------- 5 files changed, 106 insertions(+), 112 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b38ae2c84c1..7e99337d58b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,19 @@ and this project adheres to [Semantic Versioning](https://semver.org). - Fixed a near-tip sync restart loop when a timed-out `AwaitUtxo` lookup in the transaction verifier was converted to `InternalDowncastError` instead of a missing transparent input. +- The dual-stack Zakura stall watchdog no longer shuts down the Zakura header- + and block-sync reactors when it falls back to legacy `ChainSync`. Killing + them turned every fallback into a fleet-wide Zakura outage: the fallback + node — often the only peer with working block ingest — stopped advertising + statuses, serving headers and bodies, and forwarding `NewBlock`s, which + pinned every Zakura peer's frontier and starved zakura-only nodes until a + manual restart. Legacy `ChainSync` now resumes as the body-sync driver while + the Zakura reactors stay alive, follow local commits through the chain-tip + mirror, and keep serving the Zakura network as a bridge. Fallback engagement + is counted in `sync.zakura.legacy_fallback.engaged`. Inbound block-gossip + ingest failures and queue drops — previously silent at default log levels, + which made tip-ingest freezes unattributable — are now logged at info and + counted (`gossip.ingest.failed.count`, `gossip.ingest.duplicate.count`). - Fixed Zakura nodes gossiping and following side-chain blocks. An inbound `NewBlock` that committed as a side chain (for example a testnet min-difficulty branch) still advanced the node's Zakura header and verified diff --git a/zebrad/src/commands/start.rs b/zebrad/src/commands/start.rs index 6e20001c166..8f378bcb8a3 100644 --- a/zebrad/src/commands/start.rs +++ b/zebrad/src/commands/start.rs @@ -73,7 +73,7 @@ //! //! Some of the diagnostic features are optional, and need to be enabled at compile-time. -mod zakura; +pub(crate) mod zakura; use std::{net::SocketAddr, path::Path, sync::Arc}; @@ -980,20 +980,13 @@ impl StartCmd { let syncer_task_handle = if use_zakura_block_sync(&config.network) { info!("Zakura block sync is replacing the legacy ChainSync body downloader"); // Only dual-stack nodes (Zakura + legacy peers) fall back to legacy ChainSync on a - // Zakura stall; a Zakura-only node has no legacy peers to drive body sync. When the - // fallback fires it first cancels the Zakura endpoint shutdown token (stopping the - // header- and block-sync drivers) so the two commit pipelines never run at once. + // Zakura stall; a Zakura-only node has no legacy peers to drive body sync. The + // fallback resumes legacy ChainSync as the body-sync driver while the Zakura + // reactors stay alive as a serving/advertising bridge for zakura-only peers. let legacy_fallback = config.network.v2_p2p && config.network.legacy_p2p; tokio::spawn( syncer - .bootstrap_genesis_then_pause( - read_only_state_service.clone(), - legacy_fallback, - zakura_endpoint.clone(), - zakura_endpoint - .as_ref() - .and_then(|endpoint| endpoint.header_sync_shutdown()), - ) + .bootstrap_genesis_then_pause(read_only_state_service.clone(), legacy_fallback) .in_current_span(), ) } else { diff --git a/zebrad/src/components/inbound/downloads.rs b/zebrad/src/components/inbound/downloads.rs index 57bc3000739..95faffe9208 100644 --- a/zebrad/src/components/inbound/downloads.rs +++ b/zebrad/src/components/inbound/downloads.rs @@ -214,7 +214,26 @@ where let (result, hash) = match join_result.expect("block download and verify tasks must not panic") { Ok(hash) => (Ok(hash), hash), - Err((e, hash, advertiser_addr)) => (Err((e, advertiser_addr)), hash), + Err((e, hash, advertiser_addr)) => { + // A failed gossip ingest is the only way a node whose + // syncer is parked (Zakura mode) falls behind the + // network, so every non-duplicate failure must be + // visible at default log levels: silent failures here + // previously made fleet-wide tip-ingest freezes + // unattributable after the fact. + if crate::commands::start::zakura::block_verify_error_is_duplicate(&e) { + debug!(?hash, "gossiped block was already known"); + metrics::counter!("gossip.ingest.duplicate.count").increment(1); + } else { + info!( + ?hash, + error = ?e, + "gossiped block download or verify failed" + ); + metrics::counter!("gossip.ingest.failed.count").increment(1); + } + (Err((e, advertiser_addr)), hash) + } }; if let Some((_, Some(source))) = this.cancel_handles.remove(&hash) { let source_count = this @@ -310,7 +329,7 @@ where } if self.queue_len() >= self.full_verify_concurrency_limit { - debug!( + info!( ?hash, queue_len = self.queue_len(), concurrency_limit = self.full_verify_concurrency_limit, @@ -328,7 +347,7 @@ where let source_count = self.source_counts.get(source).copied().unwrap_or_default(); let source_limit = source.max_in_flight(self.full_verify_concurrency_limit); if source_count >= source_limit { - debug!( + info!( ?hash, ?source, source_count, @@ -499,7 +518,7 @@ where let block_height = block .coinbase_height() .ok_or_else(|| { - debug!( + info!( ?hash, "gossiped block with no height: dropped downloaded block" ); @@ -510,7 +529,7 @@ where .map_err(|e| (e, None))?; if block_height > max_lookahead_height { - debug!( + info!( ?hash, ?block_height, ?tip_height, @@ -522,7 +541,7 @@ where Err("gossiped block height too far ahead").map_err(|e| (e.into(), None))?; } else if block_height < min_accepted_height { - debug!( + info!( ?hash, ?block_height, ?tip_height, diff --git a/zebrad/src/components/sync.rs b/zebrad/src/components/sync.rs index 37af7aa76a4..d24e8582cd1 100644 --- a/zebrad/src/components/sync.rs +++ b/zebrad/src/components/sync.rs @@ -516,22 +516,23 @@ where } } -/// Cancels and drains the Zakura sync drivers before the legacy [`ChainSync::sync`] loop resumes. +/// Records that legacy [`ChainSync::sync`] is resuming as the body-sync driver +/// while the Zakura reactors stay alive. /// -/// The cancellation token stops new Zakura sync work. Awaiting the endpoint-owned tasks makes the -/// hand-off a commit barrier: already-started Zakura block applies finish before legacy can submit -/// commits through the same verifier and state pipeline. No-op when Zakura networking is absent. -async fn stop_zakura_sync( - zakura_endpoint: Option<&zn::zakura::ZakuraEndpoint>, - zakura_shutdown: &Option, -) { - if let Some(token) = zakura_shutdown { - token.cancel(); - } - - if let Some(endpoint) = zakura_endpoint { - endpoint.shutdown_sync_tasks().await; - } +/// The Zakura header- and block-sync reactors are deliberately NOT cancelled: +/// killing them turned every fallback into a fleet-wide Zakura outage — the +/// fallback node is often the only peer with working ingest, and with its +/// reactors dead it stops advertising statuses, serving headers/bodies, and +/// forwarding `NewBlock`s, starving every zakura-only peer and pinning the +/// other nodes' frontiers (they then fall back too). Both engines coexist +/// safely: the reactors are frontier-driven, so once legacy commits ahead of +/// the Zakura frontier they stop requesting and quiesce into a serving role, +/// following local commits through the chain-tip mirror — the same way +/// gossip-driven commits already coexist with the legacy syncer. Duplicate +/// commits from the overlap window are rejected by the verifier as usual. +fn engage_legacy_fallback_alongside_zakura() { + metrics::counter!("sync.zakura.legacy_fallback.engaged").increment(1); + metrics::gauge!("sync.zakura.legacy_fallback.active").set(1.0); } /// Sync configuration section. @@ -953,26 +954,25 @@ where /// Zakura body-sync peers, and parking forever there leaves it stuck at genesis. /// /// `legacy_fallback` (set when the node runs both stacks, `v2_p2p && legacy_p2p`) - /// controls the recovery path. When `true`, a Zakura stall first cancels - /// `zakura_shutdown` — the endpoint shutdown token shared by the Zakura header- and - /// block-sync drivers — so they stop before the legacy [`ChainSync::sync`] loop - /// resumes, ensuring only one body-sync committer is ever active (two at once break - /// the state-commit pipeline's accounting and can deadlock the node). When `false` - /// (a Zakura-only node, where falling back to absent legacy peers is pointless), - /// the watchdog never switches: it parks and warns (once per stall window) so a - /// stalled, eclipsed, or peerless node is visible in the logs. + /// controls the recovery path. When `true`, a Zakura stall resumes the legacy + /// [`ChainSync::sync`] loop as the body-sync driver while the Zakura header- and + /// block-sync reactors stay alive: they follow local commits through the chain-tip + /// mirror and quiesce into a serving/advertising role, so the fallback node keeps + /// acting as a Zakura bridge for zakura-only peers (see + /// [`engage_legacy_fallback_alongside_zakura`]). When `false` (a Zakura-only node, + /// where falling back to absent legacy peers is pointless), the watchdog never + /// switches: it parks and warns (once per stall window) so a stalled, eclipsed, or + /// peerless node is visible in the logs. /// /// `read_state` answers [`ReadRequest::BestHeaderTip`](zs::ReadRequest::BestHeaderTip) /// for the legacy-informed cross-check, which only probes legacy peers when /// the verified tip is frozen and the node looks caught up to its own header /// frontier. - #[instrument(skip(self, read_state, zakura_endpoint, zakura_shutdown))] + #[instrument(skip(self, read_state))] pub async fn bootstrap_genesis_then_pause( mut self, mut read_state: RS, legacy_fallback: bool, - zakura_endpoint: Option, - zakura_shutdown: Option, ) -> Result<(), Report> where RS: Service @@ -1029,11 +1029,11 @@ where verified_tip = ?verified_height, header_tip = ?header_tip_height, stall = ?ZAKURA_BODY_SYNC_STALL_TIMEOUT, - "Zakura body sync is not closing the gap to the network tip; stopping \ - Zakura sync drivers and falling back to legacy ChainSync so legacy peers \ - can drive body sync" + "Zakura body sync is not closing the gap to the network tip; resuming \ + legacy ChainSync as the body-sync driver while Zakura keeps serving \ + peers and following local commits" ); - stop_zakura_sync(zakura_endpoint.as_ref(), &zakura_shutdown).await; + engage_legacy_fallback_alongside_zakura(); return self.sync().await; } ZakuraWatchdogAction::ProbeLegacyPeers => { @@ -1049,10 +1049,10 @@ where header_tip = ?header_tip_height, ?blocks_ahead, "Zakura body sync is frozen while legacy peers advertise a much \ - higher tip; stopping Zakura sync drivers and falling back to legacy \ - ChainSync so it can drive body sync" + higher tip; resuming legacy ChainSync as the body-sync driver \ + while Zakura keeps serving peers and following local commits" ); - stop_zakura_sync(zakura_endpoint.as_ref(), &zakura_shutdown).await; + engage_legacy_fallback_alongside_zakura(); return self.sync().await; } } diff --git a/zebrad/src/components/sync/tests/fallback.rs b/zebrad/src/components/sync/tests/fallback.rs index ae67059bf04..b885e14655a 100644 --- a/zebrad/src/components/sync/tests/fallback.rs +++ b/zebrad/src/components/sync/tests/fallback.rs @@ -1,18 +1,15 @@ //! Tests for the Zakura body-sync stall watchdog //! ([`ChainSync::bootstrap_genesis_then_pause`]). //! -//! These exercise the pure decision function [`zakura_block_sync_stalled`] and the -//! [`stop_zakura_sync`] hand-off helper directly, so they are deterministic and need -//! no clock, services, or live `ChainTip`. - -use tokio_util::sync::CancellationToken; +//! These exercise the pure decision function [`zakura_block_sync_stalled`] directly, +//! so they are deterministic and need no clock, services, or live `ChainTip`. use zebra_chain::{block::Height, chain_sync_status::ChainSyncStatus}; use super::super::{ - legacy_probe_supports_fallback, stop_zakura_sync, zakura_block_sync_stalled, - zakura_sync_status_length, zakura_watchdog_action, SyncStatus, ZakuraLegacyProbe, - ZakuraStallTracker, ZakuraWatchdogAction, ZAKURA_LEGACY_BEHIND_THRESHOLD, + legacy_probe_supports_fallback, zakura_block_sync_stalled, zakura_sync_status_length, + zakura_watchdog_action, SyncStatus, ZakuraLegacyProbe, ZakuraStallTracker, + ZakuraWatchdogAction, ZAKURA_LEGACY_BEHIND_THRESHOLD, }; /// The original height-only rule, reproduced here only to demonstrate the F-88602 @@ -214,11 +211,14 @@ fn frozen_but_materially_behind_leaves_probe_to_gap_rule() { } } -#[tokio::test] -async fn stalled_zakura_with_legacy_fallback_cancels_the_shutdown_token() { +/// The fallback decision fires on a frozen verified tip, and the hand-off keeps +/// the Zakura reactors alive: legacy ChainSync resumes as the body-sync driver +/// while Zakura quiesces into a serving/advertising bridge. Killing the +/// reactors here previously turned every fallback into a fleet-wide Zakura +/// outage (the fallback node is often the only peer with working ingest). +#[test] +fn stalled_zakura_with_legacy_fallback_keeps_zakura_reactors_alive() { let max_idle_polls = 3; - let token = CancellationToken::new(); - let driver_view = token.child_token(); let mut tracker = ZakuraStallTracker::new(Some(Height(0))); let mut legacy_probe = ZakuraLegacyProbe::new(Some(Height(0))); @@ -241,18 +241,14 @@ async fn stalled_zakura_with_legacy_fallback_cancels_the_shutdown_token() { ZakuraWatchdogAction::FallbackToLegacy, "a frozen verified tip must trigger legacy fallback when it is enabled" ); - stop_zakura_sync(None, &Some(token)).await; - assert!( - driver_view.is_cancelled(), - "falling back to legacy must cancel the Zakura sync drivers' shutdown token" - ); + // The hand-off is now driver-only: nothing in the fallback path may cancel + // Zakura work, so there is no shutdown token to assert on. + super::super::engage_legacy_fallback_alongside_zakura(); } #[test] fn stalled_zakura_without_legacy_fallback_keeps_waiting() { let max_idle_polls = 3; - let token = CancellationToken::new(); - let driver_view = token.child_token(); let mut tracker = ZakuraStallTracker::new(Some(Height(0))); let mut legacy_probe = ZakuraLegacyProbe::new(Some(Height(0))); @@ -281,18 +277,15 @@ fn stalled_zakura_without_legacy_fallback_keeps_waiting() { saw_warn_only, "Zakura-only stalls should still produce the warn-only watchdog action" ); - assert!( - !driver_view.is_cancelled(), - "warn-only Zakura stalls must not cancel the Zakura shutdown token" - ); } -#[tokio::test] -async fn frozen_zero_gap_with_legacy_peers_ahead_cancels_the_shutdown_token() { +/// A frozen tip that looks caught up cross-checks legacy peers, and a probe at +/// or above the behind threshold engages the legacy fallback — which keeps the +/// Zakura reactors alive rather than cancelling anything. +#[test] +fn frozen_zero_gap_with_legacy_peers_ahead_engages_fallback() { let max_idle_polls = 5; let frozen = Some(Height(1_000)); - let token = CancellationToken::new(); - let driver_view = token.child_token(); let mut tracker = ZakuraStallTracker::new(frozen); let mut legacy_probe = ZakuraLegacyProbe::new(frozen); @@ -318,11 +311,7 @@ async fn frozen_zero_gap_with_legacy_peers_ahead_cancels_the_shutdown_token() { "legacy peers at or above the behind threshold must trigger fallback" ); - stop_zakura_sync(None, &Some(token)).await; - assert!( - driver_view.is_cancelled(), - "legacy-informed fallback must cancel the Zakura sync drivers' shutdown token" - ); + super::super::engage_legacy_fallback_alongside_zakura(); } #[test] @@ -405,33 +394,13 @@ fn zakura_sync_status_lengths_drive_existing_mempool_gate() { ); } -/// The point of this test is to lock in the fallback behavior: when Zebra decides to stop using -/// Zakura sync and fall back to legacy sync, it must signal the running Zakura driver tasks to shut down. -/// This asserts that the shutdown token is cancelled when the fallback occurs. -#[tokio::test] -async fn fallback_cancels_the_zakura_shutdown_token() { - let token = CancellationToken::new(); - assert!( - !token.is_cancelled(), - "precondition: a fresh token is not cancelled" - ); - - // A child token stands in for the drivers' observed shutdown: cancelling the shared token the - // watchdog holds must propagate to what the drivers actually await. - let driver_view = token.child_token(); - - stop_zakura_sync(None, &Some(token)).await; - - assert!( - driver_view.is_cancelled(), - "falling back to legacy must cancel the Zakura sync drivers' shutdown token" - ); -} - -/// On a Zakura-only node there is no endpoint shutdown token, so the hand-off helper must be a -/// no-op rather than panic. -#[tokio::test] -async fn stop_zakura_sync_is_a_noop_without_a_token() { - // Must not panic. - stop_zakura_sync(None, &None).await; +/// Locks in the fallback behavior: engaging legacy fallback must not signal any +/// Zakura shutdown — the reactors stay alive as a serving bridge. This test +/// documents the intentional inversion of the old "fallback cancels the Zakura +/// shutdown token" contract. +#[test] +fn fallback_does_not_cancel_any_zakura_shutdown_token() { + // The hand-off helper takes no token and cancels nothing; it only records + // the mode switch. Must not panic. + super::super::engage_legacy_fallback_alongside_zakura(); } From a1d409226d886221012fb0d384a677c9f84dfe46 Mon Sep 17 00:00:00 2001 From: roman Date: Sun, 5 Jul 2026 23:52:54 -0600 Subject: [PATCH 05/27] fix(state): don't panic the write task when a fork marker is above a truncated chain tip Invalidating a chain suffix (Zakura fork recovery) can leave last_fork_height above the new tip; recent_fork_length deliberately returns None for that, but the progress-bar metrics caller expected Some and panicked the block write task, crash-looping the node. Metrics display code must never panic. --- zebra-state/src/service/non_finalized_state.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/zebra-state/src/service/non_finalized_state.rs b/zebra-state/src/service/non_finalized_state.rs index 337cf01ff1c..2baff78c2be 100644 --- a/zebra-state/src/service/non_finalized_state.rs +++ b/zebra-state/src/service/non_finalized_state.rs @@ -1002,11 +1002,14 @@ impl NonFinalizedState { // For now, we don't show any work here, see the deleted code in PR #7087. let mut desc = String::new(); - if let Some(recent_fork_height) = chain.recent_fork_height() { - let recent_fork_length = chain - .recent_fork_length() - .expect("just checked recent fork height"); - + // Both are `None`-checked: invalidating a chain suffix can leave + // `last_fork_height` above the truncated tip, making the length + // `None` while the height is `Some`. This is metrics-only display + // code and must never panic the block write task (it crash-looped + // a node when the Zakura fork-recovery invalidation first hit it). + if let (Some(recent_fork_height), Some(recent_fork_length)) = + (chain.recent_fork_height(), chain.recent_fork_length()) + { let mut plural = "s"; if recent_fork_length == 1 { plural = ""; From 86895c61fccceddf48dad87c2ca1d5380f3d86fd Mon Sep 17 00:00:00 2001 From: roman Date: Mon, 6 Jul 2026 00:10:26 -0600 Subject: [PATCH 06/27] fix(zakura): reconcile a body suffix stranded across a crashed fork recovery If the node exits between a durable header reorg and its body invalidation, later header commits find the rows already replaced and report no conflict, so the event-based invalidation never fires again and the body chain stays stranded on the abandoned branch forever. Committed ranges overlapping the body tip are now compared against the best body chain, and the first stranded body block is invalidated (one extra state read per commit only when the range overlaps committed bodies). --- zebra-state/src/request.rs | 11 ++ zebra-state/src/service.rs | 4 + .../service/finalized_state/zebra_db/block.rs | 6 +- zebrad/src/commands/start.rs | 109 ++++++++++++++- .../start/zakura/header_sync_driver.rs | 124 ++++++++++++++++++ zebrad/src/commands/start/zakura/mod.rs | 4 +- 6 files changed, 250 insertions(+), 8 deletions(-) diff --git a/zebra-state/src/request.rs b/zebra-state/src/request.rs index b834aa935cd..8718db6075b 100644 --- a/zebra-state/src/request.rs +++ b/zebra-state/src/request.rs @@ -1568,6 +1568,16 @@ pub enum ReadRequest { /// * [`ReadResponse::BlockHash(None)`](ReadResponse::BlockHash) otherwise. BestChainBlockHash(block::Height), + /// Looks up the hash in the stored Zakura header chain at the given height. + /// + /// Unlike [`ReadRequest::BestChainBlockHash`], this never falls back to the + /// best body chain: it answers what the Zakura header store holds, so the + /// two chains can be compared to detect a body suffix stranded off the + /// header chain. + /// + /// Returns [`ReadResponse::BlockHash(Option)`](crate::ReadResponse::BlockHash). + ZakuraHeaderHash(block::Height), + /// Get state information from the best block chain. /// /// Returns [`ReadResponse::ChainInfo(info)`](ReadResponse::ChainInfo) where `info` is a @@ -1653,6 +1663,7 @@ impl ReadRequest { } ReadRequest::BestChainNextMedianTimePast => "best_chain_next_median_time_past", ReadRequest::BestChainBlockHash(_) => "best_chain_block_hash", + ReadRequest::ZakuraHeaderHash(_) => "zakura_header_hash", #[cfg(feature = "indexer")] ReadRequest::SpendingTransactionId(_) => "spending_transaction_id", ReadRequest::ChainInfo => "chain_info", diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs index 9b4b4279f84..e9f26409754 100644 --- a/zebra-state/src/service.rs +++ b/zebra-state/src/service.rs @@ -2037,6 +2037,10 @@ impl Service for ReadStateService { read::hash_by_height(state.latest_best_chain(), &state.db, height), )), + ReadRequest::ZakuraHeaderHash(height) => { + Ok(ReadResponse::BlockHash(state.db.zakura_header_hash(height))) + } + // Used by get_block_template and getblockchaininfo RPCs. ReadRequest::ChainInfo => { // # Correctness diff --git a/zebra-state/src/service/finalized_state/zebra_db/block.rs b/zebra-state/src/service/finalized_state/zebra_db/block.rs index 53fb8ae2c99..82b0a8770a5 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block.rs @@ -609,7 +609,11 @@ impl ZebraDb { } #[allow(clippy::unwrap_in_result)] - fn zakura_header_hash(&self, height: block::Height) -> Option { + /// Returns the hash stored in the Zakura header chain at `height`, if any. + /// + /// This reads the header store directly, without preferring the best body + /// chain like the serving reads do, so callers can compare the two chains. + pub fn zakura_header_hash(&self, height: block::Height) -> Option { let hash_by_height = self.db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); self.db.zs_get(&hash_by_height, &height) } diff --git a/zebrad/src/commands/start.rs b/zebrad/src/commands/start.rs index 8f378bcb8a3..a3a8241bbad 100644 --- a/zebrad/src/commands/start.rs +++ b/zebrad/src/commands/start.rs @@ -2123,11 +2123,11 @@ mod zakura_header_sync_driver_tests { coalesce_ready_needed_block_queries, coalesce_stale_needed_block_queries, commit_block_sync_body, drive_block_sync_actions, drive_zakura_header_sync_actions, header_range_commit_failure_kind, notify_block_sync_header_tip, query_block_sync_frontiers, - query_block_sync_needed_blocks, root_covered_query_best_header_tip, - tree_aux_roots_for_served_header_range, verified_block_tip_from_state, BlockApplyClass, - BlocksyncThroughputProbe, ZakuraHeaderSyncDriverHandles, - ZAKURA_BLOCK_SYNC_CHECKPOINT_FRONTIER_REFRESH_INTERVAL, ZAKURA_BLOCK_SYNC_DRIVER_TIMEOUT, - ZAKURA_BLOCK_SYNC_MISSING_BODY_WINDOW, + query_block_sync_needed_blocks, reconcile_stranded_body_suffix, + root_covered_query_best_header_tip, tree_aux_roots_for_served_header_range, + verified_block_tip_from_state, BlockApplyClass, BlocksyncThroughputProbe, + ZakuraHeaderSyncDriverHandles, ZAKURA_BLOCK_SYNC_CHECKPOINT_FRONTIER_REFRESH_INTERVAL, + ZAKURA_BLOCK_SYNC_DRIVER_TIMEOUT, ZAKURA_BLOCK_SYNC_MISSING_BODY_WINDOW, }; fn mainnet_block(bytes: &[u8]) -> Arc { @@ -2903,6 +2903,105 @@ mod zakura_header_sync_driver_tests { endpoint.shutdown().await; } + /// `reconcile_stranded_body_suffix` is a no-op when the body chain matches + /// the Zakura header store, and invalidates the first stranded body block + /// when the committed suffix sits on a branch the header chain reorged away + /// from (e.g. after a crash between a durable header reorg and its body + /// invalidation, when later commits report no header conflict). + #[tokio::test] + async fn stranded_body_suffix_is_reconciled_by_invalidation() { + use std::sync::atomic::{AtomicBool, Ordering}; + + let fork_height = block::Height(99); + let tip_height = block::Height(100); + let shared_hash = block::Hash([0x99; 32]); + let honest_hash_100 = block::Hash([0xAA; 32]); + let stranded_hash_100 = block::Hash([0xBB; 32]); + + // Case 1: the header store matches the body tip; no write happens. + let agree_read = service_fn(move |request: zebra_state::ReadRequest| async move { + match request { + zebra_state::ReadRequest::Tip => Ok::<_, zebra_state::BoxError>( + zebra_state::ReadResponse::Tip(Some((tip_height, honest_hash_100))), + ), + zebra_state::ReadRequest::ZakuraHeaderHash(height) if height == tip_height => { + Ok(zebra_state::ReadResponse::BlockHash(Some(honest_hash_100))) + } + request => panic!("unexpected read request: {request:?}"), + } + }); + let no_write = service_fn(|request: zebra_state::Request| async move { + panic!("agreeing chains must not trigger state writes: {request:?}"); + #[allow(unreachable_code)] + Ok::<_, zebra_state::BoxError>(zebra_state::Response::Committed(block::Hash([0; 32]))) + }); + reconcile_stranded_body_suffix( + no_write, + agree_read, + &zebra_network::zakura::ZakuraTrace::noop(), + ) + .await; + + // Case 2: the body tip diverges from the header store at 100 and the + // chains agree at 99, so the stranded body block at 100 is invalidated. + let stranded_read = service_fn(move |request: zebra_state::ReadRequest| async move { + match request { + zebra_state::ReadRequest::Tip => Ok::<_, zebra_state::BoxError>( + zebra_state::ReadResponse::Tip(Some((tip_height, stranded_hash_100))), + ), + zebra_state::ReadRequest::ZakuraHeaderHash(height) => { + let hash = if height == tip_height { + honest_hash_100 + } else if height == fork_height { + shared_hash + } else { + panic!("unexpected header height: {height:?}"); + }; + Ok(zebra_state::ReadResponse::BlockHash(Some(hash))) + } + zebra_state::ReadRequest::BestChainBlockHash(height) => { + let hash = if height == fork_height { + shared_hash + } else if height == tip_height { + stranded_hash_100 + } else { + panic!("unexpected body height: {height:?}"); + }; + Ok(zebra_state::ReadResponse::BlockHash(Some(hash))) + } + request => panic!("unexpected read request: {request:?}"), + } + }); + let invalidated = Arc::new(AtomicBool::new(false)); + let saw_invalidate = invalidated.clone(); + let write = service_fn(move |request: zebra_state::Request| { + let saw_invalidate = saw_invalidate.clone(); + async move { + match request { + zebra_state::Request::InvalidateBlock(hash) => { + assert_eq!( + hash, stranded_hash_100, + "the stranded body block at the first divergent height is invalidated" + ); + saw_invalidate.store(true, Ordering::SeqCst); + Ok::<_, zebra_state::BoxError>(zebra_state::Response::Invalidated(hash)) + } + request => panic!("unexpected state request: {request:?}"), + } + } + }); + reconcile_stranded_body_suffix( + write, + stranded_read, + &zebra_network::zakura::ZakuraTrace::noop(), + ) + .await; + assert!( + invalidated.load(Ordering::SeqCst), + "a stranded body suffix must be invalidated" + ); + } + #[tokio::test] async fn block_sync_driver_coalesces_stale_needed_queries() { let (action_tx, mut action_rx) = mpsc::channel(8); diff --git a/zebrad/src/commands/start/zakura/header_sync_driver.rs b/zebrad/src/commands/start/zakura/header_sync_driver.rs index 3f69c5f641a..10549fb247b 100644 --- a/zebrad/src/commands/start/zakura/header_sync_driver.rs +++ b/zebrad/src/commands/start/zakura/header_sync_driver.rs @@ -949,6 +949,12 @@ pub(crate) async fn drive_zakura_header_sync_actions { publish_header_frontier( @@ -1210,6 +1216,124 @@ where } } +/// Detects and repairs a body chain stranded off the stored Zakura header chain. +/// +/// Compares the best body chain against the Zakura header store at the body +/// tip. When they disagree, the committed body suffix is on a branch the +/// header chain reorged away from — a state that can outlive the event-based +/// invalidation in the commit path (for example when the node exits between a +/// durable header reorg and its body invalidation: later commits find the +/// header rows already replaced, report no conflict, and nothing else repairs +/// the bodies). Walks down to the fork point (bounded by the state's reorg +/// window) and invalidates the first stranded body block, resetting the chain +/// tip so block sync re-downloads the header chain's branch. +/// +/// Cheap when the chains agree (the normal case): two state reads. +pub(crate) async fn reconcile_stranded_body_suffix( + state: State, + read_state: ReadState, + trace: &ZakuraTrace, +) where + State: Service< + zebra_state::Request, + Response = zebra_state::Response, + Error = zebra_state::BoxError, + > + Clone + + Send + + 'static, + State::Future: Send + 'static, + ReadState: Service< + zebra_state::ReadRequest, + Response = zebra_state::ReadResponse, + Error = zebra_state::BoxError, + > + Clone + + Send + + 'static, + ReadState::Future: Send + 'static, +{ + let header_hash_at = |height: block::Height| { + let read_state = read_state.clone(); + async move { + match read_state + .oneshot(zebra_state::ReadRequest::ZakuraHeaderHash(height)) + .await + { + Ok(zebra_state::ReadResponse::BlockHash(hash)) => hash, + _ => None, + } + } + }; + let body_hash_at = |height: block::Height| { + let read_state = read_state.clone(); + async move { + match read_state + .oneshot(zebra_state::ReadRequest::BestChainBlockHash(height)) + .await + { + Ok(zebra_state::ReadResponse::BlockHash(hash)) => hash, + _ => None, + } + } + }; + + let body_tip = match read_state + .clone() + .oneshot(zebra_state::ReadRequest::Tip) + .await + { + Ok(zebra_state::ReadResponse::Tip(tip)) => tip, + _ => None, + }; + let Some((body_tip_height, body_tip_hash)) = body_tip else { + return; + }; + + // Fast path: no header row (header sync has not covered this height) or a + // matching one means the body suffix is on the header chain. + let Some(header_tip_hash) = header_hash_at(body_tip_height).await else { + return; + }; + if header_tip_hash == body_tip_hash { + return; + } + + // The body suffix is stranded. Walk down to the fork point; the state + // cannot reorg deeper than its reorg window, so bound the walk. + let mut stranded_at = body_tip_height; + let mut stranded_header_hash = header_tip_hash; + for _ in 0..zebra_state::MAX_BLOCK_REORG_HEIGHT { + let Ok(parent) = stranded_at.previous() else { + break; + }; + let (Some(body_hash), Some(header_hash)) = + (body_hash_at(parent).await, header_hash_at(parent).await) + else { + break; + }; + if body_hash == header_hash { + break; + } + stranded_at = parent; + stranded_header_hash = header_hash; + } + + metrics::counter!("sync.header.fork_recovery.body_suffix_reconciled").increment(1); + warn!( + ?body_tip_height, + ?body_tip_hash, + ?stranded_at, + "committed body suffix is stranded off the header chain; reconciling" + ); + invalidate_reorged_body_suffix( + state, + read_state, + stranded_at, + Some(stranded_header_hash), + trace, + ) + .await; +} + /// Drops the committed body suffix stranded by a header-chain reorg. /// /// `reorged_at` is the first height where a committed header range replaced a diff --git a/zebrad/src/commands/start/zakura/mod.rs b/zebrad/src/commands/start/zakura/mod.rs index bff8f791fd8..9c8397d7ffc 100644 --- a/zebrad/src/commands/start/zakura/mod.rs +++ b/zebrad/src/commands/start/zakura/mod.rs @@ -26,8 +26,8 @@ pub(crate) use frontier::{query_block_sync_frontiers, verified_block_tip_from_st pub(crate) use header_sync_driver::{ block_roots_cover_range, block_sync_chain_tip_event, body_sizes_for_served_header_range, chain_tip_mirror_frontier_change, header_range_commit_failure_kind, - notify_block_sync_header_tip, root_covered_query_best_header_tip, - tree_aux_roots_for_served_header_range, + notify_block_sync_header_tip, reconcile_stranded_body_suffix, + root_covered_query_best_header_tip, tree_aux_roots_for_served_header_range, }; pub(crate) use header_sync_driver::{ drive_zakura_header_sync_actions, mirror_zakura_full_block_commits, From 4d771657f68b9a9e44c0e6860ad8414a2915f617 Mon Sep 17 00:00:00 2001 From: roman Date: Mon, 6 Jul 2026 00:57:00 -0600 Subject: [PATCH 07/27] fix(zakura): drain the Zakura apply pipeline before legacy fallback drives commits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two sync engines submitting bulk commits concurrently race in the applying queue, so the fallback is now a commit barrier again — without killing the reactors: an apply gate stops new Zakura block applies once fallback engages and the watchdog waits for in-flight applies to finish (bounded; each apply has its own driver timeout) before legacy ChainSync takes the pipeline. Serving, statuses, header commits, and NewBlock accepts continue unaffected. --- zebrad/src/commands/start.rs | 25 ++++- .../start/zakura/block_sync_driver.rs | 54 ++++++++--- zebrad/src/commands/start/zakura/mod.rs | 95 +++++++++++++++++++ zebrad/src/components/sync.rs | 20 +++- zebrad/src/components/sync/tests/fallback.rs | 59 +++++++++--- 5 files changed, 223 insertions(+), 30 deletions(-) diff --git a/zebrad/src/commands/start.rs b/zebrad/src/commands/start.rs index a3a8241bbad..29925e33dd9 100644 --- a/zebrad/src/commands/start.rs +++ b/zebrad/src/commands/start.rs @@ -663,6 +663,10 @@ impl StartCmd { ) .await; + // Gates the Zakura bulk-apply pipeline so the legacy fallback can drain + // in-flight applies before driving commits through the same pipeline. + let zakura_apply_gate = zakura::ZakuraApplyGate::new(); + if let Some(endpoint) = zakura_endpoint.clone() { let trace = endpoint.trace(); if let (Some(header_sync), Some(shutdown), Some(actions)) = ( @@ -706,6 +710,7 @@ impl StartCmd { config.sync.zakura_block_apply_concurrency_limit, trace.clone(), blocksync_throughput_probe.clone(), + zakura_apply_gate.clone(), shutdown.clone().cancelled_owned(), ) .in_current_span(), @@ -986,7 +991,11 @@ impl StartCmd { let legacy_fallback = config.network.v2_p2p && config.network.legacy_p2p; tokio::spawn( syncer - .bootstrap_genesis_then_pause(read_only_state_service.clone(), legacy_fallback) + .bootstrap_genesis_then_pause( + read_only_state_service.clone(), + legacy_fallback, + zakura_apply_gate.clone(), + ) .in_current_span(), ) } else { @@ -3178,6 +3187,7 @@ mod zakura_header_sync_driver_tests { sync::DEFAULT_ZAKURA_BLOCK_APPLY_CONCURRENCY_LIMIT, zebra_network::zakura::ZakuraTrace::noop(), None, + super::zakura::ZakuraApplyGate::new(), async move { let _ = shutdown_rx.await; }, @@ -3320,6 +3330,7 @@ mod zakura_header_sync_driver_tests { sync::DEFAULT_ZAKURA_BLOCK_APPLY_CONCURRENCY_LIMIT, trace.clone(), Some(probe), + super::zakura::ZakuraApplyGate::new(), async move { let _ = shutdown_rx.await; }, @@ -3469,6 +3480,7 @@ mod zakura_header_sync_driver_tests { sync::DEFAULT_ZAKURA_BLOCK_APPLY_CONCURRENCY_LIMIT, zebra_network::zakura::ZakuraTrace::noop(), None, + super::zakura::ZakuraApplyGate::new(), async move { let _ = shutdown_rx.await; }, @@ -3582,6 +3594,7 @@ mod zakura_header_sync_driver_tests { sync::DEFAULT_ZAKURA_BLOCK_APPLY_CONCURRENCY_LIMIT, zebra_network::zakura::ZakuraTrace::noop(), None, + super::zakura::ZakuraApplyGate::new(), async move { let _ = shutdown_rx.await; }, @@ -3683,6 +3696,7 @@ mod zakura_header_sync_driver_tests { 1, zebra_network::zakura::ZakuraTrace::noop(), None, + super::zakura::ZakuraApplyGate::new(), async move { let _ = shutdown_rx.await; }, @@ -3785,6 +3799,7 @@ mod zakura_header_sync_driver_tests { 1, zebra_network::zakura::ZakuraTrace::noop(), None, + super::zakura::ZakuraApplyGate::new(), async move { let _ = shutdown_rx.await; }, @@ -3896,6 +3911,7 @@ mod zakura_header_sync_driver_tests { 1, zebra_network::zakura::ZakuraTrace::noop(), None, + super::zakura::ZakuraApplyGate::new(), async move { let _ = shutdown_rx.await; }, @@ -4297,6 +4313,7 @@ mod zakura_header_sync_driver_tests { sync::DEFAULT_ZAKURA_BLOCK_APPLY_CONCURRENCY_LIMIT, zebra_network::zakura::ZakuraTrace::noop(), None, + super::zakura::ZakuraApplyGate::new(), async move { let _ = shutdown_rx.await; }, @@ -4390,6 +4407,7 @@ mod zakura_header_sync_driver_tests { zebra_consensus::MAX_CHECKPOINT_HEIGHT_GAP, zebra_network::zakura::ZakuraTrace::noop(), None, + super::zakura::ZakuraApplyGate::new(), async move { let _ = shutdown_rx.await; }, @@ -4510,6 +4528,7 @@ mod zakura_header_sync_driver_tests { sync::DEFAULT_ZAKURA_BLOCK_APPLY_CONCURRENCY_LIMIT, zebra_network::zakura::ZakuraTrace::noop(), None, + super::zakura::ZakuraApplyGate::new(), async move { let _ = shutdown_rx.await; }, @@ -4633,6 +4652,7 @@ mod zakura_header_sync_driver_tests { sync::DEFAULT_ZAKURA_BLOCK_APPLY_CONCURRENCY_LIMIT, trace, None, + super::zakura::ZakuraApplyGate::new(), async move { let _ = shutdown_rx.await; }, @@ -4769,6 +4789,7 @@ mod zakura_header_sync_driver_tests { sync::DEFAULT_ZAKURA_BLOCK_APPLY_CONCURRENCY_LIMIT, zebra_network::zakura::ZakuraTrace::noop(), None, + super::zakura::ZakuraApplyGate::new(), async move { let _ = shutdown_rx.await; }, @@ -4898,6 +4919,7 @@ mod zakura_header_sync_driver_tests { sync::DEFAULT_ZAKURA_BLOCK_APPLY_CONCURRENCY_LIMIT, zebra_network::zakura::ZakuraTrace::noop(), None, + super::zakura::ZakuraApplyGate::new(), async move { let _ = shutdown_rx.await; }, @@ -5052,6 +5074,7 @@ mod zakura_header_sync_driver_tests { sync::DEFAULT_ZAKURA_BLOCK_APPLY_CONCURRENCY_LIMIT, zebra_network::zakura::ZakuraTrace::noop(), None, + super::zakura::ZakuraApplyGate::new(), async move { let _ = shutdown_rx.await; }, diff --git a/zebrad/src/commands/start/zakura/block_sync_driver.rs b/zebrad/src/commands/start/zakura/block_sync_driver.rs index c20b2c5f7e4..6928ce3b70f 100644 --- a/zebrad/src/commands/start/zakura/block_sync_driver.rs +++ b/zebrad/src/commands/start/zakura/block_sync_driver.rs @@ -111,6 +111,7 @@ pub(crate) async fn drive_block_sync_actions( combined_apply_limit: usize, trace: ZakuraTrace, throughput_probe: Option, + apply_gate: std::sync::Arc, shutdown: impl Future + Send + 'static, ) where ReadState: Service< @@ -159,6 +160,7 @@ pub(crate) async fn drive_block_sync_actions( if shutting_down { if let Some(completed) = in_flight_applies.next().await { handle_completed_block_apply( + &apply_gate, completed, &mut pending_applies, &mut in_flight_applies, @@ -185,6 +187,7 @@ pub(crate) async fn drive_block_sync_actions( if !in_flight_applies.is_empty() { if let Some(Some(completed)) = in_flight_applies.next().now_or_never() { handle_completed_block_apply( + &apply_gate, completed, &mut pending_applies, &mut in_flight_applies, @@ -226,6 +229,7 @@ pub(crate) async fn drive_block_sync_actions( continue; }; handle_completed_block_apply( + &apply_gate, completed, &mut pending_applies, &mut in_flight_applies, @@ -535,6 +539,7 @@ pub(crate) async fn drive_block_sync_actions( block, }); drain_pending_block_applies( + &apply_gate, &mut pending_applies, &mut in_flight_applies, &mut checkpoint_in_flight, @@ -651,6 +656,7 @@ pub(crate) fn coalesce_stale_needed_block_queries( #[allow(clippy::too_many_arguments)] fn handle_completed_block_apply( + apply_gate: &std::sync::Arc, completed: BlockApplyCompletion, pending_applies: &mut VecDeque, in_flight_applies: &mut FuturesUnordered>, @@ -685,6 +691,7 @@ fn handle_completed_block_apply( observe_block_apply_completion(completed, checkpoint_frontier_refresh); drain_pending_block_applies( + apply_gate, pending_applies, in_flight_applies, checkpoint_in_flight, @@ -704,6 +711,7 @@ fn handle_completed_block_apply( #[allow(clippy::too_many_arguments)] fn drain_pending_block_applies( + apply_gate: &std::sync::Arc, pending_applies: &mut VecDeque, in_flight_applies: &mut FuturesUnordered>, checkpoint_in_flight: &mut usize, @@ -732,6 +740,14 @@ fn drain_pending_block_applies( BlockVerifier::Error: std::fmt::Debug + Send + Sync + 'static, BlockVerifier::Future: Send + 'static, { + // Once the pipeline has yielded to legacy ChainSync, start no new applies: + // two engines driving bulk commits concurrently race in the applying + // queue. Queued blocks stay pending; legacy commits cover their heights + // and the sequencer releases them as its frontier advances. + if apply_gate.is_yielded() { + return; + } + // The checkpoint verifier can hold a complete range until its checkpoint is // reached. Keep room for the current range and the next complete range. let checkpoint_pipeline_apply_limit = checkpoint_apply_limit.saturating_mul(2); @@ -763,19 +779,33 @@ fn drain_pending_block_applies( } let class = pending.class; + let Some(permit) = apply_gate.begin_apply() else { + // Yielded while draining: put the block back and stop starting + // applies. Its in-flight accounting was already incremented above, + // so hand it back too. + decrement_in_flight_apply_count(class, checkpoint_in_flight, full_in_flight); + pending_applies.push_front(pending); + return; + }; + let apply = apply_block_sync_body( + block_verifier.clone(), + latest_chain_tip.clone(), + endpoint.clone(), + read_state.clone(), + block_sync.clone(), + pending.token, + pending.block, + class, + trace.clone(), + throughput_probe.clone(), + ); in_flight_applies.push( - apply_block_sync_body( - block_verifier.clone(), - latest_chain_tip.clone(), - endpoint.clone(), - read_state.clone(), - block_sync.clone(), - pending.token, - pending.block, - class, - trace.clone(), - throughput_probe.clone(), - ) + async move { + // Holds the apply-gate slot for the whole apply, so the legacy + // fallback's drain barrier sees it until completion. + let _permit = permit; + apply.await + } .boxed(), ); } diff --git a/zebrad/src/commands/start/zakura/mod.rs b/zebrad/src/commands/start/zakura/mod.rs index 9c8397d7ffc..11acfefd6a0 100644 --- a/zebrad/src/commands/start/zakura/mod.rs +++ b/zebrad/src/commands/start/zakura/mod.rs @@ -7,6 +7,101 @@ use zebra_network::zakura::{ ZakuraTrace, COMMIT_STATE_TABLE, }; +/// Gates the Zakura block-sync bulk-apply pipeline so the legacy `ChainSync` +/// fallback can drain it before driving commits through the same verifier and +/// state pipeline. +/// +/// Two sync engines submitting bulk commits concurrently race in the applying +/// queue (ordering races can poison the parent-error map and trip the +/// same-hash commit lockout), so the fallback must be a commit barrier: once +/// yielded, the block-sync driver starts no new applies, and the watchdog +/// waits for in-flight applies to finish before resuming legacy sync. The +/// Zakura reactors stay alive throughout — serving, statuses, and header +/// commits are unaffected; only bulk body applies are gated. +#[derive(Debug)] +pub(crate) struct ZakuraApplyGate { + yielded: std::sync::atomic::AtomicBool, + in_flight: std::sync::atomic::AtomicUsize, + drained: tokio::sync::Notify, +} + +/// Tracks one in-flight Zakura block apply; dropping it releases the slot and +/// wakes a pending [`ZakuraApplyGate::yield_and_drain`]. +#[derive(Debug)] +pub(crate) struct ZakuraApplyPermit(std::sync::Arc); + +impl ZakuraApplyGate { + pub(crate) fn new() -> std::sync::Arc { + std::sync::Arc::new(Self { + yielded: std::sync::atomic::AtomicBool::new(false), + in_flight: std::sync::atomic::AtomicUsize::new(0), + drained: tokio::sync::Notify::new(), + }) + } + + /// Whether the pipeline has been yielded to legacy sync. + pub(crate) fn is_yielded(&self) -> bool { + self.yielded.load(std::sync::atomic::Ordering::SeqCst) + } + + /// Returns a permit for one block apply, or `None` once the pipeline has + /// been yielded to legacy sync. + pub(crate) fn begin_apply(self: &std::sync::Arc) -> Option { + if self.is_yielded() { + return None; + } + self.in_flight + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + // Re-check after reserving so a yield that lands between the check and + // the increment still sees an accurate in-flight count. + if self.is_yielded() { + self.release(); + return None; + } + Some(ZakuraApplyPermit(self.clone())) + } + + fn release(&self) { + if self + .in_flight + .fetch_sub(1, std::sync::atomic::Ordering::SeqCst) + == 1 + { + self.drained.notify_waiters(); + } + } + + /// Yields the apply pipeline to legacy sync and waits until in-flight + /// applies drain, bounded by `timeout` (each apply already has its own + /// driver timeout, so the bound is a backstop, not the primary limit). + pub(crate) async fn yield_and_drain(&self, timeout: Duration) { + self.yielded + .store(true, std::sync::atomic::Ordering::SeqCst); + let deadline = tokio::time::Instant::now() + timeout; + loop { + let drained = self.drained.notified(); + let in_flight = self.in_flight.load(std::sync::atomic::Ordering::SeqCst); + if in_flight == 0 { + return; + } + if tokio::time::timeout_at(deadline, drained).await.is_err() { + tracing::warn!( + in_flight, + "timed out draining Zakura block applies before legacy fallback; \ + remaining applies resolve through their own driver timeouts" + ); + return; + } + } + } +} + +impl Drop for ZakuraApplyPermit { + fn drop(&mut self) { + self.0.release(); + } +} + pub(crate) mod block_sync_driver; pub(crate) mod frontier; pub(crate) mod header_sync_driver; diff --git a/zebrad/src/components/sync.rs b/zebrad/src/components/sync.rs index d24e8582cd1..d6326c1fad5 100644 --- a/zebrad/src/components/sync.rs +++ b/zebrad/src/components/sync.rs @@ -530,9 +530,18 @@ where /// following local commits through the chain-tip mirror — the same way /// gossip-driven commits already coexist with the legacy syncer. Duplicate /// commits from the overlap window are rejected by the verifier as usual. -fn engage_legacy_fallback_alongside_zakura() { +async fn engage_legacy_fallback_alongside_zakura( + apply_gate: &crate::commands::start::zakura::ZakuraApplyGate, +) { metrics::counter!("sync.zakura.legacy_fallback.engaged").increment(1); metrics::gauge!("sync.zakura.legacy_fallback.active").set(1.0); + // Commit barrier: two engines driving bulk commits concurrently race in + // the applying queue, so stop new Zakura applies and drain the in-flight + // ones before legacy ChainSync takes the pipeline. Each apply has its own + // driver timeout, so the drain bound is a backstop. + apply_gate + .yield_and_drain(std::time::Duration::from_secs(60)) + .await; } /// Sync configuration section. @@ -968,11 +977,12 @@ where /// for the legacy-informed cross-check, which only probes legacy peers when /// the verified tip is frozen and the node looks caught up to its own header /// frontier. - #[instrument(skip(self, read_state))] - pub async fn bootstrap_genesis_then_pause( + #[instrument(skip(self, read_state, apply_gate))] + pub(crate) async fn bootstrap_genesis_then_pause( mut self, mut read_state: RS, legacy_fallback: bool, + apply_gate: std::sync::Arc, ) -> Result<(), Report> where RS: Service @@ -1033,7 +1043,7 @@ where legacy ChainSync as the body-sync driver while Zakura keeps serving \ peers and following local commits" ); - engage_legacy_fallback_alongside_zakura(); + engage_legacy_fallback_alongside_zakura(&apply_gate).await; return self.sync().await; } ZakuraWatchdogAction::ProbeLegacyPeers => { @@ -1052,7 +1062,7 @@ where higher tip; resuming legacy ChainSync as the body-sync driver \ while Zakura keeps serving peers and following local commits" ); - engage_legacy_fallback_alongside_zakura(); + engage_legacy_fallback_alongside_zakura(&apply_gate).await; return self.sync().await; } } diff --git a/zebrad/src/components/sync/tests/fallback.rs b/zebrad/src/components/sync/tests/fallback.rs index b885e14655a..9132a97d608 100644 --- a/zebrad/src/components/sync/tests/fallback.rs +++ b/zebrad/src/components/sync/tests/fallback.rs @@ -241,9 +241,18 @@ fn stalled_zakura_with_legacy_fallback_keeps_zakura_reactors_alive() { ZakuraWatchdogAction::FallbackToLegacy, "a frozen verified tip must trigger legacy fallback when it is enabled" ); - // The hand-off is now driver-only: nothing in the fallback path may cancel - // Zakura work, so there is no shutdown token to assert on. - super::super::engage_legacy_fallback_alongside_zakura(); + // The hand-off drains the apply gate but cancels no Zakura work: with no + // in-flight applies this returns immediately, and the gate ends yielded. + let gate = crate::commands::start::zakura::ZakuraApplyGate::new(); + futures::executor::block_on(super::super::engage_legacy_fallback_alongside_zakura(&gate)); + assert!( + gate.is_yielded(), + "fallback must yield the Zakura apply gate" + ); + assert!( + gate.begin_apply().is_none(), + "no new Zakura applies may start after the fallback engages" + ); } #[test] @@ -311,7 +320,12 @@ fn frozen_zero_gap_with_legacy_peers_ahead_engages_fallback() { "legacy peers at or above the behind threshold must trigger fallback" ); - super::super::engage_legacy_fallback_alongside_zakura(); + let gate = crate::commands::start::zakura::ZakuraApplyGate::new(); + futures::executor::block_on(super::super::engage_legacy_fallback_alongside_zakura(&gate)); + assert!( + gate.is_yielded(), + "fallback must yield the Zakura apply gate" + ); } #[test] @@ -395,12 +409,33 @@ fn zakura_sync_status_lengths_drive_existing_mempool_gate() { } /// Locks in the fallback behavior: engaging legacy fallback must not signal any -/// Zakura shutdown — the reactors stay alive as a serving bridge. This test -/// documents the intentional inversion of the old "fallback cancels the Zakura -/// shutdown token" contract. -#[test] -fn fallback_does_not_cancel_any_zakura_shutdown_token() { - // The hand-off helper takes no token and cancels nothing; it only records - // the mode switch. Must not panic. - super::super::engage_legacy_fallback_alongside_zakura(); +/// Zakura shutdown — the reactors stay alive as a serving bridge — but it must +/// be a commit barrier: in-flight Zakura applies drain before it returns, and +/// no new applies can start afterwards. This documents the intentional +/// inversion of the old "fallback cancels the Zakura shutdown token" contract. +#[tokio::test(start_paused = true)] +async fn fallback_drains_the_apply_gate_without_cancelling_zakura() { + let gate = crate::commands::start::zakura::ZakuraApplyGate::new(); + let permit = gate.begin_apply().expect("applies run before the fallback"); + + let drain_gate = gate.clone(); + let drain = tokio::spawn(async move { + super::super::engage_legacy_fallback_alongside_zakura(&drain_gate).await + }); + + // The barrier must wait for the in-flight apply... + tokio::task::yield_now().await; + assert!( + !drain.is_finished(), + "the drain waits for in-flight applies" + ); + assert!( + gate.begin_apply().is_none(), + "no new Zakura applies may start once the fallback begins" + ); + + // ...and complete as soon as it finishes. + drop(permit); + drain.await.expect("drain task completes"); + assert!(gate.is_yielded()); } From 1fa3595e212d078a37ac8a6be089e7e3665be95e Mon Sep 17 00:00:00 2001 From: roman Date: Mon, 6 Jul 2026 01:51:50 -0600 Subject: [PATCH 08/27] feat(zakura): re-promote Zakura to body-sync driver after legacy catch-up Fallback was permanent per process lifetime: bootstrap_genesis_then_pause exited into the legacy sync loop and nothing ever handed the driver role back, so every min-difficulty burst demoted one more node's Zakura sync to a serve-only bridge until restart. The watchdog is now a cycle: on fallback it yields+drains the apply gate, legacy drives until caught up and stable (three consecutive exhausted sync rounds advancing the tip by at most two blocks), then the gate is returned to Zakura and the watchdog resumes. Hand-back happens between legacy rounds, so no legacy bulk work is in flight when Zakura resumes driving. --- CHANGELOG.md | 7 + zebrad/src/commands/start/zakura/mod.rs | 10 + zebrad/src/components/sync.rs | 213 ++++++++++++++----- zebrad/src/components/sync/tests/fallback.rs | 47 ++++ 4 files changed, 221 insertions(+), 56 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e99337d58b..5ada0ce846e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,13 @@ and this project adheres to [Semantic Versioning](https://semver.org). - Fixed a near-tip sync restart loop when a timed-out `AwaitUtxo` lookup in the transaction verifier was converted to `InternalDowncastError` instead of a missing transparent input. +- After a legacy fallback, the dual-stack watchdog now re-promotes Zakura to + the body-sync driver once legacy `ChainSync` is caught up and stable (three + consecutive exhausted sync rounds with the tip advancing by at most two + blocks). Fallback was previously permanent per process lifetime, so every + min-difficulty burst demoted one more node's Zakura sync to a serve-only + bridge until restart. The hand-back reuses the apply-gate commit barrier in + reverse and is counted in `sync.zakura.repromoted`. - The dual-stack Zakura stall watchdog no longer shuts down the Zakura header- and block-sync reactors when it falls back to legacy `ChainSync`. Killing them turned every fallback into a fleet-wide Zakura outage: the fallback diff --git a/zebrad/src/commands/start/zakura/mod.rs b/zebrad/src/commands/start/zakura/mod.rs index 11acfefd6a0..3aba77053f6 100644 --- a/zebrad/src/commands/start/zakura/mod.rs +++ b/zebrad/src/commands/start/zakura/mod.rs @@ -71,6 +71,16 @@ impl ZakuraApplyGate { } } + /// Returns the apply pipeline to Zakura after a legacy catch-up completes. + /// + /// Safe without a drain in this direction: the caller only re-promotes + /// after the legacy sync loop has returned between rounds, so no legacy + /// bulk work is in flight when Zakura resumes driving. + pub(crate) fn unyield(&self) { + self.yielded + .store(false, std::sync::atomic::Ordering::SeqCst); + } + /// Yields the apply pipeline to legacy sync and waits until in-flight /// applies drain, bounded by `timeout` (each apply already has its own /// driver timeout, so the bound is a backstop, not the primary limit). diff --git a/zebrad/src/components/sync.rs b/zebrad/src/components/sync.rs index d6326c1fad5..fa9a1a79304 100644 --- a/zebrad/src/components/sync.rs +++ b/zebrad/src/components/sync.rs @@ -530,6 +530,53 @@ where /// following local commits through the chain-tip mirror — the same way /// gossip-driven commits already coexist with the legacy syncer. Duplicate /// commits from the overlap window are rejected by the verifier as usual. +/// Consecutive stable legacy sync rounds required before Zakura is +/// re-promoted to the driver role after a fallback. +const ZAKURA_REPROMOTE_STABLE_ROUNDS: usize = 3; + +/// Maximum state-tip advance (in blocks) a legacy sync round may make and +/// still count as "stable" for re-promotion: at-tip rounds only pick up the +/// occasional gossip-adjacent block, while catch-up rounds move hundreds. +const ZAKURA_REPROMOTE_MAX_ROUND_PROGRESS: u32 = 2; + +/// Counts consecutive stable legacy sync rounds for the re-promotion decision. +#[derive(Debug, Default)] +struct RepromotionTracker { + stable_rounds: usize, +} + +impl RepromotionTracker { + /// Records one legacy sync round; returns `true` when enough consecutive + /// stable rounds have accumulated to re-promote Zakura. + fn record_round( + &mut self, + exhausted: bool, + before: Option, + after: Option, + ) -> bool { + let progressed = match (before, after) { + (Some(before), Some(after)) => after.0.saturating_sub(before.0), + // Unknown tips cannot prove stability. + _ => u32::MAX, + }; + if exhausted && progressed <= ZAKURA_REPROMOTE_MAX_ROUND_PROGRESS { + self.stable_rounds += 1; + } else { + self.stable_rounds = 0; + } + self.stable_rounds >= ZAKURA_REPROMOTE_STABLE_ROUNDS + } +} + +/// Returns the bulk-apply pipeline to Zakura after a completed legacy +/// catch-up, and records the mode switch. +fn repromote_zakura(apply_gate: &crate::commands::start::zakura::ZakuraApplyGate) { + apply_gate.unyield(); + metrics::counter!("sync.zakura.repromoted").increment(1); + metrics::gauge!("sync.zakura.legacy_fallback.active").set(0.0); + warn!("re-promoted Zakura sync to the body-sync driver after legacy catch-up"); +} + async fn engage_legacy_fallback_alongside_zakura( apply_gate: &crate::commands::start::zakura::ZakuraApplyGate, ) { @@ -1002,74 +1049,128 @@ where / ZAKURA_BODY_SYNC_STALL_POLL.as_secs()) .max(1); - let initial_tip = self.latest_chain_tip.best_tip_height(); - let mut tracker = ZakuraStallTracker::new(initial_tip); - let mut legacy_probe = ZakuraLegacyProbe::new(initial_tip); - loop { - sleep(ZAKURA_BODY_SYNC_STALL_POLL).await; - - let verified_height = self.latest_chain_tip.best_tip_height(); - let header_tip_height = best_header_tip_height(&mut read_state).await; - if let Some(sync_length) = zakura_sync_status_length(verified_height, header_tip_height) - { - self.recent_syncs.push_extend_tips_length(sync_length); - } - - match zakura_watchdog_action( - &mut tracker, - &mut legacy_probe, - verified_height, - header_tip_height, - max_idle_polls, - legacy_fallback, - ) { - ZakuraWatchdogAction::ContinueWaiting => continue, - ZakuraWatchdogAction::WarnOnly => { - warn!( - verified_tip = ?verified_height, - header_tip = ?header_tip_height, - stall = ?ZAKURA_BODY_SYNC_STALL_TIMEOUT, - "Zakura body sync is not closing the gap to the network tip; legacy \ - fallback disabled (legacy_p2p is off), continuing to wait for Zakura" - ); - continue; - } - ZakuraWatchdogAction::FallbackToLegacy => { - warn!( - verified_tip = ?verified_height, - header_tip = ?header_tip_height, - stall = ?ZAKURA_BODY_SYNC_STALL_TIMEOUT, - "Zakura body sync is not closing the gap to the network tip; resuming \ - legacy ChainSync as the body-sync driver while Zakura keeps serving \ - peers and following local commits" - ); - engage_legacy_fallback_alongside_zakura(&apply_gate).await; - return self.sync().await; + // Fallback is a cycle, not a ratchet: when Zakura stalls, legacy + // ChainSync drives until the node is caught up and stable, then the + // apply gate is returned to Zakura and the watchdog resumes. Without + // re-promotion, every burst permanently demoted one more node's Zakura + // sync to a serve-only bridge. + 'drive_cycle: loop { + let initial_tip = self.latest_chain_tip.best_tip_height(); + let mut tracker = ZakuraStallTracker::new(initial_tip); + let mut legacy_probe = ZakuraLegacyProbe::new(initial_tip); + loop { + sleep(ZAKURA_BODY_SYNC_STALL_POLL).await; + + let verified_height = self.latest_chain_tip.best_tip_height(); + let header_tip_height = best_header_tip_height(&mut read_state).await; + if let Some(sync_length) = + zakura_sync_status_length(verified_height, header_tip_height) + { + self.recent_syncs.push_extend_tips_length(sync_length); } - ZakuraWatchdogAction::ProbeLegacyPeers => { - let blocks_ahead = self.legacy_peers_blocks_ahead().await; - info!( - verified_tip = ?verified_height, - ?blocks_ahead, - "Zakura watchdog probed legacy peers for connected blocks ahead" - ); - if legacy_probe_supports_fallback(blocks_ahead) { + + match zakura_watchdog_action( + &mut tracker, + &mut legacy_probe, + verified_height, + header_tip_height, + max_idle_polls, + legacy_fallback, + ) { + ZakuraWatchdogAction::ContinueWaiting => continue, + ZakuraWatchdogAction::WarnOnly => { warn!( verified_tip = ?verified_height, header_tip = ?header_tip_height, - ?blocks_ahead, - "Zakura body sync is frozen while legacy peers advertise a much \ - higher tip; resuming legacy ChainSync as the body-sync driver \ - while Zakura keeps serving peers and following local commits" + stall = ?ZAKURA_BODY_SYNC_STALL_TIMEOUT, + "Zakura body sync is not closing the gap to the network tip; legacy \ + fallback disabled (legacy_p2p is off), continuing to wait for Zakura" + ); + continue; + } + ZakuraWatchdogAction::FallbackToLegacy => { + warn!( + verified_tip = ?verified_height, + header_tip = ?header_tip_height, + stall = ?ZAKURA_BODY_SYNC_STALL_TIMEOUT, + "Zakura body sync is not closing the gap to the network tip; resuming \ + legacy ChainSync as the body-sync driver while Zakura keeps serving \ + peers and following local commits" ); engage_legacy_fallback_alongside_zakura(&apply_gate).await; - return self.sync().await; + self.sync_until_caught_up_and_stable().await?; + repromote_zakura(&apply_gate); + continue 'drive_cycle; + } + ZakuraWatchdogAction::ProbeLegacyPeers => { + let blocks_ahead = self.legacy_peers_blocks_ahead().await; + info!( + verified_tip = ?verified_height, + ?blocks_ahead, + "Zakura watchdog probed legacy peers for connected blocks ahead" + ); + if legacy_probe_supports_fallback(blocks_ahead) { + warn!( + verified_tip = ?verified_height, + header_tip = ?header_tip_height, + ?blocks_ahead, + "Zakura body sync is frozen while legacy peers advertise a much \ + higher tip; resuming legacy ChainSync as the body-sync driver \ + while Zakura keeps serving peers and following local commits" + ); + engage_legacy_fallback_alongside_zakura(&apply_gate).await; + self.sync_until_caught_up_and_stable().await?; + repromote_zakura(&apply_gate); + continue 'drive_cycle; + } } } } } } + /// Drives legacy sync rounds until the node is caught up and the tip is + /// stable, then returns so Zakura can be re-promoted to the driver role. + /// + /// "Caught up and stable" means [`ZAKURA_REPROMOTE_STABLE_ROUNDS`] + /// consecutive sync rounds each exhausted their prospective tip set while + /// advancing the state tip by at most + /// [`ZAKURA_REPROMOTE_MAX_ROUND_PROGRESS`] blocks — legacy is no longer + /// adding anything gossip would not. Returns between rounds, so no legacy + /// bulk downloads are in flight when the caller re-promotes Zakura. + async fn sync_until_caught_up_and_stable(&mut self) -> Result<(), Report> { + let mut tracker = RepromotionTracker::default(); + loop { + let before = self.latest_chain_tip.best_tip_height(); + let round = self.try_to_sync().await; + if round.is_err() { + self.downloads.cancel_all(); + } + let after = self.latest_chain_tip.best_tip_height(); + + if tracker.record_round(round.is_ok(), before, after) { + info!( + state_tip = ?after, + "legacy catch-up is complete and stable; re-promoting Zakura sync" + ); + return Ok(()); + } + + self.update_metrics(); + let restart_delay = if self.is_regtest { + REGTEST_SYNC_RESTART_DELAY + } else { + SYNC_RESTART_SLEEP + }; + info!( + timeout = ?restart_delay, + state_tip = ?after, + "waiting to restart sync" + ); + sleep(restart_delay).await; + } + } + /// Probes the legacy peer set for how far ahead the network is on **our** /// chain, returning the longest run of peer-offered headers that extends a /// block we already have (`None` if no peer answered). diff --git a/zebrad/src/components/sync/tests/fallback.rs b/zebrad/src/components/sync/tests/fallback.rs index 9132a97d608..15ee7f8457f 100644 --- a/zebrad/src/components/sync/tests/fallback.rs +++ b/zebrad/src/components/sync/tests/fallback.rs @@ -439,3 +439,50 @@ async fn fallback_drains_the_apply_gate_without_cancelling_zakura() { drain.await.expect("drain task completes"); assert!(gate.is_yielded()); } + +/// Re-promotion requires consecutive stable rounds: catch-up rounds (large +/// tip advances) and failed rounds reset the count; unknown tips never count. +#[test] +fn repromotion_requires_consecutive_stable_rounds() { + use super::super::RepromotionTracker; + + let mut tracker = RepromotionTracker::default(); + + // Catch-up rounds: big progress, never re-promotes. + assert!(!tracker.record_round(true, Some(Height(0)), Some(Height(500)))); + assert!(!tracker.record_round(true, Some(Height(500)), Some(Height(900)))); + + // Two stable rounds are not enough... + assert!(!tracker.record_round(true, Some(Height(900)), Some(Height(901)))); + assert!(!tracker.record_round(true, Some(Height(901)), Some(Height(901)))); + // ...a failed round resets... + assert!(!tracker.record_round(false, Some(Height(901)), Some(Height(901)))); + assert!(!tracker.record_round(true, Some(Height(901)), Some(Height(902)))); + assert!(!tracker.record_round(true, Some(Height(902)), Some(Height(902)))); + // ...and the third consecutive stable round re-promotes. + assert!(tracker.record_round(true, Some(Height(902)), Some(Height(902)))); + + // Unknown tips cannot prove stability. + let mut tracker = RepromotionTracker::default(); + assert!(!tracker.record_round(true, None, Some(Height(1)))); + assert!(!tracker.record_round(true, Some(Height(1)), None)); +} + +/// The apply gate round-trips: yielded blocks new applies, unyield restores them. +#[tokio::test] +async fn apply_gate_unyield_restores_zakura_applies() { + let gate = crate::commands::start::zakura::ZakuraApplyGate::new(); + assert!(gate.begin_apply().is_some()); + + gate.yield_and_drain(std::time::Duration::from_secs(1)) + .await; + assert!(gate.is_yielded()); + assert!(gate.begin_apply().is_none()); + + gate.unyield(); + assert!(!gate.is_yielded()); + assert!( + gate.begin_apply().is_some(), + "re-promotion must let Zakura applies start again" + ); +} From a433abe623912261425865bf83d93df4a8f11be3 Mon Sep 17 00:00:00 2001 From: roman Date: Mon, 6 Jul 2026 02:28:16 -0600 Subject: [PATCH 09/27] fix(zakura): stop floor-GC from leaving stale block-sync liveness deadlines MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the download floor passed a request (other peers delivered its heights), the GC removed it but only disarmed the liveness deadline for peers whose last delivery was newer than their last request. A healthy peer with an older delivery kept a deadline it could no longer answer, was disconnected for 'no accepted block progress' with zero outstanding, and parked in the 180s no-progress cooldown — at tip this serially exiled peers until the block-sync peer set collapsed (reproduced live 5→0 after a fleet-wide restart, wedging body sync at the next gap). Heights satisfied below the floor now always clear an idle peer's deadline; unresponsive peers are still caught by request timeouts, which deliberately keep the deadline armed. --- CHANGELOG.md | 14 ++ .../src/zakura/block_sync/peer_routine.rs | 132 +++++++++++++++++- 2 files changed, 143 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ada0ce846e..fcd12ba349a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,20 @@ and this project adheres to [Semantic Versioning](https://semver.org). - Fixed a near-tip sync restart loop when a timed-out `AwaitUtxo` lookup in the transaction verifier was converted to `InternalDowncastError` instead of a missing transparent input. +- Fixed Zakura block-sync peers being serially disconnected and parked at the + chain tip by a stale liveness deadline. When the download floor passed a + request because other peers' deliveries satisfied its heights, the floor GC + removed the request but only cleared the liveness deadline for peers whose + last delivery was newer than their last request — a healthy peer with an + older delivery kept a deadline it could no longer answer, was disconnected + for "no accepted block progress" with zero outstanding requests, and parked + in the 180-second no-progress cooldown. At tip this exiled peers one by one + until the block-sync peer set collapsed (reproduced live: 5 → 0 peers within + ~25 minutes of a fleet-wide restart, wedging body sync when the next gap + formed). Heights satisfied below the floor now always clear an idle peer's + deadline; genuinely unresponsive peers are still caught by request timeouts, + which deliberately keep the deadline armed. The liveness disconnect log was + also promoted to info. - After a legacy fallback, the dual-stack watchdog now re-promotes Zakura to the body-sync driver once legacy `ChainSync` is caught up and stable (three consecutive exhausted sync rounds with the tip advancing by at most two diff --git a/zebra-network/src/zakura/block_sync/peer_routine.rs b/zebra-network/src/zakura/block_sync/peer_routine.rs index 3625d439246..0b2d44bb229 100644 --- a/zebra-network/src/zakura/block_sync/peer_routine.rs +++ b/zebra-network/src/zakura/block_sync/peer_routine.rs @@ -1141,7 +1141,7 @@ impl PeerRoutine { now + self.config.effective_no_progress_peer_cooldown(), ); self.trace_protocol_reject_liveness(error); - tracing::debug!( + tracing::info!( peer = ?self.peer, outstanding = self.window.outstanding.len(), "disconnecting Zakura block-sync peer after no accepted block progress" @@ -1185,7 +1185,18 @@ impl PeerRoutine { } if removed { self.publish_outstanding(); - self.window.disarm_liveness_after_progress_if_idle(); + // Heights the floor passed were satisfied by the network; this peer + // no longer owes them, so a liveness deadline armed by those + // requests must not survive the drain. The after-progress disarm is + // not enough: a peer that delivered earlier and then had a NEWER + // request GC'd here has `request_at > block_at`, kept its stale + // deadline, and was disconnected at tip with zero outstanding — + // serially exiling healthy peers into the no-progress cooldown + // until the block-sync peer set collapsed. Dead-session detection + // is unaffected: an unanswered request expires through + // `expire_due_timeouts`, which deliberately leaves the deadline + // armed. + self.window.clear_liveness_if_idle(); } } @@ -2324,8 +2335,11 @@ mod tests { use super::super::peer_registry::PeerRegistry; use super::super::request::BlockSizeEstimate; + use super::super::request::{BlockRangeRequest, ExpectedBlock}; use super::super::sequencer_task::{initial_view, SequencerControlInput}; - use super::super::state::{ByteBudget, ThroughputMeter}; + use super::super::state::{ + ByteBudget, LivenessOutcome, OutstandingBlockRange, ReceivedBlockTracker, ThroughputMeter, + }; use super::super::work_queue::WorkQueue; use super::super::{BlockSyncFrontiers, BlockSyncPeerSession, ZakuraBlockSyncConfig}; use super::PeerRoutine; @@ -2591,4 +2605,116 @@ mod tests { request_timeout )); } + + /// Floor-GC of a request satisfied by other peers must clear the liveness + /// deadline: the peer no longer owes those heights. Before the fix, a peer + /// that had delivered earlier and then had a NEWER request GC'd below the + /// floor kept a stale deadline (`request_at > block_at` fails the + /// after-progress disarm) and was disconnected at tip with zero + /// outstanding, then parked in the 180 s no-progress cooldown — serially + /// exiling healthy peers until the block-sync peer set collapsed + /// (reproduced live: 5 → 0 peers after a fleet-wide restart). + #[tokio::test] + async fn floor_gc_clears_stale_liveness_deadline() { + let config = ZakuraBlockSyncConfig::default(); + let budget = ByteBudget::new(1_000_000); + let work = Arc::new(WorkQueue::new(block::Height(0))); + + let cancel = CancellationToken::new(); + let (out_send, _out_recv) = framed_channel(16); + let (_in_send, in_recv) = framed_channel(16); + let peer = ZakuraPeerId::new(vec![8u8; 32]).expect("test peer id is within bounds"); + let session = BlockSyncPeerSession::for_test(peer.clone(), out_send, cancel.clone()); + + let (sequencer_input_tx, _sequencer_input_rx) = mpsc::channel(16); + let (control_tx, _control_rx) = mpsc::unbounded_channel(); + let (actions_tx, _actions_rx) = mpsc::channel(16); + let (routine_to_reactor_tx, _routine_to_reactor_rx) = mpsc::channel(16); + // The download floor sits above the request below, as if other peers + // delivered those heights and the sequencer committed them. + let (_view_tx, view_rx) = watch::channel(initial_view(BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(100), + verified_block_hash: block::Hash([0; 32]), + })); + + let mut routine = PeerRoutine::new( + peer, + session, + in_recv, + config, + 0, + budget, + work, + Arc::new(PeerRegistry::new()), + Arc::new(Mutex::new(ThroughputMeter::new(Instant::now()))), + sequencer_input_tx, + Arc::new(AtomicU64::new(0)), + control_tx, + actions_tx, + routine_to_reactor_tx, + view_rx, + cancel, + ZakuraTrace::noop(), + ); + + let now = Instant::now(); + let liveness = Duration::from_secs(30); + + // The peer delivered a block for an earlier request... + routine + .window + .arm_liveness(now - Duration::from_secs(120), liveness); + routine + .window + .note_block_progress(now - Duration::from_secs(90), liveness); + + // ...then received a newer request (deadline re-armed) whose heights + // the floor later passed. + routine + .window + .arm_liveness(now - Duration::from_secs(60), liveness); + routine.window.outstanding.push(OutstandingBlockRange { + request: BlockRangeRequest { + start_height: block::Height(99), + count: 2, + anchor_hash: block::Hash([9; 32]), + estimated_bytes: 0, + expected_blocks: vec![ + ExpectedBlock { + height: block::Height(99), + hash: block::Hash([99; 32]), + estimated_bytes: 0, + }, + ExpectedBlock { + height: block::Height(100), + hash: block::Hash([100; 32]), + estimated_bytes: 0, + }, + ], + }, + queued_at: now - Duration::from_secs(60), + deadline: now + Duration::from_secs(60), + delivery_snapshot: routine + .window + .delivery_snapshot(now - Duration::from_secs(60)), + delivered_bytes: 0, + received: ReceivedBlockTracker::default(), + }); + + routine.gc_committed_outstanding(); + + assert!( + routine.window.outstanding.is_empty(), + "the floor passed the whole request, so GC must remove it" + ); + assert_eq!( + routine.window.block_liveness_deadline, None, + "a request satisfied below the floor must not leave a stale liveness deadline" + ); + assert!( + matches!(routine.window.check_liveness(now), LivenessOutcome::Ok), + "the peer owes nothing and must not be disconnected" + ); + } } From 600a67e2fa3199c53af6309c598e8c118745e80f Mon Sep 17 00:00:00 2001 From: roman Date: Mon, 6 Jul 2026 02:45:04 -0600 Subject: [PATCH 10/27] fix(zakura): publish chain-tip resets verbatim from the chain-tip mirror MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The mirror maxed a TipAction::Reset tip against the latest_chain_tip watch, which may not have observed the reset yet; the stale higher tip turned the downstream VerifiedReset into a no-op, pinning the block-sync sequencer one block above the real tip after a fork-recovery invalidation — the missing parent was never requested and every body commit timed out until restart. On Reset the action's tip is authoritative; anti-regression maxing now applies only to Grow. --- CHANGELOG.md | 9 +++++ zebrad/src/commands/start.rs | 25 +++++++++++++ .../start/zakura/header_sync_driver.rs | 36 +++++++++++++++---- zebrad/src/commands/start/zakura/mod.rs | 4 +-- 4 files changed, 66 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fcd12ba349a..7c706cd9b54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,15 @@ and this project adheres to [Semantic Versioning](https://semver.org). - Fixed a near-tip sync restart loop when a timed-out `AwaitUtxo` lookup in the transaction verifier was converted to `InternalDowncastError` instead of a missing transparent input. +- Fixed a chain-tip reset (fork-recovery invalidation, `invalidateblock`) + being silently discarded by the Zakura chain-tip mirror. The mirror maxed + the reset tip against a `latest_chain_tip` watch that may not have observed + the reset yet, republished the stale higher tip, and turned the downstream + `VerifiedReset` into a no-op — the block-sync sequencer then pinned one + block above the real tip, never requested the missing parent, and every + body commit timed out until restart. On `TipAction::Reset` the action's tip + is now published verbatim; the anti-regression maxing only applies to + `Grow`. - Fixed Zakura block-sync peers being serially disconnected and parked at the chain tip by a stale liveness deadline. When the download floor passed a request because other peers' deliveries satisfied its heights, the floor GC diff --git a/zebrad/src/commands/start.rs b/zebrad/src/commands/start.rs index 29925e33dd9..9e6784ec79c 100644 --- a/zebrad/src/commands/start.rs +++ b/zebrad/src/commands/start.rs @@ -2917,6 +2917,31 @@ mod zakura_header_sync_driver_tests { /// when the committed suffix sits on a branch the header chain reorged away /// from (e.g. after a crash between a durable header reorg and its body /// invalidation, when later commits report no header conflict). + /// On `TipAction::Reset` the mirror must publish the action's tip verbatim: + /// maxing it against a `latest_chain_tip` watch that has not yet observed + /// the reset republishes the stale higher tip, turns the `VerifiedReset` + /// into a no-op, and pins the block-sync sequencer one block above the + /// real tip (observed live after a fork-recovery invalidation: every body + /// commit then timed out waiting for a parent that was never requested). + #[test] + fn chain_tip_mirror_reset_tip_is_authoritative() { + use super::zakura::chain_tip_mirror_verified_tip; + + let reset_tip = (block::Height(100), block::Hash([1; 32])); + let stale_watch_tip = Some((block::Height(101), block::Hash([2; 32]))); + let finalized = Some((block::Height(90), block::Hash([3; 32]))); + let reset = zebra_state::TipAction::Reset { + height: reset_tip.0, + hash: reset_tip.1, + }; + + assert_eq!( + chain_tip_mirror_verified_tip(&reset, finalized, reset_tip, stale_watch_tip), + reset_tip, + "a reset's tip is authoritative and must not be maxed against stale sources" + ); + } + #[tokio::test] async fn stranded_body_suffix_is_reconciled_by_invalidation() { use std::sync::atomic::{AtomicBool, Ordering}; diff --git a/zebrad/src/commands/start/zakura/header_sync_driver.rs b/zebrad/src/commands/start/zakura/header_sync_driver.rs index 10549fb247b..b60aef2c34f 100644 --- a/zebrad/src/commands/start/zakura/header_sync_driver.rs +++ b/zebrad/src/commands/start/zakura/header_sync_driver.rs @@ -1553,13 +1553,11 @@ pub(crate) async fn mirror_zakura_full_block_commits( insert_cs_height(row, cs_trace::FINALIZED_HEIGHT, finalized_height); }, ); - let action_tip = Some((height, hash)); - let verified_block_tip = - verified_block_tip_from_state(finalized_tip, action_tip, (height, hash)); - let verified_block_tip = verified_block_tip_from_state( - Some(verified_block_tip), + let verified_block_tip = chain_tip_mirror_verified_tip( + &action, + finalized_tip, + (height, hash), latest_chain_tip.best_tip_height_and_hash(), - verified_block_tip, ); emit_commit_state( @@ -1704,6 +1702,32 @@ pub(crate) fn block_sync_chain_tip_event( } } +/// The verified-body tip the chain-tip mirror publishes for a tip action. +/// +/// On `Grow`, the action tip is maxed against the finalized tip and the +/// `latest_chain_tip` watch so a lagging source cannot regress the frontier. +/// On `Reset` the action's tip IS the authoritative post-reset chain tip: +/// maxing it against a watch that has not yet observed the reset republishes +/// the stale higher tip, turns the downstream `VerifiedReset` into a no-op, +/// and pins the block-sync sequencer one block above the real tip — it then +/// never requests the missing parent and every body commit times out +/// (observed live after a fork-recovery invalidation). +pub(crate) fn chain_tip_mirror_verified_tip( + action: &zebra_state::TipAction, + finalized_tip: Option<(block::Height, block::Hash)>, + action_tip: (block::Height, block::Hash), + latest_tip: Option<(block::Height, block::Hash)>, +) -> (block::Height, block::Hash) { + match action { + zebra_state::TipAction::Reset { .. } => action_tip, + zebra_state::TipAction::Grow { .. } => { + let verified_block_tip = + verified_block_tip_from_state(finalized_tip, Some(action_tip), action_tip); + verified_block_tip_from_state(Some(verified_block_tip), latest_tip, verified_block_tip) + } + } +} + pub(crate) fn chain_tip_mirror_frontier_change( action: &zebra_state::TipAction, previous_verified_body: block::Height, diff --git a/zebrad/src/commands/start/zakura/mod.rs b/zebrad/src/commands/start/zakura/mod.rs index 3aba77053f6..750ba4242a0 100644 --- a/zebrad/src/commands/start/zakura/mod.rs +++ b/zebrad/src/commands/start/zakura/mod.rs @@ -130,8 +130,8 @@ pub(crate) use frontier::{query_block_sync_frontiers, verified_block_tip_from_st #[cfg(test)] pub(crate) use header_sync_driver::{ block_roots_cover_range, block_sync_chain_tip_event, body_sizes_for_served_header_range, - chain_tip_mirror_frontier_change, header_range_commit_failure_kind, - notify_block_sync_header_tip, reconcile_stranded_body_suffix, + chain_tip_mirror_frontier_change, chain_tip_mirror_verified_tip, + header_range_commit_failure_kind, notify_block_sync_header_tip, reconcile_stranded_body_suffix, root_covered_query_best_header_tip, tree_aux_roots_for_served_header_range, }; pub(crate) use header_sync_driver::{ From 9c967e992d71ec6b0a64405e777b854ddd74a110 Mon Sep 17 00:00:00 2001 From: roman Date: Mon, 6 Jul 2026 03:48:37 -0600 Subject: [PATCH 11/27] debug(consensus): attribute slow block commits to their pipeline stage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A ~45-minute uniform commit-stall (every Zakura block-sync commit timing out while the state write task, runtime workers, and parent-wait queue were all idle) could not be attributed from logs: the driver's timeout says only that the verifier did not answer. Log the two stages that can silently hold a block — the transaction-verification drain (UTXO awaits on out-of-order parents serialize here, bounded by the 6-minute UTXO_LOOKUP_TIMEOUT) and the state commit await (parent-waiting queue / write-task backlog) — at warn when they exceed 20s, and log timed-out and slow (>20s) transparent UTXO lookups with their outpoint. --- zebra-consensus/src/block.rs | 31 +++++++++++++++++++++++++++--- zebra-consensus/src/transaction.rs | 21 +++++++++++++++++++- 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/zebra-consensus/src/block.rs b/zebra-consensus/src/block.rs index ad8c16556f8..dd5920a581f 100644 --- a/zebra-consensus/src/block.rs +++ b/zebra-consensus/src/block.rs @@ -303,6 +303,7 @@ where let mut block_miner_fees = Ok(Amount::zero()); use futures::StreamExt; + let tx_checks_started = std::time::Instant::now(); while let Some(result) = async_checks.next().await { tracing::trace!(?result, remaining = async_checks.len()); let response = result @@ -323,6 +324,18 @@ where } } + // A slow drain here means transaction verification (usually a UTXO + // await on an out-of-order parent) held the block; this stage log + // attributes driver-side commit timeouts to their pipeline stage. + if tx_checks_started.elapsed() > std::time::Duration::from_secs(20) { + tracing::warn!( + ?height, + ?hash, + elapsed = ?tx_checks_started.elapsed(), + "slow transaction verification stage while committing block" + ); + } + // Check the summed block totals if sigops > MAX_BLOCK_SIGOPS { @@ -378,13 +391,25 @@ where }; } - match state_service + let state_commit_started = std::time::Instant::now(); + let state_commit_result = state_service .ready() .await .map_err(|source| VerifyBlockError::StateService { source, hash })? .call(zs::Request::CommitSemanticallyVerifiedBlock(prepared_block)) - .await - { + .await; + // A slow response here means the block sat in the state's + // parent-waiting queue or behind the write task; this stage log + // attributes driver-side commit timeouts to their pipeline stage. + if state_commit_started.elapsed() > std::time::Duration::from_secs(20) { + tracing::warn!( + ?height, + ?hash, + elapsed = ?state_commit_started.elapsed(), + "slow state commit stage while committing block" + ); + } + match state_commit_result { Ok(zs::Response::Committed(committed_hash)) => { assert_eq!(committed_hash, hash, "state must commit correct hash"); Ok(hash) diff --git a/zebra-consensus/src/transaction.rs b/zebra-consensus/src/transaction.rs index 0635c2f9bac..7860110549e 100644 --- a/zebra-consensus/src/transaction.rs +++ b/zebra-consensus/src/transaction.rs @@ -743,14 +743,33 @@ where utxo } else { + let lookup_started = std::time::Instant::now(); let response = state .clone() .oneshot(zebra_state::Request::AwaitUtxo(*outpoint)) .await .map_err(|boxed_error| match boxed_error.downcast::() { - Ok(_) => TransactionError::TransparentInputNotFound, + Ok(_) => { + // A timed-out lookup held this transaction's whole + // block for UTXO_LOOKUP_TIMEOUT; chains of these + // serialize into multi-minute commit stalls that + // are otherwise unattributable from logs. + tracing::warn!( + ?outpoint, + elapsed = ?lookup_started.elapsed(), + "transparent input UTXO lookup timed out" + ); + TransactionError::TransparentInputNotFound + } Err(boxed_error) => TransactionError::from(boxed_error), })?; + if lookup_started.elapsed() > std::time::Duration::from_secs(20) { + tracing::warn!( + ?outpoint, + elapsed = ?lookup_started.elapsed(), + "slow transparent input UTXO lookup delayed transaction verification" + ); + } if let zebra_state::Response::Utxo(utxo) = response { utxo From 3e5b6eb4ca876df7b729aa91868fb066fe99e20f Mon Sep 17 00:00:00 2001 From: roman Date: Mon, 6 Jul 2026 04:49:44 -0600 Subject: [PATCH 12/27] fix(zakura): defer block-sync liveness disconnects during local commit stalls 'No accepted block progress' cannot be blamed on a peer while the node's own apply pipeline is not landing commits: during a ~45-minute verifier-layer commit stall the liveness deadline exiled healthy peers one by one into the 180s no-progress cooldown (observed live: parks +4 on t2, t4 down to 2 peers). When the sequencer view shows bodies in the applying queue, extend the deadline instead of disconnecting; a genuinely dead peer is caught as soon as the pipeline drains, and its timed-out requests were already requeued by expire_due_timeouts. --- CHANGELOG.md | 7 ++ .../src/zakura/block_sync/peer_routine.rs | 102 +++++++++++++++++- 2 files changed, 105 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c706cd9b54..e27339610af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,13 @@ and this project adheres to [Semantic Versioning](https://semver.org). - Fixed a near-tip sync restart loop when a timed-out `AwaitUtxo` lookup in the transaction verifier was converted to `InternalDowncastError` instead of a missing transparent input. +- Zakura block-sync liveness disconnects are now deferred while the local + apply pipeline holds unfinished bodies: "no accepted block progress" cannot + be blamed on a peer while the node's own commits are not landing. During a + ~45-minute local commit stall the old behavior exiled healthy peers one by + one into the no-progress cooldown. A genuinely dead peer is still caught as + soon as the pipeline drains. Deferrals are counted in + `sync.block.liveness.deferred_local_stall`. - Fixed a chain-tip reset (fork-recovery invalidation, `invalidateblock`) being silently discarded by the Zakura chain-tip mirror. The mirror maxed the reset tip against a `latest_chain_tip` watch that may not have observed diff --git a/zebra-network/src/zakura/block_sync/peer_routine.rs b/zebra-network/src/zakura/block_sync/peer_routine.rs index 0b2d44bb229..55aa9380339 100644 --- a/zebra-network/src/zakura/block_sync/peer_routine.rs +++ b/zebra-network/src/zakura/block_sync/peer_routine.rs @@ -1125,14 +1125,34 @@ impl PeerRoutine { // continuous backpressure): plausibly transient local write congestion, not // a dead peer. While outbound is full the select loop does not drain inbound // frames (`if outbound_queue_has_capacity`), so a block the peer already sent - // may be waiting behind our write side. Grant one short, BOUNDED grace. This - // is the *only* liveness extension: a peer that stopped reading holds outbound - // full past `request_timeout`, falls through to the disconnect arm, and is - // disconnected at the liveness deadline — it cannot dodge the timer. + // may be waiting behind our write side. Grant one short, BOUNDED grace: a peer + // that stopped reading holds outbound full past `request_timeout`, falls + // through to the disconnect arm, and is disconnected at the liveness + // deadline — it cannot dodge the timer. self.window .extend_liveness_deadline(now, self.config.request_timeout); Ok(()) } + LivenessOutcome::Disconnect if self.sequencer_view.borrow().applying_len > 0 => { + // Bodies are sitting in the LOCAL apply pipeline, so "no accepted + // block progress" cannot be blamed on this peer: nothing can make + // accepted progress while our own commits are not landing. During a + // 45-minute verifier-layer commit stall this arm's absence exiled + // healthy peers one by one into the no-progress cooldown (observed + // live: t2 parks +4, t4 down to 2 peers). Defer the disconnect; a + // genuinely dead peer is caught as soon as the pipeline drains + // (`applying_len == 0`), and its timed-out requests were already + // requeued to other peers by `expire_due_timeouts`. + self.window + .extend_liveness_deadline(now, self.config.request_timeout); + metrics::counter!("sync.block.liveness.deferred_local_stall").increment(1); + tracing::debug!( + peer = ?self.peer, + applying_len = self.sequencer_view.borrow().applying_len, + "deferring block-sync liveness disconnect: local apply pipeline is stalled" + ); + Ok(()) + } LivenessOutcome::Disconnect => { let error = "block-sync peer made no accepted block progress before liveness deadline"; @@ -2717,4 +2737,78 @@ mod tests { "the peer owes nothing and must not be disconnected" ); } + + /// An expired liveness deadline must not disconnect a peer while the LOCAL + /// apply pipeline holds unfinished bodies: nothing can make "accepted block + /// progress" while our own commits are not landing. During a live + /// verifier-layer commit stall this exiled healthy peers one by one into + /// the no-progress cooldown until the peer set collapsed. Once the + /// pipeline drains, the same expired deadline disconnects as before. + #[tokio::test] + async fn liveness_disconnect_is_deferred_while_local_applies_are_stalled() { + let config = ZakuraBlockSyncConfig::default(); + let budget = ByteBudget::new(1_000_000); + let work = Arc::new(WorkQueue::new(block::Height(0))); + + let cancel = CancellationToken::new(); + let (out_send, _out_recv) = framed_channel(16); + let (_in_send, in_recv) = framed_channel(16); + let peer = ZakuraPeerId::new(vec![9u8; 32]).expect("test peer id is within bounds"); + let session = BlockSyncPeerSession::for_test(peer.clone(), out_send, cancel.clone()); + + let (sequencer_input_tx, _sequencer_input_rx) = mpsc::channel(16); + let (control_tx, _control_rx) = mpsc::unbounded_channel(); + let (actions_tx, _actions_rx) = mpsc::channel(16); + let (routine_to_reactor_tx, _routine_to_reactor_rx) = mpsc::channel(16); + let mut view = initial_view(BlockSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(100), + verified_block_hash: block::Hash([0; 32]), + }); + view.applying_len = 60; + let (view_tx, view_rx) = watch::channel(view.clone()); + + let mut routine = PeerRoutine::new( + peer, + session, + in_recv, + config, + 0, + budget, + work, + Arc::new(PeerRegistry::new()), + Arc::new(Mutex::new(ThroughputMeter::new(Instant::now()))), + sequencer_input_tx, + Arc::new(AtomicU64::new(0)), + control_tx, + actions_tx, + routine_to_reactor_tx, + view_rx, + cancel, + ZakuraTrace::noop(), + ); + + let now = Instant::now(); + // An armed deadline that expired 10s ago. + routine + .window + .arm_liveness(now - Duration::from_secs(60), Duration::from_secs(50)); + + assert!( + routine.check_block_liveness(now).is_ok(), + "an expired deadline must be deferred while local applies are stalled" + ); + + // The pipeline drains: the same expired deadline now disconnects. + view.applying_len = 0; + view_tx + .send(view) + .expect("view receiver is held by the routine"); + // The deferral extended the deadline by request_timeout; move past it. + let later = now + routine.config.request_timeout + Duration::from_secs(1); + assert!( + routine.check_block_liveness(later).is_err(), + "a drained pipeline must restore the liveness disconnect" + ); + } } From c3734e54e4531da3f3487afedfc7ad9d4b1011ef Mon Sep 17 00:00:00 2001 From: roman Date: Mon, 6 Jul 2026 04:50:27 -0600 Subject: [PATCH 13/27] chore(zakura): fix clippy clone_on_copy in liveness deferral test --- zebra-network/src/zakura/block_sync/peer_routine.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zebra-network/src/zakura/block_sync/peer_routine.rs b/zebra-network/src/zakura/block_sync/peer_routine.rs index 55aa9380339..5408800dd54 100644 --- a/zebra-network/src/zakura/block_sync/peer_routine.rs +++ b/zebra-network/src/zakura/block_sync/peer_routine.rs @@ -2766,7 +2766,7 @@ mod tests { verified_block_hash: block::Hash([0; 32]), }); view.applying_len = 60; - let (view_tx, view_rx) = watch::channel(view.clone()); + let (view_tx, view_rx) = watch::channel(view); let mut routine = PeerRoutine::new( peer, From c9480b5a21cf086053c537f56e40e5956fff1879 Mon Sep 17 00:00:00 2001 From: roman Date: Mon, 6 Jul 2026 08:05:29 -0600 Subject: [PATCH 14/27] debug(zakura): log the state error for scored header-range commit failures InvalidPeerRange rejections score the peer, but the CommitHeaderRangeError variant behind them was only visible at debug level: a stranded-context wedge (every peer's response for the same range rejected identically for hours, observed live at height 4148005) was indistinguishable from real peer misbehavior at default log levels. --- .../start/zakura/header_sync_driver.rs | 31 ++++++++++++++----- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/zebrad/src/commands/start/zakura/header_sync_driver.rs b/zebrad/src/commands/start/zakura/header_sync_driver.rs index b60aef2c34f..73989294e3e 100644 --- a/zebrad/src/commands/start/zakura/header_sync_driver.rs +++ b/zebrad/src/commands/start/zakura/header_sync_driver.rs @@ -789,14 +789,29 @@ pub(crate) async fn drive_zakura_header_sync_actions Date: Mon, 6 Jul 2026 08:19:30 -0600 Subject: [PATCH 15/27] fix(zakura): walk back on repeated header context mismatches instead of scoring peers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A fork-recovery re-commit can leave stale rows in the stored difficulty window; every extension range is then rejected with InvalidDifficultyThreshold, each rejection scored and disconnected the honest serving peer, and the wedge survived restarts (on-disk rows). No link failure ever fires in this state, so the existing walk-back trigger never engages. Classify ValidateContextError commit failures as ContextMismatch: never score the peer, and count them as stale-anchor evidence — the quorum (3 failures, 2 distinct peers) starts the same exponential walk-back, and each deepening re-commits a longer range that rewrites the poisoned window. --- CHANGELOG.md | 10 ++++ .../src/zakura/header_sync/events.rs | 6 ++ .../src/zakura/header_sync/reactor.rs | 21 +++++++ zebra-network/src/zakura/header_sync/tests.rs | 56 +++++++++++++++++++ .../src/zakura/header_sync/validation.rs | 1 + .../start/zakura/header_sync_driver.rs | 19 ++++++- 6 files changed, 110 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e27339610af..b8d9e58ee7c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,16 @@ and this project adheres to [Semantic Versioning](https://semver.org). - Fixed a near-tip sync restart loop when a timed-out `AwaitUtxo` lookup in the transaction verifier was converted to `InternalDowncastError` instead of a missing transparent input. +- Fixed a permanent header-sync wedge where a fork-recovery re-commit left + stale rows in the stored difficulty-validation window: every extension + range was then rejected with `InvalidDifficultyThreshold`, each rejection + scored and disconnected the (honest) serving peer, and the wedge survived + restarts because the stale rows are on disk (observed live at height + 4148005 for hours). Contextual validation failures are now classified as + `ContextMismatch`: they no longer score peers, and repeated identical + failures from independent peers feed the fork-recovery walk-back — each + deepening re-commits a longer range, rewriting the poisoned window until + the chain extends again. - Zakura block-sync liveness disconnects are now deferred while the local apply pipeline holds unfinished bodies: "no accepted block progress" cannot be blamed on a peer while the node's own commits are not landing. During a diff --git a/zebra-network/src/zakura/header_sync/events.rs b/zebra-network/src/zakura/header_sync/events.rs index 78b9b866d84..83d332333cc 100644 --- a/zebra-network/src/zakura/header_sync/events.rs +++ b/zebra-network/src/zakura/header_sync/events.rs @@ -456,6 +456,12 @@ pub enum HeaderSyncMisbehavior { pub enum HeaderSyncCommitFailureKind { /// The supplied headers failed contextual validation or checkpoint consistency. InvalidPeerRange, + /// The range failed contextual validation (difficulty, median-time) against + /// this node's stored header context. When independent peers fail + /// identically, the stored context itself is suspect (a reorg left stale + /// rows in the validation window), so this must never score the peer and + /// instead feeds fork-recovery walk-back evidence. + ContextMismatch, /// Local storage/channel/resource failure; do not score the peer. Local, } diff --git a/zebra-network/src/zakura/header_sync/reactor.rs b/zebra-network/src/zakura/header_sync/reactor.rs index 0219ed1f429..f60fe50a77f 100644 --- a/zebra-network/src/zakura/header_sync/reactor.rs +++ b/zebra-network/src/zakura/header_sync/reactor.rs @@ -687,6 +687,27 @@ impl HeaderSyncReactor { self.report_misbehavior(peer.clone(), HeaderSyncMisbehavior::InvalidRange) .await; } + if kind == HeaderSyncCommitFailureKind::ContextMismatch { + // The range failed contextual validation against OUR stored header + // context. One peer failing could be a bad range; independent peers + // failing identically means the stored validation window itself is + // stale (a reorg left old-branch rows in it) — every extension is + // rejected forever and no link failure ever fires, so this is the + // only signal that can trigger recovery. Reuse the stale-anchor + // quorum and walk the anchor back; each deepening re-commits a + // longer range, rewriting the poisoned window (observed live: + // InvalidDifficultyThreshold at 4148005 from every peer, surviving + // restarts, with honest peers disconnected one by one). + self.state.stale_anchor.record(peer.clone()); + metrics::counter!("sync.header.stale_anchor.context_mismatch").increment(1); + if self.state.stale_anchor.should_reanchor() { + if self.state.best_header_tip > self.state.verified_block_tip { + self.reanchor_to_verified_block_tip().await; + } else { + self.begin_fork_recovery_walk_back(); + } + } + } let key = PendingCommitKey { peer, start_height, diff --git a/zebra-network/src/zakura/header_sync/tests.rs b/zebra-network/src/zakura/header_sync/tests.rs index b40079532a2..6c41e4f6669 100644 --- a/zebra-network/src/zakura/header_sync/tests.rs +++ b/zebra-network/src/zakura/header_sync/tests.rs @@ -2092,6 +2092,62 @@ async fn invalid_async_header_commit_failure_reports_peer_disconnect() { } } +#[tokio::test(flavor = "current_thread")] +async fn context_mismatch_commit_failures_walk_back_without_scoring_peers() { + let network = regtest_network(); + let mut fixture = spawn_test_reactor(startup_for( + network.clone(), + (block::Height(0), network.genesis_hash()), + None, + )); + let peer_a = peer(63); + let peer_b = peer(64); + + connect_peer(&fixture, peer_a.clone()).await; + connect_peer(&fixture, peer_b.clone()).await; + + // Three contextual rejections from two independent peers: the stored + // validation window is suspect, so the reactor must begin a walk-back + // (QueryReanchorTarget) and must NOT score either peer — they are honest; + // the stale rows are ours. + for peer_id in [&peer_a, &peer_b, &peer_a] { + fixture + .handle + .send(HeaderSyncEvent::HeaderRangeCommitFailed { + peer: peer_id.clone(), + start_height: block::Height(1), + count: 1, + kind: HeaderSyncCommitFailureKind::ContextMismatch, + }) + .await + .unwrap(); + } + + let mut saw_reanchor_query = false; + for _ in 0..12 { + match tokio::time::timeout( + std::time::Duration::from_millis(500), + fixture.actions.recv(), + ) + .await + { + Ok(Some(HeaderSyncAction::Misbehavior { peer, reason })) => { + panic!("honest peer {peer:?} scored for a local context mismatch: {reason:?}") + } + Ok(Some(HeaderSyncAction::QueryReanchorTarget { .. })) => { + saw_reanchor_query = true; + break; + } + Ok(Some(_)) => continue, + Ok(None) | Err(_) => break, + } + } + assert!( + saw_reanchor_query, + "repeated context mismatches from independent peers must start a walk-back" + ); +} + #[tokio::test(flavor = "current_thread")] async fn peer_disconnect_removes_outstanding_requests_for_that_peer() { let network = Network::Mainnet; diff --git a/zebra-network/src/zakura/header_sync/validation.rs b/zebra-network/src/zakura/header_sync/validation.rs index 6e16519e27a..a28a21f16a5 100644 --- a/zebra-network/src/zakura/header_sync/validation.rs +++ b/zebra-network/src/zakura/header_sync/validation.rs @@ -101,6 +101,7 @@ pub(super) fn misbehavior_reason_label(reason: HeaderSyncMisbehavior) -> &'stati pub(super) fn commit_failure_reason_label(kind: HeaderSyncCommitFailureKind) -> &'static str { match kind { HeaderSyncCommitFailureKind::InvalidPeerRange => "invalid_peer_range", + HeaderSyncCommitFailureKind::ContextMismatch => "context_mismatch", HeaderSyncCommitFailureKind::Local => "local", } } diff --git a/zebrad/src/commands/start/zakura/header_sync_driver.rs b/zebrad/src/commands/start/zakura/header_sync_driver.rs index 73989294e3e..13cd1e2f2a5 100644 --- a/zebrad/src/commands/start/zakura/header_sync_driver.rs +++ b/zebrad/src/commands/start/zakura/header_sync_driver.rs @@ -794,7 +794,11 @@ pub(crate) async fn drive_zakura_header_sync_actions { + | zebra_state::CommitHeaderRangeError::ConflictingFullBlockHeader { .. } => { HeaderSyncCommitFailureKind::InvalidPeerRange } + // Contextual failures (difficulty, median-time) validate the range + // against OUR stored header context; when the context holds stale + // reorg rows every honest peer fails identically (observed live: + // InvalidDifficultyThreshold at 4148005 from all peers, for hours). + // Never score the peer for these — the reactor treats them as + // stale-anchor evidence and walks back instead. + zebra_state::CommitHeaderRangeError::ValidateContextError(_) => { + HeaderSyncCommitFailureKind::ContextMismatch + } _ => HeaderSyncCommitFailureKind::Local, } } @@ -1951,6 +1963,7 @@ fn trace_state_read_error( fn commit_failure_result_label(kind: HeaderSyncCommitFailureKind) -> &'static str { match kind { HeaderSyncCommitFailureKind::InvalidPeerRange => "invalid_peer_range", + HeaderSyncCommitFailureKind::ContextMismatch => "context_mismatch", HeaderSyncCommitFailureKind::Local => "local_error", } } From 6f8b4ad88c46dbd8bbd97c274d42dfd2f58b4c59 Mon Sep 17 00:00:00 2001 From: roman Date: Mon, 6 Jul 2026 09:29:37 -0600 Subject: [PATCH 16/27] fix(zakura): classify UnknownAnchor header-commit failures as ContextMismatch UnknownAnchor cannot be the peer's fault: the anchor is this node's own request bookkeeping, and the error fires when the reactor's in-memory frontier references a hash the header store does not have (observed live at 4148376 after the frontier advanced over diverged store rows). It was classified InvalidPeerRange, so honest peers were scored and disconnected and no recovery trigger ever fired. As ContextMismatch it feeds the stale-anchor quorum; the walk-back re-anchors from the store, which is exactly the self-correction this state needs. --- .../src/commands/start/zakura/header_sync_driver.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/zebrad/src/commands/start/zakura/header_sync_driver.rs b/zebrad/src/commands/start/zakura/header_sync_driver.rs index 13cd1e2f2a5..f7d1eded447 100644 --- a/zebrad/src/commands/start/zakura/header_sync_driver.rs +++ b/zebrad/src/commands/start/zakura/header_sync_driver.rs @@ -1492,7 +1492,6 @@ pub(crate) fn header_range_commit_failure_kind( | zebra_state::CommitHeaderRangeError::BodySizeCountMismatch { .. } | zebra_state::CommitHeaderRangeError::TreeAuxRootCountMismatch { .. } | zebra_state::CommitHeaderRangeError::TreeAuxRootHeightMismatch { .. } - | zebra_state::CommitHeaderRangeError::UnknownAnchor { .. } | zebra_state::CommitHeaderRangeError::HeightOverflow | zebra_state::CommitHeaderRangeError::ImmutableConflict { .. } | zebra_state::CommitHeaderRangeError::ReorgTooDeep { .. } @@ -1506,7 +1505,15 @@ pub(crate) fn header_range_commit_failure_kind( // InvalidDifficultyThreshold at 4148005 from all peers, for hours). // Never score the peer for these — the reactor treats them as // stale-anchor evidence and walks back instead. - zebra_state::CommitHeaderRangeError::ValidateContextError(_) => { + // UnknownAnchor cannot be the peer's fault at all: the anchor is our + // own request bookkeeping, and it fails when the reactor's in-memory + // frontier references a hash the header store does not have (observed + // live at 4148376 — a mirror-advanced frontier over a store whose + // suffix rows diverged). The walk-back re-anchors from the STORE + // (`QueryReanchorTarget`), which is exactly the self-correction this + // state needs. + zebra_state::CommitHeaderRangeError::ValidateContextError(_) + | zebra_state::CommitHeaderRangeError::UnknownAnchor { .. } => { HeaderSyncCommitFailureKind::ContextMismatch } _ => HeaderSyncCommitFailureKind::Local, From c89e76a338a479a3987553c8a39b277e3ef73114 Mon Sep 17 00:00:00 2001 From: roman Date: Mon, 6 Jul 2026 16:35:19 -0600 Subject: [PATCH 17/27] fix(state): validate linkage in zakura header-store writers Close the three write-path corruption bugs proven by the Phase-1 header-store coherence suite (PR #490): 1. prepare_header_range_batch_with_roots now rejects ranges whose first header does not link to the anchor or whose headers are not internally contiguous (new CommitHeaderRangeError::UnlinkedRange, classified as a local non-scoring failure in zebrad). 2. The range insert loop skips heights that already have a committed block, so re-deliveries can no longer strand provisional zakura rows below the body tip. 3. prepare_zakura_header_from_committed_block refuses seeds that do not link to the stored row below them as silent no-ops; header-range sync converges the store instead (scenario s11). Flip the four *_upholds_invariants twins to permanent regression gates, delete the corruption_repro_* tests, remove the discovery masking, and un-ignore the random invariant sweep (clean at PROPTEST_CASES=4096). --- CHANGELOG.md | 15 +++ zebra-state/src/error.rs | 15 +++ .../service/finalized_state/zebra_db/block.rs | 95 ++++++++++++++----- zebra-state/src/service/write.rs | 23 ++++- zebrad/src/commands/start.rs | 18 ++++ .../start/zakura/header_sync_driver.rs | 6 ++ 6 files changed, 148 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b8d9e58ee7c..0f3dfdeee6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,21 @@ and this project adheres to [Semantic Versioning](https://semver.org). ### Fixed +- Fixed three Zakura header-store write paths that could leave the on-disk + header store internally incoherent after chain forks, causing nodes to + reject valid headers from honest peers (`InvalidDifficultyThreshold` / + `UnknownAnchor`) and wedge below the network tip until manual intervention. + Header ranges must now link to their anchor and be internally contiguous + (rejected with the new `UnlinkedRange` error otherwise, which is classified + as a local, non-peer-scoring failure), header ranges re-delivered over + heights that already have committed block bodies no longer re-insert + provisional header rows below the body tip, and the header row seeded from + a committed best-chain block is skipped when it does not link to the stored + row below it (header-range sync converges the store instead). +- Fixed dual-stack Zakura fallback shutting down the Zakura serving layer. When + the stall watchdog resumes legacy `ChainSync`, Zakura now keeps its header- + and block-sync reactors alive as a serving/advertising bridge while an apply + gate drains in-flight Zakura body applies before legacy sync drives commits. - Fixed legacy peers being disconnected for returning empty `FindBlocks` or `FindHeaders` responses when Zebra is at or near the network tip. - Kept an already-active mempool and `getblocktemplate` mining RPCs running diff --git a/zebra-state/src/error.rs b/zebra-state/src/error.rs index aaf3f124b5e..84a9720dee8 100644 --- a/zebra-state/src/error.rs +++ b/zebra-state/src/error.rs @@ -258,6 +258,21 @@ pub enum CommitHeaderRangeError { #[error("header height overflow")] HeightOverflow, + /// A header in the range does not link to the anchor or to its predecessor, + /// so committing it would break the header store's linkage invariant. + #[error( + "header at {height:?} links to {actual_parent} instead of its predecessor {expected_parent}" + )] + UnlinkedRange { + /// Height of the first header that fails to link. + height: block::Height, + /// The hash of the row the header must link to (the anchor, or the + /// previous header in the range). + expected_parent: block::Hash, + /// The header's actual `previous_block_hash`. + actual_parent: block::Hash, + }, + /// A committed immutable header conflicts with the requested header. #[error("header at finalized height {height:?} conflicts with an existing header")] ImmutableConflict { diff --git a/zebra-state/src/service/finalized_state/zebra_db/block.rs b/zebra-state/src/service/finalized_state/zebra_db/block.rs index 82b0a8770a5..c1a32007597 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block.rs @@ -1802,6 +1802,12 @@ impl DiskWriteBatch { /// Full block verification is authoritative for the stored body. If a /// provisional Zakura header at this height differs, replace it with the /// block-derived header and drop stale provisional descendants. + /// + /// A block whose parent hash does not match the stored header row below + /// `height` is skipped without error: writing it would leave a gap or + /// broken link in the header store, so the store instead waits for + /// header-range sync to deliver the connecting rows (see the linkage + /// refusal below). #[allow(clippy::unwrap_in_result)] pub fn prepare_zakura_header_from_committed_block( &mut self, @@ -1826,6 +1832,32 @@ impl DiskWriteBatch { return Ok(()); } + // Seeds fire at non-finalized best-*tip* commits, so an nf best-tip + // jump (a fork switch between nf chains, or a restart restoring the nf + // backup) can seed a height whose parent row is missing or belongs to + // another branch. Writing that row would break the header store's + // linkage invariant on disk and poison difficulty-adjustment windows + // (`header_store_coherence` bug 3). Refuse the write instead: the + // header store briefly lags the nf chain, and header-range sync + // converges it onto the new branch through the linkage-checked range + // path. The merged header view (full-block rows first, then zakura + // rows) is the same view the chain walk reads. + let hash_by_height = db.cf_handle("hash_by_height").unwrap(); + let parent_hash: Option = height.previous().ok().and_then(|parent_height| { + db.zs_get(&hash_by_height, &parent_height) + .or_else(|| db.zs_get(&zakura_hash_by_height, &parent_height)) + }); + if parent_hash != Some(block.header.previous_block_hash) { + tracing::debug!( + ?height, + ?hash, + parent = ?block.header.previous_block_hash, + stored_parent = ?parent_hash, + "skipping Zakura header seed that does not link to the stored row below it" + ); + return Ok(()); + } + if existing_zakura_header.is_some_and(|existing_header| existing_header != block.header) { let best_header_tip: Option<(block::Height, block::Hash)> = db.zs_last_key_value(&zakura_hash_by_height); @@ -2036,6 +2068,14 @@ impl DiskWriteBatch { let mut first_conflicting_height = None; let mut validated_headers = Vec::with_capacity(headers.len()); + // Each header must link to the anchor (for the first header) or to its + // predecessor in the range. Without this check, a range anchored at the + // same-height hash of a *different* branch can pass contextual + // difficulty validation and commit a suffix that does not link to the + // row below it — an on-disk linkage violation reachable from a single + // peer response. + let mut expected_parent = anchor; + for (index, header) in headers.iter().enumerate() { let offset = u32::try_from(index + 1).map_err(|_| CommitHeaderRangeError::HeightOverflow)?; @@ -2044,6 +2084,16 @@ impl DiskWriteBatch { let hash = block::Hash::from(&**header); let body_size = body_sizes[index]; let roots = &tree_aux_roots[index]; + + if header.previous_block_hash != expected_parent { + return Err(CommitHeaderRangeError::UnlinkedRange { + height, + expected_parent, + actual_parent: header.previous_block_hash, + }); + } + expected_parent = hash; + if roots.height != height { return Err(CommitHeaderRangeError::TreeAuxRootHeightMismatch { expected_height: height, @@ -2166,6 +2216,14 @@ impl DiskWriteBatch { for (index, (height, hash, header, body_size)) in validated_headers.into_iter().enumerate() { + // Finalized block heights already have authoritative block rows and + // verified roots, even when pruning has removed their transactions. + // Re-delivered headers must not recreate provisional zakura rows + // there, because those rows are only trimmed during body commit. + if zebra_db.contains_height(height) { + continue; + } + let same_header = zebra_db.zakura_header_hash(height) == Some(hash); let advertised_body_size = match ( same_header, @@ -2185,29 +2243,20 @@ impl DiskWriteBatch { } else { self.zs_delete(&body_size_by_height, height); } - // A height with a committed body already has a *verified* row in the - // serving index, written by the body-commit batch. Peer-supplied roots - // are unauthenticated until verify-before-commit, and that verification - // only ever runs above the body tip — so a header range re-delivered - // over committed heights (a header store behind the body store, or a - // late range response racing body sync) must never overwrite the - // verified row: committed roots win on any overlap (design §9). - if !zebra_db.contains_body_at_height(height) { - let roots = &tree_aux_roots[index]; - self.zs_insert( - &roots_by_height, - height, - CommitmentRootsByHeight { - sapling: roots.sapling_root, - orchard: roots.orchard_root, - ironwood: roots.ironwood_root, - sapling_tx: roots.sapling_tx, - orchard_tx: roots.orchard_tx, - ironwood_tx: roots.ironwood_tx, - auth_data_root: roots.auth_data_root, - }, - ); - } + let roots = &tree_aux_roots[index]; + self.zs_insert( + &roots_by_height, + height, + CommitmentRootsByHeight { + sapling: roots.sapling_root, + orchard: roots.orchard_root, + ironwood: roots.ironwood_root, + sapling_tx: roots.sapling_tx, + orchard_tx: roots.orchard_tx, + ironwood_tx: roots.ironwood_tx, + auth_data_root: roots.auth_data_root, + }, + ); } Ok(HeaderRangeCommitOutcome { diff --git a/zebra-state/src/service/write.rs b/zebra-state/src/service/write.rs index 588078056f1..c5f59ec9630 100644 --- a/zebra-state/src/service/write.rs +++ b/zebra-state/src/service/write.rs @@ -744,7 +744,7 @@ mod tests { use crate::{ arbitrary::Prepare, service::{ - finalized_state::FinalizedState, + finalized_state::{DiskWriteBatch, FinalizedState, WriteDisk}, non_finalized_state::NonFinalizedState, write::{ seed_zakura_header_from_committed_block, @@ -781,6 +781,27 @@ mod tests { let mut non_finalized_state = NonFinalizedState::new(&network); + // The seed path refuses rows that do not link to the stored header row + // below them, and the fake chain's parent block is not otherwise + // committed to this state, so store its hash as a provisional Zakura + // row (the consensus `hash_by_height` row cannot be written alone: a + // finalized tip implies note commitment trees exist). + let parent_height = parent + .coinbase_height() + .expect("test vector block has a coinbase height"); + let zakura_hash_by_height = finalized_state + .db + .db() + .cf_handle("zakura_header_hash_by_height") + .unwrap(); + let mut batch = DiskWriteBatch::new(); + batch.zs_insert(&zakura_hash_by_height, parent_height, parent.hash()); + finalized_state + .db + .db() + .write(batch) + .expect("parent hash row writes"); + non_finalized_state .commit_new_chain(best_block.clone().prepare(), &finalized_state) .expect("best block commits to a new chain"); diff --git a/zebrad/src/commands/start.rs b/zebrad/src/commands/start.rs index 9e6784ec79c..4ec3daeb17f 100644 --- a/zebrad/src/commands/start.rs +++ b/zebrad/src/commands/start.rs @@ -2216,6 +2216,24 @@ mod zakura_header_sync_driver_tests { ); } + #[test] + fn unlinked_range_is_local_header_sync_commit_failure() { + // The reactor validates every response's linkage against the requested + // anchor before committing with that anchor, so the store's own + // linkage check failing means local anchor/response pairing broke, + // not peer misbehavior. + let error = zebra_state::CommitHeaderRangeError::UnlinkedRange { + height: block::Height(1), + expected_parent: block::Hash([0; 32]), + actual_parent: block::Hash([1; 32]), + }; + + assert_eq!( + header_range_commit_failure_kind(&error), + HeaderSyncCommitFailureKind::Local + ); + } + #[test] fn served_header_body_size_hints_align_with_served_heights() { let start = block::Height(10); diff --git a/zebrad/src/commands/start/zakura/header_sync_driver.rs b/zebrad/src/commands/start/zakura/header_sync_driver.rs index f7d1eded447..a0826087f71 100644 --- a/zebrad/src/commands/start/zakura/header_sync_driver.rs +++ b/zebrad/src/commands/start/zakura/header_sync_driver.rs @@ -1484,6 +1484,12 @@ pub(crate) fn header_range_commit_failure_kind( // fork. Treat it as non-scoring so this stays a liveness/correctness guard, // not peer punishment. | zebra_state::CommitHeaderRangeError::LowerWorkConflict { .. } + // The reactor already validates every peer response against the requested + // anchor and for internal continuity (`validate_header_range_links`) and + // scores linkage failures there, then commits with that same anchor. So the + // store's own linkage check failing means the local anchor/response pairing + // went wrong, not that the peer misbehaved. + | zebra_state::CommitHeaderRangeError::UnlinkedRange { .. } | zebra_state::CommitHeaderRangeError::CommitResponseDropped => { HeaderSyncCommitFailureKind::Local } From 59dca921ca29116dd0f9352d3f0beeae613ec5b6 Mon Sep 17 00:00:00 2001 From: roman Date: Mon, 6 Jul 2026 18:49:37 -0600 Subject: [PATCH 18/27] canary: cherry-pick #494 startup audit + self-repair (tests dropped in pick) --- .../src/service/finalized_state/zebra_db.rs | 11 + .../service/finalized_state/zebra_db/block.rs | 2 + .../zebra_db/block/startup_audit.rs | 428 ++++++++++++++++++ 3 files changed, 441 insertions(+) create mode 100644 zebra-state/src/service/finalized_state/zebra_db/block/startup_audit.rs diff --git a/zebra-state/src/service/finalized_state/zebra_db.rs b/zebra-state/src/service/finalized_state/zebra_db.rs index e7ba5325b2f..25170417702 100644 --- a/zebra-state/src/service/finalized_state/zebra_db.rs +++ b/zebra-state/src/service/finalized_state/zebra_db.rs @@ -175,6 +175,17 @@ impl ZebraDb { ) } + // Audit the zakura header store's on-disk invariants and truncate any + // incoherent suffix, so a store corrupted by an earlier binary + // self-heals at startup instead of wedging header sync (headers are + // re-fetchable, so correctness beats preserved rows). Read-only + // instances cannot repair; their reads surface any corruption as + // explicit `StoreIncoherentError`s instead. + if !read_only { + db.audit_and_repair_zakura_header_store() + .expect("startup header-store repair write failed: RocksDB is unavailable"); + } + db.spawn_format_change(format_change); db diff --git a/zebra-state/src/service/finalized_state/zebra_db/block.rs b/zebra-state/src/service/finalized_state/zebra_db/block.rs index c1a32007597..cde76e7fc4d 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block.rs @@ -58,6 +58,8 @@ use crate::{ #[cfg(feature = "indexer")] use crate::request::Spend; +mod startup_audit; + #[cfg(test)] mod tests; diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/startup_audit.rs b/zebra-state/src/service/finalized_state/zebra_db/block/startup_audit.rs new file mode 100644 index 00000000000..8a042a1e38d --- /dev/null +++ b/zebra-state/src/service/finalized_state/zebra_db/block/startup_audit.rs @@ -0,0 +1,428 @@ +//! Startup audit and self-repair for the zakura header store. +//! +//! The zakura header store is five height-indexed column families acting as a +//! replicated view of the canonical header chain above the finalized tip. Its +//! invariants (hash↔height bijection, parent linkage anchored at the finalized +//! tip, no gaps or stranded rows below the header tip) are enforced by the +//! writers, but a store corrupted by an earlier binary stays corrupted on disk +//! and wedges header sync: consensus reads surface the damage as +//! [`StoreIncoherentError`](crate::error::StoreIncoherentError) and refuse to +//! feed stale rows into validation, so the node can no longer make progress +//! past the poisoned window. +//! +//! This module runs the store audit once at [`ZebraDb`] startup and repairs +//! any violation by truncating the zakura column families to the last +//! coherent height in one atomic batch. Headers are re-fetchable — header +//! sync re-downloads the truncated suffix — so correctness beats preserved +//! rows: any residual write-path bug in this class becomes a self-healing, +//! observable transient instead of a permanent on-disk wedge. +//! +//! The audit cost is `O(header frontier)`: the zakura column families only +//! hold rows above the finalized tip (plus any stale rows this audit exists +//! to remove), and the verified commitment-roots history below the tip is +//! never scanned. + +use std::{ + collections::{BTreeMap, HashMap}, + sync::Arc, +}; + +use zebra_chain::block::{self, Height}; + +use super::{ + AdvertisedBodySize, ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT, ZAKURA_HEADER_BY_HEIGHT, + ZAKURA_HEADER_HASH_BY_HEIGHT, ZAKURA_HEADER_HEIGHT_BY_HASH, +}; +use crate::service::finalized_state::{ + disk_db::{DiskWriteBatch, WriteDisk}, + disk_format::shielded::CommitmentRootsByHeight, + zebra_db::ZebraDb, + COMMITMENT_ROOTS_BY_HEIGHT, +}; + +/// How many violations are included in the repair log line. The full list can +/// be as long as the stranded suffix; the first few identify the fault shape. +const LOGGED_VIOLATIONS: usize = 8; + +/// A single header-store invariant violation found by the startup audit. +/// +/// Every violation is repaired by deleting rows; the variants exist for +/// logging and for test assertions on the fault shape. The fields are +/// diagnostic payload rendered through `Debug` in the repair warning. +#[allow(dead_code)] +#[derive(Clone, Debug)] +pub(crate) enum ZakuraStoreViolation { + /// A zakura row at a height with a committed block (`contains_height`). + /// + /// Committed heights have authoritative full-block rows, and the zakura + /// row at a height is trimmed when its body commits, so a surviving row + /// here is stale (the pre-guard re-delivery bug shape). The predicate is + /// the consensus `hash_by_height` row — which pruning retains — not body + /// presence, so stale rows at pruned heights are found too. + StaleRowAtCommittedHeight { + /// The column family holding the stale row. + cf: &'static str, + /// The committed height of the stale row. + height: Height, + }, + + /// A `hash_by_height` row at `height` has no `header_by_height` row. + MissingHeaderRow { + /// The height with a hash row but no header row. + height: Height, + }, + + /// The header stored at `height` is not the block its hash row names. + HeaderHashMismatch { + /// The height of the divergent rows. + height: Height, + /// The hash the height→hash index names. + indexed: block::Hash, + /// The stored header's actual hash. + computed: block::Hash, + }, + + /// The header at `height` does not link to the stored row below it. + BrokenLinkage { + /// The height of the header whose parent link failed to resolve. + height: Height, + /// The parent hash the header claims (`previous_block_hash`). + expected_parent: block::Hash, + /// The hash actually stored at `height - 1`. + actual_below: block::Hash, + }, + + /// The `height_by_hash` index is missing or wrong for the hash stored at + /// `height` (a forward bijection failure). + WrongHeightByHash { + /// The height whose hash failed the round-trip. + height: Height, + /// The hash stored at `height`. + hash: block::Hash, + /// The height the hash→height index reports, if any. + indexed: Option, + }, + + /// A row above the last coherent height (a stranded suffix: rows above a + /// gap, above a broken link, or above the linked chain's tip). + RowAboveLastCoherent { + /// The column family holding the stranded row. + cf: &'static str, + /// The stranded height. + height: Height, + }, + + /// A `height_by_hash` entry whose target height stores a different hash + /// (or no row at all): a stranded reverse-index row. + OrphanHeightByHash { + /// The hash of the stranded entry. + hash: block::Hash, + /// The height the entry points at. + points_at: Height, + }, +} + +/// The outcome of a startup repair: what was found and what was deleted. +/// +/// The fields are diagnostic payload: read by test assertions and rendered +/// through `Debug`. +#[allow(dead_code)] +#[derive(Debug)] +pub(crate) struct ZakuraStoreRepair { + /// The last height that passed every audit check, and its hash. + /// + /// Rows above it were truncated. `None` means the store had zakura rows + /// but no finalized tip to anchor them at, so all rows were removed. + pub last_coherent: Option<(Height, block::Hash)>, + + /// The number of rows deleted across all five column families. + pub deleted_rows: usize, + + /// Every violation found, in audit order. + pub violations: Vec, +} + +impl ZebraDb { + /// Audits the zakura header store invariants and repairs any violation by + /// deleting the offending rows in one atomic batch. + /// + /// Checks, over the whole zakura store (which only holds the header + /// frontier above the finalized tip, so this is cheap): + /// + /// - **bijection**: `hash_by_height` ↔ `height_by_hash` are mutually + /// inverse, and every stored header is the block its hash row names; + /// - **linkage**: rows chain by `previous_block_hash` from the finalized + /// tip hash upward; + /// - **tip integrity**: no rows in any zakura column family above the + /// last linked height, no gaps below it, and no zakura rows at + /// committed heights. + /// + /// On violation, emits the `state.zakura.header_store.incoherent` metric + /// and a warning, then truncates the zakura column families to the last + /// coherent height (and removes stale rows below the finalized tip and + /// orphaned reverse-index entries). Header sync re-downloads the + /// truncated suffix. A store that would fail the linkage-verified + /// consensus reads ([`StoreIncoherentError`](crate::error::StoreIncoherentError)) + /// is always repaired by this audit, because the audit checks are a + /// superset of the read-path checks over the same rows. + /// + /// Returns `Ok(None)` if the store is coherent (nothing is written), or + /// the repair summary after a successful repair write. + /// + /// Verified commitment roots at committed heights are never touched: the + /// roots column family is only scanned above the finalized tip. + pub(crate) fn audit_and_repair_zakura_header_store( + &self, + ) -> Result, rocksdb::Error> { + // Databases opened through `ZebraDb::new` without the zakura column + // families have no header store to audit. + let ( + Some(header_cf), + Some(hash_cf), + Some(height_by_hash_cf), + Some(body_size_cf), + Some(roots_cf), + ) = ( + self.db.cf_handle(ZAKURA_HEADER_BY_HEIGHT), + self.db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT), + self.db.cf_handle(ZAKURA_HEADER_HEIGHT_BY_HASH), + self.db.cf_handle(ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT), + self.db.cf_handle(COMMITMENT_ROOTS_BY_HEIGHT), + ) + else { + return Ok(None); + }; + + let finalized_tip = self.tip(); + + let headers: BTreeMap> = + self.db.zs_forward_range_iter(&header_cf, ..).collect(); + let hashes: BTreeMap = + self.db.zs_forward_range_iter(&hash_cf, ..).collect(); + let heights_by_hash: Vec<(block::Hash, Height)> = self + .db + .zs_forward_range_iter(&height_by_hash_cf, ..) + .collect(); + let body_size_heights: Vec = self + .db + .zs_forward_range_iter::<_, Height, AdvertisedBodySize, _>(&body_size_cf, ..) + .map(|(height, _)| height) + .collect(); + + // The roots column family also holds verified rows at committed + // heights (written by body commits, kept through pruning); those are + // not zakura frontier rows and are never audited or repaired. Only + // rows above the finalized tip are provisional header-sync data. + let provisional_roots_start = + finalized_tip.and_then(|(tip_height, _)| tip_height.next().ok()); + let provisional_root_heights: Vec = match (finalized_tip, provisional_roots_start) { + // The finalized tip is at the maximum height: no frontier can + // exist above it. + (Some(_), None) => Vec::new(), + (Some(_), Some(start)) => self + .db + .zs_forward_range_iter::<_, Height, CommitmentRootsByHeight, _>(&roots_cf, start..) + .map(|(height, _)| height) + .collect(), + // No finalized tip: every roots row is provisional. + (None, _) => self + .db + .zs_forward_range_iter::<_, Height, CommitmentRootsByHeight, _>(&roots_cf, ..) + .map(|(height, _)| height) + .collect(), + }; + + if headers.is_empty() + && hashes.is_empty() + && heights_by_hash.is_empty() + && body_size_heights.is_empty() + && provisional_root_heights.is_empty() + { + return Ok(None); + } + + let mut violations = Vec::new(); + + // Walk the linked chain upward from the finalized tip, verifying at + // each height that the hash and header rows agree, the header links + // to the row below, and the reverse index round-trips. The walk stops + // at the first missing hash row (the candidate chain tip) or the + // first violation; everything it passed is the coherent prefix. + let forward_index: HashMap = heights_by_hash.iter().copied().collect(); + let last_coherent = finalized_tip.map(|(anchor_height, anchor_hash)| { + let (mut last_height, mut last_hash) = (anchor_height, anchor_hash); + + while let Ok(height) = last_height.next() { + let Some(&hash) = hashes.get(&height) else { + break; + }; + let Some(header) = headers.get(&height) else { + violations.push(ZakuraStoreViolation::MissingHeaderRow { height }); + break; + }; + + let computed = block::Hash::from(&**header); + if computed != hash { + violations.push(ZakuraStoreViolation::HeaderHashMismatch { + height, + indexed: hash, + computed, + }); + break; + } + + if header.previous_block_hash != last_hash { + violations.push(ZakuraStoreViolation::BrokenLinkage { + height, + expected_parent: header.previous_block_hash, + actual_below: last_hash, + }); + break; + } + + let indexed = forward_index.get(&hash).copied(); + if indexed != Some(height) { + violations.push(ZakuraStoreViolation::WrongHeightByHash { + height, + hash, + indexed, + }); + break; + } + + (last_height, last_hash) = (height, hash); + } + + (last_height, last_hash) + }); + + // Committed heights have authoritative full-block rows: the consensus + // `hash_by_height` rows are contiguous from genesis to the finalized + // tip and retained by pruning, so `height <= finalized tip` is + // exactly `contains_height` — the same predicate as the writers' + // committed-height insert gate. (A body-presence predicate would miss + // stale rows at pruned heights.) + let committed = + |height: Height| finalized_tip.is_some_and(|(tip_height, _)| height <= tip_height); + // The coherent frontier window: strictly above the finalized tip, at + // or below the last height the linkage walk verified. + let in_window = |height: Height| { + !committed(height) + && last_coherent.is_some_and(|(last_height, _)| height <= last_height) + }; + + let mut batch = DiskWriteBatch::new(); + let mut deleted_rows = 0; + + // Height-keyed zakura rows survive only inside the coherent window. + let height_keyed: [(&'static str, &rocksdb::ColumnFamilyRef<'_>, Vec); 3] = [ + ( + ZAKURA_HEADER_BY_HEIGHT, + &header_cf, + headers.keys().copied().collect(), + ), + ( + ZAKURA_HEADER_HASH_BY_HEIGHT, + &hash_cf, + hashes.keys().copied().collect(), + ), + ( + ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT, + &body_size_cf, + body_size_heights, + ), + ]; + for (cf_name, cf, heights) in &height_keyed { + for &height in heights { + if in_window(height) { + continue; + } + + violations.push(if committed(height) { + ZakuraStoreViolation::StaleRowAtCommittedHeight { + cf: cf_name, + height, + } + } else { + ZakuraStoreViolation::RowAboveLastCoherent { + cf: cf_name, + height, + } + }); + + batch.zs_delete(*cf, height); + deleted_rows += 1; + } + } + + // Reverse-index entries survive only when their target height is + // inside the window and stores exactly their hash. This removes the + // reverse rows of every deleted hash row, plus entries orphaned by + // earlier overwrites that never cleaned up the displaced hash. + for &(hash, points_at) in &heights_by_hash { + let target_matches = hashes.get(&points_at) == Some(&hash); + if in_window(points_at) && target_matches { + continue; + } + + // Entries whose forward row is deleted above are repair fallout, + // not separate faults; only a live-but-disagreeing target is a + // distinct violation shape worth reporting. + if target_matches { + violations.push(if committed(points_at) { + ZakuraStoreViolation::StaleRowAtCommittedHeight { + cf: ZAKURA_HEADER_HEIGHT_BY_HASH, + height: points_at, + } + } else { + ZakuraStoreViolation::RowAboveLastCoherent { + cf: ZAKURA_HEADER_HEIGHT_BY_HASH, + height: points_at, + } + }); + } else { + violations.push(ZakuraStoreViolation::OrphanHeightByHash { hash, points_at }); + } + + batch.zs_delete(&height_by_hash_cf, hash); + deleted_rows += 1; + } + + // Provisional roots above the window are part of the stranded suffix. + for &height in &provisional_root_heights { + if in_window(height) { + continue; + } + + violations.push(ZakuraStoreViolation::RowAboveLastCoherent { + cf: COMMITMENT_ROOTS_BY_HEIGHT, + height, + }); + batch.zs_delete(&roots_cf, height); + deleted_rows += 1; + } + + if violations.is_empty() { + return Ok(None); + } + + metrics::counter!("state.zakura.header_store.incoherent").increment(1); + tracing::warn!( + ?finalized_tip, + ?last_coherent, + deleted_rows, + violation_count = violations.len(), + first_violations = ?&violations[..violations.len().min(LOGGED_VIOLATIONS)], + "zakura header store failed its startup coherence audit; \ + truncating to the last coherent height so header sync re-downloads the rest" + ); + + self.db.write(batch)?; + + Ok(Some(ZakuraStoreRepair { + last_coherent, + deleted_rows, + violations, + })) + } +} From 839114cc221bff72ffd27988d62fb95e00714093 Mon Sep 17 00:00:00 2001 From: roman Date: Mon, 6 Jul 2026 18:57:24 -0600 Subject: [PATCH 19/27] log the startup audit pass for soak observability --- .../zebra_db/block/startup_audit.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/startup_audit.rs b/zebra-state/src/service/finalized_state/zebra_db/block/startup_audit.rs index 8a042a1e38d..994d4de9586 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/startup_audit.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/startup_audit.rs @@ -238,6 +238,12 @@ impl ZebraDb { && body_size_heights.is_empty() && provisional_root_heights.is_empty() { + // Log the pass so operators can verify the audit ran on this boot. + tracing::info!( + ?finalized_tip, + frontier_rows = 0, + "zakura header store passed its startup coherence audit (empty frontier)" + ); return Ok(None); } @@ -403,6 +409,14 @@ impl ZebraDb { } if violations.is_empty() { + // Log the pass so operators can verify the audit ran on this boot + // (the fleet soak requires observing it finding nothing). + tracing::info!( + ?finalized_tip, + ?last_coherent, + frontier_rows = hashes.len(), + "zakura header store passed its startup coherence audit" + ); return Ok(None); } From d98ff3cd34bef9791007bd6c2a28893f0830ae0f Mon Sep 17 00:00:00 2001 From: roman Date: Mon, 6 Jul 2026 16:49:20 -0600 Subject: [PATCH 20/27] simplify comment --- .../src/service/finalized_state/zebra_db/block.rs | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/zebra-state/src/service/finalized_state/zebra_db/block.rs b/zebra-state/src/service/finalized_state/zebra_db/block.rs index cde76e7fc4d..df8a7032a51 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block.rs @@ -1834,16 +1834,9 @@ impl DiskWriteBatch { return Ok(()); } - // Seeds fire at non-finalized best-*tip* commits, so an nf best-tip - // jump (a fork switch between nf chains, or a restart restoring the nf - // backup) can seed a height whose parent row is missing or belongs to - // another branch. Writing that row would break the header store's - // linkage invariant on disk and poison difficulty-adjustment windows - // (`header_store_coherence` bug 3). Refuse the write instead: the - // header store briefly lags the nf chain, and header-range sync - // converges it onto the new branch through the linkage-checked range - // path. The merged header view (full-block rows first, then zakura - // rows) is the same view the chain walk reads. + // Seeds can jump to a non-finalized best tip whose parent is not the + // stored row below it. Refuse those seeds so the header store stays + // linked; header-range sync will later deliver the missing rows. let hash_by_height = db.cf_handle("hash_by_height").unwrap(); let parent_hash: Option = height.previous().ok().and_then(|parent_height| { db.zs_get(&hash_by_height, &parent_height) From d7c8e58b4a52701d4b5dbac5387d5d9650d9da64 Mon Sep 17 00:00:00 2001 From: roman Date: Mon, 6 Jul 2026 17:39:40 -0600 Subject: [PATCH 21/27] fix(state): verify zakura header-store invariants in consensus reads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pillar 2 of the REORG_PLAN: a corrupted header store can no longer poison difficulty validation — readers surface the storage fault explicitly. - recent_header_context verifies, at every step of its walk, that the stored header is the block its hash row names (HeaderHashMismatch), that it links to the row below (BrokenLinkage), and that no row is missing below a stored one (Gap), returning the new StoreIncoherentError instead of a poisoned or silently shortened difficulty window. - The range writer's anchor round-trip distinguishes a bijection violation in our own indexes (StoreIncoherent::BijectionMismatch) from a genuinely unknown anchor (UnknownAnchor). - CommitHeaderRangeError::StoreIncoherent is classified as a local, non-peer-scoring commit failure in zebrad: the range was rejected because our store cannot supply trustworthy context, not because the peer's range was shown invalid. New reads.rs suite in header_store_coherence hand-corrupts the column families and pins the reader/writer behavior for all four fault shapes, including side-effect freedom of the rejections. --- CHANGELOG.md | 8 ++ zebra-state/src/error.rs | 71 ++++++++++++ zebra-state/src/lib.rs | 3 +- .../service/finalized_state/zebra_db/block.rs | 103 ++++++++++++++---- .../zebra_db/block/tests/vectors.rs | 12 +- zebrad/src/commands/start.rs | 19 ++++ .../start/zakura/header_sync_driver.rs | 6 + 7 files changed, 199 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f3dfdeee6d..505f8444bc3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,14 @@ and this project adheres to [Semantic Versioning](https://semver.org). ### Fixed +- Zakura header-store consensus reads now verify store invariants as they + walk: the difficulty-context walk checks that every stored header is the + block its hash row names and links to the row below it, and the anchor + lookup distinguishes a corrupted index round-trip from a genuinely unknown + anchor. A corrupted store now surfaces as an explicit local + `StoreIncoherent` error (never scored against peers) instead of poisoning + difficulty validation and rejecting honest headers with + `InvalidDifficultyThreshold`. - Fixed three Zakura header-store write paths that could leave the on-disk header store internally incoherent after chain forks, causing nodes to reject valid headers from honest peers (`InvalidDifficultyThreshold` / diff --git a/zebra-state/src/error.rs b/zebra-state/src/error.rs index 84a9720dee8..40533f13fb6 100644 --- a/zebra-state/src/error.rs +++ b/zebra-state/src/error.rs @@ -195,6 +195,68 @@ impl From for CommitCheckpointVerifiedError { } } +/// An internal invariant of the zakura header store was found violated while +/// reading it. +/// +/// This is a **local storage fault**, never evidence about a peer: readers +/// return it instead of feeding rows from more than one branch (or from beside +/// a gap) into consensus validation, where the corruption would otherwise +/// surface as a misleading validation failure (`InvalidDifficultyThreshold`, +/// `UnknownAnchor`) attributed to whoever supplied the input being validated. +#[derive(Debug, Error, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum StoreIncoherentError { + /// The header row at `height` does not link to the stored row below it. + #[error( + "header store incoherent: header at {height:?} links to {expected_parent} but the stored row below is {actual_below}" + )] + BrokenLinkage { + /// Height of the header whose parent link failed to resolve. + height: block::Height, + /// The parent hash the header claims (`previous_block_hash`). + expected_parent: block::Hash, + /// The hash actually stored at `height - 1`. + actual_below: block::Hash, + }, + + /// A header row exists at `height` but the row below it is missing. + #[error( + "header store incoherent: no stored row at {missing:?} below the header at {height:?}" + )] + Gap { + /// Height of the stored header above the gap. + height: block::Height, + /// The missing height (`height - 1`). + missing: block::Height, + }, + + /// The header row at `height` is not the block its hash row names. + #[error( + "header store incoherent: header stored at {height:?} hashes to {computed} but the hash row names {indexed}" + )] + HeaderHashMismatch { + /// Height of the divergent rows. + height: block::Height, + /// The hash the height→hash index names. + indexed: block::Hash, + /// The stored header's actual hash. + computed: block::Hash, + }, + + /// The hash→height and height→hash indexes disagree about a hash. + #[error( + "header store incoherent: hash {hash} is indexed at {height:?} but that height stores {stored:?}" + )] + BijectionMismatch { + /// The hash whose round-trip failed. + hash: block::Hash, + /// The height the hash→height index reports for it. + height: block::Height, + /// What the height→hash index stores there instead. + stored: Option, + }, +} + /// An error describing why a header-only range could not be committed. #[derive(Debug, Error, Clone, PartialEq, Eq)] #[non_exhaustive] @@ -323,6 +385,15 @@ pub enum CommitHeaderRangeError { height: block::Height, }, + /// The local header store was found internally incoherent while reading + /// the context needed to validate the range. + /// + /// This is a local storage fault, not a peer validation failure: the range + /// was rejected because the store cannot supply trustworthy context, not + /// because the range itself was shown invalid. + #[error("header store incoherent while validating range: {0}")] + StoreIncoherent(#[from] StoreIncoherentError), + /// Contextual header validation failed. #[error("could not contextually validate header")] ValidateContextError(#[from] Box), diff --git a/zebra-state/src/lib.rs b/zebra-state/src/lib.rs index 7b9bd04b08c..b5b3706bcfb 100644 --- a/zebra-state/src/lib.rs +++ b/zebra-state/src/lib.rs @@ -43,7 +43,8 @@ pub use config::{ pub use constants::{state_database_format_version_in_code, MAX_BLOCK_REORG_HEIGHT}; pub use error::{ BoxError, CloneError, CommitBlockError, CommitCheckpointVerifiedError, CommitHeaderRangeError, - CommitSemanticallyVerifiedError, DuplicateNullifierError, ValidateContextError, + CommitSemanticallyVerifiedError, DuplicateNullifierError, StoreIncoherentError, + ValidateContextError, }; pub use request::{ CheckpointVerifiedBlock, CommitSemanticallyVerifiedBlockRequest, HashOrHeight, MappedRequest, diff --git a/zebra-state/src/service/finalized_state/zebra_db/block.rs b/zebra-state/src/service/finalized_state/zebra_db/block.rs index df8a7032a51..6662b559df7 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block.rs @@ -36,7 +36,7 @@ use crate::{ constants::{ MAX_BLOCK_REORG_HEIGHT, MAX_HEADER_SYNC_HEIGHT_RANGE, MAX_PRUNE_HEIGHTS_PER_COMMIT, }, - error::{CommitCheckpointVerifiedError, CommitHeaderRangeError}, + error::{CommitCheckpointVerifiedError, CommitHeaderRangeError, StoreIncoherentError}, request::FinalizedBlock, response::HeaderRangeCommitOutcome, service::check, @@ -547,31 +547,85 @@ impl ZebraDb { } /// Returns recent header difficulty/time context in reverse height order, - /// starting at `height`. + /// starting at `height`, verifying `previous_block_hash` linkage at every + /// step of the walk. + /// + /// Returns an empty context when there is no stored row at `height` (the + /// caller decides whether that anchor is unknown), and a shorter-than-span + /// context when the walk reaches genesis. + /// + /// # Errors + /// + /// Returns [`StoreIncoherentError`] when the walk finds a header row that + /// is not the block its hash row names, a row that does not link to the + /// row below it, or a gap below a stored row. Feeding such a window into + /// difficulty validation would mix rows from more than one branch (or + /// shift the adjustment window), producing `InvalidDifficultyThreshold` + /// rejections of honest input — the reader surfaces the storage fault + /// explicitly instead. The per-row hash check costs one header hash per + /// consumed row, negligible next to the validation the window feeds. pub fn recent_header_context( &self, height: block::Height, - ) -> Vec<( - zebra_chain::work::difficulty::CompactDifficulty, - DateTime, - )> { + ) -> Result< + Vec<( + zebra_chain::work::difficulty::CompactDifficulty, + DateTime, + )>, + StoreIncoherentError, + > { let mut context = Vec::with_capacity(check::difficulty::POW_ADJUSTMENT_BLOCK_SPAN); - let mut current_height = Some(height); - while let Some(height) = current_height { - let Some((_hash, header)) = self.header_by_height(height) else { - break; - }; + let Some((mut hash, mut header)) = self.header_by_height(height) else { + return Ok(context); + }; + let mut height = height; + + loop { + let computed = block::Hash::from(&*header); + if computed != hash { + return Err(StoreIncoherentError::HeaderHashMismatch { + height, + indexed: hash, + computed, + }); + } context.push((header.difficulty_threshold, header.time)); + if context.len() == check::difficulty::POW_ADJUSTMENT_BLOCK_SPAN { - break; + return Ok(context); } + let Ok(below) = height.previous() else { + // The walk reached genesis: a short context is legitimate, and + // the difficulty functions handle it (MedianTime clamps + // negative heights to zero). + return Ok(context); + }; - current_height = height.previous().ok(); - } + let Some((below_hash, below_header)) = self.header_by_height(below) else { + // Rows must be contiguous from genesis up to the header tip + // (full-block rows below the body tip — retained even under + // pruning — and zakura rows above it), so a missing row below + // a stored one is a gap, not the end of history. + return Err(StoreIncoherentError::Gap { + height, + missing: below, + }); + }; - context + if header.previous_block_hash != below_hash { + return Err(StoreIncoherentError::BrokenLinkage { + height, + expected_parent: header.previous_block_hash, + actual_below: below_hash, + }); + } + + height = below; + hash = below_hash; + header = below_header; + } } /// Returns header-known, body-missing heights. @@ -2042,17 +2096,26 @@ impl DiskWriteBatch { .or_else(|| (anchor == zebra_db.network().genesis_hash()).then_some(block::Height(0))) .ok_or(CommitHeaderRangeError::UnknownAnchor { anchor })?; - if anchor != zebra_db.network().genesis_hash() - && zebra_db.header_hash(anchor_height) != Some(anchor) - { - return Err(CommitHeaderRangeError::UnknownAnchor { anchor }); + // The hash→height index knows the anchor, so a failed height→hash + // round-trip is a bijection violation in our own store — a local + // storage fault, not an unknown anchor supplied by the caller. + if anchor != zebra_db.network().genesis_hash() { + let stored = zebra_db.header_hash(anchor_height); + if stored != Some(anchor) { + return Err(StoreIncoherentError::BijectionMismatch { + hash: anchor, + height: anchor_height, + stored, + } + .into()); + } } let finalized_height = zebra_db.finalized_tip_height(); let best_header_tip = zebra_db.best_header_tip().map(|(height, _)| height); let checkpoints = zebra_db.network().checkpoint_list(); - let mut recent_headers = zebra_db.recent_header_context(anchor_height); + let mut recent_headers = zebra_db.recent_header_context(anchor_height)?; if recent_headers.is_empty() { if anchor == zebra_db.network().genesis_hash() && anchor_height == block::Height(0) { return Err(CommitHeaderRangeError::MissingGenesisAnchor { anchor }); diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs index bd6f1108d42..9f6f4e314b8 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs @@ -1023,10 +1023,16 @@ fn header_range_commit_rejects_non_current_anchor_hash() { let block2 = mainnet_block(2); let alternate_block2 = alternate_header(stale_anchor, &block2.header, 1); + // A hash→height entry whose height→hash row names a different block is a + // bijection violation in our own indexes, reported as a local storage + // fault rather than an unknown anchor. Either way the stale anchor cannot + // be committed on. let mut batch = DiskWriteBatch::new(); assert!(matches!( batch.prepare_header_range_batch(&state, stale_anchor, &[alternate_block2], &[0]), - Err(CommitHeaderRangeError::UnknownAnchor { anchor }) if anchor == stale_anchor + Err(CommitHeaderRangeError::StoreIncoherent( + crate::error::StoreIncoherentError::BijectionMismatch { hash, height, stored }, + )) if hash == stale_anchor && height == Height(1) && stored == Some(block1.hash()) )); assert_eq!(state.hash(Height(1)), None); @@ -1318,7 +1324,9 @@ fn synthetic_headers_from_state( ) -> Vec> { let network = state.network(); let template = mainnet_block(1); - let mut context = state.recent_header_context(anchor_height); + let mut context = state + .recent_header_context(anchor_height) + .expect("test store is coherent"); let mut previous_hash = anchor_hash; let mut previous_height = anchor_height; let mut nonce_tag = nonce_seed; diff --git a/zebrad/src/commands/start.rs b/zebrad/src/commands/start.rs index 4ec3daeb17f..6326a0af5f9 100644 --- a/zebrad/src/commands/start.rs +++ b/zebrad/src/commands/start.rs @@ -2216,6 +2216,25 @@ mod zakura_header_sync_driver_tests { ); } + #[test] + fn store_incoherent_is_local_header_sync_commit_failure() { + // A range rejected because our own header rows failed a + // linkage/bijection check is a local storage fault; scoring peers for + // it recreates the disconnect-honest-peers failure mode. + let error = zebra_state::CommitHeaderRangeError::StoreIncoherent( + zebra_state::StoreIncoherentError::BrokenLinkage { + height: block::Height(2), + expected_parent: block::Hash([0; 32]), + actual_below: block::Hash([1; 32]), + }, + ); + + assert_eq!( + header_range_commit_failure_kind(&error), + HeaderSyncCommitFailureKind::Local + ); + } + #[test] fn unlinked_range_is_local_header_sync_commit_failure() { // The reactor validates every response's linkage against the requested diff --git a/zebrad/src/commands/start/zakura/header_sync_driver.rs b/zebrad/src/commands/start/zakura/header_sync_driver.rs index a0826087f71..ab8a84165a4 100644 --- a/zebrad/src/commands/start/zakura/header_sync_driver.rs +++ b/zebrad/src/commands/start/zakura/header_sync_driver.rs @@ -1490,6 +1490,12 @@ pub(crate) fn header_range_commit_failure_kind( // store's own linkage check failing means the local anchor/response pairing // went wrong, not that the peer misbehaved. | zebra_state::CommitHeaderRangeError::UnlinkedRange { .. } + // Store incoherence is by definition a local storage fault: the range was + // rejected because our own header rows failed a linkage/bijection check + // while reading validation context, not because the peer's range was shown + // invalid. Scoring peers for it recreates the disconnect-honest-peers + // failure mode from the 2026-07-06 incidents. + | zebra_state::CommitHeaderRangeError::StoreIncoherent(_) | zebra_state::CommitHeaderRangeError::CommitResponseDropped => { HeaderSyncCommitFailureKind::Local } From b893243430f6176159b59514545c03e67b01a7b8 Mon Sep 17 00:00:00 2001 From: roman Date: Mon, 6 Jul 2026 19:48:55 -0600 Subject: [PATCH 22/27] canary: route StoreIncoherent to ContextMismatch (walk-back) on the fleet branch --- zebrad/src/commands/start.rs | 8 +++++--- .../commands/start/zakura/header_sync_driver.rs | 14 +++++++------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/zebrad/src/commands/start.rs b/zebrad/src/commands/start.rs index 6326a0af5f9..7fd96e512d8 100644 --- a/zebrad/src/commands/start.rs +++ b/zebrad/src/commands/start.rs @@ -2217,10 +2217,12 @@ mod zakura_header_sync_driver_tests { } #[test] - fn store_incoherent_is_local_header_sync_commit_failure() { + fn store_incoherent_triggers_walk_back_not_peer_scoring() { // A range rejected because our own header rows failed a // linkage/bijection check is a local storage fault; scoring peers for - // it recreates the disconnect-honest-peers failure mode. + // it recreates the disconnect-honest-peers failure mode. On this + // branch it routes to ContextMismatch so the walk-back re-anchors + // from the store below the damage. let error = zebra_state::CommitHeaderRangeError::StoreIncoherent( zebra_state::StoreIncoherentError::BrokenLinkage { height: block::Height(2), @@ -2231,7 +2233,7 @@ mod zakura_header_sync_driver_tests { assert_eq!( header_range_commit_failure_kind(&error), - HeaderSyncCommitFailureKind::Local + HeaderSyncCommitFailureKind::ContextMismatch ); } diff --git a/zebrad/src/commands/start/zakura/header_sync_driver.rs b/zebrad/src/commands/start/zakura/header_sync_driver.rs index ab8a84165a4..a6e10528497 100644 --- a/zebrad/src/commands/start/zakura/header_sync_driver.rs +++ b/zebrad/src/commands/start/zakura/header_sync_driver.rs @@ -1490,12 +1490,6 @@ pub(crate) fn header_range_commit_failure_kind( // store's own linkage check failing means the local anchor/response pairing // went wrong, not that the peer misbehaved. | zebra_state::CommitHeaderRangeError::UnlinkedRange { .. } - // Store incoherence is by definition a local storage fault: the range was - // rejected because our own header rows failed a linkage/bijection check - // while reading validation context, not because the peer's range was shown - // invalid. Scoring peers for it recreates the disconnect-honest-peers - // failure mode from the 2026-07-06 incidents. - | zebra_state::CommitHeaderRangeError::StoreIncoherent(_) | zebra_state::CommitHeaderRangeError::CommitResponseDropped => { HeaderSyncCommitFailureKind::Local } @@ -1525,7 +1519,13 @@ pub(crate) fn header_range_commit_failure_kind( // (`QueryReanchorTarget`), which is exactly the self-correction this // state needs. zebra_state::CommitHeaderRangeError::ValidateContextError(_) - | zebra_state::CommitHeaderRangeError::UnknownAnchor { .. } => { + | zebra_state::CommitHeaderRangeError::UnknownAnchor { .. } + // Store incoherence is a local storage fault (our own header rows failed + // a linkage/bijection check while reading validation context — never the + // peer's doing, so never scored), and the walk-back is its cure: it + // re-anchors from the store below the damage and re-commits through the + // fork point, while the startup/post-reorg audits repair the rows. + | zebra_state::CommitHeaderRangeError::StoreIncoherent(_) => { HeaderSyncCommitFailureKind::ContextMismatch } _ => HeaderSyncCommitFailureKind::Local, From 1e0416822b23268eb40f46a0a155aaf3d224ac7a Mon Sep 17 00:00:00 2001 From: roman Date: Mon, 6 Jul 2026 19:30:44 -0600 Subject: [PATCH 23/27] fix(state): own zakura header-store mutations with a single suffix-replacement primitive --- CHANGELOG.md | 9 + .../src/service/finalized_state/disk_db.rs | 19 ++ .../service/finalized_state/zebra_db/block.rs | 287 ++++++++++-------- .../zebra_db/block/canonical_suffix.rs | 263 ++++++++++++++++ .../zebra_db/block/startup_audit.rs | 8 +- zebra-state/src/service/write.rs | 8 +- 6 files changed, 454 insertions(+), 140 deletions(-) create mode 100644 zebra-state/src/service/finalized_state/zebra_db/block/canonical_suffix.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 505f8444bc3..f2ba10ff913 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,15 @@ and this project adheres to [Semantic Versioning](https://semver.org). - Verified-commitment-trees fast sync is now enabled by default when checkpoint sync is enabled. Operators can keep checkpoint sync but opt out of the new path by setting `consensus.vct_fast_sync = false`. +- All Zakura header-store writers (header-range commits, best-chain seeds, and + body-commit releases) now change chain membership through a single + suffix-replacement primitive that deletes everything above the fork point by + scanning the actual on-disk rows of all five column families — so rows can no + longer be stranded by bounded delete loops — and verifies linkage as a hard + precondition. Any batch that replaces a header suffix (a header reorg) is + followed by the store coherence audit, repairing and reporting + (`state.zakura.header_store.incoherent`) any residual violation at the moment + it happens instead of leaving a latent on-disk fault. - Refreshed the default Testnet Zakura bootstrap peer identities (`DEFAULT_TESTNET_ZAKURA_BOOTSTRAP_PEERS`) after the Testnet fleet's iroh node keys were rotated. The previous hardcoded node IDs were stale, so a fresh node diff --git a/zebra-state/src/service/finalized_state/disk_db.rs b/zebra-state/src/service/finalized_state/disk_db.rs index 7af2be19519..34ffe76d9a3 100644 --- a/zebra-state/src/service/finalized_state/disk_db.rs +++ b/zebra-state/src/service/finalized_state/disk_db.rs @@ -122,6 +122,11 @@ pub struct DiskDb { pub struct DiskWriteBatch { /// The inner RocksDB write batch. batch: rocksdb::WriteBatch, + + /// Rows deleted by zakura header-store suffix replacements staged in this + /// batch. Nonzero means the batch performs a header reorg; the write site + /// uses this to run the post-reorg store audit after committing. + zakura_replaced_rows: usize, } impl Debug for DiskWriteBatch { @@ -548,8 +553,22 @@ impl DiskWriteBatch { pub fn new() -> Self { DiskWriteBatch { batch: rocksdb::WriteBatch::default(), + zakura_replaced_rows: 0, } } + + /// Records that a zakura suffix replacement staged in this batch deleted + /// `deleted_rows` rows. + pub(crate) fn note_zakura_suffix_replacement(&mut self, deleted_rows: usize) { + self.zakura_replaced_rows += deleted_rows; + } + + /// Returns the number of rows deleted by zakura suffix replacements + /// staged in this batch. Nonzero means this batch performs a header + /// reorg, and the write site should audit the store after committing it. + pub(crate) fn zakura_suffix_replaced_rows(&self) -> usize { + self.zakura_replaced_rows + } } impl DiskDb { diff --git a/zebra-state/src/service/finalized_state/zebra_db/block.rs b/zebra-state/src/service/finalized_state/zebra_db/block.rs index 6662b559df7..8b7a9fb4b91 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block.rs @@ -41,7 +41,7 @@ use crate::{ response::HeaderRangeCommitOutcome, service::check, service::finalized_state::{ - disk_db::{DiskDb, DiskWriteBatch, ReadDisk, WriteDisk}, + disk_db::{DiskWriteBatch, ReadDisk, WriteDisk}, disk_format::{ block::TransactionLocation, shielded::CommitmentRootsByHeight, @@ -58,8 +58,11 @@ use crate::{ #[cfg(feature = "indexer")] use crate::request::Spend; +mod canonical_suffix; mod startup_audit; +pub(crate) use canonical_suffix::CanonicalHeaderRow; + #[cfg(test)] mod tests; @@ -69,7 +72,13 @@ const ZAKURA_HEADER_BY_HEIGHT: &str = "zakura_header_by_height"; pub const ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT: &str = "zakura_header_body_size_by_height"; #[derive(Copy, Clone, Debug, Eq, PartialEq)] -struct AdvertisedBodySize(u32); +pub(crate) struct AdvertisedBodySize(u32); + +/// Builds an [`AdvertisedBodySize`] for tests that plant raw rows. +#[cfg(test)] +pub(crate) fn test_body_size(size: u32) -> AdvertisedBodySize { + AdvertisedBodySize(size) +} impl AdvertisedBodySize { fn new(size: u32) -> Option { @@ -1283,12 +1292,15 @@ impl ZebraDb { // Track batch commit latency for observability let batch_start = std::time::Instant::now(); + let zakura_replaced_rows = batch.zakura_suffix_replaced_rows(); self.db .write(batch) .expect("unexpected rocksdb error while writing block"); metrics::histogram!("zebra.state.rocksdb.batch_commit.duration_seconds") .record(batch_start.elapsed().as_secs_f64()); + self.audit_zakura_header_store_after_reorg(zakura_replaced_rows); + tracing::trace!(?source, "committed block from"); Ok(finalized.hash) @@ -1325,12 +1337,39 @@ impl ZebraDb { block: &Arc, ) -> Result<(), CommitHeaderRangeError> { let mut batch = DiskWriteBatch::new(); - batch.prepare_zakura_header_from_committed_block(&self.db, height, block)?; + batch.prepare_zakura_header_from_committed_block(self, height, block)?; + let zakura_replaced_rows = batch.zakura_suffix_replaced_rows(); self.db .write(batch) .map_err(|error| CommitHeaderRangeError::StorageWriteError { error: error.to_string(), - }) + })?; + + self.audit_zakura_header_store_after_reorg(zakura_replaced_rows); + + Ok(()) + } + + /// Audits the zakura header store after a batch that replaced a header + /// suffix (`replaced_rows > 0`), repairing any violation. + /// + /// This is the after-every-reorg-batch half of the store audit (the + /// startup half runs in [`ZebraDb::new`]): any residual writer bug in + /// this class becomes a repaired, observable transient at the moment it + /// happens instead of a latent on-disk fault. Suffix replacements are + /// rare (real chain forks), so the `O(header frontier)` audit cost is + /// not on the steady-state commit path. + pub(crate) fn audit_zakura_header_store_after_reorg(&self, replaced_rows: usize) { + if replaced_rows == 0 { + return; + } + + if let Err(error) = self.audit_and_repair_zakura_header_store() { + tracing::warn!( + ?error, + "post-reorg zakura header store audit failed to write its repair" + ); + } } } @@ -1796,7 +1835,7 @@ impl DiskWriteBatch { // heights with no committed body (the frontier above the body tip). // This is unconditional so it also cleans up rows left by a prior run // that had `enable_zakura_header_seed_from_committed_blocks` enabled. - self.prepare_zakura_header_release_from_committed_block(db, *height, block)?; + self.prepare_zakura_header_release_from_committed_block(zebra_db, *height, block)?; // Index the block header, hash, and height. This also restores the // verified full block row after any provisional cleanup above. @@ -1856,8 +1895,10 @@ impl DiskWriteBatch { /// committed full block. /// /// Full block verification is authoritative for the stored body. If a - /// provisional Zakura header at this height differs, replace it with the - /// block-derived header and drop stale provisional descendants. + /// provisional Zakura header at this height differs, the seeded block is + /// the new best chain at its height: the suffix from this height upward + /// is replaced through [`Self::set_canonical_suffix`], dropping stale + /// provisional descendants of the displaced row. /// /// A block whose parent hash does not match the stored header row below /// `height` is skipped without error: writing it would leave a gap or @@ -1867,23 +1908,17 @@ impl DiskWriteBatch { #[allow(clippy::unwrap_in_result)] pub fn prepare_zakura_header_from_committed_block( &mut self, - db: &DiskDb, + zebra_db: &ZebraDb, height: block::Height, block: &Arc, ) -> Result<(), CommitHeaderRangeError> { - let zakura_header_by_height = db.cf_handle(ZAKURA_HEADER_BY_HEIGHT).unwrap(); - let zakura_hash_by_height = db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); - let zakura_height_by_hash = db.cf_handle(ZAKURA_HEADER_HEIGHT_BY_HASH).unwrap(); - let zakura_body_size_by_height = db.cf_handle(ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT).unwrap(); - let zakura_roots_by_height = db.cf_handle(COMMITMENT_ROOTS_BY_HEIGHT).unwrap(); - let tx_by_loc = db.cf_handle("tx_by_loc").unwrap(); - let hash = block.hash(); - let existing_zakura_header: Option> = - db.zs_get(&zakura_header_by_height, &height); - if existing_zakura_header.as_ref() == Some(&block.header) - && db.zs_get::<_, _, block::Hash>(&zakura_hash_by_height, &height) == Some(hash) + // An identical stored row needs no reconciliation, and the frontier + // above it survives (the common case: header sync ran ahead of the + // non-finalized best chain). + if zebra_db.zakura_header(height).as_ref() == Some(&block.header) + && zebra_db.zakura_header_hash(height) == Some(hash) { return Ok(()); } @@ -1891,11 +1926,10 @@ impl DiskWriteBatch { // Seeds can jump to a non-finalized best tip whose parent is not the // stored row below it. Refuse those seeds so the header store stays // linked; header-range sync will later deliver the missing rows. - let hash_by_height = db.cf_handle("hash_by_height").unwrap(); - let parent_hash: Option = height.previous().ok().and_then(|parent_height| { - db.zs_get(&hash_by_height, &parent_height) - .or_else(|| db.zs_get(&zakura_hash_by_height, &parent_height)) - }); + let parent_hash: Option = height + .previous() + .ok() + .and_then(|parent_height| zebra_db.header_hash(parent_height)); if parent_hash != Some(block.header.previous_block_hash) { tracing::debug!( ?height, @@ -1907,50 +1941,19 @@ impl DiskWriteBatch { return Ok(()); } - if existing_zakura_header.is_some_and(|existing_header| existing_header != block.header) { - let best_header_tip: Option<(block::Height, block::Hash)> = - db.zs_last_key_value(&zakura_hash_by_height); - - if let Some((best_header_tip, _)) = best_header_tip { - for old_height in height.0..=best_header_tip.0 { - let old_height = block::Height(old_height); - - if old_height != height - && db.zs_contains( - &tx_by_loc, - &TransactionLocation::min_for_height(old_height), - ) - { - return Err(CommitHeaderRangeError::ConflictingFullBlockHeader { - height: old_height, - }); - } + let fork_height = height + .previous() + .expect("the linkage refusal above required a stored parent row below this height"); - if let Some(old_hash) = - db.zs_get::<_, _, block::Hash>(&zakura_hash_by_height, &old_height) - { - self.zs_delete(&zakura_height_by_hash, old_hash); - } - - self.zs_delete(&zakura_hash_by_height, old_height); - self.zs_delete(&zakura_header_by_height, old_height); - self.zs_delete(&zakura_body_size_by_height, old_height); - self.zs_delete(&zakura_roots_by_height, old_height); - } - } - } else if let Some(old_hash) = - db.zs_get::<_, _, block::Hash>(&zakura_hash_by_height, &height) - { - if old_hash != hash { - self.zs_delete(&zakura_height_by_hash, old_hash); - } - } - - self.zs_insert(&zakura_header_by_height, height, &block.header); - self.zs_insert(&zakura_hash_by_height, height, hash); - self.zs_insert(&zakura_height_by_hash, hash, height); - - Ok(()) + self.set_canonical_suffix( + zebra_db, + (fork_height, block.header.previous_block_hash), + &[CanonicalHeaderRow { + header: block.header.clone(), + advertised_body_size: None, + roots: None, + }], + ) } /// Prepare a database batch that releases the Zakura header store entry for a @@ -1969,16 +1972,16 @@ impl DiskWriteBatch { #[allow(clippy::unwrap_in_result)] pub fn prepare_zakura_header_release_from_committed_block( &mut self, - db: &DiskDb, + zebra_db: &ZebraDb, height: block::Height, block: &Arc, ) -> Result<(), CommitHeaderRangeError> { + let db = &zebra_db.db; let zakura_header_by_height = db.cf_handle(ZAKURA_HEADER_BY_HEIGHT).unwrap(); let zakura_hash_by_height = db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); let zakura_height_by_hash = db.cf_handle(ZAKURA_HEADER_HEIGHT_BY_HASH).unwrap(); let zakura_body_size_by_height = db.cf_handle(ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT).unwrap(); let zakura_roots_by_height = db.cf_handle(COMMITMENT_ROOTS_BY_HEIGHT).unwrap(); - let tx_by_loc = db.cf_handle("tx_by_loc").unwrap(); let existing_zakura_header: Option> = db.zs_get(&zakura_header_by_height, &height); @@ -1989,36 +1992,15 @@ impl DiskWriteBatch { return Ok(()); } - // A committed block whose header conflicts with the provisional chain at - // this height invalidates the provisional descendants built on top of it. - // Drop them, but never overwrite a height that already has a committed body. + // A committed block whose header conflicts with the provisional chain + // at this height invalidates the provisional descendants built on top + // of it: truncate the suffix above this height through the owner + // primitive (its committed-body interlock refuses to touch any height + // that already has a body, as the old bounded loop did). The empty + // replacement is anchored at this block, whose full-block row is + // staged in this same batch. if existing_zakura_header.is_some_and(|existing_header| existing_header != block.header) { - let zakura_tip: Option<(block::Height, block::Hash)> = - db.zs_last_key_value(&zakura_hash_by_height); - - if let Some((zakura_tip, _)) = zakura_tip { - for descendant in (height.0 + 1)..=zakura_tip.0 { - let descendant = block::Height(descendant); - - if db.zs_contains(&tx_by_loc, &TransactionLocation::min_for_height(descendant)) - { - return Err(CommitHeaderRangeError::ConflictingFullBlockHeader { - height: descendant, - }); - } - - if let Some(old_hash) = - db.zs_get::<_, _, block::Hash>(&zakura_hash_by_height, &descendant) - { - self.zs_delete(&zakura_height_by_hash, old_hash); - } - - self.zs_delete(&zakura_hash_by_height, descendant); - self.zs_delete(&zakura_header_by_height, descendant); - self.zs_delete(&zakura_body_size_by_height, descendant); - self.zs_delete(&zakura_roots_by_height, descendant); - } - } + self.set_canonical_suffix(zebra_db, (height, block.hash()), &[])?; } // Release the provisional row at this height. @@ -2082,9 +2064,6 @@ impl DiskWriteBatch { }); } - let header_by_height = zebra_db.db.cf_handle(ZAKURA_HEADER_BY_HEIGHT).unwrap(); - let hash_by_height = zebra_db.db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); - let height_by_hash = zebra_db.db.cf_handle(ZAKURA_HEADER_HEIGHT_BY_HASH).unwrap(); let body_size_by_height = zebra_db .db .cf_handle(ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT) @@ -2124,6 +2103,7 @@ impl DiskWriteBatch { } let mut first_conflicting_height = None; + let mut first_missing_height = None; let mut validated_headers = Vec::with_capacity(headers.len()); // Each header must link to the anchor (for the first header) or to its @@ -2190,6 +2170,8 @@ impl DiskWriteBatch { first_conflicting_height.get_or_insert(height); } + } else { + first_missing_height.get_or_insert(height); } check::header_is_valid_for_recent_chain( @@ -2251,29 +2233,27 @@ impl DiskWriteBatch { } } - if let (Some(first_conflicting_height), Some(best_header_tip)) = - (first_conflicting_height, best_header_tip) - { - for height in first_conflicting_height.0..=best_header_tip.0 { - let height = block::Height(height); - - if zebra_db.contains_body_at_height(height) { - return Err(CommitHeaderRangeError::ConflictingFullBlockHeader { height }); - } - - if let Some(old_hash) = zebra_db.zakura_header_hash(height) { - self.zs_delete(&height_by_hash, old_hash); - } + // The suffix replacement starts at the first conflicting height, or — + // for a pure extension — at the first height without a stored row. + // Rows below it match the stored chain (a conflict below a missing + // row is impossible: rows are contiguous up to the header tip), so + // chain membership only changes from `replace_from` upward. `None` + // means the whole range re-delivered rows the store already has. + let replace_from = match (first_conflicting_height, first_missing_height) { + (Some(conflict), _) => Some(conflict), + (None, missing) => missing, + }; - self.zs_delete(&hash_by_height, height); - self.zs_delete(&header_by_height, height); - self.zs_delete(&body_size_by_height, height); - self.zs_delete(&roots_by_height, height); + // Refresh the advisory rows of re-delivered matching headers: merge + // the body-size hints and re-stage the provisional roots. These + // writes never change which header is stored at a height, so they + // stay outside the suffix-replacement primitive. + for (index, (height, _hash, _header, body_size)) in validated_headers.iter().enumerate() { + let height = *height; + if replace_from.is_some_and(|replace_from| height >= replace_from) { + break; } - } - for (index, (height, hash, header, body_size)) in validated_headers.into_iter().enumerate() - { // Finalized block heights already have authoritative block rows and // verified roots, even when pruning has removed their transactions. // Re-delivered headers must not recreate provisional zakura rows @@ -2282,25 +2262,17 @@ impl DiskWriteBatch { continue; } - let same_header = zebra_db.zakura_header_hash(height) == Some(hash); - let advertised_body_size = match ( - same_header, + let merged_body_size = match ( zebra_db.advertised_body_size(height), - AdvertisedBodySize::new(body_size).map(AdvertisedBodySize::get), + AdvertisedBodySize::new(*body_size).map(AdvertisedBodySize::get), ) { - (true, existing, Some(new)) => Some(existing.unwrap_or(0).max(new)), - (true, existing, None) => existing, - (false, _existing, new) => new, + (existing, Some(new)) => Some(existing.unwrap_or(0).max(new)), + (existing, None) => existing, }; - - self.zs_insert(&header_by_height, height, header); - self.zs_insert(&hash_by_height, height, hash); - self.zs_insert(&height_by_hash, hash, height); - if let Some(body_size) = advertised_body_size.and_then(AdvertisedBodySize::new) { + if let Some(body_size) = merged_body_size.and_then(AdvertisedBodySize::new) { self.zs_insert(&body_size_by_height, height, body_size); - } else { - self.zs_delete(&body_size_by_height, height); } + let roots = &tree_aux_roots[index]; self.zs_insert( &roots_by_height, @@ -2317,6 +2289,51 @@ impl DiskWriteBatch { ); } + + // Chain membership changes only through the suffix-replacement + // primitive: total deletion above the fork point in all five column + // families, then the new rows. Rows above the incoming range's end + // that belonged to the replaced branch are deleted with it. + if let Some(replace_from) = replace_from { + let fork_height = replace_from + .previous() + .map_err(|_| CommitHeaderRangeError::HeightOverflow)?; + let fork_hash = if fork_height == anchor_height { + anchor + } else { + // The fork row is inside the incoming range and matched the + // store during validation. + let fork_index = usize::try_from(fork_height.0 - anchor_height.0 - 1) + .expect("fork height is above the anchor"); + validated_headers[fork_index].1 + }; + + let new_rows: Vec = validated_headers + .iter() + .enumerate() + .filter(|(_, (height, ..))| *height >= replace_from) + .map(|(index, (_height, _hash, header, body_size))| { + let roots = &tree_aux_roots[index]; + CanonicalHeaderRow { + header: (*header).clone(), + advertised_body_size: AdvertisedBodySize::new(*body_size) + .map(AdvertisedBodySize::get), + roots: Some(CommitmentRootsByHeight { + sapling: roots.sapling_root, + orchard: roots.orchard_root, + ironwood: roots.ironwood_root, + sapling_tx: roots.sapling_tx, + orchard_tx: roots.orchard_tx, + ironwood_tx: roots.ironwood_tx, + auth_data_root: roots.auth_data_root, + }), + } + }) + .collect(); + + self.set_canonical_suffix(zebra_db, (fork_height, fork_hash), &new_rows)?; + } + Ok(HeaderRangeCommitOutcome { tip_hash: block::Hash::from(&**headers.last().expect("headers is non-empty")), reorged_at: first_conflicting_height, diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/canonical_suffix.rs b/zebra-state/src/service/finalized_state/zebra_db/block/canonical_suffix.rs new file mode 100644 index 00000000000..9300af7f245 --- /dev/null +++ b/zebra-state/src/service/finalized_state/zebra_db/block/canonical_suffix.rs @@ -0,0 +1,263 @@ +//! The single owner of zakura header-store chain membership: +//! [`DiskWriteBatch::set_canonical_suffix`] (REORG_PLAN Pillar 1). +//! +//! The zakura header store is five column families acting as a replicated +//! view of the canonical header chain above the finalized tip. Historically +//! it was mutated by four independent writers, each with its own bounded +//! delete loop; every proven corruption class (unlinked anchors, re-inserts +//! below the body tip, unlinked seeds, stranded suffixes) came from one of +//! those writers preserving the store invariants only under assumptions +//! about the others. +//! +//! This module replaces the per-writer delete/insert logic with one +//! primitive: *replace the suffix above a fork point*. Deletion is +//! total-above-fork by construction — it iterates the actual on-disk rows of +//! every column family, never a cached or computed tip — so stranding is +//! impossible. Linkage is a hard precondition, not a validation step callers +//! can forget. Committed heights are refused for deletes and inserts alike. +//! +//! Callers stay responsible for *policy*: contextual validation, checkpoint +//! conflicts, the cumulative-work gate, and reorg-depth limits all happen +//! before the primitive is invoked. The primitive owns *mechanism*: which +//! rows exist after a suffix replacement, in all five column families, in +//! one atomic batch. + +use std::sync::Arc; + +use zebra_chain::block::{self, Height}; + +use super::{ + AdvertisedBodySize, ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT, ZAKURA_HEADER_BY_HEIGHT, + ZAKURA_HEADER_HASH_BY_HEIGHT, ZAKURA_HEADER_HEIGHT_BY_HASH, +}; +use crate::{ + error::{CommitHeaderRangeError, StoreIncoherentError}, + service::finalized_state::{ + disk_db::{DiskWriteBatch, WriteDisk}, + disk_format::shielded::CommitmentRootsByHeight, + zebra_db::ZebraDb, + COMMITMENT_ROOTS_BY_HEIGHT, + }, +}; + +/// One header row of a new canonical suffix. +/// +/// The row's height is implied by its position: `fork_point + 1 + index`. +/// Its hash is computed from the header inside the primitive, so a caller +/// cannot stage a hash that disagrees with the stored header. +#[derive(Clone, Debug)] +pub(crate) struct CanonicalHeaderRow { + /// The header to store. + pub header: Arc, + + /// The advisory body-size hint, if known. Not consensus data; `None` + /// stores no row (and removes any stale hint left at this height). + pub advertised_body_size: Option, + + /// Provisional commitment roots for this height, if the caller has them. + /// `None` stores no row (the seed path has no roots for its block). + pub roots: Option, +} + +impl DiskWriteBatch { + /// Replaces the zakura header-store suffix above `fork_point` with + /// `new_rows`, in this batch. + /// + /// This is the **only** way any path may change which headers the zakura + /// store considers canonical above a height. In one atomic batch it: + /// + /// 1. deletes every row strictly above `fork_point` in all five header + /// column families — iterating each family's actual on-disk rows, so + /// stranded rows above gaps are removed too — including every + /// `height_by_hash` entry pointing above the fork (displaced and + /// already-orphaned entries alike), then + /// 2. writes `new_rows` at consecutive heights starting at + /// `fork_point + 1`. + /// + /// # Preconditions (hard errors, nothing staged on failure) + /// + /// - `fork_point` must be at or above the finalized tip + /// ([`CommitHeaderRangeError::ImmutableConflict`]): committed heights + /// are immutable through this primitive, which also guarantees no + /// insert lands at a committed height (the re-delivery bug shape). + /// - When `new_rows` is non-empty, the stored header row at + /// `fork_point.0` must be `fork_point.1` + /// ([`StoreIncoherentError::BijectionMismatch`]): the caller's fork + /// decision must describe the store being mutated. + /// - `new_rows[0]` must link to `fork_point.1` and every row to its + /// predecessor ([`CommitHeaderRangeError::UnlinkedRange`]): linkage is + /// structural here, not a caller obligation. + /// - No height above `fork_point` may hold a committed full block body + /// ([`CommitHeaderRangeError::ConflictingFullBlockHeader`]). Bodies + /// above the finalized tip cannot exist, so this is a safety interlock: + /// reaching it means the caller's orchestration is buggy, and the + /// switch aborts loudly instead of deleting a body's header row. + /// + /// `new_rows` may be empty: that truncates the store to `fork_point` + /// (the body-commit path drops conflicting provisional descendants this + /// way). An empty replacement skips the fork-row check, because the + /// caller may be staging a new full-block row at the fork height in this + /// same batch, which reads through `zebra_db` cannot see yet. + #[allow(clippy::unwrap_in_result)] + pub(crate) fn set_canonical_suffix( + &mut self, + zebra_db: &ZebraDb, + fork_point: (Height, block::Hash), + new_rows: &[CanonicalHeaderRow], + ) -> Result<(), CommitHeaderRangeError> { + let (fork_height, fork_hash) = fork_point; + + // Committed heights are immutable: the finalized tip is the highest + // committed block, so requiring the fork at or above it keeps every + // delete *and* insert strictly above committed history (valid on + // pruned stores too, where bodies are absent but heights committed). + if let Some(finalized_tip_height) = zebra_db.finalized_tip_height() { + if fork_height < finalized_tip_height { + return Err(CommitHeaderRangeError::ImmutableConflict { + height: fork_height + .next() + .map_err(|_| CommitHeaderRangeError::HeightOverflow)?, + }); + } + } + + // The fork row named by the caller must be the row on disk: a + // mismatch means the fork decision describes a different store state + // than the one being mutated. + if !new_rows.is_empty() { + let stored = zebra_db + .header_hash(fork_height) + .or_else(|| (fork_hash == zebra_db.network().genesis_hash()).then_some(fork_hash)); + if stored != Some(fork_hash) { + return Err(StoreIncoherentError::BijectionMismatch { + hash: fork_hash, + height: fork_height, + stored, + } + .into()); + } + } + + // Structural linkage: the new suffix must chain from the fork row. + let mut expected_parent = fork_hash; + let mut hashes = Vec::with_capacity(new_rows.len()); + for (index, row) in new_rows.iter().enumerate() { + let offset = + u32::try_from(index + 1).map_err(|_| CommitHeaderRangeError::HeightOverflow)?; + let height = + (fork_height + i64::from(offset)).ok_or(CommitHeaderRangeError::HeightOverflow)?; + + if row.header.previous_block_hash != expected_parent { + return Err(CommitHeaderRangeError::UnlinkedRange { + height, + expected_parent, + actual_parent: row.header.previous_block_hash, + }); + } + + let hash = block::Hash::from(&*row.header); + expected_parent = hash; + hashes.push((height, hash)); + } + + let header_cf = zebra_db.db.cf_handle(ZAKURA_HEADER_BY_HEIGHT).unwrap(); + let hash_cf = zebra_db.db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); + let height_by_hash_cf = zebra_db.db.cf_handle(ZAKURA_HEADER_HEIGHT_BY_HASH).unwrap(); + let body_size_cf = zebra_db + .db + .cf_handle(ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT) + .unwrap(); + let roots_cf = zebra_db.db.cf_handle(COMMITMENT_ROOTS_BY_HEIGHT).unwrap(); + + let Ok(delete_from) = fork_height.next() else { + // The fork is at the maximum height: nothing can exist above it, + // and the linkage loop already rejected any new rows. + return Ok(()); + }; + + // Total-above-fork deletion: walk the actual on-disk rows of every + // column family from the fork upward. No cached or computed tip + // bounds the loops, so rows stranded above gaps are deleted too. + // + // The committed-body interlock runs on the stored hash rows *and* + // the insert heights before anything is staged. + let stored_hash_rows: Vec<(Height, block::Hash)> = zebra_db + .db + .zs_forward_range_iter(&hash_cf, delete_from..) + .collect(); + for &(height, _) in &stored_hash_rows { + if zebra_db.contains_body_at_height(height) { + return Err(CommitHeaderRangeError::ConflictingFullBlockHeader { height }); + } + } + for &(height, _) in &hashes { + if zebra_db.contains_body_at_height(height) { + return Err(CommitHeaderRangeError::ConflictingFullBlockHeader { height }); + } + } + + let mut deleted_rows = 0; + for (height, _displaced_hash) in stored_hash_rows { + self.zs_delete(&hash_cf, height); + deleted_rows += 1; + } + + // The reverse index is hash-keyed, so deriving its deletions from the + // hash rows would miss entries already orphaned by earlier damage (a + // deleted hash row leaves its reverse entry behind). Scan the whole + // reverse index — it is frontier-sized — and delete every entry + // pointing above the fork, displaced and orphaned alike. + for (hash, points_at) in zebra_db + .db + .zs_forward_range_iter::<_, block::Hash, Height, _>(&height_by_hash_cf, ..) + { + if points_at > fork_height { + self.zs_delete(&height_by_hash_cf, hash); + deleted_rows += 1; + } + } + for (height, _) in zebra_db + .db + .zs_forward_range_iter::<_, Height, Arc, _>(&header_cf, delete_from..) + { + self.zs_delete(&header_cf, height); + deleted_rows += 1; + } + for (height, _) in zebra_db + .db + .zs_forward_range_iter::<_, Height, AdvertisedBodySize, _>(&body_size_cf, delete_from..) + { + self.zs_delete(&body_size_cf, height); + deleted_rows += 1; + } + for (height, _) in zebra_db + .db + .zs_forward_range_iter::<_, Height, CommitmentRootsByHeight, _>( + &roots_cf, + delete_from.., + ) + { + self.zs_delete(&roots_cf, height); + deleted_rows += 1; + } + + // A batch that deletes rows performs a header reorg: the write site + // audits the store after committing it (the Pillar-3 hook). + self.note_zakura_suffix_replacement(deleted_rows); + + // Write the new suffix. + for (row, &(height, hash)) in new_rows.iter().zip(&hashes) { + self.zs_insert(&header_cf, height, &row.header); + self.zs_insert(&hash_cf, height, hash); + self.zs_insert(&height_by_hash_cf, hash, height); + if let Some(body_size) = row.advertised_body_size.and_then(AdvertisedBodySize::new) { + self.zs_insert(&body_size_cf, height, body_size); + } + if let Some(roots) = &row.roots { + self.zs_insert(&roots_cf, height, *roots); + } + } + + Ok(()) + } +} diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/startup_audit.rs b/zebra-state/src/service/finalized_state/zebra_db/block/startup_audit.rs index 994d4de9586..5d355f77d13 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/startup_audit.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/startup_audit.rs @@ -238,11 +238,11 @@ impl ZebraDb { && body_size_heights.is_empty() && provisional_root_heights.is_empty() { - // Log the pass so operators can verify the audit ran on this boot. + // Log the pass so operators can verify the audit ran. tracing::info!( ?finalized_tip, frontier_rows = 0, - "zakura header store passed its startup coherence audit (empty frontier)" + "zakura header store passed its coherence audit (empty frontier)" ); return Ok(None); } @@ -415,7 +415,7 @@ impl ZebraDb { ?finalized_tip, ?last_coherent, frontier_rows = hashes.len(), - "zakura header store passed its startup coherence audit" + "zakura header store passed its coherence audit" ); return Ok(None); } @@ -427,7 +427,7 @@ impl ZebraDb { deleted_rows, violation_count = violations.len(), first_violations = ?&violations[..violations.len().min(LOGGED_VIOLATIONS)], - "zakura header store failed its startup coherence audit; \ + "zakura header store failed its coherence audit; \ truncating to the last coherent height so header sync re-downloads the rest" ); diff --git a/zebra-state/src/service/write.rs b/zebra-state/src/service/write.rs index c5f59ec9630..a9b995588c2 100644 --- a/zebra-state/src/service/write.rs +++ b/zebra-state/src/service/write.rs @@ -146,10 +146,16 @@ fn commit_header_range( &tree_aux_roots, ) .and_then(|outcome| { + let zakura_replaced_rows = batch.zakura_suffix_replaced_rows(); finalized_state .db .write_batch(batch) - .map(|()| outcome) + .map(|()| { + finalized_state + .db + .audit_zakura_header_store_after_reorg(zakura_replaced_rows); + outcome + }) .map_err(|error| { tracing::error!(?error, "failed to write validated header range"); From c4345ff9e8ff7029057c9f291584e01c126c2034 Mon Sep 17 00:00:00 2001 From: roman Date: Mon, 6 Jul 2026 20:03:13 -0600 Subject: [PATCH 24/27] feat(state): sequence branch switches as evaluate, body rollback, then header rewrite (Pillar 1a) --- zebra-state/src/response.rs | 4 + .../service/finalized_state/zebra_db/block.rs | 9 +- .../src/service/non_finalized_state.rs | 47 ++- zebra-state/src/service/tests.rs | 2 + zebra-state/src/service/write.rs | 349 ++++++++++++++++++ .../start/zakura/header_sync_driver.rs | 35 +- 6 files changed, 417 insertions(+), 29 deletions(-) diff --git a/zebra-state/src/response.rs b/zebra-state/src/response.rs index e1f617fb95b..d0e7fe85d90 100644 --- a/zebra-state/src/response.rs +++ b/zebra-state/src/response.rs @@ -36,6 +36,10 @@ pub struct HeaderRangeCommitOutcome { /// conflicting header suffix, when the commit reorged the stored header /// chain. `None` when the range only extended or duplicated stored headers. pub reorged_at: Option, + /// The new branch's hash at `reorged_at`, when the commit reorged the + /// stored header chain. Used by the switch orchestration to recognize the + /// stranded old-branch body suffix without re-reading state. + pub reorged_to_hash: Option, } #[derive(Clone, Debug, PartialEq, Eq)] diff --git a/zebra-state/src/service/finalized_state/zebra_db/block.rs b/zebra-state/src/service/finalized_state/zebra_db/block.rs index 8b7a9fb4b91..612d6e895b3 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block.rs @@ -76,6 +76,7 @@ pub(crate) struct AdvertisedBodySize(u32); /// Builds an [`AdvertisedBodySize`] for tests that plant raw rows. #[cfg(test)] +#[allow(dead_code)] // used by the coherence suite, dropped from canary picks pub(crate) fn test_body_size(size: u32) -> AdvertisedBodySize { AdvertisedBodySize(size) } @@ -2289,7 +2290,6 @@ impl DiskWriteBatch { ); } - // Chain membership changes only through the suffix-replacement // primitive: total deletion above the fork point in all five column // families, then the new rows. Rows above the incoming range's end @@ -2337,6 +2337,13 @@ impl DiskWriteBatch { Ok(HeaderRangeCommitOutcome { tip_hash: block::Hash::from(&**headers.last().expect("headers is non-empty")), reorged_at: first_conflicting_height, + reorged_to_hash: first_conflicting_height.map(|reorged_at| { + validated_headers + .iter() + .find(|(height, ..)| *height == reorged_at) + .expect("the first conflicting height is inside the validated range") + .1 + }), }) } diff --git a/zebra-state/src/service/non_finalized_state.rs b/zebra-state/src/service/non_finalized_state.rs index 2baff78c2be..b602f4589df 100644 --- a/zebra-state/src/service/non_finalized_state.rs +++ b/zebra-state/src/service/non_finalized_state.rs @@ -370,21 +370,25 @@ impl NonFinalizedState { Ok(()) } - /// Invalidate block with hash `block_hash` and all descendants from the non-finalized state. Insert - /// the new chain into the chain_set and discard the previous. + /// Removes the block with hash `block_hash` and all its descendants from + /// the non-finalized state, returning the removed blocks. Inserts the + /// truncated chain into the chain_set and discards the previous. #[allow(clippy::unwrap_in_result)] - pub fn invalidate_block(&mut self, block_hash: Hash) -> Result { + fn remove_block_and_descendants( + &mut self, + block_hash: Hash, + ) -> Result, InvalidateError> { let Some(chain) = self.find_chain(|chain| chain.contains_block_hash(block_hash)) else { return Err(InvalidateError::BlockNotFound(block_hash)); }; - let invalidated_blocks = if chain.non_finalized_root_hash() == block_hash { + let removed_blocks = if chain.non_finalized_root_hash() == block_hash { let chain_tip_hash = chain.non_finalized_tip_hash(); self.chain_set .retain(|chain| chain.non_finalized_tip_hash() != chain_tip_hash); chain.blocks.values().cloned().collect() } else { - let (new_chain, invalidated_blocks) = chain + let (new_chain, removed_blocks) = chain .invalidate_block(block_hash) .expect("already checked that chain contains hash"); @@ -394,9 +398,25 @@ impl NonFinalizedState { chain_set.retain(|c| !c.contains_block_hash(block_hash)) }); - invalidated_blocks + removed_blocks }; + self.update_metrics_for_chains(); + self.update_metrics_bars(); + + Ok(removed_blocks) + } + + /// Invalidate block with hash `block_hash` and all descendants from the non-finalized state. Insert + /// the new chain into the chain_set and discard the previous. + /// + /// The removed blocks are recorded in the invalidated list: they are + /// refused on re-delivery until explicitly reconsidered. For a reorg + /// rollback, use [`Self::rollback_block`] instead. + #[allow(clippy::unwrap_in_result)] + pub fn invalidate_block(&mut self, block_hash: Hash) -> Result { + let invalidated_blocks = self.remove_block_and_descendants(block_hash)?; + // TODO: Allow for invalidating multiple block hashes at a given height (#9552). self.invalidated_blocks.insert( invalidated_blocks @@ -411,9 +431,20 @@ impl NonFinalizedState { self.invalidated_blocks.shift_remove_index(0); } - self.update_metrics_for_chains(); - self.update_metrics_bars(); + Ok(block_hash) + } + /// Rolls back the block with hash `block_hash` and all its descendants + /// from the non-finalized state, without recording them as invalidated. + /// + /// A branch-switch rollback is a chain-selection outcome, not a verdict + /// on the blocks: the rolled-back branch must revalidate and recommit + /// normally the moment it wins the work race again. Recording it in the + /// invalidated list would refuse the re-delivery + /// (`BlockPreviouslyInvalidated`) and strand the node if the winning + /// branch later evaporates. + pub fn rollback_block(&mut self, block_hash: Hash) -> Result { + self.remove_block_and_descendants(block_hash)?; Ok(block_hash) } diff --git a/zebra-state/src/service/tests.rs b/zebra-state/src/service/tests.rs index 675e4ac4001..316ee2ebc46 100644 --- a/zebra-state/src/service/tests.rs +++ b/zebra-state/src/service/tests.rs @@ -541,6 +541,7 @@ async fn header_only_service_requests_preserve_body_boundary() -> std::result::R Response::CommittedHeaderRange(HeaderRangeCommitOutcome { tip_hash: block2_hash, reorged_at: None, + reorged_to_hash: None, }), ); @@ -801,6 +802,7 @@ async fn commit_header_range_completes_while_in_finalized_write_phase( Response::CommittedHeaderRange(HeaderRangeCommitOutcome { tip_hash: block2_hash, reorged_at: None, + reorged_to_hash: None, }) ); diff --git a/zebra-state/src/service/write.rs b/zebra-state/src/service/write.rs index a9b995588c2..1ba12af6386 100644 --- a/zebra-state/src/service/write.rs +++ b/zebra-state/src/service/write.rs @@ -128,8 +128,36 @@ fn update_latest_chain_channels( tip_block_height } +/// Commits a validated header range, orchestrating the branch switch when the +/// range reorgs the stored header chain (REORG_PLAN Pillar 1a). +/// +/// The switch is sequenced so every crash intermediate is coherent: +/// +/// 1. **Evaluate**: the batch preparation validates the whole candidate range +/// in memory (linkage, checkpoints, contextual difficulty, the strictly- +/// greater cumulative-work gate) and decides the fork point, staging disk +/// writes only through the suffix-replacement primitive. A rejected +/// candidate leaves every store untouched. +/// 2. **Body rollback**: if the fork strands a committed non-finalized body +/// suffix (the old branch's blocks above the fork), invalidate it *before* +/// the header rewrite reaches disk, so the header store never names a +/// branch the body store still extends past the fork. Invalidation is a +/// rollback, not a ban: re-delivered old-branch blocks revalidate and +/// recommit normally if that branch later wins again. +/// 3. **Header rewrite**: write the prepared batch (one atomic suffix +/// replacement), then run the post-reorg store audit. +/// +/// A crash between steps 2 and 3 leaves the node "behind" on a coherent +/// store: the non-finalized backup restores or resync re-delivers whichever +/// branch currently wins, and the ordinary sync flow re-runs the switch — +/// recovery is never a special code path. +#[allow(clippy::too_many_arguments)] fn commit_header_range( finalized_state: &FinalizedState, + non_finalized_state: &mut NonFinalizedState, + chain_tip_sender: &mut ChainTipSender, + non_finalized_state_sender: &watch::Sender, + backup_dir_path: Option<&Path>, anchor: block::Hash, headers: Vec>, body_sizes: Vec, @@ -146,6 +174,19 @@ fn commit_header_range( &tree_aux_roots, ) .and_then(|outcome| { + if let (Some(reorged_at), Some(reorged_to_hash)) = + (outcome.reorged_at, outcome.reorged_to_hash) + { + invalidate_stranded_body_suffix( + non_finalized_state, + chain_tip_sender, + non_finalized_state_sender, + backup_dir_path, + reorged_at, + reorged_to_hash, + ); + } + let zakura_replaced_rows = batch.zakura_suffix_replaced_rows(); finalized_state .db @@ -168,6 +209,82 @@ fn commit_header_range( let _ = rsp_tx.send(result); } +/// Rolls back the non-finalized body suffix stranded by a header-chain reorg +/// (step 2 of the switch orchestration in [`commit_header_range`]). +/// +/// `reorged_at` is the first height where the winning header range replaces a +/// conflicting stored header; `reorged_to_hash` is the new branch's hash +/// there. If the best body chain still holds a different block at that +/// height, that block and its descendants are invalidated and the truncated +/// chain is published, so block-gap discovery reanchors at the fork and +/// re-downloads the new branch's bodies. Without this, body sync keeps +/// extending the stale branch above a header store that no longer names it. +fn invalidate_stranded_body_suffix( + non_finalized_state: &mut NonFinalizedState, + chain_tip_sender: &mut ChainTipSender, + non_finalized_state_sender: &watch::Sender, + backup_dir_path: Option<&Path>, + reorged_at: block::Height, + reorged_to_hash: block::Hash, +) { + let Some(old_hash) = non_finalized_state.best_hash(reorged_at) else { + // No committed body at the reorged height; nothing to roll back. + return; + }; + if old_hash == reorged_to_hash { + // The body chain is already on the new branch. + return; + } + + tracing::warn!( + ?reorged_at, + ?old_hash, + new_hash = ?reorged_to_hash, + "header reorg crossed the committed body suffix; rolling back the stranded branch so block sync re-downloads the new one" + ); + + match non_finalized_state.rollback_block(old_hash) { + Ok(hash) => { + metrics::counter!("sync.header.fork_recovery.body_suffix_invalidated").increment(1); + tracing::info!( + ?reorged_at, + ?hash, + "rolled back the stranded body suffix before the header rewrite" + ); + } + Err(error) => { + // Leave the header commit to proceed: the stranded suffix is + // rediscovered on the next reorging commit or at restart, and the + // ordinary flow re-runs this rollback. + tracing::warn!( + ?reorged_at, + ?old_hash, + ?error, + "failed to roll back the stranded body suffix after a header reorg" + ); + return; + } + } + + // Publish the truncated chain so the tip watchers see the rollback. + // Invalidating the non-finalized root can empty the chain set entirely; + // `update_latest_chain_channels` expects a best chain, so skip the + // publish in that case (the tip watch keeps the stale tip until the new + // branch's bodies commit — block sync follows the header store, not the + // tip watch, so re-download still reanchors correctly). + if non_finalized_state.best_chain().is_some() { + update_latest_chain_channels( + non_finalized_state, + chain_tip_sender, + non_finalized_state_sender, + backup_dir_path, + ); + } else { + let _ = non_finalized_state_sender.send(non_finalized_state.clone()); + let _ = chain_tip_sender; + } +} + /// A worker task that reads, validates, and writes blocks to the /// `finalized_state` or `non_finalized_state`. struct WriteBlockWorkerTask { @@ -358,6 +475,10 @@ impl WriteBlockWorkerTask { }) => { commit_header_range( finalized_state, + non_finalized_state, + chain_tip_sender, + non_finalized_state_sender, + backup_dir_path.as_deref(), anchor, headers, body_sizes, @@ -551,6 +672,10 @@ impl WriteBlockWorkerTask { } => { commit_header_range( finalized_state, + non_finalized_state, + chain_tip_sender, + non_finalized_state_sender, + backup_dir_path.as_deref(), anchor, headers, body_sizes, @@ -838,4 +963,228 @@ mod tests { vec![(best_height, best_block.hash(), best_block.header.clone())], ); } + + /// Builds a three-block non-finalized chain plus the channel plumbing the + /// switch orchestration needs. + fn orchestration_fixture( + network: &Network, + ) -> ( + NonFinalizedState, + FinalizedState, + Vec>, + crate::service::ChainTipSender, + crate::LatestChainTip, + crate::ChainTipChange, + tokio::sync::watch::Sender, + tokio::sync::watch::Receiver, + ) { + let block1: Arc = Arc::new( + network + .test_block(653599, 583999) + .expect("test block exists"), + ); + let block2 = block1.make_fake_child().set_work(10); + let block3 = block2.make_fake_child().set_work(1); + + let mut non_finalized_state = NonFinalizedState::new(network); + let finalized_state = FinalizedState::new( + &Config::ephemeral(), + network, + #[cfg(feature = "elasticsearch")] + false, + ); + finalized_state.set_finalized_value_pool(ValueBalance::fake_populated_pool()); + + non_finalized_state + .commit_new_chain(block1.clone().prepare(), &finalized_state) + .expect("chain root commits"); + non_finalized_state + .commit_block(block2.clone().prepare(), &finalized_state) + .expect("child commits"); + non_finalized_state + .commit_block(block3.clone().prepare(), &finalized_state) + .expect("grandchild commits"); + + let (chain_tip_sender, latest_chain_tip, chain_tip_change) = + crate::service::ChainTipSender::new(None, network); + let (nf_sender, nf_receiver) = tokio::sync::watch::channel(NonFinalizedState::new(network)); + + ( + non_finalized_state, + finalized_state, + vec![block1, block2, block3], + chain_tip_sender, + latest_chain_tip, + chain_tip_change, + nf_sender, + nf_receiver, + ) + } + + /// A stranded suffix is rolled back from the reorged height up, keeping + /// the shared prefix, and the truncated chain is published. + #[test] + fn stranded_body_suffix_rolls_back_from_the_fork() { + let _init_guard = zebra_test::init(); + let network = Network::Mainnet; + let ( + mut non_finalized_state, + _finalized_state, + blocks, + mut chain_tip_sender, + _latest_chain_tip, + _chain_tip_change, + nf_sender, + nf_receiver, + ) = orchestration_fixture(&network); + + let reorged_at = blocks[1].coinbase_height().expect("fake child has height"); + super::invalidate_stranded_body_suffix( + &mut non_finalized_state, + &mut chain_tip_sender, + &nf_sender, + None, + reorged_at, + zebra_chain::block::Hash([0xAB; 32]), + ); + + let best_chain = non_finalized_state + .best_chain() + .expect("the shared prefix survives"); + assert!(best_chain.contains_block_hash(blocks[0].hash())); + assert!(!best_chain.contains_block_hash(blocks[1].hash())); + assert!(!best_chain.contains_block_hash(blocks[2].hash())); + + // The truncated chain was published to the watch channel. + assert!(nf_receiver + .borrow() + .best_chain() + .is_some_and(|chain| !chain.contains_block_hash(blocks[1].hash()))); + } + + /// Invalidation is a rollback, not a ban: the rolled-back block + /// revalidates and recommits if its branch wins again later. + #[test] + fn stranded_suffix_rollback_is_not_a_ban() { + let _init_guard = zebra_test::init(); + let network = Network::Mainnet; + let ( + mut non_finalized_state, + finalized_state, + blocks, + mut chain_tip_sender, + _latest_chain_tip, + _chain_tip_change, + nf_sender, + _nf_receiver, + ) = orchestration_fixture(&network); + + let reorged_at = blocks[1].coinbase_height().expect("fake child has height"); + super::invalidate_stranded_body_suffix( + &mut non_finalized_state, + &mut chain_tip_sender, + &nf_sender, + None, + reorged_at, + zebra_chain::block::Hash([0xAB; 32]), + ); + + non_finalized_state + .commit_block(blocks[1].clone().prepare(), &finalized_state) + .expect("a rolled-back block recommits when its branch wins again"); + assert!(non_finalized_state + .best_chain() + .expect("chain is non-empty") + .contains_block_hash(blocks[1].hash())); + } + + /// Rolling back from the non-finalized root empties the chain set: the + /// orchestration must skip the tip publish instead of panicking on the + /// empty state. + #[test] + fn whole_suffix_rollback_skips_publish_without_panicking() { + let _init_guard = zebra_test::init(); + let network = Network::Mainnet; + let ( + mut non_finalized_state, + _finalized_state, + blocks, + mut chain_tip_sender, + _latest_chain_tip, + _chain_tip_change, + nf_sender, + nf_receiver, + ) = orchestration_fixture(&network); + + let reorged_at = blocks[0].coinbase_height().expect("root has height"); + super::invalidate_stranded_body_suffix( + &mut non_finalized_state, + &mut chain_tip_sender, + &nf_sender, + None, + reorged_at, + zebra_chain::block::Hash([0xAB; 32]), + ); + + assert!(non_finalized_state.best_chain().is_none()); + // The emptied state was still published to the watch channel. + assert!(nf_receiver.borrow().best_chain().is_none()); + } + + /// Heights the body chain does not reach, and heights where the body + /// chain already matches the new branch, are left untouched. + #[test] + fn rollback_is_a_noop_when_bodies_match_or_are_absent() { + let _init_guard = zebra_test::init(); + let network = Network::Mainnet; + let ( + mut non_finalized_state, + _finalized_state, + blocks, + mut chain_tip_sender, + _latest_chain_tip, + _chain_tip_change, + nf_sender, + _nf_receiver, + ) = orchestration_fixture(&network); + + // The body chain already holds the "new" hash at the reorged height. + let reorged_at = blocks[1].coinbase_height().expect("fake child has height"); + super::invalidate_stranded_body_suffix( + &mut non_finalized_state, + &mut chain_tip_sender, + &nf_sender, + None, + reorged_at, + blocks[1].hash(), + ); + assert_eq!( + non_finalized_state + .best_chain() + .expect("chain untouched") + .blocks + .len(), + 3 + ); + + // A reorg entirely above the body tip has nothing to roll back. + let above_tip = + (blocks[2].coinbase_height().expect("has height") + 100).expect("height in range"); + super::invalidate_stranded_body_suffix( + &mut non_finalized_state, + &mut chain_tip_sender, + &nf_sender, + None, + above_tip, + zebra_chain::block::Hash([0xAB; 32]), + ); + assert_eq!( + non_finalized_state + .best_chain() + .expect("chain untouched") + .blocks + .len(), + 3 + ); + } } diff --git a/zebrad/src/commands/start/zakura/header_sync_driver.rs b/zebrad/src/commands/start/zakura/header_sync_driver.rs index a6e10528497..4bb55062bb3 100644 --- a/zebrad/src/commands/start/zakura/header_sync_driver.rs +++ b/zebrad/src/commands/start/zakura/header_sync_driver.rs @@ -662,9 +662,6 @@ pub(crate) async fn drive_zakura_header_sync_actions { let tip_hash = outcome.tip_hash; if let Some(reorged_at) = outcome.reorged_at { - // The offset fits usize: it is bounded by the - // committed range length, which is far below - // u32::MAX and platform usize on all targets. - let new_hash = committed_headers - .get(reorged_at.0.saturating_sub(start_height.0) as usize) - .map(|header| block::Hash::from(header.as_ref())); - invalidate_reorged_body_suffix( - state.clone(), - read_state.clone(), - reorged_at, - new_hash, - &trace, - ) - .await; + // The stranded body suffix (if any) was already + // rolled back by the state's switch orchestration, + // before the header rewrite reached disk. + metrics::counter!("sync.header.reorg_commits").increment(1); + info!( + ?reorged_at, + ?tip_hash, + "header range commit reorged the stored header chain" + ); } emit_commit_state( &trace, @@ -1353,10 +1345,13 @@ pub(crate) async fn reconcile_stranded_body_suffix( .await; } -/// Drops the committed body suffix stranded by a header-chain reorg. +/// Drops a committed body suffix found stranded off the header chain. /// -/// `reorged_at` is the first height where a committed header range replaced a -/// conflicting stored header; `new_hash` is the new branch's hash there. If +/// Commit-time stranding is rolled back inside the state's switch +/// orchestration (before the header rewrite reaches disk); this driver-side +/// path serves the startup/periodic reconciliation sweep, which repairs +/// stranding left behind by older binaries or missed events. `reorged_at` is +/// the first stranded height; `new_hash` is the header chain's hash there. If /// the best body chain still holds a different block at that height, that /// block and its descendants are invalidated, resetting the chain tip to the /// fork point. The chain-tip mirror then publishes the reset frontier From 63894f022442b51b24b80e6c0cf8456534699004 Mon Sep 17 00:00:00 2001 From: roman Date: Mon, 6 Jul 2026 21:08:43 -0600 Subject: [PATCH 25/27] test(state): restore the header-store coherence harness dropped by the canary picks Cherry-picks the test content of #490/#493/#494/#495 onto the fleet lineage: the header_store_coherence suite (fabricate/oracle/ops/audit/ scenarios/prop/reads/startup_audit/primitive), the shared block-test helpers, and the proptest regression seeds. The only adaptation is the fleet range-writer API: prepare_header_range_batch returns HeaderRangeCommitOutcome here, so the two helpers that returned the tip hash now unwrap .tip_hash. --- .../tests/header_store_coherence/prop.txt | 7 + .../finalized_state/zebra_db/block/tests.rs | 2 + .../zebra_db/block/tests/common.rs | 149 ++++ .../block/tests/header_store_coherence.rs | 38 + .../tests/header_store_coherence/README.md | 104 +++ .../tests/header_store_coherence/audit.rs | 503 ++++++++++++ .../tests/header_store_coherence/fabricate.rs | 445 +++++++++++ .../block/tests/header_store_coherence/ops.rs | 468 +++++++++++ .../tests/header_store_coherence/oracle.rs | 272 +++++++ .../tests/header_store_coherence/primitive.rs | 282 +++++++ .../tests/header_store_coherence/prop.rs | 88 +++ .../tests/header_store_coherence/reads.rs | 281 +++++++ .../tests/header_store_coherence/scenarios.rs | 736 ++++++++++++++++++ .../header_store_coherence/startup_audit.rs | 539 +++++++++++++ 14 files changed, 3914 insertions(+) create mode 100644 zebra-state/proptest-regressions/service/finalized_state/zebra_db/block/tests/header_store_coherence/prop.txt create mode 100644 zebra-state/src/service/finalized_state/zebra_db/block/tests/common.rs create mode 100644 zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence.rs create mode 100644 zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/README.md create mode 100644 zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/audit.rs create mode 100644 zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/fabricate.rs create mode 100644 zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/ops.rs create mode 100644 zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/oracle.rs create mode 100644 zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/primitive.rs create mode 100644 zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/prop.rs create mode 100644 zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/reads.rs create mode 100644 zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/scenarios.rs create mode 100644 zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/startup_audit.rs diff --git a/zebra-state/proptest-regressions/service/finalized_state/zebra_db/block/tests/header_store_coherence/prop.txt b/zebra-state/proptest-regressions/service/finalized_state/zebra_db/block/tests/header_store_coherence/prop.txt new file mode 100644 index 00000000000..f94e6abf17a --- /dev/null +++ b/zebra-state/proptest-regressions/service/finalized_state/zebra_db/block/tests/header_store_coherence/prop.txt @@ -0,0 +1,7 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc 8b5b37acb73d1b9a201144aef9f359d3be3df29827c8672852dd1e14f5deac26 # shrinks to ops = [Seed { source: Trunk, index: 1 }] diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests.rs index e3b1e5269d5..b7d89ad8027 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests.rs @@ -20,6 +20,8 @@ use crate::{ Config, }; +mod common; +mod header_store_coherence; mod prune; mod snapshot; mod vectors; diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/common.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/common.rs new file mode 100644 index 00000000000..249be03aff6 --- /dev/null +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/common.rs @@ -0,0 +1,149 @@ +//! Shared test helpers for finalized-state block and header-store tests. +//! +//! These helpers are used by both the fixed test vectors (`vectors.rs`) and the +//! header-store coherence harness (`header_store_coherence`). + +use std::{path::Path, sync::Arc}; + +use zebra_chain::{ + block::{self, Block, Height}, + orchard, + parallel::commitment_aux::BlockCommitmentRoots, + parameters::{testnet, Network, Network::Mainnet}, + sapling, + serialization::ZcashDeserializeInto, + work::difficulty::ParameterDifficulty, +}; +use zebra_test::vectors::MAINNET_BLOCKS; + +use crate::{ + constants::{state_database_format_version_in_code, STATE_DATABASE_KIND}, + request::{FinalizedBlock, Treestate}, + service::finalized_state::{disk_db::DiskWriteBatch, ZebraDb, STATE_COLUMN_FAMILIES_IN_CODE}, + CheckpointVerifiedBlock, Config, +}; + +/// Returns an ephemeral or configured state database with `genesis` committed as a full block. +pub(super) fn state_with_genesis_config( + network: &Network, + genesis: Arc, + config: Config, +) -> ZebraDb { + let state = ZebraDb::new( + &config, + STATE_DATABASE_KIND, + &state_database_format_version_in_code(), + network, + true, + STATE_COLUMN_FAMILIES_IN_CODE + .iter() + .map(ToString::to_string), + false, + ); + + write_full_block_header_and_transactions(&state, genesis.clone()); + + state +} + +/// Returns a persistent state config rooted at `cache_dir`, for close-and-reopen tests. +pub(super) fn persistent_config(cache_dir: &Path) -> Config { + Config { + cache_dir: cache_dir.to_owned(), + ephemeral: false, + debug_skip_non_finalized_state_backup_task: true, + ..Config::default() + } +} + +/// Opens (or reopens) a persistent state database from `config`. +pub(super) fn persistent_state(config: &Config, network: &Network) -> ZebraDb { + ZebraDb::new( + config, + STATE_DATABASE_KIND, + &state_database_format_version_in_code(), + network, + true, + STATE_COLUMN_FAMILIES_IN_CODE + .iter() + .map(ToString::to_string), + false, + ) +} + +/// Returns a configured testnet with only the implicit genesis checkpoint, so header +/// commits above genesis take the contextual validation path. +pub(super) fn no_extra_checkpoint_test_network(genesis_hash: block::Hash) -> Network { + testnet::Parameters::build() + .with_network_name("HeaderReorgTest") + .expect("test network name is valid") + .with_genesis_hash(genesis_hash) + .expect("test genesis hash is valid") + .with_target_difficulty_limit(Mainnet.target_difficulty_limit()) + .expect("mainnet difficulty limit is valid for test network") + .with_activation_heights(testnet::ConfiguredActivationHeights { + canopy: Some(1), + ..Default::default() + }) + .expect("test activation heights are valid") + .clear_funding_streams() + .clear_checkpoints() + .expect("genesis-only checkpoints are valid") + .to_network() + .expect("test network is valid") +} + +/// Deserializes the mainnet test vector block at `height`. +pub(super) fn mainnet_block(height: u32) -> Arc { + MAINNET_BLOCKS + .get(&height) + .expect("test vector exists") + .zcash_deserialize_into::>() + .expect("mainnet test block deserializes") +} + +/// Fabricates provisional commitment roots for `height`, with the zeroed +/// auth-data root marking them as unverified. +pub(super) fn root_at(height: Height) -> BlockCommitmentRoots { + BlockCommitmentRoots { + height, + sapling_root: sapling::tree::NoteCommitmentTree::default().root(), + orchard_root: orchard::tree::NoteCommitmentTree::default().root(), + ironwood_root: zebra_chain::ironwood::tree::NoteCommitmentTree::default().root(), + sapling_tx: 0, + orchard_tx: 0, + ironwood_tx: 0, + auth_data_root: zebra_chain::block::merkle::AuthDataRoot::from([0u8; 32]), + } +} + +/// Commits a header range through the production write path, panicking on rejection. +pub(super) fn commit_header_range( + state: &ZebraDb, + anchor: block::Hash, + headers: &[Arc], +) -> block::Hash { + let mut batch = DiskWriteBatch::new(); + let body_sizes = vec![0; headers.len()]; + let outcome = batch + .prepare_header_range_batch(state, anchor, headers, &body_sizes) + .expect("header range is valid"); + state + .write_batch(batch) + .expect("header range batch writes successfully"); + outcome.tip_hash +} + +/// Commits `block`'s header and transaction data (the body-commit batch shape), +/// without treestates or value pools. +pub(super) fn write_full_block_header_and_transactions(state: &ZebraDb, block: Arc) { + let checkpoint_verified = CheckpointVerifiedBlock::from(block); + let finalized = + FinalizedBlock::from_checkpoint_verified(checkpoint_verified, Treestate::default()); + + let mut batch = DiskWriteBatch::new(); + batch + .prepare_block_header_and_transaction_data_batch(state, &finalized, true, None) + .expect("full block header and transaction batch is valid"); + state.db.write(batch).expect("full block batch writes"); +} diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence.rs new file mode 100644 index 00000000000..b0adbdc2c23 --- /dev/null +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence.rs @@ -0,0 +1,38 @@ +//! Zakura header-store coherence harness. +//! +//! A deterministic, model-based test harness for the zakura header store: an +//! in-memory oracle models the *specified* behavior (best-cumulative-work +//! canonical chain with total suffix replacement above the fork point), an op +//! alphabet drives the real `DiskWriteBatch` write paths, and an audit checks +//! the store invariants after every mutation: +//! +//! - **I1 (bijection):** `hash_by_height` ↔ `height_by_hash` are mutually inverse. +//! - **I2 (linkage):** rows chain by `previous_block_hash` from the finalized tip +//! up to the last row in the height index. +//! - **I3 (tip):** `best_header_tip()` is the tip of that linked chain — no orphan +//! rows above it, no gaps below it. +//! - **A4 (oracle):** the linked chain equals the model's expected canonical chain. +//! +//! # Scope +//! +//! This harness exercises the finalized-store writers only: +//! `prepare_header_range_batch_with_roots`, the body-commit batch +//! (`prepare_block_header_and_transaction_data_batch` + the finalization roots +//! delete), the release path (`prepare_zakura_header_release_from_committed_block`), +//! and the seed path (`seed_zakura_header_from_committed_block`). +//! +//! Deliberately out of scope (covered by reactor-level tests or excluded from +//! the op alphabet): `Request::InvalidateBlock` / `ReconsiderBlock` +//! (non-finalized-state level), commitment-roots staging +//! (`insert_zakura_header_commitment_roots`), `rollback_finalized_state`, and +//! pruning. + +mod audit; +mod fabricate; +mod ops; +mod oracle; +mod primitive; +mod prop; +mod reads; +mod scenarios; +mod startup_audit; diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/README.md b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/README.md new file mode 100644 index 00000000000..cf788dcf19b --- /dev/null +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/README.md @@ -0,0 +1,104 @@ +# `header_store_coherence` — zakura header-store coherence suite + +A deterministic, model-based test harness for the zakura header store: it +drives the store's real write paths through sequences of production-shaped +operations and checks the store's structural invariants after every single +mutation. No network, no tokio, no mining. + +## What it tests + +The zakura header store is five height-indexed RocksDB column families +(`zakura_header_by_height`, `zakura_header_hash_by_height`, +`zakura_header_height_by_hash`, `zakura_header_body_size_by_height`, +`commitment_roots_by_height`) mutated by several independent writers in +`zebra_db/block.rs`. Every reader assumes, but nothing enforces, that: + +- **I1 (bijection):** `hash_by_height` ↔ `height_by_hash` are mutually inverse. +- **I2 (linkage):** rows chain by `previous_block_hash` from the finalized tip + up to the last height-index row. +- **I3 (tip):** `best_header_tip()` is the tip of that linked chain — no + orphan rows above it, no gaps below it, and no zakura rows at heights that + already have a committed body (the frontier-overlay rule). + +When these are violated, readers silently feed stale rows into difficulty +validation or anchor resolution, which surfaces later as +`InvalidDifficultyThreshold` or `UnknownAnchor` failures against honest peers. +This suite makes such violations visible at the moment of the corrupting +write instead. + +## How it works + +| Module | Role | +| --- | --- | +| `fabricate.rs` | Pure header/branch fabrication. Every threshold is computed with the store's own `AdjustedDifficulty` logic, so all fabricated chains pass real contextual validation. Work divergence between branches comes from block spacing (`Fast` = target/16, `Slow` = 4×target); branches are tens of headers long because the difficulty adjustment drifts only ~2%/block. Builds a fixed `Universe`: a 60-header trunk, a fork at height 50, and branches **A** (26 fast headers, high work), **B** (30 slow headers — longer than A but lower total work, so height order and work order disagree), **B_ext** (B's first 4 headers plus a fast continuation that out-works A), and **C** (5 headers off A's second header). | +| `audit.rs` | The invariant audit: A1 (bijection, both directions), A2 (linkage walk upward from the finalized tip over the merged header view), A3 (tip integrity, gaps, frontier overlay, aux-row backing), and A4 (the on-disk chain equals the model's expected canonical chain). Also `dump_store`, a comparable snapshot of all five column families used to assert that rejected commits are side-effect free and that reopens preserve the store byte-for-byte. | +| `oracle.rs` | An in-memory model of the store's _specified_ behavior: a single linked canonical chain, best-cumulative-work selection with strict improvement, total suffix replacement above the first conflicting height, and a sequential body tip. It predicts whether each op must be accepted or rejected. | +| `ops.rs` | The op alphabet, each op mapped to one real production write-batch shape: `CommitHeaderRange` → `prepare_header_range_batch_with_roots`; `CommitBody` / `Finalize` → `prepare_block_header_and_transaction_data_batch` plus the finalization roots delete (which runs the release path internally); `Seed` → `seed_zakura_header_from_committed_block` (the non-finalized best-chain commit hook); `Reopen` → shutdown and reopen of the persistent store. The `Harness` executes ops, cross-checks the oracle's prediction against the store's response, and audits after every mutation; failures come back as a transcribable `FailureReport` (the executed op prefix plus every violation found). | +| `scenarios.rs` | Scripted production event shapes (s01–s11): simple reorgs, lower-work rejections and their later reversal, split-range and walk-back deliveries, body commits racing header reorgs, reorgs to a lower height, double reorgs at one fork point, activity across the difficulty-adjustment window edge, restarts at every boundary, seed/range interplay, and refused-seed convergence. Also holds the `*_upholds_invariants` regression gates below. | +| `prop.rs` | The permanent random sweep: random op sequences over the fixed universe, shrunk to minimal counterexamples on any audit failure. | +| `reads.rs` | Read-path coherence (Pillar 2): hand-corrupts the column families and asserts `recent_header_context` / the anchor round-trip report `StoreIncoherentError` (`HeaderHashMismatch`, `BrokenLinkage`, `Gap`, `BijectionMismatch`) instead of feeding stale rows into difficulty validation, and that the range writer's `StoreIncoherent` rejection is side-effect free. | + +## Fixed corruption bugs gated by this suite + +Phase 1 of this suite proved three write-path corruption bug classes, each +pinned by a `corruption_repro_*` test that deterministically demonstrated the +violation. The write-path fixes (`REORG_PLAN.md` Phase 1.5) closed all three; +the repro tests were removed with the fixes, and their +`_upholds_invariants` twins now assert the fixed behavior over the same +op sequences as permanent regression gates: + +1. **Unlinked-anchor commit** (`unlinked_anchor_commit_upholds_invariants`). + `prepare_header_range_batch_with_roots` used to accept ranges without + checking that `headers[0].previous_block_hash == anchor` or any intra-range + linkage, so a range anchored at a same-height hash of a different branch + could pass difficulty validation and commit a suffix that did not link to + the row below it — an on-disk I2 violation reachable from a single + untrusted peer response. The writer now rejects such ranges with + `CommitHeaderRangeError::UnlinkedRange`. +2. **Re-delivery over committed bodies** + (`redelivery_over_bodies_upholds_invariants`). The range insert loop used + to gate only its _roots_ write on `contains_body_at_height`, so a header + range re-delivered over heights whose bodies were committed in the + meantime re-inserted zakura rows below the body tip that nothing ever + trims again (the release trim already ran at body-commit time) — a + permanent I3 frontier-overlay violation. The gate now covers every zakura + row write: heights that already have a committed block are skipped + entirely (checked via `contains_height`, so it also holds for pruned + heights whose bodies are gone but whose authoritative rows remain). +3. **Unlinked seed** (`seed_above_gap_upholds_invariants`, + `seed_fork_switch_upholds_invariants`; found by the proptest and shrunk to + a single op). `prepare_zakura_header_from_committed_block` used to write + its row with no linkage precondition. Seeds fire only at non-finalized + best-_tip_ commits, so any best-tip jump (a fork switch between + non-finalized chains, or a restart that restores the non-finalized backup) + seeded a height whose parent row was missing or belonged to another + branch: a gap or broken link on disk, and a generator of poisoned + difficulty-adjustment windows. The seed path now refuses a seed that does + not link to the stored row below it as a silent no-op — the header store + briefly lags the non-finalized chain, and header-range sync converges it + (`s11_refused_seed_converges_via_range_delivery`). + +## Running + +```bash +# The whole suite (scripted scenarios, audits, regression gates, random sweep): +cargo test -p zebra-state --lib header_store_coherence + +# The random sweep at discovery depth (any failure = a NEW violation class): +PROPTEST_CASES=4096 cargo test -p zebra-state --lib \ + header_store_coherence::prop +``` + +Shrunk proptest counterexamples should be transcribed into `scenarios.rs` as +hardcoded scenarios (seed-independent pinning); the file under +`zebra-state/proptest-regressions/` pins the seeds as a backstop and is +checked in. + +## Scope + +Finalized-store writers only. Out of scope: `Request::InvalidateBlock` / +`ReconsiderBlock` (they operate on the non-finalized state), commitment-roots +staging (`insert_zakura_header_commitment_roots`), `rollback_finalized_state`, +and pruning. The body-commit op omits the verified-roots write from the trees +batch (it needs treestates the harness does not model); the audit treats a +missing verified-roots row as acceptable. diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/audit.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/audit.rs new file mode 100644 index 00000000000..05a8226d703 --- /dev/null +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/audit.rs @@ -0,0 +1,503 @@ +//! The store audit: checks the header-store invariants I1–I3 (as audit checks +//! A1–A3) after every mutation, plus the oracle comparison (A4). +//! +//! The audit window is `finalized_tip ..= last row in any header CF`. All five +//! column families are scanned in full — the chains in these tests are tiny. + +use std::{ + collections::{BTreeMap, HashMap}, + sync::Arc, +}; + +use zebra_chain::block::{self, Height}; + +use super::super::super::{ + AdvertisedBodySize, ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT, ZAKURA_HEADER_BY_HEIGHT, + ZAKURA_HEADER_HASH_BY_HEIGHT, ZAKURA_HEADER_HEIGHT_BY_HASH, +}; +use crate::service::finalized_state::{ + disk_format::shielded::CommitmentRootsByHeight, ZebraDb, COMMITMENT_ROOTS_BY_HEIGHT, +}; + +/// A single store-invariant violation found by [`audit_store`]. +/// +/// Tests match on the variants; the fields are diagnostic payload rendered +/// through `Debug` in failure reports. +#[allow(dead_code)] +#[derive(Clone, Debug)] +pub(crate) enum Violation { + /// A1: a `hash_by_height` row's hash has no `height_by_hash` entry. + MissingHeightByHash { height: Height, hash: block::Hash }, + /// A1: a `hash_by_height` row's hash maps back to a different height. + WrongHeightByHash { + hash: block::Hash, + expected: Height, + actual: Height, + }, + /// A1: a `height_by_hash` entry whose target height row is missing or holds + /// a different hash — a stranded reverse-index row. + OrphanHeightByHash { + hash: block::Hash, + points_at: Height, + height_row: Option, + }, + /// A1: the header CF and hash CF disagree at a height: one row is missing, + /// or the stored header does not hash to the stored hash. + HeaderHashRowMismatch { + height: Height, + header_row_hash: Option, + hash_row: Option, + }, + /// A2: the header at `height` does not link to the row below it. + BrokenLinkage { + height: Height, + prev_in_header: block::Hash, + hash_below: block::Hash, + }, + /// A3: a zakura header CF holds a row at a height with a committed body + /// (the frontier-overlay rule: zakura rows live only above the body store). + ZakuraRowAtBodyHeight { cf: &'static str, height: Height }, + /// A3: a header CF holds a row above the last linked height. + RowAboveLastLinked { cf: &'static str, height: Height }, + /// A3: the linked chain stops at a gap, with rows stranded above it. + GapBelowTip { missing_height: Height }, + /// A3: `best_header_tip()` is not the tip of the linked chain. + BestHeaderTipMismatch { + reported: Option<(Height, block::Hash)>, + last_linked: (Height, block::Hash), + }, + /// A3: a `body_size` or `roots` row at a height without a backing header row. + AuxRowWithoutHeader { cf: &'static str, height: Height }, + /// A4: the linked on-disk chain disagrees with the expected canonical chain. + CanonicalMismatch { + height: Height, + on_disk: Option, + expected: Option, + }, +} + +/// A full, comparable snapshot of the five header-store column families. +/// +/// Used for "rejections are side-effect free" and reopen-survival assertions. +#[derive(Clone, Debug, PartialEq)] +pub(crate) struct StoreDump { + pub headers: BTreeMap>, + pub hashes: BTreeMap, + /// `height_by_hash` rows in RocksDB key order (hash has no `Ord`). + pub heights_by_hash: Vec<(block::Hash, Height)>, + pub body_sizes: BTreeMap, + pub roots: BTreeMap, +} + +pub(crate) fn dump_store(state: &ZebraDb) -> StoreDump { + let header_cf = state.db.cf_handle(ZAKURA_HEADER_BY_HEIGHT).unwrap(); + let hash_cf = state.db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); + let height_cf = state.db.cf_handle(ZAKURA_HEADER_HEIGHT_BY_HASH).unwrap(); + let body_size_cf = state + .db + .cf_handle(ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT) + .unwrap(); + let roots_cf = state.db.cf_handle(COMMITMENT_ROOTS_BY_HEIGHT).unwrap(); + + StoreDump { + headers: state + .db + .zs_forward_range_iter::<_, Height, Arc, _>(&header_cf, ..) + .collect(), + hashes: state + .db + .zs_forward_range_iter::<_, Height, block::Hash, _>(&hash_cf, ..) + .collect(), + heights_by_hash: state + .db + .zs_forward_range_iter::<_, block::Hash, Height, _>(&height_cf, ..) + .collect(), + body_sizes: state + .db + .zs_forward_range_iter::<_, Height, AdvertisedBodySize, _>(&body_size_cf, ..) + .map(|(height, size)| (height, size.get())) + .collect(), + roots: state + .db + .zs_forward_range_iter::<_, Height, CommitmentRootsByHeight, _>(&roots_cf, ..) + .collect(), + } +} + +/// Checks A1 (bijection), A2 (linkage), and A3 (tip integrity) on the store. +/// +/// Returns every violation found; an empty vec means the store is coherent. +pub(crate) fn audit_store(state: &ZebraDb) -> Vec { + let dump = dump_store(state); + let mut violations = Vec::new(); + + // A1: hash_by_height -> height_by_hash roundtrip. + let heights_by_hash: HashMap = + dump.heights_by_hash.iter().copied().collect(); + for (&height, &hash) in &dump.hashes { + match heights_by_hash.get(&hash) { + Some(&actual) if actual == height => {} + Some(&actual) => violations.push(Violation::WrongHeightByHash { + hash, + expected: height, + actual, + }), + None => violations.push(Violation::MissingHeightByHash { height, hash }), + } + } + + // A1 reverse: every height_by_hash entry points at a matching height row. + for &(hash, points_at) in &dump.heights_by_hash { + if dump.hashes.get(&points_at) != Some(&hash) { + violations.push(Violation::OrphanHeightByHash { + hash, + points_at, + height_row: dump.hashes.get(&points_at).copied(), + }); + } + } + + // A1: header rows and hash rows agree, both ways. + for height in dump + .headers + .keys() + .chain(dump.hashes.keys()) + .copied() + .collect::>() + { + let header_row_hash = dump + .headers + .get(&height) + .map(|header| block::Hash::from(&**header)); + let hash_row = dump.hashes.get(&height).copied(); + if header_row_hash != hash_row { + violations.push(Violation::HeaderHashRowMismatch { + height, + header_row_hash, + hash_row, + }); + } + } + + // A2: walk the merged header view up from the finalized (body) tip, + // verifying linkage at each step. + let (body_tip_height, body_tip_hash) = state + .tip() + .expect("harness states always have a committed genesis"); + let mut last_linked = (body_tip_height, body_tip_hash); + let mut linkage_broken = false; + let mut next_height = body_tip_height.next().ok(); + while let Some(height) = next_height { + let Some((hash, header)) = state.header_by_height(height) else { + break; + }; + if header.previous_block_hash != last_linked.1 { + violations.push(Violation::BrokenLinkage { + height, + prev_in_header: header.previous_block_hash, + hash_below: last_linked.1, + }); + linkage_broken = true; + break; + } + last_linked = (height, hash); + next_height = height.next().ok(); + } + + // A3: no zakura rows at body-backed heights (the roots CF is excluded — it + // legitimately holds verified rows at body heights). + let zakura_height_keyed: [(&'static str, Vec); 4] = [ + ( + ZAKURA_HEADER_BY_HEIGHT, + dump.headers.keys().copied().collect(), + ), + ( + ZAKURA_HEADER_HASH_BY_HEIGHT, + dump.hashes.keys().copied().collect(), + ), + ( + ZAKURA_HEADER_HEIGHT_BY_HASH, + dump.heights_by_hash + .iter() + .map(|&(_hash, height)| height) + .collect(), + ), + ( + ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT, + dump.body_sizes.keys().copied().collect(), + ), + ]; + for (cf, heights) in &zakura_height_keyed { + for &height in heights { + if state.contains_body_at_height(height) { + violations.push(Violation::ZakuraRowAtBodyHeight { cf, height }); + } + } + } + + // A3: no rows above the last linked height, in any header CF. + let mut rows_above = false; + for (cf, heights) in &zakura_height_keyed { + for &height in heights { + if height > last_linked.0 { + violations.push(Violation::RowAboveLastLinked { cf, height }); + rows_above = true; + } + } + } + for &height in dump.roots.keys() { + if height > last_linked.0 { + violations.push(Violation::RowAboveLastLinked { + cf: COMMITMENT_ROOTS_BY_HEIGHT, + height, + }); + rows_above = true; + } + } + if rows_above && !linkage_broken { + // The walk stopped at a missing height with rows stranded above it. + violations.push(Violation::GapBelowTip { + missing_height: last_linked + .0 + .next() + .expect("linked tip is far below the max height"), + }); + } + + // A3: best_header_tip() is the tip of the linked chain. + let reported = state.best_header_tip(); + if reported != Some(last_linked) { + violations.push(Violation::BestHeaderTipMismatch { + reported, + last_linked, + }); + } + + // A3: body-size rows require a backing zakura header row; roots rows + // require a zakura header row or a committed body. + for &height in dump.body_sizes.keys() { + if !dump.hashes.contains_key(&height) { + violations.push(Violation::AuxRowWithoutHeader { + cf: ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT, + height, + }); + } + } + for &height in dump.roots.keys() { + if !dump.hashes.contains_key(&height) && !state.contains_body_at_height(height) { + violations.push(Violation::AuxRowWithoutHeader { + cf: COMMITMENT_ROOTS_BY_HEIGHT, + height, + }); + } + } + + violations +} + +/// Checks A4: the merged on-disk chain above genesis equals `expected` +/// (the oracle's canonical chain), over the union of both domains. +pub(crate) fn audit_against_expected_chain( + state: &ZebraDb, + expected: &BTreeMap, +) -> Vec { + let mut violations = Vec::new(); + + let last_expected = expected.keys().next_back().copied().unwrap_or(Height(0)); + let last_on_disk = state.best_header_tip().map_or(Height(0), |(h, _)| h); + let last = last_expected.max(last_on_disk); + + for height in 1..=last.0 { + let height = Height(height); + let on_disk = state.header_hash(height); + let expected_hash = expected.get(&height).copied(); + if on_disk != expected_hash { + violations.push(Violation::CanonicalMismatch { + height, + on_disk, + expected: expected_hash, + }); + } + } + + violations +} + +#[cfg(test)] +mod tests { + use super::super::super::common::{commit_header_range, state_with_genesis_config}; + use super::super::fabricate::{Universe, BRANCH_A, FORK_HEIGHT}; + use super::*; + use crate::{ + service::finalized_state::disk_db::{DiskWriteBatch, WriteDisk}, + Config, + }; + + fn assert_clean(state: &ZebraDb) { + let violations = audit_store(state); + assert!( + violations.is_empty(), + "unexpected violations: {violations:?}" + ); + } + + /// The audit is green on stores produced by clean write sequences. + #[test] + fn audit_is_green_on_coherent_stores() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + + // Genesis-only store. + let state = state_with_genesis_config( + &universe.network, + universe.genesis.clone(), + Config::ephemeral(), + ); + assert_clean(&state); + + // Trunk committed up to the fork, then a branch on top. + let trunk_headers: Vec<_> = universe.trunk[..FORK_HEIGHT as usize] + .iter() + .map(|fab| fab.header.clone()) + .collect(); + commit_header_range(&state, universe.genesis.hash(), &trunk_headers); + assert_clean(&state); + + let branch = &universe.branches[BRANCH_A]; + let branch_headers: Vec<_> = branch + .headers + .iter() + .map(|fab| fab.header.clone()) + .collect(); + commit_header_range(&state, branch.fork_parent.1, &branch_headers); + assert_clean(&state); + } + + /// Each hand-made corruption fires the right violation variant. + #[test] + fn audit_detects_hand_made_corruption() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + + let corrupted_state = || { + let state = state_with_genesis_config( + &universe.network, + universe.genesis.clone(), + Config::ephemeral(), + ); + let trunk_headers: Vec<_> = universe.trunk[..FORK_HEIGHT as usize] + .iter() + .map(|fab| fab.header.clone()) + .collect(); + commit_header_range(&state, universe.genesis.hash(), &trunk_headers); + state + }; + + // Delete one height_by_hash row: forward bijection breaks. + let state = corrupted_state(); + let height_cf = state.db.cf_handle(ZAKURA_HEADER_HEIGHT_BY_HASH).unwrap(); + let mut batch = DiskWriteBatch::new(); + batch.zs_delete(&height_cf, universe.trunk_at(10).hash); + state.db.write(batch).expect("raw delete writes"); + let violations = audit_store(&state); + assert!( + violations + .iter() + .any(|v| matches!(v, Violation::MissingHeightByHash { height, .. } if *height == Height(10))), + "expected MissingHeightByHash: {violations:?}" + ); + + // Insert a stray height_by_hash row: reverse bijection breaks. + let state = corrupted_state(); + let height_cf = state.db.cf_handle(ZAKURA_HEADER_HEIGHT_BY_HASH).unwrap(); + let stray_hash = universe.branches[BRANCH_A].headers[0].hash; + let mut batch = DiskWriteBatch::new(); + batch.zs_insert(&height_cf, stray_hash, Height(12)); + state.db.write(batch).expect("raw insert writes"); + let violations = audit_store(&state); + assert!( + violations.iter().any( + |v| matches!(v, Violation::OrphanHeightByHash { hash, .. } if *hash == stray_hash) + ), + "expected OrphanHeightByHash: {violations:?}" + ); + + // Replace a mid-chain header row with a foreign header: the header/hash + // rows disagree and linkage breaks above the corrupted height. + let state = corrupted_state(); + let header_cf = state.db.cf_handle(ZAKURA_HEADER_BY_HEIGHT).unwrap(); + let foreign_header = universe.branches[BRANCH_A].headers[3].header.clone(); + let mut batch = DiskWriteBatch::new(); + batch.zs_insert(&header_cf, Height(20), foreign_header); + state.db.write(batch).expect("raw insert writes"); + let violations = audit_store(&state); + assert!( + violations + .iter() + .any(|v| matches!(v, Violation::HeaderHashRowMismatch { height, .. } if *height == Height(20))), + "expected HeaderHashRowMismatch: {violations:?}" + ); + + // Delete a mid-chain hash row: the linked walk stops below it and every + // surviving row above is stranded. + let state = corrupted_state(); + let hash_cf = state.db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); + let mut batch = DiskWriteBatch::new(); + batch.zs_delete(&hash_cf, Height(15)); + state.db.write(batch).expect("raw delete writes"); + let violations = audit_store(&state); + assert!( + violations + .iter() + .any(|v| matches!(v, Violation::RowAboveLastLinked { height, .. } if *height > Height(15))), + "expected RowAboveLastLinked: {violations:?}" + ); + assert!( + violations + .iter() + .any(|v| matches!(v, Violation::GapBelowTip { missing_height } if *missing_height == Height(15))), + "expected GapBelowTip: {violations:?}" + ); + assert!( + violations + .iter() + .any(|v| matches!(v, Violation::BestHeaderTipMismatch { .. })), + "expected BestHeaderTipMismatch: {violations:?}" + ); + } + + /// A4 reports disagreements with an expected canonical chain. + #[test] + fn audit_detects_canonical_mismatch() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + + let state = state_with_genesis_config( + &universe.network, + universe.genesis.clone(), + Config::ephemeral(), + ); + let trunk_headers: Vec<_> = universe.trunk[..FORK_HEIGHT as usize] + .iter() + .map(|fab| fab.header.clone()) + .collect(); + commit_header_range(&state, universe.genesis.hash(), &trunk_headers); + + let mut expected: BTreeMap = universe.trunk[..FORK_HEIGHT as usize] + .iter() + .map(|fab| (fab.height, fab.hash)) + .collect(); + assert!(audit_against_expected_chain(&state, &expected).is_empty()); + + // Claim one extra expected height: the store must be reported behind. + let next = &universe.branches[BRANCH_A].headers[0]; + expected.insert(next.height, next.hash); + let violations = audit_against_expected_chain(&state, &expected); + assert!( + violations + .iter() + .any(|v| matches!(v, Violation::CanonicalMismatch { height, on_disk: None, .. } if *height == next.height)), + "expected CanonicalMismatch: {violations:?}" + ); + } +} diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/fabricate.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/fabricate.rs new file mode 100644 index 00000000000..bfc118235a3 --- /dev/null +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/fabricate.rs @@ -0,0 +1,445 @@ +//! Pure header, body, and branch-universe fabrication for the coherence harness. +//! +//! Headers are fabricated (not mined): each header's `difficulty_threshold` is +//! computed with the same [`AdjustedDifficulty`] logic the validator uses, so the +//! real DAA runs on every branch. Work divergence between branches comes from +//! block spacing: faster-than-target spacing drives thresholds below the +//! difficulty limit (more work per header), slower-than-target spacing drifts +//! them back up (less work per header). The testnet minimum-difficulty rule is +//! irrelevant here — it only applies above `TESTNET_MINIMUM_DIFFICULTY_START_HEIGHT` +//! (299,188), far above these chains. + +use std::sync::Arc; + +use chrono::{DateTime, Duration, Utc}; +use zebra_chain::{ + block::{self, Block, Height}, + parallel::commitment_aux::BlockCommitmentRoots, + parameters::{Network, NetworkUpgrade}, + transparent, + work::difficulty::{CompactDifficulty, ParameterDifficulty, PartialCumulativeWork, Work}, +}; + +use super::super::common::{ + commit_header_range, mainnet_block, no_extra_checkpoint_test_network, root_at, + state_with_genesis_config, +}; +use crate::{ + service::check::difficulty::{AdjustedDifficulty, POW_ADJUSTMENT_BLOCK_SPAN}, + Config, +}; + +/// A difficulty context window: `(difficulty_threshold, time)` pairs in reverse +/// height order, starting from the previous block — the exact shape consumed by +/// [`AdjustedDifficulty::new_from_header_time`] and produced by +/// `ZebraDb::recent_header_context`. +pub(crate) type DifficultyContext = Vec<(CompactDifficulty, DateTime)>; + +/// Block spacing for fabricated headers, relative to the network target spacing. +#[derive(Copy, Clone, Debug)] +pub(crate) enum Spacing { + /// A sixteenth of target spacing: the DAA lowers thresholds + /// (more work per header). + Fast, + /// Four times target spacing: the DAA raises thresholds toward the + /// difficulty limit (less work per header). + Slow, +} + +impl Spacing { + fn duration(self, network: &Network, height: Height) -> Duration { + let target = NetworkUpgrade::target_spacing_for_height(network, height); + let duration = match self { + Spacing::Fast => target / 16, + Spacing::Slow => target * 4, + }; + // Header times serialize as whole seconds; sub-second precision would + // be lost on the store round-trip and desync the difficulty context. + Duration::seconds(duration.num_seconds().max(1)) + } +} + +/// A fabricated header and everything the op alphabet needs to commit it. +#[derive(Clone, Debug)] +pub(crate) struct FabHeader { + pub height: Height, + pub hash: block::Hash, + pub header: Arc, + /// The header's proof-of-work amount, from its fabricated difficulty threshold. + pub work: Work, + /// Provisional commitment roots for this height. + pub roots: BlockCommitmentRoots, + /// Advertised body size committed alongside the header (0 = unknown). + pub body_size: u32, +} + +/// Fabricates a linked run of headers on top of `anchor`. +/// +/// `context` must be the difficulty context at the anchor (anchor first). Each +/// header gets the validator-expected difficulty threshold for its fabricated +/// time, so the run passes `check::header_is_valid_for_recent_chain` when +/// committed on a store whose context at the anchor matches `context`. +pub(crate) fn fabricate_headers( + network: &Network, + anchor: (Height, block::Hash), + mut context: DifficultyContext, + spacings: &[Spacing], + nonce_seed: u8, +) -> Vec { + let template = mainnet_block(1); + let (mut previous_height, mut previous_hash) = anchor; + let mut nonce_tag = nonce_seed; + + spacings + .iter() + .map(|spacing| { + let candidate_height = previous_height + .next() + .expect("test header height remains in range"); + let previous_time = context + .first() + .expect("anchor difficulty context is available") + .1; + let candidate_time = previous_time + spacing.duration(network, candidate_height); + let expected_difficulty = AdjustedDifficulty::new_from_header_time( + candidate_time, + previous_height, + network, + context.iter().copied(), + ) + .expected_difficulty_threshold(); + + let mut header = *template.header; + header.previous_block_hash = previous_hash; + header.time = candidate_time; + header.difficulty_threshold = expected_difficulty; + header.nonce.0[0] = header.nonce.0[0].wrapping_add(nonce_tag); + nonce_tag = nonce_tag.wrapping_add(1); + + let header = Arc::new(header); + let hash = block::Hash::from(&*header); + previous_hash = hash; + previous_height = candidate_height; + context.insert(0, (header.difficulty_threshold, header.time)); + context.truncate(POW_ADJUSTMENT_BLOCK_SPAN); + + FabHeader { + height: candidate_height, + hash, + header, + work: expected_difficulty + .to_work() + .expect("fabricated difficulty threshold always converts to work"), + roots: root_at(candidate_height), + body_size: 0, + } + }) + .collect() +} + +/// Extends a difficulty context with fabricated headers (oldest to newest). +pub(crate) fn extend_context( + mut context: DifficultyContext, + headers: &[FabHeader], +) -> DifficultyContext { + for fab in headers { + context.insert(0, (fab.header.difficulty_threshold, fab.header.time)); + } + context.truncate(POW_ADJUSTMENT_BLOCK_SPAN); + context +} + +/// Fabricates a full block whose header is `fab.header` and whose single +/// coinbase transaction commits to `fab.height`, so `CheckpointVerifiedBlock` +/// derives the right height. The body-commit batch does no merkle or parent +/// validation, so a template body with a rewritten coinbase height suffices. +pub(crate) fn fabricate_body(fab: &FabHeader) -> Arc { + let template = mainnet_block(1); + let mut block = Block::clone(&template); + + let mut tx = block.transactions.remove(0); + let input = match Arc::make_mut(&mut tx) { + zebra_chain::transaction::Transaction::V1 { inputs, .. } => &mut inputs[0], + _ => panic!("mainnet block 1 has a V1 coinbase transaction"), + }; + match input { + transparent::Input::Coinbase { height, .. } => *height = fab.height, + _ => panic!("mainnet block 1 transaction 0 is a coinbase"), + } + block.transactions.insert(0, tx); + block.header = fab.header.clone(); + + Arc::new(block) +} + +/// Sums the work of a fabricated header run. +pub(crate) fn total_work(headers: &[FabHeader]) -> PartialCumulativeWork { + let mut total = PartialCumulativeWork::zero(); + for fab in headers { + total += fab.work; + } + total +} + +/// A branch of fabricated headers off a known parent. +#[derive(Clone, Debug)] +pub(crate) struct BranchDef { + /// The `(height, hash)` of the header this branch builds on. + pub fork_parent: (Height, block::Hash), + pub headers: Vec, +} + +/// Height of the trunk header the main fork branches build on. +pub(crate) const FORK_HEIGHT: u32 = 50; +/// Number of trunk headers above genesis. +pub(crate) const TRUNK_LEN: usize = 60; + +/// Branch indexes into [`Universe::branches`]. +pub(crate) const BRANCH_A: usize = 0; +pub(crate) const BRANCH_B: usize = 1; +pub(crate) const BRANCH_B_EXT: usize = 2; +pub(crate) const BRANCH_C: usize = 3; + +/// The fixed, deterministic block-tree every scenario and proptest case runs over. +/// +/// Work divergence comes from the real DAA reacting to block spacing. The DAA +/// response is slow — the median timespan compares medians ~17 blocks apart, +/// damped 4× and bounded to −16%/+32%, and the median-time-past lag hides a +/// spacing change for its first ~6 blocks — so per-header work drifts only +/// ~2%/block. Branches must be tens of headers long for the work drift to beat +/// branch-length differences: +/// +/// - `trunk`: 60 fast-spacing headers over genesis (the DAA is live and +/// thresholds are well below the limit at the fork point, height 50). +/// - Branch `A` (index 0): 26 fast headers off trunk@50 — the high-work fork. +/// - Branch `B` (index 1): 30 slow headers off trunk@50 — longer than `A` but +/// lower total work, so height order and work order disagree. +/// - Branch `B_ext` (index 2): `B`'s first 4 headers plus a fast continuation, +/// long enough to carry strictly more work than `A` (the flipped work +/// balance scenario s02 needs). +/// - Branch `C` (index 3): 5 fast headers off `A`'s second header — a nested fork. +pub(crate) struct Universe { + pub network: Network, + pub genesis: Arc, + pub trunk: Vec, + pub branches: Vec, +} + +impl Universe { + pub fn new() -> Self { + let genesis = mainnet_block(0); + let network = no_extra_checkpoint_test_network(genesis.hash()); + let genesis_anchor = (Height(0), genesis.hash()); + let genesis_context: DifficultyContext = + vec![(genesis.header.difficulty_threshold, genesis.header.time)]; + + let trunk = fabricate_headers( + &network, + genesis_anchor, + genesis_context.clone(), + &[Spacing::Fast; TRUNK_LEN], + 0x10, + ); + + let fork_index = FORK_HEIGHT as usize - 1; + let fork_parent = (trunk[fork_index].height, trunk[fork_index].hash); + let fork_context = extend_context(genesis_context, &trunk[..=fork_index]); + + // The DAA must be live at the fork point, or fast/slow spacing cannot + // produce work divergence between the branches. + let limit_work = network + .target_difficulty_limit() + .to_compact() + .to_work() + .expect("difficulty limit converts to work"); + assert!( + trunk[fork_index].work > limit_work, + "trunk fast spacing must drive thresholds below the difficulty limit \ + before the fork point" + ); + + let branch_a = BranchDef { + fork_parent, + headers: fabricate_headers( + &network, + fork_parent, + fork_context.clone(), + &[Spacing::Fast; 26], + 0x40, + ), + }; + + let branch_b = BranchDef { + fork_parent, + headers: fabricate_headers( + &network, + fork_parent, + fork_context.clone(), + &[Spacing::Slow; 30], + 0x80, + ), + }; + + // B's first 4 headers plus a fast continuation, extended until the + // whole branch carries strictly more work than A, plus margin. + let b_prefix = branch_b.headers[..4].to_vec(); + let b_prefix_tip = (b_prefix[3].height, b_prefix[3].hash); + let b_prefix_context = extend_context(fork_context.clone(), &b_prefix); + let a_work = total_work(&branch_a.headers); + let mut continuation_len = 1; + let branch_b_ext = loop { + assert!( + continuation_len <= 64, + "branch B_ext should out-work branch A within 64 fast headers" + ); + let continuation = fabricate_headers( + &network, + b_prefix_tip, + b_prefix_context.clone(), + &vec![Spacing::Fast; continuation_len], + 0xC0, + ); + let mut headers = b_prefix.clone(); + headers.extend(continuation); + if total_work(&headers) > a_work { + // Two margin headers so split-range deliveries also out-work A. + let continuation = fabricate_headers( + &network, + b_prefix_tip, + b_prefix_context.clone(), + &vec![Spacing::Fast; continuation_len + 2], + 0xC0, + ); + let mut headers = b_prefix; + headers.extend(continuation); + break BranchDef { + fork_parent, + headers, + }; + } + continuation_len += 1; + }; + + let c_parent = (branch_a.headers[1].height, branch_a.headers[1].hash); + let c_context = extend_context(fork_context, &branch_a.headers[..2]); + let branch_c = BranchDef { + fork_parent: c_parent, + headers: fabricate_headers(&network, c_parent, c_context, &[Spacing::Fast; 5], 0xE0), + }; + + let universe = Universe { + network, + genesis, + trunk, + branches: vec![branch_a, branch_b, branch_b_ext, branch_c], + }; + universe.assert_work_orderings(); + universe + } + + /// The invariants the branch construction promises to every scenario. + fn assert_work_orderings(&self) { + let a = &self.branches[BRANCH_A]; + let b = &self.branches[BRANCH_B]; + let b_ext = &self.branches[BRANCH_B_EXT]; + + assert!( + b.headers.len() > a.headers.len(), + "branch B must be longer than branch A" + ); + assert!( + total_work(&a.headers) > total_work(&b.headers), + "branch A must carry more work than the longer branch B \ + (A: {:?}, B: {:?})", + total_work(&a.headers), + total_work(&b.headers), + ); + assert!( + total_work(&b_ext.headers) > total_work(&a.headers), + "branch B_ext must carry more work than branch A \ + (B_ext: {:?}, A: {:?})", + total_work(&b_ext.headers), + total_work(&a.headers), + ); + assert_eq!( + b.headers[..4] + .iter() + .map(|fab| fab.hash) + .collect::>(), + b_ext.headers[..4] + .iter() + .map(|fab| fab.hash) + .collect::>(), + "branch B_ext must re-use branch B's first four headers" + ); + } + + /// The trunk header at `height` (heights start at 1). + pub fn trunk_at(&self, height: u32) -> &FabHeader { + &self.trunk[height as usize - 1] + } +} + +/// Proves the fabricated universe passes the real contextual validation +/// (`check::header_is_valid_for_recent_chain`) on every branch: the whole +/// trunk, and each branch on a store holding the chain up to its fork parent. +#[test] +fn fabricated_universe_commits_cleanly() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + + let headers_of = |fabs: &[FabHeader]| -> Vec> { + fabs.iter().map(|fab| fab.header.clone()).collect() + }; + + // The whole trunk, in one range from the genesis anchor. + let state = state_with_genesis_config( + &universe.network, + universe.genesis.clone(), + Config::ephemeral(), + ); + commit_header_range( + &state, + universe.genesis.hash(), + &headers_of(&universe.trunk), + ); + let trunk_tip = universe.trunk.last().expect("trunk is non-empty"); + assert_eq!( + state.best_header_tip(), + Some((trunk_tip.height, trunk_tip.hash)), + ); + + // Each branch, on a fresh store holding the chain up to its fork parent. + for branch_index in [BRANCH_A, BRANCH_B, BRANCH_B_EXT, BRANCH_C] { + let branch = &universe.branches[branch_index]; + let state = state_with_genesis_config( + &universe.network, + universe.genesis.clone(), + Config::ephemeral(), + ); + + commit_header_range( + &state, + universe.genesis.hash(), + &headers_of(&universe.trunk[..FORK_HEIGHT as usize]), + ); + if branch_index == BRANCH_C { + // C forks off A's second header, so A's prefix must be present. + let a_prefix = &universe.branches[BRANCH_A].headers[..2]; + commit_header_range( + &state, + universe.branches[BRANCH_A].fork_parent.1, + &headers_of(a_prefix), + ); + } + + commit_header_range(&state, branch.fork_parent.1, &headers_of(&branch.headers)); + + let branch_tip = branch.headers.last().expect("branches are non-empty"); + assert_eq!( + state.best_header_tip(), + Some((branch_tip.height, branch_tip.hash)), + "branch {branch_index} commits to the tip", + ); + } +} diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/ops.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/ops.rs new file mode 100644 index 00000000000..ad36b6556d1 --- /dev/null +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/ops.rs @@ -0,0 +1,468 @@ +//! The op alphabet and the harness that drives the real store writers. +//! +//! Each op maps to one production write-batch shape: +//! +//! - [`Op::CommitHeaderRange`] → `prepare_header_range_batch_with_roots` +//! (the header-sync range commit); +//! - [`Op::CommitBody`] → `prepare_block_header_and_transaction_data_batch` +//! plus the finalization roots delete (internally runs the release path); +//! - [`Op::Seed`] → `ZebraDb::seed_zakura_header_from_committed_block` +//! (the non-finalized best-chain commit hook); +//! - [`Op::Finalize`] → sequential body commits along the expected canonical +//! chain; +//! - [`Op::Reopen`] → close and reopen the database (restart survival). +//! +//! After every op the harness cross-checks the oracle's prediction against the +//! store's response (rejections must be side-effect free), then runs the full +//! audit. Any violation or prediction mismatch fails the sequence with a +//! transcribable [`FailureReport`]. + +use std::sync::{Arc, LazyLock}; + +use zebra_chain::block; + +use super::super::common::{ + persistent_config, persistent_state, write_full_block_header_and_transactions, +}; +use super::{ + audit::{audit_against_expected_chain, audit_store, dump_store, Violation}, + fabricate::{fabricate_body, FabHeader, Universe}, + oracle::{Oracle, Prediction, ResolvedRange}, +}; +use crate::{ + error::{CommitCheckpointVerifiedError, CommitHeaderRangeError}, + request::{FinalizedBlock, Treestate}, + service::finalized_state::{ + disk_db::{DiskWriteBatch, WriteDisk}, + ZebraDb, COMMITMENT_ROOTS_BY_HEIGHT, + }, + CheckpointVerifiedBlock, Config, +}; + +/// Which fabricated chain an op draws rows from. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Source { + Trunk, + Branch(usize), +} + +/// How a header range names its anchor. +#[derive(Clone, Copy, Debug)] +pub(crate) enum Anchor { + /// The natural anchor: the row below `offset` (the branch fork parent or + /// genesis for `offset == 0`). + Auto, + /// The genesis hash, regardless of offset. + Genesis, + /// The trunk header at this height. + TrunkAt(u32), + /// `universe.branches[i].headers[j]`'s hash. + BranchAt(usize, usize), +} + +/// One mutation of the store. +#[derive(Clone, Debug)] +pub(crate) enum Op { + CommitHeaderRange { + source: Source, + offset: usize, + len: usize, + anchor: Anchor, + }, + CommitBody { + source: Source, + index: usize, + }, + Seed { + source: Source, + index: usize, + }, + Finalize { + count: usize, + }, + Reopen, +} + +/// How the store responded to an executed op. +#[derive(Debug)] +pub(crate) enum OpOutcome { + Accepted, + Rejected(RejectError), + /// The op does not apply in the current state and was not executed. + Skipped(#[allow(dead_code)] &'static str), +} + +#[derive(Debug)] +pub(crate) enum RejectError { + HeaderRange(CommitHeaderRangeError), + /// Diagnostic payload rendered through `Debug` in failure reports. + Body(#[allow(dead_code)] Box), +} + +impl OpOutcome { + pub fn header_range_error(&self) -> &CommitHeaderRangeError { + match self { + OpOutcome::Rejected(RejectError::HeaderRange(error)) => error, + other => panic!("expected a header-range rejection, got {other:?}"), + } + } +} + +/// Why a sequence failed: the executed op prefix (last op is the failing one) +/// plus everything found after it. All fields are diagnostic payloads rendered +/// through `Debug` in test failure output. +#[derive(Debug)] +pub(crate) struct FailureReport { + /// The transcribable op sequence. + #[allow(dead_code)] + pub executed: Vec, + #[allow(dead_code)] + pub violations: Vec, + #[allow(dead_code)] + pub mismatches: Vec, +} + +/// The shared fabricated universe. Built once — it is deterministic and +/// immutable, and fabrication is the slowest part of a harness construction. +pub(crate) fn universe() -> &'static Universe { + static UNIVERSE: LazyLock = LazyLock::new(Universe::new); + &UNIVERSE +} + +pub(crate) struct Harness { + pub universe: &'static Universe, + pub oracle: Oracle, + config: Config, + state: Option, + executed: Vec, + _tempdir: tempfile::TempDir, +} + +impl Harness { + /// A harness over a fresh persistent store holding only genesis. + /// + /// The store is always persistent (on a tempdir) so `Reopen` behaves the + /// same wherever it appears in a sequence. + pub fn new() -> Self { + let universe = universe(); + let tempdir = tempfile::tempdir().expect("test tempdir is available"); + let config = persistent_config(tempdir.path()); + let state = persistent_state(&config, &universe.network); + write_full_block_header_and_transactions(&state, universe.genesis.clone()); + + Harness { + universe, + oracle: Oracle::new(universe), + config, + state: Some(state), + executed: Vec::new(), + _tempdir: tempdir, + } + } + + pub fn state(&self) -> &ZebraDb { + self.state + .as_ref() + .expect("state is only vacated inside Reopen") + } + + fn rows_of(&self, source: Source) -> &[FabHeader] { + match source { + Source::Trunk => &self.universe.trunk, + Source::Branch(index) => &self.universe.branches[index].headers, + } + } + + fn resolve_anchor(&self, source: Source, offset: usize, anchor: Anchor) -> block::Hash { + match anchor { + Anchor::Auto => { + if offset == 0 { + match source { + Source::Trunk => self.universe.genesis.hash(), + Source::Branch(index) => self.universe.branches[index].fork_parent.1, + } + } else { + self.rows_of(source)[offset - 1].hash + } + } + Anchor::Genesis => self.universe.genesis.hash(), + Anchor::TrunkAt(height) => { + // Clamp so randomly generated anchors always resolve. + let height = height.clamp(1, self.universe.trunk.len() as u32); + self.universe.trunk_at(height).hash + } + Anchor::BranchAt(branch, index) => { + let headers = + &self.universe.branches[branch % self.universe.branches.len()].headers; + headers[index % headers.len()].hash + } + } + } + + fn resolve_range( + &self, + source: Source, + offset: usize, + len: usize, + anchor: Anchor, + ) -> Option { + let rows = self.rows_of(source); + if offset >= rows.len() || len == 0 { + return None; + } + let end = (offset + len).min(rows.len()); + Some(ResolvedRange { + anchor: self.resolve_anchor(source, offset, anchor), + rows: rows[offset..end].to_vec(), + }) + } + + /// Runs one op: execute, cross-check the oracle, audit the store. + pub fn run(&mut self, op: &Op) -> Result { + self.executed.push(op.clone()); + let mut mismatches = Vec::new(); + + let outcome = match op { + Op::CommitHeaderRange { + source, + offset, + len, + anchor, + } => { + let Some(range) = self.resolve_range(*source, *offset, *len, *anchor) else { + return self.finish(OpOutcome::Skipped("range out of bounds"), mismatches); + }; + match self.oracle.predict_header_range(&range) { + Prediction::Skip(reason) => { + return self.finish(OpOutcome::Skipped(reason), mismatches) + } + prediction => { + let dump_before = dump_store(self.state()); + let result = execute_header_range(self.state(), &range); + match (&prediction, &result) { + (Prediction::Accept, Ok(())) => { + self.oracle.apply_header_range(&range); + } + (Prediction::Reject(_), Err(_)) => { + if dump_store(self.state()) != dump_before { + mismatches.push(format!( + "rejected header range mutated the store: {op:?}" + )); + } + } + (Prediction::Accept, Err(error)) => { + mismatches.push(format!( + "oracle predicted accept, store rejected with {error:?}: {op:?}" + )); + } + (Prediction::Reject(kind), Ok(())) => { + mismatches.push(format!( + "oracle predicted rejection ({kind:?}), store accepted: {op:?}" + )); + // Keep A4 meaningful for the rest of the report. + self.oracle.apply_header_range(&range); + } + (Prediction::Skip(_), _) => unreachable!("skips return early"), + } + match result { + Ok(()) => OpOutcome::Accepted, + Err(error) => OpOutcome::Rejected(RejectError::HeaderRange(error)), + } + } + } + } + + Op::CommitBody { source, index } => { + let rows = self.rows_of(*source); + let Some(fab) = rows.get(*index).cloned() else { + return self.finish(OpOutcome::Skipped("body index out of bounds"), mismatches); + }; + match self.oracle.predict_body(&fab) { + Prediction::Skip(reason) => { + return self.finish(OpOutcome::Skipped(reason), mismatches) + } + _ => match execute_body(self.state(), &fab) { + Ok(()) => { + self.oracle.apply_body(&fab); + OpOutcome::Accepted + } + Err(error) => { + mismatches.push(format!( + "oracle predicted accept, body commit rejected with {error:?}: {op:?}" + )); + OpOutcome::Rejected(RejectError::Body(Box::new(error))) + } + }, + } + } + + Op::Seed { source, index } => { + let rows = self.rows_of(*source); + let Some(fab) = rows.get(*index).cloned() else { + return self.finish(OpOutcome::Skipped("seed index out of bounds"), mismatches); + }; + match self.oracle.predict_seed(&fab) { + Prediction::Skip(reason) => { + return self.finish(OpOutcome::Skipped(reason), mismatches) + } + _ => { + // A seed whose parent is not the stored row below it is + // refused by the store as a silent no-op (the header + // store briefly lags the non-finalized chain until a + // linkage-checked header range converges it), so the + // call must succeed without mutating anything. + let parent_linked = self.oracle.seed_is_parent_linked(&fab); + let dump_before = (!parent_linked).then(|| dump_store(self.state())); + let block = fabricate_body(&fab); + match self + .state() + .seed_zakura_header_from_committed_block(fab.height, &block) + { + Ok(()) => { + if parent_linked { + self.oracle.apply_seed(&fab); + } else if dump_store(self.state()) + != dump_before.expect("dumped before an unlinked seed") + { + mismatches.push(format!( + "refused unlinked seed mutated the store: {op:?}" + )); + } + OpOutcome::Accepted + } + Err(error) => { + mismatches.push(format!( + "oracle predicted accept, seed rejected with {error:?}: {op:?}" + )); + OpOutcome::Rejected(RejectError::HeaderRange(error)) + } + } + } + } + } + + Op::Finalize { count } => { + let mut finalized_any = false; + for _ in 0..*count { + let next = self.oracle.next_body_height(); + let Some(&hash) = self.oracle.canonical_chain().get(&next) else { + break; + }; + let fab = self + .oracle + .fab_for(hash) + .expect("canonical hashes come from the universe") + .clone(); + match execute_body(self.state(), &fab) { + Ok(()) => { + self.oracle.apply_body(&fab); + finalized_any = true; + } + Err(error) => { + mismatches.push(format!( + "finalizing canonical height {next:?} rejected with {error:?}" + )); + break; + } + } + } + if finalized_any || !mismatches.is_empty() { + OpOutcome::Accepted + } else { + return self.finish( + OpOutcome::Skipped("no canonical row above the body tip"), + mismatches, + ); + } + } + + Op::Reopen => { + let dump_before = dump_store(self.state()); + let mut state = self.state.take().expect("state is present before Reopen"); + state.shutdown(true); + drop(state); + let state = persistent_state(&self.config, &self.universe.network); + self.state = Some(state); + if dump_store(self.state()) != dump_before { + mismatches.push("store changed across a reopen".to_string()); + } + OpOutcome::Accepted + } + }; + + self.finish(outcome, mismatches) + } + + /// Audits the store and closes out an op. + fn finish( + &mut self, + outcome: OpOutcome, + mismatches: Vec, + ) -> Result { + let mut violations = audit_store(self.state()); + violations.extend(audit_against_expected_chain( + self.state(), + self.oracle.canonical_chain(), + )); + + if violations.is_empty() && mismatches.is_empty() { + Ok(outcome) + } else { + Err(FailureReport { + executed: self.executed.clone(), + violations, + mismatches, + }) + } + } + + /// Runs a whole sequence, stopping at the first violation. + pub fn run_all(&mut self, ops: &[Op]) -> Result, FailureReport> { + ops.iter().map(|op| self.run(op)).collect() + } +} + +fn execute_header_range( + state: &ZebraDb, + range: &ResolvedRange, +) -> Result<(), CommitHeaderRangeError> { + let headers: Vec> = + range.rows.iter().map(|fab| fab.header.clone()).collect(); + let body_sizes: Vec = range.rows.iter().map(|fab| fab.body_size).collect(); + let roots: Vec<_> = range.rows.iter().map(|fab| fab.roots.clone()).collect(); + + let mut batch = DiskWriteBatch::new(); + batch.prepare_header_range_batch_with_roots( + state, + range.anchor, + &headers, + &body_sizes, + &roots, + )?; + state + .write_batch(batch) + .expect("header range batch writes successfully"); + Ok(()) +} + +/// The body-commit batch shape: header and transaction data, the internal +/// zakura release, and the finalization delete of the provisional roots row +/// (`prepare_block_batch`). The verified-roots write from the trees batch is +/// deliberately absent — it needs treestates the harness does not model, and +/// the audit treats a missing verified-roots row as acceptable. +fn execute_body(state: &ZebraDb, fab: &FabHeader) -> Result<(), CommitCheckpointVerifiedError> { + let block = fabricate_body(fab); + let checkpoint_verified = CheckpointVerifiedBlock::from(block); + let finalized = + FinalizedBlock::from_checkpoint_verified(checkpoint_verified, Treestate::default()); + + let mut batch = DiskWriteBatch::new(); + batch.prepare_block_header_and_transaction_data_batch(state, &finalized, true, None)?; + let roots_cf = state.db.cf_handle(COMMITMENT_ROOTS_BY_HEIGHT).unwrap(); + batch.zs_delete(&roots_cf, fab.height); + state + .db + .write(batch) + .expect("body batch writes successfully"); + Ok(()) +} diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/oracle.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/oracle.rs new file mode 100644 index 00000000000..574b695afc6 --- /dev/null +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/oracle.rs @@ -0,0 +1,272 @@ +//! The in-memory oracle: a trivially correct model of the header store's +//! *specified* behavior. +//! +//! The model is a single linked canonical chain above genesis, plus a +//! sequential body tip. Chain selection is best-cumulative-work with strict +//! improvement, and an accepted branch switch replaces the whole suffix above +//! the first conflicting height — nothing of the losing branch survives. +//! Rejected commits change nothing. + +use std::collections::{BTreeMap, HashMap}; + +use zebra_chain::{ + block::{self, Height}, + work::difficulty::PartialCumulativeWork, +}; + +use super::fabricate::{FabHeader, Universe}; + +/// What the oracle expects an op to do. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum Prediction { + /// The store must accept the op. + Accept, + /// The store must reject the op and remain byte-identical. + Reject(RejectKind), + /// The op does not apply in the current state (unrealistic in production); + /// the harness must not execute it. + Skip(&'static str), +} + +/// Why the oracle expects a rejection. Kinds are advisory — the runner only +/// cross-checks the accept/reject axis; scripted scenarios assert exact error +/// variants themselves. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum RejectKind { + /// The anchor hash is not on the expected canonical chain. + UnknownAnchor, + /// The range conflicts below the finalized (body) tip. + ImmutableConflict, + /// The conflicting suffix does not carry strictly more work. + LowerWork, + /// The range is malformed (wrong anchor linkage or heights); the store is + /// expected to reject it through its linkage check (`UnlinkedRange`). + Malformed, +} + +/// A header range resolved against the universe: the anchor hash and the +/// fabricated rows to commit. +#[derive(Clone, Debug)] +pub(crate) struct ResolvedRange { + pub anchor: block::Hash, + pub rows: Vec, +} + +pub(crate) struct Oracle { + /// The expected canonical chain above genesis. + committed: BTreeMap, + /// The sequential body/finalized tip. Bodies exist at `0..=body_tip`. + body_tip: Height, + /// Every fabricated header in the universe, by hash. + index: HashMap, + genesis_hash: block::Hash, +} + +impl Oracle { + pub fn new(universe: &Universe) -> Self { + let mut index = HashMap::new(); + for fab in universe + .trunk + .iter() + .chain(universe.branches.iter().flat_map(|b| b.headers.iter())) + { + index.insert(fab.hash, fab.clone()); + } + + Oracle { + committed: BTreeMap::new(), + body_tip: Height(0), + index, + genesis_hash: universe.genesis.hash(), + } + } + + /// The expected canonical chain above genesis. + pub fn canonical_chain(&self) -> &BTreeMap { + &self.committed + } + + pub fn body_tip(&self) -> Height { + self.body_tip + } + + /// The next height a body commit may target (bodies are sequential). + pub fn next_body_height(&self) -> Height { + Height(self.body_tip.0 + 1) + } + + /// The fabricated header behind a canonical-chain hash. + pub fn fab_for(&self, hash: block::Hash) -> Option<&FabHeader> { + self.index.get(&hash) + } + + fn anchor_height(&self, anchor: block::Hash) -> Option { + if anchor == self.genesis_hash { + return Some(Height(0)); + } + self.committed + .iter() + .find(|(_, &hash)| hash == anchor) + .map(|(&height, _)| height) + } + + /// The first height in `rows` whose hash conflicts with the expected chain. + fn first_conflict(&self, rows: &[FabHeader]) -> Option { + rows.iter() + .find(|row| { + self.committed + .get(&row.height) + .is_some_and(|&existing| existing != row.hash) + }) + .map(|row| row.height) + } + + fn suffix_work_from(&self, from: Height) -> PartialCumulativeWork { + let mut work = PartialCumulativeWork::zero(); + for (_height, hash) in self.committed.range(from..) { + work += self + .index + .get(hash) + .expect("committed hashes come from the universe") + .work; + } + work + } + + /// Predicts the outcome of a header-range commit. + pub fn predict_header_range(&self, range: &ResolvedRange) -> Prediction { + if range.rows.is_empty() { + return Prediction::Skip("empty range"); + } + + let Some(anchor_height) = self.anchor_height(range.anchor) else { + return Prediction::Reject(RejectKind::UnknownAnchor); + }; + + // A well-formed range links to its anchor and starts right above it. + // Anything else must be rejected by the store's contextual validation. + let first = &range.rows[0]; + if first.header.previous_block_hash != range.anchor + || first.height != Height(anchor_height.0 + 1) + { + return Prediction::Reject(RejectKind::Malformed); + } + + if let Some(first_conflict) = self.first_conflict(&range.rows) { + if first_conflict <= self.body_tip { + return Prediction::Reject(RejectKind::ImmutableConflict); + } + + let existing_work = self.suffix_work_from(first_conflict); + let mut new_work = PartialCumulativeWork::zero(); + for row in &range.rows { + if row.height >= first_conflict { + new_work += row.work; + } + } + if new_work <= existing_work { + return Prediction::Reject(RejectKind::LowerWork); + } + } + + Prediction::Accept + } + + /// Applies an accepted header-range commit to the model: total suffix + /// replacement above the first conflicting height, then insert the rows. + pub fn apply_header_range(&mut self, range: &ResolvedRange) { + if let Some(first_conflict) = self.first_conflict(&range.rows) { + self.committed.split_off(&first_conflict); + } + for row in &range.rows { + self.committed.insert(row.height, row.hash); + } + } + + /// Predicts a body commit. Bodies are sequential in production, so only + /// `body_tip + 1` is realistic; the block may belong to any branch (a body + /// of the losing branch racing a header reorg). + pub fn predict_body(&self, fab: &FabHeader) -> Prediction { + if fab.height != self.next_body_height() { + return Prediction::Skip("bodies commit sequentially at body_tip + 1"); + } + Prediction::Accept + } + + /// Applies a body commit: the verified body wins over provisional headers, + /// truncating them when it conflicts. + pub fn apply_body(&mut self, fab: &FabHeader) { + if self.committed.get(&fab.height) != Some(&fab.hash) { + self.committed.split_off(&fab.height); + } + self.committed.insert(fab.height, fab.hash); + self.body_tip = fab.height; + } + + /// The hash at `height` in `fab`'s fabricated ancestry (walking + /// `previous_block_hash` links through the universe index). + fn ancestor_hash_at(&self, fab: &FabHeader, height: Height) -> block::Hash { + let mut current = fab.clone(); + while current.height.0 > height.0 + 1 { + current = self + .index + .get(¤t.header.previous_block_hash) + .expect("fabricated ancestry stays inside the universe down to genesis") + .clone(); + } + current.header.previous_block_hash + } + + /// Predicts a seed write (the non-finalized best-chain commit hook). + /// The non-finalized state only + /// holds blocks above the finalized tip, on chains rooted at it — a block + /// whose ancestry does not pass through the finalized tip cannot be an + /// nf best tip, so seeding it is unrealistic. Canonicality is decided by + /// the non-finalized state, so the seed bypasses the header-store work + /// gate. Note that an nf best-*tip* jump (a fork switch between nf chains, + /// or an nf-backup restore) legitimately seeds a height whose parent row + /// is missing or belongs to another branch. + pub fn predict_seed(&self, fab: &FabHeader) -> Prediction { + if fab.height <= self.body_tip { + return Prediction::Skip("seeds only happen above the finalized tip"); + } + + let attach_point = self.ancestor_hash_at(fab, self.body_tip); + let finalized_hash = if self.body_tip == Height(0) { + self.genesis_hash + } else { + *self + .committed + .get(&self.body_tip) + .expect("body heights stay on the expected chain") + }; + if attach_point != finalized_hash { + return Prediction::Skip("block is not attachable to the non-finalized state"); + } + + Prediction::Accept + } + + /// Whether a seeded block's parent is the expected canonical row below it. + /// The store refuses seeds that are not parent-linked as silent no-ops + /// (writing them would strand a row the chain walk cannot reach); the + /// harness uses this to decide whether a successful seed call must have + /// mutated the store or left it untouched. + pub fn seed_is_parent_linked(&self, fab: &FabHeader) -> bool { + let parent_height = Height(fab.height.0 - 1); + if parent_height == Height(0) { + fab.header.previous_block_hash == self.genesis_hash + } else { + self.committed.get(&parent_height) == Some(&fab.header.previous_block_hash) + } + } + + /// Applies a seed write: the seeded block is the new best chain at its + /// height, truncating any conflicting suffix. + pub fn apply_seed(&mut self, fab: &FabHeader) { + if self.committed.get(&fab.height) != Some(&fab.hash) { + self.committed.split_off(&fab.height); + } + self.committed.insert(fab.height, fab.hash); + } +} diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/primitive.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/primitive.rs new file mode 100644 index 00000000000..fc9c702ae17 --- /dev/null +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/primitive.rs @@ -0,0 +1,282 @@ +//! Contract tests for `set_canonical_suffix`, the single owner of zakura +//! header-store chain membership (REORG_PLAN Pillar 1). +//! +//! The coherence harness exercises the primitive through the production +//! writers (range, seed, release); these tests pin the primitive's own +//! contract directly: preconditions reject without staging anything, +//! deletion is total-above-fork across all five column families (including +//! rows stranded above hand-made gaps), an empty replacement truncates, and +//! a pure append stages no deletions (so the post-reorg audit hook does not +//! fire for it). + +use zebra_chain::block::Height; + +use super::super::super::CanonicalHeaderRow; +use super::super::super::{ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT, ZAKURA_HEADER_HASH_BY_HEIGHT}; +use super::super::common::{ + commit_header_range, state_with_genesis_config, write_full_block_header_and_transactions, +}; +use super::{ + audit::{audit_store, dump_store}, + fabricate::{fabricate_body, FabHeader, Universe, BRANCH_A, FORK_HEIGHT}, +}; +use crate::{ + error::{CommitHeaderRangeError, StoreIncoherentError}, + service::finalized_state::{ + disk_db::{DiskWriteBatch, WriteDisk}, + disk_format::block::TransactionLocation, + RawBytes, ZebraDb, + }, + Config, +}; + +fn trunk_state(universe: &Universe) -> ZebraDb { + let state = state_with_genesis_config( + &universe.network, + universe.genesis.clone(), + Config::ephemeral(), + ); + let trunk_headers: Vec<_> = universe.trunk[..FORK_HEIGHT as usize] + .iter() + .map(|fab| fab.header.clone()) + .collect(); + commit_header_range(&state, universe.genesis.hash(), &trunk_headers); + state +} + +fn rows_of(fabs: &[FabHeader]) -> Vec { + fabs.iter() + .map(|fab| CanonicalHeaderRow { + header: fab.header.clone(), + advertised_body_size: None, + roots: None, + }) + .collect() +} + +fn assert_clean(state: &ZebraDb) { + let violations = audit_store(state); + assert!( + violations.is_empty(), + "unexpected violations: {violations:?}" + ); +} + +/// New rows that do not link to the fork row are rejected before anything is +/// staged. +#[test] +fn rejects_unlinked_new_rows_without_staging() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe); + let original = dump_store(&state); + + // Branch-A rows link at the fork height (50), not at 30. + let rows = rows_of(&universe.branches[BRANCH_A].headers[..3]); + let mut batch = DiskWriteBatch::new(); + let error = batch + .set_canonical_suffix(&state, (Height(30), universe.trunk_at(30).hash), &rows) + .expect_err("rows do not link to trunk@30"); + assert!( + matches!( + error, + CommitHeaderRangeError::UnlinkedRange { height, .. } if height == Height(31) + ), + "expected UnlinkedRange at 31, got {error:?}" + ); + assert_eq!(batch.zakura_suffix_replaced_rows(), 0); + + state.write_batch(batch).expect("empty batch writes"); + assert_eq!(dump_store(&state), original, "a rejection stages nothing"); +} + +/// A fork hash that is not the stored row at the fork height is rejected: +/// the caller's fork decision must describe the store being mutated. +#[test] +fn rejects_stale_fork_row() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe); + let original = dump_store(&state); + + // These rows link to branch-A's fourth row, which is not stored at 30. + let foreign_fork = &universe.branches[BRANCH_A].headers[3]; + let rows = rows_of(&universe.branches[BRANCH_A].headers[4..6]); + let mut batch = DiskWriteBatch::new(); + let error = batch + .set_canonical_suffix(&state, (Height(30), foreign_fork.hash), &rows) + .expect_err("the stored row at 30 is the trunk's"); + assert!( + matches!( + error, + CommitHeaderRangeError::StoreIncoherent(StoreIncoherentError::BijectionMismatch { + height, + .. + }) if height == Height(30) + ), + "expected BijectionMismatch at the fork row, got {error:?}" + ); + assert_eq!(batch.zakura_suffix_replaced_rows(), 0); + assert_eq!(dump_store(&state), original); +} + +/// A fork below the finalized tip is rejected: committed heights are +/// immutable through the primitive. +#[test] +fn rejects_fork_below_finalized_tip() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe); + + // Commit the body of trunk height 1: the finalized tip moves to 1. + write_full_block_header_and_transactions(&state, fabricate_body(&universe.trunk[0])); + assert_eq!(state.finalized_tip_height(), Some(Height(1))); + + let rows = rows_of(&universe.trunk[..2]); + let mut batch = DiskWriteBatch::new(); + let error = batch + .set_canonical_suffix(&state, (Height(0), universe.genesis.hash()), &rows) + .expect_err("the fork is below the finalized tip"); + assert!( + matches!( + error, + CommitHeaderRangeError::ImmutableConflict { height } if height == Height(1) + ), + "expected ImmutableConflict, got {error:?}" + ); + assert_eq!(batch.zakura_suffix_replaced_rows(), 0); +} + +/// The committed-body safety interlock: a body row above the fork aborts the +/// replacement before anything is staged, even when the finalized-tip bound +/// cannot see it. +#[test] +fn body_above_fork_aborts_loudly() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe); + + // Hand-plant a body marker above the finalized tip — a state the + // orchestration must never produce. + let tx_by_loc = state.db.cf_handle("tx_by_loc").unwrap(); + let mut batch = DiskWriteBatch::new(); + batch.zs_insert( + &tx_by_loc, + TransactionLocation::min_for_height(Height(45)), + RawBytes::new_raw_bytes(vec![0u8]), + ); + state.db.write(batch).expect("raw insert writes"); + + let rows = rows_of(&universe.trunk[30..35]); + let mut batch = DiskWriteBatch::new(); + let error = batch + .set_canonical_suffix(&state, (Height(30), universe.trunk_at(30).hash), &rows) + .expect_err("a body above the fork must abort the replacement"); + assert!( + matches!( + error, + CommitHeaderRangeError::ConflictingFullBlockHeader { height } if height == Height(45) + ), + "expected ConflictingFullBlockHeader at 45, got {error:?}" + ); + assert_eq!(batch.zakura_suffix_replaced_rows(), 0); +} + +/// Deletion is total-above-fork: rows stranded above a hand-made gap and +/// stray rows in single column families are deleted along with the linked +/// suffix, in all five column families. +#[test] +fn truncation_is_total_above_fork_including_strays() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe); + + // A gap at 40 strands rows 41..=50; a stray body-size row sits at 60, + // above the zakura tip. + let hash_cf = state.db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); + let body_size_cf = state + .db + .cf_handle(ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT) + .unwrap(); + let mut batch = DiskWriteBatch::new(); + batch.zs_delete(&hash_cf, Height(40)); + batch.zs_insert( + &body_size_cf, + Height(60), + super::super::super::test_body_size(7), + ); + state.db.write(batch).expect("raw writes succeed"); + + let rows = rows_of(&universe.trunk[30..35]); + let mut batch = DiskWriteBatch::new(); + batch + .set_canonical_suffix(&state, (Height(30), universe.trunk_at(30).hash), &rows) + .expect("the replacement is well-formed"); + assert!(batch.zakura_suffix_replaced_rows() > 0); + state.write_batch(batch).expect("replacement batch writes"); + + assert_eq!( + state.best_header_tip(), + Some((Height(35), universe.trunk_at(35).hash)) + ); + let dump = dump_store(&state); + assert_eq!(dump.hashes.keys().next_back(), Some(&Height(35))); + assert!(!dump.body_sizes.contains_key(&Height(60))); + assert_clean(&state); +} + +/// An empty replacement truncates the store to the fork point. +#[test] +fn empty_replacement_truncates_to_fork() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe); + + let mut batch = DiskWriteBatch::new(); + batch + .set_canonical_suffix(&state, (Height(40), universe.trunk_at(40).hash), &[]) + .expect("an empty replacement is a truncation"); + assert!(batch.zakura_suffix_replaced_rows() > 0); + state.write_batch(batch).expect("truncation batch writes"); + + assert_eq!( + state.best_header_tip(), + Some((Height(40), universe.trunk_at(40).hash)) + ); + assert_clean(&state); +} + +/// A pure append deletes nothing, so it does not count as a reorg (the +/// post-reorg audit hook stays quiet for the steady-state extension path). +#[test] +fn pure_append_replaces_nothing() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe); + + // Branch A forks at the trunk tip, so its rows extend the stored chain. + let rows = rows_of(&universe.branches[BRANCH_A].headers[..3]); + let mut batch = DiskWriteBatch::new(); + batch + .set_canonical_suffix( + &state, + (Height(FORK_HEIGHT), universe.trunk_at(FORK_HEIGHT).hash), + &rows, + ) + .expect("an extension at the tip is well-formed"); + assert_eq!( + batch.zakura_suffix_replaced_rows(), + 0, + "a pure append is not a reorg" + ); + state.write_batch(batch).expect("append batch writes"); + + assert_eq!( + state.best_header_tip(), + Some(( + Height(FORK_HEIGHT + 3), + universe.branches[BRANCH_A].headers[2].hash + )) + ); + assert_clean(&state); +} diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/prop.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/prop.rs new file mode 100644 index 00000000000..4a363411354 --- /dev/null +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/prop.rs @@ -0,0 +1,88 @@ +//! Discovery proptest: random op sequences over the fixed universe, shrunk on +//! any audit failure to a minimal, transcribable counterexample. +//! +//! The sweep runs 64 cases in CI by default; widen it manually with: +//! +//! ```text +//! PROPTEST_CASES=4096 cargo test -p zebra-state --lib \ +//! header_store_coherence::prop -- --nocapture +//! ``` +//! +//! Every shrunk counterexample must be transcribed into `scenarios.rs` as a +//! hardcoded scenario (the primary, seed-independent pinning mechanism); the +//! proptest regression file under `zebra-state/proptest-regressions/` pins +//! the seed as a backstop. + +use std::env; + +use proptest::prelude::*; + +use super::{ + fabricate::TRUNK_LEN, + ops::{universe, Anchor, Harness, Op, Source}, +}; + +fn source_strategy() -> impl Strategy { + let branch_count = universe().branches.len(); + prop_oneof![ + 1 => Just(Source::Trunk), + 4 => (0..branch_count).prop_map(Source::Branch), + ] +} + +fn anchor_strategy() -> impl Strategy { + let branch_count = universe().branches.len(); + prop_oneof![ + // Mostly the natural anchor, so sequences build interesting states. + 9 => Just(Anchor::Auto), + // Sometimes adversarial: cross-chain and stale anchors. + 1 => prop_oneof![ + (1..=TRUNK_LEN as u32).prop_map(Anchor::TrunkAt), + (0..branch_count, 0..32usize).prop_map(|(branch, index)| Anchor::BranchAt(branch, index)), + Just(Anchor::Genesis), + ], + ] +} + +fn op_strategy() -> impl Strategy { + prop_oneof![ + 9 => (source_strategy(), 0..TRUNK_LEN, 1..=12usize, anchor_strategy()).prop_map( + |(source, offset, len, anchor)| Op::CommitHeaderRange { + source, + offset, + len, + anchor, + } + ), + 2 => (source_strategy(), 0..40usize) + .prop_map(|(source, index)| Op::CommitBody { source, index }), + 2 => (source_strategy(), 0..40usize).prop_map(|(source, index)| Op::Seed { source, index }), + 3 => (1..8usize).prop_map(|count| Op::Finalize { count }), + 1 => Just(Op::Reopen), + ] +} + +fn ops_strategy() -> impl Strategy> { + proptest::collection::vec(op_strategy(), 1..40) +} + +fn proptest_cases() -> u32 { + env::var("PROPTEST_CASES") + .ok() + .and_then(|cases| cases.parse().ok()) + .unwrap_or(64) +} + +proptest! { + #![proptest_config(ProptestConfig::with_cases(proptest_cases()))] + + /// The permanent invariant sweep: any failure is a store-corruption bug. + #[test] + fn prop_random_sequences_uphold_invariants(ops in ops_strategy()) { + let _init_guard = zebra_test::init(); + let mut harness = Harness::new(); + if let Err(report) = harness.run_all(&ops) { + prop_assert!(false, "store invariants violated:\n{report:#?}"); + } + } +} diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/reads.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/reads.rs new file mode 100644 index 00000000000..cceaf89d4da --- /dev/null +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/reads.rs @@ -0,0 +1,281 @@ +//! Read-path coherence: consensus readers verify store invariants as they +//! walk, surfacing corruption as an explicit [`StoreIncoherentError`] instead +//! of feeding stale rows into difficulty validation (REORG_PLAN Pillar 2). +//! +//! The write path refuses to create incoherent stores (the Phase-1.5 guards), +//! so these tests corrupt the column families directly — the same hand-made +//! corruption technique as the audit's own tests — and assert that: +//! +//! - `recent_header_context` returns `BrokenLinkage`/`Gap` instead of a +//! poisoned or silently shortened difficulty window, and +//! - the range writer rejects with `StoreIncoherent` (a local storage fault, +//! never a peer-attributed validation failure) and leaves the store +//! untouched, and +//! - the anchor round-trip distinguishes a bijection violation in our own +//! indexes (`BijectionMismatch`) from a genuinely unknown anchor. + +use std::sync::Arc; + +use zebra_chain::block::{self, Height}; + +use super::super::super::{ZAKURA_HEADER_BY_HEIGHT, ZAKURA_HEADER_HASH_BY_HEIGHT}; +use super::super::common::{commit_header_range, state_with_genesis_config}; +use super::{ + audit::dump_store, + fabricate::{Universe, BRANCH_A, FORK_HEIGHT}, +}; +use crate::{ + error::{CommitHeaderRangeError, StoreIncoherentError}, + service::finalized_state::{ + disk_db::{DiskWriteBatch, WriteDisk}, + ZebraDb, + }, + Config, +}; + +/// A store holding genesis plus the trunk up to the fork height, built through +/// the production write path. +fn trunk_state(universe: &Universe) -> ZebraDb { + let state = state_with_genesis_config( + &universe.network, + universe.genesis.clone(), + Config::ephemeral(), + ); + let trunk_headers: Vec<_> = universe.trunk[..FORK_HEIGHT as usize] + .iter() + .map(|fab| fab.header.clone()) + .collect(); + commit_header_range(&state, universe.genesis.hash(), &trunk_headers); + state +} + +/// Re-delivers a slice of trunk headers through the production range writer. +fn redeliver_trunk( + state: &ZebraDb, + universe: &Universe, + anchor_height: u32, + len: usize, +) -> Result { + let anchor = universe.trunk_at(anchor_height).hash; + let headers: Vec> = universe.trunk + [anchor_height as usize..anchor_height as usize + len] + .iter() + .map(|fab| fab.header.clone()) + .collect(); + let body_sizes = vec![0; headers.len()]; + let mut batch = DiskWriteBatch::new(); + let result = batch.prepare_header_range_batch(state, anchor, &headers, &body_sizes); + if result.is_ok() { + state + .write_batch(batch) + .expect("header range batch writes successfully"); + } + result.map(|outcome| outcome.tip_hash) +} + +/// A coherent store yields a full-span window mid-chain and a legitimately +/// short window near genesis. +#[test] +fn coherent_walks_return_ok_contexts() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe); + + let full = state + .recent_header_context(Height(40)) + .expect("coherent store walks cleanly"); + assert_eq!( + full.len(), + crate::service::check::difficulty::POW_ADJUSTMENT_BLOCK_SPAN + ); + + // Heights 5..=0 inclusive: six rows, then the walk stops at genesis. + let short = state + .recent_header_context(Height(5)) + .expect("a short walk ending at genesis is legitimate"); + assert_eq!(short.len(), 6); + + // A missing anchor row is not incoherence; the caller decides what an + // unknown anchor means. + let missing = state + .recent_header_context(Height(FORK_HEIGHT + 10)) + .expect("a missing starting row is not a violation"); + assert!(missing.is_empty()); +} + +/// A header row that is not the block its hash row names (the incident-shaped +/// poison: a stale row's (threshold, time) feeding the DAA window): the walk +/// reports the divergence instead of consuming the row, and the range writer +/// maps it to a side-effect-free `StoreIncoherent` rejection. +#[test] +fn foreign_header_row_is_reported_not_validated() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe); + + // Overwrite the header row at height 20 with a branch-A header while the + // hash rows keep claiming the trunk. + let header_cf = state.db.cf_handle(ZAKURA_HEADER_BY_HEIGHT).unwrap(); + let foreign_header = universe.branches[BRANCH_A].headers[3].header.clone(); + let mut batch = DiskWriteBatch::new(); + batch.zs_insert(&header_cf, Height(20), foreign_header); + state.db.write(batch).expect("raw insert writes"); + + let error = state + .recent_header_context(Height(30)) + .expect_err("the walk crosses the corrupted row"); + assert!( + matches!( + error, + StoreIncoherentError::HeaderHashMismatch { height, .. } if height == Height(20) + ), + "expected HeaderHashMismatch at the corrupted row, got {error:?}" + ); + + // The writer surfaces the same fault as a local rejection: the range is + // not blamed (no contextual-validation error) and nothing is written. + let dump_before = dump_store(&state); + let error = redeliver_trunk(&state, &universe, 30, 5) + .expect_err("validation context crosses the corrupted row"); + assert!( + matches!( + error, + CommitHeaderRangeError::StoreIncoherent( + StoreIncoherentError::HeaderHashMismatch { .. } + ) + ), + "expected StoreIncoherent(HeaderHashMismatch), got {error:?}" + ); + assert_eq!( + dump_store(&state), + dump_before, + "a StoreIncoherent rejection must be side-effect free" + ); +} + +/// A self-consistent foreign row (its header *is* the block its hash row +/// names, but it belongs to another branch): the row above it no longer links +/// down, and the walk reports the broken link from above. +#[test] +fn broken_linkage_is_reported_not_validated() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe); + + // Replace both the header and hash rows at height 20 with branch-A's + // fourth row: internally consistent, but trunk@21 does not link to it. + let foreign = &universe.branches[BRANCH_A].headers[3]; + let header_cf = state.db.cf_handle(ZAKURA_HEADER_BY_HEIGHT).unwrap(); + let hash_cf = state.db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); + let mut batch = DiskWriteBatch::new(); + batch.zs_insert(&header_cf, Height(20), foreign.header.clone()); + batch.zs_insert(&hash_cf, Height(20), foreign.hash); + state.db.write(batch).expect("raw insert writes"); + + let error = state + .recent_header_context(Height(30)) + .expect_err("the walk crosses the corrupted row"); + assert!( + matches!( + error, + StoreIncoherentError::BrokenLinkage { height, actual_below, .. } + if height == Height(21) && actual_below == foreign.hash + ), + "expected BrokenLinkage above the corrupted row, got {error:?}" + ); + + let error = redeliver_trunk(&state, &universe, 30, 5) + .expect_err("validation context crosses the corrupted row"); + assert!( + matches!( + error, + CommitHeaderRangeError::StoreIncoherent(StoreIncoherentError::BrokenLinkage { .. }) + ), + "expected StoreIncoherent(BrokenLinkage), got {error:?}" + ); +} + +/// A gap mid-window: a missing row below a stored one is incoherence, not the +/// end of history — a silently shortened window would shift the difficulty +/// adjustment instead of failing. +#[test] +fn gap_is_reported_not_shortened() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe); + + let header_cf = state.db.cf_handle(ZAKURA_HEADER_BY_HEIGHT).unwrap(); + let hash_cf = state.db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); + let mut batch = DiskWriteBatch::new(); + batch.zs_delete(&header_cf, Height(15)); + batch.zs_delete(&hash_cf, Height(15)); + state.db.write(batch).expect("raw delete writes"); + + let error = state + .recent_header_context(Height(25)) + .expect_err("the walk crosses the gap"); + assert!( + matches!( + error, + StoreIncoherentError::Gap { height, missing } + if height == Height(16) && missing == Height(15) + ), + "expected Gap below height 16, got {error:?}" + ); + + let error = + redeliver_trunk(&state, &universe, 25, 5).expect_err("validation context crosses the gap"); + assert!( + matches!( + error, + CommitHeaderRangeError::StoreIncoherent(StoreIncoherentError::Gap { .. }) + ), + "expected StoreIncoherent(Gap), got {error:?}" + ); +} + +/// A hash→height entry whose height→hash row disagrees is a bijection +/// violation in our own indexes: the anchor round-trip reports it as +/// `StoreIncoherent`, distinct from a genuinely unknown anchor. +#[test] +fn anchor_bijection_violation_is_store_incoherent_not_unknown() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe); + + // The hash row at height 30 now names a different block, while + // height_by_hash still maps the trunk hash to 30. + let hash_cf = state.db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); + let stray_hash = universe.branches[BRANCH_A].headers[0].hash; + let mut batch = DiskWriteBatch::new(); + batch.zs_insert(&hash_cf, Height(30), stray_hash); + state.db.write(batch).expect("raw insert writes"); + + let error = redeliver_trunk(&state, &universe, 30, 5) + .expect_err("the anchor round-trip fails on the corrupted index"); + assert!( + matches!( + error, + CommitHeaderRangeError::StoreIncoherent(StoreIncoherentError::BijectionMismatch { + hash, + height, + stored: Some(stored), + }) if hash == universe.trunk_at(30).hash + && height == Height(30) + && stored == stray_hash + ), + "expected StoreIncoherent(BijectionMismatch), got {error:?}" + ); + + // An anchor the store has never heard of stays UnknownAnchor. + let unknown = universe.branches[BRANCH_A].headers[10].hash; + let headers = vec![universe.trunk[30].header.clone()]; + let mut batch = DiskWriteBatch::new(); + let error = batch + .prepare_header_range_batch(&state, unknown, &headers, &[0]) + .expect_err("an unindexed anchor is unknown"); + assert!( + matches!(error, CommitHeaderRangeError::UnknownAnchor { .. }), + "expected UnknownAnchor, got {error:?}" + ); +} diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/scenarios.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/scenarios.rs new file mode 100644 index 00000000000..6e41141a72b --- /dev/null +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/scenarios.rs @@ -0,0 +1,736 @@ +//! Scripted scenarios: production event shapes replayed against the store. +//! +//! Every scenario runs with a per-op audit (via `Harness::run`), so any +//! passing scenario is a regression gate on the whole write sequence, not +//! just its final assertions. +//! +//! The `*_upholds_invariants` tests at the bottom are the permanent +//! regression gates for the three write-path corruption bugs this suite +//! originally proved with `corruption_repro_*` twins (removed together with +//! the write-path fixes; see the module README for the bug histories). + +use zebra_chain::block::Height; + +use super::{ + fabricate::{BRANCH_A, BRANCH_B, BRANCH_B_EXT, BRANCH_C, FORK_HEIGHT, TRUNK_LEN}, + ops::{universe, Anchor, Harness, Op, OpOutcome, Source}, +}; +use crate::error::CommitHeaderRangeError; + +/// The whole trunk in one range from the genesis anchor. +fn commit_trunk() -> Op { + Op::CommitHeaderRange { + source: Source::Trunk, + offset: 0, + len: TRUNK_LEN, + anchor: Anchor::Auto, + } +} + +/// A full branch in one range from its fork parent. +fn commit_branch(branch: usize) -> Op { + Op::CommitHeaderRange { + source: Source::Branch(branch), + offset: 0, + len: usize::MAX / 2, + anchor: Anchor::Auto, + } +} + +fn branch_tip(branch: usize) -> (Height, zebra_chain::block::Hash) { + let fab = universe().branches[branch] + .headers + .last() + .expect("branches are non-empty"); + (fab.height, fab.hash) +} + +fn assert_accepted(outcome: &OpOutcome) { + assert!( + matches!(outcome, OpOutcome::Accepted), + "expected acceptance, got {outcome:?}" + ); +} + +/// s01: extend with the trunk, finalize its base, reorg to the higher-work +/// branch A, and survive a restart. +#[test] +fn s01_extend_then_simple_reorg() { + let _init_guard = zebra_test::init(); + let mut harness = Harness::new(); + + let outcomes = harness + .run_all(&[ + commit_trunk(), + Op::Finalize { count: 5 }, + commit_branch(BRANCH_A), + Op::Reopen, + ]) + .expect("clean sequence has no violations"); + outcomes.iter().for_each(assert_accepted); + + assert_eq!( + harness.state().best_header_tip(), + Some(branch_tip(BRANCH_A)), + ); + assert_eq!(harness.oracle.body_tip(), Height(5)); +} + +/// s02: losing a work comparison once is not a +/// lasting verdict. Branch B first loses to A (`LowerWorkConflict`, store +/// untouched — checked by the harness), then B's extended version wins and +/// becomes canonical at the same fork point. +#[test] +fn s02_lower_work_conflict_non_terminal() { + let _init_guard = zebra_test::init(); + let mut harness = Harness::new(); + + harness + .run_all(&[commit_trunk(), commit_branch(BRANCH_A)]) + .expect("setup is clean"); + + // B is longer but carries less work: rejected, side-effect free. + let outcome = harness + .run(&commit_branch(BRANCH_B)) + .expect("rejection leaves the store coherent"); + match outcome.header_range_error() { + CommitHeaderRangeError::LowerWorkConflict { + height, + existing_work, + new_work, + } => { + assert_eq!(*height, Height(FORK_HEIGHT + 1)); + assert!( + new_work <= existing_work, + "the rejected suffix must not out-work the incumbent" + ); + } + other => panic!("expected LowerWorkConflict, got {other:?}"), + } + assert_eq!( + harness.state().best_header_tip(), + Some(branch_tip(BRANCH_A)), + "A stays canonical after the rejection", + ); + + // B_ext re-includes B's rejected prefix and now out-works A: the same + // fork point switches the moment the work balance flips. + let outcome = harness + .run(&commit_branch(BRANCH_B_EXT)) + .expect("the winning delivery is clean"); + assert_accepted(&outcome); + assert_eq!( + harness.state().best_header_tip(), + Some(branch_tip(BRANCH_B_EXT)), + ); +} + +/// s02b: the winning branch arrives split into two ranges. The first range +/// must already out-work A on its own (a losing prefix is rejected — that +/// shape is s03's walk-back). +#[test] +fn s02b_lower_work_then_win_split_ranges() { + let _init_guard = zebra_test::init(); + let mut harness = Harness::new(); + + harness + .run_all(&[commit_trunk(), commit_branch(BRANCH_A)]) + .expect("setup is clean"); + harness + .run(&commit_branch(BRANCH_B)) + .expect("rejection leaves the store coherent"); + + // B_ext minus its two margin headers still out-works A by construction. + let b_ext_len = universe().branches[BRANCH_B_EXT].headers.len(); + let outcomes = harness + .run_all(&[ + Op::CommitHeaderRange { + source: Source::Branch(BRANCH_B_EXT), + offset: 0, + len: b_ext_len - 2, + anchor: Anchor::Auto, + }, + Op::CommitHeaderRange { + source: Source::Branch(BRANCH_B_EXT), + offset: b_ext_len - 2, + len: 2, + anchor: Anchor::Auto, + }, + ]) + .expect("split delivery is clean"); + outcomes.iter().for_each(assert_accepted); + assert_eq!( + harness.state().best_header_tip(), + Some(branch_tip(BRANCH_B_EXT)), + ); +} + +/// s02d: a restart between the losing delivery and the winning one must not +/// change the outcome (nothing about the rejection may persist). +#[test] +fn s02d_lower_work_then_win_with_reopen_between() { + let _init_guard = zebra_test::init(); + let mut harness = Harness::new(); + + harness + .run_all(&[commit_trunk(), commit_branch(BRANCH_A)]) + .expect("setup is clean"); + harness + .run(&commit_branch(BRANCH_B)) + .expect("rejection leaves the store coherent"); + let outcomes = harness + .run_all(&[Op::Reopen, commit_branch(BRANCH_B_EXT), Op::Reopen]) + .expect("post-restart winning delivery is clean"); + outcomes.iter().for_each(assert_accepted); + assert_eq!( + harness.state().best_header_tip(), + Some(branch_tip(BRANCH_B_EXT)), + ); +} + +/// s03: the walk-back re-commit shape. The upper half of the winning branch +/// arrives first and fails with `UnknownAnchor`; the lower half lands (a +/// reorg); the upper half is re-delivered and extends it. +#[test] +fn s03_walkback_split_recommit() { + let _init_guard = zebra_test::init(); + let mut harness = Harness::new(); + + harness + .run_all(&[commit_trunk(), commit_branch(BRANCH_A)]) + .expect("setup is clean"); + + let b_ext_len = universe().branches[BRANCH_B_EXT].headers.len(); + let split = b_ext_len - 2; + + // Upper half first: its anchor is not committed yet. + let outcome = harness + .run(&Op::CommitHeaderRange { + source: Source::Branch(BRANCH_B_EXT), + offset: split, + len: 2, + anchor: Anchor::Auto, + }) + .expect("rejection leaves the store coherent"); + assert!( + matches!( + outcome.header_range_error(), + CommitHeaderRangeError::UnknownAnchor { .. } + ), + "expected UnknownAnchor, got {outcome:?}" + ); + + // Walk back: lower half (a winning reorg), then the upper half again. + let outcomes = harness + .run_all(&[ + Op::CommitHeaderRange { + source: Source::Branch(BRANCH_B_EXT), + offset: 0, + len: split, + anchor: Anchor::Auto, + }, + Op::CommitHeaderRange { + source: Source::Branch(BRANCH_B_EXT), + offset: split, + len: 2, + anchor: Anchor::Auto, + }, + ]) + .expect("walk-back re-commit is clean"); + outcomes.iter().for_each(assert_accepted); + assert_eq!( + harness.state().best_header_tip(), + Some(branch_tip(BRANCH_B_EXT)), + ); +} + +/// s04: a body commit racing a header reorg. After headers reorg to branch A, +/// body sync (still sequential on the old chain) commits the trunk block at +/// the fork's first height: the verified body wins and truncates A's rows. +/// Re-delivering A over the now-body-backed height must be refused. +#[test] +fn s04_body_commit_racing_header_reorg() { + let _init_guard = zebra_test::init(); + let mut harness = Harness::new(); + + harness + .run_all(&[ + commit_trunk(), + Op::Finalize { + count: FORK_HEIGHT as usize, + }, + commit_branch(BRANCH_A), + ]) + .expect("setup is clean"); + assert_eq!(harness.oracle.body_tip(), Height(FORK_HEIGHT)); + + // The racing body: the old-branch block right above the fork. + let outcome = harness + .run(&Op::CommitBody { + source: Source::Trunk, + index: FORK_HEIGHT as usize, // trunk row at height FORK_HEIGHT + 1 + }) + .expect("the verified body wins cleanly"); + assert_accepted(&outcome); + assert_eq!( + harness.state().best_header_tip(), + Some(( + Height(FORK_HEIGHT + 1), + universe().trunk_at(FORK_HEIGHT + 1).hash + )), + "the body truncated A's provisional rows", + ); + + // A cannot displace a committed body through the header path. + let outcome = harness + .run(&commit_branch(BRANCH_A)) + .expect("rejection leaves the store coherent"); + assert!( + matches!( + outcome.header_range_error(), + CommitHeaderRangeError::ImmutableConflict { .. } + | CommitHeaderRangeError::ConflictingFullBlockHeader { .. } + ), + "expected a body-conflict rejection, got {outcome:?}" + ); +} + +/// s05: the release path's frontier trim. Sequential body commits release +/// matching provisional rows one by one; header re-delivery *above* the body +/// tip is idempotent; restarts change nothing. (Re-delivery *over* body-backed +/// heights skips those heights — `redelivery_over_bodies_upholds_invariants`.) +#[test] +fn s05_release_trim_then_redelivery() { + let _init_guard = zebra_test::init(); + let mut harness = Harness::new(); + + let outcomes = harness + .run_all(&[ + commit_trunk(), + Op::Finalize { count: 5 }, + Op::Reopen, + Op::Finalize { count: 5 }, + // Re-delivery anchored at the body tip, over header-only heights. + Op::CommitHeaderRange { + source: Source::Trunk, + offset: 10, + len: 10, + anchor: Anchor::Auto, + }, + Op::Reopen, + ]) + .expect("release trim and redelivery are clean"); + outcomes.iter().for_each(assert_accepted); + assert_eq!(harness.oracle.body_tip(), Height(10)); + + let trunk_tip = universe().trunk_at(TRUNK_LEN as u32); + assert_eq!( + harness.state().best_header_tip(), + Some((trunk_tip.height, trunk_tip.hash)), + ); +} + +/// s06: a reorg to a *lower* height. The long low-work branch B is canonical; +/// the short high-work branch A replaces it, so the tip height decreases and +/// every one of B's rows above A's tip must be gone. +#[test] +fn s06_reorg_to_lower_height() { + let _init_guard = zebra_test::init(); + let mut harness = Harness::new(); + + let outcomes = harness + .run_all(&[commit_trunk(), commit_branch(BRANCH_B)]) + .expect("setup is clean"); + outcomes.iter().for_each(assert_accepted); + assert_eq!( + harness.state().best_header_tip(), + Some(branch_tip(BRANCH_B)), + ); + + let outcomes = harness + .run_all(&[commit_branch(BRANCH_A), Op::Reopen]) + .expect("the lower-height reorg strands nothing"); + outcomes.iter().for_each(assert_accepted); + assert_eq!( + harness.state().best_header_tip(), + Some(branch_tip(BRANCH_A)), + "the tip height decreased to A's tip", + ); +} + +/// s07: double reorg at the same fork point (B → A → B_ext), then the losing +/// branch's re-delivery is refused without side effects. +#[test] +fn s07_double_reorg_same_fork_point() { + let _init_guard = zebra_test::init(); + let mut harness = Harness::new(); + + let outcomes = harness + .run_all(&[ + commit_trunk(), + commit_branch(BRANCH_B), + commit_branch(BRANCH_A), + commit_branch(BRANCH_B_EXT), + ]) + .expect("double reorg is clean"); + outcomes.iter().for_each(assert_accepted); + assert_eq!( + harness.state().best_header_tip(), + Some(branch_tip(BRANCH_B_EXT)), + ); + + let outcome = harness + .run(&commit_branch(BRANCH_A)) + .expect("rejection leaves the store coherent"); + assert!( + matches!( + outcome.header_range_error(), + CommitHeaderRangeError::LowerWorkConflict { .. } + ), + "expected LowerWorkConflict, got {outcome:?}" + ); +} + +/// s08: activity across the DAA window edge. After a reorg whose fork point is +/// more than `POW_ADJUSTMENT_BLOCK_SPAN` below the tip, re-deliveries whose +/// difficulty context spans both the branch and the trunk must validate, and +/// a branch whose ancestry lost the reorg must be unknown. +#[test] +fn s08_reorg_across_daa_window_edge() { + let _init_guard = zebra_test::init(); + let mut harness = Harness::new(); + + harness + .run_all(&[ + commit_trunk(), + commit_branch(BRANCH_A), + commit_branch(BRANCH_B_EXT), + ]) + .expect("setup is clean"); + + // The fork is deeper than the 28-height DAA window below the tip. + let (tip_height, _) = branch_tip(BRANCH_B_EXT); + assert!(tip_height.0 - FORK_HEIGHT > 28); + + // Idempotent re-delivery anchored mid-branch: its context spans the + // branch/trunk boundary. + let outcome = harness + .run(&Op::CommitHeaderRange { + source: Source::Branch(BRANCH_B_EXT), + offset: 10, + len: 10, + anchor: Anchor::Auto, + }) + .expect("re-delivery across the window edge is clean"); + assert_accepted(&outcome); + + // C forks off A, which lost the reorg: its anchor is unknown now. + let outcome = harness + .run(&commit_branch(BRANCH_C)) + .expect("rejection leaves the store coherent"); + assert!( + matches!( + outcome.header_range_error(), + CommitHeaderRangeError::UnknownAnchor { .. } + ), + "expected UnknownAnchor, got {outcome:?}" + ); +} + +/// s09: s03 with a restart at every boundary — pins the crash-recovery +/// property that recovery is never a special code path. +#[test] +fn s09_restart_between_walkback_and_recommit() { + let _init_guard = zebra_test::init(); + let mut harness = Harness::new(); + + harness + .run_all(&[commit_trunk(), commit_branch(BRANCH_A)]) + .expect("setup is clean"); + + let b_ext_len = universe().branches[BRANCH_B_EXT].headers.len(); + let split = b_ext_len - 2; + + let outcome = harness + .run(&Op::CommitHeaderRange { + source: Source::Branch(BRANCH_B_EXT), + offset: split, + len: 2, + anchor: Anchor::Auto, + }) + .expect("rejection leaves the store coherent"); + assert!(matches!( + outcome.header_range_error(), + CommitHeaderRangeError::UnknownAnchor { .. } + )); + + let outcomes = harness + .run_all(&[ + Op::Reopen, + Op::CommitHeaderRange { + source: Source::Branch(BRANCH_B_EXT), + offset: 0, + len: split, + anchor: Anchor::Auto, + }, + Op::Reopen, + Op::CommitHeaderRange { + source: Source::Branch(BRANCH_B_EXT), + offset: split, + len: 2, + anchor: Anchor::Auto, + }, + Op::Reopen, + ]) + .expect("restart-interleaved walk-back is clean"); + outcomes.iter().for_each(assert_accepted); + assert_eq!( + harness.state().best_header_tip(), + Some(branch_tip(BRANCH_B_EXT)), + ); +} + +/// s10: the seed path (non-finalized best-chain commits) interleaved with +/// header ranges: seeds switch the canonical row at a height, truncating +/// conflicting suffixes, and header sync extends on top of seeded rows. +#[test] +fn s10_seed_interplay() { + let _init_guard = zebra_test::init(); + let mut harness = Harness::new(); + + let outcomes = harness + .run_all(&[ + commit_trunk(), + // The non-finalized best chain switches to A: seed its first rows. + Op::Seed { + source: Source::Branch(BRANCH_A), + index: 0, + }, + Op::Seed { + source: Source::Branch(BRANCH_A), + index: 1, + }, + // Header sync extends A on top of the seeded rows. + Op::CommitHeaderRange { + source: Source::Branch(BRANCH_A), + offset: 2, + len: usize::MAX / 2, + anchor: Anchor::Auto, + }, + Op::Reopen, + // The non-finalized best chain switches to B: the seed truncates A. + Op::Seed { + source: Source::Branch(BRANCH_B), + index: 0, + }, + Op::CommitHeaderRange { + source: Source::Branch(BRANCH_B), + offset: 1, + len: usize::MAX / 2, + anchor: Anchor::Auto, + }, + Op::Reopen, + ]) + .expect("seed interplay is clean"); + outcomes.iter().for_each(assert_accepted); + assert_eq!( + harness.state().best_header_tip(), + Some(branch_tip(BRANCH_B)), + ); +} + +/// s11: a refused (unlinked) seed converges through header-range sync. The +/// zakura store follows branch A (seeded at the fork height); the +/// non-finalized best chain switches to branch B and its new best tip (B's +/// *second* row) is seeded. The store refuses the unlinked seed as a no-op — +/// the header store briefly lags the nf chain — and the later linked range +/// delivery of B converges it onto the new branch. +#[test] +fn s11_refused_seed_converges_via_range_delivery() { + let _init_guard = zebra_test::init(); + let mut harness = Harness::new(); + + let outcomes = harness + .run_all(&[ + commit_trunk(), + Op::Seed { + source: Source::Branch(BRANCH_A), + index: 0, + }, + ]) + .expect("setup is clean"); + outcomes.iter().for_each(assert_accepted); + let a_first = &universe().branches[BRANCH_A].headers[0]; + assert_eq!( + harness.state().best_header_tip(), + Some((a_first.height, a_first.hash)), + ); + + // The nf best tip jumps to B[1]; its parent row is not stored, so the + // seed is refused without touching the store (checked by the harness). + let outcome = harness + .run(&Op::Seed { + source: Source::Branch(BRANCH_B), + index: 1, + }) + .expect("the refused seed leaves the store coherent"); + assert_accepted(&outcome); + assert_eq!( + harness.state().best_header_tip(), + Some((a_first.height, a_first.hash)), + "the refused seed must not move the header tip", + ); + + // Header sync catches up with the nf chain: B arrives as a linked range + // and out-works the stored A suffix, converging the store onto B. + let outcome = harness + .run(&commit_branch(BRANCH_B)) + .expect("the converging range delivery is clean"); + assert_accepted(&outcome); + assert_eq!( + harness.state().best_header_tip(), + Some(branch_tip(BRANCH_B)), + ); +} + +// --------------------------------------------------------------------------- +// Regression gates for the three proven write-path corruption bugs. Each was +// originally pinned by a `corruption_repro_*` test that demonstrated the +// violation; those were removed with the write-path fixes, and these twins +// now assert the fixed behavior over the exact same op sequences. +// --------------------------------------------------------------------------- + +/// The unlinked-anchor commit (bug 1): a range of branch-A headers anchored at +/// the same-height *trunk* hash passes contextual difficulty validation — the +/// two fast chains have identical (time, threshold) sequences — so before the +/// linkage check in `prepare_header_range_batch_with_roots`, it committed a +/// suffix that did not link to the row below it: an on-disk I2 violation +/// reachable from a single untrusted peer response. +fn unlinked_anchor_ops() -> Vec { + vec![ + commit_trunk(), + Op::CommitHeaderRange { + source: Source::Branch(BRANCH_A), + offset: 1, + len: usize::MAX / 2, + // The trunk hash at A[0]'s height: same height, wrong chain. + anchor: Anchor::TrunkAt(FORK_HEIGHT + 1), + }, + ] +} + +/// The store rejects unlinked ranges with `UnlinkedRange` and stays coherent. +#[test] +fn unlinked_anchor_commit_upholds_invariants() { + let _init_guard = zebra_test::init(); + let mut harness = Harness::new(); + let outcomes = harness + .run_all(&unlinked_anchor_ops()) + .expect("the store rejects unlinked ranges and stays coherent"); + assert!( + matches!( + outcomes[1].header_range_error(), + CommitHeaderRangeError::UnlinkedRange { .. } + ), + "expected UnlinkedRange, got {:?}", + outcomes[1] + ); +} + +/// Re-delivery over committed bodies (bug 2): before the committed-height +/// gate covered the header/hash/height/body-size writes (it originally gated +/// only the *roots* write), a header range re-delivered over heights whose +/// bodies had since been committed re-inserted zakura rows *below* the body +/// tip — rows the release trim (which already ran at body-commit time) never +/// removes. +fn redelivery_over_bodies_ops() -> Vec { + vec![ + commit_trunk(), + Op::Finalize { count: 10 }, + // Re-delivery spanning body-backed heights 6..=10 and header-only + // heights above: accepted, but the body-backed heights are skipped. + Op::CommitHeaderRange { + source: Source::Trunk, + offset: 5, + len: 15, + anchor: Anchor::Auto, + }, + ] +} + +/// Re-delivery over bodies leaves no zakura rows below the body tip. +#[test] +fn redelivery_over_bodies_upholds_invariants() { + let _init_guard = zebra_test::init(); + let mut harness = Harness::new(); + let outcomes = harness + .run_all(&redelivery_over_bodies_ops()) + .expect("re-delivery over bodies leaves no zakura rows below the body tip"); + outcomes.iter().for_each(assert_accepted); +} + +/// Seed above a gap (bug 3, minimal shape — found by the discovery proptest +/// and shrunk to a single op): before the parent-linkage refusal in +/// `prepare_zakura_header_from_committed_block`, seeding a block whose parent +/// row is absent left a row the chain walk cannot reach. +fn seed_above_gap_ops() -> Vec { + vec![Op::Seed { + source: Source::Trunk, + index: 1, // trunk height 2; nothing exists at height 1 + }] +} + +/// A seed above a gap is refused as a no-op instead of stranding a row. +#[test] +fn seed_above_gap_upholds_invariants() { + let _init_guard = zebra_test::init(); + let mut harness = Harness::new(); + harness + .run_all(&seed_above_gap_ops()) + .expect("a seed above a gap must not strand an unreachable row"); + + let genesis_hash = universe().genesis.hash(); + assert_eq!( + harness.state().best_header_tip(), + Some((Height(0), genesis_hash)), + "the refused seed must not move the header tip", + ); +} + +/// Seed fork switch (bug 3, production shape): the zakura store follows +/// branch A; the non-finalized best chain switches to branch B and its new +/// best tip (B's *second* row) is seeded. Before the parent-linkage refusal, +/// the seed's non-conflict arm inserted the row directly over A's truncated +/// parent — broken linkage on disk, the poisoned-DAA-window generator from +/// the production incident table. (`s11` proves the refused seed converges +/// later through range delivery.) +fn seed_fork_switch_ops() -> Vec { + vec![ + commit_trunk(), + Op::Seed { + source: Source::Branch(BRANCH_A), + index: 0, + }, + Op::Seed { + source: Source::Branch(BRANCH_B), + index: 1, + }, + ] +} + +/// A fork-switch seed keeps the store linked (the unlinked seed is refused). +#[test] +fn seed_fork_switch_upholds_invariants() { + let _init_guard = zebra_test::init(); + let mut harness = Harness::new(); + harness + .run_all(&seed_fork_switch_ops()) + .expect("a fork-switch seed must keep the store linked"); + + let a_first = &universe().branches[BRANCH_A].headers[0]; + assert_eq!( + harness.state().best_header_tip(), + Some((a_first.height, a_first.hash)), + "the store stays on A until a linked delivery of B arrives", + ); +} diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/startup_audit.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/startup_audit.rs new file mode 100644 index 00000000000..1ece0ee84cf --- /dev/null +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/startup_audit.rs @@ -0,0 +1,539 @@ +//! Startup-audit self-repair: a hand-corrupted zakura header store heals when +//! the database is (re)opened (REORG_PLAN Pillar 3, startup half). +//! +//! The write path refuses to create incoherent stores (the Phase-1.5 guards) +//! and the consensus readers refuse to consume them (Pillar 2), so these tests +//! corrupt the column families directly — the same hand-made corruption +//! technique as the audit's and read-path tests — and assert that: +//! +//! - the startup audit finds the violation and truncates the store to the +//! last coherent height in all five column families, leaving a store the +//! full coherence audit passes and header sync can re-download onto; +//! - the repair happens on the real startup path (`ZebraDb::new` at reopen) +//! and persists: a second reopen finds a coherent store and changes nothing; +//! - repair is minimal where truncation is not needed (an orphaned +//! reverse-index row, stale rows at committed heights); +//! - a store that failed the Pillar-2 linkage-verified reads passes them +//! after the heal; and +//! - the `state.zakura.header_store.incoherent` metric is emitted exactly +//! when a repair runs. + +use std::{ + collections::HashMap, + sync::{Arc, Mutex}, +}; + +use zebra_chain::block::Height; + +use super::super::super::startup_audit::ZakuraStoreViolation; +use super::super::super::{ + ZAKURA_HEADER_BY_HEIGHT, ZAKURA_HEADER_HASH_BY_HEIGHT, ZAKURA_HEADER_HEIGHT_BY_HASH, +}; +use super::super::common::{ + commit_header_range, persistent_config, persistent_state, state_with_genesis_config, +}; +use super::{ + audit::{audit_store, dump_store, StoreDump}, + fabricate::{Universe, BRANCH_A, FORK_HEIGHT}, +}; +use crate::{ + error::{CommitHeaderRangeError, StoreIncoherentError}, + service::finalized_state::{ + disk_db::{DiskWriteBatch, WriteDisk}, + ZebraDb, + }, + Config, +}; + +/// A store holding genesis plus the trunk up to the fork height, built through +/// the production write path, on the given (persistent or ephemeral) config. +fn trunk_state(universe: &Universe, config: Config) -> ZebraDb { + let state = state_with_genesis_config(&universe.network, universe.genesis.clone(), config); + let trunk_headers: Vec<_> = universe.trunk[..FORK_HEIGHT as usize] + .iter() + .map(|fab| fab.header.clone()) + .collect(); + commit_header_range(&state, universe.genesis.hash(), &trunk_headers); + state +} + +/// Closes `state` and reopens the same database, running the startup audit. +fn reopen(state: ZebraDb, config: &Config, universe: &Universe) -> ZebraDb { + let mut state = state; + state.shutdown(true); + drop(state); + persistent_state(config, &universe.network) +} + +/// The expected store after truncating everything above `last` (the original +/// coherent dump with all five column families filtered to `..= last`). +fn truncated(dump: &StoreDump, last: Height) -> StoreDump { + StoreDump { + headers: dump + .headers + .iter() + .filter(|(height, _)| **height <= last) + .map(|(height, header)| (*height, header.clone())) + .collect(), + hashes: dump + .hashes + .iter() + .filter(|(height, _)| **height <= last) + .map(|(height, hash)| (*height, *hash)) + .collect(), + heights_by_hash: dump + .heights_by_hash + .iter() + .filter(|(_, points_at)| *points_at <= last) + .copied() + .collect(), + body_sizes: dump + .body_sizes + .iter() + .filter(|(height, _)| **height <= last) + .map(|(height, size)| (*height, *size)) + .collect(), + roots: dump + .roots + .iter() + .filter(|(height, _)| **height <= last) + .map(|(height, roots)| (*height, *roots)) + .collect(), + } +} + +fn assert_clean(state: &ZebraDb) { + let violations = audit_store(state); + assert!( + violations.is_empty(), + "expected a coherent store after repair: {violations:?}" + ); +} + +/// A coherent store is untouched: the direct audit reports nothing, and a +/// reopen through the real startup path changes no rows. +#[test] +fn startup_audit_is_noop_on_coherent_store() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let tempdir = tempfile::tempdir().expect("test tempdir is available"); + let config = persistent_config(tempdir.path()); + let state = trunk_state(&universe, config.clone()); + let original = dump_store(&state); + + let repair = state + .audit_and_repair_zakura_header_store() + .expect("audit reads and writes succeed"); + assert!( + repair.is_none(), + "no repair expected on a coherent store: {repair:?}" + ); + assert_eq!(dump_store(&state), original); + + let state = reopen(state, &config, &universe); + assert_eq!( + dump_store(&state), + original, + "a reopen must not change a coherent store" + ); +} + +/// Broken linkage mid-frontier (a self-consistent foreign row spliced into the +/// height index — the incident-shaped poison): the audit truncates to the row +/// below the splice, the heal persists across a further reopen, and header +/// sync re-downloads the truncated suffix back to the original store. +#[test] +fn startup_heals_broken_linkage_then_resync_converges() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let tempdir = tempfile::tempdir().expect("test tempdir is available"); + let config = persistent_config(tempdir.path()); + let state = trunk_state(&universe, config.clone()); + let original = dump_store(&state); + + // Replace the header and hash rows at height 20 with branch-A's fourth + // row: internally consistent, but it does not link to trunk@19. + let foreign = &universe.branches[BRANCH_A].headers[3]; + let header_cf = state.db.cf_handle(ZAKURA_HEADER_BY_HEIGHT).unwrap(); + let hash_cf = state.db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); + let mut batch = DiskWriteBatch::new(); + batch.zs_insert(&header_cf, Height(20), foreign.header.clone()); + batch.zs_insert(&hash_cf, Height(20), foreign.hash); + state.db.write(batch).expect("raw insert writes"); + + // The corruption is visible to the Pillar-2 reads before the heal. + let error = state + .recent_header_context(Height(30)) + .expect_err("the walk crosses the corrupted row"); + assert!(matches!(error, StoreIncoherentError::BrokenLinkage { .. })); + + let repair = state + .audit_and_repair_zakura_header_store() + .expect("audit reads and writes succeed") + .expect("the corrupted store is repaired"); + assert_eq!( + repair.last_coherent, + Some((Height(19), universe.trunk_at(19).hash)) + ); + assert!( + repair.violations.iter().any(|violation| matches!( + violation, + ZakuraStoreViolation::BrokenLinkage { height, actual_below, .. } + if *height == Height(20) && *actual_below == universe.trunk_at(19).hash + )), + "expected BrokenLinkage at the splice: {:?}", + repair.violations + ); + + // Everything above the last coherent height is gone, in all five CFs. + assert_eq!(dump_store(&state), truncated(&original, Height(19))); + assert_clean(&state); + state + .recent_header_context(Height(19)) + .expect("the healed store walks cleanly"); + + // The heal persists: a reopen through the startup path changes nothing. + let state = reopen(state, &config, &universe); + assert_eq!(dump_store(&state), truncated(&original, Height(19))); + + // Header sync re-downloads the truncated suffix and converges back onto + // the original chain. + let redelivered: Vec<_> = universe.trunk[19..FORK_HEIGHT as usize] + .iter() + .map(|fab| fab.header.clone()) + .collect(); + commit_header_range(&state, universe.trunk_at(19).hash, &redelivered); + assert_eq!(dump_store(&state), original); + assert_clean(&state); +} + +/// A gap mid-frontier strands every row above it: reopening alone (no direct +/// audit call) heals the store, proving the `ZebraDb::new` hook fires. +#[test] +fn startup_heals_gap_and_stranded_rows_on_reopen() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let tempdir = tempfile::tempdir().expect("test tempdir is available"); + let config = persistent_config(tempdir.path()); + let state = trunk_state(&universe, config.clone()); + let original = dump_store(&state); + + let header_cf = state.db.cf_handle(ZAKURA_HEADER_BY_HEIGHT).unwrap(); + let hash_cf = state.db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); + let mut batch = DiskWriteBatch::new(); + batch.zs_delete(&header_cf, Height(15)); + batch.zs_delete(&hash_cf, Height(15)); + state.db.write(batch).expect("raw delete writes"); + + let state = reopen(state, &config, &universe); + + // Rows 16..=50 were stranded above the gap; the audit truncated to 14. + assert_eq!(dump_store(&state), truncated(&original, Height(14))); + assert_clean(&state); + assert_eq!( + state.best_header_tip(), + Some((Height(14), universe.trunk_at(14).hash)) + ); +} + +/// A missing reverse-index row breaks the hash↔height bijection: the audit +/// truncates to the row below it. +#[test] +fn startup_heals_missing_reverse_index_row() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe, Config::ephemeral()); + let original = dump_store(&state); + + let height_by_hash_cf = state.db.cf_handle(ZAKURA_HEADER_HEIGHT_BY_HASH).unwrap(); + let mut batch = DiskWriteBatch::new(); + batch.zs_delete(&height_by_hash_cf, universe.trunk_at(10).hash); + state.db.write(batch).expect("raw delete writes"); + + let repair = state + .audit_and_repair_zakura_header_store() + .expect("audit reads and writes succeed") + .expect("the corrupted store is repaired"); + assert_eq!( + repair.last_coherent, + Some((Height(9), universe.trunk_at(9).hash)) + ); + assert!( + repair.violations.iter().any(|violation| matches!( + violation, + ZakuraStoreViolation::WrongHeightByHash { height, indexed: None, .. } + if *height == Height(10) + )), + "expected WrongHeightByHash at the deleted entry: {:?}", + repair.violations + ); + assert_eq!(dump_store(&state), truncated(&original, Height(9))); + assert_clean(&state); +} + +/// An orphaned reverse-index row (a stranded `height_by_hash` entry from an +/// overwrite that never cleaned up the displaced hash) is removed on its own: +/// the linked chain is coherent, so nothing is truncated. +#[test] +fn startup_removes_orphan_reverse_row_without_truncating() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe, Config::ephemeral()); + let original = dump_store(&state); + + let height_by_hash_cf = state.db.cf_handle(ZAKURA_HEADER_HEIGHT_BY_HASH).unwrap(); + let stray_hash = universe.branches[BRANCH_A].headers[0].hash; + let mut batch = DiskWriteBatch::new(); + batch.zs_insert(&height_by_hash_cf, stray_hash, Height(12)); + state.db.write(batch).expect("raw insert writes"); + + let repair = state + .audit_and_repair_zakura_header_store() + .expect("audit reads and writes succeed") + .expect("the corrupted store is repaired"); + assert_eq!( + repair.last_coherent, + Some((Height(FORK_HEIGHT), universe.trunk_at(FORK_HEIGHT).hash)), + "an orphan reverse row must not shorten the coherent chain" + ); + assert_eq!(repair.deleted_rows, 1); + assert!( + repair.violations.iter().any(|violation| matches!( + violation, + ZakuraStoreViolation::OrphanHeightByHash { hash, points_at } + if *hash == stray_hash && *points_at == Height(12) + )), + "expected OrphanHeightByHash for the stray entry: {:?}", + repair.violations + ); + assert_eq!(dump_store(&state), original); + assert_clean(&state); +} + +/// Stale zakura rows at a committed height (the pre-guard re-delivery shape: +/// rows the release trim never touches again) are removed without disturbing +/// the frontier above. +#[test] +fn startup_removes_stale_rows_at_committed_heights() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe, Config::ephemeral()); + let original = dump_store(&state); + + // Genesis is the only committed block; plant zakura rows at its height. + let header_cf = state.db.cf_handle(ZAKURA_HEADER_BY_HEIGHT).unwrap(); + let hash_cf = state.db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); + let height_by_hash_cf = state.db.cf_handle(ZAKURA_HEADER_HEIGHT_BY_HASH).unwrap(); + let genesis_hash = universe.genesis.hash(); + let mut batch = DiskWriteBatch::new(); + batch.zs_insert(&header_cf, Height(0), universe.genesis.header.clone()); + batch.zs_insert(&hash_cf, Height(0), genesis_hash); + batch.zs_insert(&height_by_hash_cf, genesis_hash, Height(0)); + state.db.write(batch).expect("raw insert writes"); + + let repair = state + .audit_and_repair_zakura_header_store() + .expect("audit reads and writes succeed") + .expect("the corrupted store is repaired"); + assert_eq!( + repair.last_coherent, + Some((Height(FORK_HEIGHT), universe.trunk_at(FORK_HEIGHT).hash)), + "stale committed-height rows must not shorten the frontier" + ); + assert_eq!(repair.deleted_rows, 3); + assert!( + repair.violations.iter().all(|violation| matches!( + violation, + ZakuraStoreViolation::StaleRowAtCommittedHeight { height, .. } + if *height == Height(0) + )), + "expected only StaleRowAtCommittedHeight at genesis: {:?}", + repair.violations + ); + assert_eq!(dump_store(&state), original); + assert_clean(&state); +} + +/// The reads.rs anchor-corruption shape (a hash row overwritten with a foreign +/// hash) makes the Pillar-2 range writer reject with `StoreIncoherent`; after +/// the heal the same delivery commits. This pins the Pillar-2 interaction: any +/// store those reads refuse is healed by this audit. +#[test] +fn startup_heal_unblocks_linkage_verified_reads() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe, Config::ephemeral()); + let original = dump_store(&state); + + // The hash row at height 30 now names a different block, while the header + // row and `height_by_hash` still claim the trunk. + let hash_cf = state.db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); + let stray_hash = universe.branches[BRANCH_A].headers[0].hash; + let mut batch = DiskWriteBatch::new(); + batch.zs_insert(&hash_cf, Height(30), stray_hash); + state.db.write(batch).expect("raw insert writes"); + + // A re-delivered range anchored at trunk@30 is rejected as a local + // storage fault before the heal. + let anchor = universe.trunk_at(30).hash; + let headers: Vec<_> = universe.trunk[30..35] + .iter() + .map(|fab| fab.header.clone()) + .collect(); + let body_sizes = vec![0; headers.len()]; + let mut batch = DiskWriteBatch::new(); + let error = batch + .prepare_header_range_batch(&state, anchor, &headers, &body_sizes) + .expect_err("the anchor round-trip fails on the corrupted index"); + assert!(matches!( + error, + CommitHeaderRangeError::StoreIncoherent(StoreIncoherentError::BijectionMismatch { .. }) + )); + + let repair = state + .audit_and_repair_zakura_header_store() + .expect("audit reads and writes succeed") + .expect("the corrupted store is repaired"); + assert_eq!( + repair.last_coherent, + Some((Height(29), universe.trunk_at(29).hash)) + ); + assert!( + repair.violations.iter().any(|violation| matches!( + violation, + ZakuraStoreViolation::HeaderHashMismatch { height, indexed, .. } + if *height == Height(30) && *indexed == stray_hash + )), + "expected HeaderHashMismatch at the overwritten hash row: {:?}", + repair.violations + ); + assert_eq!(dump_store(&state), truncated(&original, Height(29))); + assert_clean(&state); + + // The same fork of history now commits, anchored at the healed tip. + let redelivered: Vec<_> = universe.trunk[29..FORK_HEIGHT as usize] + .iter() + .map(|fab| fab.header.clone()) + .collect(); + commit_header_range(&state, universe.trunk_at(29).hash, &redelivered); + assert_eq!(dump_store(&state), original); + assert_clean(&state); +} + +/// A minimal local metrics recorder capturing counter increments by name. +#[derive(Clone, Default)] +struct CounterCapture(Arc>>); + +impl CounterCapture { + fn get(&self, name: &str) -> u64 { + self.0 + .lock() + .expect("counter capture lock is never poisoned") + .get(name) + .copied() + .unwrap_or(0) + } +} + +struct CaptureHandle { + name: String, + store: Arc>>, +} + +impl metrics::CounterFn for CaptureHandle { + fn increment(&self, value: u64) { + *self + .store + .lock() + .expect("counter capture lock is never poisoned") + .entry(self.name.clone()) + .or_insert(0) += value; + } + + fn absolute(&self, value: u64) { + self.store + .lock() + .expect("counter capture lock is never poisoned") + .insert(self.name.clone(), value); + } +} + +impl metrics::Recorder for CounterCapture { + fn describe_counter( + &self, + _: metrics::KeyName, + _: Option, + _: metrics::SharedString, + ) { + } + + fn describe_gauge( + &self, + _: metrics::KeyName, + _: Option, + _: metrics::SharedString, + ) { + } + + fn describe_histogram( + &self, + _: metrics::KeyName, + _: Option, + _: metrics::SharedString, + ) { + } + + fn register_counter(&self, key: &metrics::Key, _: &metrics::Metadata<'_>) -> metrics::Counter { + metrics::Counter::from_arc(Arc::new(CaptureHandle { + name: key.name().to_string(), + store: self.0.clone(), + })) + } + + fn register_gauge(&self, _: &metrics::Key, _: &metrics::Metadata<'_>) -> metrics::Gauge { + metrics::Gauge::noop() + } + + fn register_histogram( + &self, + _: &metrics::Key, + _: &metrics::Metadata<'_>, + ) -> metrics::Histogram { + metrics::Histogram::noop() + } +} + +/// The `state.zakura.header_store.incoherent` metric fires exactly when a +/// startup repair runs: once for the healing reopen, and not at all for the +/// clean reopen after it. +#[test] +fn startup_repair_emits_incoherent_metric() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let tempdir = tempfile::tempdir().expect("test tempdir is available"); + let config = persistent_config(tempdir.path()); + let state = trunk_state(&universe, config.clone()); + + let header_cf = state.db.cf_handle(ZAKURA_HEADER_BY_HEIGHT).unwrap(); + let hash_cf = state.db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); + let mut batch = DiskWriteBatch::new(); + batch.zs_delete(&header_cf, Height(15)); + batch.zs_delete(&hash_cf, Height(15)); + state.db.write(batch).expect("raw delete writes"); + + const METRIC: &str = "state.zakura.header_store.incoherent"; + + let healing = CounterCapture::default(); + let state = metrics::with_local_recorder(&healing, || reopen(state, &config, &universe)); + assert_eq!( + healing.get(METRIC), + 1, + "the healing reopen emits the metric" + ); + assert_clean(&state); + + let clean = CounterCapture::default(); + let state = metrics::with_local_recorder(&clean, || reopen(state, &config, &universe)); + assert_eq!(clean.get(METRIC), 0, "a clean reopen emits nothing"); + assert_clean(&state); +} From 9a75ea32ce7a442a1879a271e3ea0c57f1e29f89 Mon Sep 17 00:00:00 2001 From: roman Date: Mon, 6 Jul 2026 21:37:50 -0600 Subject: [PATCH 26/27] test(state): drive reorging header-range commits end-to-end through the switch orchestration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the T1 suite from INTEGRATION_TEST_PLAN.md: request → body rollback → header rewrite → response through the real write-worker function, against a real header store and non-finalized state. - T1.1 reorging commit: store switches, stranded suffix rolls back to the fork, truncated chain published, response reports the reorg, and the post-reorg audit runs (clean run asserts no repair metric; a planted stray row asserts the hook repairs and emits it). - T1.2 kill-between-batches (the REORG_PLAN §2 crash-table commitment): rollback ran, prepared batch dropped unwritten — the intermediate is coherent, a reopen passes the startup audit with no repair, and re-running the same commit converges (recovery is the ordinary flow). - T1.3 whole-suffix switch: a reorg at the nf root empties the chain set without panicking and publishes the emptied state. - T1.4 losing-then-winning branch: lower-work conflict rejected with byte-identical stores, higher-work extension switches, the original branch switches back, and its rolled-back bodies recommit through the full validation path — rollback-not-ban end to end. Fixture design per the spike recorded in the plan: fabricated bodies need V4 coinbases (fabricate_v4_body) and a network whose upgrades activate above the test heights (pre_heartwood_test_network), so nf commits run in the pre-Heartwood regime like the existing fake-mainnet fixtures. CounterCapture moves to the shared test helpers; the block test modules become pub(crate) so the write tests can reuse the coherence fixtures. --- zebra-state/src/service/finalized_state.rs | 2 +- .../service/finalized_state/zebra_db/block.rs | 2 +- .../finalized_state/zebra_db/block/tests.rs | 4 +- .../zebra_db/block/tests/common.rs | 105 ++- .../block/tests/header_store_coherence.rs | 4 +- .../tests/header_store_coherence/fabricate.rs | 66 ++ .../header_store_coherence/startup_audit.rs | 89 +-- zebra-state/src/service/write.rs | 2 + .../write/switch_orchestration_tests.rs | 632 ++++++++++++++++++ 9 files changed, 803 insertions(+), 103 deletions(-) create mode 100644 zebra-state/src/service/write/switch_orchestration_tests.rs diff --git a/zebra-state/src/service/finalized_state.rs b/zebra-state/src/service/finalized_state.rs index 98337f5c13e..0d69b006ddc 100644 --- a/zebra-state/src/service/finalized_state.rs +++ b/zebra-state/src/service/finalized_state.rs @@ -79,7 +79,7 @@ pub(crate) mod commitment_aux_verify; mod disk_db; mod disk_format; mod vct; -mod zebra_db; +pub(crate) mod zebra_db; use vct::{VctCommitState, VctState, VctWriteData}; diff --git a/zebra-state/src/service/finalized_state/zebra_db/block.rs b/zebra-state/src/service/finalized_state/zebra_db/block.rs index 612d6e895b3..444de13e22a 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block.rs @@ -64,7 +64,7 @@ mod startup_audit; pub(crate) use canonical_suffix::CanonicalHeaderRow; #[cfg(test)] -mod tests; +pub(crate) mod tests; const ZAKURA_HEADER_HASH_BY_HEIGHT: &str = "zakura_header_hash_by_height"; const ZAKURA_HEADER_HEIGHT_BY_HASH: &str = "zakura_header_height_by_hash"; diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests.rs index b7d89ad8027..76b27a839c6 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests.rs @@ -20,8 +20,8 @@ use crate::{ Config, }; -mod common; -mod header_store_coherence; +pub(crate) mod common; +pub(crate) mod header_store_coherence; mod prune; mod snapshot; mod vectors; diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/common.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/common.rs index 249be03aff6..aeeaaa1c0d5 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/common.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/common.rs @@ -3,7 +3,11 @@ //! These helpers are used by both the fixed test vectors (`vectors.rs`) and the //! header-store coherence harness (`header_store_coherence`). -use std::{path::Path, sync::Arc}; +use std::{ + collections::HashMap, + path::Path, + sync::{Arc, Mutex}, +}; use zebra_chain::{ block::{self, Block, Height}, @@ -24,7 +28,7 @@ use crate::{ }; /// Returns an ephemeral or configured state database with `genesis` committed as a full block. -pub(super) fn state_with_genesis_config( +pub(crate) fn state_with_genesis_config( network: &Network, genesis: Arc, config: Config, @@ -47,7 +51,7 @@ pub(super) fn state_with_genesis_config( } /// Returns a persistent state config rooted at `cache_dir`, for close-and-reopen tests. -pub(super) fn persistent_config(cache_dir: &Path) -> Config { +pub(crate) fn persistent_config(cache_dir: &Path) -> Config { Config { cache_dir: cache_dir.to_owned(), ephemeral: false, @@ -57,7 +61,7 @@ pub(super) fn persistent_config(cache_dir: &Path) -> Config { } /// Opens (or reopens) a persistent state database from `config`. -pub(super) fn persistent_state(config: &Config, network: &Network) -> ZebraDb { +pub(crate) fn persistent_state(config: &Config, network: &Network) -> ZebraDb { ZebraDb::new( config, STATE_DATABASE_KIND, @@ -73,7 +77,7 @@ pub(super) fn persistent_state(config: &Config, network: &Network) -> ZebraDb { /// Returns a configured testnet with only the implicit genesis checkpoint, so header /// commits above genesis take the contextual validation path. -pub(super) fn no_extra_checkpoint_test_network(genesis_hash: block::Hash) -> Network { +pub(crate) fn no_extra_checkpoint_test_network(genesis_hash: block::Hash) -> Network { testnet::Parameters::build() .with_network_name("HeaderReorgTest") .expect("test network name is valid") @@ -94,7 +98,7 @@ pub(super) fn no_extra_checkpoint_test_network(genesis_hash: block::Hash) -> Net } /// Deserializes the mainnet test vector block at `height`. -pub(super) fn mainnet_block(height: u32) -> Arc { +pub(crate) fn mainnet_block(height: u32) -> Arc { MAINNET_BLOCKS .get(&height) .expect("test vector exists") @@ -104,7 +108,7 @@ pub(super) fn mainnet_block(height: u32) -> Arc { /// Fabricates provisional commitment roots for `height`, with the zeroed /// auth-data root marking them as unverified. -pub(super) fn root_at(height: Height) -> BlockCommitmentRoots { +pub(crate) fn root_at(height: Height) -> BlockCommitmentRoots { BlockCommitmentRoots { height, sapling_root: sapling::tree::NoteCommitmentTree::default().root(), @@ -118,7 +122,7 @@ pub(super) fn root_at(height: Height) -> BlockCommitmentRoots { } /// Commits a header range through the production write path, panicking on rejection. -pub(super) fn commit_header_range( +pub(crate) fn commit_header_range( state: &ZebraDb, anchor: block::Hash, headers: &[Arc], @@ -136,7 +140,7 @@ pub(super) fn commit_header_range( /// Commits `block`'s header and transaction data (the body-commit batch shape), /// without treestates or value pools. -pub(super) fn write_full_block_header_and_transactions(state: &ZebraDb, block: Arc) { +pub(crate) fn write_full_block_header_and_transactions(state: &ZebraDb, block: Arc) { let checkpoint_verified = CheckpointVerifiedBlock::from(block); let finalized = FinalizedBlock::from_checkpoint_verified(checkpoint_verified, Treestate::default()); @@ -147,3 +151,86 @@ pub(super) fn write_full_block_header_and_transactions(state: &ZebraDb, block: A .expect("full block header and transaction batch is valid"); state.db.write(batch).expect("full block batch writes"); } + +/// A minimal local metrics recorder capturing counter increments by name. +#[derive(Clone, Default)] +pub(crate) struct CounterCapture(Arc>>); + +impl CounterCapture { + pub(crate) fn get(&self, name: &str) -> u64 { + self.0 + .lock() + .expect("counter capture lock is never poisoned") + .get(name) + .copied() + .unwrap_or(0) + } +} + +struct CaptureHandle { + name: String, + store: Arc>>, +} + +impl metrics::CounterFn for CaptureHandle { + fn increment(&self, value: u64) { + *self + .store + .lock() + .expect("counter capture lock is never poisoned") + .entry(self.name.clone()) + .or_insert(0) += value; + } + + fn absolute(&self, value: u64) { + self.store + .lock() + .expect("counter capture lock is never poisoned") + .insert(self.name.clone(), value); + } +} + +impl metrics::Recorder for CounterCapture { + fn describe_counter( + &self, + _: metrics::KeyName, + _: Option, + _: metrics::SharedString, + ) { + } + + fn describe_gauge( + &self, + _: metrics::KeyName, + _: Option, + _: metrics::SharedString, + ) { + } + + fn describe_histogram( + &self, + _: metrics::KeyName, + _: Option, + _: metrics::SharedString, + ) { + } + + fn register_counter(&self, key: &metrics::Key, _: &metrics::Metadata<'_>) -> metrics::Counter { + metrics::Counter::from_arc(Arc::new(CaptureHandle { + name: key.name().to_string(), + store: self.0.clone(), + })) + } + + fn register_gauge(&self, _: &metrics::Key, _: &metrics::Metadata<'_>) -> metrics::Gauge { + metrics::Gauge::noop() + } + + fn register_histogram( + &self, + _: &metrics::Key, + _: &metrics::Metadata<'_>, + ) -> metrics::Histogram { + metrics::Histogram::noop() + } +} diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence.rs index b0adbdc2c23..b669ff4f10d 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence.rs @@ -27,8 +27,8 @@ //! (`insert_zakura_header_commitment_roots`), `rollback_finalized_state`, and //! pruning. -mod audit; -mod fabricate; +pub(crate) mod audit; +pub(crate) mod fabricate; mod ops; mod oracle; mod primitive; diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/fabricate.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/fabricate.rs index bfc118235a3..504bf71f8f7 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/fabricate.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/fabricate.rs @@ -172,6 +172,72 @@ pub(crate) fn fabricate_body(fab: &FabHeader) -> Arc { Arc::new(block) } +/// Like [`fabricate_body`], but with the coinbase rebuilt as a V4 transaction, +/// since the non-finalized state rejects pre-Sapling transaction versions +/// ("older transaction versions only exist in finalized blocks"). +pub(crate) fn fabricate_v4_body(fab: &FabHeader) -> Arc { + let template = fabricate_body(fab); + let mut block = Block::clone(&template); + + let old_tx = block.transactions.remove(0); + let (inputs, outputs, lock_time) = match &*old_tx { + zebra_chain::transaction::Transaction::V1 { + inputs, + outputs, + lock_time, + } => (inputs.clone(), outputs.clone(), *lock_time), + _ => panic!("template coinbase is V1"), + }; + block.transactions.insert( + 0, + Arc::new(zebra_chain::transaction::Transaction::V4 { + inputs, + outputs, + lock_time, + expiry_height: Height(0), + joinsplit_data: None, + sapling_shielded_data: None, + }), + ); + + Arc::new(block) +} + +/// A test network whose upgrades all activate far above the fabricated heights. +/// +/// Non-finalized commits then run in the pre-Heartwood regime: the history +/// tree stays empty, the header commitment field parses as +/// `PreSaplingReserved`, and no chain-history check runs — fabricated bodies +/// pass the non-finalized contextual validation. The header store side is +/// unaffected: its checkpoint handling is per-height hash equality, so the +/// unreachable high checkpoint (which satisfies the builder's +/// mandatory-checkpoint coverage) changes nothing at test heights. +pub(crate) fn pre_heartwood_test_network(genesis_hash: block::Hash) -> Network { + use zebra_chain::parameters::{testnet, Network::Mainnet}; + + testnet::Parameters::build() + .with_network_name("HeaderReorgNfTest") + .expect("test network name is valid") + .with_genesis_hash(genesis_hash) + .expect("test genesis hash is valid") + .with_target_difficulty_limit(Mainnet.target_difficulty_limit()) + .expect("mainnet difficulty limit is valid for test network") + .with_activation_heights(testnet::ConfiguredActivationHeights { + canopy: Some(200_000), + ..Default::default() + }) + .expect("test activation heights are valid") + .clear_funding_streams() + .with_checkpoints(testnet::ConfiguredCheckpoints::HeightsAndHashes(vec![ + (Height(0), genesis_hash), + // Covers the canopy mandatory checkpoint; never reached by tests. + (Height(199_999), block::Hash([0xCC; 32])), + ])) + .expect("high dummy checkpoint coverage is valid") + .to_network() + .expect("test network is valid") +} + /// Sums the work of a fabricated header run. pub(crate) fn total_work(headers: &[FabHeader]) -> PartialCumulativeWork { let mut total = PartialCumulativeWork::zero(); diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/startup_audit.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/startup_audit.rs index 1ece0ee84cf..5e760842332 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/startup_audit.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/startup_audit.rs @@ -18,11 +18,6 @@ //! - the `state.zakura.header_store.incoherent` metric is emitted exactly //! when a repair runs. -use std::{ - collections::HashMap, - sync::{Arc, Mutex}, -}; - use zebra_chain::block::Height; use super::super::super::startup_audit::ZakuraStoreViolation; @@ -31,6 +26,7 @@ use super::super::super::{ }; use super::super::common::{ commit_header_range, persistent_config, persistent_state, state_with_genesis_config, + CounterCapture, }; use super::{ audit::{audit_store, dump_store, StoreDump}, @@ -420,89 +416,6 @@ fn startup_heal_unblocks_linkage_verified_reads() { assert_clean(&state); } -/// A minimal local metrics recorder capturing counter increments by name. -#[derive(Clone, Default)] -struct CounterCapture(Arc>>); - -impl CounterCapture { - fn get(&self, name: &str) -> u64 { - self.0 - .lock() - .expect("counter capture lock is never poisoned") - .get(name) - .copied() - .unwrap_or(0) - } -} - -struct CaptureHandle { - name: String, - store: Arc>>, -} - -impl metrics::CounterFn for CaptureHandle { - fn increment(&self, value: u64) { - *self - .store - .lock() - .expect("counter capture lock is never poisoned") - .entry(self.name.clone()) - .or_insert(0) += value; - } - - fn absolute(&self, value: u64) { - self.store - .lock() - .expect("counter capture lock is never poisoned") - .insert(self.name.clone(), value); - } -} - -impl metrics::Recorder for CounterCapture { - fn describe_counter( - &self, - _: metrics::KeyName, - _: Option, - _: metrics::SharedString, - ) { - } - - fn describe_gauge( - &self, - _: metrics::KeyName, - _: Option, - _: metrics::SharedString, - ) { - } - - fn describe_histogram( - &self, - _: metrics::KeyName, - _: Option, - _: metrics::SharedString, - ) { - } - - fn register_counter(&self, key: &metrics::Key, _: &metrics::Metadata<'_>) -> metrics::Counter { - metrics::Counter::from_arc(Arc::new(CaptureHandle { - name: key.name().to_string(), - store: self.0.clone(), - })) - } - - fn register_gauge(&self, _: &metrics::Key, _: &metrics::Metadata<'_>) -> metrics::Gauge { - metrics::Gauge::noop() - } - - fn register_histogram( - &self, - _: &metrics::Key, - _: &metrics::Metadata<'_>, - ) -> metrics::Histogram { - metrics::Histogram::noop() - } -} - /// The `state.zakura.header_store.incoherent` metric fires exactly when a /// startup repair runs: once for the healing reopen, and not at all for the /// clean reopen after it. diff --git a/zebra-state/src/service/write.rs b/zebra-state/src/service/write.rs index 1ba12af6386..ff706ca69dd 100644 --- a/zebra-state/src/service/write.rs +++ b/zebra-state/src/service/write.rs @@ -39,6 +39,8 @@ use crate::service::{ non_finalized_state::Chain, }; +#[cfg(test)] +mod switch_orchestration_tests; mod vct_write; use vct_write::VctWriteManager; diff --git a/zebra-state/src/service/write/switch_orchestration_tests.rs b/zebra-state/src/service/write/switch_orchestration_tests.rs new file mode 100644 index 00000000000..1ed572eaec9 --- /dev/null +++ b/zebra-state/src/service/write/switch_orchestration_tests.rs @@ -0,0 +1,632 @@ +//! End-to-end tests for the Pillar-1a branch-switch orchestration: a +//! reorging `CommitHeaderRange` driven through the real worker function +//! ([`super::commit_header_range`]), against a real header store and +//! non-finalized state (INTEGRATION_TEST_PLAN.md T1). +//! +//! Fixture design (per the spike recorded in the plan): one fabricated +//! universe on [`pre_heartwood_test_network`], where non-finalized commits +//! run in the pre-Heartwood regime and accept fabricated V4-coinbase bodies. +//! Branches fork above a shared non-finalized prefix so rolled-back blocks +//! can recommit through the full validation path (their parent stays in the +//! non-finalized state). +//! +//! ```text +//! genesis ── trunk (1..=50) ── prefix (51..=53) ─┬─ A (54..=79, fast) +//! header rows: trunk + prefix + one branch ├─ B (54..=83, slow: lower work than A) +//! nf bodies: prefix + a few A blocks ├─ W (54..=83, fast: higher work than A) +//! └─ A_ext (A + 6 fast: higher work than W) +//! ``` + +use std::sync::Arc; + +use tokio::sync::{oneshot, watch}; +use zebra_chain::{ + block::{self, Block, Height}, + parameters::Network, +}; + +use crate::{ + arbitrary::Prepare, + error::CommitHeaderRangeError, + service::{ + finalized_state::{ + zebra_db::block::tests::{ + common::{mainnet_block, persistent_config, CounterCapture}, + header_store_coherence::{ + audit::{audit_store, dump_store}, + fabricate::{ + extend_context, fabricate_headers, fabricate_v4_body, + pre_heartwood_test_network, total_work, DifficultyContext, FabHeader, + Spacing, + }, + }, + }, + DiskWriteBatch, FinalizedState, WriteDisk, ZebraDb, + }, + non_finalized_state::NonFinalizedState, + ChainTipSender, + }, + Config, HeaderRangeCommitOutcome, +}; + +/// The first height where the branches conflict with each other. +const REORG_HEIGHT: Height = Height(54); + +/// The repair metric emitted by the store audits. +const INCOHERENT_METRIC: &str = "state.zakura.header_store.incoherent"; + +/// The counter emitted when the orchestration rolls back a stranded suffix. +const ROLLBACK_METRIC: &str = "sync.header.fork_recovery.body_suffix_invalidated"; + +/// The fixed branch universe every switch test runs over. +struct SwitchUniverse { + network: Network, + genesis: Arc, + /// Heights 1..=50, fast spacing. + trunk: Vec, + /// Heights 51..=53: the shared prefix, committed both as header rows and + /// as non-finalized bodies. + prefix: Vec, + /// 26 fast headers off the prefix tip (54..=79). + branch_a: Vec, + /// 30 slow headers off the prefix tip (54..=83) — longer than `A` but + /// lower total work. + branch_b: Vec, + /// 30 fast headers off the prefix tip (54..=83) — strictly more work + /// than `A` (same per-header thresholds, four more headers). + branch_w: Vec, + /// `A` plus a 6-header fast continuation (54..=85) — strictly more work + /// than `W`. + branch_a_ext: Vec, +} + +impl SwitchUniverse { + fn new() -> Self { + let genesis = mainnet_block(0); + let network = pre_heartwood_test_network(genesis.hash()); + let genesis_context: DifficultyContext = + vec![(genesis.header.difficulty_threshold, genesis.header.time)]; + + let trunk = fabricate_headers( + &network, + (Height(0), genesis.hash()), + genesis_context.clone(), + &[Spacing::Fast; 50], + 0x10, + ); + let trunk_tip = trunk.last().expect("trunk is non-empty"); + let trunk_context = extend_context(genesis_context, &trunk); + + let prefix = fabricate_headers( + &network, + (trunk_tip.height, trunk_tip.hash), + trunk_context.clone(), + &[Spacing::Fast; 3], + 0x20, + ); + let prefix_tip = prefix.last().expect("prefix is non-empty"); + let fork_parent = (prefix_tip.height, prefix_tip.hash); + let fork_context = extend_context(trunk_context, &prefix); + + let branch_a = fabricate_headers( + &network, + fork_parent, + fork_context.clone(), + &[Spacing::Fast; 26], + 0x40, + ); + let branch_b = fabricate_headers( + &network, + fork_parent, + fork_context.clone(), + &[Spacing::Slow; 30], + 0x80, + ); + let branch_w = fabricate_headers( + &network, + fork_parent, + fork_context.clone(), + &[Spacing::Fast; 30], + 0xC0, + ); + + let a_tip = branch_a.last().expect("branch A is non-empty"); + let a_context = extend_context(fork_context, &branch_a); + let mut branch_a_ext = branch_a.clone(); + branch_a_ext.extend(fabricate_headers( + &network, + (a_tip.height, a_tip.hash), + a_context, + &[Spacing::Fast; 6], + 0x60, + )); + + let universe = SwitchUniverse { + network, + genesis, + trunk, + prefix, + branch_a, + branch_b, + branch_w, + branch_a_ext, + }; + + // The work orderings every test relies on. + assert!( + total_work(&universe.branch_a) > total_work(&universe.branch_b), + "branch A must out-work the longer slow branch B" + ); + assert!( + total_work(&universe.branch_w) > total_work(&universe.branch_a), + "branch W must out-work branch A" + ); + assert!( + total_work(&universe.branch_a_ext) > total_work(&universe.branch_w), + "extended branch A must out-work branch W" + ); + + universe + } + + fn prefix_tip(&self) -> &FabHeader { + self.prefix.last().expect("prefix is non-empty") + } +} + +/// The channel plumbing around the states, mirroring the write worker's. +struct Plumbing { + chain_tip_sender: ChainTipSender, + _latest_chain_tip: crate::LatestChainTip, + _chain_tip_change: crate::ChainTipChange, + nf_sender: watch::Sender, + nf_receiver: watch::Receiver, +} + +impl Plumbing { + fn new(network: &Network) -> Self { + let (chain_tip_sender, latest_chain_tip, chain_tip_change) = + ChainTipSender::new(None, network); + let (nf_sender, nf_receiver) = watch::channel(NonFinalizedState::new(network)); + Plumbing { + chain_tip_sender, + _latest_chain_tip: latest_chain_tip, + _chain_tip_change: chain_tip_change, + nf_sender, + nf_receiver, + } + } +} + +/// Commits `headers` as header rows through the production range writer, +/// panicking on rejection (fixture setup only). +fn commit_rows(db: &ZebraDb, anchor: block::Hash, headers: &[FabHeader]) { + let mut batch = DiskWriteBatch::new(); + let arcs: Vec<_> = headers.iter().map(|fab| fab.header.clone()).collect(); + let body_sizes: Vec<_> = headers.iter().map(|fab| fab.body_size).collect(); + batch + .prepare_header_range_batch(db, anchor, &arcs, &body_sizes) + .expect("fixture header range is valid"); + db.write_batch(batch).expect("fixture header range writes"); +} + +/// Builds the standard fixture: genesis finalized with treestates; header +/// rows for trunk + prefix + branch A; non-finalized bodies for the prefix +/// (when `include_prefix_bodies`) plus the first `a_bodies` branch-A blocks. +fn fixture( + universe: &SwitchUniverse, + config: Config, + include_prefix_bodies: bool, + a_bodies: usize, +) -> (FinalizedState, NonFinalizedState, Plumbing) { + let mut finalized_state = FinalizedState::new( + &config, + &universe.network, + #[cfg(feature = "elasticsearch")] + false, + ); + finalized_state + .commit_finalized_direct(universe.genesis.clone().into(), None, None, "switch tests") + .expect("genesis commits with treestates"); + + commit_rows( + &finalized_state.db, + universe.genesis.hash(), + &universe.trunk, + ); + let trunk_tip = universe.trunk.last().expect("trunk is non-empty"); + commit_rows(&finalized_state.db, trunk_tip.hash, &universe.prefix); + commit_rows( + &finalized_state.db, + universe.prefix_tip().hash, + &universe.branch_a, + ); + + let mut non_finalized_state = NonFinalizedState::new(&universe.network); + let mut bodies: Vec<&FabHeader> = Vec::new(); + if include_prefix_bodies { + bodies.extend(&universe.prefix); + } + bodies.extend(&universe.branch_a[..a_bodies]); + for (index, fab) in bodies.iter().enumerate() { + let prepared = fabricate_v4_body(fab).prepare(); + if index == 0 { + non_finalized_state + .commit_new_chain(prepared, &finalized_state) + .expect("fixture chain root commits"); + } else { + non_finalized_state + .commit_block(prepared, &finalized_state) + .expect("fixture chain block commits"); + } + } + + let plumbing = Plumbing::new(&universe.network); + (finalized_state, non_finalized_state, plumbing) +} + +/// Drives a header range through the real switch orchestration and returns +/// its response. +#[allow(clippy::unwrap_in_result)] +fn deliver( + finalized_state: &FinalizedState, + non_finalized_state: &mut NonFinalizedState, + plumbing: &mut Plumbing, + anchor: block::Hash, + headers: &[FabHeader], +) -> Result { + let (rsp_tx, mut rsp_rx) = oneshot::channel(); + super::commit_header_range( + finalized_state, + non_finalized_state, + &mut plumbing.chain_tip_sender, + &plumbing.nf_sender, + None, + anchor, + headers.iter().map(|fab| fab.header.clone()).collect(), + headers.iter().map(|fab| fab.body_size).collect(), + headers.iter().map(|fab| fab.roots.clone()).collect(), + rsp_tx, + ); + rsp_rx + .try_recv() + .expect("the orchestration always sends a response") +} + +/// Asserts the stored header chain above the prefix is exactly `branch`. +fn assert_store_holds_branch(db: &ZebraDb, universe: &SwitchUniverse, branch: &[FabHeader]) { + for fab in universe.trunk.iter().chain(&universe.prefix).chain(branch) { + assert_eq!( + db.zakura_header_hash(fab.height), + Some(fab.hash), + "stored hash row at {:?} matches the expected branch", + fab.height, + ); + } + let branch_tip = branch.last().expect("branches are non-empty"); + let above = branch_tip + .height + .next() + .expect("test heights stay in range"); + assert_eq!( + db.zakura_header_hash(above), + None, + "no stray row above the branch tip", + ); + assert_eq!( + db.best_header_tip(), + Some((branch_tip.height, branch_tip.hash)), + "the header tip is the branch tip", + ); +} + +/// T1.1: a conflicting higher-work range delivered through the worker +/// switches the header store, rolls the stranded body suffix back to the +/// fork, publishes the truncated chain, and reports the reorg in the +/// response — with the post-reorg audit running clean. +#[test] +fn reorging_commit_switches_store_and_rolls_back_stranded_bodies() { + let _init_guard = zebra_test::init(); + let universe = SwitchUniverse::new(); + let (finalized_state, mut non_finalized_state, mut plumbing) = + fixture(&universe, Config::ephemeral(), true, 3); + + let capture = CounterCapture::default(); + let outcome = metrics::with_local_recorder(&capture, || { + deliver( + &finalized_state, + &mut non_finalized_state, + &mut plumbing, + universe.prefix_tip().hash, + &universe.branch_w, + ) + }) + .expect("the higher-work conflicting range commits"); + + // The response reports the switch. + let w_tip = universe.branch_w.last().expect("branch W is non-empty"); + assert_eq!(outcome.tip_hash, w_tip.hash); + assert_eq!(outcome.reorged_at, Some(REORG_HEIGHT)); + assert_eq!(outcome.reorged_to_hash, Some(universe.branch_w[0].hash)); + + // The header store switched to W and stayed coherent. + assert_store_holds_branch(&finalized_state.db, &universe, &universe.branch_w); + assert!(audit_store(&finalized_state.db).is_empty()); + + // The stranded body suffix was rolled back to the fork, keeping the + // shared prefix, and the truncated chain was published. + let best_chain = non_finalized_state + .best_chain() + .expect("the shared prefix survives"); + assert!(best_chain.contains_block_hash(universe.prefix_tip().hash)); + assert!(!best_chain.contains_block_hash(universe.branch_a[0].hash)); + assert!(plumbing + .nf_receiver + .borrow() + .best_chain() + .is_some_and(|chain| !chain.contains_block_hash(universe.branch_a[0].hash))); + + // Exactly one rollback ran, and the post-reorg audit found nothing. + assert_eq!(capture.get(ROLLBACK_METRIC), 1); + assert_eq!(capture.get(INCOHERENT_METRIC), 0); +} + +/// T1.1 (repair variant): the post-reorg audit hook runs on the reorging +/// commit and repairs a hand-planted stray row, emitting the repair metric. +#[test] +fn reorging_commit_audit_hook_repairs_planted_stray_row() { + let _init_guard = zebra_test::init(); + let universe = SwitchUniverse::new(); + let (finalized_state, mut non_finalized_state, mut plumbing) = + fixture(&universe, Config::ephemeral(), true, 3); + + // An orphan reverse-index row pointing *below* the fork: invisible to + // the delivery path (which never scans the reverse index) and outside + // the suffix the reorg batch replaces, so only the audit hook can + // repair it. (A stray above the new tip would be swept by the + // suffix-replacement primitive itself before the audit looks.) + let height_by_hash = finalized_state + .db + .db() + .cf_handle("zakura_header_height_by_hash") + .expect("column family exists"); + let mut batch = DiskWriteBatch::new(); + batch.zs_insert(&height_by_hash, block::Hash([0xEE; 32]), Height(20)); + finalized_state + .db + .db() + .write(batch) + .expect("stray row writes"); + assert!( + !audit_store(&finalized_state.db).is_empty(), + "the planted row is a store violation" + ); + + let capture = CounterCapture::default(); + metrics::with_local_recorder(&capture, || { + deliver( + &finalized_state, + &mut non_finalized_state, + &mut plumbing, + universe.prefix_tip().hash, + &universe.branch_w, + ) + }) + .expect("the higher-work conflicting range commits"); + + assert_eq!( + capture.get(INCOHERENT_METRIC), + 1, + "the post-reorg audit repaired the stray row" + ); + assert!(audit_store(&finalized_state.db).is_empty()); + assert_store_holds_branch(&finalized_state.db, &universe, &universe.branch_w); +} + +/// T1.2: a crash between the body rollback and the header rewrite leaves +/// the crash-table intermediate — header store unchanged and coherent, nf +/// truncated — the startup audit passes it without repair, and re-running +/// the same commit converges to the switched state (recovery is the +/// ordinary flow). +#[test] +fn kill_between_rollback_and_header_rewrite_recovers_through_ordinary_flow() { + let _init_guard = zebra_test::init(); + let universe = SwitchUniverse::new(); + let tempdir = tempfile::tempdir().expect("test tempdir is available"); + let config = persistent_config(tempdir.path()); + let (finalized_state, mut non_finalized_state, mut plumbing) = + fixture(&universe, config.clone(), true, 3); + + let store_before = dump_store(&finalized_state.db); + + // Step 1 (evaluate): prepare the batch, deciding the fork point. + let arcs: Vec<_> = universe + .branch_w + .iter() + .map(|fab| fab.header.clone()) + .collect(); + let body_sizes: Vec<_> = universe.branch_w.iter().map(|fab| fab.body_size).collect(); + let roots: Vec<_> = universe + .branch_w + .iter() + .map(|fab| fab.roots.clone()) + .collect(); + let mut batch = DiskWriteBatch::new(); + let outcome = batch + .prepare_header_range_batch_with_roots( + &finalized_state.db, + universe.prefix_tip().hash, + &arcs, + &body_sizes, + &roots, + ) + .expect("the range evaluates to a reorging batch"); + assert_eq!(outcome.reorged_at, Some(REORG_HEIGHT)); + + // Step 2 (body rollback), then crash before step 3: the prepared batch + // is dropped unwritten. + super::invalidate_stranded_body_suffix( + &mut non_finalized_state, + &mut plumbing.chain_tip_sender, + &plumbing.nf_sender, + None, + REORG_HEIGHT, + universe.branch_w[0].hash, + ); + drop(batch); + + // The crash intermediate: the header store is unchanged (still A) and + // coherent; only the nf suffix is gone. + assert_eq!(dump_store(&finalized_state.db), store_before); + assert!(audit_store(&finalized_state.db).is_empty()); + assert_store_holds_branch(&finalized_state.db, &universe, &universe.branch_a); + let best_chain = non_finalized_state + .best_chain() + .expect("the shared prefix survives"); + assert!(!best_chain.contains_block_hash(universe.branch_a[0].hash)); + + // Restart: the startup audit passes the intermediate without repair. + drop(finalized_state); + let capture = CounterCapture::default(); + let finalized_state = metrics::with_local_recorder(&capture, || { + FinalizedState::new( + &config, + &universe.network, + #[cfg(feature = "elasticsearch")] + false, + ) + }); + assert_eq!( + capture.get(INCOHERENT_METRIC), + 0, + "the crash intermediate is coherent: no startup repair" + ); + assert_eq!(dump_store(&finalized_state.db), store_before); + + // Recovery is the ordinary flow: the same commit re-runs end-to-end and + // converges to the switched state. + let outcome = deliver( + &finalized_state, + &mut non_finalized_state, + &mut plumbing, + universe.prefix_tip().hash, + &universe.branch_w, + ) + .expect("the re-run commits"); + assert_eq!(outcome.reorged_at, Some(REORG_HEIGHT)); + assert_store_holds_branch(&finalized_state.db, &universe, &universe.branch_w); + assert!(audit_store(&finalized_state.db).is_empty()); +} + +/// T1.3: a reorg at the non-finalized root empties the chain set without +/// panicking, publishes the emptied state, and still switches the store. +#[test] +fn whole_suffix_switch_empties_non_finalized_state_without_panicking() { + let _init_guard = zebra_test::init(); + let universe = SwitchUniverse::new(); + // No prefix bodies: the nf chain roots at the reorged height itself. + let (finalized_state, mut non_finalized_state, mut plumbing) = + fixture(&universe, Config::ephemeral(), false, 3); + + let outcome = deliver( + &finalized_state, + &mut non_finalized_state, + &mut plumbing, + universe.prefix_tip().hash, + &universe.branch_w, + ) + .expect("the higher-work conflicting range commits"); + + assert_eq!(outcome.reorged_at, Some(REORG_HEIGHT)); + assert!( + non_finalized_state.best_chain().is_none(), + "rolling back the nf root empties the chain set" + ); + assert!( + plumbing.nf_receiver.borrow().best_chain().is_none(), + "the emptied state was published" + ); + assert_store_holds_branch(&finalized_state.db, &universe, &universe.branch_w); + assert!(audit_store(&finalized_state.db).is_empty()); +} + +/// T1.4: a lower-work conflict is rejected with no side effects; the +/// higher-work extension switches; the original branch switches back when +/// it out-works the winner; and its rolled-back bodies recommit through the +/// full validation path — rollback is not a ban, end to end. +#[test] +fn losing_then_winning_branch_switches_back_and_recommits() { + let _init_guard = zebra_test::init(); + let universe = SwitchUniverse::new(); + let (finalized_state, mut non_finalized_state, mut plumbing) = + fixture(&universe, Config::ephemeral(), true, 2); + + let store_before = dump_store(&finalized_state.db); + let nf_before = non_finalized_state.clone(); + + // A lower-work conflicting range is rejected with no side effects. + let error = deliver( + &finalized_state, + &mut non_finalized_state, + &mut plumbing, + universe.prefix_tip().hash, + &universe.branch_b, + ) + .expect_err("the lower-work range is rejected"); + assert!( + matches!(error, CommitHeaderRangeError::LowerWorkConflict { .. }), + "unexpected rejection: {error:?}" + ); + assert_eq!( + dump_store(&finalized_state.db), + store_before, + "a rejected candidate leaves the store untouched" + ); + assert!( + non_finalized_state.eq_internal_state(&nf_before), + "a rejected candidate leaves the nf state untouched" + ); + + // The higher-work branch W wins: bodies roll back to the prefix. + let outcome = deliver( + &finalized_state, + &mut non_finalized_state, + &mut plumbing, + universe.prefix_tip().hash, + &universe.branch_w, + ) + .expect("the higher-work range commits"); + assert_eq!(outcome.reorged_at, Some(REORG_HEIGHT)); + assert_store_holds_branch(&finalized_state.db, &universe, &universe.branch_w); + + // Branch A, extended past W's work, switches back. + let outcome = deliver( + &finalized_state, + &mut non_finalized_state, + &mut plumbing, + universe.prefix_tip().hash, + &universe.branch_a_ext, + ) + .expect("the extended original branch switches back"); + assert_eq!(outcome.reorged_at, Some(REORG_HEIGHT)); + assert_eq!(outcome.reorged_to_hash, Some(universe.branch_a[0].hash)); + assert_store_holds_branch(&finalized_state.db, &universe, &universe.branch_a_ext); + assert!(audit_store(&finalized_state.db).is_empty()); + + // The rolled-back A bodies recommit through the full validation path + // (contextual checks included): rollback is not a ban. + for fab in &universe.branch_a[..2] { + let prepared = fabricate_v4_body(fab).prepare(); + super::validate_and_commit_non_finalized( + &finalized_state.db, + &mut non_finalized_state, + prepared, + ) + .expect("a rolled-back block recommits when its branch wins again"); + } + let best_chain = non_finalized_state + .best_chain() + .expect("chain is non-empty"); + assert!(best_chain.contains_block_hash(universe.branch_a[0].hash)); + assert!(best_chain.contains_block_hash(universe.branch_a[1].hash)); +} From 65d0f412370bfe482d73ecc03e550944ce3ac760 Mon Sep 17 00:00:00 2001 From: roman Date: Mon, 6 Jul 2026 22:16:46 -0600 Subject: [PATCH 27/27] test(zakura): cover reorg scenarios at the reactor and driver layers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the T3 suite from INTEGRATION_TEST_PLAN.md (Phase-5 Layer 2), following the walk-back reactor test patterns: - T3.1 lower-work conflict at the peer layer: a served range rejected as a lower-work fork reports the non-scoring Local kind — the peer is never scored or disconnected, keeps receiving requests, and a later successful commit completes the switch. Plus the zebrad classification test pinning LowerWorkConflict → Local. - T3.2 the #496 flow at the driver: a CommittedHeaderRange outcome with reorged_at set must not drive any follow-up state request (the state's switch orchestration already rolled the stranded suffix back) — the mock state panics on anything but the commit itself, and the frontier still advances through the ordinary success path. - T3.3 store-incoherence walk-back to convergence: ContextMismatch failures from independent peers on every commit start a walk-back with no one scored, the frontier re-anchors below the verified tip from the store's reanchor target, and the re-delivered range re-commits through the fork point. (StoreIncoherent → ContextMismatch classification and the store-side repair are covered by the existing zebrad and startup-audit tests.) --- zebra-network/src/zakura/header_sync/tests.rs | 356 ++++++++++++++++++ zebrad/src/commands/start.rs | 139 +++++++ 2 files changed, 495 insertions(+) diff --git a/zebra-network/src/zakura/header_sync/tests.rs b/zebra-network/src/zakura/header_sync/tests.rs index 6c41e4f6669..a2e3eae6b11 100644 --- a/zebra-network/src/zakura/header_sync/tests.rs +++ b/zebra-network/src/zakura/header_sync/tests.rs @@ -5320,3 +5320,359 @@ async fn misbehavior_is_recorded_without_disconnecting_the_peer() { "misbehavior is record-only: an InvalidStatus peer must NOT be disconnected", ); } + +/// Scenario 1a at the peer layer (INTEGRATION_TEST_PLAN.md T3.1): a peer +/// whose served range is rejected as a lower-work conflicting fork gets a +/// non-scoring `Local` commit failure (the driver maps +/// `CommitHeaderRangeError::LowerWorkConflict` to that kind). The peer must +/// never be scored or disconnected and must keep receiving requests; a later +/// higher-work commit (decided in the state) completes the switch. +#[tokio::test(flavor = "current_thread")] +async fn lower_work_conflict_keeps_peer_serving_until_switch_completes() { + let header1 = mainnet_header(&BLOCK_MAINNET_1_BYTES); + let header2 = mainnet_header(&BLOCK_MAINNET_2_BYTES); + let header3 = mainnet_header(&BLOCK_MAINNET_3_BYTES); + let header4 = mainnet_header(&BLOCK_MAINNET_4_BYTES); + // Anchors above genesis must be checkpoints; pin block 3 as one on a + // network whose genesis is the mainnet genesis, so the mainnet test + // headers link above the anchor. + let (network, anchor_hash) = + checkpoint_testnet_with_hash(block::Height(3), block::Hash::from(header3.as_ref())); + let anchor = (block::Height(3), anchor_hash); + let mut fixture = spawn_test_reactor(startup_for(network, anchor, Some(anchor))); + let mut tip = fixture.handle.subscribe_tip(); + let peer_a = peer(83); + let peer_b = peer(84); + + // Only peer A advertises: every request must go to it. Peer B stays + // connected but silent. + connect_peer(&fixture, peer_a.clone()).await; + connect_peer(&fixture, peer_b.clone()).await; + advertise_tip( + &fixture, + peer_a.clone(), + anchor.0, + block::Height(4), + DEFAULT_HS_RANGE, + 1, + ) + .await; + + // Serves peer A's requests — the checkpoint backfill below the anchor + // (which must be answered, or it pins the peer's single effective + // request slot) and the forward range at height 4 — until the forward + // range reaches the committer. Panics on any misbehavior report. + async fn serve_until_forward_commit( + fixture: &mut ReactorFixture, + expected_peer: &ZakuraPeerId, + backfill: &[Arc], + forward: Arc, + checkpoint_hash: block::Hash, + ) { + loop { + match next_non_query_action(&mut fixture.actions).await { + HeaderSyncAction::SendMessage { + peer, + msg: HeaderSyncMessage::GetHeaders { start_height, .. }, + } => { + assert_eq!( + &peer, expected_peer, + "only the advertising peer receives requests" + ); + let msg = if start_height == block::Height(4) { + headers_message_from(start_height, vec![forward.clone()]) + } else { + assert_eq!(start_height, block::Height(1), "unexpected request"); + headers_message_from(start_height, backfill.to_vec()) + }; + fixture + .handle + .send(HeaderSyncEvent::WireMessage { peer, msg }) + .await + .unwrap(); + } + HeaderSyncAction::CommitHeaderRange { + peer, start_height, .. + } if start_height == block::Height(1) => { + // The finalized checkpoint backfill commits cleanly. + assert_eq!(&peer, expected_peer); + fixture + .handle + .send(HeaderSyncEvent::HeaderRangeCommitted { + start_height, + tip_height: block::Height(3), + tip_hash: checkpoint_hash, + }) + .await + .unwrap(); + } + HeaderSyncAction::CommitHeaderRange { + peer, + anchor: commit_anchor, + start_height, + .. + } => { + assert_eq!(&peer, expected_peer); + assert_eq!(commit_anchor, checkpoint_hash); + assert_eq!(start_height, block::Height(4)); + return; + } + HeaderSyncAction::Misbehavior { peer, reason } => { + panic!("unexpected misbehavior from {peer:?}: {reason:?}"); + } + _ => {} + } + } + } + + // Peer A serves its (lower-work) branch; the reactor hands it to the + // committer. + let backfill = [header1.clone(), header2.clone(), header3.clone()]; + serve_until_forward_commit( + &mut fixture, + &peer_a, + &backfill, + header4.clone(), + anchor_hash, + ) + .await; + + // The state rejects the range as a lower-work conflicting fork; the + // driver reports the non-scoring `Local` kind. + fixture + .handle + .send(HeaderSyncEvent::HeaderRangeCommitFailed { + peer: peer_a.clone(), + start_height: block::Height(4), + count: 1, + kind: HeaderSyncCommitFailureKind::Local, + }) + .await + .unwrap(); + + // The scheduler hands out ranges on event ticks; a fresh peer connection + // provides one (in production events flow continuously). + connect_peer(&fixture, peer(85)).await; + + // Peer A was not scored and stays in rotation: the retried range goes + // back to it and reaches the committer again. This time the state (which + // owns the work comparison) accepts — the switch completes. + serve_until_forward_commit( + &mut fixture, + &peer_a, + &backfill, + header4.clone(), + anchor_hash, + ) + .await; + let tip_hash = block::Hash::from(header4.as_ref()); + fixture + .handle + .send(HeaderSyncEvent::HeaderRangeCommitted { + start_height: block::Height(4), + tip_height: block::Height(4), + tip_hash, + }) + .await + .unwrap(); + tip.changed().await.unwrap(); + assert_eq!(*tip.borrow(), (block::Height(4), tip_hash)); +} + +/// `StoreIncoherent`-shaped commit failures walking back and converging +/// (INTEGRATION_TEST_PLAN.md T3.3): the driver classifies store incoherence +/// as `ContextMismatch` (covered in zebrad's classification tests); here the +/// reactor receives that kind from independent peers on every commit, starts +/// a walk-back without scoring anyone, re-anchors from the store's reanchor +/// target, and converges by re-committing through the fork. (The store-side +/// repair the walk-back races against is covered by the startup-audit suite.) +#[tokio::test(flavor = "current_thread")] +async fn context_mismatch_walk_back_reanchors_and_converges_without_scoring() { + let header1 = mainnet_header(&BLOCK_MAINNET_1_BYTES); + let header2 = mainnet_header(&BLOCK_MAINNET_2_BYTES); + let header3 = mainnet_header(&BLOCK_MAINNET_3_BYTES); + let header4 = mainnet_header(&BLOCK_MAINNET_4_BYTES); + let by_height = [ + header1.clone(), + header2.clone(), + header3.clone(), + header4.clone(), + ]; + let serve = |start_height: block::Height, count: u32| { + let start = start_height.0 as usize; + let served: Vec<_> = by_height[start - 1..(start - 1 + count as usize).min(4)].to_vec(); + headers_message_from(start_height, served) + }; + // The network's genesis is the mainnet genesis (so the mainnet test + // headers link), with the mandatory checkpoint pinned to block 3. + let (network, checkpoint_hash) = + checkpoint_testnet_with_hash(block::Height(3), block::Hash::from(header3.as_ref())); + // Anchor at genesis (the walk-back floor is max(finalized, anchor), and + // this test needs to observe a re-anchor below the verified tip); the + // durable store already holds headers up to the checkpoint. + let anchor = (block::Height(0), network.genesis_hash()); + let stored_tip = (block::Height(3), checkpoint_hash); + let mut startup = HeaderSyncStartup::new( + network, + anchor, + HeaderSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: stored_tip.0, + verified_block_hash: stored_tip.1, + }, + Some(stored_tip), + ZakuraHeaderSyncConfig::default(), + LOCAL_MAX_MESSAGE_BYTES, + ); + startup.range_state_actions_enabled = true; + let mut fixture = spawn_test_reactor(startup); + let mut tip = fixture.handle.subscribe_tip(); + // Three peers: a failed range stays assigned to the peers that already + // served it (ContextMismatch keeps hedging distinctness), so reaching + // the stale-anchor quorum needs a third independent server. + let peers = [peer(86), peer(87), peer(88)]; + + for peer_id in peers.iter().cloned() { + connect_peer(&fixture, peer_id.clone()).await; + advertise_tip( + &fixture, + peer_id, + block::Height(0), + block::Height(4), + DEFAULT_HS_RANGE, + 1, + ) + .await; + } + + // Every served range reaches the committer and fails ContextMismatch + // (the live shape of a corrupted store: the linkage-verified reads + // reject every commit attempt, whoever serves it). The reactor must not + // score anyone and must eventually query a walk-back reanchor target. + let target = loop { + match next_action(&mut fixture.actions).await { + HeaderSyncAction::QueryReanchorTarget { height } => break height, + HeaderSyncAction::SendMessage { + peer, + msg: + HeaderSyncMessage::GetHeaders { + start_height, + count, + .. + }, + } => { + fixture + .handle + .send(HeaderSyncEvent::WireMessage { + peer, + msg: serve(start_height, count), + }) + .await + .unwrap(); + } + HeaderSyncAction::CommitHeaderRange { + peer, + start_height, + headers, + .. + } => { + fixture + .handle + .send(HeaderSyncEvent::HeaderRangeCommitFailed { + peer, + start_height, + count: headers.len() as u32, + kind: HeaderSyncCommitFailureKind::ContextMismatch, + }) + .await + .unwrap(); + } + HeaderSyncAction::Misbehavior { peer, reason } => { + panic!("honest peer {peer:?} scored for a local context mismatch: {reason:?}") + } + _ => {} + } + }; + assert_eq!( + target, + block::Height(2), + "the walk-back starts one below the verified tip" + ); + + // The store (repaired by the audits) answers the reanchor query. + fixture + .handle + .send(HeaderSyncEvent::ReanchorTargetLoaded { + height: target, + hash: Some(block::Hash::from(header2.as_ref())), + }) + .await + .unwrap(); + tip.changed().await.unwrap(); + assert_eq!( + *tip.borrow_and_update(), + (block::Height(2), block::Hash::from(header2.as_ref())) + ); + + // The re-anchored frontier re-requests through the fork; the store is + // healthy again, so commits succeed and sync converges — still with no + // one scored. The recommit through the fork must anchor at the reanchor + // hash. + let final_tip = (block::Height(4), block::Hash::from(header4.as_ref())); + let mut saw_fork_recommit = false; + while *tip.borrow_and_update() != final_tip { + match next_action(&mut fixture.actions).await { + HeaderSyncAction::SendMessage { + peer, + msg: + HeaderSyncMessage::GetHeaders { + start_height, + count, + .. + }, + } => { + fixture + .handle + .send(HeaderSyncEvent::WireMessage { + peer, + msg: serve(start_height, count), + }) + .await + .unwrap(); + } + HeaderSyncAction::CommitHeaderRange { + anchor: commit_anchor, + start_height, + headers, + .. + } => { + if start_height == block::Height(3) { + assert_eq!( + commit_anchor, + block::Hash::from(header2.as_ref()), + "the recommit through the fork anchors at the reanchor target" + ); + saw_fork_recommit = true; + } + let tip_index = start_height.0 as usize - 1 + headers.len() - 1; + fixture + .handle + .send(HeaderSyncEvent::HeaderRangeCommitted { + start_height, + tip_height: block::Height(start_height.0 + headers.len() as u32 - 1), + tip_hash: block::Hash::from(by_height[tip_index].as_ref()), + }) + .await + .unwrap(); + } + HeaderSyncAction::Misbehavior { peer, reason } => { + panic!("honest peer {peer:?} scored during walk-back recovery: {reason:?}") + } + _ => {} + } + } + assert!( + saw_fork_recommit, + "convergence must re-commit the range at the fork point" + ); +} diff --git a/zebrad/src/commands/start.rs b/zebrad/src/commands/start.rs index 7fd96e512d8..3e36112d593 100644 --- a/zebrad/src/commands/start.rs +++ b/zebrad/src/commands/start.rs @@ -2255,6 +2255,145 @@ mod zakura_header_sync_driver_tests { ); } + #[test] + fn lower_work_conflict_is_local_header_sync_commit_failure() { + // A lower-work conflicting range is individually valid — the peer + // offered a worse fork, which is not misbehavior. Scoring it would + // disconnect honest peers serving a losing branch (scenario 1a), so + // the rejection must stay a non-scoring local failure. + let error = zebra_state::CommitHeaderRangeError::LowerWorkConflict { + height: block::Height(54), + existing_work: 100, + new_work: 50, + }; + + assert_eq!( + header_range_commit_failure_kind(&error), + HeaderSyncCommitFailureKind::Local + ); + } + + /// A reorging `CommitHeaderRange` outcome must not make the driver send + /// any follow-up state request: the state's switch orchestration already + /// rolled the stranded body suffix back before the header rewrite + /// (REORG_PLAN Pillar 1a), so a separate driver-side `InvalidateBlock` + /// would be a double invalidation (INTEGRATION_TEST_PLAN.md T3.2). + #[tokio::test] + async fn reorging_header_commit_sends_no_driver_side_invalidation() { + let network = zebra_chain::parameters::Network::Mainnet; + let genesis_hash = network.genesis_hash(); + let mut config = zebra_network::Config { + network: network.clone(), + ..zebra_network::Config::default() + }; + config.zakura.listen_addr = None; + let endpoint = zebra_network::zakura::spawn_zakura_endpoint_with_header_sync_driver( + &config, + |_supervisor, _trace| Arc::new(NoopZakuraService) as Arc, + Some(ZakuraHeaderSyncDriverStartup { + frontiers: HeaderSyncFrontiers { + finalized_height: block::Height(0), + verified_block_tip: block::Height(0), + verified_block_hash: genesis_hash, + }, + best_header_tip: Some((block::Height(0), genesis_hash)), + verified_block_tip_hash: genesis_hash, + }), + ) + .await + .expect("Zakura endpoint starts") + .expect("v2_p2p starts an endpoint"); + + let committed_header = mainnet_block(&BLOCK_MAINNET_1_BYTES).header.clone(); + let tip_hash = block::Hash::from(committed_header.as_ref()); + + let (action_tx, action_rx) = mpsc::channel(4); + let (shutdown_tx, shutdown_rx) = oneshot::channel(); + let handles = ZakuraHeaderSyncDriverHandles { + endpoint: endpoint.clone(), + header_sync: endpoint + .header_sync() + .expect("driver startup starts header sync"), + }; + // The only state request the driver may make is the commit itself; a + // reorging outcome must not trigger InvalidateBlock (or anything + // else) — the mock panics the driver task on any other request. + let state = service_fn(move |request: zebra_state::Request| async move { + match request { + zebra_state::Request::CommitHeaderRange { .. } => { + Ok::<_, zebra_state::BoxError>(zebra_state::Response::CommittedHeaderRange( + zebra_state::HeaderRangeCommitOutcome { + tip_hash, + reorged_at: Some(block::Height(1)), + reorged_to_hash: Some(tip_hash), + }, + )) + } + request => panic!( + "a reorging header commit must not drive further state writes: {request:?}" + ), + } + }); + let read_state = service_fn(|request: zebra_state::ReadRequest| async move { + panic!("unexpected read request during a reorging commit: {request:?}"); + #[allow(unreachable_code)] + Ok::<_, zebra_state::BoxError>(zebra_state::ReadResponse::Tip(None)) + }); + let verifier = service_fn(|request: zebra_consensus::Request| async move { + panic!("unexpected verifier request during a reorging commit: {request:?}"); + #[allow(unreachable_code)] + Ok::<_, zebra_consensus::BoxError>(block::Hash([0; 32])) + }); + let driver = tokio::spawn(drive_zakura_header_sync_actions( + action_rx, + handles, + state, + read_state, + verifier, + zebra_network::zakura::ZakuraTrace::noop(), + async move { + let _ = shutdown_rx.await; + }, + )); + + let peer = + zebra_network::zakura::ZakuraPeerId::new(vec![6; 32]).expect("test peer id is valid"); + action_tx + .send(zebra_network::zakura::HeaderSyncAction::CommitHeaderRange { + peer, + anchor: genesis_hash, + start_height: block::Height(1), + headers: vec![committed_header], + body_sizes: vec![0], + tree_aux_roots: Vec::new(), + finalized: false, + }) + .await + .expect("driver action channel stays open"); + + // The reorging commit completes through the ordinary success path: + // the frontier advances to the committed tip (published after the + // state call, so any follow-up write would already have panicked). + tokio::time::timeout(Duration::from_secs(5), async { + loop { + let update = endpoint + .current_sync_frontier() + .expect("exchange remains available"); + if update.frontier.best_header.height == block::Height(1) { + assert_eq!(update.frontier.best_header.hash, tip_hash); + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("a reorging commit still advances the header frontier"); + + let _ = shutdown_tx.send(()); + driver.await.expect("driver task exits cleanly"); + endpoint.shutdown().await; + } + #[test] fn served_header_body_size_hints_align_with_served_heights() { let start = block::Height(10);