diff --git a/CHANGELOG.md b/CHANGELOG.md index 679fb4e12cc..cef6035086b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,15 @@ and this project adheres to [Semantic Versioning](https://semver.org). - Verified-commitment-trees fast sync is now enabled by default when checkpoint sync is enabled. Operators can keep checkpoint sync but opt out of the new path by setting `consensus.vct_fast_sync = false`. +- All Zakura header-store writers (header-range commits, best-chain seeds, and + body-commit releases) now change chain membership through a single + suffix-replacement primitive that deletes everything above the fork point by + scanning the actual on-disk rows of all five column families — so rows can no + longer be stranded by bounded delete loops — and verifies linkage as a hard + precondition. Any batch that replaces a header suffix (a header reorg) is + followed by the store coherence audit, repairing and reporting + (`state.zakura.header_store.incoherent`) any residual violation at the moment + it happens instead of leaving a latent on-disk fault. - Refreshed the default Testnet Zakura bootstrap peer identities (`DEFAULT_TESTNET_ZAKURA_BOOTSTRAP_PEERS`) after the Testnet fleet's iroh node keys were rotated. The previous hardcoded node IDs were stale, so a fresh node @@ -35,6 +44,33 @@ and this project adheres to [Semantic Versioning](https://semver.org). ### Fixed +- Zebra now audits the Zakura header store when the state database opens and + self-repairs any incoherence (broken linkage, hash↔height index mismatches, + gaps with stranded rows above them, stale rows at committed heights) by + truncating the Zakura column families to the last coherent height in one + atomic batch, then letting header sync re-download the truncated suffix. + Stores corrupted by earlier binaries now heal on restart instead of staying + wedged below the network tip; each repair emits a warning and the + `state.zakura.header_store.incoherent` metric. +- Zakura header-store consensus reads now verify store invariants as they + walk: the difficulty-context walk checks that every stored header is the + block its hash row names and links to the row below it, and the anchor + lookup distinguishes a corrupted index round-trip from a genuinely unknown + anchor. A corrupted store now surfaces as an explicit local + `StoreIncoherent` error (never scored against peers) instead of poisoning + difficulty validation and rejecting honest headers with + `InvalidDifficultyThreshold`. +- Fixed three Zakura header-store write paths that could leave the on-disk + header store internally incoherent after chain forks, causing nodes to + reject valid headers from honest peers (`InvalidDifficultyThreshold` / + `UnknownAnchor`) and wedge below the network tip until manual intervention. + Header ranges must now link to their anchor and be internally contiguous + (rejected with the new `UnlinkedRange` error otherwise, which is classified + as a local, non-peer-scoring failure), header ranges re-delivered over + heights that already have committed block bodies no longer re-insert + provisional header rows below the body tip, and the header row seeded from + a committed best-chain block is skipped when it does not link to the stored + row below it (header-range sync converges the store instead). - Fixed dual-stack Zakura fallback shutting down the Zakura serving layer. When the stall watchdog resumes legacy `ChainSync`, Zakura now keeps its header- and block-sync reactors alive as a serving/advertising bridge while an apply diff --git a/Cargo.lock b/Cargo.lock index 5b397748cff..dc19b1b4066 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1349,9 +1349,9 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.18" +version = "0.9.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" dependencies = [ "crossbeam-utils", ] diff --git a/deny.toml b/deny.toml index 3ee845cc4e6..1261b6ea9e9 100644 --- a/deny.toml +++ b/deny.toml @@ -13,11 +13,9 @@ yanked = "deny" # Remove each ignore entry as the dependency chain is upgraded. ignore = [ "RUSTSEC-2021-0139", # ansi_term — transitive via abscissa_core -> structopt - "RUSTSEC-2022-0104", # structopt (maintenance mode) — transitive via abscissa_core + "RUSTSEC-2022-0104", # structopt (maintenance mode) — transitive via abscissa_core and zebra-utils "RUSTSEC-2024-0375", # atty (unmaintained) — transitive via abscissa_core -> structopt - "RUSTSEC-2021-0145", # atty (unsound) — transitive via abscissa_core -> structopt "RUSTSEC-2024-0370", # proc-macro-error — transitive via abscissa_core -> structopt - "RUSTSEC-2022-0104", # structopt (unmaintained) — direct dep of zebra-utils; no safe upgrade, migrate to clap 4 "RUSTSEC-2026-0173", # proc-macro-error2 (unmaintained) — transitive via getset "RUSTSEC-2025-0119", # number_prefix — transitive via indicatif "RUSTSEC-2025-0141", # bincode — direct dependency diff --git a/zebra-state/src/error.rs b/zebra-state/src/error.rs index aaf3f124b5e..40533f13fb6 100644 --- a/zebra-state/src/error.rs +++ b/zebra-state/src/error.rs @@ -195,6 +195,68 @@ impl From for CommitCheckpointVerifiedError { } } +/// An internal invariant of the zakura header store was found violated while +/// reading it. +/// +/// This is a **local storage fault**, never evidence about a peer: readers +/// return it instead of feeding rows from more than one branch (or from beside +/// a gap) into consensus validation, where the corruption would otherwise +/// surface as a misleading validation failure (`InvalidDifficultyThreshold`, +/// `UnknownAnchor`) attributed to whoever supplied the input being validated. +#[derive(Debug, Error, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum StoreIncoherentError { + /// The header row at `height` does not link to the stored row below it. + #[error( + "header store incoherent: header at {height:?} links to {expected_parent} but the stored row below is {actual_below}" + )] + BrokenLinkage { + /// Height of the header whose parent link failed to resolve. + height: block::Height, + /// The parent hash the header claims (`previous_block_hash`). + expected_parent: block::Hash, + /// The hash actually stored at `height - 1`. + actual_below: block::Hash, + }, + + /// A header row exists at `height` but the row below it is missing. + #[error( + "header store incoherent: no stored row at {missing:?} below the header at {height:?}" + )] + Gap { + /// Height of the stored header above the gap. + height: block::Height, + /// The missing height (`height - 1`). + missing: block::Height, + }, + + /// The header row at `height` is not the block its hash row names. + #[error( + "header store incoherent: header stored at {height:?} hashes to {computed} but the hash row names {indexed}" + )] + HeaderHashMismatch { + /// Height of the divergent rows. + height: block::Height, + /// The hash the height→hash index names. + indexed: block::Hash, + /// The stored header's actual hash. + computed: block::Hash, + }, + + /// The hash→height and height→hash indexes disagree about a hash. + #[error( + "header store incoherent: hash {hash} is indexed at {height:?} but that height stores {stored:?}" + )] + BijectionMismatch { + /// The hash whose round-trip failed. + hash: block::Hash, + /// The height the hash→height index reports for it. + height: block::Height, + /// What the height→hash index stores there instead. + stored: Option, + }, +} + /// An error describing why a header-only range could not be committed. #[derive(Debug, Error, Clone, PartialEq, Eq)] #[non_exhaustive] @@ -258,6 +320,21 @@ pub enum CommitHeaderRangeError { #[error("header height overflow")] HeightOverflow, + /// A header in the range does not link to the anchor or to its predecessor, + /// so committing it would break the header store's linkage invariant. + #[error( + "header at {height:?} links to {actual_parent} instead of its predecessor {expected_parent}" + )] + UnlinkedRange { + /// Height of the first header that fails to link. + height: block::Height, + /// The hash of the row the header must link to (the anchor, or the + /// previous header in the range). + expected_parent: block::Hash, + /// The header's actual `previous_block_hash`. + actual_parent: block::Hash, + }, + /// A committed immutable header conflicts with the requested header. #[error("header at finalized height {height:?} conflicts with an existing header")] ImmutableConflict { @@ -308,6 +385,15 @@ pub enum CommitHeaderRangeError { height: block::Height, }, + /// The local header store was found internally incoherent while reading + /// the context needed to validate the range. + /// + /// This is a local storage fault, not a peer validation failure: the range + /// was rejected because the store cannot supply trustworthy context, not + /// because the range itself was shown invalid. + #[error("header store incoherent while validating range: {0}")] + StoreIncoherent(#[from] StoreIncoherentError), + /// Contextual header validation failed. #[error("could not contextually validate header")] ValidateContextError(#[from] Box), diff --git a/zebra-state/src/lib.rs b/zebra-state/src/lib.rs index 015f190c428..88159fbd242 100644 --- a/zebra-state/src/lib.rs +++ b/zebra-state/src/lib.rs @@ -43,7 +43,8 @@ pub use config::{ pub use constants::{state_database_format_version_in_code, MAX_BLOCK_REORG_HEIGHT}; pub use error::{ BoxError, CloneError, CommitBlockError, CommitCheckpointVerifiedError, CommitHeaderRangeError, - CommitSemanticallyVerifiedError, DuplicateNullifierError, ValidateContextError, + CommitSemanticallyVerifiedError, DuplicateNullifierError, StoreIncoherentError, + ValidateContextError, }; pub use request::{ CheckpointVerifiedBlock, CommitSemanticallyVerifiedBlockRequest, HashOrHeight, MappedRequest, diff --git a/zebra-state/src/service/finalized_state/disk_db.rs b/zebra-state/src/service/finalized_state/disk_db.rs index 7af2be19519..34ffe76d9a3 100644 --- a/zebra-state/src/service/finalized_state/disk_db.rs +++ b/zebra-state/src/service/finalized_state/disk_db.rs @@ -122,6 +122,11 @@ pub struct DiskDb { pub struct DiskWriteBatch { /// The inner RocksDB write batch. batch: rocksdb::WriteBatch, + + /// Rows deleted by zakura header-store suffix replacements staged in this + /// batch. Nonzero means the batch performs a header reorg; the write site + /// uses this to run the post-reorg store audit after committing. + zakura_replaced_rows: usize, } impl Debug for DiskWriteBatch { @@ -548,8 +553,22 @@ impl DiskWriteBatch { pub fn new() -> Self { DiskWriteBatch { batch: rocksdb::WriteBatch::default(), + zakura_replaced_rows: 0, } } + + /// Records that a zakura suffix replacement staged in this batch deleted + /// `deleted_rows` rows. + pub(crate) fn note_zakura_suffix_replacement(&mut self, deleted_rows: usize) { + self.zakura_replaced_rows += deleted_rows; + } + + /// Returns the number of rows deleted by zakura suffix replacements + /// staged in this batch. Nonzero means this batch performs a header + /// reorg, and the write site should audit the store after committing it. + pub(crate) fn zakura_suffix_replaced_rows(&self) -> usize { + self.zakura_replaced_rows + } } impl DiskDb { diff --git a/zebra-state/src/service/finalized_state/zebra_db.rs b/zebra-state/src/service/finalized_state/zebra_db.rs index e7ba5325b2f..25170417702 100644 --- a/zebra-state/src/service/finalized_state/zebra_db.rs +++ b/zebra-state/src/service/finalized_state/zebra_db.rs @@ -175,6 +175,17 @@ impl ZebraDb { ) } + // Audit the zakura header store's on-disk invariants and truncate any + // incoherent suffix, so a store corrupted by an earlier binary + // self-heals at startup instead of wedging header sync (headers are + // re-fetchable, so correctness beats preserved rows). Read-only + // instances cannot repair; their reads surface any corruption as + // explicit `StoreIncoherentError`s instead. + if !read_only { + db.audit_and_repair_zakura_header_store() + .expect("startup header-store repair write failed: RocksDB is unavailable"); + } + db.spawn_format_change(format_change); db diff --git a/zebra-state/src/service/finalized_state/zebra_db/block.rs b/zebra-state/src/service/finalized_state/zebra_db/block.rs index 8feead51841..d82a2d48bd3 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block.rs @@ -36,11 +36,11 @@ use crate::{ constants::{ MAX_BLOCK_REORG_HEIGHT, MAX_HEADER_SYNC_HEIGHT_RANGE, MAX_PRUNE_HEIGHTS_PER_COMMIT, }, - error::{CommitCheckpointVerifiedError, CommitHeaderRangeError}, + error::{CommitCheckpointVerifiedError, CommitHeaderRangeError, StoreIncoherentError}, request::FinalizedBlock, service::check, service::finalized_state::{ - disk_db::{DiskDb, DiskWriteBatch, ReadDisk, WriteDisk}, + disk_db::{DiskWriteBatch, ReadDisk, WriteDisk}, disk_format::{ block::TransactionLocation, shielded::CommitmentRootsByHeight, @@ -57,6 +57,11 @@ use crate::{ #[cfg(feature = "indexer")] use crate::request::Spend; +mod canonical_suffix; +mod startup_audit; + +pub(crate) use canonical_suffix::CanonicalHeaderRow; + #[cfg(test)] mod tests; @@ -66,7 +71,13 @@ const ZAKURA_HEADER_BY_HEIGHT: &str = "zakura_header_by_height"; pub const ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT: &str = "zakura_header_body_size_by_height"; #[derive(Copy, Clone, Debug, Eq, PartialEq)] -struct AdvertisedBodySize(u32); +pub(crate) struct AdvertisedBodySize(u32); + +/// Builds an [`AdvertisedBodySize`] for tests that plant raw rows. +#[cfg(test)] +pub(crate) fn test_body_size(size: u32) -> AdvertisedBodySize { + AdvertisedBodySize(size) +} impl AdvertisedBodySize { fn new(size: u32) -> Option { @@ -544,31 +555,85 @@ impl ZebraDb { } /// Returns recent header difficulty/time context in reverse height order, - /// starting at `height`. + /// starting at `height`, verifying `previous_block_hash` linkage at every + /// step of the walk. + /// + /// Returns an empty context when there is no stored row at `height` (the + /// caller decides whether that anchor is unknown), and a shorter-than-span + /// context when the walk reaches genesis. + /// + /// # Errors + /// + /// Returns [`StoreIncoherentError`] when the walk finds a header row that + /// is not the block its hash row names, a row that does not link to the + /// row below it, or a gap below a stored row. Feeding such a window into + /// difficulty validation would mix rows from more than one branch (or + /// shift the adjustment window), producing `InvalidDifficultyThreshold` + /// rejections of honest input — the reader surfaces the storage fault + /// explicitly instead. The per-row hash check costs one header hash per + /// consumed row, negligible next to the validation the window feeds. pub fn recent_header_context( &self, height: block::Height, - ) -> Vec<( - zebra_chain::work::difficulty::CompactDifficulty, - DateTime, - )> { + ) -> Result< + Vec<( + zebra_chain::work::difficulty::CompactDifficulty, + DateTime, + )>, + StoreIncoherentError, + > { let mut context = Vec::with_capacity(check::difficulty::POW_ADJUSTMENT_BLOCK_SPAN); - let mut current_height = Some(height); - while let Some(height) = current_height { - let Some((_hash, header)) = self.header_by_height(height) else { - break; - }; + let Some((mut hash, mut header)) = self.header_by_height(height) else { + return Ok(context); + }; + let mut height = height; + + loop { + let computed = block::Hash::from(&*header); + if computed != hash { + return Err(StoreIncoherentError::HeaderHashMismatch { + height, + indexed: hash, + computed, + }); + } context.push((header.difficulty_threshold, header.time)); + if context.len() == check::difficulty::POW_ADJUSTMENT_BLOCK_SPAN { - break; + return Ok(context); } + let Ok(below) = height.previous() else { + // The walk reached genesis: a short context is legitimate, and + // the difficulty functions handle it (MedianTime clamps + // negative heights to zero). + return Ok(context); + }; - current_height = height.previous().ok(); - } + let Some((below_hash, below_header)) = self.header_by_height(below) else { + // Rows must be contiguous from genesis up to the header tip + // (full-block rows below the body tip — retained even under + // pruning — and zakura rows above it), so a missing row below + // a stored one is a gap, not the end of history. + return Err(StoreIncoherentError::Gap { + height, + missing: below, + }); + }; + + if header.previous_block_hash != below_hash { + return Err(StoreIncoherentError::BrokenLinkage { + height, + expected_parent: header.previous_block_hash, + actual_below: below_hash, + }); + } - context + height = below; + hash = below_hash; + header = below_header; + } } /// Returns header-known, body-missing heights. @@ -1222,12 +1287,15 @@ impl ZebraDb { // Track batch commit latency for observability let batch_start = std::time::Instant::now(); + let zakura_replaced_rows = batch.zakura_suffix_replaced_rows(); self.db .write(batch) .expect("unexpected rocksdb error while writing block"); metrics::histogram!("zebra.state.rocksdb.batch_commit.duration_seconds") .record(batch_start.elapsed().as_secs_f64()); + self.audit_zakura_header_store_after_reorg(zakura_replaced_rows); + tracing::trace!(?source, "committed block from"); Ok(finalized.hash) @@ -1264,12 +1332,39 @@ impl ZebraDb { block: &Arc, ) -> Result<(), CommitHeaderRangeError> { let mut batch = DiskWriteBatch::new(); - batch.prepare_zakura_header_from_committed_block(&self.db, height, block)?; + batch.prepare_zakura_header_from_committed_block(self, height, block)?; + let zakura_replaced_rows = batch.zakura_suffix_replaced_rows(); self.db .write(batch) .map_err(|error| CommitHeaderRangeError::StorageWriteError { error: error.to_string(), - }) + })?; + + self.audit_zakura_header_store_after_reorg(zakura_replaced_rows); + + Ok(()) + } + + /// Audits the zakura header store after a batch that replaced a header + /// suffix (`replaced_rows > 0`), repairing any violation. + /// + /// This is the after-every-reorg-batch half of the store audit (the + /// startup half runs in [`ZebraDb::new`]): any residual writer bug in + /// this class becomes a repaired, observable transient at the moment it + /// happens instead of a latent on-disk fault. Suffix replacements are + /// rare (real chain forks), so the `O(header frontier)` audit cost is + /// not on the steady-state commit path. + pub(crate) fn audit_zakura_header_store_after_reorg(&self, replaced_rows: usize) { + if replaced_rows == 0 { + return; + } + + if let Err(error) = self.audit_and_repair_zakura_header_store() { + tracing::warn!( + ?error, + "post-reorg zakura header store audit failed to write its repair" + ); + } } } @@ -1735,7 +1830,7 @@ impl DiskWriteBatch { // heights with no committed body (the frontier above the body tip). // This is unconditional so it also cleans up rows left by a prior run // that had `enable_zakura_header_seed_from_committed_blocks` enabled. - self.prepare_zakura_header_release_from_committed_block(db, *height, block)?; + self.prepare_zakura_header_release_from_committed_block(zebra_db, *height, block)?; // Index the block header, hash, and height. This also restores the // verified full block row after any provisional cleanup above. @@ -1795,76 +1890,65 @@ impl DiskWriteBatch { /// committed full block. /// /// Full block verification is authoritative for the stored body. If a - /// provisional Zakura header at this height differs, replace it with the - /// block-derived header and drop stale provisional descendants. + /// provisional Zakura header at this height differs, the seeded block is + /// the new best chain at its height: the suffix from this height upward + /// is replaced through [`Self::set_canonical_suffix`], dropping stale + /// provisional descendants of the displaced row. + /// + /// A block whose parent hash does not match the stored header row below + /// `height` is skipped without error: writing it would leave a gap or + /// broken link in the header store, so the store instead waits for + /// header-range sync to deliver the connecting rows (see the linkage + /// refusal below). #[allow(clippy::unwrap_in_result)] pub fn prepare_zakura_header_from_committed_block( &mut self, - db: &DiskDb, + zebra_db: &ZebraDb, height: block::Height, block: &Arc, ) -> Result<(), CommitHeaderRangeError> { - let zakura_header_by_height = db.cf_handle(ZAKURA_HEADER_BY_HEIGHT).unwrap(); - let zakura_hash_by_height = db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); - let zakura_height_by_hash = db.cf_handle(ZAKURA_HEADER_HEIGHT_BY_HASH).unwrap(); - let zakura_body_size_by_height = db.cf_handle(ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT).unwrap(); - let zakura_roots_by_height = db.cf_handle(COMMITMENT_ROOTS_BY_HEIGHT).unwrap(); - let tx_by_loc = db.cf_handle("tx_by_loc").unwrap(); - let hash = block.hash(); - let existing_zakura_header: Option> = - db.zs_get(&zakura_header_by_height, &height); - if existing_zakura_header.as_ref() == Some(&block.header) - && db.zs_get::<_, _, block::Hash>(&zakura_hash_by_height, &height) == Some(hash) + // An identical stored row needs no reconciliation, and the frontier + // above it survives (the common case: header sync ran ahead of the + // non-finalized best chain). + if zebra_db.zakura_header(height).as_ref() == Some(&block.header) + && zebra_db.zakura_header_hash(height) == Some(hash) { return Ok(()); } - if existing_zakura_header.is_some_and(|existing_header| existing_header != block.header) { - let best_header_tip: Option<(block::Height, block::Hash)> = - db.zs_last_key_value(&zakura_hash_by_height); - - if let Some((best_header_tip, _)) = best_header_tip { - for old_height in height.0..=best_header_tip.0 { - let old_height = block::Height(old_height); - - if old_height != height - && db.zs_contains( - &tx_by_loc, - &TransactionLocation::min_for_height(old_height), - ) - { - return Err(CommitHeaderRangeError::ConflictingFullBlockHeader { - height: old_height, - }); - } - - if let Some(old_hash) = - db.zs_get::<_, _, block::Hash>(&zakura_hash_by_height, &old_height) - { - self.zs_delete(&zakura_height_by_hash, old_hash); - } - - self.zs_delete(&zakura_hash_by_height, old_height); - self.zs_delete(&zakura_header_by_height, old_height); - self.zs_delete(&zakura_body_size_by_height, old_height); - self.zs_delete(&zakura_roots_by_height, old_height); - } - } - } else if let Some(old_hash) = - db.zs_get::<_, _, block::Hash>(&zakura_hash_by_height, &height) - { - if old_hash != hash { - self.zs_delete(&zakura_height_by_hash, old_hash); - } + // Seeds can jump to a non-finalized best tip whose parent is not the + // stored row below it. Refuse those seeds so the header store stays + // linked; header-range sync will later deliver the missing rows. + let parent_hash: Option = height + .previous() + .ok() + .and_then(|parent_height| zebra_db.header_hash(parent_height)); + if parent_hash != Some(block.header.previous_block_hash) { + tracing::debug!( + ?height, + ?hash, + parent = ?block.header.previous_block_hash, + stored_parent = ?parent_hash, + "skipping Zakura header seed that does not link to the stored row below it" + ); + return Ok(()); } - self.zs_insert(&zakura_header_by_height, height, &block.header); - self.zs_insert(&zakura_hash_by_height, height, hash); - self.zs_insert(&zakura_height_by_hash, hash, height); + let fork_height = height + .previous() + .expect("the linkage refusal above required a stored parent row below this height"); - Ok(()) + self.set_canonical_suffix( + zebra_db, + (fork_height, block.header.previous_block_hash), + &[CanonicalHeaderRow { + header: block.header.clone(), + advertised_body_size: None, + roots: None, + }], + ) } /// Prepare a database batch that releases the Zakura header store entry for a @@ -1883,16 +1967,16 @@ impl DiskWriteBatch { #[allow(clippy::unwrap_in_result)] pub fn prepare_zakura_header_release_from_committed_block( &mut self, - db: &DiskDb, + zebra_db: &ZebraDb, height: block::Height, block: &Arc, ) -> Result<(), CommitHeaderRangeError> { + let db = &zebra_db.db; let zakura_header_by_height = db.cf_handle(ZAKURA_HEADER_BY_HEIGHT).unwrap(); let zakura_hash_by_height = db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); let zakura_height_by_hash = db.cf_handle(ZAKURA_HEADER_HEIGHT_BY_HASH).unwrap(); let zakura_body_size_by_height = db.cf_handle(ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT).unwrap(); let zakura_roots_by_height = db.cf_handle(COMMITMENT_ROOTS_BY_HEIGHT).unwrap(); - let tx_by_loc = db.cf_handle("tx_by_loc").unwrap(); let existing_zakura_header: Option> = db.zs_get(&zakura_header_by_height, &height); @@ -1903,36 +1987,15 @@ impl DiskWriteBatch { return Ok(()); } - // A committed block whose header conflicts with the provisional chain at - // this height invalidates the provisional descendants built on top of it. - // Drop them, but never overwrite a height that already has a committed body. + // A committed block whose header conflicts with the provisional chain + // at this height invalidates the provisional descendants built on top + // of it: truncate the suffix above this height through the owner + // primitive (its committed-body interlock refuses to touch any height + // that already has a body, as the old bounded loop did). The empty + // replacement is anchored at this block, whose full-block row is + // staged in this same batch. if existing_zakura_header.is_some_and(|existing_header| existing_header != block.header) { - let zakura_tip: Option<(block::Height, block::Hash)> = - db.zs_last_key_value(&zakura_hash_by_height); - - if let Some((zakura_tip, _)) = zakura_tip { - for descendant in (height.0 + 1)..=zakura_tip.0 { - let descendant = block::Height(descendant); - - if db.zs_contains(&tx_by_loc, &TransactionLocation::min_for_height(descendant)) - { - return Err(CommitHeaderRangeError::ConflictingFullBlockHeader { - height: descendant, - }); - } - - if let Some(old_hash) = - db.zs_get::<_, _, block::Hash>(&zakura_hash_by_height, &descendant) - { - self.zs_delete(&zakura_height_by_hash, old_hash); - } - - self.zs_delete(&zakura_hash_by_height, descendant); - self.zs_delete(&zakura_header_by_height, descendant); - self.zs_delete(&zakura_body_size_by_height, descendant); - self.zs_delete(&zakura_roots_by_height, descendant); - } - } + self.set_canonical_suffix(zebra_db, (height, block.hash()), &[])?; } // Release the provisional row at this height. @@ -1996,9 +2059,6 @@ impl DiskWriteBatch { }); } - let header_by_height = zebra_db.db.cf_handle(ZAKURA_HEADER_BY_HEIGHT).unwrap(); - let hash_by_height = zebra_db.db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); - let height_by_hash = zebra_db.db.cf_handle(ZAKURA_HEADER_HEIGHT_BY_HASH).unwrap(); let body_size_by_height = zebra_db .db .cf_handle(ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT) @@ -2010,17 +2070,26 @@ impl DiskWriteBatch { .or_else(|| (anchor == zebra_db.network().genesis_hash()).then_some(block::Height(0))) .ok_or(CommitHeaderRangeError::UnknownAnchor { anchor })?; - if anchor != zebra_db.network().genesis_hash() - && zebra_db.header_hash(anchor_height) != Some(anchor) - { - return Err(CommitHeaderRangeError::UnknownAnchor { anchor }); + // The hash→height index knows the anchor, so a failed height→hash + // round-trip is a bijection violation in our own store — a local + // storage fault, not an unknown anchor supplied by the caller. + if anchor != zebra_db.network().genesis_hash() { + let stored = zebra_db.header_hash(anchor_height); + if stored != Some(anchor) { + return Err(StoreIncoherentError::BijectionMismatch { + hash: anchor, + height: anchor_height, + stored, + } + .into()); + } } let finalized_height = zebra_db.finalized_tip_height(); let best_header_tip = zebra_db.best_header_tip().map(|(height, _)| height); let checkpoints = zebra_db.network().checkpoint_list(); - let mut recent_headers = zebra_db.recent_header_context(anchor_height); + let mut recent_headers = zebra_db.recent_header_context(anchor_height)?; if recent_headers.is_empty() { if anchor == zebra_db.network().genesis_hash() && anchor_height == block::Height(0) { return Err(CommitHeaderRangeError::MissingGenesisAnchor { anchor }); @@ -2029,8 +2098,17 @@ impl DiskWriteBatch { } let mut first_conflicting_height = None; + let mut first_missing_height = None; let mut validated_headers = Vec::with_capacity(headers.len()); + // Each header must link to the anchor (for the first header) or to its + // predecessor in the range. Without this check, a range anchored at the + // same-height hash of a *different* branch can pass contextual + // difficulty validation and commit a suffix that does not link to the + // row below it — an on-disk linkage violation reachable from a single + // peer response. + let mut expected_parent = anchor; + for (index, header) in headers.iter().enumerate() { let offset = u32::try_from(index + 1).map_err(|_| CommitHeaderRangeError::HeightOverflow)?; @@ -2039,6 +2117,16 @@ impl DiskWriteBatch { let hash = block::Hash::from(&**header); let body_size = body_sizes[index]; let roots = &tree_aux_roots[index]; + + if header.previous_block_hash != expected_parent { + return Err(CommitHeaderRangeError::UnlinkedRange { + height, + expected_parent, + actual_parent: header.previous_block_hash, + }); + } + expected_parent = hash; + if roots.height != height { return Err(CommitHeaderRangeError::TreeAuxRootHeightMismatch { expected_height: height, @@ -2077,6 +2165,8 @@ impl DiskWriteBatch { first_conflicting_height.get_or_insert(height); } + } else { + first_missing_height.get_or_insert(height); } check::header_is_valid_for_recent_chain( @@ -2138,71 +2228,104 @@ impl DiskWriteBatch { } } - if let (Some(first_conflicting_height), Some(best_header_tip)) = - (first_conflicting_height, best_header_tip) - { - for height in first_conflicting_height.0..=best_header_tip.0 { - let height = block::Height(height); - - if zebra_db.contains_body_at_height(height) { - return Err(CommitHeaderRangeError::ConflictingFullBlockHeader { height }); - } + // The suffix replacement starts at the first conflicting height, or — + // for a pure extension — at the first height without a stored row. + // Rows below it match the stored chain (a conflict below a missing + // row is impossible: rows are contiguous up to the header tip), so + // chain membership only changes from `replace_from` upward. `None` + // means the whole range re-delivered rows the store already has. + let replace_from = match (first_conflicting_height, first_missing_height) { + (Some(conflict), _) => Some(conflict), + (None, missing) => missing, + }; - if let Some(old_hash) = zebra_db.zakura_header_hash(height) { - self.zs_delete(&height_by_hash, old_hash); - } + // Refresh the advisory rows of re-delivered matching headers: merge + // the body-size hints and re-stage the provisional roots. These + // writes never change which header is stored at a height, so they + // stay outside the suffix-replacement primitive. + for (index, (height, _hash, _header, body_size)) in validated_headers.iter().enumerate() { + let height = *height; + if replace_from.is_some_and(|replace_from| height >= replace_from) { + break; + } - self.zs_delete(&hash_by_height, height); - self.zs_delete(&header_by_height, height); - self.zs_delete(&body_size_by_height, height); - self.zs_delete(&roots_by_height, height); + // Finalized block heights already have authoritative block rows and + // verified roots, even when pruning has removed their transactions. + // Re-delivered headers must not recreate provisional zakura rows + // there, because those rows are only trimmed during body commit. + if zebra_db.contains_height(height) { + continue; } - } - for (index, (height, hash, header, body_size)) in validated_headers.into_iter().enumerate() - { - let same_header = zebra_db.zakura_header_hash(height) == Some(hash); - let advertised_body_size = match ( - same_header, + let merged_body_size = match ( zebra_db.advertised_body_size(height), - AdvertisedBodySize::new(body_size).map(AdvertisedBodySize::get), + AdvertisedBodySize::new(*body_size).map(AdvertisedBodySize::get), ) { - (true, existing, Some(new)) => Some(existing.unwrap_or(0).max(new)), - (true, existing, None) => existing, - (false, _existing, new) => new, + (existing, Some(new)) => Some(existing.unwrap_or(0).max(new)), + (existing, None) => existing, }; - - self.zs_insert(&header_by_height, height, header); - self.zs_insert(&hash_by_height, height, hash); - self.zs_insert(&height_by_hash, hash, height); - if let Some(body_size) = advertised_body_size.and_then(AdvertisedBodySize::new) { + if let Some(body_size) = merged_body_size.and_then(AdvertisedBodySize::new) { self.zs_insert(&body_size_by_height, height, body_size); - } else { - self.zs_delete(&body_size_by_height, height); - } - // A height with a committed body already has a *verified* row in the - // serving index, written by the body-commit batch. Peer-supplied roots - // are unauthenticated until verify-before-commit, and that verification - // only ever runs above the body tip — so a header range re-delivered - // over committed heights (a header store behind the body store, or a - // late range response racing body sync) must never overwrite the - // verified row: committed roots win on any overlap (design §9). - if !zebra_db.contains_body_at_height(height) { - let roots = &tree_aux_roots[index]; - self.zs_insert( - &roots_by_height, - height, - CommitmentRootsByHeight { - sapling: roots.sapling_root, - orchard: roots.orchard_root, - ironwood: roots.ironwood_root, - sapling_tx: roots.sapling_tx, - orchard_tx: roots.orchard_tx, - ironwood_tx: roots.ironwood_tx, - auth_data_root: roots.auth_data_root, - }, - ); } + + let roots = &tree_aux_roots[index]; + self.zs_insert( + &roots_by_height, + height, + CommitmentRootsByHeight { + sapling: roots.sapling_root, + orchard: roots.orchard_root, + ironwood: roots.ironwood_root, + sapling_tx: roots.sapling_tx, + orchard_tx: roots.orchard_tx, + ironwood_tx: roots.ironwood_tx, + auth_data_root: roots.auth_data_root, + }, + ); + } + + // Chain membership changes only through the suffix-replacement + // primitive: total deletion above the fork point in all five column + // families, then the new rows. Rows above the incoming range's end + // that belonged to the replaced branch are deleted with it. + if let Some(replace_from) = replace_from { + let fork_height = replace_from + .previous() + .map_err(|_| CommitHeaderRangeError::HeightOverflow)?; + let fork_hash = if fork_height == anchor_height { + anchor + } else { + // The fork row is inside the incoming range and matched the + // store during validation. + let fork_index = usize::try_from(fork_height.0 - anchor_height.0 - 1) + .expect("fork height is above the anchor"); + validated_headers[fork_index].1 + }; + + let new_rows: Vec = validated_headers + .iter() + .enumerate() + .filter(|(_, (height, ..))| *height >= replace_from) + .map(|(index, (_height, _hash, header, body_size))| { + let roots = &tree_aux_roots[index]; + CanonicalHeaderRow { + header: (*header).clone(), + advertised_body_size: AdvertisedBodySize::new(*body_size) + .map(AdvertisedBodySize::get), + roots: Some(CommitmentRootsByHeight { + sapling: roots.sapling_root, + orchard: roots.orchard_root, + ironwood: roots.ironwood_root, + sapling_tx: roots.sapling_tx, + orchard_tx: roots.orchard_tx, + ironwood_tx: roots.ironwood_tx, + auth_data_root: roots.auth_data_root, + }), + } + }) + .collect(); + + self.set_canonical_suffix(zebra_db, (fork_height, fork_hash), &new_rows)?; } Ok(block::Hash::from( diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/canonical_suffix.rs b/zebra-state/src/service/finalized_state/zebra_db/block/canonical_suffix.rs new file mode 100644 index 00000000000..9300af7f245 --- /dev/null +++ b/zebra-state/src/service/finalized_state/zebra_db/block/canonical_suffix.rs @@ -0,0 +1,263 @@ +//! The single owner of zakura header-store chain membership: +//! [`DiskWriteBatch::set_canonical_suffix`] (REORG_PLAN Pillar 1). +//! +//! The zakura header store is five column families acting as a replicated +//! view of the canonical header chain above the finalized tip. Historically +//! it was mutated by four independent writers, each with its own bounded +//! delete loop; every proven corruption class (unlinked anchors, re-inserts +//! below the body tip, unlinked seeds, stranded suffixes) came from one of +//! those writers preserving the store invariants only under assumptions +//! about the others. +//! +//! This module replaces the per-writer delete/insert logic with one +//! primitive: *replace the suffix above a fork point*. Deletion is +//! total-above-fork by construction — it iterates the actual on-disk rows of +//! every column family, never a cached or computed tip — so stranding is +//! impossible. Linkage is a hard precondition, not a validation step callers +//! can forget. Committed heights are refused for deletes and inserts alike. +//! +//! Callers stay responsible for *policy*: contextual validation, checkpoint +//! conflicts, the cumulative-work gate, and reorg-depth limits all happen +//! before the primitive is invoked. The primitive owns *mechanism*: which +//! rows exist after a suffix replacement, in all five column families, in +//! one atomic batch. + +use std::sync::Arc; + +use zebra_chain::block::{self, Height}; + +use super::{ + AdvertisedBodySize, ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT, ZAKURA_HEADER_BY_HEIGHT, + ZAKURA_HEADER_HASH_BY_HEIGHT, ZAKURA_HEADER_HEIGHT_BY_HASH, +}; +use crate::{ + error::{CommitHeaderRangeError, StoreIncoherentError}, + service::finalized_state::{ + disk_db::{DiskWriteBatch, WriteDisk}, + disk_format::shielded::CommitmentRootsByHeight, + zebra_db::ZebraDb, + COMMITMENT_ROOTS_BY_HEIGHT, + }, +}; + +/// One header row of a new canonical suffix. +/// +/// The row's height is implied by its position: `fork_point + 1 + index`. +/// Its hash is computed from the header inside the primitive, so a caller +/// cannot stage a hash that disagrees with the stored header. +#[derive(Clone, Debug)] +pub(crate) struct CanonicalHeaderRow { + /// The header to store. + pub header: Arc, + + /// The advisory body-size hint, if known. Not consensus data; `None` + /// stores no row (and removes any stale hint left at this height). + pub advertised_body_size: Option, + + /// Provisional commitment roots for this height, if the caller has them. + /// `None` stores no row (the seed path has no roots for its block). + pub roots: Option, +} + +impl DiskWriteBatch { + /// Replaces the zakura header-store suffix above `fork_point` with + /// `new_rows`, in this batch. + /// + /// This is the **only** way any path may change which headers the zakura + /// store considers canonical above a height. In one atomic batch it: + /// + /// 1. deletes every row strictly above `fork_point` in all five header + /// column families — iterating each family's actual on-disk rows, so + /// stranded rows above gaps are removed too — including every + /// `height_by_hash` entry pointing above the fork (displaced and + /// already-orphaned entries alike), then + /// 2. writes `new_rows` at consecutive heights starting at + /// `fork_point + 1`. + /// + /// # Preconditions (hard errors, nothing staged on failure) + /// + /// - `fork_point` must be at or above the finalized tip + /// ([`CommitHeaderRangeError::ImmutableConflict`]): committed heights + /// are immutable through this primitive, which also guarantees no + /// insert lands at a committed height (the re-delivery bug shape). + /// - When `new_rows` is non-empty, the stored header row at + /// `fork_point.0` must be `fork_point.1` + /// ([`StoreIncoherentError::BijectionMismatch`]): the caller's fork + /// decision must describe the store being mutated. + /// - `new_rows[0]` must link to `fork_point.1` and every row to its + /// predecessor ([`CommitHeaderRangeError::UnlinkedRange`]): linkage is + /// structural here, not a caller obligation. + /// - No height above `fork_point` may hold a committed full block body + /// ([`CommitHeaderRangeError::ConflictingFullBlockHeader`]). Bodies + /// above the finalized tip cannot exist, so this is a safety interlock: + /// reaching it means the caller's orchestration is buggy, and the + /// switch aborts loudly instead of deleting a body's header row. + /// + /// `new_rows` may be empty: that truncates the store to `fork_point` + /// (the body-commit path drops conflicting provisional descendants this + /// way). An empty replacement skips the fork-row check, because the + /// caller may be staging a new full-block row at the fork height in this + /// same batch, which reads through `zebra_db` cannot see yet. + #[allow(clippy::unwrap_in_result)] + pub(crate) fn set_canonical_suffix( + &mut self, + zebra_db: &ZebraDb, + fork_point: (Height, block::Hash), + new_rows: &[CanonicalHeaderRow], + ) -> Result<(), CommitHeaderRangeError> { + let (fork_height, fork_hash) = fork_point; + + // Committed heights are immutable: the finalized tip is the highest + // committed block, so requiring the fork at or above it keeps every + // delete *and* insert strictly above committed history (valid on + // pruned stores too, where bodies are absent but heights committed). + if let Some(finalized_tip_height) = zebra_db.finalized_tip_height() { + if fork_height < finalized_tip_height { + return Err(CommitHeaderRangeError::ImmutableConflict { + height: fork_height + .next() + .map_err(|_| CommitHeaderRangeError::HeightOverflow)?, + }); + } + } + + // The fork row named by the caller must be the row on disk: a + // mismatch means the fork decision describes a different store state + // than the one being mutated. + if !new_rows.is_empty() { + let stored = zebra_db + .header_hash(fork_height) + .or_else(|| (fork_hash == zebra_db.network().genesis_hash()).then_some(fork_hash)); + if stored != Some(fork_hash) { + return Err(StoreIncoherentError::BijectionMismatch { + hash: fork_hash, + height: fork_height, + stored, + } + .into()); + } + } + + // Structural linkage: the new suffix must chain from the fork row. + let mut expected_parent = fork_hash; + let mut hashes = Vec::with_capacity(new_rows.len()); + for (index, row) in new_rows.iter().enumerate() { + let offset = + u32::try_from(index + 1).map_err(|_| CommitHeaderRangeError::HeightOverflow)?; + let height = + (fork_height + i64::from(offset)).ok_or(CommitHeaderRangeError::HeightOverflow)?; + + if row.header.previous_block_hash != expected_parent { + return Err(CommitHeaderRangeError::UnlinkedRange { + height, + expected_parent, + actual_parent: row.header.previous_block_hash, + }); + } + + let hash = block::Hash::from(&*row.header); + expected_parent = hash; + hashes.push((height, hash)); + } + + let header_cf = zebra_db.db.cf_handle(ZAKURA_HEADER_BY_HEIGHT).unwrap(); + let hash_cf = zebra_db.db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); + let height_by_hash_cf = zebra_db.db.cf_handle(ZAKURA_HEADER_HEIGHT_BY_HASH).unwrap(); + let body_size_cf = zebra_db + .db + .cf_handle(ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT) + .unwrap(); + let roots_cf = zebra_db.db.cf_handle(COMMITMENT_ROOTS_BY_HEIGHT).unwrap(); + + let Ok(delete_from) = fork_height.next() else { + // The fork is at the maximum height: nothing can exist above it, + // and the linkage loop already rejected any new rows. + return Ok(()); + }; + + // Total-above-fork deletion: walk the actual on-disk rows of every + // column family from the fork upward. No cached or computed tip + // bounds the loops, so rows stranded above gaps are deleted too. + // + // The committed-body interlock runs on the stored hash rows *and* + // the insert heights before anything is staged. + let stored_hash_rows: Vec<(Height, block::Hash)> = zebra_db + .db + .zs_forward_range_iter(&hash_cf, delete_from..) + .collect(); + for &(height, _) in &stored_hash_rows { + if zebra_db.contains_body_at_height(height) { + return Err(CommitHeaderRangeError::ConflictingFullBlockHeader { height }); + } + } + for &(height, _) in &hashes { + if zebra_db.contains_body_at_height(height) { + return Err(CommitHeaderRangeError::ConflictingFullBlockHeader { height }); + } + } + + let mut deleted_rows = 0; + for (height, _displaced_hash) in stored_hash_rows { + self.zs_delete(&hash_cf, height); + deleted_rows += 1; + } + + // The reverse index is hash-keyed, so deriving its deletions from the + // hash rows would miss entries already orphaned by earlier damage (a + // deleted hash row leaves its reverse entry behind). Scan the whole + // reverse index — it is frontier-sized — and delete every entry + // pointing above the fork, displaced and orphaned alike. + for (hash, points_at) in zebra_db + .db + .zs_forward_range_iter::<_, block::Hash, Height, _>(&height_by_hash_cf, ..) + { + if points_at > fork_height { + self.zs_delete(&height_by_hash_cf, hash); + deleted_rows += 1; + } + } + for (height, _) in zebra_db + .db + .zs_forward_range_iter::<_, Height, Arc, _>(&header_cf, delete_from..) + { + self.zs_delete(&header_cf, height); + deleted_rows += 1; + } + for (height, _) in zebra_db + .db + .zs_forward_range_iter::<_, Height, AdvertisedBodySize, _>(&body_size_cf, delete_from..) + { + self.zs_delete(&body_size_cf, height); + deleted_rows += 1; + } + for (height, _) in zebra_db + .db + .zs_forward_range_iter::<_, Height, CommitmentRootsByHeight, _>( + &roots_cf, + delete_from.., + ) + { + self.zs_delete(&roots_cf, height); + deleted_rows += 1; + } + + // A batch that deletes rows performs a header reorg: the write site + // audits the store after committing it (the Pillar-3 hook). + self.note_zakura_suffix_replacement(deleted_rows); + + // Write the new suffix. + for (row, &(height, hash)) in new_rows.iter().zip(&hashes) { + self.zs_insert(&header_cf, height, &row.header); + self.zs_insert(&hash_cf, height, hash); + self.zs_insert(&height_by_hash_cf, hash, height); + if let Some(body_size) = row.advertised_body_size.and_then(AdvertisedBodySize::new) { + self.zs_insert(&body_size_cf, height, body_size); + } + if let Some(roots) = &row.roots { + self.zs_insert(&roots_cf, height, *roots); + } + } + + Ok(()) + } +} diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/startup_audit.rs b/zebra-state/src/service/finalized_state/zebra_db/block/startup_audit.rs new file mode 100644 index 00000000000..5d355f77d13 --- /dev/null +++ b/zebra-state/src/service/finalized_state/zebra_db/block/startup_audit.rs @@ -0,0 +1,442 @@ +//! Startup audit and self-repair for the zakura header store. +//! +//! The zakura header store is five height-indexed column families acting as a +//! replicated view of the canonical header chain above the finalized tip. Its +//! invariants (hash↔height bijection, parent linkage anchored at the finalized +//! tip, no gaps or stranded rows below the header tip) are enforced by the +//! writers, but a store corrupted by an earlier binary stays corrupted on disk +//! and wedges header sync: consensus reads surface the damage as +//! [`StoreIncoherentError`](crate::error::StoreIncoherentError) and refuse to +//! feed stale rows into validation, so the node can no longer make progress +//! past the poisoned window. +//! +//! This module runs the store audit once at [`ZebraDb`] startup and repairs +//! any violation by truncating the zakura column families to the last +//! coherent height in one atomic batch. Headers are re-fetchable — header +//! sync re-downloads the truncated suffix — so correctness beats preserved +//! rows: any residual write-path bug in this class becomes a self-healing, +//! observable transient instead of a permanent on-disk wedge. +//! +//! The audit cost is `O(header frontier)`: the zakura column families only +//! hold rows above the finalized tip (plus any stale rows this audit exists +//! to remove), and the verified commitment-roots history below the tip is +//! never scanned. + +use std::{ + collections::{BTreeMap, HashMap}, + sync::Arc, +}; + +use zebra_chain::block::{self, Height}; + +use super::{ + AdvertisedBodySize, ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT, ZAKURA_HEADER_BY_HEIGHT, + ZAKURA_HEADER_HASH_BY_HEIGHT, ZAKURA_HEADER_HEIGHT_BY_HASH, +}; +use crate::service::finalized_state::{ + disk_db::{DiskWriteBatch, WriteDisk}, + disk_format::shielded::CommitmentRootsByHeight, + zebra_db::ZebraDb, + COMMITMENT_ROOTS_BY_HEIGHT, +}; + +/// How many violations are included in the repair log line. The full list can +/// be as long as the stranded suffix; the first few identify the fault shape. +const LOGGED_VIOLATIONS: usize = 8; + +/// A single header-store invariant violation found by the startup audit. +/// +/// Every violation is repaired by deleting rows; the variants exist for +/// logging and for test assertions on the fault shape. The fields are +/// diagnostic payload rendered through `Debug` in the repair warning. +#[allow(dead_code)] +#[derive(Clone, Debug)] +pub(crate) enum ZakuraStoreViolation { + /// A zakura row at a height with a committed block (`contains_height`). + /// + /// Committed heights have authoritative full-block rows, and the zakura + /// row at a height is trimmed when its body commits, so a surviving row + /// here is stale (the pre-guard re-delivery bug shape). The predicate is + /// the consensus `hash_by_height` row — which pruning retains — not body + /// presence, so stale rows at pruned heights are found too. + StaleRowAtCommittedHeight { + /// The column family holding the stale row. + cf: &'static str, + /// The committed height of the stale row. + height: Height, + }, + + /// A `hash_by_height` row at `height` has no `header_by_height` row. + MissingHeaderRow { + /// The height with a hash row but no header row. + height: Height, + }, + + /// The header stored at `height` is not the block its hash row names. + HeaderHashMismatch { + /// The height of the divergent rows. + height: Height, + /// The hash the height→hash index names. + indexed: block::Hash, + /// The stored header's actual hash. + computed: block::Hash, + }, + + /// The header at `height` does not link to the stored row below it. + BrokenLinkage { + /// The height of the header whose parent link failed to resolve. + height: Height, + /// The parent hash the header claims (`previous_block_hash`). + expected_parent: block::Hash, + /// The hash actually stored at `height - 1`. + actual_below: block::Hash, + }, + + /// The `height_by_hash` index is missing or wrong for the hash stored at + /// `height` (a forward bijection failure). + WrongHeightByHash { + /// The height whose hash failed the round-trip. + height: Height, + /// The hash stored at `height`. + hash: block::Hash, + /// The height the hash→height index reports, if any. + indexed: Option, + }, + + /// A row above the last coherent height (a stranded suffix: rows above a + /// gap, above a broken link, or above the linked chain's tip). + RowAboveLastCoherent { + /// The column family holding the stranded row. + cf: &'static str, + /// The stranded height. + height: Height, + }, + + /// A `height_by_hash` entry whose target height stores a different hash + /// (or no row at all): a stranded reverse-index row. + OrphanHeightByHash { + /// The hash of the stranded entry. + hash: block::Hash, + /// The height the entry points at. + points_at: Height, + }, +} + +/// The outcome of a startup repair: what was found and what was deleted. +/// +/// The fields are diagnostic payload: read by test assertions and rendered +/// through `Debug`. +#[allow(dead_code)] +#[derive(Debug)] +pub(crate) struct ZakuraStoreRepair { + /// The last height that passed every audit check, and its hash. + /// + /// Rows above it were truncated. `None` means the store had zakura rows + /// but no finalized tip to anchor them at, so all rows were removed. + pub last_coherent: Option<(Height, block::Hash)>, + + /// The number of rows deleted across all five column families. + pub deleted_rows: usize, + + /// Every violation found, in audit order. + pub violations: Vec, +} + +impl ZebraDb { + /// Audits the zakura header store invariants and repairs any violation by + /// deleting the offending rows in one atomic batch. + /// + /// Checks, over the whole zakura store (which only holds the header + /// frontier above the finalized tip, so this is cheap): + /// + /// - **bijection**: `hash_by_height` ↔ `height_by_hash` are mutually + /// inverse, and every stored header is the block its hash row names; + /// - **linkage**: rows chain by `previous_block_hash` from the finalized + /// tip hash upward; + /// - **tip integrity**: no rows in any zakura column family above the + /// last linked height, no gaps below it, and no zakura rows at + /// committed heights. + /// + /// On violation, emits the `state.zakura.header_store.incoherent` metric + /// and a warning, then truncates the zakura column families to the last + /// coherent height (and removes stale rows below the finalized tip and + /// orphaned reverse-index entries). Header sync re-downloads the + /// truncated suffix. A store that would fail the linkage-verified + /// consensus reads ([`StoreIncoherentError`](crate::error::StoreIncoherentError)) + /// is always repaired by this audit, because the audit checks are a + /// superset of the read-path checks over the same rows. + /// + /// Returns `Ok(None)` if the store is coherent (nothing is written), or + /// the repair summary after a successful repair write. + /// + /// Verified commitment roots at committed heights are never touched: the + /// roots column family is only scanned above the finalized tip. + pub(crate) fn audit_and_repair_zakura_header_store( + &self, + ) -> Result, rocksdb::Error> { + // Databases opened through `ZebraDb::new` without the zakura column + // families have no header store to audit. + let ( + Some(header_cf), + Some(hash_cf), + Some(height_by_hash_cf), + Some(body_size_cf), + Some(roots_cf), + ) = ( + self.db.cf_handle(ZAKURA_HEADER_BY_HEIGHT), + self.db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT), + self.db.cf_handle(ZAKURA_HEADER_HEIGHT_BY_HASH), + self.db.cf_handle(ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT), + self.db.cf_handle(COMMITMENT_ROOTS_BY_HEIGHT), + ) + else { + return Ok(None); + }; + + let finalized_tip = self.tip(); + + let headers: BTreeMap> = + self.db.zs_forward_range_iter(&header_cf, ..).collect(); + let hashes: BTreeMap = + self.db.zs_forward_range_iter(&hash_cf, ..).collect(); + let heights_by_hash: Vec<(block::Hash, Height)> = self + .db + .zs_forward_range_iter(&height_by_hash_cf, ..) + .collect(); + let body_size_heights: Vec = self + .db + .zs_forward_range_iter::<_, Height, AdvertisedBodySize, _>(&body_size_cf, ..) + .map(|(height, _)| height) + .collect(); + + // The roots column family also holds verified rows at committed + // heights (written by body commits, kept through pruning); those are + // not zakura frontier rows and are never audited or repaired. Only + // rows above the finalized tip are provisional header-sync data. + let provisional_roots_start = + finalized_tip.and_then(|(tip_height, _)| tip_height.next().ok()); + let provisional_root_heights: Vec = match (finalized_tip, provisional_roots_start) { + // The finalized tip is at the maximum height: no frontier can + // exist above it. + (Some(_), None) => Vec::new(), + (Some(_), Some(start)) => self + .db + .zs_forward_range_iter::<_, Height, CommitmentRootsByHeight, _>(&roots_cf, start..) + .map(|(height, _)| height) + .collect(), + // No finalized tip: every roots row is provisional. + (None, _) => self + .db + .zs_forward_range_iter::<_, Height, CommitmentRootsByHeight, _>(&roots_cf, ..) + .map(|(height, _)| height) + .collect(), + }; + + if headers.is_empty() + && hashes.is_empty() + && heights_by_hash.is_empty() + && body_size_heights.is_empty() + && provisional_root_heights.is_empty() + { + // Log the pass so operators can verify the audit ran. + tracing::info!( + ?finalized_tip, + frontier_rows = 0, + "zakura header store passed its coherence audit (empty frontier)" + ); + return Ok(None); + } + + let mut violations = Vec::new(); + + // Walk the linked chain upward from the finalized tip, verifying at + // each height that the hash and header rows agree, the header links + // to the row below, and the reverse index round-trips. The walk stops + // at the first missing hash row (the candidate chain tip) or the + // first violation; everything it passed is the coherent prefix. + let forward_index: HashMap = heights_by_hash.iter().copied().collect(); + let last_coherent = finalized_tip.map(|(anchor_height, anchor_hash)| { + let (mut last_height, mut last_hash) = (anchor_height, anchor_hash); + + while let Ok(height) = last_height.next() { + let Some(&hash) = hashes.get(&height) else { + break; + }; + let Some(header) = headers.get(&height) else { + violations.push(ZakuraStoreViolation::MissingHeaderRow { height }); + break; + }; + + let computed = block::Hash::from(&**header); + if computed != hash { + violations.push(ZakuraStoreViolation::HeaderHashMismatch { + height, + indexed: hash, + computed, + }); + break; + } + + if header.previous_block_hash != last_hash { + violations.push(ZakuraStoreViolation::BrokenLinkage { + height, + expected_parent: header.previous_block_hash, + actual_below: last_hash, + }); + break; + } + + let indexed = forward_index.get(&hash).copied(); + if indexed != Some(height) { + violations.push(ZakuraStoreViolation::WrongHeightByHash { + height, + hash, + indexed, + }); + break; + } + + (last_height, last_hash) = (height, hash); + } + + (last_height, last_hash) + }); + + // Committed heights have authoritative full-block rows: the consensus + // `hash_by_height` rows are contiguous from genesis to the finalized + // tip and retained by pruning, so `height <= finalized tip` is + // exactly `contains_height` — the same predicate as the writers' + // committed-height insert gate. (A body-presence predicate would miss + // stale rows at pruned heights.) + let committed = + |height: Height| finalized_tip.is_some_and(|(tip_height, _)| height <= tip_height); + // The coherent frontier window: strictly above the finalized tip, at + // or below the last height the linkage walk verified. + let in_window = |height: Height| { + !committed(height) + && last_coherent.is_some_and(|(last_height, _)| height <= last_height) + }; + + let mut batch = DiskWriteBatch::new(); + let mut deleted_rows = 0; + + // Height-keyed zakura rows survive only inside the coherent window. + let height_keyed: [(&'static str, &rocksdb::ColumnFamilyRef<'_>, Vec); 3] = [ + ( + ZAKURA_HEADER_BY_HEIGHT, + &header_cf, + headers.keys().copied().collect(), + ), + ( + ZAKURA_HEADER_HASH_BY_HEIGHT, + &hash_cf, + hashes.keys().copied().collect(), + ), + ( + ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT, + &body_size_cf, + body_size_heights, + ), + ]; + for (cf_name, cf, heights) in &height_keyed { + for &height in heights { + if in_window(height) { + continue; + } + + violations.push(if committed(height) { + ZakuraStoreViolation::StaleRowAtCommittedHeight { + cf: cf_name, + height, + } + } else { + ZakuraStoreViolation::RowAboveLastCoherent { + cf: cf_name, + height, + } + }); + + batch.zs_delete(*cf, height); + deleted_rows += 1; + } + } + + // Reverse-index entries survive only when their target height is + // inside the window and stores exactly their hash. This removes the + // reverse rows of every deleted hash row, plus entries orphaned by + // earlier overwrites that never cleaned up the displaced hash. + for &(hash, points_at) in &heights_by_hash { + let target_matches = hashes.get(&points_at) == Some(&hash); + if in_window(points_at) && target_matches { + continue; + } + + // Entries whose forward row is deleted above are repair fallout, + // not separate faults; only a live-but-disagreeing target is a + // distinct violation shape worth reporting. + if target_matches { + violations.push(if committed(points_at) { + ZakuraStoreViolation::StaleRowAtCommittedHeight { + cf: ZAKURA_HEADER_HEIGHT_BY_HASH, + height: points_at, + } + } else { + ZakuraStoreViolation::RowAboveLastCoherent { + cf: ZAKURA_HEADER_HEIGHT_BY_HASH, + height: points_at, + } + }); + } else { + violations.push(ZakuraStoreViolation::OrphanHeightByHash { hash, points_at }); + } + + batch.zs_delete(&height_by_hash_cf, hash); + deleted_rows += 1; + } + + // Provisional roots above the window are part of the stranded suffix. + for &height in &provisional_root_heights { + if in_window(height) { + continue; + } + + violations.push(ZakuraStoreViolation::RowAboveLastCoherent { + cf: COMMITMENT_ROOTS_BY_HEIGHT, + height, + }); + batch.zs_delete(&roots_cf, height); + deleted_rows += 1; + } + + if violations.is_empty() { + // Log the pass so operators can verify the audit ran on this boot + // (the fleet soak requires observing it finding nothing). + tracing::info!( + ?finalized_tip, + ?last_coherent, + frontier_rows = hashes.len(), + "zakura header store passed its coherence audit" + ); + return Ok(None); + } + + metrics::counter!("state.zakura.header_store.incoherent").increment(1); + tracing::warn!( + ?finalized_tip, + ?last_coherent, + deleted_rows, + violation_count = violations.len(), + first_violations = ?&violations[..violations.len().min(LOGGED_VIOLATIONS)], + "zakura header store failed its coherence audit; \ + truncating to the last coherent height so header sync re-downloads the rest" + ); + + self.db.write(batch)?; + + Ok(Some(ZakuraStoreRepair { + last_coherent, + deleted_rows, + violations, + })) + } +} diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence.rs index 81b87d2b4ae..b0adbdc2c23 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence.rs @@ -31,5 +31,8 @@ mod audit; mod fabricate; mod ops; mod oracle; +mod primitive; mod prop; +mod reads; mod scenarios; +mod startup_audit; diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/README.md b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/README.md index d23a5205a0b..cf788dcf19b 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/README.md +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/README.md @@ -34,65 +34,65 @@ write instead. | `audit.rs` | The invariant audit: A1 (bijection, both directions), A2 (linkage walk upward from the finalized tip over the merged header view), A3 (tip integrity, gaps, frontier overlay, aux-row backing), and A4 (the on-disk chain equals the model's expected canonical chain). Also `dump_store`, a comparable snapshot of all five column families used to assert that rejected commits are side-effect free and that reopens preserve the store byte-for-byte. | | `oracle.rs` | An in-memory model of the store's _specified_ behavior: a single linked canonical chain, best-cumulative-work selection with strict improvement, total suffix replacement above the first conflicting height, and a sequential body tip. It predicts whether each op must be accepted or rejected. | | `ops.rs` | The op alphabet, each op mapped to one real production write-batch shape: `CommitHeaderRange` → `prepare_header_range_batch_with_roots`; `CommitBody` / `Finalize` → `prepare_block_header_and_transaction_data_batch` plus the finalization roots delete (which runs the release path internally); `Seed` → `seed_zakura_header_from_committed_block` (the non-finalized best-chain commit hook); `Reopen` → shutdown and reopen of the persistent store. The `Harness` executes ops, cross-checks the oracle's prediction against the store's response, and audits after every mutation; failures come back as a transcribable `FailureReport` (the executed op prefix plus every violation found). | -| `scenarios.rs` | Scripted production event shapes (s01–s10): simple reorgs, lower-work rejections and their later reversal, split-range and walk-back deliveries, body commits racing header reorgs, reorgs to a lower height, double reorgs at one fork point, activity across the difficulty-adjustment window edge, restarts at every boundary, and seed/range interplay. Also holds the `corruption_repro_*` tests below. | -| `prop.rs` | Discovery proptests: random op sequences over the fixed universe, shrunk to minimal counterexamples on any audit failure. Both are `#[ignore]`d because the store has known bugs today; see "Running". | - -## Known corruption bugs pinned by this suite - -Each bug is pinned by a pair of tests: `corruption_repro_` **passes -today** and deterministically demonstrates the violation, and -`_upholds_invariants` is `#[ignore]`d and asserts the correct behavior. -When the write path is fixed, the repro fails loudly (forcing re-triage) and -the twin gets un-ignored as the permanent regression test. - -1. **Unlinked-anchor commit** (`corruption_repro_unlinked_anchor_commit`). - `prepare_header_range_batch_with_roots` never checks that - `headers[0].previous_block_hash == anchor`, nor any intra-range linkage. A - range anchored at a same-height hash of a different branch passes - difficulty validation and commits a suffix that does not link to the row - below it — an on-disk I2 violation reachable from a single untrusted peer - response. +| `scenarios.rs` | Scripted production event shapes (s01–s11): simple reorgs, lower-work rejections and their later reversal, split-range and walk-back deliveries, body commits racing header reorgs, reorgs to a lower height, double reorgs at one fork point, activity across the difficulty-adjustment window edge, restarts at every boundary, seed/range interplay, and refused-seed convergence. Also holds the `*_upholds_invariants` regression gates below. | +| `prop.rs` | The permanent random sweep: random op sequences over the fixed universe, shrunk to minimal counterexamples on any audit failure. | +| `reads.rs` | Read-path coherence (Pillar 2): hand-corrupts the column families and asserts `recent_header_context` / the anchor round-trip report `StoreIncoherentError` (`HeaderHashMismatch`, `BrokenLinkage`, `Gap`, `BijectionMismatch`) instead of feeding stale rows into difficulty validation, and that the range writer's `StoreIncoherent` rejection is side-effect free. | + +## Fixed corruption bugs gated by this suite + +Phase 1 of this suite proved three write-path corruption bug classes, each +pinned by a `corruption_repro_*` test that deterministically demonstrated the +violation. The write-path fixes (`REORG_PLAN.md` Phase 1.5) closed all three; +the repro tests were removed with the fixes, and their +`_upholds_invariants` twins now assert the fixed behavior over the same +op sequences as permanent regression gates: + +1. **Unlinked-anchor commit** (`unlinked_anchor_commit_upholds_invariants`). + `prepare_header_range_batch_with_roots` used to accept ranges without + checking that `headers[0].previous_block_hash == anchor` or any intra-range + linkage, so a range anchored at a same-height hash of a different branch + could pass difficulty validation and commit a suffix that did not link to + the row below it — an on-disk I2 violation reachable from a single + untrusted peer response. The writer now rejects such ranges with + `CommitHeaderRangeError::UnlinkedRange`. 2. **Re-delivery over committed bodies** - (`corruption_repro_redelivery_over_bodies`). The range insert loop gates - only its _roots_ write on `contains_body_at_height`; the - header/hash/height/body-size writes are unconditional. A header range - re-delivered over heights whose bodies were committed in the meantime - re-inserts zakura rows below the body tip, and nothing ever trims them - again (the release trim already ran at body-commit time) — a permanent I3 - frontier-overlay violation. -3. **Unlinked seed** (`corruption_repro_seed_above_gap`, - `corruption_repro_seed_fork_switch`; found by the proptest and shrunk to a - single op). `prepare_zakura_header_from_committed_block` writes its row - with no linkage or anchor precondition. Seeds fire only at non-finalized + (`redelivery_over_bodies_upholds_invariants`). The range insert loop used + to gate only its _roots_ write on `contains_body_at_height`, so a header + range re-delivered over heights whose bodies were committed in the + meantime re-inserted zakura rows below the body tip that nothing ever + trims again (the release trim already ran at body-commit time) — a + permanent I3 frontier-overlay violation. The gate now covers every zakura + row write: heights that already have a committed block are skipped + entirely (checked via `contains_height`, so it also holds for pruned + heights whose bodies are gone but whose authoritative rows remain). +3. **Unlinked seed** (`seed_above_gap_upholds_invariants`, + `seed_fork_switch_upholds_invariants`; found by the proptest and shrunk to + a single op). `prepare_zakura_header_from_committed_block` used to write + its row with no linkage precondition. Seeds fire only at non-finalized best-_tip_ commits, so any best-tip jump (a fork switch between non-finalized chains, or a restart that restores the non-finalized backup) - seeds a height whose parent row is missing or belongs to another branch: - a gap or broken link on disk, and a generator of poisoned - difficulty-adjustment windows. - -A discovery sweep of 2048 random sequences with these three shapes excluded -found no further violation class at that depth. + seeded a height whose parent row was missing or belonged to another + branch: a gap or broken link on disk, and a generator of poisoned + difficulty-adjustment windows. The seed path now refuses a seed that does + not link to the stored row below it as a silent no-op — the header store + briefly lags the non-finalized chain, and header-range sync converges it + (`s11_refused_seed_converges_via_range_delivery`). ## Running ```bash -# The whole suite (scripted scenarios, audits, corruption repros): +# The whole suite (scripted scenarios, audits, regression gates, random sweep): cargo test -p zebra-state --lib header_store_coherence -# Discovery sweep beyond the known bugs (any failure = a NEW violation class): +# The random sweep at discovery depth (any failure = a NEW violation class): PROPTEST_CASES=4096 cargo test -p zebra-state --lib \ - header_store_coherence::prop::prop_discovery -- --ignored - -# The full random sweep (fails today on the known shapes; becomes a permanent -# regression gate once the write paths are fixed): -cargo test -p zebra-state --lib \ - header_store_coherence::prop::prop_random_sequences -- --ignored + header_store_coherence::prop ``` Shrunk proptest counterexamples should be transcribed into `scenarios.rs` as -hardcoded `corruption_repro_*`/`#[ignore]` pairs (seed-independent pinning); -the file under `zebra-state/proptest-regressions/` pins the seeds as a -backstop and is checked in. +hardcoded scenarios (seed-independent pinning); the file under +`zebra-state/proptest-regressions/` pins the seeds as a backstop and is +checked in. ## Scope diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/audit.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/audit.rs index 05a8226d703..54b6f78bba8 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/audit.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/audit.rs @@ -274,7 +274,10 @@ pub(crate) fn audit_store(state: &ZebraDb) -> Vec { } // A3: body-size rows require a backing zakura header row; roots rows - // require a zakura header row or a committed body. + // require a zakura header row or a committed height. The committed + // predicate is `contains_height` (the consensus hash row, which pruning + // retains), not body presence: verified roots rows at committed heights + // survive pruning and are legitimate, body or no body. for &height in dump.body_sizes.keys() { if !dump.hashes.contains_key(&height) { violations.push(Violation::AuxRowWithoutHeader { @@ -284,7 +287,7 @@ pub(crate) fn audit_store(state: &ZebraDb) -> Vec { } } for &height in dump.roots.keys() { - if !dump.hashes.contains_key(&height) && !state.contains_body_at_height(height) { + if !dump.hashes.contains_key(&height) && !state.contains_height(height) { violations.push(Violation::AuxRowWithoutHeader { cf: COMMITMENT_ROOTS_BY_HEIGHT, height, diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/ops.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/ops.rs index 05480931678..a3a041b1c13 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/ops.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/ops.rs @@ -10,6 +10,8 @@ //! (the non-finalized best-chain commit hook); //! - [`Op::Finalize`] → sequential body commits along the expected canonical //! chain; +//! - [`Op::Prune`] → `DiskWriteBatch::prepare_prune_batch` (online pruning of +//! raw transactions below the body tip; no chain-membership change); //! - [`Op::Reopen`] → close and reopen the database (restart survival). //! //! After every op the harness cross-checks the oracle's prediction against the @@ -80,6 +82,11 @@ pub(crate) enum Op { Finalize { count: usize, }, + /// Prunes raw transactions from the last pruning marker up to (but not + /// including) `min(until, body tip)`, through the production prune batch. + Prune { + until: u32, + }, Reopen, } @@ -109,13 +116,16 @@ impl OpOutcome { } /// Why a sequence failed: the executed op prefix (last op is the failing one) -/// plus everything found after it. +/// plus everything found after it. All fields are diagnostic payloads rendered +/// through `Debug` in test failure output. #[derive(Debug)] pub(crate) struct FailureReport { - /// The transcribable op sequence, rendered through `Debug` in reports. + /// The transcribable op sequence. #[allow(dead_code)] pub executed: Vec, + #[allow(dead_code)] pub violations: Vec, + #[allow(dead_code)] pub mismatches: Vec, } @@ -132,10 +142,6 @@ pub(crate) struct Harness { config: Config, state: Option, executed: Vec, - /// Skip the op shapes that trigger the *known* corruption bugs (the - /// `corruption_repro_*` scenarios), so proptest discovery can search for - /// new violation classes instead of rediscovering the known ones. - avoid_known_corruptions: bool, _tempdir: tempfile::TempDir, } @@ -157,19 +163,10 @@ impl Harness { config, state: Some(state), executed: Vec::new(), - avoid_known_corruptions: false, _tempdir: tempdir, } } - /// A harness that skips the known corruption shapes (see - /// [`Harness::avoid_known_corruptions`]). - pub fn new_avoiding_known_corruptions() -> Self { - let mut harness = Self::new(); - harness.avoid_known_corruptions = true; - harness - } - pub fn state(&self) -> &ZebraDb { self.state .as_ref() @@ -242,32 +239,6 @@ impl Harness { let Some(range) = self.resolve_range(*source, *offset, *len, *anchor) else { return self.finish(OpOutcome::Skipped("range out of bounds"), mismatches); }; - if self.avoid_known_corruptions { - // Known bug 1: unlinked-anchor ranges are accepted instead - // of rejected (corruption_repro_unlinked_anchor_commit). - if matches!( - self.oracle.predict_header_range(&range), - Prediction::Reject(super::oracle::RejectKind::Malformed) - ) { - return self.finish( - OpOutcome::Skipped("known corruption shape: unlinked anchor"), - mismatches, - ); - } - // Known bug 2: accepted re-deliveries touching body-backed - // heights strand zakura rows below the body tip - // (corruption_repro_redelivery_over_bodies). - if range - .rows - .iter() - .any(|row| row.height <= self.oracle.body_tip()) - { - return self.finish( - OpOutcome::Skipped("known corruption shape: re-delivery over bodies"), - mismatches, - ); - } - } match self.oracle.predict_header_range(&range) { Prediction::Skip(reason) => { return self.finish(OpOutcome::Skipped(reason), mismatches) @@ -337,27 +308,33 @@ impl Harness { let Some(fab) = rows.get(*index).cloned() else { return self.finish(OpOutcome::Skipped("seed index out of bounds"), mismatches); }; - // Known bug 3: a seed whose parent is not the committed row - // below it writes an unlinkable row - // (corruption_repro_seed_above_gap / _seed_fork_switch). - if self.avoid_known_corruptions && !self.oracle.seed_is_parent_linked(&fab) { - return self.finish( - OpOutcome::Skipped("known corruption shape: unlinked seed"), - mismatches, - ); - } match self.oracle.predict_seed(&fab) { Prediction::Skip(reason) => { return self.finish(OpOutcome::Skipped(reason), mismatches) } _ => { + // A seed whose parent is not the stored row below it is + // refused by the store as a silent no-op (the header + // store briefly lags the non-finalized chain until a + // linkage-checked header range converges it), so the + // call must succeed without mutating anything. + let parent_linked = self.oracle.seed_is_parent_linked(&fab); + let dump_before = (!parent_linked).then(|| dump_store(self.state())); let block = fabricate_body(&fab); match self .state() .seed_zakura_header_from_committed_block(fab.height, &block) { Ok(()) => { - self.oracle.apply_seed(&fab); + if parent_linked { + self.oracle.apply_seed(&fab); + } else if dump_store(self.state()) + != dump_before.expect("dumped before an unlinked seed") + { + mismatches.push(format!( + "refused unlinked seed mutated the store: {op:?}" + )); + } OpOutcome::Accepted } Err(error) => { @@ -406,6 +383,50 @@ impl Harness { } } + Op::Prune { until } => { + // Pruning applies below the finalized body tip and never + // re-prunes below the existing marker (the marker is + // monotone in production). + let Some(tip) = self.state().finalized_tip_height() else { + return self.finish(OpOutcome::Skipped("no finalized tip"), mismatches); + }; + let from = self + .state() + .lowest_retained_height() + .unwrap_or(block::Height(1)); + let until = block::Height((*until).min(tip.0)); + if until <= from { + return self.finish(OpOutcome::Skipped("empty prune range"), mismatches); + } + + let dump_before = dump_store(self.state()); + let mut batch = DiskWriteBatch::new(); + batch.prepare_prune_batch(self.state(), from, until); + self.state() + .write_batch(batch) + .expect("prune batch writes successfully"); + + // Pruning removes raw transactions and advances the marker; + // chain membership and the zakura header store must be + // untouched (the oracle's predictions are unchanged). + if dump_store(self.state()) != dump_before { + mismatches.push(format!("pruning mutated the zakura header store: {op:?}")); + } + for height in from.0..until.0 { + let height = block::Height(height); + if self.state().contains_body_at_height(height) { + mismatches + .push(format!("pruned height {height:?} still has a body: {op:?}")); + } + if !self.state().contains_height(height) { + mismatches.push(format!( + "pruning removed the consensus hash row at {height:?}: {op:?}" + )); + } + } + OpOutcome::Accepted + } + Op::Reopen => { let dump_before = dump_store(self.state()); let mut state = self.state.take().expect("state is present before Reopen"); diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/oracle.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/oracle.rs index d28872f3055..574b695afc6 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/oracle.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/oracle.rs @@ -40,7 +40,7 @@ pub(crate) enum RejectKind { /// The conflicting suffix does not carry strictly more work. LowerWork, /// The range is malformed (wrong anchor linkage or heights); the store is - /// expected to reject it through contextual validation. + /// expected to reject it through its linkage check (`UnlinkedRange`). Malformed, } @@ -248,8 +248,10 @@ impl Oracle { } /// Whether a seeded block's parent is the expected canonical row below it. - /// Seeds that are not parent-linked are the known seed-path corruption - /// shape (they write a row the store cannot link). + /// The store refuses seeds that are not parent-linked as silent no-ops + /// (writing them would strand a row the chain walk cannot reach); the + /// harness uses this to decide whether a successful seed call must have + /// mutated the store or left it untouched. pub fn seed_is_parent_linked(&self, fab: &FabHeader) -> bool { let parent_height = Height(fab.height.0 - 1); if parent_height == Height(0) { diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/primitive.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/primitive.rs new file mode 100644 index 00000000000..fc9c702ae17 --- /dev/null +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/primitive.rs @@ -0,0 +1,282 @@ +//! Contract tests for `set_canonical_suffix`, the single owner of zakura +//! header-store chain membership (REORG_PLAN Pillar 1). +//! +//! The coherence harness exercises the primitive through the production +//! writers (range, seed, release); these tests pin the primitive's own +//! contract directly: preconditions reject without staging anything, +//! deletion is total-above-fork across all five column families (including +//! rows stranded above hand-made gaps), an empty replacement truncates, and +//! a pure append stages no deletions (so the post-reorg audit hook does not +//! fire for it). + +use zebra_chain::block::Height; + +use super::super::super::CanonicalHeaderRow; +use super::super::super::{ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT, ZAKURA_HEADER_HASH_BY_HEIGHT}; +use super::super::common::{ + commit_header_range, state_with_genesis_config, write_full_block_header_and_transactions, +}; +use super::{ + audit::{audit_store, dump_store}, + fabricate::{fabricate_body, FabHeader, Universe, BRANCH_A, FORK_HEIGHT}, +}; +use crate::{ + error::{CommitHeaderRangeError, StoreIncoherentError}, + service::finalized_state::{ + disk_db::{DiskWriteBatch, WriteDisk}, + disk_format::block::TransactionLocation, + RawBytes, ZebraDb, + }, + Config, +}; + +fn trunk_state(universe: &Universe) -> ZebraDb { + let state = state_with_genesis_config( + &universe.network, + universe.genesis.clone(), + Config::ephemeral(), + ); + let trunk_headers: Vec<_> = universe.trunk[..FORK_HEIGHT as usize] + .iter() + .map(|fab| fab.header.clone()) + .collect(); + commit_header_range(&state, universe.genesis.hash(), &trunk_headers); + state +} + +fn rows_of(fabs: &[FabHeader]) -> Vec { + fabs.iter() + .map(|fab| CanonicalHeaderRow { + header: fab.header.clone(), + advertised_body_size: None, + roots: None, + }) + .collect() +} + +fn assert_clean(state: &ZebraDb) { + let violations = audit_store(state); + assert!( + violations.is_empty(), + "unexpected violations: {violations:?}" + ); +} + +/// New rows that do not link to the fork row are rejected before anything is +/// staged. +#[test] +fn rejects_unlinked_new_rows_without_staging() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe); + let original = dump_store(&state); + + // Branch-A rows link at the fork height (50), not at 30. + let rows = rows_of(&universe.branches[BRANCH_A].headers[..3]); + let mut batch = DiskWriteBatch::new(); + let error = batch + .set_canonical_suffix(&state, (Height(30), universe.trunk_at(30).hash), &rows) + .expect_err("rows do not link to trunk@30"); + assert!( + matches!( + error, + CommitHeaderRangeError::UnlinkedRange { height, .. } if height == Height(31) + ), + "expected UnlinkedRange at 31, got {error:?}" + ); + assert_eq!(batch.zakura_suffix_replaced_rows(), 0); + + state.write_batch(batch).expect("empty batch writes"); + assert_eq!(dump_store(&state), original, "a rejection stages nothing"); +} + +/// A fork hash that is not the stored row at the fork height is rejected: +/// the caller's fork decision must describe the store being mutated. +#[test] +fn rejects_stale_fork_row() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe); + let original = dump_store(&state); + + // These rows link to branch-A's fourth row, which is not stored at 30. + let foreign_fork = &universe.branches[BRANCH_A].headers[3]; + let rows = rows_of(&universe.branches[BRANCH_A].headers[4..6]); + let mut batch = DiskWriteBatch::new(); + let error = batch + .set_canonical_suffix(&state, (Height(30), foreign_fork.hash), &rows) + .expect_err("the stored row at 30 is the trunk's"); + assert!( + matches!( + error, + CommitHeaderRangeError::StoreIncoherent(StoreIncoherentError::BijectionMismatch { + height, + .. + }) if height == Height(30) + ), + "expected BijectionMismatch at the fork row, got {error:?}" + ); + assert_eq!(batch.zakura_suffix_replaced_rows(), 0); + assert_eq!(dump_store(&state), original); +} + +/// A fork below the finalized tip is rejected: committed heights are +/// immutable through the primitive. +#[test] +fn rejects_fork_below_finalized_tip() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe); + + // Commit the body of trunk height 1: the finalized tip moves to 1. + write_full_block_header_and_transactions(&state, fabricate_body(&universe.trunk[0])); + assert_eq!(state.finalized_tip_height(), Some(Height(1))); + + let rows = rows_of(&universe.trunk[..2]); + let mut batch = DiskWriteBatch::new(); + let error = batch + .set_canonical_suffix(&state, (Height(0), universe.genesis.hash()), &rows) + .expect_err("the fork is below the finalized tip"); + assert!( + matches!( + error, + CommitHeaderRangeError::ImmutableConflict { height } if height == Height(1) + ), + "expected ImmutableConflict, got {error:?}" + ); + assert_eq!(batch.zakura_suffix_replaced_rows(), 0); +} + +/// The committed-body safety interlock: a body row above the fork aborts the +/// replacement before anything is staged, even when the finalized-tip bound +/// cannot see it. +#[test] +fn body_above_fork_aborts_loudly() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe); + + // Hand-plant a body marker above the finalized tip — a state the + // orchestration must never produce. + let tx_by_loc = state.db.cf_handle("tx_by_loc").unwrap(); + let mut batch = DiskWriteBatch::new(); + batch.zs_insert( + &tx_by_loc, + TransactionLocation::min_for_height(Height(45)), + RawBytes::new_raw_bytes(vec![0u8]), + ); + state.db.write(batch).expect("raw insert writes"); + + let rows = rows_of(&universe.trunk[30..35]); + let mut batch = DiskWriteBatch::new(); + let error = batch + .set_canonical_suffix(&state, (Height(30), universe.trunk_at(30).hash), &rows) + .expect_err("a body above the fork must abort the replacement"); + assert!( + matches!( + error, + CommitHeaderRangeError::ConflictingFullBlockHeader { height } if height == Height(45) + ), + "expected ConflictingFullBlockHeader at 45, got {error:?}" + ); + assert_eq!(batch.zakura_suffix_replaced_rows(), 0); +} + +/// Deletion is total-above-fork: rows stranded above a hand-made gap and +/// stray rows in single column families are deleted along with the linked +/// suffix, in all five column families. +#[test] +fn truncation_is_total_above_fork_including_strays() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe); + + // A gap at 40 strands rows 41..=50; a stray body-size row sits at 60, + // above the zakura tip. + let hash_cf = state.db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); + let body_size_cf = state + .db + .cf_handle(ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT) + .unwrap(); + let mut batch = DiskWriteBatch::new(); + batch.zs_delete(&hash_cf, Height(40)); + batch.zs_insert( + &body_size_cf, + Height(60), + super::super::super::test_body_size(7), + ); + state.db.write(batch).expect("raw writes succeed"); + + let rows = rows_of(&universe.trunk[30..35]); + let mut batch = DiskWriteBatch::new(); + batch + .set_canonical_suffix(&state, (Height(30), universe.trunk_at(30).hash), &rows) + .expect("the replacement is well-formed"); + assert!(batch.zakura_suffix_replaced_rows() > 0); + state.write_batch(batch).expect("replacement batch writes"); + + assert_eq!( + state.best_header_tip(), + Some((Height(35), universe.trunk_at(35).hash)) + ); + let dump = dump_store(&state); + assert_eq!(dump.hashes.keys().next_back(), Some(&Height(35))); + assert!(!dump.body_sizes.contains_key(&Height(60))); + assert_clean(&state); +} + +/// An empty replacement truncates the store to the fork point. +#[test] +fn empty_replacement_truncates_to_fork() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe); + + let mut batch = DiskWriteBatch::new(); + batch + .set_canonical_suffix(&state, (Height(40), universe.trunk_at(40).hash), &[]) + .expect("an empty replacement is a truncation"); + assert!(batch.zakura_suffix_replaced_rows() > 0); + state.write_batch(batch).expect("truncation batch writes"); + + assert_eq!( + state.best_header_tip(), + Some((Height(40), universe.trunk_at(40).hash)) + ); + assert_clean(&state); +} + +/// A pure append deletes nothing, so it does not count as a reorg (the +/// post-reorg audit hook stays quiet for the steady-state extension path). +#[test] +fn pure_append_replaces_nothing() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe); + + // Branch A forks at the trunk tip, so its rows extend the stored chain. + let rows = rows_of(&universe.branches[BRANCH_A].headers[..3]); + let mut batch = DiskWriteBatch::new(); + batch + .set_canonical_suffix( + &state, + (Height(FORK_HEIGHT), universe.trunk_at(FORK_HEIGHT).hash), + &rows, + ) + .expect("an extension at the tip is well-formed"); + assert_eq!( + batch.zakura_suffix_replaced_rows(), + 0, + "a pure append is not a reorg" + ); + state.write_batch(batch).expect("append batch writes"); + + assert_eq!( + state.best_header_tip(), + Some(( + Height(FORK_HEIGHT + 3), + universe.branches[BRANCH_A].headers[2].hash + )) + ); + assert_clean(&state); +} diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/prop.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/prop.rs index e867768b56e..8b39d126982 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/prop.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/prop.rs @@ -1,22 +1,17 @@ //! Discovery proptest: random op sequences over the fixed universe, shrunk on //! any audit failure to a minimal, transcribable counterexample. //! -//! Both tests are `#[ignore]`d: the store has known corruption bugs today (the -//! `corruption_repro_*` scenarios), so an always-on random sweep cannot be a CI -//! gate yet. Run discovery manually: +//! The sweep runs 64 cases in CI by default; widen it manually with: //! //! ```text //! PROPTEST_CASES=4096 cargo test -p zebra-state --lib \ -//! header_store_coherence::prop -- --ignored --nocapture +//! header_store_coherence::prop -- --nocapture //! ``` //! //! Every shrunk counterexample must be transcribed into `scenarios.rs` as a -//! hardcoded `corruption_repro_*` / `#[ignore]`d-invariant pair (the primary, -//! seed-independent pinning mechanism); the proptest regression file under -//! `zebra-state/proptest-regressions/` pins the seed as a backstop. -//! -//! When the write-path fixes land, un-ignore -//! `prop_random_sequences_uphold_invariants` as a permanent regression sweep. +//! hardcoded scenario (the primary, seed-independent pinning mechanism); the +//! proptest regression file under `zebra-state/proptest-regressions/` pins +//! the seed as a backstop. use std::env; @@ -63,6 +58,7 @@ fn op_strategy() -> impl Strategy { .prop_map(|(source, index)| Op::CommitBody { source, index }), 2 => (source_strategy(), 0..40usize).prop_map(|(source, index)| Op::Seed { source, index }), 3 => (1..8usize).prop_map(|count| Op::Finalize { count }), + 1 => (2..=TRUNK_LEN as u32).prop_map(|until| Op::Prune { until }), 1 => Just(Op::Reopen), ] } @@ -81,10 +77,8 @@ fn proptest_cases() -> u32 { proptest! { #![proptest_config(ProptestConfig::with_cases(proptest_cases()))] - /// The full invariant sweep. Fails today on the known corruption shapes; - /// becomes the permanent regression gate once the write paths are fixed. + /// The permanent invariant sweep: any failure is a store-corruption bug. #[test] - #[ignore = "known zakura header-store corruption: un-ignore with the write-path fix"] fn prop_random_sequences_uphold_invariants(ops in ops_strategy()) { let _init_guard = zebra_test::init(); let mut harness = Harness::new(); @@ -92,17 +86,4 @@ proptest! { prop_assert!(false, "store invariants violated:\n{report:#?}"); } } - - /// Discovery beyond the known bugs: the harness skips the known - /// corruption shapes, so any failure here is a *new* violation class. - /// Run manually with a large PROPTEST_CASES while the known bugs are open. - #[test] - #[ignore = "discovery sweep: run manually"] - fn prop_discovery_avoiding_known_corruptions(ops in ops_strategy()) { - let _init_guard = zebra_test::init(); - let mut harness = Harness::new_avoiding_known_corruptions(); - if let Err(report) = harness.run_all(&ops) { - prop_assert!(false, "new store-corruption class found:\n{report:#?}"); - } - } } diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/reads.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/reads.rs new file mode 100644 index 00000000000..08196889c11 --- /dev/null +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/reads.rs @@ -0,0 +1,281 @@ +//! Read-path coherence: consensus readers verify store invariants as they +//! walk, surfacing corruption as an explicit [`StoreIncoherentError`] instead +//! of feeding stale rows into difficulty validation (REORG_PLAN Pillar 2). +//! +//! The write path refuses to create incoherent stores (the Phase-1.5 guards), +//! so these tests corrupt the column families directly — the same hand-made +//! corruption technique as the audit's own tests — and assert that: +//! +//! - `recent_header_context` returns `BrokenLinkage`/`Gap` instead of a +//! poisoned or silently shortened difficulty window, and +//! - the range writer rejects with `StoreIncoherent` (a local storage fault, +//! never a peer-attributed validation failure) and leaves the store +//! untouched, and +//! - the anchor round-trip distinguishes a bijection violation in our own +//! indexes (`BijectionMismatch`) from a genuinely unknown anchor. + +use std::sync::Arc; + +use zebra_chain::block::{self, Height}; + +use super::super::super::{ZAKURA_HEADER_BY_HEIGHT, ZAKURA_HEADER_HASH_BY_HEIGHT}; +use super::super::common::{commit_header_range, state_with_genesis_config}; +use super::{ + audit::dump_store, + fabricate::{Universe, BRANCH_A, FORK_HEIGHT}, +}; +use crate::{ + error::{CommitHeaderRangeError, StoreIncoherentError}, + service::finalized_state::{ + disk_db::{DiskWriteBatch, WriteDisk}, + ZebraDb, + }, + Config, +}; + +/// A store holding genesis plus the trunk up to the fork height, built through +/// the production write path. +fn trunk_state(universe: &Universe) -> ZebraDb { + let state = state_with_genesis_config( + &universe.network, + universe.genesis.clone(), + Config::ephemeral(), + ); + let trunk_headers: Vec<_> = universe.trunk[..FORK_HEIGHT as usize] + .iter() + .map(|fab| fab.header.clone()) + .collect(); + commit_header_range(&state, universe.genesis.hash(), &trunk_headers); + state +} + +/// Re-delivers a slice of trunk headers through the production range writer. +fn redeliver_trunk( + state: &ZebraDb, + universe: &Universe, + anchor_height: u32, + len: usize, +) -> Result { + let anchor = universe.trunk_at(anchor_height).hash; + let headers: Vec> = universe.trunk + [anchor_height as usize..anchor_height as usize + len] + .iter() + .map(|fab| fab.header.clone()) + .collect(); + let body_sizes = vec![0; headers.len()]; + let mut batch = DiskWriteBatch::new(); + let result = batch.prepare_header_range_batch(state, anchor, &headers, &body_sizes); + if result.is_ok() { + state + .write_batch(batch) + .expect("header range batch writes successfully"); + } + result +} + +/// A coherent store yields a full-span window mid-chain and a legitimately +/// short window near genesis. +#[test] +fn coherent_walks_return_ok_contexts() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe); + + let full = state + .recent_header_context(Height(40)) + .expect("coherent store walks cleanly"); + assert_eq!( + full.len(), + crate::service::check::difficulty::POW_ADJUSTMENT_BLOCK_SPAN + ); + + // Heights 5..=0 inclusive: six rows, then the walk stops at genesis. + let short = state + .recent_header_context(Height(5)) + .expect("a short walk ending at genesis is legitimate"); + assert_eq!(short.len(), 6); + + // A missing anchor row is not incoherence; the caller decides what an + // unknown anchor means. + let missing = state + .recent_header_context(Height(FORK_HEIGHT + 10)) + .expect("a missing starting row is not a violation"); + assert!(missing.is_empty()); +} + +/// A header row that is not the block its hash row names (the incident-shaped +/// poison: a stale row's (threshold, time) feeding the DAA window): the walk +/// reports the divergence instead of consuming the row, and the range writer +/// maps it to a side-effect-free `StoreIncoherent` rejection. +#[test] +fn foreign_header_row_is_reported_not_validated() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe); + + // Overwrite the header row at height 20 with a branch-A header while the + // hash rows keep claiming the trunk. + let header_cf = state.db.cf_handle(ZAKURA_HEADER_BY_HEIGHT).unwrap(); + let foreign_header = universe.branches[BRANCH_A].headers[3].header.clone(); + let mut batch = DiskWriteBatch::new(); + batch.zs_insert(&header_cf, Height(20), foreign_header); + state.db.write(batch).expect("raw insert writes"); + + let error = state + .recent_header_context(Height(30)) + .expect_err("the walk crosses the corrupted row"); + assert!( + matches!( + error, + StoreIncoherentError::HeaderHashMismatch { height, .. } if height == Height(20) + ), + "expected HeaderHashMismatch at the corrupted row, got {error:?}" + ); + + // The writer surfaces the same fault as a local rejection: the range is + // not blamed (no contextual-validation error) and nothing is written. + let dump_before = dump_store(&state); + let error = redeliver_trunk(&state, &universe, 30, 5) + .expect_err("validation context crosses the corrupted row"); + assert!( + matches!( + error, + CommitHeaderRangeError::StoreIncoherent( + StoreIncoherentError::HeaderHashMismatch { .. } + ) + ), + "expected StoreIncoherent(HeaderHashMismatch), got {error:?}" + ); + assert_eq!( + dump_store(&state), + dump_before, + "a StoreIncoherent rejection must be side-effect free" + ); +} + +/// A self-consistent foreign row (its header *is* the block its hash row +/// names, but it belongs to another branch): the row above it no longer links +/// down, and the walk reports the broken link from above. +#[test] +fn broken_linkage_is_reported_not_validated() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe); + + // Replace both the header and hash rows at height 20 with branch-A's + // fourth row: internally consistent, but trunk@21 does not link to it. + let foreign = &universe.branches[BRANCH_A].headers[3]; + let header_cf = state.db.cf_handle(ZAKURA_HEADER_BY_HEIGHT).unwrap(); + let hash_cf = state.db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); + let mut batch = DiskWriteBatch::new(); + batch.zs_insert(&header_cf, Height(20), foreign.header.clone()); + batch.zs_insert(&hash_cf, Height(20), foreign.hash); + state.db.write(batch).expect("raw insert writes"); + + let error = state + .recent_header_context(Height(30)) + .expect_err("the walk crosses the corrupted row"); + assert!( + matches!( + error, + StoreIncoherentError::BrokenLinkage { height, actual_below, .. } + if height == Height(21) && actual_below == foreign.hash + ), + "expected BrokenLinkage above the corrupted row, got {error:?}" + ); + + let error = redeliver_trunk(&state, &universe, 30, 5) + .expect_err("validation context crosses the corrupted row"); + assert!( + matches!( + error, + CommitHeaderRangeError::StoreIncoherent(StoreIncoherentError::BrokenLinkage { .. }) + ), + "expected StoreIncoherent(BrokenLinkage), got {error:?}" + ); +} + +/// A gap mid-window: a missing row below a stored one is incoherence, not the +/// end of history — a silently shortened window would shift the difficulty +/// adjustment instead of failing. +#[test] +fn gap_is_reported_not_shortened() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe); + + let header_cf = state.db.cf_handle(ZAKURA_HEADER_BY_HEIGHT).unwrap(); + let hash_cf = state.db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); + let mut batch = DiskWriteBatch::new(); + batch.zs_delete(&header_cf, Height(15)); + batch.zs_delete(&hash_cf, Height(15)); + state.db.write(batch).expect("raw delete writes"); + + let error = state + .recent_header_context(Height(25)) + .expect_err("the walk crosses the gap"); + assert!( + matches!( + error, + StoreIncoherentError::Gap { height, missing } + if height == Height(16) && missing == Height(15) + ), + "expected Gap below height 16, got {error:?}" + ); + + let error = + redeliver_trunk(&state, &universe, 25, 5).expect_err("validation context crosses the gap"); + assert!( + matches!( + error, + CommitHeaderRangeError::StoreIncoherent(StoreIncoherentError::Gap { .. }) + ), + "expected StoreIncoherent(Gap), got {error:?}" + ); +} + +/// A hash→height entry whose height→hash row disagrees is a bijection +/// violation in our own indexes: the anchor round-trip reports it as +/// `StoreIncoherent`, distinct from a genuinely unknown anchor. +#[test] +fn anchor_bijection_violation_is_store_incoherent_not_unknown() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe); + + // The hash row at height 30 now names a different block, while + // height_by_hash still maps the trunk hash to 30. + let hash_cf = state.db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); + let stray_hash = universe.branches[BRANCH_A].headers[0].hash; + let mut batch = DiskWriteBatch::new(); + batch.zs_insert(&hash_cf, Height(30), stray_hash); + state.db.write(batch).expect("raw insert writes"); + + let error = redeliver_trunk(&state, &universe, 30, 5) + .expect_err("the anchor round-trip fails on the corrupted index"); + assert!( + matches!( + error, + CommitHeaderRangeError::StoreIncoherent(StoreIncoherentError::BijectionMismatch { + hash, + height, + stored: Some(stored), + }) if hash == universe.trunk_at(30).hash + && height == Height(30) + && stored == stray_hash + ), + "expected StoreIncoherent(BijectionMismatch), got {error:?}" + ); + + // An anchor the store has never heard of stays UnknownAnchor. + let unknown = universe.branches[BRANCH_A].headers[10].hash; + let headers = vec![universe.trunk[30].header.clone()]; + let mut batch = DiskWriteBatch::new(); + let error = batch + .prepare_header_range_batch(&state, unknown, &headers, &[0]) + .expect_err("an unindexed anchor is unknown"); + assert!( + matches!(error, CommitHeaderRangeError::UnknownAnchor { .. }), + "expected UnknownAnchor, got {error:?}" + ); +} diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/scenarios.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/scenarios.rs index 8515937c34c..072bd4eb4df 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/scenarios.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/scenarios.rs @@ -4,16 +4,14 @@ //! passing scenario is a regression gate on the whole write sequence, not //! just its final assertions. //! -//! The `corruption_repro_*` tests at the bottom deliberately demonstrate -//! store-invariant violations that exist today: they PASS while the bug -//! exists and fail loudly once the write path is fixed, forcing re-triage. -//! Their `#[ignore]`d twins assert the true invariant, to be un-ignored -//! together with the write-path fix. +//! The `*_upholds_invariants` tests at the bottom are the permanent +//! regression gates for the three write-path corruption bugs this suite +//! originally proved with `corruption_repro_*` twins (removed together with +//! the write-path fixes; see the module README for the bug histories). use zebra_chain::block::Height; use super::{ - audit::Violation, fabricate::{BRANCH_A, BRANCH_B, BRANCH_B_EXT, BRANCH_C, FORK_HEIGHT, TRUNK_LEN}, ops::{universe, Anchor, Harness, Op, OpOutcome, Source}, }; @@ -300,7 +298,7 @@ fn s04_body_commit_racing_header_reorg() { /// s05: the release path's frontier trim. Sequential body commits release /// matching provisional rows one by one; header re-delivery *above* the body /// tip is idempotent; restarts change nothing. (Re-delivery *over* body-backed -/// heights strands rows — that is `corruption_repro_redelivery_over_bodies`.) +/// heights skips those heights — `redelivery_over_bodies_upholds_invariants`.) #[test] fn s05_release_trim_then_redelivery() { let _init_guard = zebra_test::init(); @@ -541,19 +539,108 @@ fn s10_seed_interplay() { ); } +/// s11: online pruning below the body tip is membership-neutral — the zakura +/// frontier, chain selection, reorgs, and restart survival are unaffected — +/// and it leaves the pruned-store shape (consensus hash rows retained, +/// bodies gone) the startup audit's committed-height predicate relies on. +#[test] +fn s11_prune_below_body_tip_is_membership_neutral() { + let _init_guard = zebra_test::init(); + let mut harness = Harness::new(); + + let outcomes = harness + .run_all(&[ + commit_trunk(), + Op::Finalize { count: 10 }, + Op::Prune { until: 8 }, + // A reorg above the fork still works over a pruned base. + commit_branch(BRANCH_A), + Op::Reopen, + // Deeper pruning after the reorg and restart is still clean. + Op::Finalize { count: 2 }, + Op::Prune { until: 12 }, + ]) + .expect("pruning sequence has no violations"); + outcomes.iter().for_each(assert_accepted); + + assert_eq!( + harness.state().best_header_tip(), + Some(branch_tip(BRANCH_A)), + ); + // The pruned-store shape: heights stay committed, bodies are gone. + assert!(harness.state().contains_height(Height(5))); + assert!(!harness.state().contains_body_at_height(Height(5))); + assert!(!harness.state().contains_body_at_height(Height(11))); + assert!(harness.state().contains_body_at_height(Height(12))); +} + +/// s11: a refused (unlinked) seed converges through header-range sync. The +/// zakura store follows branch A (seeded at the fork height); the +/// non-finalized best chain switches to branch B and its new best tip (B's +/// *second* row) is seeded. The store refuses the unlinked seed as a no-op — +/// the header store briefly lags the nf chain — and the later linked range +/// delivery of B converges it onto the new branch. +#[test] +fn s11_refused_seed_converges_via_range_delivery() { + let _init_guard = zebra_test::init(); + let mut harness = Harness::new(); + + let outcomes = harness + .run_all(&[ + commit_trunk(), + Op::Seed { + source: Source::Branch(BRANCH_A), + index: 0, + }, + ]) + .expect("setup is clean"); + outcomes.iter().for_each(assert_accepted); + let a_first = &universe().branches[BRANCH_A].headers[0]; + assert_eq!( + harness.state().best_header_tip(), + Some((a_first.height, a_first.hash)), + ); + + // The nf best tip jumps to B[1]; its parent row is not stored, so the + // seed is refused without touching the store (checked by the harness). + let outcome = harness + .run(&Op::Seed { + source: Source::Branch(BRANCH_B), + index: 1, + }) + .expect("the refused seed leaves the store coherent"); + assert_accepted(&outcome); + assert_eq!( + harness.state().best_header_tip(), + Some((a_first.height, a_first.hash)), + "the refused seed must not move the header tip", + ); + + // Header sync catches up with the nf chain: B arrives as a linked range + // and out-works the stored A suffix, converging the store onto B. + let outcome = harness + .run(&commit_branch(BRANCH_B)) + .expect("the converging range delivery is clean"); + assert_accepted(&outcome); + assert_eq!( + harness.state().best_header_tip(), + Some(branch_tip(BRANCH_B)), + ); +} + // --------------------------------------------------------------------------- -// Corruption reproductions: deterministic sequences that violate the store -// invariants today. See the module doc for the pass/ignore convention. +// Regression gates for the three proven write-path corruption bugs. Each was +// originally pinned by a `corruption_repro_*` test that demonstrated the +// violation; those were removed with the write-path fixes, and these twins +// now assert the fixed behavior over the exact same op sequences. // --------------------------------------------------------------------------- -/// The unlinked-anchor commit: `prepare_header_range_batch_with_roots` never -/// checks that `headers[0].previous_block_hash == anchor` (or any intra-range -/// linkage). A range of branch-A headers anchored at the same-height *trunk* -/// hash passes contextual difficulty validation — the two fast chains have -/// identical (time, threshold) sequences — and commits a suffix that does not -/// link to the row below it: an on-disk I2 violation. Production reach: a -/// header range is untrusted peer input; nothing upstream of the store -/// re-checks the anchor linkage. +/// The unlinked-anchor commit (bug 1): a range of branch-A headers anchored at +/// the same-height *trunk* hash passes contextual difficulty validation — the +/// two fast chains have identical (time, threshold) sequences — so before the +/// linkage check in `prepare_header_range_batch_with_roots`, it committed a +/// suffix that did not link to the row below it: an on-disk I2 violation +/// reachable from a single untrusted peer response. fn unlinked_anchor_ops() -> Vec { vec![ commit_trunk(), @@ -567,58 +654,36 @@ fn unlinked_anchor_ops() -> Vec { ] } -/// PASSES while the bug exists (proves the repro); flips when the writer is -/// fixed, forcing this pair to be re-triaged. +/// The store rejects unlinked ranges with `UnlinkedRange` and stays coherent. #[test] -fn corruption_repro_unlinked_anchor_commit() { - let _init_guard = zebra_test::init(); - let mut harness = Harness::new(); - - let report = harness - .run_all(&unlinked_anchor_ops()) - .expect_err("the unlinked-anchor commit corrupts the store today"); - - assert!( - report - .violations - .iter() - .any(|violation| matches!(violation, Violation::BrokenLinkage { height, .. } if *height == Height(FORK_HEIGHT + 2))), - "expected BrokenLinkage right above the spliced anchor: {report:?}" - ); - assert!( - !report.mismatches.is_empty(), - "the oracle rejects this range; the store accepted it: {report:?}" - ); -} - -/// The true invariant. Un-ignore when the write-path fix lands. -#[test] -#[ignore = "known zakura header-store corruption: un-ignore with the write-path fix"] fn unlinked_anchor_commit_upholds_invariants() { let _init_guard = zebra_test::init(); let mut harness = Harness::new(); - harness + let outcomes = harness .run_all(&unlinked_anchor_ops()) .expect("the store rejects unlinked ranges and stays coherent"); + assert!( + matches!( + outcomes[1].header_range_error(), + CommitHeaderRangeError::UnlinkedRange { .. } + ), + "expected UnlinkedRange, got {:?}", + outcomes[1] + ); } -/// Re-delivery over committed bodies: the range insert loop -/// (`prepare_header_range_batch_with_roots`) gates only its *roots* write on -/// `contains_body_at_height` — the header/hash/height/body-size writes are -/// unconditional. A header range re-delivered over heights whose bodies have -/// since been committed (a header store behind the body store, or a late range -/// response racing body sync — the exact scenario the roots gate's own comment -/// describes) re-inserts zakura rows *below* the body tip. That breaks the -/// frontier-overlay invariant ("the Zakura header store only ever holds -/// heights with no committed body", block.rs release-path doc): the release -/// trim already ran at body-commit time, so nothing ever removes these rows. +/// Re-delivery over committed bodies (bug 2): before the committed-height +/// gate covered the header/hash/height/body-size writes (it originally gated +/// only the *roots* write), a header range re-delivered over heights whose +/// bodies had since been committed re-inserted zakura rows *below* the body +/// tip — rows the release trim (which already ran at body-commit time) never +/// removes. fn redelivery_over_bodies_ops() -> Vec { vec![ commit_trunk(), Op::Finalize { count: 10 }, // Re-delivery spanning body-backed heights 6..=10 and header-only - // heights above. The rows match the canonical chain, so the store - // accepts — but the write strands zakura rows under the body tip. + // heights above: accepted, but the body-backed heights are skipped. Op::CommitHeaderRange { source: Source::Trunk, offset: 5, @@ -628,50 +693,21 @@ fn redelivery_over_bodies_ops() -> Vec { ] } -/// PASSES while the bug exists (proves the repro); flips when the writer is -/// fixed, forcing this pair to be re-triaged. -#[test] -fn corruption_repro_redelivery_over_bodies() { - let _init_guard = zebra_test::init(); - let mut harness = Harness::new(); - - let report = harness - .run_all(&redelivery_over_bodies_ops()) - .expect_err("re-delivery over committed bodies strands zakura rows today"); - - assert!( - report - .violations - .iter() - .any(|violation| matches!(violation, Violation::ZakuraRowAtBodyHeight { height, .. } if *height <= Height(10))), - "expected zakura rows stranded at body-backed heights: {report:?}" - ); - assert!( - report.mismatches.is_empty(), - "acceptance itself is correct chain selection; the write effects are the bug: {report:?}" - ); -} - -/// The true invariant. Un-ignore when the write-path fix lands. +/// Re-delivery over bodies leaves no zakura rows below the body tip. #[test] -#[ignore = "known zakura header-store corruption: un-ignore with the write-path fix"] fn redelivery_over_bodies_upholds_invariants() { let _init_guard = zebra_test::init(); let mut harness = Harness::new(); - harness + let outcomes = harness .run_all(&redelivery_over_bodies_ops()) .expect("re-delivery over bodies leaves no zakura rows below the body tip"); + outcomes.iter().for_each(assert_accepted); } -/// Seed above a gap — found by `prop_random_sequences_uphold_invariants` and -/// shrunk to a single op. The seed path -/// (`prepare_zakura_header_from_committed_block`) writes header/hash/height -/// rows at its height with **no linkage or anchor precondition**: seeding a -/// block whose parent row is absent leaves a row the chain walk cannot reach. -/// Production shape: seeds fire only at non-finalized best-*tip* commits, so -/// any nf best-tip jump — a fork switch between nf chains, or a restart that -/// restores the nf backup and then commits on top — seeds a height whose -/// parent row was never written. +/// Seed above a gap (bug 3, minimal shape — found by the discovery proptest +/// and shrunk to a single op): before the parent-linkage refusal in +/// `prepare_zakura_header_from_committed_block`, seeding a block whose parent +/// row is absent left a row the chain walk cannot reach. fn seed_above_gap_ops() -> Vec { vec![Op::Seed { source: Source::Trunk, @@ -679,51 +715,30 @@ fn seed_above_gap_ops() -> Vec { }] } -/// PASSES while the bug exists (proves the repro); flips when the writer is -/// fixed, forcing this pair to be re-triaged. +/// A seed above a gap is refused as a no-op instead of stranding a row. #[test] -fn corruption_repro_seed_above_gap() { - let _init_guard = zebra_test::init(); - let mut harness = Harness::new(); - - let report = harness - .run_all(&seed_above_gap_ops()) - .expect_err("seeding above a gap strands an unreachable row today"); - - assert!( - report - .violations - .iter() - .any(|violation| matches!(violation, Violation::RowAboveLastLinked { height, .. } if *height == Height(2))), - "expected the seeded row to be unreachable from the finalized tip: {report:?}" - ); - assert!( - report - .violations - .iter() - .any(|violation| matches!(violation, Violation::BestHeaderTipMismatch { .. })), - "expected best_header_tip to point above the linked chain: {report:?}" - ); -} - -/// The true invariant. Un-ignore when the write-path fix lands. -#[test] -#[ignore = "known zakura header-store corruption: un-ignore with the write-path fix"] fn seed_above_gap_upholds_invariants() { let _init_guard = zebra_test::init(); let mut harness = Harness::new(); harness .run_all(&seed_above_gap_ops()) .expect("a seed above a gap must not strand an unreachable row"); + + let genesis_hash = universe().genesis.hash(); + assert_eq!( + harness.state().best_header_tip(), + Some((Height(0), genesis_hash)), + "the refused seed must not move the header tip", + ); } -/// Seed fork switch — the production-shaped variant of the seed linkage hole. -/// The zakura store follows branch A (seeded at the fork height); the -/// non-finalized best chain switches to branch B and its new best tip (B's -/// *second* row) is seeded. The seed's non-conflict arm inserts the row -/// directly — B's parent row was truncated, so the store now holds -/// `A[0]` at the fork height and `B[1]` right above it: broken linkage on -/// disk, exactly the poisoned-DAA-window generator from the incident table. +/// Seed fork switch (bug 3, production shape): the zakura store follows +/// branch A; the non-finalized best chain switches to branch B and its new +/// best tip (B's *second* row) is seeded. Before the parent-linkage refusal, +/// the seed's non-conflict arm inserted the row directly over A's truncated +/// parent — broken linkage on disk, the poisoned-DAA-window generator from +/// the production incident table. (`s11` proves the refused seed converges +/// later through range delivery.) fn seed_fork_switch_ops() -> Vec { vec![ commit_trunk(), @@ -738,33 +753,19 @@ fn seed_fork_switch_ops() -> Vec { ] } -/// PASSES while the bug exists (proves the repro); flips when the writer is -/// fixed, forcing this pair to be re-triaged. +/// A fork-switch seed keeps the store linked (the unlinked seed is refused). #[test] -fn corruption_repro_seed_fork_switch() { - let _init_guard = zebra_test::init(); - let mut harness = Harness::new(); - - let report = harness - .run_all(&seed_fork_switch_ops()) - .expect_err("a fork-switch seed breaks linkage today"); - - assert!( - report - .violations - .iter() - .any(|violation| matches!(violation, Violation::BrokenLinkage { height, .. } if *height == Height(FORK_HEIGHT + 2))), - "expected broken linkage right above the fork height: {report:?}" - ); -} - -/// The true invariant. Un-ignore when the write-path fix lands. -#[test] -#[ignore = "known zakura header-store corruption: un-ignore with the write-path fix"] fn seed_fork_switch_upholds_invariants() { let _init_guard = zebra_test::init(); let mut harness = Harness::new(); harness .run_all(&seed_fork_switch_ops()) .expect("a fork-switch seed must keep the store linked"); + + let a_first = &universe().branches[BRANCH_A].headers[0]; + assert_eq!( + harness.state().best_header_tip(), + Some((a_first.height, a_first.hash)), + "the store stays on A until a linked delivery of B arrives", + ); } diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/startup_audit.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/startup_audit.rs new file mode 100644 index 00000000000..933aa3e7b89 --- /dev/null +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/startup_audit.rs @@ -0,0 +1,664 @@ +//! Startup-audit self-repair: a hand-corrupted zakura header store heals when +//! the database is (re)opened (REORG_PLAN Pillar 3, startup half). +//! +//! The write path refuses to create incoherent stores (the Phase-1.5 guards) +//! and the consensus readers refuse to consume them (Pillar 2), so these tests +//! corrupt the column families directly — the same hand-made corruption +//! technique as the audit's and read-path tests — and assert that: +//! +//! - the startup audit finds the violation and truncates the store to the +//! last coherent height in all five column families, leaving a store the +//! full coherence audit passes and header sync can re-download onto; +//! - the repair happens on the real startup path (`ZebraDb::new` at reopen) +//! and persists: a second reopen finds a coherent store and changes nothing; +//! - repair is minimal where truncation is not needed (an orphaned +//! reverse-index row, stale rows at committed heights); +//! - a store that failed the Pillar-2 linkage-verified reads passes them +//! after the heal; and +//! - the `state.zakura.header_store.incoherent` metric is emitted exactly +//! when a repair runs. + +use std::{ + collections::HashMap, + sync::{Arc, Mutex}, +}; + +use zebra_chain::block::Height; + +use super::super::super::startup_audit::ZakuraStoreViolation; +use super::super::super::{ + test_body_size, ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT, ZAKURA_HEADER_BY_HEIGHT, + ZAKURA_HEADER_HASH_BY_HEIGHT, ZAKURA_HEADER_HEIGHT_BY_HASH, +}; +use super::super::common::{ + commit_header_range, persistent_config, persistent_state, state_with_genesis_config, + write_full_block_header_and_transactions, +}; +use super::{ + audit::{audit_store, dump_store, StoreDump}, + fabricate::{fabricate_body, Universe, BRANCH_A, FORK_HEIGHT}, +}; +use crate::{ + error::{CommitHeaderRangeError, StoreIncoherentError}, + service::finalized_state::{ + disk_db::{DiskWriteBatch, WriteDisk}, + disk_format::shielded::CommitmentRootsByHeight, + ZebraDb, COMMITMENT_ROOTS_BY_HEIGHT, + }, + Config, +}; + +/// A store holding genesis plus the trunk up to the fork height, built through +/// the production write path, on the given (persistent or ephemeral) config. +fn trunk_state(universe: &Universe, config: Config) -> ZebraDb { + let state = state_with_genesis_config(&universe.network, universe.genesis.clone(), config); + let trunk_headers: Vec<_> = universe.trunk[..FORK_HEIGHT as usize] + .iter() + .map(|fab| fab.header.clone()) + .collect(); + commit_header_range(&state, universe.genesis.hash(), &trunk_headers); + state +} + +/// Closes `state` and reopens the same database, running the startup audit. +fn reopen(state: ZebraDb, config: &Config, universe: &Universe) -> ZebraDb { + let mut state = state; + state.shutdown(true); + drop(state); + persistent_state(config, &universe.network) +} + +/// The expected store after truncating everything above `last` (the original +/// coherent dump with all five column families filtered to `..= last`). +fn truncated(dump: &StoreDump, last: Height) -> StoreDump { + StoreDump { + headers: dump + .headers + .iter() + .filter(|(height, _)| **height <= last) + .map(|(height, header)| (*height, header.clone())) + .collect(), + hashes: dump + .hashes + .iter() + .filter(|(height, _)| **height <= last) + .map(|(height, hash)| (*height, *hash)) + .collect(), + heights_by_hash: dump + .heights_by_hash + .iter() + .filter(|(_, points_at)| *points_at <= last) + .copied() + .collect(), + body_sizes: dump + .body_sizes + .iter() + .filter(|(height, _)| **height <= last) + .map(|(height, size)| (*height, *size)) + .collect(), + roots: dump + .roots + .iter() + .filter(|(height, _)| **height <= last) + .map(|(height, roots)| (*height, *roots)) + .collect(), + } +} + +fn assert_clean(state: &ZebraDb) { + let violations = audit_store(state); + assert!( + violations.is_empty(), + "expected a coherent store after repair: {violations:?}" + ); +} + +/// A coherent store is untouched: the direct audit reports nothing, and a +/// reopen through the real startup path changes no rows. +#[test] +fn startup_audit_is_noop_on_coherent_store() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let tempdir = tempfile::tempdir().expect("test tempdir is available"); + let config = persistent_config(tempdir.path()); + let state = trunk_state(&universe, config.clone()); + let original = dump_store(&state); + + let repair = state + .audit_and_repair_zakura_header_store() + .expect("audit reads and writes succeed"); + assert!( + repair.is_none(), + "no repair expected on a coherent store: {repair:?}" + ); + assert_eq!(dump_store(&state), original); + + let state = reopen(state, &config, &universe); + assert_eq!( + dump_store(&state), + original, + "a reopen must not change a coherent store" + ); +} + +/// Broken linkage mid-frontier (a self-consistent foreign row spliced into the +/// height index — the incident-shaped poison): the audit truncates to the row +/// below the splice, the heal persists across a further reopen, and header +/// sync re-downloads the truncated suffix back to the original store. +#[test] +fn startup_heals_broken_linkage_then_resync_converges() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let tempdir = tempfile::tempdir().expect("test tempdir is available"); + let config = persistent_config(tempdir.path()); + let state = trunk_state(&universe, config.clone()); + let original = dump_store(&state); + + // Replace the header and hash rows at height 20 with branch-A's fourth + // row: internally consistent, but it does not link to trunk@19. + let foreign = &universe.branches[BRANCH_A].headers[3]; + let header_cf = state.db.cf_handle(ZAKURA_HEADER_BY_HEIGHT).unwrap(); + let hash_cf = state.db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); + let mut batch = DiskWriteBatch::new(); + batch.zs_insert(&header_cf, Height(20), foreign.header.clone()); + batch.zs_insert(&hash_cf, Height(20), foreign.hash); + state.db.write(batch).expect("raw insert writes"); + + // The corruption is visible to the Pillar-2 reads before the heal. + let error = state + .recent_header_context(Height(30)) + .expect_err("the walk crosses the corrupted row"); + assert!(matches!(error, StoreIncoherentError::BrokenLinkage { .. })); + + let repair = state + .audit_and_repair_zakura_header_store() + .expect("audit reads and writes succeed") + .expect("the corrupted store is repaired"); + assert_eq!( + repair.last_coherent, + Some((Height(19), universe.trunk_at(19).hash)) + ); + assert!( + repair.violations.iter().any(|violation| matches!( + violation, + ZakuraStoreViolation::BrokenLinkage { height, actual_below, .. } + if *height == Height(20) && *actual_below == universe.trunk_at(19).hash + )), + "expected BrokenLinkage at the splice: {:?}", + repair.violations + ); + + // Everything above the last coherent height is gone, in all five CFs. + assert_eq!(dump_store(&state), truncated(&original, Height(19))); + assert_clean(&state); + state + .recent_header_context(Height(19)) + .expect("the healed store walks cleanly"); + + // The heal persists: a reopen through the startup path changes nothing. + let state = reopen(state, &config, &universe); + assert_eq!(dump_store(&state), truncated(&original, Height(19))); + + // Header sync re-downloads the truncated suffix and converges back onto + // the original chain. + let redelivered: Vec<_> = universe.trunk[19..FORK_HEIGHT as usize] + .iter() + .map(|fab| fab.header.clone()) + .collect(); + commit_header_range(&state, universe.trunk_at(19).hash, &redelivered); + assert_eq!(dump_store(&state), original); + assert_clean(&state); +} + +/// A gap mid-frontier strands every row above it: reopening alone (no direct +/// audit call) heals the store, proving the `ZebraDb::new` hook fires. +#[test] +fn startup_heals_gap_and_stranded_rows_on_reopen() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let tempdir = tempfile::tempdir().expect("test tempdir is available"); + let config = persistent_config(tempdir.path()); + let state = trunk_state(&universe, config.clone()); + let original = dump_store(&state); + + let header_cf = state.db.cf_handle(ZAKURA_HEADER_BY_HEIGHT).unwrap(); + let hash_cf = state.db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); + let mut batch = DiskWriteBatch::new(); + batch.zs_delete(&header_cf, Height(15)); + batch.zs_delete(&hash_cf, Height(15)); + state.db.write(batch).expect("raw delete writes"); + + let state = reopen(state, &config, &universe); + + // Rows 16..=50 were stranded above the gap; the audit truncated to 14. + assert_eq!(dump_store(&state), truncated(&original, Height(14))); + assert_clean(&state); + assert_eq!( + state.best_header_tip(), + Some((Height(14), universe.trunk_at(14).hash)) + ); +} + +/// A missing reverse-index row breaks the hash↔height bijection: the audit +/// truncates to the row below it. +#[test] +fn startup_heals_missing_reverse_index_row() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe, Config::ephemeral()); + let original = dump_store(&state); + + let height_by_hash_cf = state.db.cf_handle(ZAKURA_HEADER_HEIGHT_BY_HASH).unwrap(); + let mut batch = DiskWriteBatch::new(); + batch.zs_delete(&height_by_hash_cf, universe.trunk_at(10).hash); + state.db.write(batch).expect("raw delete writes"); + + let repair = state + .audit_and_repair_zakura_header_store() + .expect("audit reads and writes succeed") + .expect("the corrupted store is repaired"); + assert_eq!( + repair.last_coherent, + Some((Height(9), universe.trunk_at(9).hash)) + ); + assert!( + repair.violations.iter().any(|violation| matches!( + violation, + ZakuraStoreViolation::WrongHeightByHash { height, indexed: None, .. } + if *height == Height(10) + )), + "expected WrongHeightByHash at the deleted entry: {:?}", + repair.violations + ); + assert_eq!(dump_store(&state), truncated(&original, Height(9))); + assert_clean(&state); +} + +/// An orphaned reverse-index row (a stranded `height_by_hash` entry from an +/// overwrite that never cleaned up the displaced hash) is removed on its own: +/// the linked chain is coherent, so nothing is truncated. +#[test] +fn startup_removes_orphan_reverse_row_without_truncating() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe, Config::ephemeral()); + let original = dump_store(&state); + + let height_by_hash_cf = state.db.cf_handle(ZAKURA_HEADER_HEIGHT_BY_HASH).unwrap(); + let stray_hash = universe.branches[BRANCH_A].headers[0].hash; + let mut batch = DiskWriteBatch::new(); + batch.zs_insert(&height_by_hash_cf, stray_hash, Height(12)); + state.db.write(batch).expect("raw insert writes"); + + let repair = state + .audit_and_repair_zakura_header_store() + .expect("audit reads and writes succeed") + .expect("the corrupted store is repaired"); + assert_eq!( + repair.last_coherent, + Some((Height(FORK_HEIGHT), universe.trunk_at(FORK_HEIGHT).hash)), + "an orphan reverse row must not shorten the coherent chain" + ); + assert_eq!(repair.deleted_rows, 1); + assert!( + repair.violations.iter().any(|violation| matches!( + violation, + ZakuraStoreViolation::OrphanHeightByHash { hash, points_at } + if *hash == stray_hash && *points_at == Height(12) + )), + "expected OrphanHeightByHash for the stray entry: {:?}", + repair.violations + ); + assert_eq!(dump_store(&state), original); + assert_clean(&state); +} + +/// Stale zakura rows at a committed height (the pre-guard re-delivery shape: +/// rows the release trim never touches again) are removed without disturbing +/// the frontier above. +#[test] +fn startup_removes_stale_rows_at_committed_heights() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe, Config::ephemeral()); + let original = dump_store(&state); + + // Genesis is the only committed block; plant zakura rows at its height. + let header_cf = state.db.cf_handle(ZAKURA_HEADER_BY_HEIGHT).unwrap(); + let hash_cf = state.db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); + let height_by_hash_cf = state.db.cf_handle(ZAKURA_HEADER_HEIGHT_BY_HASH).unwrap(); + let genesis_hash = universe.genesis.hash(); + let mut batch = DiskWriteBatch::new(); + batch.zs_insert(&header_cf, Height(0), universe.genesis.header.clone()); + batch.zs_insert(&hash_cf, Height(0), genesis_hash); + batch.zs_insert(&height_by_hash_cf, genesis_hash, Height(0)); + state.db.write(batch).expect("raw insert writes"); + + let repair = state + .audit_and_repair_zakura_header_store() + .expect("audit reads and writes succeed") + .expect("the corrupted store is repaired"); + assert_eq!( + repair.last_coherent, + Some((Height(FORK_HEIGHT), universe.trunk_at(FORK_HEIGHT).hash)), + "stale committed-height rows must not shorten the frontier" + ); + assert_eq!(repair.deleted_rows, 3); + assert!( + repair.violations.iter().all(|violation| matches!( + violation, + ZakuraStoreViolation::StaleRowAtCommittedHeight { height, .. } + if *height == Height(0) + )), + "expected only StaleRowAtCommittedHeight at genesis: {:?}", + repair.violations + ); + assert_eq!(dump_store(&state), original); + assert_clean(&state); +} + +/// The reads.rs anchor-corruption shape (a hash row overwritten with a foreign +/// hash) makes the Pillar-2 range writer reject with `StoreIncoherent`; after +/// the heal the same delivery commits. This pins the Pillar-2 interaction: any +/// store those reads refuse is healed by this audit. +#[test] +fn startup_heal_unblocks_linkage_verified_reads() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe, Config::ephemeral()); + let original = dump_store(&state); + + // The hash row at height 30 now names a different block, while the header + // row and `height_by_hash` still claim the trunk. + let hash_cf = state.db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); + let stray_hash = universe.branches[BRANCH_A].headers[0].hash; + let mut batch = DiskWriteBatch::new(); + batch.zs_insert(&hash_cf, Height(30), stray_hash); + state.db.write(batch).expect("raw insert writes"); + + // A re-delivered range anchored at trunk@30 is rejected as a local + // storage fault before the heal. + let anchor = universe.trunk_at(30).hash; + let headers: Vec<_> = universe.trunk[30..35] + .iter() + .map(|fab| fab.header.clone()) + .collect(); + let body_sizes = vec![0; headers.len()]; + let mut batch = DiskWriteBatch::new(); + let error = batch + .prepare_header_range_batch(&state, anchor, &headers, &body_sizes) + .expect_err("the anchor round-trip fails on the corrupted index"); + assert!(matches!( + error, + CommitHeaderRangeError::StoreIncoherent(StoreIncoherentError::BijectionMismatch { .. }) + )); + + let repair = state + .audit_and_repair_zakura_header_store() + .expect("audit reads and writes succeed") + .expect("the corrupted store is repaired"); + assert_eq!( + repair.last_coherent, + Some((Height(29), universe.trunk_at(29).hash)) + ); + assert!( + repair.violations.iter().any(|violation| matches!( + violation, + ZakuraStoreViolation::HeaderHashMismatch { height, indexed, .. } + if *height == Height(30) && *indexed == stray_hash + )), + "expected HeaderHashMismatch at the overwritten hash row: {:?}", + repair.violations + ); + assert_eq!(dump_store(&state), truncated(&original, Height(29))); + assert_clean(&state); + + // The same fork of history now commits, anchored at the healed tip. + let redelivered: Vec<_> = universe.trunk[29..FORK_HEIGHT as usize] + .iter() + .map(|fab| fab.header.clone()) + .collect(); + commit_header_range(&state, universe.trunk_at(29).hash, &redelivered); + assert_eq!(dump_store(&state), original); + assert_clean(&state); +} + +/// A minimal local metrics recorder capturing counter increments by name. +#[derive(Clone, Default)] +struct CounterCapture(Arc>>); + +impl CounterCapture { + fn get(&self, name: &str) -> u64 { + self.0 + .lock() + .expect("counter capture lock is never poisoned") + .get(name) + .copied() + .unwrap_or(0) + } +} + +struct CaptureHandle { + name: String, + store: Arc>>, +} + +impl metrics::CounterFn for CaptureHandle { + fn increment(&self, value: u64) { + *self + .store + .lock() + .expect("counter capture lock is never poisoned") + .entry(self.name.clone()) + .or_insert(0) += value; + } + + fn absolute(&self, value: u64) { + self.store + .lock() + .expect("counter capture lock is never poisoned") + .insert(self.name.clone(), value); + } +} + +impl metrics::Recorder for CounterCapture { + fn describe_counter( + &self, + _: metrics::KeyName, + _: Option, + _: metrics::SharedString, + ) { + } + + fn describe_gauge( + &self, + _: metrics::KeyName, + _: Option, + _: metrics::SharedString, + ) { + } + + fn describe_histogram( + &self, + _: metrics::KeyName, + _: Option, + _: metrics::SharedString, + ) { + } + + fn register_counter(&self, key: &metrics::Key, _: &metrics::Metadata<'_>) -> metrics::Counter { + metrics::Counter::from_arc(Arc::new(CaptureHandle { + name: key.name().to_string(), + store: self.0.clone(), + })) + } + + fn register_gauge(&self, _: &metrics::Key, _: &metrics::Metadata<'_>) -> metrics::Gauge { + metrics::Gauge::noop() + } + + fn register_histogram( + &self, + _: &metrics::Key, + _: &metrics::Metadata<'_>, + ) -> metrics::Histogram { + metrics::Histogram::noop() + } +} + +/// The `state.zakura.header_store.incoherent` metric fires exactly when a +/// startup repair runs: once for the healing reopen, and not at all for the +/// clean reopen after it. +#[test] +fn startup_repair_emits_incoherent_metric() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let tempdir = tempfile::tempdir().expect("test tempdir is available"); + let config = persistent_config(tempdir.path()); + let state = trunk_state(&universe, config.clone()); + + let header_cf = state.db.cf_handle(ZAKURA_HEADER_BY_HEIGHT).unwrap(); + let hash_cf = state.db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); + let mut batch = DiskWriteBatch::new(); + batch.zs_delete(&header_cf, Height(15)); + batch.zs_delete(&hash_cf, Height(15)); + state.db.write(batch).expect("raw delete writes"); + + const METRIC: &str = "state.zakura.header_store.incoherent"; + + let healing = CounterCapture::default(); + let state = metrics::with_local_recorder(&healing, || reopen(state, &config, &universe)); + assert_eq!( + healing.get(METRIC), + 1, + "the healing reopen emits the metric" + ); + assert_clean(&state); + + let clean = CounterCapture::default(); + let state = metrics::with_local_recorder(&clean, || reopen(state, &config, &universe)); + assert_eq!(clean.get(METRIC), 0, "a clean reopen emits nothing"); + assert_clean(&state); +} + +/// Stale zakura rows at *pruned* heights are classified +/// `StaleRowAtCommittedHeight` and deleted, while everything else — the +/// zakura frontier and the roots rows at committed heights — is untouched. +/// +/// This is the test the audit's committed-height predicate was chosen for: +/// the predicate is the consensus `hash_by_height` row, which pruning +/// retains, not body presence (pruning removes bodies). A body-presence +/// predicate would misclassify these rows as frontier rows and let the +/// pre-#491 bug-2 shape (stale re-delivered rows below the body tip) +/// survive on pruned stores. +#[test] +fn startup_removes_stale_rows_at_pruned_heights() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let tempdir = tempfile::tempdir().expect("test tempdir is available"); + let config = persistent_config(tempdir.path()); + let state = trunk_state(&universe, config.clone()); + + // Commit bodies 1..=10 (releasing the zakura rows there), then prune + // raw transactions below height 8 through the production prune batch. + for fab in &universe.trunk[..10] { + write_full_block_header_and_transactions(&state, fabricate_body(fab)); + } + let mut batch = DiskWriteBatch::new(); + batch.prepare_prune_batch(&state, Height(1), Height(8)); + state.write_batch(batch).expect("prune batch writes"); + + // The pruned-store shape: committed heights whose bodies are gone. + assert!(state.contains_height(Height(5))); + assert!(!state.contains_body_at_height(Height(5))); + assert!(state.contains_body_at_height(Height(8))); + + // A verified roots row at a pruned height (pruning retains these): the + // audit must leave it untouched — committed heights are out of scope. + let roots_cf = state.db.cf_handle(COMMITMENT_ROOTS_BY_HEIGHT).unwrap(); + let mut batch = DiskWriteBatch::new(); + batch.zs_insert( + &roots_cf, + Height(5), + CommitmentRootsByHeight { + sapling: zebra_chain::sapling::tree::NoteCommitmentTree::default().root(), + orchard: zebra_chain::orchard::tree::NoteCommitmentTree::default().root(), + auth_data_root: zebra_chain::block::merkle::AuthDataRoot::from([7u8; 32]), + ironwood: zebra_chain::ironwood::tree::NoteCommitmentTree::default().root(), + sapling_tx: 0, + orchard_tx: 0, + ironwood_tx: 0, + }, + ); + state.db.write(batch).expect("raw insert writes"); + + let clean = dump_store(&state); + assert!( + clean.roots.contains_key(&Height(5)), + "the roots row at a pruned height is present and out of audit scope" + ); + + // Hand-plant stale zakura rows at pruned heights: a full row triple at + // height 5 and a body-size hint at height 3 (the pre-guard re-delivery + // shape: `contains_height` true, `contains_body_at_height` false). + let header_cf = state.db.cf_handle(ZAKURA_HEADER_BY_HEIGHT).unwrap(); + let hash_cf = state.db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); + let reverse_cf = state.db.cf_handle(ZAKURA_HEADER_HEIGHT_BY_HASH).unwrap(); + let body_size_cf = state + .db + .cf_handle(ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT) + .unwrap(); + let stale = &universe.trunk[4]; + let mut batch = DiskWriteBatch::new(); + batch.zs_insert(&header_cf, Height(5), stale.header.clone()); + batch.zs_insert(&hash_cf, Height(5), stale.hash); + batch.zs_insert(&reverse_cf, stale.hash, Height(5)); + batch.zs_insert(&body_size_cf, Height(3), test_body_size(77)); + state.db.write(batch).expect("raw insert writes"); + + let repair = state + .audit_and_repair_zakura_header_store() + .expect("audit reads and writes succeed") + .expect("the stale rows are repaired"); + assert_eq!(repair.deleted_rows, 4); + for (cf, height) in [ + (ZAKURA_HEADER_BY_HEIGHT, Height(5)), + (ZAKURA_HEADER_HASH_BY_HEIGHT, Height(5)), + (ZAKURA_HEADER_HEIGHT_BY_HASH, Height(5)), + (ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT, Height(3)), + ] { + assert!( + repair.violations.iter().any(|violation| matches!( + violation, + ZakuraStoreViolation::StaleRowAtCommittedHeight { + cf: found_cf, + height: found_height, + } if *found_cf == cf && *found_height == height + )), + "expected StaleRowAtCommittedHeight for {cf} at {height:?}: {:?}", + repair.violations + ); + } + assert_eq!( + repair.last_coherent, + Some((Height(FORK_HEIGHT), universe.trunk_at(FORK_HEIGHT).hash)), + "stale rows at pruned heights must not shorten the coherent frontier" + ); + + // The repair restores exactly the pre-corruption store: the frontier + // and the out-of-scope roots rows at committed heights survive. + assert_eq!(dump_store(&state), clean); + assert_clean(&state); + + // The same heal runs on the real startup path (reopen). + let mut batch = DiskWriteBatch::new(); + batch.zs_insert(&hash_cf, Height(6), universe.trunk[5].hash); + state.db.write(batch).expect("raw insert writes"); + let state = reopen(state, &config, &universe); + assert_eq!( + dump_store(&state), + clean, + "the startup repair removes the stale pruned-height row" + ); + assert_clean(&state); +} diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs index 858e34c092c..7bca301f624 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs @@ -1020,10 +1020,16 @@ fn header_range_commit_rejects_non_current_anchor_hash() { let block2 = mainnet_block(2); let alternate_block2 = alternate_header(stale_anchor, &block2.header, 1); + // A hash→height entry whose height→hash row names a different block is a + // bijection violation in our own indexes, reported as a local storage + // fault rather than an unknown anchor. Either way the stale anchor cannot + // be committed on. let mut batch = DiskWriteBatch::new(); assert!(matches!( batch.prepare_header_range_batch(&state, stale_anchor, &[alternate_block2], &[0]), - Err(CommitHeaderRangeError::UnknownAnchor { anchor }) if anchor == stale_anchor + Err(CommitHeaderRangeError::StoreIncoherent( + crate::error::StoreIncoherentError::BijectionMismatch { hash, height, stored }, + )) if hash == stale_anchor && height == Height(1) && stored == Some(block1.hash()) )); assert_eq!(state.hash(Height(1)), None); @@ -1246,7 +1252,9 @@ fn synthetic_headers_from_state( ) -> Vec> { let network = state.network(); let template = mainnet_block(1); - let mut context = state.recent_header_context(anchor_height); + let mut context = state + .recent_header_context(anchor_height) + .expect("test store is coherent"); let mut previous_hash = anchor_hash; let mut previous_height = anchor_height; let mut nonce_tag = nonce_seed; diff --git a/zebra-state/src/service/write.rs b/zebra-state/src/service/write.rs index 21c0ac0d974..178a97818d2 100644 --- a/zebra-state/src/service/write.rs +++ b/zebra-state/src/service/write.rs @@ -146,10 +146,16 @@ fn commit_header_range( &tree_aux_roots, ) .and_then(|hash| { + let zakura_replaced_rows = batch.zakura_suffix_replaced_rows(); finalized_state .db .write_batch(batch) - .map(|()| hash) + .map(|()| { + finalized_state + .db + .audit_zakura_header_store_after_reorg(zakura_replaced_rows); + hash + }) .map_err(|error| { tracing::error!(?error, "failed to write validated header range"); @@ -744,7 +750,7 @@ mod tests { use crate::{ arbitrary::Prepare, service::{ - finalized_state::FinalizedState, + finalized_state::{DiskWriteBatch, FinalizedState, WriteDisk}, non_finalized_state::NonFinalizedState, write::{ seed_zakura_header_from_committed_block, @@ -781,6 +787,27 @@ mod tests { let mut non_finalized_state = NonFinalizedState::new(&network); + // The seed path refuses rows that do not link to the stored header row + // below them, and the fake chain's parent block is not otherwise + // committed to this state, so store its hash as a provisional Zakura + // row (the consensus `hash_by_height` row cannot be written alone: a + // finalized tip implies note commitment trees exist). + let parent_height = parent + .coinbase_height() + .expect("test vector block has a coinbase height"); + let zakura_hash_by_height = finalized_state + .db + .db() + .cf_handle("zakura_header_hash_by_height") + .unwrap(); + let mut batch = DiskWriteBatch::new(); + batch.zs_insert(&zakura_hash_by_height, parent_height, parent.hash()); + finalized_state + .db + .db() + .write(batch) + .expect("parent hash row writes"); + non_finalized_state .commit_new_chain(best_block.clone().prepare(), &finalized_state) .expect("best block commits to a new chain"); diff --git a/zebrad/src/commands/start.rs b/zebrad/src/commands/start.rs index e66925402c7..62fa6a78d6f 100644 --- a/zebrad/src/commands/start.rs +++ b/zebrad/src/commands/start.rs @@ -2346,6 +2346,43 @@ mod zakura_header_sync_driver_tests { ); } + #[test] + fn store_incoherent_is_local_header_sync_commit_failure() { + // A range rejected because our own header rows failed a + // linkage/bijection check is a local storage fault; scoring peers for + // it recreates the disconnect-honest-peers failure mode. + let error = zebra_state::CommitHeaderRangeError::StoreIncoherent( + zebra_state::StoreIncoherentError::BrokenLinkage { + height: block::Height(2), + expected_parent: block::Hash([0; 32]), + actual_below: block::Hash([1; 32]), + }, + ); + + assert_eq!( + header_range_commit_failure_kind(&error), + HeaderSyncCommitFailureKind::Local + ); + } + + #[test] + fn unlinked_range_is_local_header_sync_commit_failure() { + // The reactor validates every response's linkage against the requested + // anchor before committing with that anchor, so the store's own + // linkage check failing means local anchor/response pairing broke, + // not peer misbehavior. + let error = zebra_state::CommitHeaderRangeError::UnlinkedRange { + height: block::Height(1), + expected_parent: block::Hash([0; 32]), + actual_parent: block::Hash([1; 32]), + }; + + assert_eq!( + header_range_commit_failure_kind(&error), + HeaderSyncCommitFailureKind::Local + ); + } + #[test] fn served_header_body_size_hints_align_with_served_heights() { let start = block::Height(10); diff --git a/zebrad/src/commands/start/zakura/header_sync_driver.rs b/zebrad/src/commands/start/zakura/header_sync_driver.rs index a1d9ca0bec7..a4d6ad9bb8b 100644 --- a/zebrad/src/commands/start/zakura/header_sync_driver.rs +++ b/zebrad/src/commands/start/zakura/header_sync_driver.rs @@ -1112,6 +1112,18 @@ pub(crate) fn header_range_commit_failure_kind( // fork. Treat it as non-scoring so this stays a liveness/correctness guard, // not peer punishment. | zebra_state::CommitHeaderRangeError::LowerWorkConflict { .. } + // The reactor already validates every peer response against the requested + // anchor and for internal continuity (`validate_header_range_links`) and + // scores linkage failures there, then commits with that same anchor. So the + // store's own linkage check failing means the local anchor/response pairing + // went wrong, not that the peer misbehaved. + | zebra_state::CommitHeaderRangeError::UnlinkedRange { .. } + // Store incoherence is by definition a local storage fault: the range was + // rejected because our own header rows failed a linkage/bijection check + // while reading validation context, not because the peer's range was shown + // invalid. Scoring peers for it recreates the disconnect-honest-peers + // failure mode from the 2026-07-06 incidents. + | zebra_state::CommitHeaderRangeError::StoreIncoherent(_) | zebra_state::CommitHeaderRangeError::CommitResponseDropped => { HeaderSyncCommitFailureKind::Local }