test(state): exercise pruning in the header-store coherence harness

CHANGELOG.md

@@ -18,6 +18,15 @@ and this project adheres to [Semantic Versioning](https://semver.org).
 - Verified-commitment-trees fast sync is now enabled by default when checkpoint
   sync is enabled. Operators can keep checkpoint sync but opt out of the new
   path by setting `consensus.vct_fast_sync = false`.
+- All Zakura header-store writers (header-range commits, best-chain seeds, and
+  body-commit releases) now change chain membership through a single
+  suffix-replacement primitive that deletes everything above the fork point by
+  scanning the actual on-disk rows of all five column families — so rows can no
+  longer be stranded by bounded delete loops — and verifies linkage as a hard
+  precondition. Any batch that replaces a header suffix (a header reorg) is
+  followed by the store coherence audit, repairing and reporting
+  (`state.zakura.header_store.incoherent`) any residual violation at the moment
+  it happens instead of leaving a latent on-disk fault.
 - Refreshed the default Testnet Zakura bootstrap peer identities
   (`DEFAULT_TESTNET_ZAKURA_BOOTSTRAP_PEERS`) after the Testnet fleet's iroh node
   keys were rotated. The previous hardcoded node IDs were stale, so a fresh node
@@ -35,6 +44,33 @@ and this project adheres to [Semantic Versioning](https://semver.org).
 
 ### Fixed
 
+- Zebra now audits the Zakura header store when the state database opens and
+  self-repairs any incoherence (broken linkage, hash↔height index mismatches,
+  gaps with stranded rows above them, stale rows at committed heights) by
+  truncating the Zakura column families to the last coherent height in one
+  atomic batch, then letting header sync re-download the truncated suffix.
+  Stores corrupted by earlier binaries now heal on restart instead of staying
+  wedged below the network tip; each repair emits a warning and the
+  `state.zakura.header_store.incoherent` metric.
+- Zakura header-store consensus reads now verify store invariants as they
+  walk: the difficulty-context walk checks that every stored header is the
+  block its hash row names and links to the row below it, and the anchor
+  lookup distinguishes a corrupted index round-trip from a genuinely unknown
+  anchor. A corrupted store now surfaces as an explicit local
+  `StoreIncoherent` error (never scored against peers) instead of poisoning
+  difficulty validation and rejecting honest headers with
+  `InvalidDifficultyThreshold`.
+- Fixed three Zakura header-store write paths that could leave the on-disk
+  header store internally incoherent after chain forks, causing nodes to
+  reject valid headers from honest peers (`InvalidDifficultyThreshold` /
+  `UnknownAnchor`) and wedge below the network tip until manual intervention.
+  Header ranges must now link to their anchor and be internally contiguous
+  (rejected with the new `UnlinkedRange` error otherwise, which is classified
+  as a local, non-peer-scoring failure), header ranges re-delivered over
+  heights that already have committed block bodies no longer re-insert
+  provisional header rows below the body tip, and the header row seeded from
+  a committed best-chain block is skipped when it does not link to the stored
+  row below it (header-range sync converges the store instead).
 - Fixed dual-stack Zakura fallback shutting down the Zakura serving layer. When
   the stall watchdog resumes legacy `ChainSync`, Zakura now keeps its header-
   and block-sync reactors alive as a serving/advertising bridge while an apply

Cargo.lock

@@ -1349,9 +1349,9 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-epoch"
-version = "0.9.18"
+version = "0.9.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
+checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f"
 dependencies = [
  "crossbeam-utils",
 ]

deny.toml

@@ -13,11 +13,9 @@ yanked = "deny"
 # Remove each ignore entry as the dependency chain is upgraded.
 ignore = [
     "RUSTSEC-2021-0139", # ansi_term — transitive via abscissa_core -> structopt
-    "RUSTSEC-2022-0104", # structopt (maintenance mode) — transitive via abscissa_core
+    "RUSTSEC-2022-0104", # structopt (maintenance mode) — transitive via abscissa_core and zebra-utils
     "RUSTSEC-2024-0375", # atty (unmaintained) — transitive via abscissa_core -> structopt
-    "RUSTSEC-2021-0145", # atty (unsound) — transitive via abscissa_core -> structopt
     "RUSTSEC-2024-0370", # proc-macro-error — transitive via abscissa_core -> structopt
-    "RUSTSEC-2022-0104", # structopt (unmaintained) — direct dep of zebra-utils; no safe upgrade, migrate to clap 4
     "RUSTSEC-2026-0173", # proc-macro-error2 (unmaintained) — transitive via getset
     "RUSTSEC-2025-0119", # number_prefix — transitive via indicatif
     "RUSTSEC-2025-0141", # bincode — direct dependency

zebra-state/src/error.rs

@@ -195,6 +195,68 @@ impl From<CommitHeaderRangeError> for CommitCheckpointVerifiedError {
     }
 }
 
+/// An internal invariant of the zakura header store was found violated while
+/// reading it.
+///
+/// This is a **local storage fault**, never evidence about a peer: readers
+/// return it instead of feeding rows from more than one branch (or from beside
+/// a gap) into consensus validation, where the corruption would otherwise
+/// surface as a misleading validation failure (`InvalidDifficultyThreshold`,
+/// `UnknownAnchor`) attributed to whoever supplied the input being validated.
+#[derive(Debug, Error, Clone, Copy, PartialEq, Eq)]
+#[non_exhaustive]
+pub enum StoreIncoherentError {
+    /// The header row at `height` does not link to the stored row below it.
+    #[error(
+        "header store incoherent: header at {height:?} links to {expected_parent} but the stored row below is {actual_below}"
+    )]
+    BrokenLinkage {
+        /// Height of the header whose parent link failed to resolve.
+        height: block::Height,
+        /// The parent hash the header claims (`previous_block_hash`).
+        expected_parent: block::Hash,
+        /// The hash actually stored at `height - 1`.
+        actual_below: block::Hash,
+    },
+
+    /// A header row exists at `height` but the row below it is missing.
+    #[error(
+        "header store incoherent: no stored row at {missing:?} below the header at {height:?}"
+    )]
+    Gap {
+        /// Height of the stored header above the gap.
+        height: block::Height,
+        /// The missing height (`height - 1`).
+        missing: block::Height,
+    },
+
+    /// The header row at `height` is not the block its hash row names.
+    #[error(
+        "header store incoherent: header stored at {height:?} hashes to {computed} but the hash row names {indexed}"
+    )]
+    HeaderHashMismatch {
+        /// Height of the divergent rows.
+        height: block::Height,
+        /// The hash the height→hash index names.
+        indexed: block::Hash,
+        /// The stored header's actual hash.
+        computed: block::Hash,
+    },
+
+    /// The hash→height and height→hash indexes disagree about a hash.
+    #[error(
+        "header store incoherent: hash {hash} is indexed at {height:?} but that height stores {stored:?}"
+    )]
+    BijectionMismatch {
+        /// The hash whose round-trip failed.
+        hash: block::Hash,
+        /// The height the hash→height index reports for it.
+        height: block::Height,
+        /// What the height→hash index stores there instead.
+        stored: Option<block::Hash>,
+    },
+}
+
 /// An error describing why a header-only range could not be committed.
 #[derive(Debug, Error, Clone, PartialEq, Eq)]
 #[non_exhaustive]
@@ -258,6 +320,21 @@ pub enum CommitHeaderRangeError {
     #[error("header height overflow")]
     HeightOverflow,
 
+    /// A header in the range does not link to the anchor or to its predecessor,
+    /// so committing it would break the header store's linkage invariant.
+    #[error(
+        "header at {height:?} links to {actual_parent} instead of its predecessor {expected_parent}"
+    )]
+    UnlinkedRange {
+        /// Height of the first header that fails to link.
+        height: block::Height,
+        /// The hash of the row the header must link to (the anchor, or the
+        /// previous header in the range).
+        expected_parent: block::Hash,
+        /// The header's actual `previous_block_hash`.
+        actual_parent: block::Hash,
+    },
+
     /// A committed immutable header conflicts with the requested header.
     #[error("header at finalized height {height:?} conflicts with an existing header")]
     ImmutableConflict {
@@ -308,6 +385,15 @@ pub enum CommitHeaderRangeError {
         height: block::Height,
     },
 
+    /// The local header store was found internally incoherent while reading
+    /// the context needed to validate the range.
+    ///
+    /// This is a local storage fault, not a peer validation failure: the range
+    /// was rejected because the store cannot supply trustworthy context, not
+    /// because the range itself was shown invalid.
+    #[error("header store incoherent while validating range: {0}")]
+    StoreIncoherent(#[from] StoreIncoherentError),
+
     /// Contextual header validation failed.
     #[error("could not contextually validate header")]
     ValidateContextError(#[from] Box<ValidateContextError>),

zebra-state/src/lib.rs

@@ -43,7 +43,8 @@ pub use config::{
 pub use constants::{state_database_format_version_in_code, MAX_BLOCK_REORG_HEIGHT};
 pub use error::{
     BoxError, CloneError, CommitBlockError, CommitCheckpointVerifiedError, CommitHeaderRangeError,
-    CommitSemanticallyVerifiedError, DuplicateNullifierError, ValidateContextError,
+    CommitSemanticallyVerifiedError, DuplicateNullifierError, StoreIncoherentError,
+    ValidateContextError,
 };
 pub use request::{
     CheckpointVerifiedBlock, CommitSemanticallyVerifiedBlockRequest, HashOrHeight, MappedRequest,

zebra-state/src/service/finalized_state/disk_db.rs

@@ -122,6 +122,11 @@ pub struct DiskDb {
 pub struct DiskWriteBatch {
     /// The inner RocksDB write batch.
     batch: rocksdb::WriteBatch,
+
+    /// Rows deleted by zakura header-store suffix replacements staged in this
+    /// batch. Nonzero means the batch performs a header reorg; the write site
+    /// uses this to run the post-reorg store audit after committing.
+    zakura_replaced_rows: usize,
 }
 
 impl Debug for DiskWriteBatch {
@@ -548,8 +553,22 @@ impl DiskWriteBatch {
     pub fn new() -> Self {
         DiskWriteBatch {
             batch: rocksdb::WriteBatch::default(),
+            zakura_replaced_rows: 0,
         }
     }
+
+    /// Records that a zakura suffix replacement staged in this batch deleted
+    /// `deleted_rows` rows.
+    pub(crate) fn note_zakura_suffix_replacement(&mut self, deleted_rows: usize) {
+        self.zakura_replaced_rows += deleted_rows;
+    }
+
+    /// Returns the number of rows deleted by zakura suffix replacements
+    /// staged in this batch. Nonzero means this batch performs a header
+    /// reorg, and the write site should audit the store after committing it.
+    pub(crate) fn zakura_suffix_replaced_rows(&self) -> usize {
+        self.zakura_replaced_rows
+    }
 }
 
 impl DiskDb {

zebra-state/src/service/finalized_state/zebra_db.rs

@@ -175,6 +175,17 @@ impl ZebraDb {
             )
         }
 
+        // Audit the zakura header store's on-disk invariants and truncate any
+        // incoherent suffix, so a store corrupted by an earlier binary
+        // self-heals at startup instead of wedging header sync (headers are
+        // re-fetchable, so correctness beats preserved rows). Read-only
+        // instances cannot repair; their reads surface any corruption as
+        // explicit `StoreIncoherentError`s instead.
+        if !read_only {
+            db.audit_and_repair_zakura_header_store()
+                .expect("startup header-store repair write failed: RocksDB is unavailable");
+        }
+
         db.spawn_format_change(format_change);
 
         db