From 63b74cbfbd984db68dc3011b7c6a6915d88e385d Mon Sep 17 00:00:00 2001 From: roman Date: Mon, 6 Jul 2026 16:35:19 -0600 Subject: [PATCH 1/8] fix(state): validate linkage in zakura header-store writers Close the three write-path corruption bugs proven by the Phase-1 header-store coherence suite (PR #490): 1. prepare_header_range_batch_with_roots now rejects ranges whose first header does not link to the anchor or whose headers are not internally contiguous (new CommitHeaderRangeError::UnlinkedRange, classified as a local non-scoring failure in zebrad). 2. The range insert loop skips heights that already have a committed block, so re-deliveries can no longer strand provisional zakura rows below the body tip. 3. prepare_zakura_header_from_committed_block refuses seeds that do not link to the stored row below them as silent no-ops; header-range sync converges the store instead (scenario s11). Flip the four *_upholds_invariants twins to permanent regression gates, delete the corruption_repro_* tests, remove the discovery masking, and un-ignore the random invariant sweep (clean at PROPTEST_CASES=4096). --- CHANGELOG.md | 11 + zebra-state/src/error.rs | 15 + .../service/finalized_state/zebra_db/block.rs | 95 +++++-- .../tests/header_store_coherence/README.md | 91 +++--- .../block/tests/header_store_coherence/ops.rs | 72 ++--- .../tests/header_store_coherence/oracle.rs | 8 +- .../tests/header_store_coherence/prop.rs | 32 +-- .../tests/header_store_coherence/scenarios.rs | 266 ++++++++---------- zebra-state/src/service/write.rs | 23 +- zebrad/src/commands/start.rs | 18 ++ .../start/zakura/header_sync_driver.rs | 6 + 11 files changed, 337 insertions(+), 300 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 679fb4e12cc..96f5da37488 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,17 @@ and this project adheres to [Semantic Versioning](https://semver.org). ### Fixed +- Fixed three Zakura header-store write paths that could leave the on-disk + header store internally incoherent after chain forks, causing nodes to + reject valid headers from honest peers (`InvalidDifficultyThreshold` / + `UnknownAnchor`) and wedge below the network tip until manual intervention. + Header ranges must now link to their anchor and be internally contiguous + (rejected with the new `UnlinkedRange` error otherwise, which is classified + as a local, non-peer-scoring failure), header ranges re-delivered over + heights that already have committed block bodies no longer re-insert + provisional header rows below the body tip, and the header row seeded from + a committed best-chain block is skipped when it does not link to the stored + row below it (header-range sync converges the store instead). - Fixed dual-stack Zakura fallback shutting down the Zakura serving layer. When the stall watchdog resumes legacy `ChainSync`, Zakura now keeps its header- and block-sync reactors alive as a serving/advertising bridge while an apply diff --git a/zebra-state/src/error.rs b/zebra-state/src/error.rs index aaf3f124b5e..84a9720dee8 100644 --- a/zebra-state/src/error.rs +++ b/zebra-state/src/error.rs @@ -258,6 +258,21 @@ pub enum CommitHeaderRangeError { #[error("header height overflow")] HeightOverflow, + /// A header in the range does not link to the anchor or to its predecessor, + /// so committing it would break the header store's linkage invariant. + #[error( + "header at {height:?} links to {actual_parent} instead of its predecessor {expected_parent}" + )] + UnlinkedRange { + /// Height of the first header that fails to link. + height: block::Height, + /// The hash of the row the header must link to (the anchor, or the + /// previous header in the range). + expected_parent: block::Hash, + /// The header's actual `previous_block_hash`. + actual_parent: block::Hash, + }, + /// A committed immutable header conflicts with the requested header. #[error("header at finalized height {height:?} conflicts with an existing header")] ImmutableConflict { diff --git a/zebra-state/src/service/finalized_state/zebra_db/block.rs b/zebra-state/src/service/finalized_state/zebra_db/block.rs index 8feead51841..1557b6fd8dc 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block.rs @@ -1797,6 +1797,12 @@ impl DiskWriteBatch { /// Full block verification is authoritative for the stored body. If a /// provisional Zakura header at this height differs, replace it with the /// block-derived header and drop stale provisional descendants. + /// + /// A block whose parent hash does not match the stored header row below + /// `height` is skipped without error: writing it would leave a gap or + /// broken link in the header store, so the store instead waits for + /// header-range sync to deliver the connecting rows (see the linkage + /// refusal below). #[allow(clippy::unwrap_in_result)] pub fn prepare_zakura_header_from_committed_block( &mut self, @@ -1821,6 +1827,32 @@ impl DiskWriteBatch { return Ok(()); } + // Seeds fire at non-finalized best-*tip* commits, so an nf best-tip + // jump (a fork switch between nf chains, or a restart restoring the nf + // backup) can seed a height whose parent row is missing or belongs to + // another branch. Writing that row would break the header store's + // linkage invariant on disk and poison difficulty-adjustment windows + // (`header_store_coherence` bug 3). Refuse the write instead: the + // header store briefly lags the nf chain, and header-range sync + // converges it onto the new branch through the linkage-checked range + // path. The merged header view (full-block rows first, then zakura + // rows) is the same view the chain walk reads. + let hash_by_height = db.cf_handle("hash_by_height").unwrap(); + let parent_hash: Option = height.previous().ok().and_then(|parent_height| { + db.zs_get(&hash_by_height, &parent_height) + .or_else(|| db.zs_get(&zakura_hash_by_height, &parent_height)) + }); + if parent_hash != Some(block.header.previous_block_hash) { + tracing::debug!( + ?height, + ?hash, + parent = ?block.header.previous_block_hash, + stored_parent = ?parent_hash, + "skipping Zakura header seed that does not link to the stored row below it" + ); + return Ok(()); + } + if existing_zakura_header.is_some_and(|existing_header| existing_header != block.header) { let best_header_tip: Option<(block::Height, block::Hash)> = db.zs_last_key_value(&zakura_hash_by_height); @@ -2031,6 +2063,14 @@ impl DiskWriteBatch { let mut first_conflicting_height = None; let mut validated_headers = Vec::with_capacity(headers.len()); + // Each header must link to the anchor (for the first header) or to its + // predecessor in the range. Without this check, a range anchored at the + // same-height hash of a *different* branch can pass contextual + // difficulty validation and commit a suffix that does not link to the + // row below it — an on-disk linkage violation reachable from a single + // peer response. + let mut expected_parent = anchor; + for (index, header) in headers.iter().enumerate() { let offset = u32::try_from(index + 1).map_err(|_| CommitHeaderRangeError::HeightOverflow)?; @@ -2039,6 +2079,16 @@ impl DiskWriteBatch { let hash = block::Hash::from(&**header); let body_size = body_sizes[index]; let roots = &tree_aux_roots[index]; + + if header.previous_block_hash != expected_parent { + return Err(CommitHeaderRangeError::UnlinkedRange { + height, + expected_parent, + actual_parent: header.previous_block_hash, + }); + } + expected_parent = hash; + if roots.height != height { return Err(CommitHeaderRangeError::TreeAuxRootHeightMismatch { expected_height: height, @@ -2161,6 +2211,14 @@ impl DiskWriteBatch { for (index, (height, hash, header, body_size)) in validated_headers.into_iter().enumerate() { + // Finalized block heights already have authoritative block rows and + // verified roots, even when pruning has removed their transactions. + // Re-delivered headers must not recreate provisional zakura rows + // there, because those rows are only trimmed during body commit. + if zebra_db.contains_height(height) { + continue; + } + let same_header = zebra_db.zakura_header_hash(height) == Some(hash); let advertised_body_size = match ( same_header, @@ -2180,29 +2238,20 @@ impl DiskWriteBatch { } else { self.zs_delete(&body_size_by_height, height); } - // A height with a committed body already has a *verified* row in the - // serving index, written by the body-commit batch. Peer-supplied roots - // are unauthenticated until verify-before-commit, and that verification - // only ever runs above the body tip — so a header range re-delivered - // over committed heights (a header store behind the body store, or a - // late range response racing body sync) must never overwrite the - // verified row: committed roots win on any overlap (design §9). - if !zebra_db.contains_body_at_height(height) { - let roots = &tree_aux_roots[index]; - self.zs_insert( - &roots_by_height, - height, - CommitmentRootsByHeight { - sapling: roots.sapling_root, - orchard: roots.orchard_root, - ironwood: roots.ironwood_root, - sapling_tx: roots.sapling_tx, - orchard_tx: roots.orchard_tx, - ironwood_tx: roots.ironwood_tx, - auth_data_root: roots.auth_data_root, - }, - ); - } + let roots = &tree_aux_roots[index]; + self.zs_insert( + &roots_by_height, + height, + CommitmentRootsByHeight { + sapling: roots.sapling_root, + orchard: roots.orchard_root, + ironwood: roots.ironwood_root, + sapling_tx: roots.sapling_tx, + orchard_tx: roots.orchard_tx, + ironwood_tx: roots.ironwood_tx, + auth_data_root: roots.auth_data_root, + }, + ); } Ok(block::Hash::from( diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/README.md b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/README.md index d23a5205a0b..4b49718607a 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/README.md +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/README.md @@ -34,65 +34,64 @@ write instead. | `audit.rs` | The invariant audit: A1 (bijection, both directions), A2 (linkage walk upward from the finalized tip over the merged header view), A3 (tip integrity, gaps, frontier overlay, aux-row backing), and A4 (the on-disk chain equals the model's expected canonical chain). Also `dump_store`, a comparable snapshot of all five column families used to assert that rejected commits are side-effect free and that reopens preserve the store byte-for-byte. | | `oracle.rs` | An in-memory model of the store's _specified_ behavior: a single linked canonical chain, best-cumulative-work selection with strict improvement, total suffix replacement above the first conflicting height, and a sequential body tip. It predicts whether each op must be accepted or rejected. | | `ops.rs` | The op alphabet, each op mapped to one real production write-batch shape: `CommitHeaderRange` → `prepare_header_range_batch_with_roots`; `CommitBody` / `Finalize` → `prepare_block_header_and_transaction_data_batch` plus the finalization roots delete (which runs the release path internally); `Seed` → `seed_zakura_header_from_committed_block` (the non-finalized best-chain commit hook); `Reopen` → shutdown and reopen of the persistent store. The `Harness` executes ops, cross-checks the oracle's prediction against the store's response, and audits after every mutation; failures come back as a transcribable `FailureReport` (the executed op prefix plus every violation found). | -| `scenarios.rs` | Scripted production event shapes (s01–s10): simple reorgs, lower-work rejections and their later reversal, split-range and walk-back deliveries, body commits racing header reorgs, reorgs to a lower height, double reorgs at one fork point, activity across the difficulty-adjustment window edge, restarts at every boundary, and seed/range interplay. Also holds the `corruption_repro_*` tests below. | -| `prop.rs` | Discovery proptests: random op sequences over the fixed universe, shrunk to minimal counterexamples on any audit failure. Both are `#[ignore]`d because the store has known bugs today; see "Running". | - -## Known corruption bugs pinned by this suite - -Each bug is pinned by a pair of tests: `corruption_repro_` **passes -today** and deterministically demonstrates the violation, and -`_upholds_invariants` is `#[ignore]`d and asserts the correct behavior. -When the write path is fixed, the repro fails loudly (forcing re-triage) and -the twin gets un-ignored as the permanent regression test. - -1. **Unlinked-anchor commit** (`corruption_repro_unlinked_anchor_commit`). - `prepare_header_range_batch_with_roots` never checks that - `headers[0].previous_block_hash == anchor`, nor any intra-range linkage. A - range anchored at a same-height hash of a different branch passes - difficulty validation and commits a suffix that does not link to the row - below it — an on-disk I2 violation reachable from a single untrusted peer - response. +| `scenarios.rs` | Scripted production event shapes (s01–s11): simple reorgs, lower-work rejections and their later reversal, split-range and walk-back deliveries, body commits racing header reorgs, reorgs to a lower height, double reorgs at one fork point, activity across the difficulty-adjustment window edge, restarts at every boundary, seed/range interplay, and refused-seed convergence. Also holds the `*_upholds_invariants` regression gates below. | +| `prop.rs` | The permanent random sweep: random op sequences over the fixed universe, shrunk to minimal counterexamples on any audit failure. | + +## Fixed corruption bugs gated by this suite + +Phase 1 of this suite proved three write-path corruption bug classes, each +pinned by a `corruption_repro_*` test that deterministically demonstrated the +violation. The write-path fixes (`REORG_PLAN.md` Phase 1.5) closed all three; +the repro tests were removed with the fixes, and their +`_upholds_invariants` twins now assert the fixed behavior over the same +op sequences as permanent regression gates: + +1. **Unlinked-anchor commit** (`unlinked_anchor_commit_upholds_invariants`). + `prepare_header_range_batch_with_roots` used to accept ranges without + checking that `headers[0].previous_block_hash == anchor` or any intra-range + linkage, so a range anchored at a same-height hash of a different branch + could pass difficulty validation and commit a suffix that did not link to + the row below it — an on-disk I2 violation reachable from a single + untrusted peer response. The writer now rejects such ranges with + `CommitHeaderRangeError::UnlinkedRange`. 2. **Re-delivery over committed bodies** - (`corruption_repro_redelivery_over_bodies`). The range insert loop gates - only its _roots_ write on `contains_body_at_height`; the - header/hash/height/body-size writes are unconditional. A header range - re-delivered over heights whose bodies were committed in the meantime - re-inserts zakura rows below the body tip, and nothing ever trims them - again (the release trim already ran at body-commit time) — a permanent I3 - frontier-overlay violation. -3. **Unlinked seed** (`corruption_repro_seed_above_gap`, - `corruption_repro_seed_fork_switch`; found by the proptest and shrunk to a - single op). `prepare_zakura_header_from_committed_block` writes its row - with no linkage or anchor precondition. Seeds fire only at non-finalized + (`redelivery_over_bodies_upholds_invariants`). The range insert loop used + to gate only its _roots_ write on `contains_body_at_height`, so a header + range re-delivered over heights whose bodies were committed in the + meantime re-inserted zakura rows below the body tip that nothing ever + trims again (the release trim already ran at body-commit time) — a + permanent I3 frontier-overlay violation. The gate now covers every zakura + row write: heights that already have a committed block are skipped + entirely (checked via `contains_height`, so it also holds for pruned + heights whose bodies are gone but whose authoritative rows remain). +3. **Unlinked seed** (`seed_above_gap_upholds_invariants`, + `seed_fork_switch_upholds_invariants`; found by the proptest and shrunk to + a single op). `prepare_zakura_header_from_committed_block` used to write + its row with no linkage precondition. Seeds fire only at non-finalized best-_tip_ commits, so any best-tip jump (a fork switch between non-finalized chains, or a restart that restores the non-finalized backup) - seeds a height whose parent row is missing or belongs to another branch: - a gap or broken link on disk, and a generator of poisoned - difficulty-adjustment windows. - -A discovery sweep of 2048 random sequences with these three shapes excluded -found no further violation class at that depth. + seeded a height whose parent row was missing or belonged to another + branch: a gap or broken link on disk, and a generator of poisoned + difficulty-adjustment windows. The seed path now refuses a seed that does + not link to the stored row below it as a silent no-op — the header store + briefly lags the non-finalized chain, and header-range sync converges it + (`s11_refused_seed_converges_via_range_delivery`). ## Running ```bash -# The whole suite (scripted scenarios, audits, corruption repros): +# The whole suite (scripted scenarios, audits, regression gates, random sweep): cargo test -p zebra-state --lib header_store_coherence -# Discovery sweep beyond the known bugs (any failure = a NEW violation class): +# The random sweep at discovery depth (any failure = a NEW violation class): PROPTEST_CASES=4096 cargo test -p zebra-state --lib \ - header_store_coherence::prop::prop_discovery -- --ignored - -# The full random sweep (fails today on the known shapes; becomes a permanent -# regression gate once the write paths are fixed): -cargo test -p zebra-state --lib \ - header_store_coherence::prop::prop_random_sequences -- --ignored + header_store_coherence::prop ``` Shrunk proptest counterexamples should be transcribed into `scenarios.rs` as -hardcoded `corruption_repro_*`/`#[ignore]` pairs (seed-independent pinning); -the file under `zebra-state/proptest-regressions/` pins the seeds as a -backstop and is checked in. +hardcoded scenarios (seed-independent pinning); the file under +`zebra-state/proptest-regressions/` pins the seeds as a backstop and is +checked in. ## Scope diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/ops.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/ops.rs index 05480931678..ad36b6556d1 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/ops.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/ops.rs @@ -109,13 +109,16 @@ impl OpOutcome { } /// Why a sequence failed: the executed op prefix (last op is the failing one) -/// plus everything found after it. +/// plus everything found after it. All fields are diagnostic payloads rendered +/// through `Debug` in test failure output. #[derive(Debug)] pub(crate) struct FailureReport { - /// The transcribable op sequence, rendered through `Debug` in reports. + /// The transcribable op sequence. #[allow(dead_code)] pub executed: Vec, + #[allow(dead_code)] pub violations: Vec, + #[allow(dead_code)] pub mismatches: Vec, } @@ -132,10 +135,6 @@ pub(crate) struct Harness { config: Config, state: Option, executed: Vec, - /// Skip the op shapes that trigger the *known* corruption bugs (the - /// `corruption_repro_*` scenarios), so proptest discovery can search for - /// new violation classes instead of rediscovering the known ones. - avoid_known_corruptions: bool, _tempdir: tempfile::TempDir, } @@ -157,19 +156,10 @@ impl Harness { config, state: Some(state), executed: Vec::new(), - avoid_known_corruptions: false, _tempdir: tempdir, } } - /// A harness that skips the known corruption shapes (see - /// [`Harness::avoid_known_corruptions`]). - pub fn new_avoiding_known_corruptions() -> Self { - let mut harness = Self::new(); - harness.avoid_known_corruptions = true; - harness - } - pub fn state(&self) -> &ZebraDb { self.state .as_ref() @@ -242,32 +232,6 @@ impl Harness { let Some(range) = self.resolve_range(*source, *offset, *len, *anchor) else { return self.finish(OpOutcome::Skipped("range out of bounds"), mismatches); }; - if self.avoid_known_corruptions { - // Known bug 1: unlinked-anchor ranges are accepted instead - // of rejected (corruption_repro_unlinked_anchor_commit). - if matches!( - self.oracle.predict_header_range(&range), - Prediction::Reject(super::oracle::RejectKind::Malformed) - ) { - return self.finish( - OpOutcome::Skipped("known corruption shape: unlinked anchor"), - mismatches, - ); - } - // Known bug 2: accepted re-deliveries touching body-backed - // heights strand zakura rows below the body tip - // (corruption_repro_redelivery_over_bodies). - if range - .rows - .iter() - .any(|row| row.height <= self.oracle.body_tip()) - { - return self.finish( - OpOutcome::Skipped("known corruption shape: re-delivery over bodies"), - mismatches, - ); - } - } match self.oracle.predict_header_range(&range) { Prediction::Skip(reason) => { return self.finish(OpOutcome::Skipped(reason), mismatches) @@ -337,27 +301,33 @@ impl Harness { let Some(fab) = rows.get(*index).cloned() else { return self.finish(OpOutcome::Skipped("seed index out of bounds"), mismatches); }; - // Known bug 3: a seed whose parent is not the committed row - // below it writes an unlinkable row - // (corruption_repro_seed_above_gap / _seed_fork_switch). - if self.avoid_known_corruptions && !self.oracle.seed_is_parent_linked(&fab) { - return self.finish( - OpOutcome::Skipped("known corruption shape: unlinked seed"), - mismatches, - ); - } match self.oracle.predict_seed(&fab) { Prediction::Skip(reason) => { return self.finish(OpOutcome::Skipped(reason), mismatches) } _ => { + // A seed whose parent is not the stored row below it is + // refused by the store as a silent no-op (the header + // store briefly lags the non-finalized chain until a + // linkage-checked header range converges it), so the + // call must succeed without mutating anything. + let parent_linked = self.oracle.seed_is_parent_linked(&fab); + let dump_before = (!parent_linked).then(|| dump_store(self.state())); let block = fabricate_body(&fab); match self .state() .seed_zakura_header_from_committed_block(fab.height, &block) { Ok(()) => { - self.oracle.apply_seed(&fab); + if parent_linked { + self.oracle.apply_seed(&fab); + } else if dump_store(self.state()) + != dump_before.expect("dumped before an unlinked seed") + { + mismatches.push(format!( + "refused unlinked seed mutated the store: {op:?}" + )); + } OpOutcome::Accepted } Err(error) => { diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/oracle.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/oracle.rs index d28872f3055..574b695afc6 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/oracle.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/oracle.rs @@ -40,7 +40,7 @@ pub(crate) enum RejectKind { /// The conflicting suffix does not carry strictly more work. LowerWork, /// The range is malformed (wrong anchor linkage or heights); the store is - /// expected to reject it through contextual validation. + /// expected to reject it through its linkage check (`UnlinkedRange`). Malformed, } @@ -248,8 +248,10 @@ impl Oracle { } /// Whether a seeded block's parent is the expected canonical row below it. - /// Seeds that are not parent-linked are the known seed-path corruption - /// shape (they write a row the store cannot link). + /// The store refuses seeds that are not parent-linked as silent no-ops + /// (writing them would strand a row the chain walk cannot reach); the + /// harness uses this to decide whether a successful seed call must have + /// mutated the store or left it untouched. pub fn seed_is_parent_linked(&self, fab: &FabHeader) -> bool { let parent_height = Height(fab.height.0 - 1); if parent_height == Height(0) { diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/prop.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/prop.rs index e867768b56e..4a363411354 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/prop.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/prop.rs @@ -1,22 +1,17 @@ //! Discovery proptest: random op sequences over the fixed universe, shrunk on //! any audit failure to a minimal, transcribable counterexample. //! -//! Both tests are `#[ignore]`d: the store has known corruption bugs today (the -//! `corruption_repro_*` scenarios), so an always-on random sweep cannot be a CI -//! gate yet. Run discovery manually: +//! The sweep runs 64 cases in CI by default; widen it manually with: //! //! ```text //! PROPTEST_CASES=4096 cargo test -p zebra-state --lib \ -//! header_store_coherence::prop -- --ignored --nocapture +//! header_store_coherence::prop -- --nocapture //! ``` //! //! Every shrunk counterexample must be transcribed into `scenarios.rs` as a -//! hardcoded `corruption_repro_*` / `#[ignore]`d-invariant pair (the primary, -//! seed-independent pinning mechanism); the proptest regression file under -//! `zebra-state/proptest-regressions/` pins the seed as a backstop. -//! -//! When the write-path fixes land, un-ignore -//! `prop_random_sequences_uphold_invariants` as a permanent regression sweep. +//! hardcoded scenario (the primary, seed-independent pinning mechanism); the +//! proptest regression file under `zebra-state/proptest-regressions/` pins +//! the seed as a backstop. use std::env; @@ -81,10 +76,8 @@ fn proptest_cases() -> u32 { proptest! { #![proptest_config(ProptestConfig::with_cases(proptest_cases()))] - /// The full invariant sweep. Fails today on the known corruption shapes; - /// becomes the permanent regression gate once the write paths are fixed. + /// The permanent invariant sweep: any failure is a store-corruption bug. #[test] - #[ignore = "known zakura header-store corruption: un-ignore with the write-path fix"] fn prop_random_sequences_uphold_invariants(ops in ops_strategy()) { let _init_guard = zebra_test::init(); let mut harness = Harness::new(); @@ -92,17 +85,4 @@ proptest! { prop_assert!(false, "store invariants violated:\n{report:#?}"); } } - - /// Discovery beyond the known bugs: the harness skips the known - /// corruption shapes, so any failure here is a *new* violation class. - /// Run manually with a large PROPTEST_CASES while the known bugs are open. - #[test] - #[ignore = "discovery sweep: run manually"] - fn prop_discovery_avoiding_known_corruptions(ops in ops_strategy()) { - let _init_guard = zebra_test::init(); - let mut harness = Harness::new_avoiding_known_corruptions(); - if let Err(report) = harness.run_all(&ops) { - prop_assert!(false, "new store-corruption class found:\n{report:#?}"); - } - } } diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/scenarios.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/scenarios.rs index 8515937c34c..6e41141a72b 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/scenarios.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/scenarios.rs @@ -4,16 +4,14 @@ //! passing scenario is a regression gate on the whole write sequence, not //! just its final assertions. //! -//! The `corruption_repro_*` tests at the bottom deliberately demonstrate -//! store-invariant violations that exist today: they PASS while the bug -//! exists and fail loudly once the write path is fixed, forcing re-triage. -//! Their `#[ignore]`d twins assert the true invariant, to be un-ignored -//! together with the write-path fix. +//! The `*_upholds_invariants` tests at the bottom are the permanent +//! regression gates for the three write-path corruption bugs this suite +//! originally proved with `corruption_repro_*` twins (removed together with +//! the write-path fixes; see the module README for the bug histories). use zebra_chain::block::Height; use super::{ - audit::Violation, fabricate::{BRANCH_A, BRANCH_B, BRANCH_B_EXT, BRANCH_C, FORK_HEIGHT, TRUNK_LEN}, ops::{universe, Anchor, Harness, Op, OpOutcome, Source}, }; @@ -300,7 +298,7 @@ fn s04_body_commit_racing_header_reorg() { /// s05: the release path's frontier trim. Sequential body commits release /// matching provisional rows one by one; header re-delivery *above* the body /// tip is idempotent; restarts change nothing. (Re-delivery *over* body-backed -/// heights strands rows — that is `corruption_repro_redelivery_over_bodies`.) +/// heights skips those heights — `redelivery_over_bodies_upholds_invariants`.) #[test] fn s05_release_trim_then_redelivery() { let _init_guard = zebra_test::init(); @@ -541,19 +539,73 @@ fn s10_seed_interplay() { ); } +/// s11: a refused (unlinked) seed converges through header-range sync. The +/// zakura store follows branch A (seeded at the fork height); the +/// non-finalized best chain switches to branch B and its new best tip (B's +/// *second* row) is seeded. The store refuses the unlinked seed as a no-op — +/// the header store briefly lags the nf chain — and the later linked range +/// delivery of B converges it onto the new branch. +#[test] +fn s11_refused_seed_converges_via_range_delivery() { + let _init_guard = zebra_test::init(); + let mut harness = Harness::new(); + + let outcomes = harness + .run_all(&[ + commit_trunk(), + Op::Seed { + source: Source::Branch(BRANCH_A), + index: 0, + }, + ]) + .expect("setup is clean"); + outcomes.iter().for_each(assert_accepted); + let a_first = &universe().branches[BRANCH_A].headers[0]; + assert_eq!( + harness.state().best_header_tip(), + Some((a_first.height, a_first.hash)), + ); + + // The nf best tip jumps to B[1]; its parent row is not stored, so the + // seed is refused without touching the store (checked by the harness). + let outcome = harness + .run(&Op::Seed { + source: Source::Branch(BRANCH_B), + index: 1, + }) + .expect("the refused seed leaves the store coherent"); + assert_accepted(&outcome); + assert_eq!( + harness.state().best_header_tip(), + Some((a_first.height, a_first.hash)), + "the refused seed must not move the header tip", + ); + + // Header sync catches up with the nf chain: B arrives as a linked range + // and out-works the stored A suffix, converging the store onto B. + let outcome = harness + .run(&commit_branch(BRANCH_B)) + .expect("the converging range delivery is clean"); + assert_accepted(&outcome); + assert_eq!( + harness.state().best_header_tip(), + Some(branch_tip(BRANCH_B)), + ); +} + // --------------------------------------------------------------------------- -// Corruption reproductions: deterministic sequences that violate the store -// invariants today. See the module doc for the pass/ignore convention. +// Regression gates for the three proven write-path corruption bugs. Each was +// originally pinned by a `corruption_repro_*` test that demonstrated the +// violation; those were removed with the write-path fixes, and these twins +// now assert the fixed behavior over the exact same op sequences. // --------------------------------------------------------------------------- -/// The unlinked-anchor commit: `prepare_header_range_batch_with_roots` never -/// checks that `headers[0].previous_block_hash == anchor` (or any intra-range -/// linkage). A range of branch-A headers anchored at the same-height *trunk* -/// hash passes contextual difficulty validation — the two fast chains have -/// identical (time, threshold) sequences — and commits a suffix that does not -/// link to the row below it: an on-disk I2 violation. Production reach: a -/// header range is untrusted peer input; nothing upstream of the store -/// re-checks the anchor linkage. +/// The unlinked-anchor commit (bug 1): a range of branch-A headers anchored at +/// the same-height *trunk* hash passes contextual difficulty validation — the +/// two fast chains have identical (time, threshold) sequences — so before the +/// linkage check in `prepare_header_range_batch_with_roots`, it committed a +/// suffix that did not link to the row below it: an on-disk I2 violation +/// reachable from a single untrusted peer response. fn unlinked_anchor_ops() -> Vec { vec![ commit_trunk(), @@ -567,58 +619,36 @@ fn unlinked_anchor_ops() -> Vec { ] } -/// PASSES while the bug exists (proves the repro); flips when the writer is -/// fixed, forcing this pair to be re-triaged. +/// The store rejects unlinked ranges with `UnlinkedRange` and stays coherent. #[test] -fn corruption_repro_unlinked_anchor_commit() { - let _init_guard = zebra_test::init(); - let mut harness = Harness::new(); - - let report = harness - .run_all(&unlinked_anchor_ops()) - .expect_err("the unlinked-anchor commit corrupts the store today"); - - assert!( - report - .violations - .iter() - .any(|violation| matches!(violation, Violation::BrokenLinkage { height, .. } if *height == Height(FORK_HEIGHT + 2))), - "expected BrokenLinkage right above the spliced anchor: {report:?}" - ); - assert!( - !report.mismatches.is_empty(), - "the oracle rejects this range; the store accepted it: {report:?}" - ); -} - -/// The true invariant. Un-ignore when the write-path fix lands. -#[test] -#[ignore = "known zakura header-store corruption: un-ignore with the write-path fix"] fn unlinked_anchor_commit_upholds_invariants() { let _init_guard = zebra_test::init(); let mut harness = Harness::new(); - harness + let outcomes = harness .run_all(&unlinked_anchor_ops()) .expect("the store rejects unlinked ranges and stays coherent"); + assert!( + matches!( + outcomes[1].header_range_error(), + CommitHeaderRangeError::UnlinkedRange { .. } + ), + "expected UnlinkedRange, got {:?}", + outcomes[1] + ); } -/// Re-delivery over committed bodies: the range insert loop -/// (`prepare_header_range_batch_with_roots`) gates only its *roots* write on -/// `contains_body_at_height` — the header/hash/height/body-size writes are -/// unconditional. A header range re-delivered over heights whose bodies have -/// since been committed (a header store behind the body store, or a late range -/// response racing body sync — the exact scenario the roots gate's own comment -/// describes) re-inserts zakura rows *below* the body tip. That breaks the -/// frontier-overlay invariant ("the Zakura header store only ever holds -/// heights with no committed body", block.rs release-path doc): the release -/// trim already ran at body-commit time, so nothing ever removes these rows. +/// Re-delivery over committed bodies (bug 2): before the committed-height +/// gate covered the header/hash/height/body-size writes (it originally gated +/// only the *roots* write), a header range re-delivered over heights whose +/// bodies had since been committed re-inserted zakura rows *below* the body +/// tip — rows the release trim (which already ran at body-commit time) never +/// removes. fn redelivery_over_bodies_ops() -> Vec { vec![ commit_trunk(), Op::Finalize { count: 10 }, // Re-delivery spanning body-backed heights 6..=10 and header-only - // heights above. The rows match the canonical chain, so the store - // accepts — but the write strands zakura rows under the body tip. + // heights above: accepted, but the body-backed heights are skipped. Op::CommitHeaderRange { source: Source::Trunk, offset: 5, @@ -628,50 +658,21 @@ fn redelivery_over_bodies_ops() -> Vec { ] } -/// PASSES while the bug exists (proves the repro); flips when the writer is -/// fixed, forcing this pair to be re-triaged. +/// Re-delivery over bodies leaves no zakura rows below the body tip. #[test] -fn corruption_repro_redelivery_over_bodies() { - let _init_guard = zebra_test::init(); - let mut harness = Harness::new(); - - let report = harness - .run_all(&redelivery_over_bodies_ops()) - .expect_err("re-delivery over committed bodies strands zakura rows today"); - - assert!( - report - .violations - .iter() - .any(|violation| matches!(violation, Violation::ZakuraRowAtBodyHeight { height, .. } if *height <= Height(10))), - "expected zakura rows stranded at body-backed heights: {report:?}" - ); - assert!( - report.mismatches.is_empty(), - "acceptance itself is correct chain selection; the write effects are the bug: {report:?}" - ); -} - -/// The true invariant. Un-ignore when the write-path fix lands. -#[test] -#[ignore = "known zakura header-store corruption: un-ignore with the write-path fix"] fn redelivery_over_bodies_upholds_invariants() { let _init_guard = zebra_test::init(); let mut harness = Harness::new(); - harness + let outcomes = harness .run_all(&redelivery_over_bodies_ops()) .expect("re-delivery over bodies leaves no zakura rows below the body tip"); + outcomes.iter().for_each(assert_accepted); } -/// Seed above a gap — found by `prop_random_sequences_uphold_invariants` and -/// shrunk to a single op. The seed path -/// (`prepare_zakura_header_from_committed_block`) writes header/hash/height -/// rows at its height with **no linkage or anchor precondition**: seeding a -/// block whose parent row is absent leaves a row the chain walk cannot reach. -/// Production shape: seeds fire only at non-finalized best-*tip* commits, so -/// any nf best-tip jump — a fork switch between nf chains, or a restart that -/// restores the nf backup and then commits on top — seeds a height whose -/// parent row was never written. +/// Seed above a gap (bug 3, minimal shape — found by the discovery proptest +/// and shrunk to a single op): before the parent-linkage refusal in +/// `prepare_zakura_header_from_committed_block`, seeding a block whose parent +/// row is absent left a row the chain walk cannot reach. fn seed_above_gap_ops() -> Vec { vec![Op::Seed { source: Source::Trunk, @@ -679,51 +680,30 @@ fn seed_above_gap_ops() -> Vec { }] } -/// PASSES while the bug exists (proves the repro); flips when the writer is -/// fixed, forcing this pair to be re-triaged. -#[test] -fn corruption_repro_seed_above_gap() { - let _init_guard = zebra_test::init(); - let mut harness = Harness::new(); - - let report = harness - .run_all(&seed_above_gap_ops()) - .expect_err("seeding above a gap strands an unreachable row today"); - - assert!( - report - .violations - .iter() - .any(|violation| matches!(violation, Violation::RowAboveLastLinked { height, .. } if *height == Height(2))), - "expected the seeded row to be unreachable from the finalized tip: {report:?}" - ); - assert!( - report - .violations - .iter() - .any(|violation| matches!(violation, Violation::BestHeaderTipMismatch { .. })), - "expected best_header_tip to point above the linked chain: {report:?}" - ); -} - -/// The true invariant. Un-ignore when the write-path fix lands. +/// A seed above a gap is refused as a no-op instead of stranding a row. #[test] -#[ignore = "known zakura header-store corruption: un-ignore with the write-path fix"] fn seed_above_gap_upholds_invariants() { let _init_guard = zebra_test::init(); let mut harness = Harness::new(); harness .run_all(&seed_above_gap_ops()) .expect("a seed above a gap must not strand an unreachable row"); + + let genesis_hash = universe().genesis.hash(); + assert_eq!( + harness.state().best_header_tip(), + Some((Height(0), genesis_hash)), + "the refused seed must not move the header tip", + ); } -/// Seed fork switch — the production-shaped variant of the seed linkage hole. -/// The zakura store follows branch A (seeded at the fork height); the -/// non-finalized best chain switches to branch B and its new best tip (B's -/// *second* row) is seeded. The seed's non-conflict arm inserts the row -/// directly — B's parent row was truncated, so the store now holds -/// `A[0]` at the fork height and `B[1]` right above it: broken linkage on -/// disk, exactly the poisoned-DAA-window generator from the incident table. +/// Seed fork switch (bug 3, production shape): the zakura store follows +/// branch A; the non-finalized best chain switches to branch B and its new +/// best tip (B's *second* row) is seeded. Before the parent-linkage refusal, +/// the seed's non-conflict arm inserted the row directly over A's truncated +/// parent — broken linkage on disk, the poisoned-DAA-window generator from +/// the production incident table. (`s11` proves the refused seed converges +/// later through range delivery.) fn seed_fork_switch_ops() -> Vec { vec![ commit_trunk(), @@ -738,33 +718,19 @@ fn seed_fork_switch_ops() -> Vec { ] } -/// PASSES while the bug exists (proves the repro); flips when the writer is -/// fixed, forcing this pair to be re-triaged. +/// A fork-switch seed keeps the store linked (the unlinked seed is refused). #[test] -fn corruption_repro_seed_fork_switch() { - let _init_guard = zebra_test::init(); - let mut harness = Harness::new(); - - let report = harness - .run_all(&seed_fork_switch_ops()) - .expect_err("a fork-switch seed breaks linkage today"); - - assert!( - report - .violations - .iter() - .any(|violation| matches!(violation, Violation::BrokenLinkage { height, .. } if *height == Height(FORK_HEIGHT + 2))), - "expected broken linkage right above the fork height: {report:?}" - ); -} - -/// The true invariant. Un-ignore when the write-path fix lands. -#[test] -#[ignore = "known zakura header-store corruption: un-ignore with the write-path fix"] fn seed_fork_switch_upholds_invariants() { let _init_guard = zebra_test::init(); let mut harness = Harness::new(); harness .run_all(&seed_fork_switch_ops()) .expect("a fork-switch seed must keep the store linked"); + + let a_first = &universe().branches[BRANCH_A].headers[0]; + assert_eq!( + harness.state().best_header_tip(), + Some((a_first.height, a_first.hash)), + "the store stays on A until a linked delivery of B arrives", + ); } diff --git a/zebra-state/src/service/write.rs b/zebra-state/src/service/write.rs index 21c0ac0d974..0452d20e642 100644 --- a/zebra-state/src/service/write.rs +++ b/zebra-state/src/service/write.rs @@ -744,7 +744,7 @@ mod tests { use crate::{ arbitrary::Prepare, service::{ - finalized_state::FinalizedState, + finalized_state::{DiskWriteBatch, FinalizedState, WriteDisk}, non_finalized_state::NonFinalizedState, write::{ seed_zakura_header_from_committed_block, @@ -781,6 +781,27 @@ mod tests { let mut non_finalized_state = NonFinalizedState::new(&network); + // The seed path refuses rows that do not link to the stored header row + // below them, and the fake chain's parent block is not otherwise + // committed to this state, so store its hash as a provisional Zakura + // row (the consensus `hash_by_height` row cannot be written alone: a + // finalized tip implies note commitment trees exist). + let parent_height = parent + .coinbase_height() + .expect("test vector block has a coinbase height"); + let zakura_hash_by_height = finalized_state + .db + .db() + .cf_handle("zakura_header_hash_by_height") + .unwrap(); + let mut batch = DiskWriteBatch::new(); + batch.zs_insert(&zakura_hash_by_height, parent_height, parent.hash()); + finalized_state + .db + .db() + .write(batch) + .expect("parent hash row writes"); + non_finalized_state .commit_new_chain(best_block.clone().prepare(), &finalized_state) .expect("best block commits to a new chain"); diff --git a/zebrad/src/commands/start.rs b/zebrad/src/commands/start.rs index e66925402c7..fe94936d94d 100644 --- a/zebrad/src/commands/start.rs +++ b/zebrad/src/commands/start.rs @@ -2346,6 +2346,24 @@ mod zakura_header_sync_driver_tests { ); } + #[test] + fn unlinked_range_is_local_header_sync_commit_failure() { + // The reactor validates every response's linkage against the requested + // anchor before committing with that anchor, so the store's own + // linkage check failing means local anchor/response pairing broke, + // not peer misbehavior. + let error = zebra_state::CommitHeaderRangeError::UnlinkedRange { + height: block::Height(1), + expected_parent: block::Hash([0; 32]), + actual_parent: block::Hash([1; 32]), + }; + + assert_eq!( + header_range_commit_failure_kind(&error), + HeaderSyncCommitFailureKind::Local + ); + } + #[test] fn served_header_body_size_hints_align_with_served_heights() { let start = block::Height(10); diff --git a/zebrad/src/commands/start/zakura/header_sync_driver.rs b/zebrad/src/commands/start/zakura/header_sync_driver.rs index a1d9ca0bec7..a3e55b9f899 100644 --- a/zebrad/src/commands/start/zakura/header_sync_driver.rs +++ b/zebrad/src/commands/start/zakura/header_sync_driver.rs @@ -1112,6 +1112,12 @@ pub(crate) fn header_range_commit_failure_kind( // fork. Treat it as non-scoring so this stays a liveness/correctness guard, // not peer punishment. | zebra_state::CommitHeaderRangeError::LowerWorkConflict { .. } + // The reactor already validates every peer response against the requested + // anchor and for internal continuity (`validate_header_range_links`) and + // scores linkage failures there, then commits with that same anchor. So the + // store's own linkage check failing means the local anchor/response pairing + // went wrong, not that the peer misbehaved. + | zebra_state::CommitHeaderRangeError::UnlinkedRange { .. } | zebra_state::CommitHeaderRangeError::CommitResponseDropped => { HeaderSyncCommitFailureKind::Local } From 4a1600ca4f8bfb7372ced8d6e2b3c9f19b173702 Mon Sep 17 00:00:00 2001 From: roman Date: Mon, 6 Jul 2026 16:43:57 -0600 Subject: [PATCH 2/8] lint --- Cargo.lock | 4 ++-- deny.toml | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5b397748cff..dc19b1b4066 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1349,9 +1349,9 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.18" +version = "0.9.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" dependencies = [ "crossbeam-utils", ] diff --git a/deny.toml b/deny.toml index 3ee845cc4e6..1261b6ea9e9 100644 --- a/deny.toml +++ b/deny.toml @@ -13,11 +13,9 @@ yanked = "deny" # Remove each ignore entry as the dependency chain is upgraded. ignore = [ "RUSTSEC-2021-0139", # ansi_term — transitive via abscissa_core -> structopt - "RUSTSEC-2022-0104", # structopt (maintenance mode) — transitive via abscissa_core + "RUSTSEC-2022-0104", # structopt (maintenance mode) — transitive via abscissa_core and zebra-utils "RUSTSEC-2024-0375", # atty (unmaintained) — transitive via abscissa_core -> structopt - "RUSTSEC-2021-0145", # atty (unsound) — transitive via abscissa_core -> structopt "RUSTSEC-2024-0370", # proc-macro-error — transitive via abscissa_core -> structopt - "RUSTSEC-2022-0104", # structopt (unmaintained) — direct dep of zebra-utils; no safe upgrade, migrate to clap 4 "RUSTSEC-2026-0173", # proc-macro-error2 (unmaintained) — transitive via getset "RUSTSEC-2025-0119", # number_prefix — transitive via indicatif "RUSTSEC-2025-0141", # bincode — direct dependency From 93d3456b323b50cc6609936707b31bbea0746b16 Mon Sep 17 00:00:00 2001 From: roman Date: Mon, 6 Jul 2026 16:49:20 -0600 Subject: [PATCH 3/8] simplify comment --- .../src/service/finalized_state/zebra_db/block.rs | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/zebra-state/src/service/finalized_state/zebra_db/block.rs b/zebra-state/src/service/finalized_state/zebra_db/block.rs index 1557b6fd8dc..d722bb0b242 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block.rs @@ -1827,16 +1827,9 @@ impl DiskWriteBatch { return Ok(()); } - // Seeds fire at non-finalized best-*tip* commits, so an nf best-tip - // jump (a fork switch between nf chains, or a restart restoring the nf - // backup) can seed a height whose parent row is missing or belongs to - // another branch. Writing that row would break the header store's - // linkage invariant on disk and poison difficulty-adjustment windows - // (`header_store_coherence` bug 3). Refuse the write instead: the - // header store briefly lags the nf chain, and header-range sync - // converges it onto the new branch through the linkage-checked range - // path. The merged header view (full-block rows first, then zakura - // rows) is the same view the chain walk reads. + // Seeds can jump to a non-finalized best tip whose parent is not the + // stored row below it. Refuse those seeds so the header store stays + // linked; header-range sync will later deliver the missing rows. let hash_by_height = db.cf_handle("hash_by_height").unwrap(); let parent_hash: Option = height.previous().ok().and_then(|parent_height| { db.zs_get(&hash_by_height, &parent_height) From 08f791a0e0337c000b44ef5aad85472741a197fb Mon Sep 17 00:00:00 2001 From: roman Date: Mon, 6 Jul 2026 17:39:40 -0600 Subject: [PATCH 4/8] fix(state): verify zakura header-store invariants in consensus reads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pillar 2 of the REORG_PLAN: a corrupted header store can no longer poison difficulty validation — readers surface the storage fault explicitly. - recent_header_context verifies, at every step of its walk, that the stored header is the block its hash row names (HeaderHashMismatch), that it links to the row below (BrokenLinkage), and that no row is missing below a stored one (Gap), returning the new StoreIncoherentError instead of a poisoned or silently shortened difficulty window. - The range writer's anchor round-trip distinguishes a bijection violation in our own indexes (StoreIncoherent::BijectionMismatch) from a genuinely unknown anchor (UnknownAnchor). - CommitHeaderRangeError::StoreIncoherent is classified as a local, non-peer-scoring commit failure in zebrad: the range was rejected because our store cannot supply trustworthy context, not because the peer's range was shown invalid. New reads.rs suite in header_store_coherence hand-corrupts the column families and pins the reader/writer behavior for all four fault shapes, including side-effect freedom of the rejections. --- CHANGELOG.md | 8 + zebra-state/src/error.rs | 71 +++++ zebra-state/src/lib.rs | 3 +- .../service/finalized_state/zebra_db/block.rs | 103 +++++-- .../block/tests/header_store_coherence.rs | 1 + .../tests/header_store_coherence/README.md | 1 + .../tests/header_store_coherence/reads.rs | 281 ++++++++++++++++++ .../zebra_db/block/tests/vectors.rs | 12 +- zebrad/src/commands/start.rs | 19 ++ .../start/zakura/header_sync_driver.rs | 6 + 10 files changed, 482 insertions(+), 23 deletions(-) create mode 100644 zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/reads.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 96f5da37488..8308484c00e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,14 @@ and this project adheres to [Semantic Versioning](https://semver.org). ### Fixed +- Zakura header-store consensus reads now verify store invariants as they + walk: the difficulty-context walk checks that every stored header is the + block its hash row names and links to the row below it, and the anchor + lookup distinguishes a corrupted index round-trip from a genuinely unknown + anchor. A corrupted store now surfaces as an explicit local + `StoreIncoherent` error (never scored against peers) instead of poisoning + difficulty validation and rejecting honest headers with + `InvalidDifficultyThreshold`. - Fixed three Zakura header-store write paths that could leave the on-disk header store internally incoherent after chain forks, causing nodes to reject valid headers from honest peers (`InvalidDifficultyThreshold` / diff --git a/zebra-state/src/error.rs b/zebra-state/src/error.rs index 84a9720dee8..40533f13fb6 100644 --- a/zebra-state/src/error.rs +++ b/zebra-state/src/error.rs @@ -195,6 +195,68 @@ impl From for CommitCheckpointVerifiedError { } } +/// An internal invariant of the zakura header store was found violated while +/// reading it. +/// +/// This is a **local storage fault**, never evidence about a peer: readers +/// return it instead of feeding rows from more than one branch (or from beside +/// a gap) into consensus validation, where the corruption would otherwise +/// surface as a misleading validation failure (`InvalidDifficultyThreshold`, +/// `UnknownAnchor`) attributed to whoever supplied the input being validated. +#[derive(Debug, Error, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum StoreIncoherentError { + /// The header row at `height` does not link to the stored row below it. + #[error( + "header store incoherent: header at {height:?} links to {expected_parent} but the stored row below is {actual_below}" + )] + BrokenLinkage { + /// Height of the header whose parent link failed to resolve. + height: block::Height, + /// The parent hash the header claims (`previous_block_hash`). + expected_parent: block::Hash, + /// The hash actually stored at `height - 1`. + actual_below: block::Hash, + }, + + /// A header row exists at `height` but the row below it is missing. + #[error( + "header store incoherent: no stored row at {missing:?} below the header at {height:?}" + )] + Gap { + /// Height of the stored header above the gap. + height: block::Height, + /// The missing height (`height - 1`). + missing: block::Height, + }, + + /// The header row at `height` is not the block its hash row names. + #[error( + "header store incoherent: header stored at {height:?} hashes to {computed} but the hash row names {indexed}" + )] + HeaderHashMismatch { + /// Height of the divergent rows. + height: block::Height, + /// The hash the height→hash index names. + indexed: block::Hash, + /// The stored header's actual hash. + computed: block::Hash, + }, + + /// The hash→height and height→hash indexes disagree about a hash. + #[error( + "header store incoherent: hash {hash} is indexed at {height:?} but that height stores {stored:?}" + )] + BijectionMismatch { + /// The hash whose round-trip failed. + hash: block::Hash, + /// The height the hash→height index reports for it. + height: block::Height, + /// What the height→hash index stores there instead. + stored: Option, + }, +} + /// An error describing why a header-only range could not be committed. #[derive(Debug, Error, Clone, PartialEq, Eq)] #[non_exhaustive] @@ -323,6 +385,15 @@ pub enum CommitHeaderRangeError { height: block::Height, }, + /// The local header store was found internally incoherent while reading + /// the context needed to validate the range. + /// + /// This is a local storage fault, not a peer validation failure: the range + /// was rejected because the store cannot supply trustworthy context, not + /// because the range itself was shown invalid. + #[error("header store incoherent while validating range: {0}")] + StoreIncoherent(#[from] StoreIncoherentError), + /// Contextual header validation failed. #[error("could not contextually validate header")] ValidateContextError(#[from] Box), diff --git a/zebra-state/src/lib.rs b/zebra-state/src/lib.rs index 015f190c428..88159fbd242 100644 --- a/zebra-state/src/lib.rs +++ b/zebra-state/src/lib.rs @@ -43,7 +43,8 @@ pub use config::{ pub use constants::{state_database_format_version_in_code, MAX_BLOCK_REORG_HEIGHT}; pub use error::{ BoxError, CloneError, CommitBlockError, CommitCheckpointVerifiedError, CommitHeaderRangeError, - CommitSemanticallyVerifiedError, DuplicateNullifierError, ValidateContextError, + CommitSemanticallyVerifiedError, DuplicateNullifierError, StoreIncoherentError, + ValidateContextError, }; pub use request::{ CheckpointVerifiedBlock, CommitSemanticallyVerifiedBlockRequest, HashOrHeight, MappedRequest, diff --git a/zebra-state/src/service/finalized_state/zebra_db/block.rs b/zebra-state/src/service/finalized_state/zebra_db/block.rs index d722bb0b242..5dab23a1272 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block.rs @@ -36,7 +36,7 @@ use crate::{ constants::{ MAX_BLOCK_REORG_HEIGHT, MAX_HEADER_SYNC_HEIGHT_RANGE, MAX_PRUNE_HEIGHTS_PER_COMMIT, }, - error::{CommitCheckpointVerifiedError, CommitHeaderRangeError}, + error::{CommitCheckpointVerifiedError, CommitHeaderRangeError, StoreIncoherentError}, request::FinalizedBlock, service::check, service::finalized_state::{ @@ -544,31 +544,85 @@ impl ZebraDb { } /// Returns recent header difficulty/time context in reverse height order, - /// starting at `height`. + /// starting at `height`, verifying `previous_block_hash` linkage at every + /// step of the walk. + /// + /// Returns an empty context when there is no stored row at `height` (the + /// caller decides whether that anchor is unknown), and a shorter-than-span + /// context when the walk reaches genesis. + /// + /// # Errors + /// + /// Returns [`StoreIncoherentError`] when the walk finds a header row that + /// is not the block its hash row names, a row that does not link to the + /// row below it, or a gap below a stored row. Feeding such a window into + /// difficulty validation would mix rows from more than one branch (or + /// shift the adjustment window), producing `InvalidDifficultyThreshold` + /// rejections of honest input — the reader surfaces the storage fault + /// explicitly instead. The per-row hash check costs one header hash per + /// consumed row, negligible next to the validation the window feeds. pub fn recent_header_context( &self, height: block::Height, - ) -> Vec<( - zebra_chain::work::difficulty::CompactDifficulty, - DateTime, - )> { + ) -> Result< + Vec<( + zebra_chain::work::difficulty::CompactDifficulty, + DateTime, + )>, + StoreIncoherentError, + > { let mut context = Vec::with_capacity(check::difficulty::POW_ADJUSTMENT_BLOCK_SPAN); - let mut current_height = Some(height); - while let Some(height) = current_height { - let Some((_hash, header)) = self.header_by_height(height) else { - break; - }; + let Some((mut hash, mut header)) = self.header_by_height(height) else { + return Ok(context); + }; + let mut height = height; + + loop { + let computed = block::Hash::from(&*header); + if computed != hash { + return Err(StoreIncoherentError::HeaderHashMismatch { + height, + indexed: hash, + computed, + }); + } context.push((header.difficulty_threshold, header.time)); + if context.len() == check::difficulty::POW_ADJUSTMENT_BLOCK_SPAN { - break; + return Ok(context); } + let Ok(below) = height.previous() else { + // The walk reached genesis: a short context is legitimate, and + // the difficulty functions handle it (MedianTime clamps + // negative heights to zero). + return Ok(context); + }; - current_height = height.previous().ok(); - } + let Some((below_hash, below_header)) = self.header_by_height(below) else { + // Rows must be contiguous from genesis up to the header tip + // (full-block rows below the body tip — retained even under + // pruning — and zakura rows above it), so a missing row below + // a stored one is a gap, not the end of history. + return Err(StoreIncoherentError::Gap { + height, + missing: below, + }); + }; - context + if header.previous_block_hash != below_hash { + return Err(StoreIncoherentError::BrokenLinkage { + height, + expected_parent: header.previous_block_hash, + actual_below: below_hash, + }); + } + + height = below; + hash = below_hash; + header = below_header; + } } /// Returns header-known, body-missing heights. @@ -2035,17 +2089,26 @@ impl DiskWriteBatch { .or_else(|| (anchor == zebra_db.network().genesis_hash()).then_some(block::Height(0))) .ok_or(CommitHeaderRangeError::UnknownAnchor { anchor })?; - if anchor != zebra_db.network().genesis_hash() - && zebra_db.header_hash(anchor_height) != Some(anchor) - { - return Err(CommitHeaderRangeError::UnknownAnchor { anchor }); + // The hash→height index knows the anchor, so a failed height→hash + // round-trip is a bijection violation in our own store — a local + // storage fault, not an unknown anchor supplied by the caller. + if anchor != zebra_db.network().genesis_hash() { + let stored = zebra_db.header_hash(anchor_height); + if stored != Some(anchor) { + return Err(StoreIncoherentError::BijectionMismatch { + hash: anchor, + height: anchor_height, + stored, + } + .into()); + } } let finalized_height = zebra_db.finalized_tip_height(); let best_header_tip = zebra_db.best_header_tip().map(|(height, _)| height); let checkpoints = zebra_db.network().checkpoint_list(); - let mut recent_headers = zebra_db.recent_header_context(anchor_height); + let mut recent_headers = zebra_db.recent_header_context(anchor_height)?; if recent_headers.is_empty() { if anchor == zebra_db.network().genesis_hash() && anchor_height == block::Height(0) { return Err(CommitHeaderRangeError::MissingGenesisAnchor { anchor }); diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence.rs index 81b87d2b4ae..ef8cea5a0a1 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence.rs @@ -32,4 +32,5 @@ mod fabricate; mod ops; mod oracle; mod prop; +mod reads; mod scenarios; diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/README.md b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/README.md index 4b49718607a..cf788dcf19b 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/README.md +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/README.md @@ -36,6 +36,7 @@ write instead. | `ops.rs` | The op alphabet, each op mapped to one real production write-batch shape: `CommitHeaderRange` → `prepare_header_range_batch_with_roots`; `CommitBody` / `Finalize` → `prepare_block_header_and_transaction_data_batch` plus the finalization roots delete (which runs the release path internally); `Seed` → `seed_zakura_header_from_committed_block` (the non-finalized best-chain commit hook); `Reopen` → shutdown and reopen of the persistent store. The `Harness` executes ops, cross-checks the oracle's prediction against the store's response, and audits after every mutation; failures come back as a transcribable `FailureReport` (the executed op prefix plus every violation found). | | `scenarios.rs` | Scripted production event shapes (s01–s11): simple reorgs, lower-work rejections and their later reversal, split-range and walk-back deliveries, body commits racing header reorgs, reorgs to a lower height, double reorgs at one fork point, activity across the difficulty-adjustment window edge, restarts at every boundary, seed/range interplay, and refused-seed convergence. Also holds the `*_upholds_invariants` regression gates below. | | `prop.rs` | The permanent random sweep: random op sequences over the fixed universe, shrunk to minimal counterexamples on any audit failure. | +| `reads.rs` | Read-path coherence (Pillar 2): hand-corrupts the column families and asserts `recent_header_context` / the anchor round-trip report `StoreIncoherentError` (`HeaderHashMismatch`, `BrokenLinkage`, `Gap`, `BijectionMismatch`) instead of feeding stale rows into difficulty validation, and that the range writer's `StoreIncoherent` rejection is side-effect free. | ## Fixed corruption bugs gated by this suite diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/reads.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/reads.rs new file mode 100644 index 00000000000..08196889c11 --- /dev/null +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/reads.rs @@ -0,0 +1,281 @@ +//! Read-path coherence: consensus readers verify store invariants as they +//! walk, surfacing corruption as an explicit [`StoreIncoherentError`] instead +//! of feeding stale rows into difficulty validation (REORG_PLAN Pillar 2). +//! +//! The write path refuses to create incoherent stores (the Phase-1.5 guards), +//! so these tests corrupt the column families directly — the same hand-made +//! corruption technique as the audit's own tests — and assert that: +//! +//! - `recent_header_context` returns `BrokenLinkage`/`Gap` instead of a +//! poisoned or silently shortened difficulty window, and +//! - the range writer rejects with `StoreIncoherent` (a local storage fault, +//! never a peer-attributed validation failure) and leaves the store +//! untouched, and +//! - the anchor round-trip distinguishes a bijection violation in our own +//! indexes (`BijectionMismatch`) from a genuinely unknown anchor. + +use std::sync::Arc; + +use zebra_chain::block::{self, Height}; + +use super::super::super::{ZAKURA_HEADER_BY_HEIGHT, ZAKURA_HEADER_HASH_BY_HEIGHT}; +use super::super::common::{commit_header_range, state_with_genesis_config}; +use super::{ + audit::dump_store, + fabricate::{Universe, BRANCH_A, FORK_HEIGHT}, +}; +use crate::{ + error::{CommitHeaderRangeError, StoreIncoherentError}, + service::finalized_state::{ + disk_db::{DiskWriteBatch, WriteDisk}, + ZebraDb, + }, + Config, +}; + +/// A store holding genesis plus the trunk up to the fork height, built through +/// the production write path. +fn trunk_state(universe: &Universe) -> ZebraDb { + let state = state_with_genesis_config( + &universe.network, + universe.genesis.clone(), + Config::ephemeral(), + ); + let trunk_headers: Vec<_> = universe.trunk[..FORK_HEIGHT as usize] + .iter() + .map(|fab| fab.header.clone()) + .collect(); + commit_header_range(&state, universe.genesis.hash(), &trunk_headers); + state +} + +/// Re-delivers a slice of trunk headers through the production range writer. +fn redeliver_trunk( + state: &ZebraDb, + universe: &Universe, + anchor_height: u32, + len: usize, +) -> Result { + let anchor = universe.trunk_at(anchor_height).hash; + let headers: Vec> = universe.trunk + [anchor_height as usize..anchor_height as usize + len] + .iter() + .map(|fab| fab.header.clone()) + .collect(); + let body_sizes = vec![0; headers.len()]; + let mut batch = DiskWriteBatch::new(); + let result = batch.prepare_header_range_batch(state, anchor, &headers, &body_sizes); + if result.is_ok() { + state + .write_batch(batch) + .expect("header range batch writes successfully"); + } + result +} + +/// A coherent store yields a full-span window mid-chain and a legitimately +/// short window near genesis. +#[test] +fn coherent_walks_return_ok_contexts() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe); + + let full = state + .recent_header_context(Height(40)) + .expect("coherent store walks cleanly"); + assert_eq!( + full.len(), + crate::service::check::difficulty::POW_ADJUSTMENT_BLOCK_SPAN + ); + + // Heights 5..=0 inclusive: six rows, then the walk stops at genesis. + let short = state + .recent_header_context(Height(5)) + .expect("a short walk ending at genesis is legitimate"); + assert_eq!(short.len(), 6); + + // A missing anchor row is not incoherence; the caller decides what an + // unknown anchor means. + let missing = state + .recent_header_context(Height(FORK_HEIGHT + 10)) + .expect("a missing starting row is not a violation"); + assert!(missing.is_empty()); +} + +/// A header row that is not the block its hash row names (the incident-shaped +/// poison: a stale row's (threshold, time) feeding the DAA window): the walk +/// reports the divergence instead of consuming the row, and the range writer +/// maps it to a side-effect-free `StoreIncoherent` rejection. +#[test] +fn foreign_header_row_is_reported_not_validated() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe); + + // Overwrite the header row at height 20 with a branch-A header while the + // hash rows keep claiming the trunk. + let header_cf = state.db.cf_handle(ZAKURA_HEADER_BY_HEIGHT).unwrap(); + let foreign_header = universe.branches[BRANCH_A].headers[3].header.clone(); + let mut batch = DiskWriteBatch::new(); + batch.zs_insert(&header_cf, Height(20), foreign_header); + state.db.write(batch).expect("raw insert writes"); + + let error = state + .recent_header_context(Height(30)) + .expect_err("the walk crosses the corrupted row"); + assert!( + matches!( + error, + StoreIncoherentError::HeaderHashMismatch { height, .. } if height == Height(20) + ), + "expected HeaderHashMismatch at the corrupted row, got {error:?}" + ); + + // The writer surfaces the same fault as a local rejection: the range is + // not blamed (no contextual-validation error) and nothing is written. + let dump_before = dump_store(&state); + let error = redeliver_trunk(&state, &universe, 30, 5) + .expect_err("validation context crosses the corrupted row"); + assert!( + matches!( + error, + CommitHeaderRangeError::StoreIncoherent( + StoreIncoherentError::HeaderHashMismatch { .. } + ) + ), + "expected StoreIncoherent(HeaderHashMismatch), got {error:?}" + ); + assert_eq!( + dump_store(&state), + dump_before, + "a StoreIncoherent rejection must be side-effect free" + ); +} + +/// A self-consistent foreign row (its header *is* the block its hash row +/// names, but it belongs to another branch): the row above it no longer links +/// down, and the walk reports the broken link from above. +#[test] +fn broken_linkage_is_reported_not_validated() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe); + + // Replace both the header and hash rows at height 20 with branch-A's + // fourth row: internally consistent, but trunk@21 does not link to it. + let foreign = &universe.branches[BRANCH_A].headers[3]; + let header_cf = state.db.cf_handle(ZAKURA_HEADER_BY_HEIGHT).unwrap(); + let hash_cf = state.db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); + let mut batch = DiskWriteBatch::new(); + batch.zs_insert(&header_cf, Height(20), foreign.header.clone()); + batch.zs_insert(&hash_cf, Height(20), foreign.hash); + state.db.write(batch).expect("raw insert writes"); + + let error = state + .recent_header_context(Height(30)) + .expect_err("the walk crosses the corrupted row"); + assert!( + matches!( + error, + StoreIncoherentError::BrokenLinkage { height, actual_below, .. } + if height == Height(21) && actual_below == foreign.hash + ), + "expected BrokenLinkage above the corrupted row, got {error:?}" + ); + + let error = redeliver_trunk(&state, &universe, 30, 5) + .expect_err("validation context crosses the corrupted row"); + assert!( + matches!( + error, + CommitHeaderRangeError::StoreIncoherent(StoreIncoherentError::BrokenLinkage { .. }) + ), + "expected StoreIncoherent(BrokenLinkage), got {error:?}" + ); +} + +/// A gap mid-window: a missing row below a stored one is incoherence, not the +/// end of history — a silently shortened window would shift the difficulty +/// adjustment instead of failing. +#[test] +fn gap_is_reported_not_shortened() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe); + + let header_cf = state.db.cf_handle(ZAKURA_HEADER_BY_HEIGHT).unwrap(); + let hash_cf = state.db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); + let mut batch = DiskWriteBatch::new(); + batch.zs_delete(&header_cf, Height(15)); + batch.zs_delete(&hash_cf, Height(15)); + state.db.write(batch).expect("raw delete writes"); + + let error = state + .recent_header_context(Height(25)) + .expect_err("the walk crosses the gap"); + assert!( + matches!( + error, + StoreIncoherentError::Gap { height, missing } + if height == Height(16) && missing == Height(15) + ), + "expected Gap below height 16, got {error:?}" + ); + + let error = + redeliver_trunk(&state, &universe, 25, 5).expect_err("validation context crosses the gap"); + assert!( + matches!( + error, + CommitHeaderRangeError::StoreIncoherent(StoreIncoherentError::Gap { .. }) + ), + "expected StoreIncoherent(Gap), got {error:?}" + ); +} + +/// A hash→height entry whose height→hash row disagrees is a bijection +/// violation in our own indexes: the anchor round-trip reports it as +/// `StoreIncoherent`, distinct from a genuinely unknown anchor. +#[test] +fn anchor_bijection_violation_is_store_incoherent_not_unknown() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe); + + // The hash row at height 30 now names a different block, while + // height_by_hash still maps the trunk hash to 30. + let hash_cf = state.db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); + let stray_hash = universe.branches[BRANCH_A].headers[0].hash; + let mut batch = DiskWriteBatch::new(); + batch.zs_insert(&hash_cf, Height(30), stray_hash); + state.db.write(batch).expect("raw insert writes"); + + let error = redeliver_trunk(&state, &universe, 30, 5) + .expect_err("the anchor round-trip fails on the corrupted index"); + assert!( + matches!( + error, + CommitHeaderRangeError::StoreIncoherent(StoreIncoherentError::BijectionMismatch { + hash, + height, + stored: Some(stored), + }) if hash == universe.trunk_at(30).hash + && height == Height(30) + && stored == stray_hash + ), + "expected StoreIncoherent(BijectionMismatch), got {error:?}" + ); + + // An anchor the store has never heard of stays UnknownAnchor. + let unknown = universe.branches[BRANCH_A].headers[10].hash; + let headers = vec![universe.trunk[30].header.clone()]; + let mut batch = DiskWriteBatch::new(); + let error = batch + .prepare_header_range_batch(&state, unknown, &headers, &[0]) + .expect_err("an unindexed anchor is unknown"); + assert!( + matches!(error, CommitHeaderRangeError::UnknownAnchor { .. }), + "expected UnknownAnchor, got {error:?}" + ); +} diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs index 858e34c092c..7bca301f624 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs @@ -1020,10 +1020,16 @@ fn header_range_commit_rejects_non_current_anchor_hash() { let block2 = mainnet_block(2); let alternate_block2 = alternate_header(stale_anchor, &block2.header, 1); + // A hash→height entry whose height→hash row names a different block is a + // bijection violation in our own indexes, reported as a local storage + // fault rather than an unknown anchor. Either way the stale anchor cannot + // be committed on. let mut batch = DiskWriteBatch::new(); assert!(matches!( batch.prepare_header_range_batch(&state, stale_anchor, &[alternate_block2], &[0]), - Err(CommitHeaderRangeError::UnknownAnchor { anchor }) if anchor == stale_anchor + Err(CommitHeaderRangeError::StoreIncoherent( + crate::error::StoreIncoherentError::BijectionMismatch { hash, height, stored }, + )) if hash == stale_anchor && height == Height(1) && stored == Some(block1.hash()) )); assert_eq!(state.hash(Height(1)), None); @@ -1246,7 +1252,9 @@ fn synthetic_headers_from_state( ) -> Vec> { let network = state.network(); let template = mainnet_block(1); - let mut context = state.recent_header_context(anchor_height); + let mut context = state + .recent_header_context(anchor_height) + .expect("test store is coherent"); let mut previous_hash = anchor_hash; let mut previous_height = anchor_height; let mut nonce_tag = nonce_seed; diff --git a/zebrad/src/commands/start.rs b/zebrad/src/commands/start.rs index fe94936d94d..62fa6a78d6f 100644 --- a/zebrad/src/commands/start.rs +++ b/zebrad/src/commands/start.rs @@ -2346,6 +2346,25 @@ mod zakura_header_sync_driver_tests { ); } + #[test] + fn store_incoherent_is_local_header_sync_commit_failure() { + // A range rejected because our own header rows failed a + // linkage/bijection check is a local storage fault; scoring peers for + // it recreates the disconnect-honest-peers failure mode. + let error = zebra_state::CommitHeaderRangeError::StoreIncoherent( + zebra_state::StoreIncoherentError::BrokenLinkage { + height: block::Height(2), + expected_parent: block::Hash([0; 32]), + actual_below: block::Hash([1; 32]), + }, + ); + + assert_eq!( + header_range_commit_failure_kind(&error), + HeaderSyncCommitFailureKind::Local + ); + } + #[test] fn unlinked_range_is_local_header_sync_commit_failure() { // The reactor validates every response's linkage against the requested diff --git a/zebrad/src/commands/start/zakura/header_sync_driver.rs b/zebrad/src/commands/start/zakura/header_sync_driver.rs index a3e55b9f899..a4d6ad9bb8b 100644 --- a/zebrad/src/commands/start/zakura/header_sync_driver.rs +++ b/zebrad/src/commands/start/zakura/header_sync_driver.rs @@ -1118,6 +1118,12 @@ pub(crate) fn header_range_commit_failure_kind( // store's own linkage check failing means the local anchor/response pairing // went wrong, not that the peer misbehaved. | zebra_state::CommitHeaderRangeError::UnlinkedRange { .. } + // Store incoherence is by definition a local storage fault: the range was + // rejected because our own header rows failed a linkage/bijection check + // while reading validation context, not because the peer's range was shown + // invalid. Scoring peers for it recreates the disconnect-honest-peers + // failure mode from the 2026-07-06 incidents. + | zebra_state::CommitHeaderRangeError::StoreIncoherent(_) | zebra_state::CommitHeaderRangeError::CommitResponseDropped => { HeaderSyncCommitFailureKind::Local } From b354be0ee8b7edd5efaa0a98947bc887fbbc4af9 Mon Sep 17 00:00:00 2001 From: roman Date: Mon, 6 Jul 2026 18:20:19 -0600 Subject: [PATCH 5/8] fix(state): audit and self-repair the zakura header store at startup --- CHANGELOG.md | 8 + .../src/service/finalized_state/zebra_db.rs | 11 + .../service/finalized_state/zebra_db/block.rs | 2 + .../zebra_db/block/startup_audit.rs | 428 ++++++++++++++ .../block/tests/header_store_coherence.rs | 1 + .../header_store_coherence/startup_audit.rs | 539 ++++++++++++++++++ 6 files changed, 989 insertions(+) create mode 100644 zebra-state/src/service/finalized_state/zebra_db/block/startup_audit.rs create mode 100644 zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/startup_audit.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 8308484c00e..7a74d83bb50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,14 @@ and this project adheres to [Semantic Versioning](https://semver.org). ### Fixed +- Zebra now audits the Zakura header store when the state database opens and + self-repairs any incoherence (broken linkage, hash↔height index mismatches, + gaps with stranded rows above them, stale rows at committed heights) by + truncating the Zakura column families to the last coherent height in one + atomic batch, then letting header sync re-download the truncated suffix. + Stores corrupted by earlier binaries now heal on restart instead of staying + wedged below the network tip; each repair emits a warning and the + `state.zakura.header_store.incoherent` metric. - Zakura header-store consensus reads now verify store invariants as they walk: the difficulty-context walk checks that every stored header is the block its hash row names and links to the row below it, and the anchor diff --git a/zebra-state/src/service/finalized_state/zebra_db.rs b/zebra-state/src/service/finalized_state/zebra_db.rs index e7ba5325b2f..25170417702 100644 --- a/zebra-state/src/service/finalized_state/zebra_db.rs +++ b/zebra-state/src/service/finalized_state/zebra_db.rs @@ -175,6 +175,17 @@ impl ZebraDb { ) } + // Audit the zakura header store's on-disk invariants and truncate any + // incoherent suffix, so a store corrupted by an earlier binary + // self-heals at startup instead of wedging header sync (headers are + // re-fetchable, so correctness beats preserved rows). Read-only + // instances cannot repair; their reads surface any corruption as + // explicit `StoreIncoherentError`s instead. + if !read_only { + db.audit_and_repair_zakura_header_store() + .expect("startup header-store repair write failed: RocksDB is unavailable"); + } + db.spawn_format_change(format_change); db diff --git a/zebra-state/src/service/finalized_state/zebra_db/block.rs b/zebra-state/src/service/finalized_state/zebra_db/block.rs index 5dab23a1272..a3451c5b864 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block.rs @@ -57,6 +57,8 @@ use crate::{ #[cfg(feature = "indexer")] use crate::request::Spend; +mod startup_audit; + #[cfg(test)] mod tests; diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/startup_audit.rs b/zebra-state/src/service/finalized_state/zebra_db/block/startup_audit.rs new file mode 100644 index 00000000000..8a042a1e38d --- /dev/null +++ b/zebra-state/src/service/finalized_state/zebra_db/block/startup_audit.rs @@ -0,0 +1,428 @@ +//! Startup audit and self-repair for the zakura header store. +//! +//! The zakura header store is five height-indexed column families acting as a +//! replicated view of the canonical header chain above the finalized tip. Its +//! invariants (hash↔height bijection, parent linkage anchored at the finalized +//! tip, no gaps or stranded rows below the header tip) are enforced by the +//! writers, but a store corrupted by an earlier binary stays corrupted on disk +//! and wedges header sync: consensus reads surface the damage as +//! [`StoreIncoherentError`](crate::error::StoreIncoherentError) and refuse to +//! feed stale rows into validation, so the node can no longer make progress +//! past the poisoned window. +//! +//! This module runs the store audit once at [`ZebraDb`] startup and repairs +//! any violation by truncating the zakura column families to the last +//! coherent height in one atomic batch. Headers are re-fetchable — header +//! sync re-downloads the truncated suffix — so correctness beats preserved +//! rows: any residual write-path bug in this class becomes a self-healing, +//! observable transient instead of a permanent on-disk wedge. +//! +//! The audit cost is `O(header frontier)`: the zakura column families only +//! hold rows above the finalized tip (plus any stale rows this audit exists +//! to remove), and the verified commitment-roots history below the tip is +//! never scanned. + +use std::{ + collections::{BTreeMap, HashMap}, + sync::Arc, +}; + +use zebra_chain::block::{self, Height}; + +use super::{ + AdvertisedBodySize, ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT, ZAKURA_HEADER_BY_HEIGHT, + ZAKURA_HEADER_HASH_BY_HEIGHT, ZAKURA_HEADER_HEIGHT_BY_HASH, +}; +use crate::service::finalized_state::{ + disk_db::{DiskWriteBatch, WriteDisk}, + disk_format::shielded::CommitmentRootsByHeight, + zebra_db::ZebraDb, + COMMITMENT_ROOTS_BY_HEIGHT, +}; + +/// How many violations are included in the repair log line. The full list can +/// be as long as the stranded suffix; the first few identify the fault shape. +const LOGGED_VIOLATIONS: usize = 8; + +/// A single header-store invariant violation found by the startup audit. +/// +/// Every violation is repaired by deleting rows; the variants exist for +/// logging and for test assertions on the fault shape. The fields are +/// diagnostic payload rendered through `Debug` in the repair warning. +#[allow(dead_code)] +#[derive(Clone, Debug)] +pub(crate) enum ZakuraStoreViolation { + /// A zakura row at a height with a committed block (`contains_height`). + /// + /// Committed heights have authoritative full-block rows, and the zakura + /// row at a height is trimmed when its body commits, so a surviving row + /// here is stale (the pre-guard re-delivery bug shape). The predicate is + /// the consensus `hash_by_height` row — which pruning retains — not body + /// presence, so stale rows at pruned heights are found too. + StaleRowAtCommittedHeight { + /// The column family holding the stale row. + cf: &'static str, + /// The committed height of the stale row. + height: Height, + }, + + /// A `hash_by_height` row at `height` has no `header_by_height` row. + MissingHeaderRow { + /// The height with a hash row but no header row. + height: Height, + }, + + /// The header stored at `height` is not the block its hash row names. + HeaderHashMismatch { + /// The height of the divergent rows. + height: Height, + /// The hash the height→hash index names. + indexed: block::Hash, + /// The stored header's actual hash. + computed: block::Hash, + }, + + /// The header at `height` does not link to the stored row below it. + BrokenLinkage { + /// The height of the header whose parent link failed to resolve. + height: Height, + /// The parent hash the header claims (`previous_block_hash`). + expected_parent: block::Hash, + /// The hash actually stored at `height - 1`. + actual_below: block::Hash, + }, + + /// The `height_by_hash` index is missing or wrong for the hash stored at + /// `height` (a forward bijection failure). + WrongHeightByHash { + /// The height whose hash failed the round-trip. + height: Height, + /// The hash stored at `height`. + hash: block::Hash, + /// The height the hash→height index reports, if any. + indexed: Option, + }, + + /// A row above the last coherent height (a stranded suffix: rows above a + /// gap, above a broken link, or above the linked chain's tip). + RowAboveLastCoherent { + /// The column family holding the stranded row. + cf: &'static str, + /// The stranded height. + height: Height, + }, + + /// A `height_by_hash` entry whose target height stores a different hash + /// (or no row at all): a stranded reverse-index row. + OrphanHeightByHash { + /// The hash of the stranded entry. + hash: block::Hash, + /// The height the entry points at. + points_at: Height, + }, +} + +/// The outcome of a startup repair: what was found and what was deleted. +/// +/// The fields are diagnostic payload: read by test assertions and rendered +/// through `Debug`. +#[allow(dead_code)] +#[derive(Debug)] +pub(crate) struct ZakuraStoreRepair { + /// The last height that passed every audit check, and its hash. + /// + /// Rows above it were truncated. `None` means the store had zakura rows + /// but no finalized tip to anchor them at, so all rows were removed. + pub last_coherent: Option<(Height, block::Hash)>, + + /// The number of rows deleted across all five column families. + pub deleted_rows: usize, + + /// Every violation found, in audit order. + pub violations: Vec, +} + +impl ZebraDb { + /// Audits the zakura header store invariants and repairs any violation by + /// deleting the offending rows in one atomic batch. + /// + /// Checks, over the whole zakura store (which only holds the header + /// frontier above the finalized tip, so this is cheap): + /// + /// - **bijection**: `hash_by_height` ↔ `height_by_hash` are mutually + /// inverse, and every stored header is the block its hash row names; + /// - **linkage**: rows chain by `previous_block_hash` from the finalized + /// tip hash upward; + /// - **tip integrity**: no rows in any zakura column family above the + /// last linked height, no gaps below it, and no zakura rows at + /// committed heights. + /// + /// On violation, emits the `state.zakura.header_store.incoherent` metric + /// and a warning, then truncates the zakura column families to the last + /// coherent height (and removes stale rows below the finalized tip and + /// orphaned reverse-index entries). Header sync re-downloads the + /// truncated suffix. A store that would fail the linkage-verified + /// consensus reads ([`StoreIncoherentError`](crate::error::StoreIncoherentError)) + /// is always repaired by this audit, because the audit checks are a + /// superset of the read-path checks over the same rows. + /// + /// Returns `Ok(None)` if the store is coherent (nothing is written), or + /// the repair summary after a successful repair write. + /// + /// Verified commitment roots at committed heights are never touched: the + /// roots column family is only scanned above the finalized tip. + pub(crate) fn audit_and_repair_zakura_header_store( + &self, + ) -> Result, rocksdb::Error> { + // Databases opened through `ZebraDb::new` without the zakura column + // families have no header store to audit. + let ( + Some(header_cf), + Some(hash_cf), + Some(height_by_hash_cf), + Some(body_size_cf), + Some(roots_cf), + ) = ( + self.db.cf_handle(ZAKURA_HEADER_BY_HEIGHT), + self.db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT), + self.db.cf_handle(ZAKURA_HEADER_HEIGHT_BY_HASH), + self.db.cf_handle(ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT), + self.db.cf_handle(COMMITMENT_ROOTS_BY_HEIGHT), + ) + else { + return Ok(None); + }; + + let finalized_tip = self.tip(); + + let headers: BTreeMap> = + self.db.zs_forward_range_iter(&header_cf, ..).collect(); + let hashes: BTreeMap = + self.db.zs_forward_range_iter(&hash_cf, ..).collect(); + let heights_by_hash: Vec<(block::Hash, Height)> = self + .db + .zs_forward_range_iter(&height_by_hash_cf, ..) + .collect(); + let body_size_heights: Vec = self + .db + .zs_forward_range_iter::<_, Height, AdvertisedBodySize, _>(&body_size_cf, ..) + .map(|(height, _)| height) + .collect(); + + // The roots column family also holds verified rows at committed + // heights (written by body commits, kept through pruning); those are + // not zakura frontier rows and are never audited or repaired. Only + // rows above the finalized tip are provisional header-sync data. + let provisional_roots_start = + finalized_tip.and_then(|(tip_height, _)| tip_height.next().ok()); + let provisional_root_heights: Vec = match (finalized_tip, provisional_roots_start) { + // The finalized tip is at the maximum height: no frontier can + // exist above it. + (Some(_), None) => Vec::new(), + (Some(_), Some(start)) => self + .db + .zs_forward_range_iter::<_, Height, CommitmentRootsByHeight, _>(&roots_cf, start..) + .map(|(height, _)| height) + .collect(), + // No finalized tip: every roots row is provisional. + (None, _) => self + .db + .zs_forward_range_iter::<_, Height, CommitmentRootsByHeight, _>(&roots_cf, ..) + .map(|(height, _)| height) + .collect(), + }; + + if headers.is_empty() + && hashes.is_empty() + && heights_by_hash.is_empty() + && body_size_heights.is_empty() + && provisional_root_heights.is_empty() + { + return Ok(None); + } + + let mut violations = Vec::new(); + + // Walk the linked chain upward from the finalized tip, verifying at + // each height that the hash and header rows agree, the header links + // to the row below, and the reverse index round-trips. The walk stops + // at the first missing hash row (the candidate chain tip) or the + // first violation; everything it passed is the coherent prefix. + let forward_index: HashMap = heights_by_hash.iter().copied().collect(); + let last_coherent = finalized_tip.map(|(anchor_height, anchor_hash)| { + let (mut last_height, mut last_hash) = (anchor_height, anchor_hash); + + while let Ok(height) = last_height.next() { + let Some(&hash) = hashes.get(&height) else { + break; + }; + let Some(header) = headers.get(&height) else { + violations.push(ZakuraStoreViolation::MissingHeaderRow { height }); + break; + }; + + let computed = block::Hash::from(&**header); + if computed != hash { + violations.push(ZakuraStoreViolation::HeaderHashMismatch { + height, + indexed: hash, + computed, + }); + break; + } + + if header.previous_block_hash != last_hash { + violations.push(ZakuraStoreViolation::BrokenLinkage { + height, + expected_parent: header.previous_block_hash, + actual_below: last_hash, + }); + break; + } + + let indexed = forward_index.get(&hash).copied(); + if indexed != Some(height) { + violations.push(ZakuraStoreViolation::WrongHeightByHash { + height, + hash, + indexed, + }); + break; + } + + (last_height, last_hash) = (height, hash); + } + + (last_height, last_hash) + }); + + // Committed heights have authoritative full-block rows: the consensus + // `hash_by_height` rows are contiguous from genesis to the finalized + // tip and retained by pruning, so `height <= finalized tip` is + // exactly `contains_height` — the same predicate as the writers' + // committed-height insert gate. (A body-presence predicate would miss + // stale rows at pruned heights.) + let committed = + |height: Height| finalized_tip.is_some_and(|(tip_height, _)| height <= tip_height); + // The coherent frontier window: strictly above the finalized tip, at + // or below the last height the linkage walk verified. + let in_window = |height: Height| { + !committed(height) + && last_coherent.is_some_and(|(last_height, _)| height <= last_height) + }; + + let mut batch = DiskWriteBatch::new(); + let mut deleted_rows = 0; + + // Height-keyed zakura rows survive only inside the coherent window. + let height_keyed: [(&'static str, &rocksdb::ColumnFamilyRef<'_>, Vec); 3] = [ + ( + ZAKURA_HEADER_BY_HEIGHT, + &header_cf, + headers.keys().copied().collect(), + ), + ( + ZAKURA_HEADER_HASH_BY_HEIGHT, + &hash_cf, + hashes.keys().copied().collect(), + ), + ( + ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT, + &body_size_cf, + body_size_heights, + ), + ]; + for (cf_name, cf, heights) in &height_keyed { + for &height in heights { + if in_window(height) { + continue; + } + + violations.push(if committed(height) { + ZakuraStoreViolation::StaleRowAtCommittedHeight { + cf: cf_name, + height, + } + } else { + ZakuraStoreViolation::RowAboveLastCoherent { + cf: cf_name, + height, + } + }); + + batch.zs_delete(*cf, height); + deleted_rows += 1; + } + } + + // Reverse-index entries survive only when their target height is + // inside the window and stores exactly their hash. This removes the + // reverse rows of every deleted hash row, plus entries orphaned by + // earlier overwrites that never cleaned up the displaced hash. + for &(hash, points_at) in &heights_by_hash { + let target_matches = hashes.get(&points_at) == Some(&hash); + if in_window(points_at) && target_matches { + continue; + } + + // Entries whose forward row is deleted above are repair fallout, + // not separate faults; only a live-but-disagreeing target is a + // distinct violation shape worth reporting. + if target_matches { + violations.push(if committed(points_at) { + ZakuraStoreViolation::StaleRowAtCommittedHeight { + cf: ZAKURA_HEADER_HEIGHT_BY_HASH, + height: points_at, + } + } else { + ZakuraStoreViolation::RowAboveLastCoherent { + cf: ZAKURA_HEADER_HEIGHT_BY_HASH, + height: points_at, + } + }); + } else { + violations.push(ZakuraStoreViolation::OrphanHeightByHash { hash, points_at }); + } + + batch.zs_delete(&height_by_hash_cf, hash); + deleted_rows += 1; + } + + // Provisional roots above the window are part of the stranded suffix. + for &height in &provisional_root_heights { + if in_window(height) { + continue; + } + + violations.push(ZakuraStoreViolation::RowAboveLastCoherent { + cf: COMMITMENT_ROOTS_BY_HEIGHT, + height, + }); + batch.zs_delete(&roots_cf, height); + deleted_rows += 1; + } + + if violations.is_empty() { + return Ok(None); + } + + metrics::counter!("state.zakura.header_store.incoherent").increment(1); + tracing::warn!( + ?finalized_tip, + ?last_coherent, + deleted_rows, + violation_count = violations.len(), + first_violations = ?&violations[..violations.len().min(LOGGED_VIOLATIONS)], + "zakura header store failed its startup coherence audit; \ + truncating to the last coherent height so header sync re-downloads the rest" + ); + + self.db.write(batch)?; + + Ok(Some(ZakuraStoreRepair { + last_coherent, + deleted_rows, + violations, + })) + } +} diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence.rs index ef8cea5a0a1..669225c0930 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence.rs @@ -34,3 +34,4 @@ mod oracle; mod prop; mod reads; mod scenarios; +mod startup_audit; diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/startup_audit.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/startup_audit.rs new file mode 100644 index 00000000000..1ece0ee84cf --- /dev/null +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/startup_audit.rs @@ -0,0 +1,539 @@ +//! Startup-audit self-repair: a hand-corrupted zakura header store heals when +//! the database is (re)opened (REORG_PLAN Pillar 3, startup half). +//! +//! The write path refuses to create incoherent stores (the Phase-1.5 guards) +//! and the consensus readers refuse to consume them (Pillar 2), so these tests +//! corrupt the column families directly — the same hand-made corruption +//! technique as the audit's and read-path tests — and assert that: +//! +//! - the startup audit finds the violation and truncates the store to the +//! last coherent height in all five column families, leaving a store the +//! full coherence audit passes and header sync can re-download onto; +//! - the repair happens on the real startup path (`ZebraDb::new` at reopen) +//! and persists: a second reopen finds a coherent store and changes nothing; +//! - repair is minimal where truncation is not needed (an orphaned +//! reverse-index row, stale rows at committed heights); +//! - a store that failed the Pillar-2 linkage-verified reads passes them +//! after the heal; and +//! - the `state.zakura.header_store.incoherent` metric is emitted exactly +//! when a repair runs. + +use std::{ + collections::HashMap, + sync::{Arc, Mutex}, +}; + +use zebra_chain::block::Height; + +use super::super::super::startup_audit::ZakuraStoreViolation; +use super::super::super::{ + ZAKURA_HEADER_BY_HEIGHT, ZAKURA_HEADER_HASH_BY_HEIGHT, ZAKURA_HEADER_HEIGHT_BY_HASH, +}; +use super::super::common::{ + commit_header_range, persistent_config, persistent_state, state_with_genesis_config, +}; +use super::{ + audit::{audit_store, dump_store, StoreDump}, + fabricate::{Universe, BRANCH_A, FORK_HEIGHT}, +}; +use crate::{ + error::{CommitHeaderRangeError, StoreIncoherentError}, + service::finalized_state::{ + disk_db::{DiskWriteBatch, WriteDisk}, + ZebraDb, + }, + Config, +}; + +/// A store holding genesis plus the trunk up to the fork height, built through +/// the production write path, on the given (persistent or ephemeral) config. +fn trunk_state(universe: &Universe, config: Config) -> ZebraDb { + let state = state_with_genesis_config(&universe.network, universe.genesis.clone(), config); + let trunk_headers: Vec<_> = universe.trunk[..FORK_HEIGHT as usize] + .iter() + .map(|fab| fab.header.clone()) + .collect(); + commit_header_range(&state, universe.genesis.hash(), &trunk_headers); + state +} + +/// Closes `state` and reopens the same database, running the startup audit. +fn reopen(state: ZebraDb, config: &Config, universe: &Universe) -> ZebraDb { + let mut state = state; + state.shutdown(true); + drop(state); + persistent_state(config, &universe.network) +} + +/// The expected store after truncating everything above `last` (the original +/// coherent dump with all five column families filtered to `..= last`). +fn truncated(dump: &StoreDump, last: Height) -> StoreDump { + StoreDump { + headers: dump + .headers + .iter() + .filter(|(height, _)| **height <= last) + .map(|(height, header)| (*height, header.clone())) + .collect(), + hashes: dump + .hashes + .iter() + .filter(|(height, _)| **height <= last) + .map(|(height, hash)| (*height, *hash)) + .collect(), + heights_by_hash: dump + .heights_by_hash + .iter() + .filter(|(_, points_at)| *points_at <= last) + .copied() + .collect(), + body_sizes: dump + .body_sizes + .iter() + .filter(|(height, _)| **height <= last) + .map(|(height, size)| (*height, *size)) + .collect(), + roots: dump + .roots + .iter() + .filter(|(height, _)| **height <= last) + .map(|(height, roots)| (*height, *roots)) + .collect(), + } +} + +fn assert_clean(state: &ZebraDb) { + let violations = audit_store(state); + assert!( + violations.is_empty(), + "expected a coherent store after repair: {violations:?}" + ); +} + +/// A coherent store is untouched: the direct audit reports nothing, and a +/// reopen through the real startup path changes no rows. +#[test] +fn startup_audit_is_noop_on_coherent_store() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let tempdir = tempfile::tempdir().expect("test tempdir is available"); + let config = persistent_config(tempdir.path()); + let state = trunk_state(&universe, config.clone()); + let original = dump_store(&state); + + let repair = state + .audit_and_repair_zakura_header_store() + .expect("audit reads and writes succeed"); + assert!( + repair.is_none(), + "no repair expected on a coherent store: {repair:?}" + ); + assert_eq!(dump_store(&state), original); + + let state = reopen(state, &config, &universe); + assert_eq!( + dump_store(&state), + original, + "a reopen must not change a coherent store" + ); +} + +/// Broken linkage mid-frontier (a self-consistent foreign row spliced into the +/// height index — the incident-shaped poison): the audit truncates to the row +/// below the splice, the heal persists across a further reopen, and header +/// sync re-downloads the truncated suffix back to the original store. +#[test] +fn startup_heals_broken_linkage_then_resync_converges() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let tempdir = tempfile::tempdir().expect("test tempdir is available"); + let config = persistent_config(tempdir.path()); + let state = trunk_state(&universe, config.clone()); + let original = dump_store(&state); + + // Replace the header and hash rows at height 20 with branch-A's fourth + // row: internally consistent, but it does not link to trunk@19. + let foreign = &universe.branches[BRANCH_A].headers[3]; + let header_cf = state.db.cf_handle(ZAKURA_HEADER_BY_HEIGHT).unwrap(); + let hash_cf = state.db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); + let mut batch = DiskWriteBatch::new(); + batch.zs_insert(&header_cf, Height(20), foreign.header.clone()); + batch.zs_insert(&hash_cf, Height(20), foreign.hash); + state.db.write(batch).expect("raw insert writes"); + + // The corruption is visible to the Pillar-2 reads before the heal. + let error = state + .recent_header_context(Height(30)) + .expect_err("the walk crosses the corrupted row"); + assert!(matches!(error, StoreIncoherentError::BrokenLinkage { .. })); + + let repair = state + .audit_and_repair_zakura_header_store() + .expect("audit reads and writes succeed") + .expect("the corrupted store is repaired"); + assert_eq!( + repair.last_coherent, + Some((Height(19), universe.trunk_at(19).hash)) + ); + assert!( + repair.violations.iter().any(|violation| matches!( + violation, + ZakuraStoreViolation::BrokenLinkage { height, actual_below, .. } + if *height == Height(20) && *actual_below == universe.trunk_at(19).hash + )), + "expected BrokenLinkage at the splice: {:?}", + repair.violations + ); + + // Everything above the last coherent height is gone, in all five CFs. + assert_eq!(dump_store(&state), truncated(&original, Height(19))); + assert_clean(&state); + state + .recent_header_context(Height(19)) + .expect("the healed store walks cleanly"); + + // The heal persists: a reopen through the startup path changes nothing. + let state = reopen(state, &config, &universe); + assert_eq!(dump_store(&state), truncated(&original, Height(19))); + + // Header sync re-downloads the truncated suffix and converges back onto + // the original chain. + let redelivered: Vec<_> = universe.trunk[19..FORK_HEIGHT as usize] + .iter() + .map(|fab| fab.header.clone()) + .collect(); + commit_header_range(&state, universe.trunk_at(19).hash, &redelivered); + assert_eq!(dump_store(&state), original); + assert_clean(&state); +} + +/// A gap mid-frontier strands every row above it: reopening alone (no direct +/// audit call) heals the store, proving the `ZebraDb::new` hook fires. +#[test] +fn startup_heals_gap_and_stranded_rows_on_reopen() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let tempdir = tempfile::tempdir().expect("test tempdir is available"); + let config = persistent_config(tempdir.path()); + let state = trunk_state(&universe, config.clone()); + let original = dump_store(&state); + + let header_cf = state.db.cf_handle(ZAKURA_HEADER_BY_HEIGHT).unwrap(); + let hash_cf = state.db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); + let mut batch = DiskWriteBatch::new(); + batch.zs_delete(&header_cf, Height(15)); + batch.zs_delete(&hash_cf, Height(15)); + state.db.write(batch).expect("raw delete writes"); + + let state = reopen(state, &config, &universe); + + // Rows 16..=50 were stranded above the gap; the audit truncated to 14. + assert_eq!(dump_store(&state), truncated(&original, Height(14))); + assert_clean(&state); + assert_eq!( + state.best_header_tip(), + Some((Height(14), universe.trunk_at(14).hash)) + ); +} + +/// A missing reverse-index row breaks the hash↔height bijection: the audit +/// truncates to the row below it. +#[test] +fn startup_heals_missing_reverse_index_row() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe, Config::ephemeral()); + let original = dump_store(&state); + + let height_by_hash_cf = state.db.cf_handle(ZAKURA_HEADER_HEIGHT_BY_HASH).unwrap(); + let mut batch = DiskWriteBatch::new(); + batch.zs_delete(&height_by_hash_cf, universe.trunk_at(10).hash); + state.db.write(batch).expect("raw delete writes"); + + let repair = state + .audit_and_repair_zakura_header_store() + .expect("audit reads and writes succeed") + .expect("the corrupted store is repaired"); + assert_eq!( + repair.last_coherent, + Some((Height(9), universe.trunk_at(9).hash)) + ); + assert!( + repair.violations.iter().any(|violation| matches!( + violation, + ZakuraStoreViolation::WrongHeightByHash { height, indexed: None, .. } + if *height == Height(10) + )), + "expected WrongHeightByHash at the deleted entry: {:?}", + repair.violations + ); + assert_eq!(dump_store(&state), truncated(&original, Height(9))); + assert_clean(&state); +} + +/// An orphaned reverse-index row (a stranded `height_by_hash` entry from an +/// overwrite that never cleaned up the displaced hash) is removed on its own: +/// the linked chain is coherent, so nothing is truncated. +#[test] +fn startup_removes_orphan_reverse_row_without_truncating() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe, Config::ephemeral()); + let original = dump_store(&state); + + let height_by_hash_cf = state.db.cf_handle(ZAKURA_HEADER_HEIGHT_BY_HASH).unwrap(); + let stray_hash = universe.branches[BRANCH_A].headers[0].hash; + let mut batch = DiskWriteBatch::new(); + batch.zs_insert(&height_by_hash_cf, stray_hash, Height(12)); + state.db.write(batch).expect("raw insert writes"); + + let repair = state + .audit_and_repair_zakura_header_store() + .expect("audit reads and writes succeed") + .expect("the corrupted store is repaired"); + assert_eq!( + repair.last_coherent, + Some((Height(FORK_HEIGHT), universe.trunk_at(FORK_HEIGHT).hash)), + "an orphan reverse row must not shorten the coherent chain" + ); + assert_eq!(repair.deleted_rows, 1); + assert!( + repair.violations.iter().any(|violation| matches!( + violation, + ZakuraStoreViolation::OrphanHeightByHash { hash, points_at } + if *hash == stray_hash && *points_at == Height(12) + )), + "expected OrphanHeightByHash for the stray entry: {:?}", + repair.violations + ); + assert_eq!(dump_store(&state), original); + assert_clean(&state); +} + +/// Stale zakura rows at a committed height (the pre-guard re-delivery shape: +/// rows the release trim never touches again) are removed without disturbing +/// the frontier above. +#[test] +fn startup_removes_stale_rows_at_committed_heights() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe, Config::ephemeral()); + let original = dump_store(&state); + + // Genesis is the only committed block; plant zakura rows at its height. + let header_cf = state.db.cf_handle(ZAKURA_HEADER_BY_HEIGHT).unwrap(); + let hash_cf = state.db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); + let height_by_hash_cf = state.db.cf_handle(ZAKURA_HEADER_HEIGHT_BY_HASH).unwrap(); + let genesis_hash = universe.genesis.hash(); + let mut batch = DiskWriteBatch::new(); + batch.zs_insert(&header_cf, Height(0), universe.genesis.header.clone()); + batch.zs_insert(&hash_cf, Height(0), genesis_hash); + batch.zs_insert(&height_by_hash_cf, genesis_hash, Height(0)); + state.db.write(batch).expect("raw insert writes"); + + let repair = state + .audit_and_repair_zakura_header_store() + .expect("audit reads and writes succeed") + .expect("the corrupted store is repaired"); + assert_eq!( + repair.last_coherent, + Some((Height(FORK_HEIGHT), universe.trunk_at(FORK_HEIGHT).hash)), + "stale committed-height rows must not shorten the frontier" + ); + assert_eq!(repair.deleted_rows, 3); + assert!( + repair.violations.iter().all(|violation| matches!( + violation, + ZakuraStoreViolation::StaleRowAtCommittedHeight { height, .. } + if *height == Height(0) + )), + "expected only StaleRowAtCommittedHeight at genesis: {:?}", + repair.violations + ); + assert_eq!(dump_store(&state), original); + assert_clean(&state); +} + +/// The reads.rs anchor-corruption shape (a hash row overwritten with a foreign +/// hash) makes the Pillar-2 range writer reject with `StoreIncoherent`; after +/// the heal the same delivery commits. This pins the Pillar-2 interaction: any +/// store those reads refuse is healed by this audit. +#[test] +fn startup_heal_unblocks_linkage_verified_reads() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe, Config::ephemeral()); + let original = dump_store(&state); + + // The hash row at height 30 now names a different block, while the header + // row and `height_by_hash` still claim the trunk. + let hash_cf = state.db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); + let stray_hash = universe.branches[BRANCH_A].headers[0].hash; + let mut batch = DiskWriteBatch::new(); + batch.zs_insert(&hash_cf, Height(30), stray_hash); + state.db.write(batch).expect("raw insert writes"); + + // A re-delivered range anchored at trunk@30 is rejected as a local + // storage fault before the heal. + let anchor = universe.trunk_at(30).hash; + let headers: Vec<_> = universe.trunk[30..35] + .iter() + .map(|fab| fab.header.clone()) + .collect(); + let body_sizes = vec![0; headers.len()]; + let mut batch = DiskWriteBatch::new(); + let error = batch + .prepare_header_range_batch(&state, anchor, &headers, &body_sizes) + .expect_err("the anchor round-trip fails on the corrupted index"); + assert!(matches!( + error, + CommitHeaderRangeError::StoreIncoherent(StoreIncoherentError::BijectionMismatch { .. }) + )); + + let repair = state + .audit_and_repair_zakura_header_store() + .expect("audit reads and writes succeed") + .expect("the corrupted store is repaired"); + assert_eq!( + repair.last_coherent, + Some((Height(29), universe.trunk_at(29).hash)) + ); + assert!( + repair.violations.iter().any(|violation| matches!( + violation, + ZakuraStoreViolation::HeaderHashMismatch { height, indexed, .. } + if *height == Height(30) && *indexed == stray_hash + )), + "expected HeaderHashMismatch at the overwritten hash row: {:?}", + repair.violations + ); + assert_eq!(dump_store(&state), truncated(&original, Height(29))); + assert_clean(&state); + + // The same fork of history now commits, anchored at the healed tip. + let redelivered: Vec<_> = universe.trunk[29..FORK_HEIGHT as usize] + .iter() + .map(|fab| fab.header.clone()) + .collect(); + commit_header_range(&state, universe.trunk_at(29).hash, &redelivered); + assert_eq!(dump_store(&state), original); + assert_clean(&state); +} + +/// A minimal local metrics recorder capturing counter increments by name. +#[derive(Clone, Default)] +struct CounterCapture(Arc>>); + +impl CounterCapture { + fn get(&self, name: &str) -> u64 { + self.0 + .lock() + .expect("counter capture lock is never poisoned") + .get(name) + .copied() + .unwrap_or(0) + } +} + +struct CaptureHandle { + name: String, + store: Arc>>, +} + +impl metrics::CounterFn for CaptureHandle { + fn increment(&self, value: u64) { + *self + .store + .lock() + .expect("counter capture lock is never poisoned") + .entry(self.name.clone()) + .or_insert(0) += value; + } + + fn absolute(&self, value: u64) { + self.store + .lock() + .expect("counter capture lock is never poisoned") + .insert(self.name.clone(), value); + } +} + +impl metrics::Recorder for CounterCapture { + fn describe_counter( + &self, + _: metrics::KeyName, + _: Option, + _: metrics::SharedString, + ) { + } + + fn describe_gauge( + &self, + _: metrics::KeyName, + _: Option, + _: metrics::SharedString, + ) { + } + + fn describe_histogram( + &self, + _: metrics::KeyName, + _: Option, + _: metrics::SharedString, + ) { + } + + fn register_counter(&self, key: &metrics::Key, _: &metrics::Metadata<'_>) -> metrics::Counter { + metrics::Counter::from_arc(Arc::new(CaptureHandle { + name: key.name().to_string(), + store: self.0.clone(), + })) + } + + fn register_gauge(&self, _: &metrics::Key, _: &metrics::Metadata<'_>) -> metrics::Gauge { + metrics::Gauge::noop() + } + + fn register_histogram( + &self, + _: &metrics::Key, + _: &metrics::Metadata<'_>, + ) -> metrics::Histogram { + metrics::Histogram::noop() + } +} + +/// The `state.zakura.header_store.incoherent` metric fires exactly when a +/// startup repair runs: once for the healing reopen, and not at all for the +/// clean reopen after it. +#[test] +fn startup_repair_emits_incoherent_metric() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let tempdir = tempfile::tempdir().expect("test tempdir is available"); + let config = persistent_config(tempdir.path()); + let state = trunk_state(&universe, config.clone()); + + let header_cf = state.db.cf_handle(ZAKURA_HEADER_BY_HEIGHT).unwrap(); + let hash_cf = state.db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); + let mut batch = DiskWriteBatch::new(); + batch.zs_delete(&header_cf, Height(15)); + batch.zs_delete(&hash_cf, Height(15)); + state.db.write(batch).expect("raw delete writes"); + + const METRIC: &str = "state.zakura.header_store.incoherent"; + + let healing = CounterCapture::default(); + let state = metrics::with_local_recorder(&healing, || reopen(state, &config, &universe)); + assert_eq!( + healing.get(METRIC), + 1, + "the healing reopen emits the metric" + ); + assert_clean(&state); + + let clean = CounterCapture::default(); + let state = metrics::with_local_recorder(&clean, || reopen(state, &config, &universe)); + assert_eq!(clean.get(METRIC), 0, "a clean reopen emits nothing"); + assert_clean(&state); +} From 6dffe6daeef5dbb6db4e3f257f0614811789ffcc Mon Sep 17 00:00:00 2001 From: roman Date: Mon, 6 Jul 2026 18:57:24 -0600 Subject: [PATCH 6/8] log the startup audit pass for soak observability --- .../zebra_db/block/startup_audit.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/startup_audit.rs b/zebra-state/src/service/finalized_state/zebra_db/block/startup_audit.rs index 8a042a1e38d..994d4de9586 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/startup_audit.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/startup_audit.rs @@ -238,6 +238,12 @@ impl ZebraDb { && body_size_heights.is_empty() && provisional_root_heights.is_empty() { + // Log the pass so operators can verify the audit ran on this boot. + tracing::info!( + ?finalized_tip, + frontier_rows = 0, + "zakura header store passed its startup coherence audit (empty frontier)" + ); return Ok(None); } @@ -403,6 +409,14 @@ impl ZebraDb { } if violations.is_empty() { + // Log the pass so operators can verify the audit ran on this boot + // (the fleet soak requires observing it finding nothing). + tracing::info!( + ?finalized_tip, + ?last_coherent, + frontier_rows = hashes.len(), + "zakura header store passed its startup coherence audit" + ); return Ok(None); } From 44f01307fae39ff1c77713bef91ca50a05c183fa Mon Sep 17 00:00:00 2001 From: roman Date: Mon, 6 Jul 2026 19:30:44 -0600 Subject: [PATCH 7/8] fix(state): own zakura header-store mutations with a single suffix-replacement primitive --- CHANGELOG.md | 9 + .../src/service/finalized_state/disk_db.rs | 19 ++ .../service/finalized_state/zebra_db/block.rs | 286 +++++++++--------- .../zebra_db/block/canonical_suffix.rs | 263 ++++++++++++++++ .../zebra_db/block/startup_audit.rs | 8 +- .../block/tests/header_store_coherence.rs | 1 + .../tests/header_store_coherence/primitive.rs | 282 +++++++++++++++++ zebra-state/src/service/write.rs | 8 +- 8 files changed, 736 insertions(+), 140 deletions(-) create mode 100644 zebra-state/src/service/finalized_state/zebra_db/block/canonical_suffix.rs create mode 100644 zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/primitive.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a74d83bb50..cef6035086b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,15 @@ and this project adheres to [Semantic Versioning](https://semver.org). - Verified-commitment-trees fast sync is now enabled by default when checkpoint sync is enabled. Operators can keep checkpoint sync but opt out of the new path by setting `consensus.vct_fast_sync = false`. +- All Zakura header-store writers (header-range commits, best-chain seeds, and + body-commit releases) now change chain membership through a single + suffix-replacement primitive that deletes everything above the fork point by + scanning the actual on-disk rows of all five column families — so rows can no + longer be stranded by bounded delete loops — and verifies linkage as a hard + precondition. Any batch that replaces a header suffix (a header reorg) is + followed by the store coherence audit, repairing and reporting + (`state.zakura.header_store.incoherent`) any residual violation at the moment + it happens instead of leaving a latent on-disk fault. - Refreshed the default Testnet Zakura bootstrap peer identities (`DEFAULT_TESTNET_ZAKURA_BOOTSTRAP_PEERS`) after the Testnet fleet's iroh node keys were rotated. The previous hardcoded node IDs were stale, so a fresh node diff --git a/zebra-state/src/service/finalized_state/disk_db.rs b/zebra-state/src/service/finalized_state/disk_db.rs index 7af2be19519..34ffe76d9a3 100644 --- a/zebra-state/src/service/finalized_state/disk_db.rs +++ b/zebra-state/src/service/finalized_state/disk_db.rs @@ -122,6 +122,11 @@ pub struct DiskDb { pub struct DiskWriteBatch { /// The inner RocksDB write batch. batch: rocksdb::WriteBatch, + + /// Rows deleted by zakura header-store suffix replacements staged in this + /// batch. Nonzero means the batch performs a header reorg; the write site + /// uses this to run the post-reorg store audit after committing. + zakura_replaced_rows: usize, } impl Debug for DiskWriteBatch { @@ -548,8 +553,22 @@ impl DiskWriteBatch { pub fn new() -> Self { DiskWriteBatch { batch: rocksdb::WriteBatch::default(), + zakura_replaced_rows: 0, } } + + /// Records that a zakura suffix replacement staged in this batch deleted + /// `deleted_rows` rows. + pub(crate) fn note_zakura_suffix_replacement(&mut self, deleted_rows: usize) { + self.zakura_replaced_rows += deleted_rows; + } + + /// Returns the number of rows deleted by zakura suffix replacements + /// staged in this batch. Nonzero means this batch performs a header + /// reorg, and the write site should audit the store after committing it. + pub(crate) fn zakura_suffix_replaced_rows(&self) -> usize { + self.zakura_replaced_rows + } } impl DiskDb { diff --git a/zebra-state/src/service/finalized_state/zebra_db/block.rs b/zebra-state/src/service/finalized_state/zebra_db/block.rs index a3451c5b864..d82a2d48bd3 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block.rs @@ -40,7 +40,7 @@ use crate::{ request::FinalizedBlock, service::check, service::finalized_state::{ - disk_db::{DiskDb, DiskWriteBatch, ReadDisk, WriteDisk}, + disk_db::{DiskWriteBatch, ReadDisk, WriteDisk}, disk_format::{ block::TransactionLocation, shielded::CommitmentRootsByHeight, @@ -57,8 +57,11 @@ use crate::{ #[cfg(feature = "indexer")] use crate::request::Spend; +mod canonical_suffix; mod startup_audit; +pub(crate) use canonical_suffix::CanonicalHeaderRow; + #[cfg(test)] mod tests; @@ -68,7 +71,13 @@ const ZAKURA_HEADER_BY_HEIGHT: &str = "zakura_header_by_height"; pub const ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT: &str = "zakura_header_body_size_by_height"; #[derive(Copy, Clone, Debug, Eq, PartialEq)] -struct AdvertisedBodySize(u32); +pub(crate) struct AdvertisedBodySize(u32); + +/// Builds an [`AdvertisedBodySize`] for tests that plant raw rows. +#[cfg(test)] +pub(crate) fn test_body_size(size: u32) -> AdvertisedBodySize { + AdvertisedBodySize(size) +} impl AdvertisedBodySize { fn new(size: u32) -> Option { @@ -1278,12 +1287,15 @@ impl ZebraDb { // Track batch commit latency for observability let batch_start = std::time::Instant::now(); + let zakura_replaced_rows = batch.zakura_suffix_replaced_rows(); self.db .write(batch) .expect("unexpected rocksdb error while writing block"); metrics::histogram!("zebra.state.rocksdb.batch_commit.duration_seconds") .record(batch_start.elapsed().as_secs_f64()); + self.audit_zakura_header_store_after_reorg(zakura_replaced_rows); + tracing::trace!(?source, "committed block from"); Ok(finalized.hash) @@ -1320,12 +1332,39 @@ impl ZebraDb { block: &Arc, ) -> Result<(), CommitHeaderRangeError> { let mut batch = DiskWriteBatch::new(); - batch.prepare_zakura_header_from_committed_block(&self.db, height, block)?; + batch.prepare_zakura_header_from_committed_block(self, height, block)?; + let zakura_replaced_rows = batch.zakura_suffix_replaced_rows(); self.db .write(batch) .map_err(|error| CommitHeaderRangeError::StorageWriteError { error: error.to_string(), - }) + })?; + + self.audit_zakura_header_store_after_reorg(zakura_replaced_rows); + + Ok(()) + } + + /// Audits the zakura header store after a batch that replaced a header + /// suffix (`replaced_rows > 0`), repairing any violation. + /// + /// This is the after-every-reorg-batch half of the store audit (the + /// startup half runs in [`ZebraDb::new`]): any residual writer bug in + /// this class becomes a repaired, observable transient at the moment it + /// happens instead of a latent on-disk fault. Suffix replacements are + /// rare (real chain forks), so the `O(header frontier)` audit cost is + /// not on the steady-state commit path. + pub(crate) fn audit_zakura_header_store_after_reorg(&self, replaced_rows: usize) { + if replaced_rows == 0 { + return; + } + + if let Err(error) = self.audit_and_repair_zakura_header_store() { + tracing::warn!( + ?error, + "post-reorg zakura header store audit failed to write its repair" + ); + } } } @@ -1791,7 +1830,7 @@ impl DiskWriteBatch { // heights with no committed body (the frontier above the body tip). // This is unconditional so it also cleans up rows left by a prior run // that had `enable_zakura_header_seed_from_committed_blocks` enabled. - self.prepare_zakura_header_release_from_committed_block(db, *height, block)?; + self.prepare_zakura_header_release_from_committed_block(zebra_db, *height, block)?; // Index the block header, hash, and height. This also restores the // verified full block row after any provisional cleanup above. @@ -1851,8 +1890,10 @@ impl DiskWriteBatch { /// committed full block. /// /// Full block verification is authoritative for the stored body. If a - /// provisional Zakura header at this height differs, replace it with the - /// block-derived header and drop stale provisional descendants. + /// provisional Zakura header at this height differs, the seeded block is + /// the new best chain at its height: the suffix from this height upward + /// is replaced through [`Self::set_canonical_suffix`], dropping stale + /// provisional descendants of the displaced row. /// /// A block whose parent hash does not match the stored header row below /// `height` is skipped without error: writing it would leave a gap or @@ -1862,23 +1903,17 @@ impl DiskWriteBatch { #[allow(clippy::unwrap_in_result)] pub fn prepare_zakura_header_from_committed_block( &mut self, - db: &DiskDb, + zebra_db: &ZebraDb, height: block::Height, block: &Arc, ) -> Result<(), CommitHeaderRangeError> { - let zakura_header_by_height = db.cf_handle(ZAKURA_HEADER_BY_HEIGHT).unwrap(); - let zakura_hash_by_height = db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); - let zakura_height_by_hash = db.cf_handle(ZAKURA_HEADER_HEIGHT_BY_HASH).unwrap(); - let zakura_body_size_by_height = db.cf_handle(ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT).unwrap(); - let zakura_roots_by_height = db.cf_handle(COMMITMENT_ROOTS_BY_HEIGHT).unwrap(); - let tx_by_loc = db.cf_handle("tx_by_loc").unwrap(); - let hash = block.hash(); - let existing_zakura_header: Option> = - db.zs_get(&zakura_header_by_height, &height); - if existing_zakura_header.as_ref() == Some(&block.header) - && db.zs_get::<_, _, block::Hash>(&zakura_hash_by_height, &height) == Some(hash) + // An identical stored row needs no reconciliation, and the frontier + // above it survives (the common case: header sync ran ahead of the + // non-finalized best chain). + if zebra_db.zakura_header(height).as_ref() == Some(&block.header) + && zebra_db.zakura_header_hash(height) == Some(hash) { return Ok(()); } @@ -1886,11 +1921,10 @@ impl DiskWriteBatch { // Seeds can jump to a non-finalized best tip whose parent is not the // stored row below it. Refuse those seeds so the header store stays // linked; header-range sync will later deliver the missing rows. - let hash_by_height = db.cf_handle("hash_by_height").unwrap(); - let parent_hash: Option = height.previous().ok().and_then(|parent_height| { - db.zs_get(&hash_by_height, &parent_height) - .or_else(|| db.zs_get(&zakura_hash_by_height, &parent_height)) - }); + let parent_hash: Option = height + .previous() + .ok() + .and_then(|parent_height| zebra_db.header_hash(parent_height)); if parent_hash != Some(block.header.previous_block_hash) { tracing::debug!( ?height, @@ -1902,50 +1936,19 @@ impl DiskWriteBatch { return Ok(()); } - if existing_zakura_header.is_some_and(|existing_header| existing_header != block.header) { - let best_header_tip: Option<(block::Height, block::Hash)> = - db.zs_last_key_value(&zakura_hash_by_height); - - if let Some((best_header_tip, _)) = best_header_tip { - for old_height in height.0..=best_header_tip.0 { - let old_height = block::Height(old_height); - - if old_height != height - && db.zs_contains( - &tx_by_loc, - &TransactionLocation::min_for_height(old_height), - ) - { - return Err(CommitHeaderRangeError::ConflictingFullBlockHeader { - height: old_height, - }); - } - - if let Some(old_hash) = - db.zs_get::<_, _, block::Hash>(&zakura_hash_by_height, &old_height) - { - self.zs_delete(&zakura_height_by_hash, old_hash); - } - - self.zs_delete(&zakura_hash_by_height, old_height); - self.zs_delete(&zakura_header_by_height, old_height); - self.zs_delete(&zakura_body_size_by_height, old_height); - self.zs_delete(&zakura_roots_by_height, old_height); - } - } - } else if let Some(old_hash) = - db.zs_get::<_, _, block::Hash>(&zakura_hash_by_height, &height) - { - if old_hash != hash { - self.zs_delete(&zakura_height_by_hash, old_hash); - } - } - - self.zs_insert(&zakura_header_by_height, height, &block.header); - self.zs_insert(&zakura_hash_by_height, height, hash); - self.zs_insert(&zakura_height_by_hash, hash, height); + let fork_height = height + .previous() + .expect("the linkage refusal above required a stored parent row below this height"); - Ok(()) + self.set_canonical_suffix( + zebra_db, + (fork_height, block.header.previous_block_hash), + &[CanonicalHeaderRow { + header: block.header.clone(), + advertised_body_size: None, + roots: None, + }], + ) } /// Prepare a database batch that releases the Zakura header store entry for a @@ -1964,16 +1967,16 @@ impl DiskWriteBatch { #[allow(clippy::unwrap_in_result)] pub fn prepare_zakura_header_release_from_committed_block( &mut self, - db: &DiskDb, + zebra_db: &ZebraDb, height: block::Height, block: &Arc, ) -> Result<(), CommitHeaderRangeError> { + let db = &zebra_db.db; let zakura_header_by_height = db.cf_handle(ZAKURA_HEADER_BY_HEIGHT).unwrap(); let zakura_hash_by_height = db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); let zakura_height_by_hash = db.cf_handle(ZAKURA_HEADER_HEIGHT_BY_HASH).unwrap(); let zakura_body_size_by_height = db.cf_handle(ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT).unwrap(); let zakura_roots_by_height = db.cf_handle(COMMITMENT_ROOTS_BY_HEIGHT).unwrap(); - let tx_by_loc = db.cf_handle("tx_by_loc").unwrap(); let existing_zakura_header: Option> = db.zs_get(&zakura_header_by_height, &height); @@ -1984,36 +1987,15 @@ impl DiskWriteBatch { return Ok(()); } - // A committed block whose header conflicts with the provisional chain at - // this height invalidates the provisional descendants built on top of it. - // Drop them, but never overwrite a height that already has a committed body. + // A committed block whose header conflicts with the provisional chain + // at this height invalidates the provisional descendants built on top + // of it: truncate the suffix above this height through the owner + // primitive (its committed-body interlock refuses to touch any height + // that already has a body, as the old bounded loop did). The empty + // replacement is anchored at this block, whose full-block row is + // staged in this same batch. if existing_zakura_header.is_some_and(|existing_header| existing_header != block.header) { - let zakura_tip: Option<(block::Height, block::Hash)> = - db.zs_last_key_value(&zakura_hash_by_height); - - if let Some((zakura_tip, _)) = zakura_tip { - for descendant in (height.0 + 1)..=zakura_tip.0 { - let descendant = block::Height(descendant); - - if db.zs_contains(&tx_by_loc, &TransactionLocation::min_for_height(descendant)) - { - return Err(CommitHeaderRangeError::ConflictingFullBlockHeader { - height: descendant, - }); - } - - if let Some(old_hash) = - db.zs_get::<_, _, block::Hash>(&zakura_hash_by_height, &descendant) - { - self.zs_delete(&zakura_height_by_hash, old_hash); - } - - self.zs_delete(&zakura_hash_by_height, descendant); - self.zs_delete(&zakura_header_by_height, descendant); - self.zs_delete(&zakura_body_size_by_height, descendant); - self.zs_delete(&zakura_roots_by_height, descendant); - } - } + self.set_canonical_suffix(zebra_db, (height, block.hash()), &[])?; } // Release the provisional row at this height. @@ -2077,9 +2059,6 @@ impl DiskWriteBatch { }); } - let header_by_height = zebra_db.db.cf_handle(ZAKURA_HEADER_BY_HEIGHT).unwrap(); - let hash_by_height = zebra_db.db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); - let height_by_hash = zebra_db.db.cf_handle(ZAKURA_HEADER_HEIGHT_BY_HASH).unwrap(); let body_size_by_height = zebra_db .db .cf_handle(ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT) @@ -2119,6 +2098,7 @@ impl DiskWriteBatch { } let mut first_conflicting_height = None; + let mut first_missing_height = None; let mut validated_headers = Vec::with_capacity(headers.len()); // Each header must link to the anchor (for the first header) or to its @@ -2185,6 +2165,8 @@ impl DiskWriteBatch { first_conflicting_height.get_or_insert(height); } + } else { + first_missing_height.get_or_insert(height); } check::header_is_valid_for_recent_chain( @@ -2246,29 +2228,27 @@ impl DiskWriteBatch { } } - if let (Some(first_conflicting_height), Some(best_header_tip)) = - (first_conflicting_height, best_header_tip) - { - for height in first_conflicting_height.0..=best_header_tip.0 { - let height = block::Height(height); - - if zebra_db.contains_body_at_height(height) { - return Err(CommitHeaderRangeError::ConflictingFullBlockHeader { height }); - } - - if let Some(old_hash) = zebra_db.zakura_header_hash(height) { - self.zs_delete(&height_by_hash, old_hash); - } + // The suffix replacement starts at the first conflicting height, or — + // for a pure extension — at the first height without a stored row. + // Rows below it match the stored chain (a conflict below a missing + // row is impossible: rows are contiguous up to the header tip), so + // chain membership only changes from `replace_from` upward. `None` + // means the whole range re-delivered rows the store already has. + let replace_from = match (first_conflicting_height, first_missing_height) { + (Some(conflict), _) => Some(conflict), + (None, missing) => missing, + }; - self.zs_delete(&hash_by_height, height); - self.zs_delete(&header_by_height, height); - self.zs_delete(&body_size_by_height, height); - self.zs_delete(&roots_by_height, height); + // Refresh the advisory rows of re-delivered matching headers: merge + // the body-size hints and re-stage the provisional roots. These + // writes never change which header is stored at a height, so they + // stay outside the suffix-replacement primitive. + for (index, (height, _hash, _header, body_size)) in validated_headers.iter().enumerate() { + let height = *height; + if replace_from.is_some_and(|replace_from| height >= replace_from) { + break; } - } - for (index, (height, hash, header, body_size)) in validated_headers.into_iter().enumerate() - { // Finalized block heights already have authoritative block rows and // verified roots, even when pruning has removed their transactions. // Re-delivered headers must not recreate provisional zakura rows @@ -2277,25 +2257,17 @@ impl DiskWriteBatch { continue; } - let same_header = zebra_db.zakura_header_hash(height) == Some(hash); - let advertised_body_size = match ( - same_header, + let merged_body_size = match ( zebra_db.advertised_body_size(height), - AdvertisedBodySize::new(body_size).map(AdvertisedBodySize::get), + AdvertisedBodySize::new(*body_size).map(AdvertisedBodySize::get), ) { - (true, existing, Some(new)) => Some(existing.unwrap_or(0).max(new)), - (true, existing, None) => existing, - (false, _existing, new) => new, + (existing, Some(new)) => Some(existing.unwrap_or(0).max(new)), + (existing, None) => existing, }; - - self.zs_insert(&header_by_height, height, header); - self.zs_insert(&hash_by_height, height, hash); - self.zs_insert(&height_by_hash, hash, height); - if let Some(body_size) = advertised_body_size.and_then(AdvertisedBodySize::new) { + if let Some(body_size) = merged_body_size.and_then(AdvertisedBodySize::new) { self.zs_insert(&body_size_by_height, height, body_size); - } else { - self.zs_delete(&body_size_by_height, height); } + let roots = &tree_aux_roots[index]; self.zs_insert( &roots_by_height, @@ -2312,6 +2284,50 @@ impl DiskWriteBatch { ); } + // Chain membership changes only through the suffix-replacement + // primitive: total deletion above the fork point in all five column + // families, then the new rows. Rows above the incoming range's end + // that belonged to the replaced branch are deleted with it. + if let Some(replace_from) = replace_from { + let fork_height = replace_from + .previous() + .map_err(|_| CommitHeaderRangeError::HeightOverflow)?; + let fork_hash = if fork_height == anchor_height { + anchor + } else { + // The fork row is inside the incoming range and matched the + // store during validation. + let fork_index = usize::try_from(fork_height.0 - anchor_height.0 - 1) + .expect("fork height is above the anchor"); + validated_headers[fork_index].1 + }; + + let new_rows: Vec = validated_headers + .iter() + .enumerate() + .filter(|(_, (height, ..))| *height >= replace_from) + .map(|(index, (_height, _hash, header, body_size))| { + let roots = &tree_aux_roots[index]; + CanonicalHeaderRow { + header: (*header).clone(), + advertised_body_size: AdvertisedBodySize::new(*body_size) + .map(AdvertisedBodySize::get), + roots: Some(CommitmentRootsByHeight { + sapling: roots.sapling_root, + orchard: roots.orchard_root, + ironwood: roots.ironwood_root, + sapling_tx: roots.sapling_tx, + orchard_tx: roots.orchard_tx, + ironwood_tx: roots.ironwood_tx, + auth_data_root: roots.auth_data_root, + }), + } + }) + .collect(); + + self.set_canonical_suffix(zebra_db, (fork_height, fork_hash), &new_rows)?; + } + Ok(block::Hash::from( &**headers.last().expect("headers is non-empty"), )) diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/canonical_suffix.rs b/zebra-state/src/service/finalized_state/zebra_db/block/canonical_suffix.rs new file mode 100644 index 00000000000..9300af7f245 --- /dev/null +++ b/zebra-state/src/service/finalized_state/zebra_db/block/canonical_suffix.rs @@ -0,0 +1,263 @@ +//! The single owner of zakura header-store chain membership: +//! [`DiskWriteBatch::set_canonical_suffix`] (REORG_PLAN Pillar 1). +//! +//! The zakura header store is five column families acting as a replicated +//! view of the canonical header chain above the finalized tip. Historically +//! it was mutated by four independent writers, each with its own bounded +//! delete loop; every proven corruption class (unlinked anchors, re-inserts +//! below the body tip, unlinked seeds, stranded suffixes) came from one of +//! those writers preserving the store invariants only under assumptions +//! about the others. +//! +//! This module replaces the per-writer delete/insert logic with one +//! primitive: *replace the suffix above a fork point*. Deletion is +//! total-above-fork by construction — it iterates the actual on-disk rows of +//! every column family, never a cached or computed tip — so stranding is +//! impossible. Linkage is a hard precondition, not a validation step callers +//! can forget. Committed heights are refused for deletes and inserts alike. +//! +//! Callers stay responsible for *policy*: contextual validation, checkpoint +//! conflicts, the cumulative-work gate, and reorg-depth limits all happen +//! before the primitive is invoked. The primitive owns *mechanism*: which +//! rows exist after a suffix replacement, in all five column families, in +//! one atomic batch. + +use std::sync::Arc; + +use zebra_chain::block::{self, Height}; + +use super::{ + AdvertisedBodySize, ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT, ZAKURA_HEADER_BY_HEIGHT, + ZAKURA_HEADER_HASH_BY_HEIGHT, ZAKURA_HEADER_HEIGHT_BY_HASH, +}; +use crate::{ + error::{CommitHeaderRangeError, StoreIncoherentError}, + service::finalized_state::{ + disk_db::{DiskWriteBatch, WriteDisk}, + disk_format::shielded::CommitmentRootsByHeight, + zebra_db::ZebraDb, + COMMITMENT_ROOTS_BY_HEIGHT, + }, +}; + +/// One header row of a new canonical suffix. +/// +/// The row's height is implied by its position: `fork_point + 1 + index`. +/// Its hash is computed from the header inside the primitive, so a caller +/// cannot stage a hash that disagrees with the stored header. +#[derive(Clone, Debug)] +pub(crate) struct CanonicalHeaderRow { + /// The header to store. + pub header: Arc, + + /// The advisory body-size hint, if known. Not consensus data; `None` + /// stores no row (and removes any stale hint left at this height). + pub advertised_body_size: Option, + + /// Provisional commitment roots for this height, if the caller has them. + /// `None` stores no row (the seed path has no roots for its block). + pub roots: Option, +} + +impl DiskWriteBatch { + /// Replaces the zakura header-store suffix above `fork_point` with + /// `new_rows`, in this batch. + /// + /// This is the **only** way any path may change which headers the zakura + /// store considers canonical above a height. In one atomic batch it: + /// + /// 1. deletes every row strictly above `fork_point` in all five header + /// column families — iterating each family's actual on-disk rows, so + /// stranded rows above gaps are removed too — including every + /// `height_by_hash` entry pointing above the fork (displaced and + /// already-orphaned entries alike), then + /// 2. writes `new_rows` at consecutive heights starting at + /// `fork_point + 1`. + /// + /// # Preconditions (hard errors, nothing staged on failure) + /// + /// - `fork_point` must be at or above the finalized tip + /// ([`CommitHeaderRangeError::ImmutableConflict`]): committed heights + /// are immutable through this primitive, which also guarantees no + /// insert lands at a committed height (the re-delivery bug shape). + /// - When `new_rows` is non-empty, the stored header row at + /// `fork_point.0` must be `fork_point.1` + /// ([`StoreIncoherentError::BijectionMismatch`]): the caller's fork + /// decision must describe the store being mutated. + /// - `new_rows[0]` must link to `fork_point.1` and every row to its + /// predecessor ([`CommitHeaderRangeError::UnlinkedRange`]): linkage is + /// structural here, not a caller obligation. + /// - No height above `fork_point` may hold a committed full block body + /// ([`CommitHeaderRangeError::ConflictingFullBlockHeader`]). Bodies + /// above the finalized tip cannot exist, so this is a safety interlock: + /// reaching it means the caller's orchestration is buggy, and the + /// switch aborts loudly instead of deleting a body's header row. + /// + /// `new_rows` may be empty: that truncates the store to `fork_point` + /// (the body-commit path drops conflicting provisional descendants this + /// way). An empty replacement skips the fork-row check, because the + /// caller may be staging a new full-block row at the fork height in this + /// same batch, which reads through `zebra_db` cannot see yet. + #[allow(clippy::unwrap_in_result)] + pub(crate) fn set_canonical_suffix( + &mut self, + zebra_db: &ZebraDb, + fork_point: (Height, block::Hash), + new_rows: &[CanonicalHeaderRow], + ) -> Result<(), CommitHeaderRangeError> { + let (fork_height, fork_hash) = fork_point; + + // Committed heights are immutable: the finalized tip is the highest + // committed block, so requiring the fork at or above it keeps every + // delete *and* insert strictly above committed history (valid on + // pruned stores too, where bodies are absent but heights committed). + if let Some(finalized_tip_height) = zebra_db.finalized_tip_height() { + if fork_height < finalized_tip_height { + return Err(CommitHeaderRangeError::ImmutableConflict { + height: fork_height + .next() + .map_err(|_| CommitHeaderRangeError::HeightOverflow)?, + }); + } + } + + // The fork row named by the caller must be the row on disk: a + // mismatch means the fork decision describes a different store state + // than the one being mutated. + if !new_rows.is_empty() { + let stored = zebra_db + .header_hash(fork_height) + .or_else(|| (fork_hash == zebra_db.network().genesis_hash()).then_some(fork_hash)); + if stored != Some(fork_hash) { + return Err(StoreIncoherentError::BijectionMismatch { + hash: fork_hash, + height: fork_height, + stored, + } + .into()); + } + } + + // Structural linkage: the new suffix must chain from the fork row. + let mut expected_parent = fork_hash; + let mut hashes = Vec::with_capacity(new_rows.len()); + for (index, row) in new_rows.iter().enumerate() { + let offset = + u32::try_from(index + 1).map_err(|_| CommitHeaderRangeError::HeightOverflow)?; + let height = + (fork_height + i64::from(offset)).ok_or(CommitHeaderRangeError::HeightOverflow)?; + + if row.header.previous_block_hash != expected_parent { + return Err(CommitHeaderRangeError::UnlinkedRange { + height, + expected_parent, + actual_parent: row.header.previous_block_hash, + }); + } + + let hash = block::Hash::from(&*row.header); + expected_parent = hash; + hashes.push((height, hash)); + } + + let header_cf = zebra_db.db.cf_handle(ZAKURA_HEADER_BY_HEIGHT).unwrap(); + let hash_cf = zebra_db.db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); + let height_by_hash_cf = zebra_db.db.cf_handle(ZAKURA_HEADER_HEIGHT_BY_HASH).unwrap(); + let body_size_cf = zebra_db + .db + .cf_handle(ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT) + .unwrap(); + let roots_cf = zebra_db.db.cf_handle(COMMITMENT_ROOTS_BY_HEIGHT).unwrap(); + + let Ok(delete_from) = fork_height.next() else { + // The fork is at the maximum height: nothing can exist above it, + // and the linkage loop already rejected any new rows. + return Ok(()); + }; + + // Total-above-fork deletion: walk the actual on-disk rows of every + // column family from the fork upward. No cached or computed tip + // bounds the loops, so rows stranded above gaps are deleted too. + // + // The committed-body interlock runs on the stored hash rows *and* + // the insert heights before anything is staged. + let stored_hash_rows: Vec<(Height, block::Hash)> = zebra_db + .db + .zs_forward_range_iter(&hash_cf, delete_from..) + .collect(); + for &(height, _) in &stored_hash_rows { + if zebra_db.contains_body_at_height(height) { + return Err(CommitHeaderRangeError::ConflictingFullBlockHeader { height }); + } + } + for &(height, _) in &hashes { + if zebra_db.contains_body_at_height(height) { + return Err(CommitHeaderRangeError::ConflictingFullBlockHeader { height }); + } + } + + let mut deleted_rows = 0; + for (height, _displaced_hash) in stored_hash_rows { + self.zs_delete(&hash_cf, height); + deleted_rows += 1; + } + + // The reverse index is hash-keyed, so deriving its deletions from the + // hash rows would miss entries already orphaned by earlier damage (a + // deleted hash row leaves its reverse entry behind). Scan the whole + // reverse index — it is frontier-sized — and delete every entry + // pointing above the fork, displaced and orphaned alike. + for (hash, points_at) in zebra_db + .db + .zs_forward_range_iter::<_, block::Hash, Height, _>(&height_by_hash_cf, ..) + { + if points_at > fork_height { + self.zs_delete(&height_by_hash_cf, hash); + deleted_rows += 1; + } + } + for (height, _) in zebra_db + .db + .zs_forward_range_iter::<_, Height, Arc, _>(&header_cf, delete_from..) + { + self.zs_delete(&header_cf, height); + deleted_rows += 1; + } + for (height, _) in zebra_db + .db + .zs_forward_range_iter::<_, Height, AdvertisedBodySize, _>(&body_size_cf, delete_from..) + { + self.zs_delete(&body_size_cf, height); + deleted_rows += 1; + } + for (height, _) in zebra_db + .db + .zs_forward_range_iter::<_, Height, CommitmentRootsByHeight, _>( + &roots_cf, + delete_from.., + ) + { + self.zs_delete(&roots_cf, height); + deleted_rows += 1; + } + + // A batch that deletes rows performs a header reorg: the write site + // audits the store after committing it (the Pillar-3 hook). + self.note_zakura_suffix_replacement(deleted_rows); + + // Write the new suffix. + for (row, &(height, hash)) in new_rows.iter().zip(&hashes) { + self.zs_insert(&header_cf, height, &row.header); + self.zs_insert(&hash_cf, height, hash); + self.zs_insert(&height_by_hash_cf, hash, height); + if let Some(body_size) = row.advertised_body_size.and_then(AdvertisedBodySize::new) { + self.zs_insert(&body_size_cf, height, body_size); + } + if let Some(roots) = &row.roots { + self.zs_insert(&roots_cf, height, *roots); + } + } + + Ok(()) + } +} diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/startup_audit.rs b/zebra-state/src/service/finalized_state/zebra_db/block/startup_audit.rs index 994d4de9586..5d355f77d13 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/startup_audit.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/startup_audit.rs @@ -238,11 +238,11 @@ impl ZebraDb { && body_size_heights.is_empty() && provisional_root_heights.is_empty() { - // Log the pass so operators can verify the audit ran on this boot. + // Log the pass so operators can verify the audit ran. tracing::info!( ?finalized_tip, frontier_rows = 0, - "zakura header store passed its startup coherence audit (empty frontier)" + "zakura header store passed its coherence audit (empty frontier)" ); return Ok(None); } @@ -415,7 +415,7 @@ impl ZebraDb { ?finalized_tip, ?last_coherent, frontier_rows = hashes.len(), - "zakura header store passed its startup coherence audit" + "zakura header store passed its coherence audit" ); return Ok(None); } @@ -427,7 +427,7 @@ impl ZebraDb { deleted_rows, violation_count = violations.len(), first_violations = ?&violations[..violations.len().min(LOGGED_VIOLATIONS)], - "zakura header store failed its startup coherence audit; \ + "zakura header store failed its coherence audit; \ truncating to the last coherent height so header sync re-downloads the rest" ); diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence.rs index 669225c0930..b0adbdc2c23 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence.rs @@ -31,6 +31,7 @@ mod audit; mod fabricate; mod ops; mod oracle; +mod primitive; mod prop; mod reads; mod scenarios; diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/primitive.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/primitive.rs new file mode 100644 index 00000000000..fc9c702ae17 --- /dev/null +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/primitive.rs @@ -0,0 +1,282 @@ +//! Contract tests for `set_canonical_suffix`, the single owner of zakura +//! header-store chain membership (REORG_PLAN Pillar 1). +//! +//! The coherence harness exercises the primitive through the production +//! writers (range, seed, release); these tests pin the primitive's own +//! contract directly: preconditions reject without staging anything, +//! deletion is total-above-fork across all five column families (including +//! rows stranded above hand-made gaps), an empty replacement truncates, and +//! a pure append stages no deletions (so the post-reorg audit hook does not +//! fire for it). + +use zebra_chain::block::Height; + +use super::super::super::CanonicalHeaderRow; +use super::super::super::{ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT, ZAKURA_HEADER_HASH_BY_HEIGHT}; +use super::super::common::{ + commit_header_range, state_with_genesis_config, write_full_block_header_and_transactions, +}; +use super::{ + audit::{audit_store, dump_store}, + fabricate::{fabricate_body, FabHeader, Universe, BRANCH_A, FORK_HEIGHT}, +}; +use crate::{ + error::{CommitHeaderRangeError, StoreIncoherentError}, + service::finalized_state::{ + disk_db::{DiskWriteBatch, WriteDisk}, + disk_format::block::TransactionLocation, + RawBytes, ZebraDb, + }, + Config, +}; + +fn trunk_state(universe: &Universe) -> ZebraDb { + let state = state_with_genesis_config( + &universe.network, + universe.genesis.clone(), + Config::ephemeral(), + ); + let trunk_headers: Vec<_> = universe.trunk[..FORK_HEIGHT as usize] + .iter() + .map(|fab| fab.header.clone()) + .collect(); + commit_header_range(&state, universe.genesis.hash(), &trunk_headers); + state +} + +fn rows_of(fabs: &[FabHeader]) -> Vec { + fabs.iter() + .map(|fab| CanonicalHeaderRow { + header: fab.header.clone(), + advertised_body_size: None, + roots: None, + }) + .collect() +} + +fn assert_clean(state: &ZebraDb) { + let violations = audit_store(state); + assert!( + violations.is_empty(), + "unexpected violations: {violations:?}" + ); +} + +/// New rows that do not link to the fork row are rejected before anything is +/// staged. +#[test] +fn rejects_unlinked_new_rows_without_staging() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe); + let original = dump_store(&state); + + // Branch-A rows link at the fork height (50), not at 30. + let rows = rows_of(&universe.branches[BRANCH_A].headers[..3]); + let mut batch = DiskWriteBatch::new(); + let error = batch + .set_canonical_suffix(&state, (Height(30), universe.trunk_at(30).hash), &rows) + .expect_err("rows do not link to trunk@30"); + assert!( + matches!( + error, + CommitHeaderRangeError::UnlinkedRange { height, .. } if height == Height(31) + ), + "expected UnlinkedRange at 31, got {error:?}" + ); + assert_eq!(batch.zakura_suffix_replaced_rows(), 0); + + state.write_batch(batch).expect("empty batch writes"); + assert_eq!(dump_store(&state), original, "a rejection stages nothing"); +} + +/// A fork hash that is not the stored row at the fork height is rejected: +/// the caller's fork decision must describe the store being mutated. +#[test] +fn rejects_stale_fork_row() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe); + let original = dump_store(&state); + + // These rows link to branch-A's fourth row, which is not stored at 30. + let foreign_fork = &universe.branches[BRANCH_A].headers[3]; + let rows = rows_of(&universe.branches[BRANCH_A].headers[4..6]); + let mut batch = DiskWriteBatch::new(); + let error = batch + .set_canonical_suffix(&state, (Height(30), foreign_fork.hash), &rows) + .expect_err("the stored row at 30 is the trunk's"); + assert!( + matches!( + error, + CommitHeaderRangeError::StoreIncoherent(StoreIncoherentError::BijectionMismatch { + height, + .. + }) if height == Height(30) + ), + "expected BijectionMismatch at the fork row, got {error:?}" + ); + assert_eq!(batch.zakura_suffix_replaced_rows(), 0); + assert_eq!(dump_store(&state), original); +} + +/// A fork below the finalized tip is rejected: committed heights are +/// immutable through the primitive. +#[test] +fn rejects_fork_below_finalized_tip() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe); + + // Commit the body of trunk height 1: the finalized tip moves to 1. + write_full_block_header_and_transactions(&state, fabricate_body(&universe.trunk[0])); + assert_eq!(state.finalized_tip_height(), Some(Height(1))); + + let rows = rows_of(&universe.trunk[..2]); + let mut batch = DiskWriteBatch::new(); + let error = batch + .set_canonical_suffix(&state, (Height(0), universe.genesis.hash()), &rows) + .expect_err("the fork is below the finalized tip"); + assert!( + matches!( + error, + CommitHeaderRangeError::ImmutableConflict { height } if height == Height(1) + ), + "expected ImmutableConflict, got {error:?}" + ); + assert_eq!(batch.zakura_suffix_replaced_rows(), 0); +} + +/// The committed-body safety interlock: a body row above the fork aborts the +/// replacement before anything is staged, even when the finalized-tip bound +/// cannot see it. +#[test] +fn body_above_fork_aborts_loudly() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe); + + // Hand-plant a body marker above the finalized tip — a state the + // orchestration must never produce. + let tx_by_loc = state.db.cf_handle("tx_by_loc").unwrap(); + let mut batch = DiskWriteBatch::new(); + batch.zs_insert( + &tx_by_loc, + TransactionLocation::min_for_height(Height(45)), + RawBytes::new_raw_bytes(vec![0u8]), + ); + state.db.write(batch).expect("raw insert writes"); + + let rows = rows_of(&universe.trunk[30..35]); + let mut batch = DiskWriteBatch::new(); + let error = batch + .set_canonical_suffix(&state, (Height(30), universe.trunk_at(30).hash), &rows) + .expect_err("a body above the fork must abort the replacement"); + assert!( + matches!( + error, + CommitHeaderRangeError::ConflictingFullBlockHeader { height } if height == Height(45) + ), + "expected ConflictingFullBlockHeader at 45, got {error:?}" + ); + assert_eq!(batch.zakura_suffix_replaced_rows(), 0); +} + +/// Deletion is total-above-fork: rows stranded above a hand-made gap and +/// stray rows in single column families are deleted along with the linked +/// suffix, in all five column families. +#[test] +fn truncation_is_total_above_fork_including_strays() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe); + + // A gap at 40 strands rows 41..=50; a stray body-size row sits at 60, + // above the zakura tip. + let hash_cf = state.db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); + let body_size_cf = state + .db + .cf_handle(ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT) + .unwrap(); + let mut batch = DiskWriteBatch::new(); + batch.zs_delete(&hash_cf, Height(40)); + batch.zs_insert( + &body_size_cf, + Height(60), + super::super::super::test_body_size(7), + ); + state.db.write(batch).expect("raw writes succeed"); + + let rows = rows_of(&universe.trunk[30..35]); + let mut batch = DiskWriteBatch::new(); + batch + .set_canonical_suffix(&state, (Height(30), universe.trunk_at(30).hash), &rows) + .expect("the replacement is well-formed"); + assert!(batch.zakura_suffix_replaced_rows() > 0); + state.write_batch(batch).expect("replacement batch writes"); + + assert_eq!( + state.best_header_tip(), + Some((Height(35), universe.trunk_at(35).hash)) + ); + let dump = dump_store(&state); + assert_eq!(dump.hashes.keys().next_back(), Some(&Height(35))); + assert!(!dump.body_sizes.contains_key(&Height(60))); + assert_clean(&state); +} + +/// An empty replacement truncates the store to the fork point. +#[test] +fn empty_replacement_truncates_to_fork() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe); + + let mut batch = DiskWriteBatch::new(); + batch + .set_canonical_suffix(&state, (Height(40), universe.trunk_at(40).hash), &[]) + .expect("an empty replacement is a truncation"); + assert!(batch.zakura_suffix_replaced_rows() > 0); + state.write_batch(batch).expect("truncation batch writes"); + + assert_eq!( + state.best_header_tip(), + Some((Height(40), universe.trunk_at(40).hash)) + ); + assert_clean(&state); +} + +/// A pure append deletes nothing, so it does not count as a reorg (the +/// post-reorg audit hook stays quiet for the steady-state extension path). +#[test] +fn pure_append_replaces_nothing() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe); + + // Branch A forks at the trunk tip, so its rows extend the stored chain. + let rows = rows_of(&universe.branches[BRANCH_A].headers[..3]); + let mut batch = DiskWriteBatch::new(); + batch + .set_canonical_suffix( + &state, + (Height(FORK_HEIGHT), universe.trunk_at(FORK_HEIGHT).hash), + &rows, + ) + .expect("an extension at the tip is well-formed"); + assert_eq!( + batch.zakura_suffix_replaced_rows(), + 0, + "a pure append is not a reorg" + ); + state.write_batch(batch).expect("append batch writes"); + + assert_eq!( + state.best_header_tip(), + Some(( + Height(FORK_HEIGHT + 3), + universe.branches[BRANCH_A].headers[2].hash + )) + ); + assert_clean(&state); +} diff --git a/zebra-state/src/service/write.rs b/zebra-state/src/service/write.rs index 0452d20e642..178a97818d2 100644 --- a/zebra-state/src/service/write.rs +++ b/zebra-state/src/service/write.rs @@ -146,10 +146,16 @@ fn commit_header_range( &tree_aux_roots, ) .and_then(|hash| { + let zakura_replaced_rows = batch.zakura_suffix_replaced_rows(); finalized_state .db .write_batch(batch) - .map(|()| hash) + .map(|()| { + finalized_state + .db + .audit_zakura_header_store_after_reorg(zakura_replaced_rows); + hash + }) .map_err(|error| { tracing::error!(?error, "failed to write validated header range"); From 60b17414fc701334cdc204599e106374a9e70a0e Mon Sep 17 00:00:00 2001 From: roman Date: Mon, 6 Jul 2026 22:30:28 -0600 Subject: [PATCH 8/8] test(state): exercise pruning in the header-store coherence harness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the REORG_PLAN §1c residual (INTEGRATION_TEST_PLAN.md T2): - Op::Prune joins the op alphabet, driving the production prepare_prune_batch below the finalized body tip with a monotone marker. Pruning is membership-neutral: the harness asserts the zakura store is byte-identical, pruned heights keep their consensus hash rows and lose their bodies, and the oracle's predictions are unchanged. Added to the discovery proptest distribution (1024-case sweep green) and pinned by scenario s11 (prune → reorg → restart → deeper prune). - startup_removes_stale_rows_at_pruned_heights: hand-planted zakura rows at actually-pruned heights (the pre-#491 bug-2 shape: contains_height true, contains_body_at_height false) are classified StaleRowAtCommittedHeight and deleted, while a verified roots row at a pruned height survives untouched — the test the audit's contains_height predicate choice was made for. - Fixes the harness-side audit model the new test exposed: the test audit's roots-row rule used body presence, which would misclassify pruning-retained verified roots rows as violations; it now uses the committed-height predicate like the production audit. --- .../tests/header_store_coherence/audit.rs | 7 +- .../block/tests/header_store_coherence/ops.rs | 51 +++++++ .../tests/header_store_coherence/prop.rs | 1 + .../tests/header_store_coherence/scenarios.rs | 35 +++++ .../header_store_coherence/startup_audit.rs | 131 +++++++++++++++++- 5 files changed, 220 insertions(+), 5 deletions(-) diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/audit.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/audit.rs index 05a8226d703..54b6f78bba8 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/audit.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/audit.rs @@ -274,7 +274,10 @@ pub(crate) fn audit_store(state: &ZebraDb) -> Vec { } // A3: body-size rows require a backing zakura header row; roots rows - // require a zakura header row or a committed body. + // require a zakura header row or a committed height. The committed + // predicate is `contains_height` (the consensus hash row, which pruning + // retains), not body presence: verified roots rows at committed heights + // survive pruning and are legitimate, body or no body. for &height in dump.body_sizes.keys() { if !dump.hashes.contains_key(&height) { violations.push(Violation::AuxRowWithoutHeader { @@ -284,7 +287,7 @@ pub(crate) fn audit_store(state: &ZebraDb) -> Vec { } } for &height in dump.roots.keys() { - if !dump.hashes.contains_key(&height) && !state.contains_body_at_height(height) { + if !dump.hashes.contains_key(&height) && !state.contains_height(height) { violations.push(Violation::AuxRowWithoutHeader { cf: COMMITMENT_ROOTS_BY_HEIGHT, height, diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/ops.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/ops.rs index ad36b6556d1..a3a041b1c13 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/ops.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/ops.rs @@ -10,6 +10,8 @@ //! (the non-finalized best-chain commit hook); //! - [`Op::Finalize`] → sequential body commits along the expected canonical //! chain; +//! - [`Op::Prune`] → `DiskWriteBatch::prepare_prune_batch` (online pruning of +//! raw transactions below the body tip; no chain-membership change); //! - [`Op::Reopen`] → close and reopen the database (restart survival). //! //! After every op the harness cross-checks the oracle's prediction against the @@ -80,6 +82,11 @@ pub(crate) enum Op { Finalize { count: usize, }, + /// Prunes raw transactions from the last pruning marker up to (but not + /// including) `min(until, body tip)`, through the production prune batch. + Prune { + until: u32, + }, Reopen, } @@ -376,6 +383,50 @@ impl Harness { } } + Op::Prune { until } => { + // Pruning applies below the finalized body tip and never + // re-prunes below the existing marker (the marker is + // monotone in production). + let Some(tip) = self.state().finalized_tip_height() else { + return self.finish(OpOutcome::Skipped("no finalized tip"), mismatches); + }; + let from = self + .state() + .lowest_retained_height() + .unwrap_or(block::Height(1)); + let until = block::Height((*until).min(tip.0)); + if until <= from { + return self.finish(OpOutcome::Skipped("empty prune range"), mismatches); + } + + let dump_before = dump_store(self.state()); + let mut batch = DiskWriteBatch::new(); + batch.prepare_prune_batch(self.state(), from, until); + self.state() + .write_batch(batch) + .expect("prune batch writes successfully"); + + // Pruning removes raw transactions and advances the marker; + // chain membership and the zakura header store must be + // untouched (the oracle's predictions are unchanged). + if dump_store(self.state()) != dump_before { + mismatches.push(format!("pruning mutated the zakura header store: {op:?}")); + } + for height in from.0..until.0 { + let height = block::Height(height); + if self.state().contains_body_at_height(height) { + mismatches + .push(format!("pruned height {height:?} still has a body: {op:?}")); + } + if !self.state().contains_height(height) { + mismatches.push(format!( + "pruning removed the consensus hash row at {height:?}: {op:?}" + )); + } + } + OpOutcome::Accepted + } + Op::Reopen => { let dump_before = dump_store(self.state()); let mut state = self.state.take().expect("state is present before Reopen"); diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/prop.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/prop.rs index 4a363411354..8b39d126982 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/prop.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/prop.rs @@ -58,6 +58,7 @@ fn op_strategy() -> impl Strategy { .prop_map(|(source, index)| Op::CommitBody { source, index }), 2 => (source_strategy(), 0..40usize).prop_map(|(source, index)| Op::Seed { source, index }), 3 => (1..8usize).prop_map(|count| Op::Finalize { count }), + 1 => (2..=TRUNK_LEN as u32).prop_map(|until| Op::Prune { until }), 1 => Just(Op::Reopen), ] } diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/scenarios.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/scenarios.rs index 6e41141a72b..072bd4eb4df 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/scenarios.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/scenarios.rs @@ -539,6 +539,41 @@ fn s10_seed_interplay() { ); } +/// s11: online pruning below the body tip is membership-neutral — the zakura +/// frontier, chain selection, reorgs, and restart survival are unaffected — +/// and it leaves the pruned-store shape (consensus hash rows retained, +/// bodies gone) the startup audit's committed-height predicate relies on. +#[test] +fn s11_prune_below_body_tip_is_membership_neutral() { + let _init_guard = zebra_test::init(); + let mut harness = Harness::new(); + + let outcomes = harness + .run_all(&[ + commit_trunk(), + Op::Finalize { count: 10 }, + Op::Prune { until: 8 }, + // A reorg above the fork still works over a pruned base. + commit_branch(BRANCH_A), + Op::Reopen, + // Deeper pruning after the reorg and restart is still clean. + Op::Finalize { count: 2 }, + Op::Prune { until: 12 }, + ]) + .expect("pruning sequence has no violations"); + outcomes.iter().for_each(assert_accepted); + + assert_eq!( + harness.state().best_header_tip(), + Some(branch_tip(BRANCH_A)), + ); + // The pruned-store shape: heights stay committed, bodies are gone. + assert!(harness.state().contains_height(Height(5))); + assert!(!harness.state().contains_body_at_height(Height(5))); + assert!(!harness.state().contains_body_at_height(Height(11))); + assert!(harness.state().contains_body_at_height(Height(12))); +} + /// s11: a refused (unlinked) seed converges through header-range sync. The /// zakura store follows branch A (seeded at the fork height); the /// non-finalized best chain switches to branch B and its new best tip (B's diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/startup_audit.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/startup_audit.rs index 1ece0ee84cf..933aa3e7b89 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/startup_audit.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/startup_audit.rs @@ -27,20 +27,23 @@ use zebra_chain::block::Height; use super::super::super::startup_audit::ZakuraStoreViolation; use super::super::super::{ - ZAKURA_HEADER_BY_HEIGHT, ZAKURA_HEADER_HASH_BY_HEIGHT, ZAKURA_HEADER_HEIGHT_BY_HASH, + test_body_size, ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT, ZAKURA_HEADER_BY_HEIGHT, + ZAKURA_HEADER_HASH_BY_HEIGHT, ZAKURA_HEADER_HEIGHT_BY_HASH, }; use super::super::common::{ commit_header_range, persistent_config, persistent_state, state_with_genesis_config, + write_full_block_header_and_transactions, }; use super::{ audit::{audit_store, dump_store, StoreDump}, - fabricate::{Universe, BRANCH_A, FORK_HEIGHT}, + fabricate::{fabricate_body, Universe, BRANCH_A, FORK_HEIGHT}, }; use crate::{ error::{CommitHeaderRangeError, StoreIncoherentError}, service::finalized_state::{ disk_db::{DiskWriteBatch, WriteDisk}, - ZebraDb, + disk_format::shielded::CommitmentRootsByHeight, + ZebraDb, COMMITMENT_ROOTS_BY_HEIGHT, }, Config, }; @@ -537,3 +540,125 @@ fn startup_repair_emits_incoherent_metric() { assert_eq!(clean.get(METRIC), 0, "a clean reopen emits nothing"); assert_clean(&state); } + +/// Stale zakura rows at *pruned* heights are classified +/// `StaleRowAtCommittedHeight` and deleted, while everything else — the +/// zakura frontier and the roots rows at committed heights — is untouched. +/// +/// This is the test the audit's committed-height predicate was chosen for: +/// the predicate is the consensus `hash_by_height` row, which pruning +/// retains, not body presence (pruning removes bodies). A body-presence +/// predicate would misclassify these rows as frontier rows and let the +/// pre-#491 bug-2 shape (stale re-delivered rows below the body tip) +/// survive on pruned stores. +#[test] +fn startup_removes_stale_rows_at_pruned_heights() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let tempdir = tempfile::tempdir().expect("test tempdir is available"); + let config = persistent_config(tempdir.path()); + let state = trunk_state(&universe, config.clone()); + + // Commit bodies 1..=10 (releasing the zakura rows there), then prune + // raw transactions below height 8 through the production prune batch. + for fab in &universe.trunk[..10] { + write_full_block_header_and_transactions(&state, fabricate_body(fab)); + } + let mut batch = DiskWriteBatch::new(); + batch.prepare_prune_batch(&state, Height(1), Height(8)); + state.write_batch(batch).expect("prune batch writes"); + + // The pruned-store shape: committed heights whose bodies are gone. + assert!(state.contains_height(Height(5))); + assert!(!state.contains_body_at_height(Height(5))); + assert!(state.contains_body_at_height(Height(8))); + + // A verified roots row at a pruned height (pruning retains these): the + // audit must leave it untouched — committed heights are out of scope. + let roots_cf = state.db.cf_handle(COMMITMENT_ROOTS_BY_HEIGHT).unwrap(); + let mut batch = DiskWriteBatch::new(); + batch.zs_insert( + &roots_cf, + Height(5), + CommitmentRootsByHeight { + sapling: zebra_chain::sapling::tree::NoteCommitmentTree::default().root(), + orchard: zebra_chain::orchard::tree::NoteCommitmentTree::default().root(), + auth_data_root: zebra_chain::block::merkle::AuthDataRoot::from([7u8; 32]), + ironwood: zebra_chain::ironwood::tree::NoteCommitmentTree::default().root(), + sapling_tx: 0, + orchard_tx: 0, + ironwood_tx: 0, + }, + ); + state.db.write(batch).expect("raw insert writes"); + + let clean = dump_store(&state); + assert!( + clean.roots.contains_key(&Height(5)), + "the roots row at a pruned height is present and out of audit scope" + ); + + // Hand-plant stale zakura rows at pruned heights: a full row triple at + // height 5 and a body-size hint at height 3 (the pre-guard re-delivery + // shape: `contains_height` true, `contains_body_at_height` false). + let header_cf = state.db.cf_handle(ZAKURA_HEADER_BY_HEIGHT).unwrap(); + let hash_cf = state.db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); + let reverse_cf = state.db.cf_handle(ZAKURA_HEADER_HEIGHT_BY_HASH).unwrap(); + let body_size_cf = state + .db + .cf_handle(ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT) + .unwrap(); + let stale = &universe.trunk[4]; + let mut batch = DiskWriteBatch::new(); + batch.zs_insert(&header_cf, Height(5), stale.header.clone()); + batch.zs_insert(&hash_cf, Height(5), stale.hash); + batch.zs_insert(&reverse_cf, stale.hash, Height(5)); + batch.zs_insert(&body_size_cf, Height(3), test_body_size(77)); + state.db.write(batch).expect("raw insert writes"); + + let repair = state + .audit_and_repair_zakura_header_store() + .expect("audit reads and writes succeed") + .expect("the stale rows are repaired"); + assert_eq!(repair.deleted_rows, 4); + for (cf, height) in [ + (ZAKURA_HEADER_BY_HEIGHT, Height(5)), + (ZAKURA_HEADER_HASH_BY_HEIGHT, Height(5)), + (ZAKURA_HEADER_HEIGHT_BY_HASH, Height(5)), + (ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT, Height(3)), + ] { + assert!( + repair.violations.iter().any(|violation| matches!( + violation, + ZakuraStoreViolation::StaleRowAtCommittedHeight { + cf: found_cf, + height: found_height, + } if *found_cf == cf && *found_height == height + )), + "expected StaleRowAtCommittedHeight for {cf} at {height:?}: {:?}", + repair.violations + ); + } + assert_eq!( + repair.last_coherent, + Some((Height(FORK_HEIGHT), universe.trunk_at(FORK_HEIGHT).hash)), + "stale rows at pruned heights must not shorten the coherent frontier" + ); + + // The repair restores exactly the pre-corruption store: the frontier + // and the out-of-scope roots rows at committed heights survive. + assert_eq!(dump_store(&state), clean); + assert_clean(&state); + + // The same heal runs on the real startup path (reopen). + let mut batch = DiskWriteBatch::new(); + batch.zs_insert(&hash_cf, Height(6), universe.trunk[5].hash); + state.db.write(batch).expect("raw insert writes"); + let state = reopen(state, &config, &universe); + assert_eq!( + dump_store(&state), + clean, + "the startup repair removes the stale pruned-height row" + ); + assert_clean(&state); +}