From 63b74cbfbd984db68dc3011b7c6a6915d88e385d Mon Sep 17 00:00:00 2001 From: roman Date: Mon, 6 Jul 2026 16:35:19 -0600 Subject: [PATCH 01/10] fix(state): validate linkage in zakura header-store writers Close the three write-path corruption bugs proven by the Phase-1 header-store coherence suite (PR #490): 1. prepare_header_range_batch_with_roots now rejects ranges whose first header does not link to the anchor or whose headers are not internally contiguous (new CommitHeaderRangeError::UnlinkedRange, classified as a local non-scoring failure in zebrad). 2. The range insert loop skips heights that already have a committed block, so re-deliveries can no longer strand provisional zakura rows below the body tip. 3. prepare_zakura_header_from_committed_block refuses seeds that do not link to the stored row below them as silent no-ops; header-range sync converges the store instead (scenario s11). Flip the four *_upholds_invariants twins to permanent regression gates, delete the corruption_repro_* tests, remove the discovery masking, and un-ignore the random invariant sweep (clean at PROPTEST_CASES=4096). --- CHANGELOG.md | 11 + zebra-state/src/error.rs | 15 + .../service/finalized_state/zebra_db/block.rs | 95 +++++-- .../tests/header_store_coherence/README.md | 91 +++--- .../block/tests/header_store_coherence/ops.rs | 72 ++--- .../tests/header_store_coherence/oracle.rs | 8 +- .../tests/header_store_coherence/prop.rs | 32 +-- .../tests/header_store_coherence/scenarios.rs | 266 ++++++++---------- zebra-state/src/service/write.rs | 23 +- zebrad/src/commands/start.rs | 18 ++ .../start/zakura/header_sync_driver.rs | 6 + 11 files changed, 337 insertions(+), 300 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 679fb4e12cc..96f5da37488 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,17 @@ and this project adheres to [Semantic Versioning](https://semver.org). ### Fixed +- Fixed three Zakura header-store write paths that could leave the on-disk + header store internally incoherent after chain forks, causing nodes to + reject valid headers from honest peers (`InvalidDifficultyThreshold` / + `UnknownAnchor`) and wedge below the network tip until manual intervention. + Header ranges must now link to their anchor and be internally contiguous + (rejected with the new `UnlinkedRange` error otherwise, which is classified + as a local, non-peer-scoring failure), header ranges re-delivered over + heights that already have committed block bodies no longer re-insert + provisional header rows below the body tip, and the header row seeded from + a committed best-chain block is skipped when it does not link to the stored + row below it (header-range sync converges the store instead). - Fixed dual-stack Zakura fallback shutting down the Zakura serving layer. When the stall watchdog resumes legacy `ChainSync`, Zakura now keeps its header- and block-sync reactors alive as a serving/advertising bridge while an apply diff --git a/zebra-state/src/error.rs b/zebra-state/src/error.rs index aaf3f124b5e..84a9720dee8 100644 --- a/zebra-state/src/error.rs +++ b/zebra-state/src/error.rs @@ -258,6 +258,21 @@ pub enum CommitHeaderRangeError { #[error("header height overflow")] HeightOverflow, + /// A header in the range does not link to the anchor or to its predecessor, + /// so committing it would break the header store's linkage invariant. + #[error( + "header at {height:?} links to {actual_parent} instead of its predecessor {expected_parent}" + )] + UnlinkedRange { + /// Height of the first header that fails to link. + height: block::Height, + /// The hash of the row the header must link to (the anchor, or the + /// previous header in the range). + expected_parent: block::Hash, + /// The header's actual `previous_block_hash`. + actual_parent: block::Hash, + }, + /// A committed immutable header conflicts with the requested header. #[error("header at finalized height {height:?} conflicts with an existing header")] ImmutableConflict { diff --git a/zebra-state/src/service/finalized_state/zebra_db/block.rs b/zebra-state/src/service/finalized_state/zebra_db/block.rs index 8feead51841..1557b6fd8dc 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block.rs @@ -1797,6 +1797,12 @@ impl DiskWriteBatch { /// Full block verification is authoritative for the stored body. If a /// provisional Zakura header at this height differs, replace it with the /// block-derived header and drop stale provisional descendants. + /// + /// A block whose parent hash does not match the stored header row below + /// `height` is skipped without error: writing it would leave a gap or + /// broken link in the header store, so the store instead waits for + /// header-range sync to deliver the connecting rows (see the linkage + /// refusal below). #[allow(clippy::unwrap_in_result)] pub fn prepare_zakura_header_from_committed_block( &mut self, @@ -1821,6 +1827,32 @@ impl DiskWriteBatch { return Ok(()); } + // Seeds fire at non-finalized best-*tip* commits, so an nf best-tip + // jump (a fork switch between nf chains, or a restart restoring the nf + // backup) can seed a height whose parent row is missing or belongs to + // another branch. Writing that row would break the header store's + // linkage invariant on disk and poison difficulty-adjustment windows + // (`header_store_coherence` bug 3). Refuse the write instead: the + // header store briefly lags the nf chain, and header-range sync + // converges it onto the new branch through the linkage-checked range + // path. The merged header view (full-block rows first, then zakura + // rows) is the same view the chain walk reads. + let hash_by_height = db.cf_handle("hash_by_height").unwrap(); + let parent_hash: Option = height.previous().ok().and_then(|parent_height| { + db.zs_get(&hash_by_height, &parent_height) + .or_else(|| db.zs_get(&zakura_hash_by_height, &parent_height)) + }); + if parent_hash != Some(block.header.previous_block_hash) { + tracing::debug!( + ?height, + ?hash, + parent = ?block.header.previous_block_hash, + stored_parent = ?parent_hash, + "skipping Zakura header seed that does not link to the stored row below it" + ); + return Ok(()); + } + if existing_zakura_header.is_some_and(|existing_header| existing_header != block.header) { let best_header_tip: Option<(block::Height, block::Hash)> = db.zs_last_key_value(&zakura_hash_by_height); @@ -2031,6 +2063,14 @@ impl DiskWriteBatch { let mut first_conflicting_height = None; let mut validated_headers = Vec::with_capacity(headers.len()); + // Each header must link to the anchor (for the first header) or to its + // predecessor in the range. Without this check, a range anchored at the + // same-height hash of a *different* branch can pass contextual + // difficulty validation and commit a suffix that does not link to the + // row below it — an on-disk linkage violation reachable from a single + // peer response. + let mut expected_parent = anchor; + for (index, header) in headers.iter().enumerate() { let offset = u32::try_from(index + 1).map_err(|_| CommitHeaderRangeError::HeightOverflow)?; @@ -2039,6 +2079,16 @@ impl DiskWriteBatch { let hash = block::Hash::from(&**header); let body_size = body_sizes[index]; let roots = &tree_aux_roots[index]; + + if header.previous_block_hash != expected_parent { + return Err(CommitHeaderRangeError::UnlinkedRange { + height, + expected_parent, + actual_parent: header.previous_block_hash, + }); + } + expected_parent = hash; + if roots.height != height { return Err(CommitHeaderRangeError::TreeAuxRootHeightMismatch { expected_height: height, @@ -2161,6 +2211,14 @@ impl DiskWriteBatch { for (index, (height, hash, header, body_size)) in validated_headers.into_iter().enumerate() { + // Finalized block heights already have authoritative block rows and + // verified roots, even when pruning has removed their transactions. + // Re-delivered headers must not recreate provisional zakura rows + // there, because those rows are only trimmed during body commit. + if zebra_db.contains_height(height) { + continue; + } + let same_header = zebra_db.zakura_header_hash(height) == Some(hash); let advertised_body_size = match ( same_header, @@ -2180,29 +2238,20 @@ impl DiskWriteBatch { } else { self.zs_delete(&body_size_by_height, height); } - // A height with a committed body already has a *verified* row in the - // serving index, written by the body-commit batch. Peer-supplied roots - // are unauthenticated until verify-before-commit, and that verification - // only ever runs above the body tip — so a header range re-delivered - // over committed heights (a header store behind the body store, or a - // late range response racing body sync) must never overwrite the - // verified row: committed roots win on any overlap (design §9). - if !zebra_db.contains_body_at_height(height) { - let roots = &tree_aux_roots[index]; - self.zs_insert( - &roots_by_height, - height, - CommitmentRootsByHeight { - sapling: roots.sapling_root, - orchard: roots.orchard_root, - ironwood: roots.ironwood_root, - sapling_tx: roots.sapling_tx, - orchard_tx: roots.orchard_tx, - ironwood_tx: roots.ironwood_tx, - auth_data_root: roots.auth_data_root, - }, - ); - } + let roots = &tree_aux_roots[index]; + self.zs_insert( + &roots_by_height, + height, + CommitmentRootsByHeight { + sapling: roots.sapling_root, + orchard: roots.orchard_root, + ironwood: roots.ironwood_root, + sapling_tx: roots.sapling_tx, + orchard_tx: roots.orchard_tx, + ironwood_tx: roots.ironwood_tx, + auth_data_root: roots.auth_data_root, + }, + ); } Ok(block::Hash::from( diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/README.md b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/README.md index d23a5205a0b..4b49718607a 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/README.md +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/README.md @@ -34,65 +34,64 @@ write instead. | `audit.rs` | The invariant audit: A1 (bijection, both directions), A2 (linkage walk upward from the finalized tip over the merged header view), A3 (tip integrity, gaps, frontier overlay, aux-row backing), and A4 (the on-disk chain equals the model's expected canonical chain). Also `dump_store`, a comparable snapshot of all five column families used to assert that rejected commits are side-effect free and that reopens preserve the store byte-for-byte. | | `oracle.rs` | An in-memory model of the store's _specified_ behavior: a single linked canonical chain, best-cumulative-work selection with strict improvement, total suffix replacement above the first conflicting height, and a sequential body tip. It predicts whether each op must be accepted or rejected. | | `ops.rs` | The op alphabet, each op mapped to one real production write-batch shape: `CommitHeaderRange` → `prepare_header_range_batch_with_roots`; `CommitBody` / `Finalize` → `prepare_block_header_and_transaction_data_batch` plus the finalization roots delete (which runs the release path internally); `Seed` → `seed_zakura_header_from_committed_block` (the non-finalized best-chain commit hook); `Reopen` → shutdown and reopen of the persistent store. The `Harness` executes ops, cross-checks the oracle's prediction against the store's response, and audits after every mutation; failures come back as a transcribable `FailureReport` (the executed op prefix plus every violation found). | -| `scenarios.rs` | Scripted production event shapes (s01–s10): simple reorgs, lower-work rejections and their later reversal, split-range and walk-back deliveries, body commits racing header reorgs, reorgs to a lower height, double reorgs at one fork point, activity across the difficulty-adjustment window edge, restarts at every boundary, and seed/range interplay. Also holds the `corruption_repro_*` tests below. | -| `prop.rs` | Discovery proptests: random op sequences over the fixed universe, shrunk to minimal counterexamples on any audit failure. Both are `#[ignore]`d because the store has known bugs today; see "Running". | - -## Known corruption bugs pinned by this suite - -Each bug is pinned by a pair of tests: `corruption_repro_` **passes -today** and deterministically demonstrates the violation, and -`_upholds_invariants` is `#[ignore]`d and asserts the correct behavior. -When the write path is fixed, the repro fails loudly (forcing re-triage) and -the twin gets un-ignored as the permanent regression test. - -1. **Unlinked-anchor commit** (`corruption_repro_unlinked_anchor_commit`). - `prepare_header_range_batch_with_roots` never checks that - `headers[0].previous_block_hash == anchor`, nor any intra-range linkage. A - range anchored at a same-height hash of a different branch passes - difficulty validation and commits a suffix that does not link to the row - below it — an on-disk I2 violation reachable from a single untrusted peer - response. +| `scenarios.rs` | Scripted production event shapes (s01–s11): simple reorgs, lower-work rejections and their later reversal, split-range and walk-back deliveries, body commits racing header reorgs, reorgs to a lower height, double reorgs at one fork point, activity across the difficulty-adjustment window edge, restarts at every boundary, seed/range interplay, and refused-seed convergence. Also holds the `*_upholds_invariants` regression gates below. | +| `prop.rs` | The permanent random sweep: random op sequences over the fixed universe, shrunk to minimal counterexamples on any audit failure. | + +## Fixed corruption bugs gated by this suite + +Phase 1 of this suite proved three write-path corruption bug classes, each +pinned by a `corruption_repro_*` test that deterministically demonstrated the +violation. The write-path fixes (`REORG_PLAN.md` Phase 1.5) closed all three; +the repro tests were removed with the fixes, and their +`_upholds_invariants` twins now assert the fixed behavior over the same +op sequences as permanent regression gates: + +1. **Unlinked-anchor commit** (`unlinked_anchor_commit_upholds_invariants`). + `prepare_header_range_batch_with_roots` used to accept ranges without + checking that `headers[0].previous_block_hash == anchor` or any intra-range + linkage, so a range anchored at a same-height hash of a different branch + could pass difficulty validation and commit a suffix that did not link to + the row below it — an on-disk I2 violation reachable from a single + untrusted peer response. The writer now rejects such ranges with + `CommitHeaderRangeError::UnlinkedRange`. 2. **Re-delivery over committed bodies** - (`corruption_repro_redelivery_over_bodies`). The range insert loop gates - only its _roots_ write on `contains_body_at_height`; the - header/hash/height/body-size writes are unconditional. A header range - re-delivered over heights whose bodies were committed in the meantime - re-inserts zakura rows below the body tip, and nothing ever trims them - again (the release trim already ran at body-commit time) — a permanent I3 - frontier-overlay violation. -3. **Unlinked seed** (`corruption_repro_seed_above_gap`, - `corruption_repro_seed_fork_switch`; found by the proptest and shrunk to a - single op). `prepare_zakura_header_from_committed_block` writes its row - with no linkage or anchor precondition. Seeds fire only at non-finalized + (`redelivery_over_bodies_upholds_invariants`). The range insert loop used + to gate only its _roots_ write on `contains_body_at_height`, so a header + range re-delivered over heights whose bodies were committed in the + meantime re-inserted zakura rows below the body tip that nothing ever + trims again (the release trim already ran at body-commit time) — a + permanent I3 frontier-overlay violation. The gate now covers every zakura + row write: heights that already have a committed block are skipped + entirely (checked via `contains_height`, so it also holds for pruned + heights whose bodies are gone but whose authoritative rows remain). +3. **Unlinked seed** (`seed_above_gap_upholds_invariants`, + `seed_fork_switch_upholds_invariants`; found by the proptest and shrunk to + a single op). `prepare_zakura_header_from_committed_block` used to write + its row with no linkage precondition. Seeds fire only at non-finalized best-_tip_ commits, so any best-tip jump (a fork switch between non-finalized chains, or a restart that restores the non-finalized backup) - seeds a height whose parent row is missing or belongs to another branch: - a gap or broken link on disk, and a generator of poisoned - difficulty-adjustment windows. - -A discovery sweep of 2048 random sequences with these three shapes excluded -found no further violation class at that depth. + seeded a height whose parent row was missing or belonged to another + branch: a gap or broken link on disk, and a generator of poisoned + difficulty-adjustment windows. The seed path now refuses a seed that does + not link to the stored row below it as a silent no-op — the header store + briefly lags the non-finalized chain, and header-range sync converges it + (`s11_refused_seed_converges_via_range_delivery`). ## Running ```bash -# The whole suite (scripted scenarios, audits, corruption repros): +# The whole suite (scripted scenarios, audits, regression gates, random sweep): cargo test -p zebra-state --lib header_store_coherence -# Discovery sweep beyond the known bugs (any failure = a NEW violation class): +# The random sweep at discovery depth (any failure = a NEW violation class): PROPTEST_CASES=4096 cargo test -p zebra-state --lib \ - header_store_coherence::prop::prop_discovery -- --ignored - -# The full random sweep (fails today on the known shapes; becomes a permanent -# regression gate once the write paths are fixed): -cargo test -p zebra-state --lib \ - header_store_coherence::prop::prop_random_sequences -- --ignored + header_store_coherence::prop ``` Shrunk proptest counterexamples should be transcribed into `scenarios.rs` as -hardcoded `corruption_repro_*`/`#[ignore]` pairs (seed-independent pinning); -the file under `zebra-state/proptest-regressions/` pins the seeds as a -backstop and is checked in. +hardcoded scenarios (seed-independent pinning); the file under +`zebra-state/proptest-regressions/` pins the seeds as a backstop and is +checked in. ## Scope diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/ops.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/ops.rs index 05480931678..ad36b6556d1 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/ops.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/ops.rs @@ -109,13 +109,16 @@ impl OpOutcome { } /// Why a sequence failed: the executed op prefix (last op is the failing one) -/// plus everything found after it. +/// plus everything found after it. All fields are diagnostic payloads rendered +/// through `Debug` in test failure output. #[derive(Debug)] pub(crate) struct FailureReport { - /// The transcribable op sequence, rendered through `Debug` in reports. + /// The transcribable op sequence. #[allow(dead_code)] pub executed: Vec, + #[allow(dead_code)] pub violations: Vec, + #[allow(dead_code)] pub mismatches: Vec, } @@ -132,10 +135,6 @@ pub(crate) struct Harness { config: Config, state: Option, executed: Vec, - /// Skip the op shapes that trigger the *known* corruption bugs (the - /// `corruption_repro_*` scenarios), so proptest discovery can search for - /// new violation classes instead of rediscovering the known ones. - avoid_known_corruptions: bool, _tempdir: tempfile::TempDir, } @@ -157,19 +156,10 @@ impl Harness { config, state: Some(state), executed: Vec::new(), - avoid_known_corruptions: false, _tempdir: tempdir, } } - /// A harness that skips the known corruption shapes (see - /// [`Harness::avoid_known_corruptions`]). - pub fn new_avoiding_known_corruptions() -> Self { - let mut harness = Self::new(); - harness.avoid_known_corruptions = true; - harness - } - pub fn state(&self) -> &ZebraDb { self.state .as_ref() @@ -242,32 +232,6 @@ impl Harness { let Some(range) = self.resolve_range(*source, *offset, *len, *anchor) else { return self.finish(OpOutcome::Skipped("range out of bounds"), mismatches); }; - if self.avoid_known_corruptions { - // Known bug 1: unlinked-anchor ranges are accepted instead - // of rejected (corruption_repro_unlinked_anchor_commit). - if matches!( - self.oracle.predict_header_range(&range), - Prediction::Reject(super::oracle::RejectKind::Malformed) - ) { - return self.finish( - OpOutcome::Skipped("known corruption shape: unlinked anchor"), - mismatches, - ); - } - // Known bug 2: accepted re-deliveries touching body-backed - // heights strand zakura rows below the body tip - // (corruption_repro_redelivery_over_bodies). - if range - .rows - .iter() - .any(|row| row.height <= self.oracle.body_tip()) - { - return self.finish( - OpOutcome::Skipped("known corruption shape: re-delivery over bodies"), - mismatches, - ); - } - } match self.oracle.predict_header_range(&range) { Prediction::Skip(reason) => { return self.finish(OpOutcome::Skipped(reason), mismatches) @@ -337,27 +301,33 @@ impl Harness { let Some(fab) = rows.get(*index).cloned() else { return self.finish(OpOutcome::Skipped("seed index out of bounds"), mismatches); }; - // Known bug 3: a seed whose parent is not the committed row - // below it writes an unlinkable row - // (corruption_repro_seed_above_gap / _seed_fork_switch). - if self.avoid_known_corruptions && !self.oracle.seed_is_parent_linked(&fab) { - return self.finish( - OpOutcome::Skipped("known corruption shape: unlinked seed"), - mismatches, - ); - } match self.oracle.predict_seed(&fab) { Prediction::Skip(reason) => { return self.finish(OpOutcome::Skipped(reason), mismatches) } _ => { + // A seed whose parent is not the stored row below it is + // refused by the store as a silent no-op (the header + // store briefly lags the non-finalized chain until a + // linkage-checked header range converges it), so the + // call must succeed without mutating anything. + let parent_linked = self.oracle.seed_is_parent_linked(&fab); + let dump_before = (!parent_linked).then(|| dump_store(self.state())); let block = fabricate_body(&fab); match self .state() .seed_zakura_header_from_committed_block(fab.height, &block) { Ok(()) => { - self.oracle.apply_seed(&fab); + if parent_linked { + self.oracle.apply_seed(&fab); + } else if dump_store(self.state()) + != dump_before.expect("dumped before an unlinked seed") + { + mismatches.push(format!( + "refused unlinked seed mutated the store: {op:?}" + )); + } OpOutcome::Accepted } Err(error) => { diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/oracle.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/oracle.rs index d28872f3055..574b695afc6 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/oracle.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/oracle.rs @@ -40,7 +40,7 @@ pub(crate) enum RejectKind { /// The conflicting suffix does not carry strictly more work. LowerWork, /// The range is malformed (wrong anchor linkage or heights); the store is - /// expected to reject it through contextual validation. + /// expected to reject it through its linkage check (`UnlinkedRange`). Malformed, } @@ -248,8 +248,10 @@ impl Oracle { } /// Whether a seeded block's parent is the expected canonical row below it. - /// Seeds that are not parent-linked are the known seed-path corruption - /// shape (they write a row the store cannot link). + /// The store refuses seeds that are not parent-linked as silent no-ops + /// (writing them would strand a row the chain walk cannot reach); the + /// harness uses this to decide whether a successful seed call must have + /// mutated the store or left it untouched. pub fn seed_is_parent_linked(&self, fab: &FabHeader) -> bool { let parent_height = Height(fab.height.0 - 1); if parent_height == Height(0) { diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/prop.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/prop.rs index e867768b56e..4a363411354 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/prop.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/prop.rs @@ -1,22 +1,17 @@ //! Discovery proptest: random op sequences over the fixed universe, shrunk on //! any audit failure to a minimal, transcribable counterexample. //! -//! Both tests are `#[ignore]`d: the store has known corruption bugs today (the -//! `corruption_repro_*` scenarios), so an always-on random sweep cannot be a CI -//! gate yet. Run discovery manually: +//! The sweep runs 64 cases in CI by default; widen it manually with: //! //! ```text //! PROPTEST_CASES=4096 cargo test -p zebra-state --lib \ -//! header_store_coherence::prop -- --ignored --nocapture +//! header_store_coherence::prop -- --nocapture //! ``` //! //! Every shrunk counterexample must be transcribed into `scenarios.rs` as a -//! hardcoded `corruption_repro_*` / `#[ignore]`d-invariant pair (the primary, -//! seed-independent pinning mechanism); the proptest regression file under -//! `zebra-state/proptest-regressions/` pins the seed as a backstop. -//! -//! When the write-path fixes land, un-ignore -//! `prop_random_sequences_uphold_invariants` as a permanent regression sweep. +//! hardcoded scenario (the primary, seed-independent pinning mechanism); the +//! proptest regression file under `zebra-state/proptest-regressions/` pins +//! the seed as a backstop. use std::env; @@ -81,10 +76,8 @@ fn proptest_cases() -> u32 { proptest! { #![proptest_config(ProptestConfig::with_cases(proptest_cases()))] - /// The full invariant sweep. Fails today on the known corruption shapes; - /// becomes the permanent regression gate once the write paths are fixed. + /// The permanent invariant sweep: any failure is a store-corruption bug. #[test] - #[ignore = "known zakura header-store corruption: un-ignore with the write-path fix"] fn prop_random_sequences_uphold_invariants(ops in ops_strategy()) { let _init_guard = zebra_test::init(); let mut harness = Harness::new(); @@ -92,17 +85,4 @@ proptest! { prop_assert!(false, "store invariants violated:\n{report:#?}"); } } - - /// Discovery beyond the known bugs: the harness skips the known - /// corruption shapes, so any failure here is a *new* violation class. - /// Run manually with a large PROPTEST_CASES while the known bugs are open. - #[test] - #[ignore = "discovery sweep: run manually"] - fn prop_discovery_avoiding_known_corruptions(ops in ops_strategy()) { - let _init_guard = zebra_test::init(); - let mut harness = Harness::new_avoiding_known_corruptions(); - if let Err(report) = harness.run_all(&ops) { - prop_assert!(false, "new store-corruption class found:\n{report:#?}"); - } - } } diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/scenarios.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/scenarios.rs index 8515937c34c..6e41141a72b 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/scenarios.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/scenarios.rs @@ -4,16 +4,14 @@ //! passing scenario is a regression gate on the whole write sequence, not //! just its final assertions. //! -//! The `corruption_repro_*` tests at the bottom deliberately demonstrate -//! store-invariant violations that exist today: they PASS while the bug -//! exists and fail loudly once the write path is fixed, forcing re-triage. -//! Their `#[ignore]`d twins assert the true invariant, to be un-ignored -//! together with the write-path fix. +//! The `*_upholds_invariants` tests at the bottom are the permanent +//! regression gates for the three write-path corruption bugs this suite +//! originally proved with `corruption_repro_*` twins (removed together with +//! the write-path fixes; see the module README for the bug histories). use zebra_chain::block::Height; use super::{ - audit::Violation, fabricate::{BRANCH_A, BRANCH_B, BRANCH_B_EXT, BRANCH_C, FORK_HEIGHT, TRUNK_LEN}, ops::{universe, Anchor, Harness, Op, OpOutcome, Source}, }; @@ -300,7 +298,7 @@ fn s04_body_commit_racing_header_reorg() { /// s05: the release path's frontier trim. Sequential body commits release /// matching provisional rows one by one; header re-delivery *above* the body /// tip is idempotent; restarts change nothing. (Re-delivery *over* body-backed -/// heights strands rows — that is `corruption_repro_redelivery_over_bodies`.) +/// heights skips those heights — `redelivery_over_bodies_upholds_invariants`.) #[test] fn s05_release_trim_then_redelivery() { let _init_guard = zebra_test::init(); @@ -541,19 +539,73 @@ fn s10_seed_interplay() { ); } +/// s11: a refused (unlinked) seed converges through header-range sync. The +/// zakura store follows branch A (seeded at the fork height); the +/// non-finalized best chain switches to branch B and its new best tip (B's +/// *second* row) is seeded. The store refuses the unlinked seed as a no-op — +/// the header store briefly lags the nf chain — and the later linked range +/// delivery of B converges it onto the new branch. +#[test] +fn s11_refused_seed_converges_via_range_delivery() { + let _init_guard = zebra_test::init(); + let mut harness = Harness::new(); + + let outcomes = harness + .run_all(&[ + commit_trunk(), + Op::Seed { + source: Source::Branch(BRANCH_A), + index: 0, + }, + ]) + .expect("setup is clean"); + outcomes.iter().for_each(assert_accepted); + let a_first = &universe().branches[BRANCH_A].headers[0]; + assert_eq!( + harness.state().best_header_tip(), + Some((a_first.height, a_first.hash)), + ); + + // The nf best tip jumps to B[1]; its parent row is not stored, so the + // seed is refused without touching the store (checked by the harness). + let outcome = harness + .run(&Op::Seed { + source: Source::Branch(BRANCH_B), + index: 1, + }) + .expect("the refused seed leaves the store coherent"); + assert_accepted(&outcome); + assert_eq!( + harness.state().best_header_tip(), + Some((a_first.height, a_first.hash)), + "the refused seed must not move the header tip", + ); + + // Header sync catches up with the nf chain: B arrives as a linked range + // and out-works the stored A suffix, converging the store onto B. + let outcome = harness + .run(&commit_branch(BRANCH_B)) + .expect("the converging range delivery is clean"); + assert_accepted(&outcome); + assert_eq!( + harness.state().best_header_tip(), + Some(branch_tip(BRANCH_B)), + ); +} + // --------------------------------------------------------------------------- -// Corruption reproductions: deterministic sequences that violate the store -// invariants today. See the module doc for the pass/ignore convention. +// Regression gates for the three proven write-path corruption bugs. Each was +// originally pinned by a `corruption_repro_*` test that demonstrated the +// violation; those were removed with the write-path fixes, and these twins +// now assert the fixed behavior over the exact same op sequences. // --------------------------------------------------------------------------- -/// The unlinked-anchor commit: `prepare_header_range_batch_with_roots` never -/// checks that `headers[0].previous_block_hash == anchor` (or any intra-range -/// linkage). A range of branch-A headers anchored at the same-height *trunk* -/// hash passes contextual difficulty validation — the two fast chains have -/// identical (time, threshold) sequences — and commits a suffix that does not -/// link to the row below it: an on-disk I2 violation. Production reach: a -/// header range is untrusted peer input; nothing upstream of the store -/// re-checks the anchor linkage. +/// The unlinked-anchor commit (bug 1): a range of branch-A headers anchored at +/// the same-height *trunk* hash passes contextual difficulty validation — the +/// two fast chains have identical (time, threshold) sequences — so before the +/// linkage check in `prepare_header_range_batch_with_roots`, it committed a +/// suffix that did not link to the row below it: an on-disk I2 violation +/// reachable from a single untrusted peer response. fn unlinked_anchor_ops() -> Vec { vec![ commit_trunk(), @@ -567,58 +619,36 @@ fn unlinked_anchor_ops() -> Vec { ] } -/// PASSES while the bug exists (proves the repro); flips when the writer is -/// fixed, forcing this pair to be re-triaged. +/// The store rejects unlinked ranges with `UnlinkedRange` and stays coherent. #[test] -fn corruption_repro_unlinked_anchor_commit() { - let _init_guard = zebra_test::init(); - let mut harness = Harness::new(); - - let report = harness - .run_all(&unlinked_anchor_ops()) - .expect_err("the unlinked-anchor commit corrupts the store today"); - - assert!( - report - .violations - .iter() - .any(|violation| matches!(violation, Violation::BrokenLinkage { height, .. } if *height == Height(FORK_HEIGHT + 2))), - "expected BrokenLinkage right above the spliced anchor: {report:?}" - ); - assert!( - !report.mismatches.is_empty(), - "the oracle rejects this range; the store accepted it: {report:?}" - ); -} - -/// The true invariant. Un-ignore when the write-path fix lands. -#[test] -#[ignore = "known zakura header-store corruption: un-ignore with the write-path fix"] fn unlinked_anchor_commit_upholds_invariants() { let _init_guard = zebra_test::init(); let mut harness = Harness::new(); - harness + let outcomes = harness .run_all(&unlinked_anchor_ops()) .expect("the store rejects unlinked ranges and stays coherent"); + assert!( + matches!( + outcomes[1].header_range_error(), + CommitHeaderRangeError::UnlinkedRange { .. } + ), + "expected UnlinkedRange, got {:?}", + outcomes[1] + ); } -/// Re-delivery over committed bodies: the range insert loop -/// (`prepare_header_range_batch_with_roots`) gates only its *roots* write on -/// `contains_body_at_height` — the header/hash/height/body-size writes are -/// unconditional. A header range re-delivered over heights whose bodies have -/// since been committed (a header store behind the body store, or a late range -/// response racing body sync — the exact scenario the roots gate's own comment -/// describes) re-inserts zakura rows *below* the body tip. That breaks the -/// frontier-overlay invariant ("the Zakura header store only ever holds -/// heights with no committed body", block.rs release-path doc): the release -/// trim already ran at body-commit time, so nothing ever removes these rows. +/// Re-delivery over committed bodies (bug 2): before the committed-height +/// gate covered the header/hash/height/body-size writes (it originally gated +/// only the *roots* write), a header range re-delivered over heights whose +/// bodies had since been committed re-inserted zakura rows *below* the body +/// tip — rows the release trim (which already ran at body-commit time) never +/// removes. fn redelivery_over_bodies_ops() -> Vec { vec![ commit_trunk(), Op::Finalize { count: 10 }, // Re-delivery spanning body-backed heights 6..=10 and header-only - // heights above. The rows match the canonical chain, so the store - // accepts — but the write strands zakura rows under the body tip. + // heights above: accepted, but the body-backed heights are skipped. Op::CommitHeaderRange { source: Source::Trunk, offset: 5, @@ -628,50 +658,21 @@ fn redelivery_over_bodies_ops() -> Vec { ] } -/// PASSES while the bug exists (proves the repro); flips when the writer is -/// fixed, forcing this pair to be re-triaged. +/// Re-delivery over bodies leaves no zakura rows below the body tip. #[test] -fn corruption_repro_redelivery_over_bodies() { - let _init_guard = zebra_test::init(); - let mut harness = Harness::new(); - - let report = harness - .run_all(&redelivery_over_bodies_ops()) - .expect_err("re-delivery over committed bodies strands zakura rows today"); - - assert!( - report - .violations - .iter() - .any(|violation| matches!(violation, Violation::ZakuraRowAtBodyHeight { height, .. } if *height <= Height(10))), - "expected zakura rows stranded at body-backed heights: {report:?}" - ); - assert!( - report.mismatches.is_empty(), - "acceptance itself is correct chain selection; the write effects are the bug: {report:?}" - ); -} - -/// The true invariant. Un-ignore when the write-path fix lands. -#[test] -#[ignore = "known zakura header-store corruption: un-ignore with the write-path fix"] fn redelivery_over_bodies_upholds_invariants() { let _init_guard = zebra_test::init(); let mut harness = Harness::new(); - harness + let outcomes = harness .run_all(&redelivery_over_bodies_ops()) .expect("re-delivery over bodies leaves no zakura rows below the body tip"); + outcomes.iter().for_each(assert_accepted); } -/// Seed above a gap — found by `prop_random_sequences_uphold_invariants` and -/// shrunk to a single op. The seed path -/// (`prepare_zakura_header_from_committed_block`) writes header/hash/height -/// rows at its height with **no linkage or anchor precondition**: seeding a -/// block whose parent row is absent leaves a row the chain walk cannot reach. -/// Production shape: seeds fire only at non-finalized best-*tip* commits, so -/// any nf best-tip jump — a fork switch between nf chains, or a restart that -/// restores the nf backup and then commits on top — seeds a height whose -/// parent row was never written. +/// Seed above a gap (bug 3, minimal shape — found by the discovery proptest +/// and shrunk to a single op): before the parent-linkage refusal in +/// `prepare_zakura_header_from_committed_block`, seeding a block whose parent +/// row is absent left a row the chain walk cannot reach. fn seed_above_gap_ops() -> Vec { vec![Op::Seed { source: Source::Trunk, @@ -679,51 +680,30 @@ fn seed_above_gap_ops() -> Vec { }] } -/// PASSES while the bug exists (proves the repro); flips when the writer is -/// fixed, forcing this pair to be re-triaged. -#[test] -fn corruption_repro_seed_above_gap() { - let _init_guard = zebra_test::init(); - let mut harness = Harness::new(); - - let report = harness - .run_all(&seed_above_gap_ops()) - .expect_err("seeding above a gap strands an unreachable row today"); - - assert!( - report - .violations - .iter() - .any(|violation| matches!(violation, Violation::RowAboveLastLinked { height, .. } if *height == Height(2))), - "expected the seeded row to be unreachable from the finalized tip: {report:?}" - ); - assert!( - report - .violations - .iter() - .any(|violation| matches!(violation, Violation::BestHeaderTipMismatch { .. })), - "expected best_header_tip to point above the linked chain: {report:?}" - ); -} - -/// The true invariant. Un-ignore when the write-path fix lands. +/// A seed above a gap is refused as a no-op instead of stranding a row. #[test] -#[ignore = "known zakura header-store corruption: un-ignore with the write-path fix"] fn seed_above_gap_upholds_invariants() { let _init_guard = zebra_test::init(); let mut harness = Harness::new(); harness .run_all(&seed_above_gap_ops()) .expect("a seed above a gap must not strand an unreachable row"); + + let genesis_hash = universe().genesis.hash(); + assert_eq!( + harness.state().best_header_tip(), + Some((Height(0), genesis_hash)), + "the refused seed must not move the header tip", + ); } -/// Seed fork switch — the production-shaped variant of the seed linkage hole. -/// The zakura store follows branch A (seeded at the fork height); the -/// non-finalized best chain switches to branch B and its new best tip (B's -/// *second* row) is seeded. The seed's non-conflict arm inserts the row -/// directly — B's parent row was truncated, so the store now holds -/// `A[0]` at the fork height and `B[1]` right above it: broken linkage on -/// disk, exactly the poisoned-DAA-window generator from the incident table. +/// Seed fork switch (bug 3, production shape): the zakura store follows +/// branch A; the non-finalized best chain switches to branch B and its new +/// best tip (B's *second* row) is seeded. Before the parent-linkage refusal, +/// the seed's non-conflict arm inserted the row directly over A's truncated +/// parent — broken linkage on disk, the poisoned-DAA-window generator from +/// the production incident table. (`s11` proves the refused seed converges +/// later through range delivery.) fn seed_fork_switch_ops() -> Vec { vec![ commit_trunk(), @@ -738,33 +718,19 @@ fn seed_fork_switch_ops() -> Vec { ] } -/// PASSES while the bug exists (proves the repro); flips when the writer is -/// fixed, forcing this pair to be re-triaged. +/// A fork-switch seed keeps the store linked (the unlinked seed is refused). #[test] -fn corruption_repro_seed_fork_switch() { - let _init_guard = zebra_test::init(); - let mut harness = Harness::new(); - - let report = harness - .run_all(&seed_fork_switch_ops()) - .expect_err("a fork-switch seed breaks linkage today"); - - assert!( - report - .violations - .iter() - .any(|violation| matches!(violation, Violation::BrokenLinkage { height, .. } if *height == Height(FORK_HEIGHT + 2))), - "expected broken linkage right above the fork height: {report:?}" - ); -} - -/// The true invariant. Un-ignore when the write-path fix lands. -#[test] -#[ignore = "known zakura header-store corruption: un-ignore with the write-path fix"] fn seed_fork_switch_upholds_invariants() { let _init_guard = zebra_test::init(); let mut harness = Harness::new(); harness .run_all(&seed_fork_switch_ops()) .expect("a fork-switch seed must keep the store linked"); + + let a_first = &universe().branches[BRANCH_A].headers[0]; + assert_eq!( + harness.state().best_header_tip(), + Some((a_first.height, a_first.hash)), + "the store stays on A until a linked delivery of B arrives", + ); } diff --git a/zebra-state/src/service/write.rs b/zebra-state/src/service/write.rs index 21c0ac0d974..0452d20e642 100644 --- a/zebra-state/src/service/write.rs +++ b/zebra-state/src/service/write.rs @@ -744,7 +744,7 @@ mod tests { use crate::{ arbitrary::Prepare, service::{ - finalized_state::FinalizedState, + finalized_state::{DiskWriteBatch, FinalizedState, WriteDisk}, non_finalized_state::NonFinalizedState, write::{ seed_zakura_header_from_committed_block, @@ -781,6 +781,27 @@ mod tests { let mut non_finalized_state = NonFinalizedState::new(&network); + // The seed path refuses rows that do not link to the stored header row + // below them, and the fake chain's parent block is not otherwise + // committed to this state, so store its hash as a provisional Zakura + // row (the consensus `hash_by_height` row cannot be written alone: a + // finalized tip implies note commitment trees exist). + let parent_height = parent + .coinbase_height() + .expect("test vector block has a coinbase height"); + let zakura_hash_by_height = finalized_state + .db + .db() + .cf_handle("zakura_header_hash_by_height") + .unwrap(); + let mut batch = DiskWriteBatch::new(); + batch.zs_insert(&zakura_hash_by_height, parent_height, parent.hash()); + finalized_state + .db + .db() + .write(batch) + .expect("parent hash row writes"); + non_finalized_state .commit_new_chain(best_block.clone().prepare(), &finalized_state) .expect("best block commits to a new chain"); diff --git a/zebrad/src/commands/start.rs b/zebrad/src/commands/start.rs index e66925402c7..fe94936d94d 100644 --- a/zebrad/src/commands/start.rs +++ b/zebrad/src/commands/start.rs @@ -2346,6 +2346,24 @@ mod zakura_header_sync_driver_tests { ); } + #[test] + fn unlinked_range_is_local_header_sync_commit_failure() { + // The reactor validates every response's linkage against the requested + // anchor before committing with that anchor, so the store's own + // linkage check failing means local anchor/response pairing broke, + // not peer misbehavior. + let error = zebra_state::CommitHeaderRangeError::UnlinkedRange { + height: block::Height(1), + expected_parent: block::Hash([0; 32]), + actual_parent: block::Hash([1; 32]), + }; + + assert_eq!( + header_range_commit_failure_kind(&error), + HeaderSyncCommitFailureKind::Local + ); + } + #[test] fn served_header_body_size_hints_align_with_served_heights() { let start = block::Height(10); diff --git a/zebrad/src/commands/start/zakura/header_sync_driver.rs b/zebrad/src/commands/start/zakura/header_sync_driver.rs index a1d9ca0bec7..a3e55b9f899 100644 --- a/zebrad/src/commands/start/zakura/header_sync_driver.rs +++ b/zebrad/src/commands/start/zakura/header_sync_driver.rs @@ -1112,6 +1112,12 @@ pub(crate) fn header_range_commit_failure_kind( // fork. Treat it as non-scoring so this stays a liveness/correctness guard, // not peer punishment. | zebra_state::CommitHeaderRangeError::LowerWorkConflict { .. } + // The reactor already validates every peer response against the requested + // anchor and for internal continuity (`validate_header_range_links`) and + // scores linkage failures there, then commits with that same anchor. So the + // store's own linkage check failing means the local anchor/response pairing + // went wrong, not that the peer misbehaved. + | zebra_state::CommitHeaderRangeError::UnlinkedRange { .. } | zebra_state::CommitHeaderRangeError::CommitResponseDropped => { HeaderSyncCommitFailureKind::Local } From 4a1600ca4f8bfb7372ced8d6e2b3c9f19b173702 Mon Sep 17 00:00:00 2001 From: roman Date: Mon, 6 Jul 2026 16:43:57 -0600 Subject: [PATCH 02/10] lint --- Cargo.lock | 4 ++-- deny.toml | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5b397748cff..dc19b1b4066 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1349,9 +1349,9 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.18" +version = "0.9.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" dependencies = [ "crossbeam-utils", ] diff --git a/deny.toml b/deny.toml index 3ee845cc4e6..1261b6ea9e9 100644 --- a/deny.toml +++ b/deny.toml @@ -13,11 +13,9 @@ yanked = "deny" # Remove each ignore entry as the dependency chain is upgraded. ignore = [ "RUSTSEC-2021-0139", # ansi_term — transitive via abscissa_core -> structopt - "RUSTSEC-2022-0104", # structopt (maintenance mode) — transitive via abscissa_core + "RUSTSEC-2022-0104", # structopt (maintenance mode) — transitive via abscissa_core and zebra-utils "RUSTSEC-2024-0375", # atty (unmaintained) — transitive via abscissa_core -> structopt - "RUSTSEC-2021-0145", # atty (unsound) — transitive via abscissa_core -> structopt "RUSTSEC-2024-0370", # proc-macro-error — transitive via abscissa_core -> structopt - "RUSTSEC-2022-0104", # structopt (unmaintained) — direct dep of zebra-utils; no safe upgrade, migrate to clap 4 "RUSTSEC-2026-0173", # proc-macro-error2 (unmaintained) — transitive via getset "RUSTSEC-2025-0119", # number_prefix — transitive via indicatif "RUSTSEC-2025-0141", # bincode — direct dependency From 93d3456b323b50cc6609936707b31bbea0746b16 Mon Sep 17 00:00:00 2001 From: roman Date: Mon, 6 Jul 2026 16:49:20 -0600 Subject: [PATCH 03/10] simplify comment --- .../src/service/finalized_state/zebra_db/block.rs | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/zebra-state/src/service/finalized_state/zebra_db/block.rs b/zebra-state/src/service/finalized_state/zebra_db/block.rs index 1557b6fd8dc..d722bb0b242 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block.rs @@ -1827,16 +1827,9 @@ impl DiskWriteBatch { return Ok(()); } - // Seeds fire at non-finalized best-*tip* commits, so an nf best-tip - // jump (a fork switch between nf chains, or a restart restoring the nf - // backup) can seed a height whose parent row is missing or belongs to - // another branch. Writing that row would break the header store's - // linkage invariant on disk and poison difficulty-adjustment windows - // (`header_store_coherence` bug 3). Refuse the write instead: the - // header store briefly lags the nf chain, and header-range sync - // converges it onto the new branch through the linkage-checked range - // path. The merged header view (full-block rows first, then zakura - // rows) is the same view the chain walk reads. + // Seeds can jump to a non-finalized best tip whose parent is not the + // stored row below it. Refuse those seeds so the header store stays + // linked; header-range sync will later deliver the missing rows. let hash_by_height = db.cf_handle("hash_by_height").unwrap(); let parent_hash: Option = height.previous().ok().and_then(|parent_height| { db.zs_get(&hash_by_height, &parent_height) From 08f791a0e0337c000b44ef5aad85472741a197fb Mon Sep 17 00:00:00 2001 From: roman Date: Mon, 6 Jul 2026 17:39:40 -0600 Subject: [PATCH 04/10] fix(state): verify zakura header-store invariants in consensus reads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pillar 2 of the REORG_PLAN: a corrupted header store can no longer poison difficulty validation — readers surface the storage fault explicitly. - recent_header_context verifies, at every step of its walk, that the stored header is the block its hash row names (HeaderHashMismatch), that it links to the row below (BrokenLinkage), and that no row is missing below a stored one (Gap), returning the new StoreIncoherentError instead of a poisoned or silently shortened difficulty window. - The range writer's anchor round-trip distinguishes a bijection violation in our own indexes (StoreIncoherent::BijectionMismatch) from a genuinely unknown anchor (UnknownAnchor). - CommitHeaderRangeError::StoreIncoherent is classified as a local, non-peer-scoring commit failure in zebrad: the range was rejected because our store cannot supply trustworthy context, not because the peer's range was shown invalid. New reads.rs suite in header_store_coherence hand-corrupts the column families and pins the reader/writer behavior for all four fault shapes, including side-effect freedom of the rejections. --- CHANGELOG.md | 8 + zebra-state/src/error.rs | 71 +++++ zebra-state/src/lib.rs | 3 +- .../service/finalized_state/zebra_db/block.rs | 103 +++++-- .../block/tests/header_store_coherence.rs | 1 + .../tests/header_store_coherence/README.md | 1 + .../tests/header_store_coherence/reads.rs | 281 ++++++++++++++++++ .../zebra_db/block/tests/vectors.rs | 12 +- zebrad/src/commands/start.rs | 19 ++ .../start/zakura/header_sync_driver.rs | 6 + 10 files changed, 482 insertions(+), 23 deletions(-) create mode 100644 zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/reads.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 96f5da37488..8308484c00e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,14 @@ and this project adheres to [Semantic Versioning](https://semver.org). ### Fixed +- Zakura header-store consensus reads now verify store invariants as they + walk: the difficulty-context walk checks that every stored header is the + block its hash row names and links to the row below it, and the anchor + lookup distinguishes a corrupted index round-trip from a genuinely unknown + anchor. A corrupted store now surfaces as an explicit local + `StoreIncoherent` error (never scored against peers) instead of poisoning + difficulty validation and rejecting honest headers with + `InvalidDifficultyThreshold`. - Fixed three Zakura header-store write paths that could leave the on-disk header store internally incoherent after chain forks, causing nodes to reject valid headers from honest peers (`InvalidDifficultyThreshold` / diff --git a/zebra-state/src/error.rs b/zebra-state/src/error.rs index 84a9720dee8..40533f13fb6 100644 --- a/zebra-state/src/error.rs +++ b/zebra-state/src/error.rs @@ -195,6 +195,68 @@ impl From for CommitCheckpointVerifiedError { } } +/// An internal invariant of the zakura header store was found violated while +/// reading it. +/// +/// This is a **local storage fault**, never evidence about a peer: readers +/// return it instead of feeding rows from more than one branch (or from beside +/// a gap) into consensus validation, where the corruption would otherwise +/// surface as a misleading validation failure (`InvalidDifficultyThreshold`, +/// `UnknownAnchor`) attributed to whoever supplied the input being validated. +#[derive(Debug, Error, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum StoreIncoherentError { + /// The header row at `height` does not link to the stored row below it. + #[error( + "header store incoherent: header at {height:?} links to {expected_parent} but the stored row below is {actual_below}" + )] + BrokenLinkage { + /// Height of the header whose parent link failed to resolve. + height: block::Height, + /// The parent hash the header claims (`previous_block_hash`). + expected_parent: block::Hash, + /// The hash actually stored at `height - 1`. + actual_below: block::Hash, + }, + + /// A header row exists at `height` but the row below it is missing. + #[error( + "header store incoherent: no stored row at {missing:?} below the header at {height:?}" + )] + Gap { + /// Height of the stored header above the gap. + height: block::Height, + /// The missing height (`height - 1`). + missing: block::Height, + }, + + /// The header row at `height` is not the block its hash row names. + #[error( + "header store incoherent: header stored at {height:?} hashes to {computed} but the hash row names {indexed}" + )] + HeaderHashMismatch { + /// Height of the divergent rows. + height: block::Height, + /// The hash the height→hash index names. + indexed: block::Hash, + /// The stored header's actual hash. + computed: block::Hash, + }, + + /// The hash→height and height→hash indexes disagree about a hash. + #[error( + "header store incoherent: hash {hash} is indexed at {height:?} but that height stores {stored:?}" + )] + BijectionMismatch { + /// The hash whose round-trip failed. + hash: block::Hash, + /// The height the hash→height index reports for it. + height: block::Height, + /// What the height→hash index stores there instead. + stored: Option, + }, +} + /// An error describing why a header-only range could not be committed. #[derive(Debug, Error, Clone, PartialEq, Eq)] #[non_exhaustive] @@ -323,6 +385,15 @@ pub enum CommitHeaderRangeError { height: block::Height, }, + /// The local header store was found internally incoherent while reading + /// the context needed to validate the range. + /// + /// This is a local storage fault, not a peer validation failure: the range + /// was rejected because the store cannot supply trustworthy context, not + /// because the range itself was shown invalid. + #[error("header store incoherent while validating range: {0}")] + StoreIncoherent(#[from] StoreIncoherentError), + /// Contextual header validation failed. #[error("could not contextually validate header")] ValidateContextError(#[from] Box), diff --git a/zebra-state/src/lib.rs b/zebra-state/src/lib.rs index 015f190c428..88159fbd242 100644 --- a/zebra-state/src/lib.rs +++ b/zebra-state/src/lib.rs @@ -43,7 +43,8 @@ pub use config::{ pub use constants::{state_database_format_version_in_code, MAX_BLOCK_REORG_HEIGHT}; pub use error::{ BoxError, CloneError, CommitBlockError, CommitCheckpointVerifiedError, CommitHeaderRangeError, - CommitSemanticallyVerifiedError, DuplicateNullifierError, ValidateContextError, + CommitSemanticallyVerifiedError, DuplicateNullifierError, StoreIncoherentError, + ValidateContextError, }; pub use request::{ CheckpointVerifiedBlock, CommitSemanticallyVerifiedBlockRequest, HashOrHeight, MappedRequest, diff --git a/zebra-state/src/service/finalized_state/zebra_db/block.rs b/zebra-state/src/service/finalized_state/zebra_db/block.rs index d722bb0b242..5dab23a1272 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block.rs @@ -36,7 +36,7 @@ use crate::{ constants::{ MAX_BLOCK_REORG_HEIGHT, MAX_HEADER_SYNC_HEIGHT_RANGE, MAX_PRUNE_HEIGHTS_PER_COMMIT, }, - error::{CommitCheckpointVerifiedError, CommitHeaderRangeError}, + error::{CommitCheckpointVerifiedError, CommitHeaderRangeError, StoreIncoherentError}, request::FinalizedBlock, service::check, service::finalized_state::{ @@ -544,31 +544,85 @@ impl ZebraDb { } /// Returns recent header difficulty/time context in reverse height order, - /// starting at `height`. + /// starting at `height`, verifying `previous_block_hash` linkage at every + /// step of the walk. + /// + /// Returns an empty context when there is no stored row at `height` (the + /// caller decides whether that anchor is unknown), and a shorter-than-span + /// context when the walk reaches genesis. + /// + /// # Errors + /// + /// Returns [`StoreIncoherentError`] when the walk finds a header row that + /// is not the block its hash row names, a row that does not link to the + /// row below it, or a gap below a stored row. Feeding such a window into + /// difficulty validation would mix rows from more than one branch (or + /// shift the adjustment window), producing `InvalidDifficultyThreshold` + /// rejections of honest input — the reader surfaces the storage fault + /// explicitly instead. The per-row hash check costs one header hash per + /// consumed row, negligible next to the validation the window feeds. pub fn recent_header_context( &self, height: block::Height, - ) -> Vec<( - zebra_chain::work::difficulty::CompactDifficulty, - DateTime, - )> { + ) -> Result< + Vec<( + zebra_chain::work::difficulty::CompactDifficulty, + DateTime, + )>, + StoreIncoherentError, + > { let mut context = Vec::with_capacity(check::difficulty::POW_ADJUSTMENT_BLOCK_SPAN); - let mut current_height = Some(height); - while let Some(height) = current_height { - let Some((_hash, header)) = self.header_by_height(height) else { - break; - }; + let Some((mut hash, mut header)) = self.header_by_height(height) else { + return Ok(context); + }; + let mut height = height; + + loop { + let computed = block::Hash::from(&*header); + if computed != hash { + return Err(StoreIncoherentError::HeaderHashMismatch { + height, + indexed: hash, + computed, + }); + } context.push((header.difficulty_threshold, header.time)); + if context.len() == check::difficulty::POW_ADJUSTMENT_BLOCK_SPAN { - break; + return Ok(context); } + let Ok(below) = height.previous() else { + // The walk reached genesis: a short context is legitimate, and + // the difficulty functions handle it (MedianTime clamps + // negative heights to zero). + return Ok(context); + }; - current_height = height.previous().ok(); - } + let Some((below_hash, below_header)) = self.header_by_height(below) else { + // Rows must be contiguous from genesis up to the header tip + // (full-block rows below the body tip — retained even under + // pruning — and zakura rows above it), so a missing row below + // a stored one is a gap, not the end of history. + return Err(StoreIncoherentError::Gap { + height, + missing: below, + }); + }; - context + if header.previous_block_hash != below_hash { + return Err(StoreIncoherentError::BrokenLinkage { + height, + expected_parent: header.previous_block_hash, + actual_below: below_hash, + }); + } + + height = below; + hash = below_hash; + header = below_header; + } } /// Returns header-known, body-missing heights. @@ -2035,17 +2089,26 @@ impl DiskWriteBatch { .or_else(|| (anchor == zebra_db.network().genesis_hash()).then_some(block::Height(0))) .ok_or(CommitHeaderRangeError::UnknownAnchor { anchor })?; - if anchor != zebra_db.network().genesis_hash() - && zebra_db.header_hash(anchor_height) != Some(anchor) - { - return Err(CommitHeaderRangeError::UnknownAnchor { anchor }); + // The hash→height index knows the anchor, so a failed height→hash + // round-trip is a bijection violation in our own store — a local + // storage fault, not an unknown anchor supplied by the caller. + if anchor != zebra_db.network().genesis_hash() { + let stored = zebra_db.header_hash(anchor_height); + if stored != Some(anchor) { + return Err(StoreIncoherentError::BijectionMismatch { + hash: anchor, + height: anchor_height, + stored, + } + .into()); + } } let finalized_height = zebra_db.finalized_tip_height(); let best_header_tip = zebra_db.best_header_tip().map(|(height, _)| height); let checkpoints = zebra_db.network().checkpoint_list(); - let mut recent_headers = zebra_db.recent_header_context(anchor_height); + let mut recent_headers = zebra_db.recent_header_context(anchor_height)?; if recent_headers.is_empty() { if anchor == zebra_db.network().genesis_hash() && anchor_height == block::Height(0) { return Err(CommitHeaderRangeError::MissingGenesisAnchor { anchor }); diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence.rs index 81b87d2b4ae..ef8cea5a0a1 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence.rs @@ -32,4 +32,5 @@ mod fabricate; mod ops; mod oracle; mod prop; +mod reads; mod scenarios; diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/README.md b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/README.md index 4b49718607a..cf788dcf19b 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/README.md +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/README.md @@ -36,6 +36,7 @@ write instead. | `ops.rs` | The op alphabet, each op mapped to one real production write-batch shape: `CommitHeaderRange` → `prepare_header_range_batch_with_roots`; `CommitBody` / `Finalize` → `prepare_block_header_and_transaction_data_batch` plus the finalization roots delete (which runs the release path internally); `Seed` → `seed_zakura_header_from_committed_block` (the non-finalized best-chain commit hook); `Reopen` → shutdown and reopen of the persistent store. The `Harness` executes ops, cross-checks the oracle's prediction against the store's response, and audits after every mutation; failures come back as a transcribable `FailureReport` (the executed op prefix plus every violation found). | | `scenarios.rs` | Scripted production event shapes (s01–s11): simple reorgs, lower-work rejections and their later reversal, split-range and walk-back deliveries, body commits racing header reorgs, reorgs to a lower height, double reorgs at one fork point, activity across the difficulty-adjustment window edge, restarts at every boundary, seed/range interplay, and refused-seed convergence. Also holds the `*_upholds_invariants` regression gates below. | | `prop.rs` | The permanent random sweep: random op sequences over the fixed universe, shrunk to minimal counterexamples on any audit failure. | +| `reads.rs` | Read-path coherence (Pillar 2): hand-corrupts the column families and asserts `recent_header_context` / the anchor round-trip report `StoreIncoherentError` (`HeaderHashMismatch`, `BrokenLinkage`, `Gap`, `BijectionMismatch`) instead of feeding stale rows into difficulty validation, and that the range writer's `StoreIncoherent` rejection is side-effect free. | ## Fixed corruption bugs gated by this suite diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/reads.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/reads.rs new file mode 100644 index 00000000000..08196889c11 --- /dev/null +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/reads.rs @@ -0,0 +1,281 @@ +//! Read-path coherence: consensus readers verify store invariants as they +//! walk, surfacing corruption as an explicit [`StoreIncoherentError`] instead +//! of feeding stale rows into difficulty validation (REORG_PLAN Pillar 2). +//! +//! The write path refuses to create incoherent stores (the Phase-1.5 guards), +//! so these tests corrupt the column families directly — the same hand-made +//! corruption technique as the audit's own tests — and assert that: +//! +//! - `recent_header_context` returns `BrokenLinkage`/`Gap` instead of a +//! poisoned or silently shortened difficulty window, and +//! - the range writer rejects with `StoreIncoherent` (a local storage fault, +//! never a peer-attributed validation failure) and leaves the store +//! untouched, and +//! - the anchor round-trip distinguishes a bijection violation in our own +//! indexes (`BijectionMismatch`) from a genuinely unknown anchor. + +use std::sync::Arc; + +use zebra_chain::block::{self, Height}; + +use super::super::super::{ZAKURA_HEADER_BY_HEIGHT, ZAKURA_HEADER_HASH_BY_HEIGHT}; +use super::super::common::{commit_header_range, state_with_genesis_config}; +use super::{ + audit::dump_store, + fabricate::{Universe, BRANCH_A, FORK_HEIGHT}, +}; +use crate::{ + error::{CommitHeaderRangeError, StoreIncoherentError}, + service::finalized_state::{ + disk_db::{DiskWriteBatch, WriteDisk}, + ZebraDb, + }, + Config, +}; + +/// A store holding genesis plus the trunk up to the fork height, built through +/// the production write path. +fn trunk_state(universe: &Universe) -> ZebraDb { + let state = state_with_genesis_config( + &universe.network, + universe.genesis.clone(), + Config::ephemeral(), + ); + let trunk_headers: Vec<_> = universe.trunk[..FORK_HEIGHT as usize] + .iter() + .map(|fab| fab.header.clone()) + .collect(); + commit_header_range(&state, universe.genesis.hash(), &trunk_headers); + state +} + +/// Re-delivers a slice of trunk headers through the production range writer. +fn redeliver_trunk( + state: &ZebraDb, + universe: &Universe, + anchor_height: u32, + len: usize, +) -> Result { + let anchor = universe.trunk_at(anchor_height).hash; + let headers: Vec> = universe.trunk + [anchor_height as usize..anchor_height as usize + len] + .iter() + .map(|fab| fab.header.clone()) + .collect(); + let body_sizes = vec![0; headers.len()]; + let mut batch = DiskWriteBatch::new(); + let result = batch.prepare_header_range_batch(state, anchor, &headers, &body_sizes); + if result.is_ok() { + state + .write_batch(batch) + .expect("header range batch writes successfully"); + } + result +} + +/// A coherent store yields a full-span window mid-chain and a legitimately +/// short window near genesis. +#[test] +fn coherent_walks_return_ok_contexts() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe); + + let full = state + .recent_header_context(Height(40)) + .expect("coherent store walks cleanly"); + assert_eq!( + full.len(), + crate::service::check::difficulty::POW_ADJUSTMENT_BLOCK_SPAN + ); + + // Heights 5..=0 inclusive: six rows, then the walk stops at genesis. + let short = state + .recent_header_context(Height(5)) + .expect("a short walk ending at genesis is legitimate"); + assert_eq!(short.len(), 6); + + // A missing anchor row is not incoherence; the caller decides what an + // unknown anchor means. + let missing = state + .recent_header_context(Height(FORK_HEIGHT + 10)) + .expect("a missing starting row is not a violation"); + assert!(missing.is_empty()); +} + +/// A header row that is not the block its hash row names (the incident-shaped +/// poison: a stale row's (threshold, time) feeding the DAA window): the walk +/// reports the divergence instead of consuming the row, and the range writer +/// maps it to a side-effect-free `StoreIncoherent` rejection. +#[test] +fn foreign_header_row_is_reported_not_validated() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe); + + // Overwrite the header row at height 20 with a branch-A header while the + // hash rows keep claiming the trunk. + let header_cf = state.db.cf_handle(ZAKURA_HEADER_BY_HEIGHT).unwrap(); + let foreign_header = universe.branches[BRANCH_A].headers[3].header.clone(); + let mut batch = DiskWriteBatch::new(); + batch.zs_insert(&header_cf, Height(20), foreign_header); + state.db.write(batch).expect("raw insert writes"); + + let error = state + .recent_header_context(Height(30)) + .expect_err("the walk crosses the corrupted row"); + assert!( + matches!( + error, + StoreIncoherentError::HeaderHashMismatch { height, .. } if height == Height(20) + ), + "expected HeaderHashMismatch at the corrupted row, got {error:?}" + ); + + // The writer surfaces the same fault as a local rejection: the range is + // not blamed (no contextual-validation error) and nothing is written. + let dump_before = dump_store(&state); + let error = redeliver_trunk(&state, &universe, 30, 5) + .expect_err("validation context crosses the corrupted row"); + assert!( + matches!( + error, + CommitHeaderRangeError::StoreIncoherent( + StoreIncoherentError::HeaderHashMismatch { .. } + ) + ), + "expected StoreIncoherent(HeaderHashMismatch), got {error:?}" + ); + assert_eq!( + dump_store(&state), + dump_before, + "a StoreIncoherent rejection must be side-effect free" + ); +} + +/// A self-consistent foreign row (its header *is* the block its hash row +/// names, but it belongs to another branch): the row above it no longer links +/// down, and the walk reports the broken link from above. +#[test] +fn broken_linkage_is_reported_not_validated() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe); + + // Replace both the header and hash rows at height 20 with branch-A's + // fourth row: internally consistent, but trunk@21 does not link to it. + let foreign = &universe.branches[BRANCH_A].headers[3]; + let header_cf = state.db.cf_handle(ZAKURA_HEADER_BY_HEIGHT).unwrap(); + let hash_cf = state.db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); + let mut batch = DiskWriteBatch::new(); + batch.zs_insert(&header_cf, Height(20), foreign.header.clone()); + batch.zs_insert(&hash_cf, Height(20), foreign.hash); + state.db.write(batch).expect("raw insert writes"); + + let error = state + .recent_header_context(Height(30)) + .expect_err("the walk crosses the corrupted row"); + assert!( + matches!( + error, + StoreIncoherentError::BrokenLinkage { height, actual_below, .. } + if height == Height(21) && actual_below == foreign.hash + ), + "expected BrokenLinkage above the corrupted row, got {error:?}" + ); + + let error = redeliver_trunk(&state, &universe, 30, 5) + .expect_err("validation context crosses the corrupted row"); + assert!( + matches!( + error, + CommitHeaderRangeError::StoreIncoherent(StoreIncoherentError::BrokenLinkage { .. }) + ), + "expected StoreIncoherent(BrokenLinkage), got {error:?}" + ); +} + +/// A gap mid-window: a missing row below a stored one is incoherence, not the +/// end of history — a silently shortened window would shift the difficulty +/// adjustment instead of failing. +#[test] +fn gap_is_reported_not_shortened() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe); + + let header_cf = state.db.cf_handle(ZAKURA_HEADER_BY_HEIGHT).unwrap(); + let hash_cf = state.db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); + let mut batch = DiskWriteBatch::new(); + batch.zs_delete(&header_cf, Height(15)); + batch.zs_delete(&hash_cf, Height(15)); + state.db.write(batch).expect("raw delete writes"); + + let error = state + .recent_header_context(Height(25)) + .expect_err("the walk crosses the gap"); + assert!( + matches!( + error, + StoreIncoherentError::Gap { height, missing } + if height == Height(16) && missing == Height(15) + ), + "expected Gap below height 16, got {error:?}" + ); + + let error = + redeliver_trunk(&state, &universe, 25, 5).expect_err("validation context crosses the gap"); + assert!( + matches!( + error, + CommitHeaderRangeError::StoreIncoherent(StoreIncoherentError::Gap { .. }) + ), + "expected StoreIncoherent(Gap), got {error:?}" + ); +} + +/// A hash→height entry whose height→hash row disagrees is a bijection +/// violation in our own indexes: the anchor round-trip reports it as +/// `StoreIncoherent`, distinct from a genuinely unknown anchor. +#[test] +fn anchor_bijection_violation_is_store_incoherent_not_unknown() { + let _init_guard = zebra_test::init(); + let universe = Universe::new(); + let state = trunk_state(&universe); + + // The hash row at height 30 now names a different block, while + // height_by_hash still maps the trunk hash to 30. + let hash_cf = state.db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); + let stray_hash = universe.branches[BRANCH_A].headers[0].hash; + let mut batch = DiskWriteBatch::new(); + batch.zs_insert(&hash_cf, Height(30), stray_hash); + state.db.write(batch).expect("raw insert writes"); + + let error = redeliver_trunk(&state, &universe, 30, 5) + .expect_err("the anchor round-trip fails on the corrupted index"); + assert!( + matches!( + error, + CommitHeaderRangeError::StoreIncoherent(StoreIncoherentError::BijectionMismatch { + hash, + height, + stored: Some(stored), + }) if hash == universe.trunk_at(30).hash + && height == Height(30) + && stored == stray_hash + ), + "expected StoreIncoherent(BijectionMismatch), got {error:?}" + ); + + // An anchor the store has never heard of stays UnknownAnchor. + let unknown = universe.branches[BRANCH_A].headers[10].hash; + let headers = vec![universe.trunk[30].header.clone()]; + let mut batch = DiskWriteBatch::new(); + let error = batch + .prepare_header_range_batch(&state, unknown, &headers, &[0]) + .expect_err("an unindexed anchor is unknown"); + assert!( + matches!(error, CommitHeaderRangeError::UnknownAnchor { .. }), + "expected UnknownAnchor, got {error:?}" + ); +} diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs index 858e34c092c..7bca301f624 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs @@ -1020,10 +1020,16 @@ fn header_range_commit_rejects_non_current_anchor_hash() { let block2 = mainnet_block(2); let alternate_block2 = alternate_header(stale_anchor, &block2.header, 1); + // A hash→height entry whose height→hash row names a different block is a + // bijection violation in our own indexes, reported as a local storage + // fault rather than an unknown anchor. Either way the stale anchor cannot + // be committed on. let mut batch = DiskWriteBatch::new(); assert!(matches!( batch.prepare_header_range_batch(&state, stale_anchor, &[alternate_block2], &[0]), - Err(CommitHeaderRangeError::UnknownAnchor { anchor }) if anchor == stale_anchor + Err(CommitHeaderRangeError::StoreIncoherent( + crate::error::StoreIncoherentError::BijectionMismatch { hash, height, stored }, + )) if hash == stale_anchor && height == Height(1) && stored == Some(block1.hash()) )); assert_eq!(state.hash(Height(1)), None); @@ -1246,7 +1252,9 @@ fn synthetic_headers_from_state( ) -> Vec> { let network = state.network(); let template = mainnet_block(1); - let mut context = state.recent_header_context(anchor_height); + let mut context = state + .recent_header_context(anchor_height) + .expect("test store is coherent"); let mut previous_hash = anchor_hash; let mut previous_height = anchor_height; let mut nonce_tag = nonce_seed; diff --git a/zebrad/src/commands/start.rs b/zebrad/src/commands/start.rs index fe94936d94d..62fa6a78d6f 100644 --- a/zebrad/src/commands/start.rs +++ b/zebrad/src/commands/start.rs @@ -2346,6 +2346,25 @@ mod zakura_header_sync_driver_tests { ); } + #[test] + fn store_incoherent_is_local_header_sync_commit_failure() { + // A range rejected because our own header rows failed a + // linkage/bijection check is a local storage fault; scoring peers for + // it recreates the disconnect-honest-peers failure mode. + let error = zebra_state::CommitHeaderRangeError::StoreIncoherent( + zebra_state::StoreIncoherentError::BrokenLinkage { + height: block::Height(2), + expected_parent: block::Hash([0; 32]), + actual_below: block::Hash([1; 32]), + }, + ); + + assert_eq!( + header_range_commit_failure_kind(&error), + HeaderSyncCommitFailureKind::Local + ); + } + #[test] fn unlinked_range_is_local_header_sync_commit_failure() { // The reactor validates every response's linkage against the requested diff --git a/zebrad/src/commands/start/zakura/header_sync_driver.rs b/zebrad/src/commands/start/zakura/header_sync_driver.rs index a3e55b9f899..a4d6ad9bb8b 100644 --- a/zebrad/src/commands/start/zakura/header_sync_driver.rs +++ b/zebrad/src/commands/start/zakura/header_sync_driver.rs @@ -1118,6 +1118,12 @@ pub(crate) fn header_range_commit_failure_kind( // store's own linkage check failing means the local anchor/response pairing // went wrong, not that the peer misbehaved. | zebra_state::CommitHeaderRangeError::UnlinkedRange { .. } + // Store incoherence is by definition a local storage fault: the range was + // rejected because our own header rows failed a linkage/bijection check + // while reading validation context, not because the peer's range was shown + // invalid. Scoring peers for it recreates the disconnect-honest-peers + // failure mode from the 2026-07-06 incidents. + | zebra_state::CommitHeaderRangeError::StoreIncoherent(_) | zebra_state::CommitHeaderRangeError::CommitResponseDropped => { HeaderSyncCommitFailureKind::Local } From d708235f37f8d5e4f5feee3c49e79b8d670587e1 Mon Sep 17 00:00:00 2001 From: roman Date: Mon, 6 Jul 2026 17:46:27 -0600 Subject: [PATCH 05/10] simplify comment --- .../block/tests/header_store_coherence/README.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/README.md b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/README.md index 4b49718607a..7f646d7ff12 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/README.md +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/README.md @@ -39,12 +39,11 @@ write instead. ## Fixed corruption bugs gated by this suite -Phase 1 of this suite proved three write-path corruption bug classes, each -pinned by a `corruption_repro_*` test that deterministically demonstrated the -violation. The write-path fixes (`REORG_PLAN.md` Phase 1.5) closed all three; -the repro tests were removed with the fixes, and their -`_upholds_invariants` twins now assert the fixed behavior over the same -op sequences as permanent regression gates: +This suite found and closed three write-path corruption bug classes. Each was +first pinned by a `corruption_repro_*` test that deterministically demonstrated +the violation; once the writer was fixed, the repro test was removed and its +`_upholds_invariants` twin now asserts the fixed behavior over the same +op sequence as a permanent regression gate: 1. **Unlinked-anchor commit** (`unlinked_anchor_commit_upholds_invariants`). `prepare_header_range_batch_with_roots` used to accept ranges without From 1a5936e46b19d65a9523b30ed274a60f0f840041 Mon Sep 17 00:00:00 2001 From: roman Date: Mon, 6 Jul 2026 18:11:40 -0600 Subject: [PATCH 06/10] fix lint --- supply-chain/config.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/supply-chain/config.toml b/supply-chain/config.toml index f7d83997820..2cd88a879f6 100644 --- a/supply-chain/config.toml +++ b/supply-chain/config.toml @@ -466,7 +466,7 @@ version = "0.8.6" criteria = "safe-to-deploy" [[exemptions.crossbeam-epoch]] -version = "0.9.18" +version = "0.9.20" criteria = "safe-to-deploy" [[exemptions.crossbeam-utils]] From e3e26c87cc6be5e3c74ab0215aa5dd57811c0278 Mon Sep 17 00:00:00 2001 From: roman Date: Mon, 6 Jul 2026 20:52:44 -0600 Subject: [PATCH 07/10] clean up comment --- .../zebra_db/block/tests/header_store_coherence/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/README.md b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/README.md index cf788dcf19b..536aebdb416 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/README.md +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/README.md @@ -36,7 +36,7 @@ write instead. | `ops.rs` | The op alphabet, each op mapped to one real production write-batch shape: `CommitHeaderRange` → `prepare_header_range_batch_with_roots`; `CommitBody` / `Finalize` → `prepare_block_header_and_transaction_data_batch` plus the finalization roots delete (which runs the release path internally); `Seed` → `seed_zakura_header_from_committed_block` (the non-finalized best-chain commit hook); `Reopen` → shutdown and reopen of the persistent store. The `Harness` executes ops, cross-checks the oracle's prediction against the store's response, and audits after every mutation; failures come back as a transcribable `FailureReport` (the executed op prefix plus every violation found). | | `scenarios.rs` | Scripted production event shapes (s01–s11): simple reorgs, lower-work rejections and their later reversal, split-range and walk-back deliveries, body commits racing header reorgs, reorgs to a lower height, double reorgs at one fork point, activity across the difficulty-adjustment window edge, restarts at every boundary, seed/range interplay, and refused-seed convergence. Also holds the `*_upholds_invariants` regression gates below. | | `prop.rs` | The permanent random sweep: random op sequences over the fixed universe, shrunk to minimal counterexamples on any audit failure. | -| `reads.rs` | Read-path coherence (Pillar 2): hand-corrupts the column families and asserts `recent_header_context` / the anchor round-trip report `StoreIncoherentError` (`HeaderHashMismatch`, `BrokenLinkage`, `Gap`, `BijectionMismatch`) instead of feeding stale rows into difficulty validation, and that the range writer's `StoreIncoherent` rejection is side-effect free. | +| `reads.rs` | Read-path coherence: hand-corrupts the column families and asserts `recent_header_context` / the anchor round-trip report `StoreIncoherentError` (`HeaderHashMismatch`, `BrokenLinkage`, `Gap`, `BijectionMismatch`) instead of feeding stale rows into difficulty validation, and that the range writer's `StoreIncoherent` rejection is side-effect free. | ## Fixed corruption bugs gated by this suite From 9170f1d49f16513144592377992f696acfdf5f22 Mon Sep 17 00:00:00 2001 From: roman Date: Mon, 6 Jul 2026 20:57:14 -0600 Subject: [PATCH 08/10] simplify comment --- .../zebra_db/block/tests/header_store_coherence/reads.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/reads.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/reads.rs index 08196889c11..84de63a5159 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/reads.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/reads.rs @@ -1,10 +1,7 @@ -//! Read-path coherence: consensus readers verify store invariants as they -//! walk, surfacing corruption as an explicit [`StoreIncoherentError`] instead -//! of feeding stale rows into difficulty validation (REORG_PLAN Pillar 2). +//! Read-path coherence tests for finalized header storage. //! -//! The write path refuses to create incoherent stores (the Phase-1.5 guards), -//! so these tests corrupt the column families directly — the same hand-made -//! corruption technique as the audit's own tests — and assert that: +//! The write path rejects incoherent stores, so these tests corrupt column +//! families directly and assert that: //! //! - `recent_header_context` returns `BrokenLinkage`/`Gap` instead of a //! poisoned or silently shortened difficulty window, and From 4459656a858149e0b585a99c57d17c0f3eac463e Mon Sep 17 00:00:00 2001 From: roman Date: Mon, 6 Jul 2026 21:01:13 -0600 Subject: [PATCH 09/10] clean up comments --- zebrad/src/commands/start/zakura/header_sync_driver.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zebrad/src/commands/start/zakura/header_sync_driver.rs b/zebrad/src/commands/start/zakura/header_sync_driver.rs index a4d6ad9bb8b..8162b6a0184 100644 --- a/zebrad/src/commands/start/zakura/header_sync_driver.rs +++ b/zebrad/src/commands/start/zakura/header_sync_driver.rs @@ -1122,7 +1122,7 @@ pub(crate) fn header_range_commit_failure_kind( // rejected because our own header rows failed a linkage/bijection check // while reading validation context, not because the peer's range was shown // invalid. Scoring peers for it recreates the disconnect-honest-peers - // failure mode from the 2026-07-06 incidents. + // failure mode. | zebra_state::CommitHeaderRangeError::StoreIncoherent(_) | zebra_state::CommitHeaderRangeError::CommitResponseDropped => { HeaderSyncCommitFailureKind::Local From 56d6678616f0570578e31fbb245296a24cc86f1f Mon Sep 17 00:00:00 2001 From: evan-forbes Date: Tue, 7 Jul 2026 06:15:11 -0500 Subject: [PATCH 10/10] fix(state): update zakura startup-audit expectations --- .../tests/header_store_coherence/startup_audit.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/startup_audit.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/startup_audit.rs index c87f00b2d79..c1dfe093de9 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/startup_audit.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/header_store_coherence/startup_audit.rs @@ -34,7 +34,7 @@ use super::{ fabricate::{Universe, BRANCH_A, FORK_HEIGHT}, }; use crate::{ - error::CommitHeaderRangeError, + error::{CommitHeaderRangeError, StoreIncoherentError}, service::finalized_state::{ disk_db::{DiskWriteBatch, WriteDisk}, ZebraDb, @@ -197,7 +197,10 @@ fn startup_heals_broken_linkage_then_resync_converges() { assert_eq!(dump_store(&state), truncated(&original, Height(19))); assert_clean(&state); assert!( - !state.recent_header_context(Height(19)).is_empty(), + !state + .recent_header_context(Height(19)) + .expect("the healed store is coherent") + .is_empty(), "the healed store supplies recent header context" ); @@ -470,8 +473,11 @@ fn startup_heal_unblocks_linkage_verified_reads() { .expect_err("the anchor round-trip fails on the corrupted index"); assert!(matches!( error, - CommitHeaderRangeError::UnknownAnchor { anchor: rejected_anchor } - if rejected_anchor == anchor + CommitHeaderRangeError::StoreIncoherent(StoreIncoherentError::BijectionMismatch { + hash, + height: Height(30), + stored + }) if hash == anchor && stored == Some(stray_hash) )); let repair = state