diff --git a/zebra-state/src/request.rs b/zebra-state/src/request.rs index b834aa935cd..b7d044b9f69 100644 --- a/zebra-state/src/request.rs +++ b/zebra-state/src/request.rs @@ -932,10 +932,15 @@ pub enum Request { /// /// A `0` value means unknown. These hints are not consensus data. body_sizes: Vec, - /// Tree-aux roots, parallel to `headers`. + /// The header-authenticated confirmed prefix of the range's tree-aux roots, aligned from the + /// first header's height. /// - /// Every non-empty Zakura header range must provide one root per header. - /// Roots are advisory until verified during block commit. + /// For a semantically-validated forward range this is exactly one shorter than `headers`: + /// the range tip's root is only authenticated once the next range delivers its successor + /// header (the one-block confirmation lag), so the caller omits it and the state persists + /// exactly the roots it is given. Checkpoint-authenticated backfill ranges carry no roots, + /// so an empty vector is also accepted (nothing is persisted). Roots remain advisory + /// (superseded by the verified row) until the block body is committed. tree_aux_roots: Vec, }, @@ -1407,6 +1412,23 @@ pub enum ReadRequest { /// Returns the highest header held on disk. BestHeaderTip, + /// Reconstructs the ZIP-221 history tree positioned at the confirmed header frontier. + /// + /// The durable history tree at `verified_block_tip` is the reconstruction base; the per-height + /// roots stored for `(verified_block_tip, best_header_tip)` — exclusive of the header tip, whose + /// root is never persisted — are folded onto it. These roots were verified when they were + /// committed, so the fold is direct (no header re-check), and the returned tree is positioned at + /// the confirmed frontier one block behind the header tip. + /// + /// Returns [`ReadResponse::BestHeaderHistoryTree`](ReadResponse::BestHeaderHistoryTree), which + /// also carries the `(height, hash)` the tree is positioned at when a header lead exists. + BestHeaderHistoryTree { + /// Verified block tip whose durable history tree is the reconstruction base. + verified_block_tip: block::Height, + /// Header tip; roots are folded up to the block below it (its own root is unpersisted). + best_header_tip: block::Height, + }, + /// Returns header-known, body-missing heights in `(verified_block_tip, best_header_tip]`. MissingBlockBodies { /// First height to consider. @@ -1635,6 +1657,7 @@ impl ReadRequest { ReadRequest::HeadersByHeightRange { .. } => "headers_by_height_range", ReadRequest::BlockRoots { .. } => "block_roots", ReadRequest::BestHeaderTip => "best_header_tip", + ReadRequest::BestHeaderHistoryTree { .. } => "best_header_history_tree", ReadRequest::MissingBlockBodies { .. } => "missing_block_bodies", ReadRequest::MissingBlockBodyMetadata { .. } => "missing_block_body_metadata", ReadRequest::BlockSizeHints { .. } => "block_size_hints", diff --git a/zebra-state/src/response.rs b/zebra-state/src/response.rs index 80991e95d54..e4b678b1cfe 100644 --- a/zebra-state/src/response.rs +++ b/zebra-state/src/response.rs @@ -395,6 +395,18 @@ pub enum ReadResponse { /// Response to [`ReadRequest::BestHeaderTip`]. BestHeaderTip(Option<(block::Height, block::Hash)>), + /// Response to [`ReadRequest::BestHeaderHistoryTree`]. + /// + /// `tree` is positioned at `frontier`, the highest contiguous confirmed header-root height the + /// fold could reach. Callers use `frontier` as the overlap anchor and resume header sync from the + /// next height. + BestHeaderHistoryTree { + /// The reconstructed history tree, positioned at `frontier`. + tree: Arc, + /// The `(height, hash)` the tree is positioned at (the confirmed contiguous frontier). + frontier: (block::Height, block::Hash), + }, + /// Response to [`ReadRequest::MissingBlockBodies`]. MissingBlockBodies(Vec), @@ -594,6 +606,7 @@ impl TryFrom for Response { | ReadResponse::ChainInfo(_) | ReadResponse::Headers(_) | ReadResponse::BestHeaderTip(_) + | ReadResponse::BestHeaderHistoryTree { .. } | ReadResponse::MissingBlockBodies(_) | ReadResponse::MissingBlockBodyMetadata(_) | ReadResponse::BlockSizeHints(_) diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs index 4641208fc79..f2b455ac047 100644 --- a/zebra-state/src/service.rs +++ b/zebra-state/src/service.rs @@ -66,6 +66,8 @@ pub mod block_iter; pub mod chain_tip; pub mod watch_receiver; +const BEST_HEADER_HISTORY_TREE_REBUILD_LOG_INTERVAL: u32 = 100_000; + pub mod check; pub(crate) mod finalized_state; @@ -1535,6 +1537,8 @@ where C: AsRef, { let capped_count = count.min(MAX_HEADER_SYNC_HEIGHT_RANGE); + let finalized_tip_height = db.finalized_tip_height(); + let best_header_tip_height = db.best_header_tip().map(|(height, _hash)| height); let mut roots = Vec::with_capacity(usize::try_from(capped_count).expect("capped root count fits in usize")); @@ -1543,18 +1547,23 @@ where break; }; - let root = if db - .finalized_tip_height() - .is_some_and(|finalized_tip| height <= finalized_tip) - { + let chain_contains_height = chain + .as_ref() + .map(|chain| chain.as_ref().contains_block_height(height)) + .unwrap_or(false); + + let mut root_source = "zakura_header"; + let root = if finalized_tip_height.is_some_and(|finalized_tip| height <= finalized_tip) { + root_source = "finalized"; finalized_state::serve_block_roots(db, height..=height) .into_iter() .next() - } else if let Some(chain) = chain - .as_ref() - .map(|chain| chain.as_ref()) - .filter(|chain| chain.contains_block_height(height)) - { + } else if chain_contains_height { + root_source = "non_finalized"; + let chain = chain + .as_ref() + .map(|chain| chain.as_ref()) + .expect("chain exists because it contains this block height"); match ( chain.sapling_tree(height.into()), chain.orchard_tree(height.into()), @@ -1598,10 +1607,36 @@ where }; let Some(root) = root else { + tracing::warn!( + ?start, + count, + capped_count, + ?height, + offset, + root_source, + ?finalized_tip_height, + ?best_header_tip_height, + chain_contains_height, + has_body = db.contains_body_at_height(height), + has_header = !db.headers_by_height_range(height, 1).is_empty(), + returned_count = roots.len(), + "Zakura BlockRoots read stopped at missing commitment root" + ); break; }; if root.height != height { + tracing::warn!( + ?start, + count, + capped_count, + ?height, + returned_height = ?root.height, + offset, + root_source, + returned_count = roots.len(), + "Zakura BlockRoots read stopped at non-contiguous commitment root" + ); break; } @@ -1611,6 +1646,214 @@ where roots } +/// Reconstructs the ZIP-221 history tree positioned at the *confirmed* header frontier. +/// +/// The durable history tree at `verified_block_tip` is the base. The per-height roots stored for +/// `(verified_block_tip, best_header_tip)` are folded onto it in bounded windows — note the range +/// is **exclusive** of `best_header_tip`. A committed forward range never persists the root of its +/// own tip block: that root is only authenticated by the *next* range's first header (the one-block +/// confirmation lag), so the highest durable root is at `best_header_tip - 1`. The returned tree is +/// therefore positioned at the confirmed frontier (`best_header_tip - 1` when the header tip leads +/// the verified body tip, otherwise the base). +/// +/// This mirrors the running reactor, which keeps its in-memory tree one block behind the header tip +/// and re-validates the tip's root through the overlapping next forward range. A restart that folded +/// the tip's root here would instead *trust* an unauthenticated peer-supplied root, so we stop one +/// block short and let the overlap re-validate it. +/// +/// Errors only if the base history tree at `verified_block_tip` is missing at a real verified tip, +/// or a fold fails. A missing header/root *above* the base is not an error but the contiguous +/// frontier, where folding stops. +/// +/// Alongside the tree, returns the frontier `(height, hash)` the tree is positioned at: the highest +/// contiguous confirmed header-root height reached (the verified base when nothing folds). The +/// caller seeds `best_header_parent_hash` and resumes header sync from `frontier + 1` off this one +/// authoritative value, so a one-block gap in the persisted roots resumes from the gap rather than +/// capping all the way back to the verified tip. +fn best_header_history_tree( + chain: Option, + db: &ZebraDb, + network: &Network, + mut verified_block_tip: block::Height, + best_header_tip: block::Height, +) -> Result< + ( + Arc, + (block::Height, block::Hash), + ), + BoxError, +> +where + C: AsRef + Clone, +{ + use zebra_chain::{history_tree::HistoryTree, parallel::commitment_aux_verify}; + + // Base tree as of the verified block tip. A missing tree is only legitimate for the empty or + // genesis state (height 0); at any real verified tip the state always holds a tree (empty + // pre-Heartwood, non-empty after), so a `None` there is an inconsistency we must not paper over + // with an empty base — that would silently stall (or, once folding starts, panic) verification. + let base_tree = match read::tree::history_tree(chain.clone(), db, verified_block_tip.into()) { + Some(tree) => tree, + None if verified_block_tip == block::Height(0) => Arc::new(HistoryTree::default()), + None => { + // The caller's `verified_block_tip` lagged the committed tip: a concurrent checkpoint or + // legacy commit advanced the finalized tip during the round-trip (the runtime lazy-rebuild + // path), and below the checkpoint the finalized history tree is stored only at the current + // tip — there is no per-height finalized history tree. Re-base at this snapshot's tip, + // read atomically from the same `chain`/`db`, which is the real committed base; the header + // frontier tracks it, so any header lead folds on top from here. + let tip = read::tip_height(chain.clone(), db).ok_or_else(|| { + BoxError::from("cannot rebuild header-tip history tree: no state tip") + })?; + verified_block_tip = tip; + match read::tree::history_tree(chain.clone(), db, tip.into()) { + Some(tree) => tree, + None if tip == block::Height(0) => Arc::new(HistoryTree::default()), + None => { + return Err(format!( + "cannot rebuild header-tip history tree: no history tree at state tip {tip:?}" + ) + .into()) + } + } + } + }; + + // Construct the MMR tree up to + // this height. + let mmr_tree_height_target = if best_header_tip > verified_block_tip { + block::Height(best_header_tip.0 - 1) + } else { + verified_block_tip + }; + let total_to_fold = mmr_tree_height_target + .0 + .saturating_sub(verified_block_tip.0); + let started_at = Instant::now(); + let mut next_progress_log = verified_block_tip + .0 + .saturating_add(BEST_HEADER_HISTORY_TREE_REBUILD_LOG_INTERVAL); + + if total_to_fold > 0 { + tracing::info!( + ?verified_block_tip, + ?best_header_tip, + target = ?mmr_tree_height_target, + total_to_fold, + "rebuilding Zakura best header history tree from durable roots" + ); + } + + let mut tree = (*base_tree).clone(); + let mut next = verified_block_tip + .next() + .map_err(|_| BoxError::from("verified block tip height overflow"))?; + + // The (height, hash) of the last folded header — the contiguous frontier the tree ends at. + // Stays `None` while the tree is still at the base. + let mut reconstructed_frontier: Option<(block::Height, block::Hash)> = None; + + while next <= mmr_tree_height_target { + // `+ 1` because the range is inclusive of `mmr_tree_height_target`. + let remaining = mmr_tree_height_target.0 - next.0 + 1; + let count = remaining.min(MAX_HEADER_SYNC_HEIGHT_RANGE); + + let headers = headers_by_height_range(chain.clone(), db, next, count); + let roots = block_roots_by_height_range(chain.clone(), db, next, count); + + // Both reads return a contiguous prefix from `next`, so the number of aligned `(header, + // root)` pairs is the shorter length. A short read means the persisted roots stop here (a + // gap, or simply the frontier): fold what is contiguous and stop — this is not an error. + let contiguous = headers.len().min(roots.len()); + if contiguous > 0 { + reconstructed_frontier = headers + .get(contiguous - 1) + .map(|(height, hash, _header)| (*height, *hash)); + + let roots_by_height = headers + .iter() + .zip(roots.iter()) + .take(contiguous) + .map(|((_height, _hash, header), root)| (header.as_ref(), root)); + + tree = commitment_aux_verify::append_confirmed_roots(network, tree, roots_by_height) + .map_err(|(height, error)| { + BoxError::from(format!( + "failed to fold header-tip history tree at {height:?}: {error}" + )) + })?; + + if total_to_fold >= BEST_HEADER_HISTORY_TREE_REBUILD_LOG_INTERVAL { + let (frontier_height, _frontier_hash) = reconstructed_frontier + .expect("frontier exists because at least one header was folded"); + if frontier_height.0 >= next_progress_log + || frontier_height == mmr_tree_height_target + { + tracing::info!( + ?verified_block_tip, + ?best_header_tip, + frontier = ?frontier_height, + target = ?mmr_tree_height_target, + folded = frontier_height.0.saturating_sub(verified_block_tip.0), + total_to_fold, + elapsed = ?started_at.elapsed(), + "rebuilding Zakura best header history tree" + ); + next_progress_log = frontier_height + .0 + .saturating_add(BEST_HEADER_HISTORY_TREE_REBUILD_LOG_INTERVAL); + } + } + } + + // `count` is capped at `MAX_HEADER_SYNC_HEIGHT_RANGE`. + if contiguous < count as usize { + break; + } + + let Some(advanced_blocks) = next.0.checked_add(count) else { + break; + }; + next = block::Height(advanced_blocks); + } + + // The frontier is the last folded height, or the verified base when nothing folded. The base + // always has a durable header hash (the verified tip is committed); genesis is the sole + // empty-state case. + let frontier = match reconstructed_frontier { + Some(frontier) => frontier, + None => { + let base_hash = read::hash_by_height(chain.clone(), db, verified_block_tip) + .or_else(|| { + (verified_block_tip == block::Height(0)).then(|| network.genesis_hash()) + }) + .ok_or_else(|| { + BoxError::from(format!( + "cannot rebuild header-tip history tree: no header hash at verified block \ + tip {verified_block_tip:?}" + )) + })?; + (verified_block_tip, base_hash) + } + }; + + if total_to_fold > 0 { + tracing::info!( + ?verified_block_tip, + ?best_header_tip, + frontier = ?frontier.0, + target = ?mmr_tree_height_target, + folded = frontier.0.0.saturating_sub(verified_block_tip.0), + total_to_fold, + elapsed = ?started_at.elapsed(), + complete = frontier.0 == mmr_tree_height_target, + "rebuilt Zakura best header history tree from durable roots" + ); + } + + Ok((Arc::new(tree), frontier)) +} + impl Service for ReadStateService { type Response = ReadResponse; type Error = BoxError; @@ -1859,6 +2102,29 @@ impl Service for ReadStateService { ) }; + let capped_count = count.min(MAX_HEADER_SYNC_HEIGHT_RANGE); + let capped_count_usize = + usize::try_from(capped_count).expect("capped root count fits in usize"); + if count > 0 && roots.len() < capped_count_usize { + tracing::warn!( + ?start_height, + count, + capped_count, + returned_count = roots.len(), + last_returned_height = ?roots.last().map(|root| root.height), + "Zakura BlockRoots read returned a short contiguous prefix" + ); + } else { + tracing::debug!( + ?start_height, + count, + capped_count, + returned_count = roots.len(), + last_returned_height = ?roots.last().map(|root| root.height), + "Zakura BlockRoots read completed" + ); + } + Ok(ReadResponse::BlockRoots(roots)) } @@ -1877,6 +2143,20 @@ impl Service for ReadStateService { )) } + ReadRequest::BestHeaderHistoryTree { + verified_block_tip, + best_header_tip, + } => { + let (tree, frontier) = best_header_history_tree( + state.latest_best_chain(), + &state.db, + &state.db.network(), + verified_block_tip, + best_header_tip, + )?; + Ok(ReadResponse::BestHeaderHistoryTree { tree, frontier }) + } + ReadRequest::MissingBlockBodies { from, limit } => { let verified_block_tip = read::tip_height(state.latest_best_chain(), &state.db); let best_header_tip = state diff --git a/zebra-state/src/service/finalized_state/zebra_db/block.rs b/zebra-state/src/service/finalized_state/zebra_db/block.rs index 8feead51841..dc64a28448d 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block.rs @@ -1957,12 +1957,22 @@ impl DiskWriteBatch { headers: &[Arc], body_sizes: &[u32], ) -> Result { - let roots = inferred_header_range_roots(zebra_db, anchor, headers.len())?; + // The commit path only accepts the confirmed prefix (one root shorter than the headers, and + // none for a single-header range), so infer exactly that many placeholder roots. + let roots = inferred_header_range_roots(zebra_db, anchor, headers.len().saturating_sub(1))?; self.prepare_header_range_batch_with_roots(zebra_db, anchor, headers, body_sizes, &roots) } /// Prepare a database batch containing a contextually validated header range - /// and one provisional tree-aux root per header. + /// and the tree-aux roots supplied for it. + /// + /// `tree_aux_roots` is the caller's confirmed prefix, aligned from the range start. For a + /// semantically-validated forward range it is exactly one shorter than `headers` (zero roots + /// for a single-header range): the range tip's root is only authenticated by the next range's + /// successor header, so it is never confirmed here. Checkpoint-authenticated backfill ranges + /// pass an empty vector and persist no roots. Both shapes are enforced, not merely expected, so + /// it is structurally impossible to persist the unconfirmed tip root and expose peer-supplied + /// data to startup history-tree reconstruction. #[allow(clippy::unwrap_in_result)] pub fn prepare_header_range_batch_with_roots( &mut self, @@ -1983,7 +1993,17 @@ impl DiskWriteBatch { }); } - if headers.len() != tree_aux_roots.len() { + // A semantically-validated header range carries its *confirmed prefix* of roots: header + // `H + 1` authenticates the root for `H`, so over `[start..=tip]` the roots confirmed are + // `[start..=tip - 1]` and the tip's own root stays unconfirmed until the next overlapping + // range delivers its successor header. That prefix is always exactly one shorter than the + // headers (zero roots for a single-header range). + // + // Checkpoint-authenticated backfill ranges (backward ranges) carry no provisional roots at + // all, so an empty vector is also accepted. Both accepted shapes keep the trust boundary a + // state invariant: the unconfirmed tip root can never be persisted, because a full-length + // (or otherwise-longer) vector is still rejected, and an empty vector persists nothing. + if !(tree_aux_roots.is_empty() || tree_aux_roots.len() + 1 == headers.len()) { return Err(CommitHeaderRangeError::TreeAuxRootCountMismatch { headers: headers.len(), roots: tree_aux_roots.len(), @@ -2038,12 +2058,15 @@ impl DiskWriteBatch { .ok_or(CommitHeaderRangeError::HeightOverflow)?; let hash = block::Hash::from(&**header); let body_size = body_sizes[index]; - let roots = &tree_aux_roots[index]; - if roots.height != height { - return Err(CommitHeaderRangeError::TreeAuxRootHeightMismatch { - expected_height: height, - root_height: roots.height, - }); + // The tip's root is omitted from the confirmed prefix, so the last header may have no + // matching root. Present roots must align with their header height. + if let Some(roots) = tree_aux_roots.get(index) { + if roots.height != height { + return Err(CommitHeaderRangeError::TreeAuxRootHeightMismatch { + expected_height: height, + root_height: roots.height, + }); + } } if let Some(expected) = checkpoints.hash(height) { @@ -2186,22 +2209,24 @@ impl DiskWriteBatch { // only ever runs above the body tip — so a header range re-delivered // over committed heights (a header store behind the body store, or a // late range response racing body sync) must never overwrite the - // verified row: committed roots win on any overlap (design §9). - if !zebra_db.contains_body_at_height(height) { - let roots = &tree_aux_roots[index]; - self.zs_insert( - &roots_by_height, - height, - CommitmentRootsByHeight { - sapling: roots.sapling_root, - orchard: roots.orchard_root, - ironwood: roots.ironwood_root, - sapling_tx: roots.sapling_tx, - orchard_tx: roots.orchard_tx, - ironwood_tx: roots.ironwood_tx, - auth_data_root: roots.auth_data_root, - }, - ); + // verified row: committed roots win on any overlap (design §9). The tip's root is absent + // from the confirmed prefix (`tree_aux_roots.get` is `None`), so it is never persisted. + if let Some(roots) = tree_aux_roots.get(index) { + if !zebra_db.contains_body_at_height(height) { + self.zs_insert( + &roots_by_height, + height, + CommitmentRootsByHeight { + sapling: roots.sapling_root, + orchard: roots.orchard_root, + ironwood: roots.ironwood_root, + sapling_tx: roots.sapling_tx, + orchard_tx: roots.orchard_tx, + ironwood_tx: roots.ironwood_tx, + auth_data_root: roots.auth_data_root, + }, + ); + } } } diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs index 92230f2e8d1..7231137302b 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs @@ -625,6 +625,10 @@ fn header_range_roots_do_not_overwrite_committed_serving_index_rows() { false, ); + let block2 = zebra_test::vectors::BLOCK_MAINNET_2_BYTES + .zcash_deserialize_into::>() + .expect("block 2 deserializes"); + write_full_block(&mut state, genesis.clone()); write_full_block(&mut state, block1.clone()); @@ -637,7 +641,9 @@ fn header_range_roots_do_not_overwrite_committed_serving_index_rows() { // Re-deliver the real header for the committed height, but with garbage roots — // exactly what a malicious serving peer can put in a `Headers` response, since - // root bytes are unauthenticated at header-commit time. + // root bytes are unauthenticated at header-commit time. The committed height must be a confirmed + // (non-tip) root of the range, so re-deliver it as the first header of a two-header range whose + // confirmed prefix is exactly the poisoned height-1 root. let mut poisoned = root_at(Height(1)); poisoned.sapling_tx = 99; poisoned.auth_data_root = zebra_chain::block::merkle::AuthDataRoot::from([0xAA; 32]); @@ -647,8 +653,8 @@ fn header_range_roots_do_not_overwrite_committed_serving_index_rows() { .prepare_header_range_batch_with_roots( &state, genesis.hash(), - std::slice::from_ref(&block1.header), - &[0], + &[block1.header.clone(), block2.header.clone()], + &[0, 0], &[poisoned], ) .expect("re-delivering the same header over a committed height is accepted"); @@ -663,6 +669,71 @@ fn header_range_roots_do_not_overwrite_committed_serving_index_rows() { ); } +/// A header-range commit persists exactly the confirmed prefix of roots it is given: the range +/// tip's root is unauthenticated (only confirmed once the next range delivers its successor header), +/// so the caller omits it and the state must not fabricate or persist one. A tip root left on disk +/// would be folded, unverified, into the startup history-tree reconstruction. +#[test] +fn header_range_commit_persists_only_the_confirmed_root_prefix() { + let _init_guard = zebra_test::init(); + let (state, genesis, block1) = mainnet_state_with_genesis(); + let block2 = mainnet_block(2); + + let headers = vec![block1.header.clone(), block2.header.clone()]; + // Confirmed prefix: one root for the two-header range. The tip (height 2) root is omitted. + let confirmed_roots = vec![root_at(Height(1))]; + + let mut batch = DiskWriteBatch::new(); + batch + .prepare_header_range_batch_with_roots( + &state, + genesis.hash(), + &headers, + &[0, 0], + &confirmed_roots, + ) + .expect("a confirmed prefix one shorter than the headers is accepted"); + state + .write_batch(batch) + .expect("header range batch writes successfully"); + + // Both headers are durable and the header tip advances to the range tip... + assert_eq!(state.best_header_tip(), Some((Height(2), block2.hash()))); + // ...but only the confirmed height has a persisted root; the tip root is absent. + assert_eq!( + state.zakura_header_commitment_roots_by_height_range(Height(1)..=Height(1)), + vec![root_at(Height(1))], + "the confirmed height keeps its provisional root", + ); + assert!( + state + .zakura_header_commitment_roots_by_height_range(Height(2)..=Height(2)) + .is_empty(), + "the unconfirmed range tip must not have a persisted root", + ); + + // An empty vector is the checkpoint-authenticated backfill shape: it is accepted and persists no + // provisional roots, so the trust boundary is never crossed. + let mut batch = DiskWriteBatch::new(); + batch + .prepare_header_range_batch_with_roots(&state, genesis.hash(), &headers, &[0, 0], &[]) + .expect("an empty root vector is accepted for checkpoint-authenticated backfill"); + + // A full-length vector would include the unauthenticated tip root, so it is rejected. + let mut batch = DiskWriteBatch::new(); + let full_roots = vec![root_at(Height(1)), root_at(Height(2))]; + assert!(matches!( + batch.prepare_header_range_batch_with_roots( + &state, + genesis.hash(), + &headers, + &[0, 0], + &full_roots, + ), + Err(CommitHeaderRangeError::TreeAuxRootCountMismatch { .. }), + )); +} + /// Pruning-readiness guard: a committed height whose body is removed (as online /// pruning deletes `tx_by_loc` rows) keeps its header readable from the retained /// consensus `block_header_by_height`, because the header readers are not gated diff --git a/zebra-state/src/service/tests.rs b/zebra-state/src/service/tests.rs index 8f363c28cdb..cd644dc79b4 100644 --- a/zebra-state/src/service/tests.rs +++ b/zebra-state/src/service/tests.rs @@ -13,6 +13,7 @@ use zebra_chain::{ block::{self, Block, CountedHeader, Height}, chain_tip::ChainTip, fmt::SummaryDebug, + history_tree::HistoryTree, orchard, parallel::commitment_aux::BlockCommitmentRoots, parameters::{Network, NetworkUpgrade}, @@ -26,10 +27,17 @@ use zebra_test::{prelude::*, transcript::Transcript}; use crate::{ arbitrary::Prepare, + constants::{state_database_format_version_in_code, STATE_DATABASE_KIND}, init_test, + request::{FinalizedBlock, Treestate}, service::{ - arbitrary::populated_state, chain_tip::TipAction, headers_by_height_range, - non_finalized_state::Chain, read, StateService, + arbitrary::populated_state, + best_header_history_tree, + chain_tip::TipAction, + finalized_state::{DiskWriteBatch, WriteDisk, ZebraDb, STATE_COLUMN_FAMILIES_IN_CODE}, + headers_by_height_range, + non_finalized_state::Chain, + read, StateService, }, tests::{ setup::{partial_nu5_chain_strategy, transaction_v4_from_coinbase}, @@ -56,6 +64,94 @@ fn roots_from_height(start: Height, count: u32) -> Vec { .collect() } +fn root_at(height: Height) -> BlockCommitmentRoots { + roots_from_height(height, 1) + .into_iter() + .next() + .expect("one root was requested") +} + +fn mainnet_block(height: u32) -> Arc { + zebra_test::vectors::MAINNET_BLOCKS + .get(&height) + .expect("mainnet test vector exists") + .zcash_deserialize_into::>() + .expect("mainnet test vector deserializes") +} + +fn mainnet_state_with_genesis() -> (ZebraDb, Arc, Arc) { + let genesis = mainnet_block(0); + let block1 = mainnet_block(1); + let state = new_ephemeral_db(&Network::Mainnet); + + write_full_block_header_and_transactions(&state, genesis.clone()); + + (state, genesis, block1) +} + +fn new_ephemeral_db(network: &Network) -> ZebraDb { + ZebraDb::new( + &Config::ephemeral(), + STATE_DATABASE_KIND, + &state_database_format_version_in_code(), + network, + true, + STATE_COLUMN_FAMILIES_IN_CODE + .iter() + .map(ToString::to_string), + false, + ) +} + +fn write_full_block_header_and_transactions(state: &ZebraDb, block: Arc) { + let checkpoint_verified = CheckpointVerifiedBlock::from(block); + let finalized = + FinalizedBlock::from_checkpoint_verified(checkpoint_verified, Treestate::default()); + + let mut batch = DiskWriteBatch::new(); + batch + .prepare_block_header_and_transaction_data_batch(state, &finalized, true, None) + .expect("full block header and transaction batch is valid"); + state.write_batch(batch).expect("full block batch writes"); +} + +fn commit_header_range_with_roots( + state: &ZebraDb, + anchor: block::Hash, + headers: &[Arc], + roots: &[BlockCommitmentRoots], +) -> block::Hash { + let mut batch = DiskWriteBatch::new(); + let body_sizes = vec![0; headers.len()]; + let committed_hash = batch + .prepare_header_range_batch_with_roots(state, anchor, headers, &body_sizes, roots) + .expect("header range with roots is valid"); + state + .write_batch(batch) + .expect("header range batch writes successfully"); + + committed_hash +} + +fn insert_zakura_header(state: &ZebraDb, height: Height, header: Arc) { + let header_by_height = state.db().cf_handle("zakura_header_by_height").unwrap(); + let hash_by_height = state + .db() + .cf_handle("zakura_header_hash_by_height") + .unwrap(); + let height_by_hash = state + .db() + .cf_handle("zakura_header_height_by_hash") + .unwrap(); + let hash = block::Hash::from(header.as_ref()); + let mut batch = DiskWriteBatch::new(); + + batch.zs_insert(&header_by_height, height, &header); + batch.zs_insert(&hash_by_height, height, hash); + batch.zs_insert(&height_by_hash, hash, height); + state.write_batch(batch).expect("test header rows write"); +} + async fn test_populated_state_responds_correctly( mut state: Buffer, Request>, ) -> Result<()> { @@ -535,7 +631,7 @@ async fn header_only_service_requests_preserve_body_boundary() -> std::result::R anchor: genesis.hash(), headers: vec![block1.header.clone(), block2.header.clone()], body_sizes: vec![999_999, 0], - tree_aux_roots: roots_from_height(Height(1), 2), + tree_aux_roots: roots_from_height(Height(1), 1), }) .await?, Response::Committed(block2_hash), @@ -787,7 +883,7 @@ async fn commit_header_range_completes_while_in_finalized_write_phase( anchor: genesis.hash(), headers: vec![block1.header.clone(), block2.header.clone()], body_sizes: vec![999_999, 0], - tree_aux_roots: roots_from_height(Height(1), 2), + tree_aux_roots: roots_from_height(Height(1), 1), }), ) .await @@ -861,6 +957,311 @@ async fn header_range_reads_include_non_finalized_best_chain_blocks() -> Result< Ok(()) } +#[test] +fn best_header_history_tree_uses_genesis_base_when_no_header_lead() { + let _init_guard = zebra_test::init(); + let (state, genesis, _block1) = mainnet_state_with_genesis(); + + let (tree, frontier) = best_header_history_tree( + None::>, + &state, + &Network::Mainnet, + Height(0), + Height(0), + ) + .expect("genesis-only state has a valid base history tree"); + + assert_eq!(frontier, (Height(0), genesis.hash())); + assert_eq!(tree.as_ref(), &HistoryTree::default()); +} + +#[test] +fn best_header_history_tree_stops_one_block_before_header_tip() { + let _init_guard = zebra_test::init(); + let (state, genesis, block1) = mainnet_state_with_genesis(); + let block2 = mainnet_block(2); + + commit_header_range_with_roots( + &state, + genesis.hash(), + &[block1.header.clone(), block2.header.clone()], + &[root_at(Height(1))], + ); + + let (tree, frontier) = best_header_history_tree( + None::>, + &state, + &Network::Mainnet, + Height(0), + Height(2), + ) + .expect("contiguous confirmed root reconstructs from genesis"); + + assert_eq!(frontier, (Height(1), block1.hash())); + assert_ne!(frontier, (Height(2), block2.hash())); + assert_eq!( + tree.hash(), + None, + "pre-Heartwood mainnet roots should not create a history tree", + ); +} + +#[test] +fn best_header_history_tree_stops_at_first_missing_root_or_header() { + let _init_guard = zebra_test::init(); + let (state, genesis, block1) = mainnet_state_with_genesis(); + let block2 = mainnet_block(2); + let block3 = mainnet_block(3); + + commit_header_range_with_roots( + &state, + genesis.hash(), + &[ + block1.header.clone(), + block2.header.clone(), + block3.header.clone(), + ], + &[root_at(Height(1)), root_at(Height(2))], + ); + state + .delete_zakura_header_commitment_roots([Height(2)]) + .expect("test root deletion succeeds"); + + let (_tree, frontier) = best_header_history_tree( + None::>, + &state, + &Network::Mainnet, + Height(0), + Height(3), + ) + .expect("reconstruction succeeds up to the contiguous prefix"); + + assert_eq!( + frontier, + (Height(1), block1.hash()), + "reconstruction stops at the last contiguous root before the gap", + ); +} + +#[test] +fn best_header_history_tree_returns_base_when_no_contiguous_fold_exists() { + let _init_guard = zebra_test::init(); + let (state, genesis, block1) = mainnet_state_with_genesis(); + let block2 = mainnet_block(2); + + commit_header_range_with_roots( + &state, + genesis.hash(), + &[block1.header.clone(), block2.header.clone()], + &[root_at(Height(1))], + ); + state + .delete_zakura_header_commitment_roots([Height(1)]) + .expect("test root deletion succeeds"); + + let (tree, frontier) = best_header_history_tree( + None::>, + &state, + &Network::Mainnet, + Height(0), + Height(2), + ) + .expect("missing roots above genesis leave the base frontier"); + + assert_eq!(frontier, (Height(0), genesis.hash())); + assert_eq!(tree.as_ref(), &HistoryTree::default()); +} + +#[test] +fn best_header_history_tree_rebases_to_committed_tip_when_verified_tip_has_no_tree() { + let _init_guard = zebra_test::init(); + let (state, genesis, block1) = mainnet_state_with_genesis(); + insert_zakura_header(&state, Height(1), block1.header.clone()); + + // `verified_block_tip` is ahead of the real committed tip (only genesis is committed) and has no + // stored history tree. The lazy rebuild re-bases at the committed tip read from the same snapshot + // instead of erroring, so it returns the committed base frontier (genesis) rather than papering + // over a real tip with an empty tree. + let (tree, frontier) = best_header_history_tree( + None::>, + &state, + &Network::Mainnet, + Height(1), + Height(1), + ) + .expect("rebuild re-bases to the committed tip when the verified tip has no stored tree"); + + assert_eq!(frontier, (Height(0), genesis.hash())); + assert_eq!(tree.as_ref(), &HistoryTree::default()); +} + +#[tokio::test(flavor = "multi_thread")] +async fn best_header_history_tree_read_request_returns_reconstructed_frontier() { + let _init_guard = zebra_test::init(); + let network = Network::Mainnet; + let (mut state_service, read_state, _, _) = + StateService::new(Config::ephemeral(), &network, Height::MAX, 0).await; + let block1 = mainnet_block(1); + let block2 = mainnet_block(2); + let genesis = mainnet_block(0); + + state_service + .ready() + .await + .expect("state service is ready") + .call(Request::CommitCheckpointVerifiedBlock( + CheckpointVerifiedBlock::from(genesis.clone()), + )) + .await + .expect("genesis block commits"); + + state_service + .ready() + .await + .expect("state service is ready") + .call(Request::CommitHeaderRange { + anchor: genesis.hash(), + headers: vec![block1.header.clone(), block2.header.clone()], + body_sizes: vec![0, 0], + tree_aux_roots: roots_from_height(Height(1), 1), + }) + .await + .expect("header range commits"); + + let response = read_state + .oneshot(ReadRequest::BestHeaderHistoryTree { + verified_block_tip: Height(0), + best_header_tip: Height(2), + }) + .await + .expect("best header history tree read succeeds"); + + let ReadResponse::BestHeaderHistoryTree { tree, frontier } = response else { + panic!("unexpected response: {response:?}"); + }; + + assert_eq!( + frontier, + (Height(1), block1.hash()), + "the read service returns the same confirmed frontier as the helper", + ); + assert_eq!( + tree.hash(), + None, + "pre-Heartwood mainnet roots should not create a history tree", + ); +} + +/// A block-level reorg below the header frontier: when a full block commits a header that conflicts +/// with the provisional header-sync chain, the conflicting provisional headers *and* their roots are +/// dropped, so no stale peer-supplied root survives to be folded by history-tree reconstruction. +#[test] +fn block_reorg_drops_conflicting_provisional_roots_before_reconstruction() { + let _init_guard = zebra_test::init(); + let (state, genesis, block1) = mainnet_state_with_genesis(); + let block2 = mainnet_block(2); + let block3 = mainnet_block(3); + + // Header sync races ahead on chain A: provisional headers 1..=3 with the confirmed root prefix + // (heights 1..=2; the tip's root is never persisted). + commit_header_range_with_roots( + &state, + genesis.hash(), + &[ + block1.header.clone(), + block2.header.clone(), + block3.header.clone(), + ], + &[root_at(Height(1)), root_at(Height(2))], + ); + assert_eq!( + state + .zakura_header_commitment_roots_by_height_range(Height(1)..=Height(2)) + .len(), + 2, + "chain A provisional roots are persisted before the reorg", + ); + + // A full block commits at height 1 with a header that conflicts with chain A (a reorg). The body + // commit path must drop chain A's provisional descendants and their roots — nothing carries a + // committed body yet, so the whole provisional suffix is reconciled away. + let mut conflicting_header = *block1.header; + conflicting_header.previous_block_hash = block::Hash([0x5a; 32]); + let conflicting_block = Arc::new(Block { + header: Arc::new(conflicting_header), + transactions: block1.transactions.clone(), + }); + assert_ne!(conflicting_block.header, block1.header); + + state + .seed_zakura_header_from_committed_block(Height(1), &conflicting_block) + .expect("conflicting body commit reconciles the provisional header store"); + + // Chain A's provisional roots at every height are gone: nothing stale survives to be folded. + assert!( + state + .zakura_header_commitment_roots_by_height_range(Height(1)..=Height(3)) + .is_empty(), + "a conflicting commit must drop all chain A provisional roots", + ); + + // Reconstruction over the reorged state folds nothing stale and returns the verified base. + let (tree, frontier) = best_header_history_tree( + None::>, + &state, + &Network::Mainnet, + Height(0), + Height(1), + ) + .expect("reconstruction succeeds after the reorg"); + assert_eq!(frontier, (Height(0), genesis.hash())); + assert_eq!(tree.as_ref(), &HistoryTree::default()); +} + +/// The tightened state invariant is enforced through the real request path: a `CommitHeaderRange` +/// carrying a full-length roots vector (one per header, *including* the unconfirmed tip) is rejected, +/// so a future or refactored caller cannot persist the unconfirmed tip root by mistake. +#[tokio::test(flavor = "multi_thread")] +async fn commit_header_range_rejects_full_length_roots_through_the_service() { + let _init_guard = zebra_test::init(); + let network = Network::Mainnet; + let (mut state_service, _read_state, _, _) = + StateService::new(Config::ephemeral(), &network, Height::MAX, 0).await; + let genesis = mainnet_block(0); + let block1 = mainnet_block(1); + let block2 = mainnet_block(2); + + state_service + .ready() + .await + .expect("state service is ready") + .call(Request::CommitCheckpointVerifiedBlock( + CheckpointVerifiedBlock::from(genesis.clone()), + )) + .await + .expect("genesis block commits"); + + // Full-length roots: one per header, including the range tip (height 2). Only the confirmed + // prefix (height 1) may be persisted, so the boundary must reject this shape outright. + let error = state_service + .ready() + .await + .expect("state service is ready") + .call(Request::CommitHeaderRange { + anchor: genesis.hash(), + headers: vec![block1.header.clone(), block2.header.clone()], + body_sizes: vec![0, 0], + tree_aux_roots: roots_from_height(Height(1), 2), + }) + .await + .expect_err("a full-length roots vector must be rejected by the tightened boundary"); + + assert!( + error.to_string().contains("does not match header count"), + "unexpected error: {error}", + ); +} + #[test] fn state_behaves_when_blocks_are_committed_in_order() -> Result<()> { let _init_guard = zebra_test::init();