From 55587dca202c26573a5bed6ede08127ab89a324d Mon Sep 17 00:00:00 2001 From: Roman Akhtariev Date: Mon, 29 Jun 2026 00:06:46 -0300 Subject: [PATCH 01/17] perf(sync): header-authenticated per-block checkpoint release (#296) * perf(sync): header-authenticated per-block checkpoint release Step 1 of decoupling checkpoint verification from block bodies. Instead of streaming checkpoint-class bodies into the legacy CheckpointVerifier (which re-accumulates ~400 blocks and releases a whole range via the backward walk, causing the burst sawtooth and range-coupled head-of-line stall), commit each checkpoint-class body individually using a state-minted provenance token. - zebra-state: AuthenticatedCheckpointHash provenance newtype (private ctor) + ReadRequest::AuthenticatedCheckpointHash. The token is minted strictly from the Zakura checkpoint-anchored header frontier (zakura_header_hash for both the height and the next-checkpoint anchor C), never from committed body state, so it cannot authenticate a height from finalized bodies. missing_block_bodies clamps checkpoint-class heights by authenticated_header_tip for airtight ordering. - zebra-consensus: CommitCheckpointAuthenticated request routed on height <= max_checkpoint_height to CheckpointVerifier::call_authenticated, a narrow committer that runs check_block, binds block.hash == token, and releases straight to CommitCheckpointVerifiedBlock with no queue, no range walk, and no verifier_progress mutation. No fallback, no mixed mode. - zebrad: Zakura block-sync driver reads the token at apply time and sends the authenticated request; a missing token is a hard invariant violation, not a fallback. Tagged checkpoint_commit_path = "zakura_authenticated". Also includes commit-pipeline/sync instrumentation: floor-gap and commit-frontier diagnostics in the block-sync reactor, timed_commit_phase on the history-tree push, and a commit-metrics-gated DiskWriteBatch::size_in_bytes for batch-size sampling. * build --- zebra-consensus/src/block/request.rs | 26 +++- zebra-consensus/src/checkpoint.rs | 108 ++++++++++++++++- zebra-consensus/src/checkpoint/tests.rs | 81 +++++++++++++ zebra-consensus/src/router.rs | 30 +++++ .../src/zakura/block_sync/reactor.rs | 34 +++++- zebra-state/src/lib.rs | 4 +- zebra-state/src/request.rs | 49 ++++++++ zebra-state/src/response.rs | 4 + zebra-state/src/service.rs | 6 + zebra-state/src/service/finalized_state.rs | 24 ++-- .../src/service/finalized_state/disk_db.rs | 8 ++ .../service/finalized_state/zebra_db/block.rs | 109 ++++++++++++++++- .../zebra_db/block/tests/vectors.rs | 24 ++++ .../start/zakura/block_sync_driver.rs | 111 +++++++++++++++--- 14 files changed, 583 insertions(+), 35 deletions(-) diff --git a/zebra-consensus/src/block/request.rs b/zebra-consensus/src/block/request.rs index d03abc2de26..284f15da1b1 100644 --- a/zebra-consensus/src/block/request.rs +++ b/zebra-consensus/src/block/request.rs @@ -6,7 +6,7 @@ use zebra_chain::{ block::{self, Block}, parameters::Network, }; -use zebra_state::CheckpointVerifiedBlock; +use zebra_state::{AuthenticatedCheckpointHash, CheckpointVerifiedBlock}; use crate::checkpoint::VerifyCheckpointError; @@ -26,6 +26,25 @@ pub enum Request { /// which can build these blocks concurrently across many download tasks. CommitCheckpointPrecomputed(CheckpointVerifiedBlock), + /// Commits a checkpoint-range block whose expected hash has already been + /// authenticated against the hardcoded checkpoint list by Zakura header sync. + /// + /// The verifier validates the block in isolation (proof of work, Merkle root, + /// height) and asserts `block.hash() == expected_hash`, then releases it to the + /// state commit pipeline immediately — it does **not** accumulate or walk the + /// checkpoint range. `expected_hash` is an [`AuthenticatedCheckpointHash`], a + /// provenance token the caller can only obtain from the state's authenticated + /// header frontier, so this request cannot be forged from a raw block hash. + /// + /// Only valid at or below the checkpoint height. There is no fallback: a height + /// above the checkpoint, or a hash mismatch, is a hard invariant violation. + CommitCheckpointAuthenticated { + /// The downloaded block body. + block: Arc, + /// The checkpoint-authenticated expected hash for this height. + expected_hash: AuthenticatedCheckpointHash, + }, + /// Performs semantic validation but skips checking proof of work, /// then asks the state to perform contextual validation. /// Does not commit the block to the state. @@ -81,6 +100,7 @@ impl Request { match self { Request::Commit(block) => Arc::clone(block), Request::CommitCheckpointPrecomputed(block) => Arc::clone(&block.block), + Request::CommitCheckpointAuthenticated { block, .. } => Arc::clone(block), Request::CheckProposal(block) => Arc::clone(block), } } @@ -88,7 +108,9 @@ impl Request { /// Returns `true` if the request is a proposal pub fn is_proposal(&self) -> bool { match self { - Request::Commit(_) | Request::CommitCheckpointPrecomputed(_) => false, + Request::Commit(_) + | Request::CommitCheckpointPrecomputed(_) + | Request::CommitCheckpointAuthenticated { .. } => false, Request::CheckProposal(_) => true, } } diff --git a/zebra-consensus/src/checkpoint.rs b/zebra-consensus/src/checkpoint.rs index 31de33bfbf3..8f643c538a3 100644 --- a/zebra-consensus/src/checkpoint.rs +++ b/zebra-consensus/src/checkpoint.rs @@ -37,7 +37,7 @@ use zebra_chain::{ }, work::equihash, }; -use zebra_state::{self as zs, CheckpointVerifiedBlock}; +use zebra_state::{self as zs, AuthenticatedCheckpointHash, CheckpointVerifiedBlock}; use crate::{ block::VerifyBlockError, @@ -900,6 +900,103 @@ where .boxed() } + /// Verify and release a Zakura header-authenticated checkpoint block. + /// + /// This is the narrow fast path: it validates the block in isolation (height, proof of work, + /// Merkle root) and asserts the body matches the checkpoint-authenticated header hash, then + /// releases it straight to the state commit pipeline. It does **not** enqueue the block, walk + /// the checkpoint range, or touch `verifier_progress`. + /// + /// Zakura checkpoint heights always take this path (there is no fallback into range + /// accumulation), so the verifier's range/progress state is never consulted for them; the + /// `check_height` bound stays correct because progress only ever lags the committed tip, which + /// is lenient (no false `AlreadyVerified`). The router enforces that `block` is at or below the + /// checkpoint height before calling this. + pub(crate) fn call_authenticated( + &mut self, + block: Arc, + expected_hash: AuthenticatedCheckpointHash, + ) -> Pin> + Send + 'static>> + { + // Per-block validity checks (height, proof of work, Merkle root), the same checks the + // accumulation path runs per block — defense in depth on top of the authenticated header. + let block = match self.check_block(block) { + Ok(block) => block, + Err(e) => return async move { Err(e) }.boxed(), + }; + + // Bind the body to the authenticated header. A mismatch means the downloaded body and the + // checkpoint-authenticated header disagree: a hard invariant violation, never recoverable. + let expected = expected_hash.hash(); + if block.hash != expected { + let (height, found) = (block.height, block.hash); + return async move { + Err(VerifyCheckpointError::AuthenticatedHashMismatch { + height, + expected, + found, + }) + } + .boxed(); + } + + self.commit_authenticated_block(block) + } + + /// Release an already-validated, authenticated checkpoint block straight to the state commit + /// pipeline. + /// + /// Mirrors the commit/reset behavior of [`Self::verify_and_commit`] but without the range + /// release channel: the block is committed directly (DB mutation stays parent-ordered in the + /// state write task), and on a commit error the verifier is reset to the state tip. + fn commit_authenticated_block( + &mut self, + block: CheckpointVerifiedBlock, + ) -> Pin> + Send + 'static>> + { + let hash = block.hash; + + let state_service = self.state_service.clone(); + let commit_checkpoint_verified = tokio::spawn(async move { + match state_service + .oneshot(zs::Request::CommitCheckpointVerifiedBlock(block)) + .map_err(VerifyCheckpointError::CommitCheckpointVerified) + .await? + { + zs::Response::Committed(committed_hash) => { + assert_eq!(committed_hash, hash, "state must commit correct hash"); + Ok(hash) + } + _ => unreachable!("wrong response for CommitCheckpointVerifiedBlock"), + } + }); + + let state_service = self.state_service.clone(); + let reset_sender = self.reset_sender.clone(); + async move { + let result = commit_checkpoint_verified.await; + let result = if zebra_chain::shutdown::is_shutting_down() { + Err(VerifyCheckpointError::ShuttingDown) + } else { + result.expect("commit_checkpoint_verified should not panic") + }; + if result.is_err() { + // Keep the verifier consistent with the state for any later legacy caller. + let tip = match state_service + .oneshot(zs::Request::Tip) + .await + .map_err(VerifyCheckpointError::Tip)? + { + zs::Response::Tip(tip) => tip, + _ => unreachable!("wrong response for Tip"), + }; + let _ = reset_sender.send(tip); + } + result + } + .boxed() + } + /// During checkpoint range processing, process all the blocks at `height`. /// /// Returns the first valid block. If there is no valid block, returns None. @@ -1203,6 +1300,15 @@ pub enum VerifyCheckpointError { }, #[error("zebra is shutting down")] ShuttingDown, + #[error( + "authenticated checkpoint body at {height:?} has hash {found:?}, \ + but the authenticated header hash is {expected:?}" + )] + AuthenticatedHashMismatch { + height: block::Height, + expected: block::Hash, + found: block::Hash, + }, } impl From for VerifyCheckpointError { diff --git a/zebra-consensus/src/checkpoint/tests.rs b/zebra-consensus/src/checkpoint/tests.rs index 098e7a61426..4e6b06ede68 100644 --- a/zebra-consensus/src/checkpoint/tests.rs +++ b/zebra-consensus/src/checkpoint/tests.rs @@ -94,6 +94,87 @@ async fn single_item_checkpoint_list() -> Result<(), Report> { Ok(()) } +#[tokio::test(flavor = "multi_thread")] +async fn authenticated_checkpoint_release_test() -> Result<(), Report> { + let _init_guard = zebra_test::init(); + + let block0 = + Arc::::zcash_deserialize(&zebra_test::vectors::BLOCK_MAINNET_GENESIS_BYTES[..])?; + let hash0 = block0.hash(); + + let genesis_checkpoint_list: BTreeMap = + [(block0.coinbase_height().unwrap(), hash0)] + .iter() + .cloned() + .collect(); + + let state_service = zebra_state::init_test(&Mainnet).await; + let mut checkpoint_verifier = + CheckpointVerifier::from_list(genesis_checkpoint_list, &Mainnet, None, state_service) + .map_err(|e| eyre!(e))?; + + // The authenticated fast path commits the block against its authenticated hash, with no other + // height queued, and without touching the range/progress state. + let token = zebra_state::AuthenticatedCheckpointHash::new_for_test(hash0); + let response = timeout( + Duration::from_secs(VERIFY_TIMEOUT_SECONDS), + checkpoint_verifier.call_authenticated(block0.clone(), token), + ) + .await + .expect("timeout should not happen") + .expect("authenticated block should commit"); + + assert_eq!(response, hash0); + // The fast path must not enqueue anything or walk/advance the checkpoint range state. + assert!(checkpoint_verifier.queued.is_empty()); + assert_eq!( + checkpoint_verifier.previous_checkpoint_height(), + BeforeGenesis + ); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn authenticated_checkpoint_hash_mismatch_test() -> Result<(), Report> { + let _init_guard = zebra_test::init(); + + let block0 = + Arc::::zcash_deserialize(&zebra_test::vectors::BLOCK_MAINNET_GENESIS_BYTES[..])?; + let hash0 = block0.hash(); + let block1 = Arc::::zcash_deserialize(&zebra_test::vectors::BLOCK_MAINNET_1_BYTES[..])?; + let wrong_hash = block1.hash(); + + let genesis_checkpoint_list: BTreeMap = + [(block0.coinbase_height().unwrap(), hash0)] + .iter() + .cloned() + .collect(); + + let state_service = zebra_state::init_test(&Mainnet).await; + let mut checkpoint_verifier = + CheckpointVerifier::from_list(genesis_checkpoint_list, &Mainnet, None, state_service) + .map_err(|e| eyre!(e))?; + + // A body whose hash disagrees with the authenticated header is a hard invariant violation: + // there is no fallback, and nothing is released or queued. + let token = zebra_state::AuthenticatedCheckpointHash::new_for_test(wrong_hash); + let result = timeout( + Duration::from_secs(VERIFY_TIMEOUT_SECONDS), + checkpoint_verifier.call_authenticated(block0.clone(), token), + ) + .await + .expect("timeout should not happen"); + + assert!(matches!( + result, + Err(VerifyCheckpointError::AuthenticatedHashMismatch { .. }) + )); + assert!(checkpoint_verifier.queued.is_empty()); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread")] async fn multi_item_checkpoint_list_test() -> Result<(), Report> { multi_item_checkpoint_list().await diff --git a/zebra-consensus/src/router.rs b/zebra-consensus/src/router.rs index 905702de21e..498ef67f228 100644 --- a/zebra-consensus/src/router.rs +++ b/zebra-consensus/src/router.rs @@ -211,6 +211,36 @@ where .boxed(); } + // A Zakura header-authenticated checkpoint body: validate in isolation and release + // immediately, no range accumulation. Policy lives here — this is checkpoint-range only. + if let Request::CommitCheckpointAuthenticated { + block, + expected_hash, + } = request + { + let max_checkpoint_height = self.max_checkpoint_height; + return match block.coinbase_height() { + Some(height) if height <= max_checkpoint_height => self + .checkpoint + .call_authenticated(block, expected_hash) + .map_err(Into::into) + .boxed(), + Some(height) => async move { + Err(VerifyCheckpointError::TooHigh { + height, + max_height: max_checkpoint_height, + })? + } + .boxed(), + None => async move { + Err(VerifyCheckpointError::CoinbaseHeight { + hash: expected_hash.hash(), + })? + } + .boxed(), + }; + } + let block = request.block(); match block.coinbase_height() { diff --git a/zebra-network/src/zakura/block_sync/reactor.rs b/zebra-network/src/zakura/block_sync/reactor.rs index 3a9ac9bed38..14f1ea61c6e 100644 --- a/zebra-network/src/zakura/block_sync/reactor.rs +++ b/zebra-network/src/zakura/block_sync/reactor.rs @@ -1888,18 +1888,46 @@ impl BlockSyncReactor { fn publish_metrics(&self) { // These lossy casts are metrics-only gauges; consensus and scheduling // continue to use the original integer values. + let view = self.last_view; + metrics::gauge!("sync.block.best_header_tip.height") .set(self.state.best_header_tip.0 as f64); - metrics::gauge!("sync.block.verified_tip.height").set(self.verified_block_tip.0 as f64); + metrics::gauge!("sync.block.verified_tip.height").set(view.verified_tip.0 as f64); + metrics::gauge!("sync.block.download_floor.height").set(view.download_floor.0 as f64); + metrics::gauge!("sync.block.commit_gap.height") + .set(view.download_floor.0.saturating_sub(view.verified_tip.0) as f64); metrics::gauge!("sync.block.missing_bodies").set(self.state.needed_heights.len() as f64); metrics::gauge!("sync.block.budget.reserved_bytes") .set(self.state.budget.reserved() as f64); + metrics::gauge!("sync.block.reorder").set(view.reorder_len as f64); + metrics::gauge!("sync.block.reorder.buffered_blocks").set(view.reorder_len as f64); metrics::gauge!("sync.block.reorder.buffered_bytes") - .set(self.last_view.reorder_buffered_bytes as f64); - metrics::gauge!("sync.block.applying").set(self.last_view.applying_len as f64); + .set(view.reorder_buffered_bytes as f64); + metrics::gauge!("sync.block.applying").set(view.applying_len as f64); + metrics::gauge!("sync.block.unsubmitted_applying") + .set(view.unsubmitted_applying_count as f64); + metrics::gauge!("sync.block.submitted_applying").set(view.submitted_applying_count as f64); // Outstanding (unreceived in-flight) heights summed across peers from the // registry (the routines own the per-peer outstanding now). metrics::gauge!("sync.block.outstanding").set(self.registry.total_unreceived() as f64); + + if let Some(floor_gap) = self.floor_gap_diagnostics(Instant::now()) { + metrics::gauge!("sync.block.floor_gap.height").set(floor_gap.height.0 as f64); + metrics::gauge!("sync.block.floor_gap.servable_peers") + .set(floor_gap.servable_peers as f64); + metrics::gauge!("sync.block.floor_gap.available_peers") + .set(floor_gap.available_peers as f64); + metrics::gauge!("sync.block.floor_gap.outstanding_peers") + .set(floor_gap.outstanding_peers as f64); + metrics::gauge!("sync.block.commit_frontier_stall.seconds").set( + floor_gap + .oldest_outstanding_ms + .map(|age| age as f64 / 1000.0) + .unwrap_or(0.0), + ); + metrics::counter!("sync.block.floor_gap.state_ticks", "state" => floor_gap.state) + .increment(1); + } } fn clamp_served_block_count(&self, start_height: block::Height, count: u32) -> u32 { diff --git a/zebra-state/src/lib.rs b/zebra-state/src/lib.rs index 6c555a22b98..397f6b7e9b5 100644 --- a/zebra-state/src/lib.rs +++ b/zebra-state/src/lib.rs @@ -46,8 +46,8 @@ pub use error::{ CommitSemanticallyVerifiedError, DuplicateNullifierError, ValidateContextError, }; pub use request::{ - CheckpointVerifiedBlock, CommitSemanticallyVerifiedBlockRequest, HashOrHeight, MappedRequest, - ReadRequest, Request, SemanticallyVerifiedBlock, + AuthenticatedCheckpointHash, CheckpointVerifiedBlock, CommitSemanticallyVerifiedBlockRequest, + HashOrHeight, MappedRequest, ReadRequest, Request, SemanticallyVerifiedBlock, }; #[cfg(feature = "indexer")] diff --git a/zebra-state/src/request.rs b/zebra-state/src/request.rs index 66415aa6857..0a60f246776 100644 --- a/zebra-state/src/request.rs +++ b/zebra-state/src/request.rs @@ -1264,6 +1264,41 @@ impl Request { } } +/// A block hash that has been positively authenticated against the hardcoded checkpoint list +/// via the Zakura header chain. +/// +/// The inner hash is private and can only be constructed by the state (after re-establishing the +/// checkpoint anchor for the height — see +/// [`ReadRequest::AuthenticatedCheckpointHash`]). This makes it impossible to fabricate one from a +/// raw `block.hash()`: it is the provenance marker the Zakura checkpoint fast-commit path relies on +/// to prove a body's expected hash came from authenticated headers, not from the body itself. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct AuthenticatedCheckpointHash(block::Hash); + +impl AuthenticatedCheckpointHash { + /// Construct an authenticated checkpoint hash. + /// + /// Crate-internal: only the state may call this, and only after positively re-establishing the + /// checkpoint anchor that authenticates `hash`. + pub(crate) fn new(hash: block::Hash) -> Self { + Self(hash) + } + + /// The authenticated block hash. + pub fn hash(&self) -> block::Hash { + self.0 + } + + /// Construct an authenticated checkpoint hash directly, bypassing the state's authentication. + /// + /// Test-only: this exists so other crates' tests can exercise the checkpoint fast-commit path. + /// It is never compiled into production builds, preserving the provenance guarantee. + #[cfg(any(test, feature = "proptest-impl"))] + pub fn new_for_test(hash: block::Hash) -> Self { + Self(hash) + } +} + #[derive(Clone, Debug, PartialEq, Eq)] /// A read-only query about the chain state, via the /// [`ReadStateService`](crate::service::ReadStateService). @@ -1486,6 +1521,19 @@ pub enum ReadRequest { /// Returns the highest header held on disk. BestHeaderTip, + /// Returns the checkpoint-authenticated hash at `height`, if `height` is at or below the + /// checkpoint-authenticated header frontier; otherwise `None`. + /// + /// The hash is positively re-established against the hardcoded checkpoint list (the next + /// checkpoint `C >= height` is re-checked, and the persisted header store is a contiguous + /// continuity-verified frontier, so continuity pins `height`'s hash to `C`). The returned + /// [`AuthenticatedCheckpointHash`] is the provenance token the Zakura checkpoint fast-commit + /// path requires. + AuthenticatedCheckpointHash { + /// The height to authenticate. + height: block::Height, + }, + /// Returns header-known, body-missing heights in `(verified_block_tip, best_header_tip]`. MissingBlockBodies { /// First height to consider. @@ -1683,6 +1731,7 @@ impl ReadRequest { ReadRequest::FindBlockHeaders { .. } => "find_block_headers", ReadRequest::HeadersByHeightRange { .. } => "headers_by_height_range", ReadRequest::BestHeaderTip => "best_header_tip", + ReadRequest::AuthenticatedCheckpointHash { .. } => "authenticated_checkpoint_hash", ReadRequest::MissingBlockBodies { .. } => "missing_block_bodies", ReadRequest::BlockSizeHints { .. } => "block_size_hints", ReadRequest::BlocksByHeightRange { .. } => "blocks_by_height_range", diff --git a/zebra-state/src/response.rs b/zebra-state/src/response.rs index 5b8316c445a..6085617037d 100644 --- a/zebra-state/src/response.rs +++ b/zebra-state/src/response.rs @@ -395,6 +395,9 @@ pub enum ReadResponse { /// Response to [`ReadRequest::BestHeaderTip`]. BestHeaderTip(Option<(block::Height, block::Hash)>), + /// Response to [`ReadRequest::AuthenticatedCheckpointHash`]. + AuthenticatedCheckpointHash(Option), + /// Response to [`ReadRequest::MissingBlockBodies`]. MissingBlockBodies(Vec), @@ -580,6 +583,7 @@ impl TryFrom for Response { | ReadResponse::ChainInfo(_) | ReadResponse::Headers(_) | ReadResponse::BestHeaderTip(_) + | ReadResponse::AuthenticatedCheckpointHash(_) | ReadResponse::MissingBlockBodies(_) | ReadResponse::BlockSizeHints(_) | ReadResponse::Blocks(_) diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs index d86c6d2376d..72332c952a4 100644 --- a/zebra-state/src/service.rs +++ b/zebra-state/src/service.rs @@ -1883,6 +1883,12 @@ impl Service for ReadStateService { ))) } + ReadRequest::AuthenticatedCheckpointHash { height } => { + Ok(ReadResponse::AuthenticatedCheckpointHash( + state.db.authenticated_checkpoint_hash(height), + )) + } + ReadRequest::MissingBlockBodies { from, limit } => { let verified_block_tip = read::tip_height(state.latest_best_chain(), &state.db); let best_header_tip = state diff --git a/zebra-state/src/service/finalized_state.rs b/zebra-state/src/service/finalized_state.rs index 8c8da85088e..459fb46b59f 100644 --- a/zebra-state/src/service/finalized_state.rs +++ b/zebra-state/src/service/finalized_state.rs @@ -1120,16 +1120,20 @@ impl FinalizedState { let history_tree_mut = Arc::make_mut(&mut history_tree); let sapling_root = note_commitment_trees.sapling.root(); let orchard_root = note_commitment_trees.orchard.root(); - history_tree_mut - .push( - &network, - block.clone(), - &sapling_root, - &orchard_root, - &Default::default(), - ) - .map_err(Arc::new) - .map_err(ValidateContextError::from)?; + let ironwood_root = Default::default(); + timed_commit_phase!( + "zebra.state.commit.history_push.duration_seconds", + history_tree_mut + .push( + &network, + block.clone(), + &sapling_root, + &orchard_root, + &ironwood_root, + ) + .map_err(Arc::new) + .map_err(ValidateContextError::from) + )?; #[cfg(feature = "commit-metrics")] metrics::histogram!("zebra.state.write.checkpoint_compute.duration_seconds") diff --git a/zebra-state/src/service/finalized_state/disk_db.rs b/zebra-state/src/service/finalized_state/disk_db.rs index d425fab2530..49261aa4e93 100644 --- a/zebra-state/src/service/finalized_state/disk_db.rs +++ b/zebra-state/src/service/finalized_state/disk_db.rs @@ -124,6 +124,14 @@ pub struct DiskWriteBatch { batch: rocksdb::WriteBatch, } +#[cfg(feature = "commit-metrics")] +impl DiskWriteBatch { + /// Returns the serialized size of this pending RocksDB write batch. + pub(crate) fn size_in_bytes(&self) -> usize { + self.batch.size_in_bytes() + } +} + impl Debug for DiskWriteBatch { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("DiskWriteBatch") diff --git a/zebra-state/src/service/finalized_state/zebra_db/block.rs b/zebra-state/src/service/finalized_state/zebra_db/block.rs index 75598a299c1..ddf691f160c 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block.rs @@ -37,7 +37,7 @@ use crate::{ MAX_BLOCK_REORG_HEIGHT, MAX_HEADER_SYNC_HEIGHT_RANGE, MAX_PRUNE_HEIGHTS_PER_COMMIT, }, error::{CommitCheckpointVerifiedError, CommitHeaderRangeError}, - request::FinalizedBlock, + request::{AuthenticatedCheckpointHash, FinalizedBlock}, service::check, service::finalized_state::{ disk_db::{DiskDb, DiskWriteBatch, ReadDisk, WriteDisk}, @@ -403,6 +403,83 @@ impl ZebraDb { } } + /// Returns the hash at `height` from a committed body, falling back to a persisted Zakura + /// header-only row. `None` if neither is present. + fn header_hash_at(&self, height: block::Height) -> Option { + self.hash(height) + .or_else(|| self.zakura_header_hash(height)) + } + + /// Returns the highest checkpoint height whose header hash is positively re-established against + /// the hardcoded checkpoint list. Every height at or below this is checkpoint-authenticated. + /// + /// Used to bound checkpoint-class body scheduling (airtight ordering): the Zakura checkpoint + /// fast-commit path has no fallback, so a checkpoint-class body must never be requested before + /// its height is authenticated here. + pub fn authenticated_header_tip(&self) -> Option { + let checkpoints = self.network().checkpoint_list(); + let persisted_tip = self.best_header_tip().map(|(height, _)| height)?; + + // Walk checkpoints down from the highest at/below the persisted tip until one is positively + // authenticated. Normally the first matches; we do not trust persistence to have checked it + // (see `authenticated_checkpoint_hash`). The body tip's enclosing checkpoint is always + // authenticated because it was committed from a verified block, so this terminates. + let mut candidate = checkpoints.max_height_in_range(..=persisted_tip); + while let Some(c) = candidate { + if self.header_hash_at(c) == checkpoints.hash(c) { + return Some(c); + } + candidate = c + .previous() + .ok() + .and_then(|below| checkpoints.max_height_in_range(..=below)); + } + + None + } + + /// Returns the Zakura-authenticated header hash at `height`, if `height` is a header-only + /// frontier height (above the committed body tip) whose enclosing checkpoint has been reached + /// and authenticated; otherwise `None`. + /// + /// The hash is taken **strictly from the Zakura header store**, never from committed body state, + /// so the token means exactly "the checkpoint-authenticated Zakura header hash for `height`". + /// Authentication is established positively at the gate: for the next checkpoint `C >= height`, + /// the persisted Zakura header hash at `C` must equal the hardcoded `checkpoint_list.hash(C)`. + /// + /// Continuity is sound because `height` and `C` are both in the un-trimmed frontier above the + /// body tip (the Zakura store releases a row only once its body commits), so every height in + /// `[height, C]` is also in the frontier; header sync persists only anchor-linked contiguous + /// ranges, so continuity from `height` up to the pinned `C` authenticates `height`'s hash. An + /// already-committed height returns `None` here (its Zakura row has been released), so the fast + /// path never re-commits or authenticates from body state. + /// + /// This is the only constructor of [`AuthenticatedCheckpointHash`]. + pub fn authenticated_checkpoint_hash( + &self, + height: block::Height, + ) -> Option { + let checkpoints = self.network().checkpoint_list(); + + // The next checkpoint at or above `height`. `None` => above the last checkpoint, i.e. not a + // checkpoint-anchored height (the caller must not fast-commit it). + let next_checkpoint = checkpoints.min_height_in_range(height..)?; + + // Construct strictly from the Zakura header frontier: `height` must be a header-only frontier + // height. (`None` for an already-committed height, so we never authenticate from body + // state.) + let header_hash = self.zakura_header_hash(height)?; + + // Positively re-check the checkpoint anchor against the Zakura frontier. If the frontier has + // not reached `C`, or the persisted hash there does not match the hardcoded checkpoint, + // `height` is not yet authenticated. + if self.zakura_header_hash(next_checkpoint) != checkpoints.hash(next_checkpoint) { + return None; + } + + Some(AuthenticatedCheckpointHash::new(header_hash)) + } + /// Returns a contiguous ascending header range from full blocks and Zakura header rows. pub fn headers_by_height_range( &self, @@ -482,10 +559,21 @@ impl ZebraDb { let count = limit.min(best_header_tip.0.saturating_sub(start.0).saturating_add(1)); + // Airtight ordering: never schedule a checkpoint-class body (`height <= checkpoint_max`) + // before header sync has checkpoint-authenticated that height. The Zakura checkpoint + // fast-commit path has no fallback, so requesting such a body early would later trip a hard + // invariant. Full-class heights (above the last checkpoint) are unaffected. + let checkpoint_max = self.network().checkpoint_list().max_height(); + let authenticated_header_tip = self.authenticated_header_tip(); + self.headers_by_height_range(start, count) .into_iter() .map(|(height, _, _)| height) .filter(|height| !self.contains_body_at_height(*height)) + .filter(|height| { + *height > checkpoint_max + || authenticated_header_tip.is_some_and(|tip| *height <= tip) + }) .take(limit as usize) .collect() } @@ -985,6 +1073,8 @@ impl ZebraDb { // `None` it serializes inline (e.g. the semantic path). let store_raw_txs = retention.stores_raw_transactions(); let db: &ZebraDb = self; + #[cfg(feature = "commit-metrics")] + let spent_reads_start = std::time::Instant::now(); let (spent_utxos, precomputed_raw_txs): ( Vec<(transparent::OutPoint, OutputLocation, transparent::Utxo)>, Option>, @@ -1035,6 +1125,9 @@ impl ZebraDb { } }, ); + #[cfg(feature = "commit-metrics")] + metrics::histogram!("zebra.state.write.spent_utxo_reads.duration_seconds") + .record(spent_reads_start.elapsed().as_secs_f64()); let spent_utxos_by_outpoint: HashMap = spent_utxos @@ -1101,6 +1194,8 @@ impl ZebraDb { // reading all of the pending merge operands (potentially hundreds), and applying pending merge operands to the // fully-merged value such that it's much faster to read entries that have been updated with insertions than it // is to read entries that have been updated with merge operations. + #[cfg(feature = "commit-metrics")] + let address_reads_start = std::time::Instant::now(); let address_balances: AddressBalanceLocationUpdates = if self.finished_format_upgrades() { AddressBalanceLocationUpdates::Insert(read_addr_locs(changed_addresses, |addr| { self.address_balance_location(addr) @@ -1110,10 +1205,15 @@ impl ZebraDb { Some(self.address_balance_location(addr)?.into_new_change()) })) }; + #[cfg(feature = "commit-metrics")] + metrics::histogram!("zebra.state.write.address_reads.duration_seconds") + .record(address_reads_start.elapsed().as_secs_f64()); let mut batch = DiskWriteBatch::new(); // In case of errors, propagate and do not write the batch. + #[cfg(feature = "commit-metrics")] + let batch_prep_start = std::time::Instant::now(); batch.prepare_block_batch( self, network, @@ -1139,6 +1239,13 @@ impl ZebraDb { // block commit path. In archive mode the plan is always `Store`, so this // is a no-op. retention.prepare_prune(&mut batch, self, &finalized); + #[cfg(feature = "commit-metrics")] + { + metrics::histogram!("zebra.state.write.batch_prep.duration_seconds") + .record(batch_prep_start.elapsed().as_secs_f64()); + metrics::histogram!("zebra.state.write.batch_bytes") + .record(batch.size_in_bytes() as f64); + } // Track batch commit latency for observability let batch_start = std::time::Instant::now(); diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs index 2c412f9488e..1cd9fb4d6d0 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs @@ -499,6 +499,30 @@ fn committed_block_releases_matching_zakura_header() { .is_none()); } +#[test] +fn authenticated_checkpoint_hash_requires_zakura_frontier_row() { + let _init_guard = zebra_test::init(); + let genesis = mainnet_block(0); + let block1 = mainnet_block(1); + let network = checkpoint_test_network(genesis.hash(), block1.hash()); + let state = state_with_genesis(&network, genesis.clone()); + + commit_header_range(&state, genesis.hash(), std::slice::from_ref(&block1.header)); + + let authenticated_hash = state + .authenticated_checkpoint_hash(Height(1)) + .expect("checkpoint-authenticated header-only frontier hash is available"); + assert_eq!(authenticated_hash.hash(), block1.hash()); + + write_full_block_header_and_transactions(&state, block1.clone()); + + assert_eq!( + state.headers_by_height_range(Height(1), 1), + vec![(Height(1), block1.hash(), block1.header.clone())], + ); + assert!(state.authenticated_checkpoint_hash(Height(1)).is_none()); +} + /// Committing a body at the bottom of a header-only frontier releases just that /// height from the Zakura header store while the higher frontier rows remain, /// and reads span the body/frontier boundary. diff --git a/zebrad/src/commands/start/zakura/block_sync_driver.rs b/zebrad/src/commands/start/zakura/block_sync_driver.rs index 1c8c405725d..85171855b7f 100644 --- a/zebrad/src/commands/start/zakura/block_sync_driver.rs +++ b/zebrad/src/commands/start/zakura/block_sync_driver.rs @@ -946,19 +946,52 @@ where // normal mode the frontier comes from re-reading committed state below. let (result, probe_frontier) = match throughput_probe.as_ref() { Some(probe) => probe.apply_block(block.as_ref()), - None => ( - commit_block_sync_body_with_stall_trace( - block_verifier.clone(), - block, - class, - &trace, - token, - height, - expected_hash, - ) - .await, - None, - ), + None => { + // For a checkpoint-class body, obtain the checkpoint-authenticated expected hash. This + // also enforces the airtight-ordering invariant: the body must not reach the verifier + // before header sync has authenticated its height (the needed-bodies gate ensures it). + let checkpoint_auth_hash = if class == BlockApplyClass::Checkpoint { + query_authenticated_checkpoint_hash(read_state.clone(), height).await + } else { + None + }; + + if class == BlockApplyClass::Checkpoint && checkpoint_auth_hash.is_none() { + // Hard sync-invariant violation: there is no fallback into range accumulation. We + // refuse the commit and surface it; airtight ordering means this should not happen. + warn!( + ?height, + ?expected_hash, + "Zakura checkpoint body is not header-authenticated; refusing fast commit" + ); + emit_commit_state( + &trace, + "checkpoint_auth_invariant", + "block_sync_driver", + |row| { + insert_cs_u64(row, cs_trace::APPLY_TOKEN, token); + insert_cs_height(row, cs_trace::HEIGHT, height); + insert_cs_hash(row, cs_trace::HASH, expected_hash); + }, + ); + (BlockApplyResult::Rejected, None) + } else { + ( + commit_block_sync_body_with_stall_trace( + block_verifier.clone(), + block, + class, + &trace, + token, + height, + expected_hash, + checkpoint_auth_hash, + ) + .await, + None, + ) + } + } }; emit_commit_state( &trace, @@ -971,6 +1004,9 @@ where insert_cs_hash(row, cs_trace::HASH, expected_hash); insert_cs_str(row, cs_trace::RESULT, block_apply_result_label(result)); insert_cs_u64(row, cs_trace::ELAPSED_MS, elapsed_ms(started)); + if class == BlockApplyClass::Checkpoint && result == BlockApplyResult::Committed { + insert_cs_str(row, "checkpoint_commit_path", "zakura_authenticated"); + } }, ); emit_commit_state( @@ -1083,6 +1119,7 @@ async fn commit_block_sync_body_with_stall_trace( token: BlockApplyToken, height: block::Height, expected_hash: block::Hash, + checkpoint_auth_hash: Option, ) -> BlockApplyResult where BlockVerifier: @@ -1090,9 +1127,16 @@ where BlockVerifier::Error: std::fmt::Debug + Send + Sync + 'static, BlockVerifier::Future: Send + 'static, { - let commit = block_verifier - .clone() - .oneshot(zebra_consensus::Request::Commit(block)); + // Checkpoint-class bodies use the header-authenticated fast path (per-block validate + release, + // no range accumulation). Full-class bodies take the normal semantic commit. + let request = match checkpoint_auth_hash { + Some(expected_hash) => zebra_consensus::Request::CommitCheckpointAuthenticated { + block, + expected_hash, + }, + None => zebra_consensus::Request::Commit(block), + }; + let commit = block_verifier.clone().oneshot(request); match class { BlockApplyClass::Checkpoint => { @@ -1189,6 +1233,41 @@ fn block_commit_timed_out( BlockApplyResult::TimedOut } +/// Reads the checkpoint-authenticated hash for `height`, if header sync has authenticated it. +/// +/// Returns the provenance token required by the Zakura checkpoint fast-commit path, or `None` if +/// the height is not (yet) checkpoint-authenticated or the read fails. +async fn query_authenticated_checkpoint_hash( + read_state: ReadState, + height: block::Height, +) -> Option +where + ReadState: Service< + zebra_state::ReadRequest, + Response = zebra_state::ReadResponse, + Error = zebra_state::BoxError, + > + Clone + + Send + + 'static, + ReadState::Future: Send + 'static, +{ + match read_state + .oneshot(zebra_state::ReadRequest::AuthenticatedCheckpointHash { height }) + .await + { + Ok(zebra_state::ReadResponse::AuthenticatedCheckpointHash(token)) => token, + Ok(_) => None, + Err(error) => { + warn!( + ?height, + ?error, + "failed to read authenticated checkpoint hash" + ); + None + } + } +} + async fn refresh_block_sync_frontiers_for_checkpoint_window( read_state: ReadState, latest_chain_tip: impl ChainTip + Clone + Send + Sync + 'static, From 0a8dede776db68fd71f8f819007cdbb855235bdc Mon Sep 17 00:00:00 2001 From: Roman Akhtariev Date: Mon, 29 Jun 2026 02:58:20 -0300 Subject: [PATCH 02/17] perf(state): run-ahead finalized-commit pipeline (default-off) (#310) Overlap the next finalized block's batch assembly with the current block's disk write, raising checkpoint-sync commit throughput from `assemble + flush` per block toward `max(assemble, flush)`. - Split the committer into assemble (`assemble_finalized_direct` / `assemble_block_batch`, no writes) and flush (`flush_finalized_direct` / `flush_block_batch`) halves. - Add `FinalizedPipeline`: in-memory tip state (history tree, note trees, value pool, vct_upgrade marker, tip cursor) + a read-through overlay for not-yet- flushed spent UTXOs and address balances, so a block assembled ahead of the durable write reads its parent's effects from memory. - Wire a dedicated disk-writer thread fed by a bounded channel (depth = backpressure); the finalized tip, commit response, and download budget all trail the flush (ack-after-flush), so crash recovery is unchanged. - Gate behind `Config::finalized_block_pipeline_depth` (default 0 = synchronous, byte-identical to the previous committer). Only the checkpoint (reorg-free) region uses the pipeline. --- CHANGELOG.md | 20 + CHANGELOG_PARAMS.md | 1 + zebra-state/src/config.rs | 20 + zebra-state/src/service/finalized_state.rs | 348 +++++++++++++++--- .../disk_format/transparent.rs | 2 + .../src/service/finalized_state/pipeline.rs | 284 ++++++++++++++ .../src/service/finalized_state/tests/prop.rs | 92 ++++- .../finalized_state/tests/transparent.rs | 1 + .../service/finalized_state/zebra_db/block.rs | 229 +++++++++--- .../service/finalized_state/zebra_db/chain.rs | 6 +- .../finalized_state/zebra_db/shielded.rs | 7 +- .../finalized_state/zebra_db/transparent.rs | 13 +- zebra-state/src/service/write.rs | 255 ++++++++++++- 13 files changed, 1174 insertions(+), 104 deletions(-) create mode 100644 zebra-state/src/service/finalized_state/pipeline.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index cb5e1a77b0f..8699a0c55d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,26 @@ and this project adheres to [Semantic Versioning](https://semver.org). `librustzcash`. The output is byte-identical: a differential property test (`native_zip244_matches_librustzcash`) asserts the native txid and auth digest match the `librustzcash` conversion across thousands of random v5 transactions. +- Add a run-ahead finalized-commit pipeline that overlaps the next block's batch + assembly with the current block's disk write, raising checkpoint-sync commit + throughput from `assemble + flush` per block toward `max(assemble, flush)`. The + committer is split into an assemble half (`assemble_finalized_direct` / + `assemble_block_batch` — treestate compute + batch build, no writes) and a flush + half (`flush_finalized_direct` / `flush_block_batch` — the rocksdb write and + post-commit bookkeeping). When enabled, the committer assembles each block's + batch on its thread and hands it to a dedicated disk-writer thread over a bounded + channel (the channel depth is the backpressure). A new `FinalizedPipeline` (in + `zebra-state/src/service/finalized_state/pipeline.rs`) carries the in-memory tip + state — history tree, note-commitment trees, value pool, `vct_upgrade_height` + marker, tip cursor — and a read-through overlay for not-yet-flushed spent UTXOs + and address balances, so a block assembled ahead of the durable write reads its + parent's effects from memory. The externally-visible finalized tip, the per-block + commit response, and the download budget all trail the durable flush + (ack-after-flush), so crash recovery is unchanged. Only the checkpoint + (reorg-free) region uses the pipeline; near the tip the committer stays + synchronous. The new `Config::finalized_block_pipeline_depth` defaults to `0` + (synchronous), so behavior is byte-identical to the previous committer until the + depth is raised. See `CHANGELOG_PARAMS.md`. - Parallelize per-block serialization in the finalized block writer. On heavy shielded blocks, serializing the raw transaction bytes (`tx_by_loc`) and computing the block size for `BlockInfo` dominate the per-block write cost. Both diff --git a/CHANGELOG_PARAMS.md b/CHANGELOG_PARAMS.md index 6ccbcca61f7..c7695aa028a 100644 --- a/CHANGELOG_PARAMS.md +++ b/CHANGELOG_PARAMS.md @@ -28,4 +28,5 @@ Keep entries **newest-first**. Each row records: | Parameter | Location | Old → New | PR | Why | | --- | --- | --- | --- | --- | +| `Config::finalized_block_pipeline_depth` | `zebra-state/src/config.rs` | _(new)_ → `0` | _(this PR)_ | Run-ahead finalized-commit pipeline depth (blocks the assembler may build ahead of the durable disk write). Defaults to `0` = synchronous (original behavior); `> 0` overlaps the next block's assembly with the current block's flush, bounded so the in-memory overlay stays small. | | `OUTBOUND_WINDOW_FLOOR_TIMEOUTS_BEFORE_DISCONNECT` | `zebra-network/src/zakura/block_sync/state.rs` | `3` → `2 * OUTBOUND_WINDOW_REDUCTION_EPOCH_TIMEOUTS` (`32`) | [#303](https://github.com/valargroup/zebra/pull/303) | Tolerate two full reduction epochs (~256s at the 8s request timeout) of floor-pinned timeouts before disconnecting a block-sync peer, instead of ~24s, so briefly-congested peers are not churned. Any successful response resets the streak. | diff --git a/zebra-state/src/config.rs b/zebra-state/src/config.rs index 9cf5a4a0a80..0b7c241d181 100644 --- a/zebra-state/src/config.rs +++ b/zebra-state/src/config.rs @@ -136,6 +136,23 @@ pub struct Config { #[serde(skip)] pub vct_fast_sync: bool, + /// Maximum number of checkpoint-verified blocks the finalized committer may + /// assemble ahead of the durable disk write (the run-ahead pipeline depth). + /// + /// When `0` (the default), the committer is fully synchronous: each block is + /// assembled and flushed to disk before the next is assembled — the original + /// behavior. When greater than `0`, the committer assembles up to this many + /// blocks ahead on the assembler thread while a dedicated disk-writer thread + /// flushes earlier blocks, so per-block wall time approaches `max(assemble, + /// flush)` instead of their sum. The externally-visible finalized tip and the + /// download budget still trail the durable flush (ack-after-flush), so crash + /// recovery is unchanged. + /// + /// Only applies below the last checkpoint (the reorg-free region); near the + /// tip the committer is always synchronous. The bound caps the in-memory + /// overlay of not-yet-flushed blocks. + pub finalized_block_pipeline_depth: usize, + /// Whether to delete the old database directories when present. /// /// Set to `true` by default. If this is set to `false`, @@ -430,6 +447,9 @@ impl Default for Config { enable_zakura_header_seed_from_committed_blocks: false, checkpoint_sync: true, vct_fast_sync: true, + // Off by default: the committer is synchronous (assemble then flush) + // until run-ahead is explicitly enabled and validated. + finalized_block_pipeline_depth: 0, delete_old_database: true, storage_mode: StorageMode::default(), debug_stop_at_height: None, diff --git a/zebra-state/src/service/finalized_state.rs b/zebra-state/src/service/finalized_state.rs index 459fb46b59f..01f2d2fba7d 100644 --- a/zebra-state/src/service/finalized_state.rs +++ b/zebra-state/src/service/finalized_state.rs @@ -116,17 +116,71 @@ pub(crate) fn spawn_note_precompute( (rx, cancel) } +/// A fully-assembled finalized-block commit that has not yet been flushed to disk. +/// +/// Produced by [`FinalizedState::assemble_finalized_direct`] (the read/compute half +/// of a block commit) and consumed by [`FinalizedState::flush_finalized_direct`] (the +/// disk-write half). Carrying the batch and its post-commit bookkeeping inputs lets +/// the run-ahead committer hand the batch to a separate disk-writer thread while it +/// assembles the next block. +pub struct AssembledCommit { + /// The fully-prepared write batch, ready to flush. + batch: DiskWriteBatch, + /// The committed block's hash. + hash: block::Hash, + /// The committed block's height. + height: block::Height, + /// The note-commitment trees after this block, threaded to the next block. + note_commitment_trees: NoteCommitmentTrees, + /// The retention plan, needed for post-flush archive-backlog bookkeeping. + retention: RetentionPlan, + /// This block's run-ahead pipeline contribution, captured only when the + /// committer is running ahead (`None` on the synchronous path). The committer + /// records it into the [`FinalizedPipeline`](pipeline::FinalizedPipeline) + /// before handing the batch to the disk-writer thread. + /// + /// Read by the disk-writer-thread integration in `write.rs`; until that lands, + /// it is only ever the `None` produced by the synchronous path. + #[allow(dead_code)] + pipeline_contribution: Option, + /// The committed block, for elasticsearch indexing after the flush. + #[cfg(feature = "elasticsearch")] + finalized_block: Arc, +} + +/// The subset of a run-ahead [`AssembledCommit`] the disk-writer thread and the +/// committer's ack handling need, returned by +/// [`FinalizedState::commit_finalized_pipelined`]. +/// +/// The overlay-specific fields of `AssembledCommit` have already been folded into +/// the pipeline by the time this is returned, so this carries only the batch to +/// flush and the scalars used to advance the chain tip and thread the trees. +pub(crate) struct PipelineAssembled { + /// The fully-prepared write batch, handed to the disk-writer thread. + pub batch: DiskWriteBatch, + /// The committed block's hash (sent on the commit response after the flush). + pub hash: block::Hash, + /// The committed block's height (acked by the writer to advance the tip). + pub height: block::Height, + /// The note-commitment trees after this block, threaded to the next block. + pub note_commitment_trees: NoteCommitmentTrees, +} + pub mod column_family; mod commitment_aux; mod commitment_aux_verify; mod disk_db; mod disk_format; +mod pipeline; mod vct; mod zebra_db; use vct::VctState; +use pipeline::BlockPipelineContribution; +pub(crate) use pipeline::FinalizedPipeline; + /// The verified-commitment-trees `tree_aux` serving read path (design §9): the per-block /// commitment roots for a height range, derived from the per-height trees. pub(crate) use commitment_aux::serve_block_roots; @@ -824,6 +878,68 @@ impl FinalizedState { } } + /// Assemble a checkpoint-verified block for the run-ahead pipeline: build its + /// write batch against the in-memory `pipeline` overlay, record the block's + /// contribution into the overlay, run the committer-owned post-assemble + /// bookkeeping, and return the batch (plus the scalars the disk-writer thread + /// needs) without writing anything to disk. + /// + /// The caller hands the returned batch to the disk-writer thread, and advances + /// the chain tip and retires the overlay once that thread acknowledges the + /// durable flush (ack-after-flush). + /// + /// # Errors + /// + /// Propagates the same assembly errors as + /// [`assemble_finalized_direct`](Self::assemble_finalized_direct) (including the + /// retryable verified-commitment-trees stalls), without mutating the overlay. + pub(crate) fn commit_finalized_pipelined( + &mut self, + finalizable_block: FinalizableBlock, + prev_note_commitment_trees: Option, + note_precompute: Option, + next_checkpoint: Option<(Arc, Option)>, + pipeline: &mut pipeline::FinalizedPipeline, + ) -> Result { + pipeline.seed(&self.db); + + let assembled = self.assemble_finalized_direct( + finalizable_block, + prev_note_commitment_trees, + note_precompute, + next_checkpoint, + Some(pipeline), + "commit checkpoint-verified request", + )?; + + let AssembledCommit { + batch, + hash, + height, + note_commitment_trees, + retention, + pipeline_contribution, + // Elasticsearch indexing is skipped on the run-ahead path. + .. + } = assembled; + + let contribution = + pipeline_contribution.expect("overlay was supplied, so a contribution is captured"); + pipeline.record_block(contribution); + + // Evict committed roots and clear the archive backlog now: a block that has + // been assembled into a pipeline batch is guaranteed to flush in order (the + // checkpoint region is reorg-free), so this need not wait for the flush. + self.pipeline_post_assemble(height, retention); + + Ok(PipelineAssembled { + batch, + hash, + height, + note_commitment_trees, + }) + } + /// Immediately commit a `finalized` block to the finalized state. /// /// This can be called either by the non-finalized state (when finalizing @@ -831,6 +947,11 @@ impl FinalizedState { /// /// Use `source` as the source of the block in log messages. /// + /// This is the synchronous (tip-mode) entry point: it + /// [assembles](Self::assemble_finalized_direct) the block's batch and + /// [flushes](Self::flush_finalized_direct) it to disk in one call. The + /// run-ahead committer calls those two halves on separate threads instead. + /// /// # Errors /// /// - Propagates any errors from writing to the DB @@ -839,6 +960,42 @@ impl FinalizedState { /// does not match the expected value #[allow(clippy::unwrap_in_result)] pub fn commit_finalized_direct( + &mut self, + finalizable_block: FinalizableBlock, + prev_note_commitment_trees: Option, + note_precompute: Option, + next_checkpoint: Option<(Arc, Option)>, + source: &str, + ) -> Result<(block::Hash, NoteCommitmentTrees), CommitCheckpointVerifiedError> { + let assembled = self.assemble_finalized_direct( + finalizable_block, + prev_note_commitment_trees, + note_precompute, + next_checkpoint, + None, + source, + )?; + Ok(self.flush_finalized_direct(assembled, source)) + } + + /// Assemble (but do not flush) the [`DiskWriteBatch`] for `finalizable_block`. + /// + /// This is the read/compute half of [`commit_finalized_direct`](Self::commit_finalized_direct): + /// it updates the note-commitment and history trees, runs the verified-commitment-trees + /// fast path or legacy recompute, checks the ordering asserts, and builds the + /// batch — but performs no disk writes. The returned [`AssembledCommit`] is + /// handed to [`flush_finalized_direct`](Self::flush_finalized_direct). + /// + /// Splitting assembly from the flush lets the run-ahead committer build the + /// next block's batch while the disk-writer thread flushes this one. + /// + /// # Errors + /// + /// - Propagates any errors from updating history and note commitment trees + /// - If `hashFinalSaplingRoot` / `hashLightClientRoot` / `hashBlockCommitments` + /// does not match the expected value + #[allow(clippy::unwrap_in_result)] + pub(crate) fn assemble_finalized_direct( &mut self, finalizable_block: FinalizableBlock, prev_note_commitment_trees: Option, @@ -849,8 +1006,12 @@ impl FinalizedState { // handoff, where the embedded final frontiers independently authenticate // this height's roots, or outside the checkpoint commit path. next_checkpoint: Option<(Arc, Option)>, + // The run-ahead pipeline's in-memory tip state and overlay. When `Some`, the + // parent history tree and ordering-assert tip cursor are read from it, and + // the block's contribution is captured into the returned `AssembledCommit`. + overlay: Option<&pipeline::FinalizedPipeline>, source: &str, - ) -> Result<(block::Hash, NoteCommitmentTrees), CommitCheckpointVerifiedError> { + ) -> Result { let ( height, hash, @@ -874,8 +1035,13 @@ impl FinalizedState { // so the commitment check below doesn't recompute it here on the // single-threaded committer. `AuthDataRoot` is `Copy`. let precomputed_auth_data_root = checkpoint_verified.auth_data_root; - let mut history_tree = self.db.history_tree(); + // The parent history tree is read fresh from disk per block, so when + // running ahead it must come from the in-memory pipeline tip instead. + let mut history_tree = overlay + .map(|overlay| overlay.history_tree()) + .unwrap_or_else(|| self.db.history_tree()); let prev_note_commitment_trees = prev_note_commitment_trees + .or_else(|| overlay.map(|overlay| overlay.note_commitment_trees())) .unwrap_or_else(|| self.db.note_commitment_trees_for_tip()); let mut note_commitment_trees = prev_note_commitment_trees.clone(); @@ -1175,11 +1341,21 @@ impl FinalizedState { } }; - let committed_tip_hash = self.db.finalized_tip_hash(); - let committed_tip_height = self.db.finalized_tip_height(); + // The finalized tip is read fresh from disk per block for the ordering + // asserts, so when running ahead it must come from the in-memory pipeline + // tip (which is seeded from disk and equals it when nothing is in flight). + let (committed_tip_hash, committed_tip_height, db_is_empty) = + match overlay.and_then(|overlay| overlay.tip()) { + Some((tip_height, tip_hash)) => (tip_hash, Some(tip_height), false), + None => ( + self.db.finalized_tip_hash(), + self.db.finalized_tip_height(), + self.db.is_empty(), + ), + }; // Assert that callers (including unit tests) get the chain order correct - if self.db.is_empty() { + if db_is_empty { assert_eq!( committed_tip_hash, finalized.block.header.previous_block_hash, "the first block added to an empty state must be a genesis block, source: {source}", @@ -1205,8 +1381,11 @@ impl FinalizedState { #[cfg(feature = "elasticsearch")] let finalized_inner_block = finalized.block.clone(); let note_commitment_trees = finalized.treestate.note_commitment_trees.clone(); + // Captured for the run-ahead pipeline's in-memory tip state (the history + // tree is otherwise read fresh from disk per block); cheap `Arc` clone. + let history_tree = finalized.treestate.history_tree.clone(); - // Run `write_block` directly on the committer thread rather than entering the + // Assemble the write batch on the committer thread rather than entering the // dedicated commit-compute pool via `install()`. // // The committer is not a member of `COMMIT_COMPUTE_POOL`, so `install()` is a @@ -1214,59 +1393,142 @@ impl FinalizedState { // picks up the job, runs it, and signals back. The look-ahead note-commitment // precompute (`spawn_note_precompute`) keeps those workers busy, so the handoff // waits on a contended pool, and that wait dominates the isolation it was meant - // to provide for `write_block`'s internal rayon (`join`/`par_iter`). Running - // `write_block` here removes the per-block round-trip; its internal rayon uses + // to provide for the batch's internal rayon (`join`/`par_iter`). Running + // assembly here removes the per-block round-trip; its internal rayon uses // the global pool instead. Measured net win on the sandblast region (see PR). let network = self.network(); - let result = self.db.write_block( + // `assemble_block_batch` also returns `finalized.hash`, which equals the + // `hash` already destructured above; keep the outer binding and ignore it. + let (batch, _hash, batch_contribution) = self.db.assemble_block_batch( finalized, prev_note_commitment_trees, &network, - source, retention, fast_anchor_roots, fast_sync_below, - ); - - if result.is_ok() { - if let Some(vct) = &self.vct { - vct.evict_committed_roots_through(height); - } + overlay, + )?; + + // When running ahead, fold this block's batch contribution together with its + // tip cursor and trees into the contribution the committer records into the + // pipeline before handing the batch to the disk-writer thread. + let pipeline_contribution = + batch_contribution.map(|contribution| BlockPipelineContribution { + tip: (height, hash), + history_tree, + note_commitment_trees: note_commitment_trees.clone(), + value_pool: contribution.value_pool, + wrote_vct_upgrade_marker: contribution.wrote_vct_upgrade_marker, + created_outputs: contribution.created_outputs, + updated_balances: contribution.updated_balances, + }); - if retention.clears_archive_backlog() { - self.checkpoint_raw_tx_archive_backlog - .store(false, Ordering::Relaxed); - } + Ok(AssembledCommit { + batch, + hash, + height, + note_commitment_trees, + retention, + pipeline_contribution, + #[cfg(feature = "elasticsearch")] + finalized_block: finalized_inner_block, + }) + } - // Save blocks to elasticsearch if the feature is enabled. + /// Flush a previously [`assemble_finalized_direct`](Self::assemble_finalized_direct)d + /// batch to disk and run the post-commit bookkeeping (root eviction, archive + /// backlog clearing, elasticsearch, and the configured stop-height check). + /// + /// This is the write half of [`commit_finalized_direct`](Self::commit_finalized_direct). + /// In run-ahead (sync) mode the disk write runs on the disk-writer thread; the + /// bookkeeping that mutates committer-owned state runs back on the assembler. + /// A rocksdb write failure is fatal, as before. + pub fn flush_finalized_direct( + &mut self, + assembled: AssembledCommit, + source: &str, + ) -> (block::Hash, NoteCommitmentTrees) { + let AssembledCommit { + batch, + hash, + height, + note_commitment_trees, + retention, + // Recorded into the pipeline by the committer before the flush, so the + // disk-writer side ignores it here. + pipeline_contribution: _, #[cfg(feature = "elasticsearch")] - self.elasticsearch(&finalized_inner_block); + finalized_block, + } = assembled; - // TODO: move the stop height check to the syncer (#3442) - if self.is_at_stop_height(height) { - tracing::info!( - ?height, - ?hash, - block_source = ?source, - "stopping at configured height, flushing database to disk" - ); + self.db.flush_block_batch(batch, source); - // POC: emit the equivalence digest + fast-path summary before exit. - self.vct_log_equivalence_digest(); + self.pipeline_post_assemble(height, retention); - // We're just about to do a forced exit, so it's ok to do a forced db shutdown - self.db.shutdown(true); + // Save blocks to elasticsearch if the feature is enabled. + #[cfg(feature = "elasticsearch")] + self.elasticsearch(&finalized_block); - // Drops tracing log output that's hasn't already been written to stdout - // since this exits before calling drop on the WorkerGuard for the logger thread. - // This is okay for now because this is test-only code - // - // TODO: Call ZebradApp.shutdown or drop its Tracing component before calling exit_process to flush logs to stdout - Self::exit_process(); - } + self.finalized_stop_at_height_if_configured(height, hash, source); + + (hash, note_commitment_trees) + } + + /// Run the committer-owned bookkeeping for a block that has been assembled + /// (and, on the synchronous path, flushed): evict its now-committed + /// verified-commitment-tree roots and clear the archive backlog flag if the + /// retention plan says it is now drained. + /// + /// In the run-ahead pipeline this runs on the assembler thread right after a + /// block is assembled (it only frees memory / clears a flag, so it does not + /// need to wait for the durable flush); the synchronous + /// [`flush_finalized_direct`](Self::flush_finalized_direct) runs it after the + /// flush. + fn pipeline_post_assemble(&self, height: block::Height, retention: RetentionPlan) { + if let Some(vct) = &self.vct { + vct.evict_committed_roots_through(height); } - result.map(|hash| (hash, note_commitment_trees)) + if retention.clears_archive_backlog() { + self.checkpoint_raw_tx_archive_backlog + .store(false, Ordering::Relaxed); + } + } + + /// If `height` is the configured `debug_stop_at_height`, flush the database and + /// exit the process. + /// + /// This must only be called once the block at `height` is durable on disk, so + /// the run-ahead pipeline calls it from the assembler after the disk-writer + /// thread acknowledges the flush of that height. + pub(crate) fn finalized_stop_at_height_if_configured( + &mut self, + height: block::Height, + hash: block::Hash, + source: &str, + ) { + // TODO: move the stop height check to the syncer (#3442) + if self.is_at_stop_height(height) { + tracing::info!( + ?height, + ?hash, + block_source = ?source, + "stopping at configured height, flushing database to disk" + ); + + // POC: emit the equivalence digest + fast-path summary before exit. + self.vct_log_equivalence_digest(); + + // We're just about to do a forced exit, so it's ok to do a forced db shutdown + self.db.shutdown(true); + + // Drops tracing log output that's hasn't already been written to stdout + // since this exits before calling drop on the WorkerGuard for the logger thread. + // This is okay for now because this is test-only code + // + // TODO: Call ZebradApp.shutdown or drop its Tracing component before calling exit_process to flush logs to stdout + Self::exit_process(); + } } /// POC: `true` when the verified-commitment-trees fast (skip-recompute) path will @@ -1508,7 +1770,7 @@ impl FinalizedState { /// Stop the process if `block_height` is greater than or equal to the /// configured stop height. - fn is_at_stop_height(&self, block_height: block::Height) -> bool { + pub(crate) fn is_at_stop_height(&self, block_height: block::Height) -> bool { let debug_stop_at_height = match self.debug_stop_at_height { Some(debug_stop_at_height) => debug_stop_at_height, None => return false, diff --git a/zebra-state/src/service/finalized_state/disk_format/transparent.rs b/zebra-state/src/service/finalized_state/disk_format/transparent.rs index bf695381b70..0afff70ef4f 100644 --- a/zebra-state/src/service/finalized_state/disk_format/transparent.rs +++ b/zebra-state/src/service/finalized_state/disk_format/transparent.rs @@ -284,9 +284,11 @@ impl From> for AddressBalanceLocati /// Represents a change in the [`AddressBalanceLocation`] of a transparent address /// in the finalized state. +#[derive(Clone)] pub struct AddressBalanceLocationChange(AddressBalanceLocationInner); /// Represents a set of updates to address balance locations in the database. +#[derive(Clone)] pub enum AddressBalanceLocationUpdates { /// A set of [`AddressBalanceLocationChange`]s that should be merged into the existing values in the database. Merge(HashMap), diff --git a/zebra-state/src/service/finalized_state/pipeline.rs b/zebra-state/src/service/finalized_state/pipeline.rs new file mode 100644 index 00000000000..c8172909357 --- /dev/null +++ b/zebra-state/src/service/finalized_state/pipeline.rs @@ -0,0 +1,284 @@ +//! In-memory state for the run-ahead finalized-commit pipeline. +//! +//! When [`Config::finalized_block_pipeline_depth`](crate::config::Config::finalized_block_pipeline_depth) +//! is greater than zero, the finalized committer assembles a block's +//! [`DiskWriteBatch`](super::DiskWriteBatch) on the assembler thread and hands it +//! to a dedicated disk-writer thread, so the next block's assembly overlaps this +//! block's flush. +//! +//! Assembling block `X + 1` while block `X` is still being flushed means `X + 1`'s +//! reads can target state that `X` created but has not yet written to disk. This +//! module holds that not-yet-durable state so those reads can be served from +//! memory before falling back to RocksDB: +//! +//! - **Threaded tip state** — the history tree, note-commitment trees, value pool, +//! `vct_upgrade_height` marker, and the (height, hash) tip cursor. These advance +//! every block and are otherwise read fresh from disk per block, so they must be +//! carried forward in memory while the assembler is ahead of the writer. +//! - **A read-through overlay** — the transparent outputs created, and the absolute +//! address balances updated, by not-yet-flushed blocks. Entries are retired once +//! the writer has flushed up to (or past) the height that wrote them, so the +//! overlay is bounded by the pipeline depth. +//! +//! This is only used below the last checkpoint, the reorg-free region, so the +//! overlay never has to roll back: a crash mid-pipeline simply resumes from the +//! durable finalized tip and re-downloads the gap. +//! +//! The committer (in [`write`](crate::service::write)) [`seed`]s the pipeline from +//! disk, [`record_block`]s each assembled block's contribution before handing its +//! batch to the disk-writer thread, and [`retire_through`]s the overlay as the +//! writer makes blocks durable. +//! +//! [`seed`]: FinalizedPipeline::seed +//! [`record_block`]: FinalizedPipeline::record_block +//! [`retire_through`]: FinalizedPipeline::retire_through + +use std::{collections::HashMap, sync::Arc}; + +use zebra_chain::{ + amount::NonNegative, + block::{self, Height}, + history_tree::HistoryTree, + transparent, + value_balance::ValueBalance, +}; + +use crate::service::finalized_state::{ + disk_format::transparent::{AddressBalanceLocation, AddressBalanceLocationUpdates}, + disk_format::OutputLocation, + ZebraDb, +}; + +use super::NoteCommitmentTrees; + +/// The pipeline-relevant outputs of [`DiskWriteBatch::prepare_block_batch`], returned +/// so the run-ahead committer can thread them forward in memory. +/// +/// [`DiskWriteBatch::prepare_block_batch`]: super::DiskWriteBatch::prepare_block_batch +pub(crate) struct BlockBatchOutputs { + /// The chain value pool after this block. + pub value_pool: ValueBalance, + /// The absolute address balances this block updated, captured only when the + /// pipeline is running ahead (`None` on the synchronous path). + pub address_balances: Option, + /// Whether this block wrote the set-once `vct_upgrade_height` marker. + pub wrote_vct_upgrade_marker: bool, +} + +/// The pipeline contribution captured by [`ZebraDb::assemble_block_batch`] for a +/// single block: its value pool, created outputs, and updated address balances. +/// Combined with the block's tip/history/note trees by the committer to form a +/// [`BlockPipelineContribution`]. +/// +/// [`ZebraDb::assemble_block_batch`]: super::ZebraDb::assemble_block_batch +pub(crate) struct PipelineBatchContribution { + /// The chain value pool after this block. + pub value_pool: ValueBalance, + /// Whether this block wrote the set-once `vct_upgrade_height` marker. + pub wrote_vct_upgrade_marker: bool, + /// The transparent outputs this block created: `(outpoint, location, utxo)`. + pub created_outputs: Vec<(transparent::OutPoint, OutputLocation, transparent::Utxo)>, + /// The absolute address balances this block updated. + pub updated_balances: Vec<(transparent::Address, AddressBalanceLocation)>, +} + +/// A transparent output created by a not-yet-flushed block. +#[derive(Clone, Debug)] +struct OverlayUtxo { + /// The on-disk location this output will occupy once flushed. + out_loc: OutputLocation, + /// The unspent output. + utxo: transparent::Utxo, + /// The height of the block that created it, for retirement. + height: Height, +} + +/// An absolute address balance written by a not-yet-flushed block. +#[derive(Clone, Debug)] +struct OverlayBalance { + /// The absolute balance/location after the latest not-yet-flushed update. + balance: AddressBalanceLocation, + /// The height of the latest block that updated it, for retirement. + height: Height, +} + +/// Everything block assembly captures about a committed-but-not-yet-flushed block, +/// so the next block's assembly can read its effects from memory. +#[derive(Clone, Debug)] +pub(crate) struct BlockPipelineContribution { + /// The committed block's height and hash (the new in-memory tip cursor). + pub tip: (Height, block::Hash), + /// The history tree after this block. + pub history_tree: Arc, + /// The note-commitment trees after this block. + pub note_commitment_trees: NoteCommitmentTrees, + /// The value pool after this block. + pub value_pool: ValueBalance, + /// Whether this block wrote the set-once `vct_upgrade_height` marker. + pub wrote_vct_upgrade_marker: bool, + /// The transparent outputs this block created: `(outpoint, location, utxo)`. + pub created_outputs: Vec<(transparent::OutPoint, OutputLocation, transparent::Utxo)>, + /// The absolute address balances this block updated. + pub updated_balances: Vec<(transparent::Address, AddressBalanceLocation)>, +} + +/// The in-memory tip state and read-through overlay for the run-ahead committer. +/// +/// Seeded lazily from disk on the first block, then advanced in memory by +/// [`record_block`](Self::record_block) and trimmed by +/// [`retire_through`](Self::retire_through) as the writer makes blocks durable. +#[derive(Debug)] +pub(crate) struct FinalizedPipeline { + /// `true` once the threaded tip state has been seeded from disk. + seeded: bool, + /// The in-memory finalized tip `(height, hash)`, ahead of the durable tip. + tip: Option<(Height, block::Hash)>, + /// The history tree after the latest assembled block. + history_tree: Arc, + /// The note-commitment trees after the latest assembled block. + note_commitment_trees: NoteCommitmentTrees, + /// The value pool after the latest assembled block. + value_pool: ValueBalance, + /// Whether a not-yet-flushed block has written the `vct_upgrade_height` marker. + vct_upgrade_marker_set: bool, + /// Transparent outputs created by not-yet-flushed blocks, keyed by outpoint. + utxos: HashMap, + /// Absolute address balances updated by not-yet-flushed blocks. + address_balances: HashMap, +} + +impl FinalizedPipeline { + /// Create an empty, unseeded pipeline. The threaded tip state is read from + /// `db` on the first [`seed`](Self::seed). + pub(crate) fn new() -> Self { + Self { + seeded: false, + tip: None, + history_tree: Arc::new(HistoryTree::default()), + note_commitment_trees: NoteCommitmentTrees::default(), + value_pool: ValueBalance::zero(), + vct_upgrade_marker_set: false, + utxos: HashMap::new(), + address_balances: HashMap::new(), + } + } + + /// Seed the threaded tip state from the durable database, if not already seeded. + /// + /// Called before assembling the first block of a pipeline run, so the in-memory + /// tip state starts exactly equal to disk. + pub(crate) fn seed(&mut self, db: &ZebraDb) { + if self.seeded { + return; + } + + self.tip = db + .finalized_tip_height() + .map(|height| (height, db.finalized_tip_hash())); + self.history_tree = db.history_tree(); + self.note_commitment_trees = db.note_commitment_trees_for_tip(); + self.value_pool = db.finalized_value_pool(); + self.vct_upgrade_marker_set = db.vct_upgrade_height().is_some(); + self.seeded = true; + } + + /// The in-memory tip `(height, hash)`, used for the commit ordering asserts. + pub(crate) fn tip(&self) -> Option<(Height, block::Hash)> { + self.tip + } + + /// The note-commitment trees after the latest assembled block (the parent + /// trees for the next block). + pub(crate) fn note_commitment_trees(&self) -> NoteCommitmentTrees { + self.note_commitment_trees.clone() + } + + /// The history tree after the latest assembled block. + pub(crate) fn history_tree(&self) -> Arc { + self.history_tree.clone() + } + + /// The value pool after the latest assembled block (the starting pool for the + /// next block). + pub(crate) fn value_pool(&self) -> ValueBalance { + self.value_pool + } + + /// Whether a not-yet-flushed block has already written the set-once + /// `vct_upgrade_height` marker, so the next block must not write it again. + pub(crate) fn vct_upgrade_marker_set(&self) -> bool { + self.vct_upgrade_marker_set + } + + /// Overlay lookup for a spent output: returns its location and value if it was + /// created by a not-yet-flushed block. + pub(crate) fn spent_utxo_override( + &self, + outpoint: &transparent::OutPoint, + ) -> Option<(OutputLocation, transparent::Utxo)> { + self.utxos + .get(outpoint) + .map(|entry| (entry.out_loc, entry.utxo.clone())) + } + + /// Overlay lookup for an address balance: returns the absolute balance if a + /// not-yet-flushed block updated it. + pub(crate) fn address_balance_override( + &self, + address: &transparent::Address, + ) -> Option { + self.address_balances + .get(address) + .map(|entry| entry.balance) + } + + /// Record a freshly-assembled block's effects, advancing the in-memory tip + /// state and overlay so the next block's assembly observes them. + pub(crate) fn record_block(&mut self, contribution: BlockPipelineContribution) { + let BlockPipelineContribution { + tip, + history_tree, + note_commitment_trees, + value_pool, + wrote_vct_upgrade_marker, + created_outputs, + updated_balances, + } = contribution; + + let (height, _hash) = tip; + + self.tip = Some(tip); + self.history_tree = history_tree; + self.note_commitment_trees = note_commitment_trees; + self.value_pool = value_pool; + self.vct_upgrade_marker_set |= wrote_vct_upgrade_marker; + + for (outpoint, out_loc, utxo) in created_outputs { + self.utxos.insert( + outpoint, + OverlayUtxo { + out_loc, + utxo, + height, + }, + ); + } + + for (address, balance) in updated_balances { + self.address_balances + .insert(address, OverlayBalance { balance, height }); + } + } + + /// Retire overlay entries the writer has made durable, after it flushes + /// `flushed_height`. + /// + /// Once a block is on disk, its created outputs and updated balances are served + /// from RocksDB, so the overlay entries written at or below the durable height + /// are redundant and can be dropped, keeping the overlay bounded by depth. + pub(crate) fn retire_through(&mut self, flushed_height: Height) { + self.utxos.retain(|_, entry| entry.height > flushed_height); + self.address_balances + .retain(|_, entry| entry.height > flushed_height); + } +} diff --git a/zebra-state/src/service/finalized_state/tests/prop.rs b/zebra-state/src/service/finalized_state/tests/prop.rs index 2e6a533c78a..4be16ce508f 100644 --- a/zebra-state/src/service/finalized_state/tests/prop.rs +++ b/zebra-state/src/service/finalized_state/tests/prop.rs @@ -21,7 +21,7 @@ use crate::{ use super::super::{ commitment_aux, serve_block_roots, vct::validate_final_frontiers_bytes, - CheckpointVerifiedBlock, DiskWriteBatch, FinalizedState, + CheckpointVerifiedBlock, DiskWriteBatch, FinalizedPipeline, FinalizedState, }; const DEFAULT_PARTIAL_CHAIN_PROPTEST_CASES: u32 = 1; @@ -176,6 +176,96 @@ fn blocks_with_v5_transactions() -> Result<()> { Ok(()) } +/// Committing a chain through the run-ahead pipeline produces byte-identical +/// finalized state to the synchronous committer. +/// +/// Every block is assembled against the in-memory overlay *before any of them are +/// flushed*, so later blocks read earlier blocks' not-yet-durable effects from the +/// overlay (spent UTXOs, address balances, value pool, history/note trees, tip +/// cursor). The batches are then flushed in height order. The resulting finalized +/// tip, height, and value pool must match a synchronous `commit_finalized_direct` +/// of the same chain. +#[test] +fn pipelined_commit_matches_synchronous() -> Result<()> { + let _init_guard = zebra_test::init(); + proptest!(ProptestConfig::with_cases(env::var("PROPTEST_CASES") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(DEFAULT_PARTIAL_CHAIN_PROPTEST_CASES)), + |((chain, count, network, _history_tree) in PreparedChain::default())| { + let blocks: Vec<_> = chain.iter().take(count).cloned().collect(); + + // Synchronous baseline. + let mut sync_state = FinalizedState::new( + &Config::ephemeral(), + &network, + #[cfg(feature = "elasticsearch")] + false, + ); + for block in &blocks { + let checkpoint_verified = CheckpointVerifiedBlock::from(block.block.clone()); + sync_state + .commit_finalized_direct( + checkpoint_verified.into(), + None, + None, + None, + "pipelined_commit_matches_synchronous sync", + ) + .unwrap(); + } + + // Run-ahead: assemble every block against the overlay (threading the + // note-commitment trees forward), buffer the batches, then flush them in + // height order — mirroring what the disk-writer thread does, single-threaded. + let mut pipe_state = FinalizedState::new( + &Config::ephemeral(), + &network, + #[cfg(feature = "elasticsearch")] + false, + ); + let mut pipeline = FinalizedPipeline::new(); + let mut prev_trees = None; + let mut pending = std::collections::VecDeque::new(); + for block in &blocks { + let checkpoint_verified = CheckpointVerifiedBlock::from(block.block.clone()); + let assembled = pipe_state + .commit_finalized_pipelined( + checkpoint_verified.into(), + prev_trees.take(), + None, + None, + &mut pipeline, + ) + .unwrap(); + prev_trees = Some(assembled.note_commitment_trees.clone()); + pending.push_back(assembled); + } + for assembled in pending { + let height = assembled.height; + pipe_state + .db + .flush_block_batch(assembled.batch, "pipelined_commit_matches_synchronous flush"); + pipeline.retire_through(height); + } + + prop_assert_eq!( + sync_state.finalized_tip_hash(), + pipe_state.finalized_tip_hash() + ); + prop_assert_eq!( + sync_state.finalized_tip_height(), + pipe_state.finalized_tip_height() + ); + prop_assert_eq!( + sync_state.finalized_value_pool(), + pipe_state.finalized_value_pool() + ); + }); + + Ok(()) +} + /// Test if committing blocks from all upgrades work correctly, to make /// sure the contextual validation done by the finalized state works. /// Also test if a block with the wrong commitment is correctly rejected. diff --git a/zebra-state/src/service/finalized_state/tests/transparent.rs b/zebra-state/src/service/finalized_state/tests/transparent.rs index c10689d1868..0cba782fe47 100644 --- a/zebra-state/src/service/finalized_state/tests/transparent.rs +++ b/zebra-state/src/service/finalized_state/tests/transparent.rs @@ -183,6 +183,7 @@ fn intra_block_self_spend_chain_in_finalized_state() { #[cfg(feature = "indexer")] &HashMap::new(), address_balances, + false, ); // Write the batch and confirm the final on-disk balance matches the consensus value diff --git a/zebra-state/src/service/finalized_state/zebra_db/block.rs b/zebra-state/src/service/finalized_state/zebra_db/block.rs index ddf691f160c..f2b80bb5058 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block.rs @@ -56,6 +56,8 @@ use crate::{ #[cfg(feature = "indexer")] use crate::request::Spend; +use super::super::pipeline::{BlockBatchOutputs, FinalizedPipeline, PipelineBatchContribution}; + #[cfg(test)] mod tests; @@ -1007,18 +1009,26 @@ impl ZebraDb { // Write block methods - /// Write `finalized` to the finalized state. + /// Commit `finalized` to the finalized state: assemble its [`DiskWriteBatch`] + /// and immediately flush it to disk. + /// + /// This is the synchronous (tip-mode) entry point. The run-ahead committer + /// instead calls [`assemble_block_batch`](Self::assemble_block_batch) on the + /// assembler thread and [`flush_block_batch`](Self::flush_block_batch) on a + /// separate disk-writer thread, so the next block's assembly overlaps this + /// block's flush. /// - /// Uses: - /// - `history_tree`: the current tip's history tree - /// - `network`: the configured network - /// - `source`: the source of the block in log messages + /// Production commits go through [`FinalizedState::assemble_finalized_direct`] + /// and [`FinalizedState::flush_finalized_direct`], which call the two halves + /// below directly; this combined wrapper is retained as a synchronous one-shot + /// for tests, so it is unused in lib-only builds. /// /// # Errors /// /// - Propagates any errors from writing to the DB /// - Propagates any errors from computing the block's chain value balance change or /// from applying the change to the chain value balance + #[allow(dead_code)] #[allow(clippy::unwrap_in_result)] #[allow(clippy::too_many_arguments)] pub(in super::super) fn write_block( @@ -1028,12 +1038,62 @@ impl ZebraDb { network: &Network, source: &str, retention: RetentionPlan, + vct_anchor_roots: Option<(sapling::tree::Root, orchard::tree::Root)>, + vct_sync_below: Option, + ) -> Result { + let (batch, hash, _contribution) = self.assemble_block_batch( + finalized, + prev_note_commitment_trees, + network, + retention, + vct_anchor_roots, + vct_sync_below, + None, + )?; + self.flush_block_batch(batch, source); + Ok(hash) + } + + /// Assemble the [`DiskWriteBatch`] for `finalized` without writing it to disk. + /// + /// This is the read/compute half of a block commit: it reads the spent UTXOs, + /// changed-address balances, and current value pool, applies the transparent + /// and shielded batch preparation, and returns the fully-built batch plus the + /// committed block hash. It performs no writes, so it can run on the assembler + /// thread (or the look-ahead) while a previously-assembled batch is being + /// flushed by the disk-writer thread. + /// + /// The reads it performs (`read_spent_utxo`, [`address_balance_location`], the + /// value pool) depend on the parent block. In run-ahead (sync) mode the parent + /// may not yet be durable on disk, so those reads must be served from the + /// in-memory pipeline overlay before falling back to disk; see the caller. + /// + /// [`address_balance_location`]: ZebraDb::address_balance_location + #[allow(clippy::too_many_arguments)] + pub(in super::super) fn assemble_block_batch( + &self, + finalized: FinalizedBlock, + prev_note_commitment_trees: Option, + network: &Network, + retention: RetentionPlan, // When `Some`, skip per-height tree writes and fold these roots into // the anchor set. vct_anchor_roots: Option<(sapling::tree::Root, orchard::tree::Root)>, // When `Some(height)`, mark the database as vct-synced. vct_sync_below: Option, - ) -> Result { + // The run-ahead pipeline's in-memory tip state and overlay. When `Some`, + // the parent-block reads (spent UTXOs, address balances, value pool, and + // the `vct_upgrade_height` marker) are served from it before falling back + // to disk, and this block's contribution is captured and returned. + overlay: Option<&FinalizedPipeline>, + ) -> Result< + ( + DiskWriteBatch, + block::Hash, + Option, + ), + CommitCheckpointVerifiedError, + > { let tx_hash_indexes: HashMap = finalized .transaction_hashes .iter() @@ -1073,6 +1133,24 @@ impl ZebraDb { // `None` it serializes inline (e.g. the semantic path). let store_raw_txs = retention.stores_raw_transactions(); let db: &ZebraDb = self; + // Resolve a spent output from the run-ahead overlay first (it may have been + // created by a not-yet-flushed block), then from disk / this block's own + // new outputs. With `overlay = None` this is exactly the disk read. + let read_one_spent = + |outpoint: transparent::OutPoint| -> (transparent::OutPoint, OutputLocation, transparent::Utxo) { + if let Some(overlay) = overlay { + if let Some((out_loc, utxo)) = overlay.spent_utxo_override(&outpoint) { + return (outpoint, out_loc, utxo); + } + } + read_spent_utxo( + db, + finalized.height, + outpoint, + &tx_hash_indexes, + &finalized.new_outputs, + ) + }; #[cfg(feature = "commit-metrics")] let spent_reads_start = std::time::Instant::now(); let (spent_utxos, precomputed_raw_txs): ( @@ -1082,31 +1160,9 @@ impl ZebraDb { || { if outpoints.len() >= super::PARALLEL_BLOCK_READ_THRESHOLD { use rayon::prelude::*; - outpoints - .into_par_iter() - .map(|outpoint| { - read_spent_utxo( - db, - finalized.height, - outpoint, - &tx_hash_indexes, - &finalized.new_outputs, - ) - }) - .collect() + outpoints.into_par_iter().map(&read_one_spent).collect() } else { - outpoints - .into_iter() - .map(|outpoint| { - read_spent_utxo( - db, - finalized.height, - outpoint, - &tx_hash_indexes, - &finalized.new_outputs, - ) - }) - .collect() + outpoints.into_iter().map(&read_one_spent).collect() } }, || { @@ -1194,27 +1250,50 @@ impl ZebraDb { // reading all of the pending merge operands (potentially hundreds), and applying pending merge operands to the // fully-merged value such that it's much faster to read entries that have been updated with insertions than it // is to read entries that have been updated with merge operations. + // Resolve an address balance from the run-ahead overlay first (it may have + // been updated by a not-yet-flushed block), then from disk. With + // `overlay = None` this is exactly the disk read. + let lookup_balance = |addr: &transparent::Address| { + if let Some(overlay) = overlay { + if let Some(balance) = overlay.address_balance_override(addr) { + return Some(balance); + } + } + self.address_balance_location(addr) + }; #[cfg(feature = "commit-metrics")] let address_reads_start = std::time::Instant::now(); let address_balances: AddressBalanceLocationUpdates = if self.finished_format_upgrades() { AddressBalanceLocationUpdates::Insert(read_addr_locs(changed_addresses, |addr| { - self.address_balance_location(addr) + lookup_balance(addr) })) } else { AddressBalanceLocationUpdates::Merge(read_addr_locs(changed_addresses, |addr| { - Some(self.address_balance_location(addr)?.into_new_change()) + Some(lookup_balance(addr)?.into_new_change()) })) }; #[cfg(feature = "commit-metrics")] metrics::histogram!("zebra.state.write.address_reads.duration_seconds") .record(address_reads_start.elapsed().as_secs_f64()); + // The value pool and `vct_upgrade_height` marker are threaded forward in + // memory by the run-ahead pipeline; with `overlay = None` they come from + // disk exactly as before. `capture_pipeline_outputs` is set only when + // running ahead, so the synchronous path avoids the balance clone. + let value_pool = overlay + .map(|overlay| overlay.value_pool()) + .unwrap_or_else(|| self.finalized_value_pool()); + let pipeline_vct_marker_set = overlay + .map(|overlay| overlay.vct_upgrade_marker_set()) + .unwrap_or(false); + let capture_pipeline_outputs = overlay.is_some(); + let mut batch = DiskWriteBatch::new(); // In case of errors, propagate and do not write the batch. #[cfg(feature = "commit-metrics")] let batch_prep_start = std::time::Instant::now(); - batch.prepare_block_batch( + let block_outputs = batch.prepare_block_batch( self, network, &finalized, @@ -1224,12 +1303,14 @@ impl ZebraDb { #[cfg(feature = "indexer")] out_loc_by_outpoint, address_balances, - self.finalized_value_pool(), + value_pool, prev_note_commitment_trees, store_raw_txs, precomputed_raw_txs, vct_anchor_roots, vct_sync_below, + pipeline_vct_marker_set, + capture_pipeline_outputs, )?; // In pruned storage mode, delete raw transaction history that has fallen @@ -1247,6 +1328,49 @@ impl ZebraDb { .record(batch.size_in_bytes() as f64); } + // When running ahead, capture this block's overlay contribution: the + // outputs it created (so the next block can spend them from memory) and + // the absolute address balances it updated. + let contribution = overlay.map(|_| { + let created_outputs = finalized + .new_outputs + .iter() + .map(|(outpoint, ordered_utxo)| { + ( + *outpoint, + lookup_out_loc(finalized.height, outpoint, &tx_hash_indexes), + ordered_utxo.utxo.clone(), + ) + }) + .collect(); + let updated_balances = match block_outputs.address_balances { + Some(AddressBalanceLocationUpdates::Insert(balances)) => { + balances.into_iter().collect() + } + // The pipeline only runs once format upgrades are finished (the + // `Insert` path), so `Merge`/`None` contribute no balances. + _ => Vec::new(), + }; + + PipelineBatchContribution { + value_pool: block_outputs.value_pool, + wrote_vct_upgrade_marker: block_outputs.wrote_vct_upgrade_marker, + created_outputs, + updated_balances, + } + }); + + Ok((batch, finalized.hash, contribution)) + } + + /// Flush a previously [`assemble_block_batch`](Self::assemble_block_batch)d + /// batch to disk. + /// + /// This is the write half of a block commit. In run-ahead (sync) mode it runs + /// on the dedicated disk-writer thread so it overlaps the next block's + /// assembly; in tip mode it runs inline. A rocksdb write failure is fatal, as + /// before. + pub(crate) fn flush_block_batch(&self, batch: DiskWriteBatch, source: &str) { // Track batch commit latency for observability let batch_start = std::time::Instant::now(); self.db @@ -1256,8 +1380,6 @@ impl ZebraDb { .record(batch_start.elapsed().as_secs_f64()); tracing::trace!(?source, "committed block from"); - - Ok(finalized.hash) } /// Writes the given batch to the database. @@ -1569,7 +1691,7 @@ impl DiskWriteBatch { /// - Propagates any errors from computing the block's chain value balance change or /// from applying the change to the chain value balance #[allow(clippy::too_many_arguments)] - pub fn prepare_block_batch( + pub(crate) fn prepare_block_batch( &mut self, zebra_db: &ZebraDb, network: &Network, @@ -1588,7 +1710,13 @@ impl DiskWriteBatch { precomputed_raw_txs: Option>, vct_anchor_roots: Option<(sapling::tree::Root, orchard::tree::Root)>, vct_sync_below: Option, - ) -> Result<(), CommitCheckpointVerifiedError> { + // When `true`, a not-yet-flushed pipeline block already wrote the set-once + // `vct_upgrade_height` marker, so this block must not write it again. + pipeline_vct_marker_set: bool, + // When `true`, capture and return this block's post-update value pool and + // absolute address balances for the run-ahead pipeline overlay. + capture_pipeline_outputs: bool, + ) -> Result { // Commit block, transaction, and note commitment tree data. self.prepare_block_header_and_transaction_data_batch( zebra_db, @@ -1609,12 +1737,21 @@ impl DiskWriteBatch { // // In Zebra we include the nullifiers and note commitments in the genesis block because it simplifies our code. self.prepare_shielded_transaction_batch(zebra_db, finalized); + + // The `vct_upgrade_height` marker is written once, by the first block this + // binary commits. In the run-ahead pipeline an earlier not-yet-flushed block + // may have already set it in memory (`pipeline_vct_marker_set`), in which case + // its disk write is still pending, so the disk read would wrongly look absent; + // suppress the duplicate write here and report which block actually wrote it. + let wrote_vct_upgrade_marker = + !pipeline_vct_marker_set && zebra_db.vct_upgrade_height().is_none(); self.prepare_trees_batch( zebra_db, finalized, prev_note_commitment_trees, vct_anchor_roots, vct_sync_below, + wrote_vct_upgrade_marker, ); // # Consensus @@ -1627,9 +1764,10 @@ impl DiskWriteBatch { // So we ignore the genesis UTXO, transparent address index, and value pool updates // for the genesis block. This also ignores genesis shielded value pool updates, but there // aren't any of those on mainnet or testnet. + let mut captured_address_balances = None; if !finalized.height.is_min() { // Commit transaction indexes - self.prepare_transparent_transaction_batch( + captured_address_balances = self.prepare_transparent_transaction_batch( zebra_db, network, finalized, @@ -1639,11 +1777,14 @@ impl DiskWriteBatch { #[cfg(feature = "indexer")] &out_loc_by_outpoint, address_balances, + capture_pipeline_outputs, ); } - // Commit UTXOs and value pools - self.prepare_chain_value_pools_batch( + // Commit UTXOs and value pools. This runs for every height, including + // genesis (which writes the initial pool and block info), matching the + // original commit path. + let new_value_pool = self.prepare_chain_value_pools_batch( zebra_db, finalized, spent_utxos_by_outpoint, @@ -1653,7 +1794,11 @@ impl DiskWriteBatch { // The block has passed contextual validation, so update the metrics block_precommit_metrics(&finalized.block, finalized.hash, finalized.height); - Ok(()) + Ok(BlockBatchOutputs { + value_pool: new_value_pool, + address_balances: captured_address_balances, + wrote_vct_upgrade_marker, + }) } /// Adds deletes for pruned raw transaction data to this batch, for the diff --git a/zebra-state/src/service/finalized_state/zebra_db/chain.rs b/zebra-state/src/service/finalized_state/zebra_db/chain.rs index e3532ad69a1..9b869ba33c1 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/chain.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/chain.rs @@ -286,13 +286,15 @@ impl DiskWriteBatch { /// [`chain_value_pool_change`]: zebra_chain::block::Block::chain_value_pool_change /// [`add_chain_value_pool_change`]: ValueBalance::add_chain_value_pool_change #[allow(clippy::unwrap_in_result)] + /// Returns the chain value pool after applying this block, so the run-ahead + /// committer can thread it forward in memory to the next block's assembly. pub fn prepare_chain_value_pools_batch( &mut self, db: &ZebraDb, finalized: &FinalizedBlock, utxos_spent_by_block: HashMap, value_pool: ValueBalance, - ) -> Result<(), ValidateContextError> { + ) -> Result, ValidateContextError> { let block_value_pool_change = finalized .block .chain_value_pool_change( @@ -364,6 +366,6 @@ impl DiskWriteBatch { &BlockInfo::new(new_value_pool, block_size as u32), ); - Ok(()) + Ok(new_value_pool) } } diff --git a/zebra-state/src/service/finalized_state/zebra_db/shielded.rs b/zebra-state/src/service/finalized_state/zebra_db/shielded.rs index bdcfc6dbf58..6d142fc6f93 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/shielded.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/shielded.rs @@ -653,6 +653,11 @@ impl DiskWriteBatch { prev_note_commitment_trees: Option, vct_anchor_roots: Option<(sapling::tree::Root, orchard::tree::Root)>, vct_sync_below: Option, + // Whether to write the set-once `vct_upgrade_height` marker for this block. + // Computed by the caller (`!already_set && marker_is_absent`) so the run-ahead + // pipeline can suppress a duplicate write when an earlier not-yet-flushed block + // already set it. + write_vct_upgrade_marker: bool, ) { let FinalizedBlock { height, @@ -670,7 +675,7 @@ impl DiskWriteBatch { // a node that upgrades above the last checkpoint (legacy path only). Set-once: the marker // is never moved, so the boundary stays stable as the chain grows. Commits are sequential, // so the absent check sees the previous block's committed marker, not a half-written batch. - if zebra_db.vct_upgrade_height().is_none() { + if write_vct_upgrade_marker { self.update_vct_upgrade_marker(zebra_db, *height); } diff --git a/zebra-state/src/service/finalized_state/zebra_db/transparent.rs b/zebra-state/src/service/finalized_state/zebra_db/transparent.rs index a5aec765587..d7ffa2c60e3 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/transparent.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/transparent.rs @@ -417,6 +417,10 @@ impl DiskWriteBatch { /// If this method returns an error, it will be propagated, /// and the batch should not be written to the database. #[allow(clippy::too_many_arguments)] + /// When `capture_balances` is true, returns the absolute address balances + /// after applying this block, so the run-ahead committer can serve the + /// next block's address reads from memory before the write is durable. + /// Returns `None` otherwise, avoiding a clone on the synchronous path. pub fn prepare_transparent_transaction_batch( &mut self, zebra_db: &ZebraDb, @@ -430,7 +434,8 @@ impl DiskWriteBatch { OutputLocation, >, mut address_balances: AddressBalanceLocationUpdates, - ) { + capture_balances: bool, + ) -> Option { let db = &zebra_db.db; let FinalizedBlock { block, height, .. } = finalized; @@ -478,7 +483,13 @@ impl DiskWriteBatch { ); } + // Capture the post-block absolute balances for the pipeline overlay before + // they are moved into the write batch (cheap clone, only when running ahead). + let captured = capture_balances.then(|| address_balances.clone()); + self.prepare_transparent_balances_batch(db, address_balances); + + captured } /// Update `address_balances` in memory for the transparent transfers in `transactions`, diff --git a/zebra-state/src/service/write.rs b/zebra-state/src/service/write.rs index 2a3f373b2cf..26110518a37 100644 --- a/zebra-state/src/service/write.rs +++ b/zebra-state/src/service/write.rs @@ -29,12 +29,14 @@ use crate::{ error::CommitHeaderRangeError, service::{ check, - finalized_state::{spawn_note_precompute, FinalizedState, ZebraDb}, + finalized_state::{ + spawn_note_precompute, DiskWriteBatch, FinalizedPipeline, FinalizedState, ZebraDb, + }, non_finalized_state::NonFinalizedState, queued_blocks::{QueuedCheckpointVerified, QueuedSemanticallyVerified}, ChainTipBlock, ChainTipSender, InvalidateError, ReconsiderError, }, - SemanticallyVerifiedBlock, ValidateContextError, + CommitCheckpointVerifiedError, SemanticallyVerifiedBlock, ValidateContextError, }; // These types are used in doc links @@ -53,6 +55,99 @@ type PendingPrecompute = ( Arc, ); +/// A finalized block batch handed to the disk-writer thread for flushing, in the +/// run-ahead committer. +/// +/// The assembler thread builds these against the in-memory overlay and sends them +/// over a bounded channel; the disk-writer thread flushes each one to disk in +/// height order, sends the per-block commit response only once durable, then acks +/// the flushed height back so the assembler can advance the chain tip and retire +/// the overlay. +struct PipelineFlush { + batch: DiskWriteBatch, + hash: block::Hash, + height: Height, + rsp_tx: oneshot::Sender>, +} + +/// The disk-writer thread loop for the run-ahead finalized committer. +/// +/// Flushes each assembled batch to disk in arrival (height) order, sends the +/// per-block commit response only once the write is durable (ack-after-flush), +/// then acks the flushed height back to the assembler. A rocksdb write failure is +/// fatal, as on the synchronous path. +fn run_finalized_writer( + db: ZebraDb, + batch_receiver: std::sync::mpsc::Receiver, + ack_sender: std::sync::mpsc::Sender, +) { + while let Ok(PipelineFlush { + batch, + hash, + height, + rsp_tx, + }) = batch_receiver.recv() + { + db.flush_block_batch(batch, "commit checkpoint-verified request (pipelined)"); + + // Ack-after-flush: the commit response only resolves once the block is + // durable on disk. + let _ = rsp_tx.send(Ok(hash)); + + // If the assembler has gone away (shutdown), stop. + if ack_sender.send(height).is_err() { + break; + } + } +} + +/// Process the disk-writer thread's flush acknowledgements: for each acked height, +/// advance the finalized chain tip, retire the now-durable overlay entries, and run +/// the configured stop-height check (which exits the process if matched). +/// +/// With `block_until_empty`, waits for every in-flight block to be acked (used at +/// the checkpoint→non-finalized handoff and after a stop-height block); otherwise +/// drains only the acks already available. +fn drain_finalized_acks( + finalized_state: &mut FinalizedState, + pipeline: &mut FinalizedPipeline, + ack_receiver: &std::sync::mpsc::Receiver, + in_flight: &mut VecDeque<(Height, block::Hash, ChainTipBlock)>, + chain_tip_sender: &mut ChainTipSender, + block_until_empty: bool, +) { + while !in_flight.is_empty() { + let acked = if block_until_empty { + match ack_receiver.recv() { + Ok(height) => height, + // The writer thread is gone; stop draining. + Err(_) => break, + } + } else { + match ack_receiver.try_recv() { + Ok(height) => height, + Err(_) => break, + } + }; + + let (height, hash, tip_block) = in_flight + .pop_front() + .expect("each ack corresponds to an in-flight block"); + debug_assert_eq!(height, acked, "the writer acks blocks in height order"); + + chain_tip_sender.set_finalized_tip(tip_block); + pipeline.retire_through(height); + + // The block is now durable, so the stop-height check (which exits the + // process) is safe to run here. + finalized_state.finalized_stop_at_height_if_configured( + height, + hash, + "commit checkpoint-verified request", + ); + } +} + /// Delay between retryable VCT root-miss commit attempts while the peer cache refills. const VCT_ROOT_RETRY_WAIT: Duration = Duration::from_millis(500); @@ -395,6 +490,45 @@ impl WriteBlockWorkerTask { let mut vct_root_stall: Option<(Height, Instant)> = None; let mut vct_root_stall_logged = false; + // Run-ahead finalized-commit pipeline. Enabled only below the last + // checkpoint (the reorg-free region, where the committer is processing the + // finalized block channel) and once format upgrades are finished (so the + // overlay serves absolute `Insert` address balances, not merge deltas). When + // active, the committer assembles each block's batch against the in-memory + // overlay and hands it to a dedicated disk-writer thread; the chain tip and + // overlay retirement trail the writer's flush acks (ack-after-flush). + let pipeline_depth = finalized_state.db.config().finalized_block_pipeline_depth; + let pipeline_active = pipeline_depth > 0 && finalized_state.db.finished_format_upgrades(); + + let mut pipeline_state: Option = None; + let mut pipeline_flush_sender: Option> = None; + let mut pipeline_ack_receiver: Option> = None; + let mut pipeline_writer_handle: Option> = None; + let mut pipeline_in_flight: VecDeque<(Height, block::Hash, ChainTipBlock)> = + VecDeque::new(); + + if pipeline_active { + // The bounded batch channel is the backpressure: at most `pipeline_depth` + // assembled-but-not-yet-flushed blocks are in flight, bounding the overlay. + let (flush_sender, flush_receiver) = + std::sync::mpsc::sync_channel::(pipeline_depth); + let (ack_sender, ack_receiver) = std::sync::mpsc::channel::(); + let writer_db = finalized_state.db.clone(); + let writer_span = Span::current(); + let handle = std::thread::Builder::new() + .name("zebra-finalized-writer".to_string()) + .spawn(move || { + writer_span + .in_scope(|| run_finalized_writer(writer_db, flush_receiver, ack_sender)) + }) + .expect("failed to spawn the finalized disk-writer thread"); + + pipeline_state = Some(FinalizedPipeline::new()); + pipeline_flush_sender = Some(flush_sender); + pipeline_ack_receiver = Some(ack_receiver); + pipeline_writer_handle = Some(handle); + } + // Write all the finalized blocks sent by the state, // until the state closes the finalized block channel's sender. loop { @@ -548,17 +682,60 @@ impl WriteBlockWorkerTask { let prev_note_commitment_trees = prev_finalized_note_commitment_trees.take(); let prev_note_commitment_trees_for_retry = prev_note_commitment_trees.clone(); - let next_block_took_vct_path = - finalized_state.vct_fast_will_apply(ordered_block.0.height); + let committed_height = ordered_block.0.height; + let next_block_took_vct_path = finalized_state.vct_fast_will_apply(committed_height); + + // Commit the block. When the run-ahead pipeline is active, assemble the + // block against the overlay and hand its batch to the disk-writer thread + // (the chain tip advances later, on the writer's flush ack); otherwise + // assemble and flush it synchronously here, advancing the tip inline. + let commit_result: Result< + NoteCommitmentTrees, + (QueuedCheckpointVerified, CommitCheckpointVerifiedError), + > = if let Some(pipeline) = pipeline_state.as_mut() { + let (checkpoint_verified, rsp_tx) = ordered_block; + let tip_block = ChainTipBlock::from(checkpoint_verified.clone()); + match finalized_state.commit_finalized_pipelined( + checkpoint_verified.clone().into(), + prev_note_commitment_trees, + note_precompute, + next_checkpoint, + pipeline, + ) { + Ok(assembled) => { + pipeline_in_flight.push_back((assembled.height, assembled.hash, tip_block)); + // Bounded send: applies backpressure once `pipeline_depth` + // blocks are already in flight to the disk-writer thread. + let _ = pipeline_flush_sender + .as_ref() + .expect("flush sender is present while the pipeline is active") + .send(PipelineFlush { + batch: assembled.batch, + hash: assembled.hash, + height: assembled.height, + rsp_tx, + }); + Ok(assembled.note_commitment_trees) + } + Err(error) => Err(((checkpoint_verified, rsp_tx), error)), + } + } else { + finalized_state + .commit_finalized( + ordered_block, + prev_note_commitment_trees, + note_precompute, + next_checkpoint, + ) + .map(|(finalized, note_commitment_trees)| { + let tip_block = ChainTipBlock::from(finalized); + chain_tip_sender.set_finalized_tip(tip_block); + note_commitment_trees + }) + }; - // Try committing the block - match finalized_state.commit_finalized( - ordered_block, - prev_note_commitment_trees, - note_precompute, - next_checkpoint, - ) { - Ok((finalized, note_commitment_trees)) => { + match commit_result { + Ok(note_commitment_trees) => { // Whether this successful commit consumed header-carried // tree-aux roots to skip the note-commitment frontier rebuild. if next_block_took_vct_path { @@ -581,9 +758,35 @@ impl WriteBlockWorkerTask { vct_root_stall_logged = false; } - let tip_block = ChainTipBlock::from(finalized); prev_finalized_note_commitment_trees = Some(note_commitment_trees); - chain_tip_sender.set_finalized_tip(tip_block); + + // In the pipeline, advance the chain tip and retire the overlay + // for any blocks the writer has already flushed. If this block is + // the configured stop height, then wait for it to become durable, + // so the stop check fires after the flush and no later block is + // assembled past it. + if let (Some(pipeline), Some(ack_receiver)) = + (pipeline_state.as_mut(), pipeline_ack_receiver.as_ref()) + { + drain_finalized_acks( + finalized_state, + pipeline, + ack_receiver, + &mut pipeline_in_flight, + chain_tip_sender, + false, + ); + if finalized_state.is_at_stop_height(committed_height) { + drain_finalized_acks( + finalized_state, + pipeline, + ack_receiver, + &mut pipeline_in_flight, + chain_tip_sender, + true, + ); + } + } } Err((ordered_block, error)) => { // Retryable VCT root stalls (an absent/evicted root, or one not yet @@ -678,6 +881,30 @@ impl WriteBlockWorkerTask { } } + // The finalized block channel has closed (the checkpoint→non-finalized + // handoff). Drain the run-ahead pipeline to durability before switching to + // the non-finalized loop: drop the batch sender so the writer finishes its + // queue and exits, advance the chain tip and retire the overlay for every + // remaining in-flight block, then join the writer thread. + if pipeline_active { + drop(pipeline_flush_sender.take()); + if let (Some(pipeline), Some(ack_receiver)) = + (pipeline_state.as_mut(), pipeline_ack_receiver.as_ref()) + { + drain_finalized_acks( + finalized_state, + pipeline, + ack_receiver, + &mut pipeline_in_flight, + chain_tip_sender, + true, + ); + } + if let Some(handle) = pipeline_writer_handle.take() { + let _ = handle.join(); + } + } + // Do this check even if the channel got closed before any finalized blocks were sent. // This can happen if we're past the finalized tip. if invalid_block_reset_sender.is_closed() { From 5113b76b3b10266fc8915866bed4106871940061 Mon Sep 17 00:00:00 2001 From: Roman Date: Mon, 29 Jun 2026 16:33:18 +0000 Subject: [PATCH 03/17] fix(state): follow in-memory pipeline tip for the next-block height check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The run-ahead finalized committer dropped every block after the first when `finalized_block_pipeline_depth > 0`. The write worker's "next valid height" guard (which drops blocks whose parent has not committed) read the *durable* disk tip via `finalized_state.db.finalized_tip_height()`. But in the pipeline the assembler runs ahead of the disk writer (ack-after-flush), so when block N+1 arrives, block N is assembled but not yet flushed — the durable tip still shows N-1, the guard computes a stale `next_valid_height`, and N+1 is dropped as "wrong height". Its dropped response surfaces as `WriteTaskExited`, which cascades the whole checkpoint range and tears the StateService down. Fix: when the pipeline is active, derive `next_valid_height` from the in-memory assembled tip (`pipeline.tip()`), falling back to the durable tip (so the first block, before the pipeline is seeded, is unchanged). The synchronous path is unaffected (no pipeline -> durable tip, as before). This only reproduces through the full StateService write worker, which the existing `FinalizedState`-only equivalence test bypasses. Adds a regression test (`pipelined_state_service_commits_chain`) that commits a chain through the real StateService with the pipeline enabled. --- zebra-state/src/service/tests.rs | 53 ++++++++++++++++++++++++++++++++ zebra-state/src/service/write.rs | 15 +++++++-- 2 files changed, 65 insertions(+), 3 deletions(-) diff --git a/zebra-state/src/service/tests.rs b/zebra-state/src/service/tests.rs index 81ad0d0c48c..f17d585961b 100644 --- a/zebra-state/src/service/tests.rs +++ b/zebra-state/src/service/tests.rs @@ -239,6 +239,59 @@ fn out_of_order_committing_strategy() -> BoxedStrategy>> { Just(blocks).prop_shuffle().boxed() } +/// Committing checkpoint-verified blocks through the real `StateService` with the +/// run-ahead pipeline enabled (`finalized_block_pipeline_depth > 0`) must succeed, +/// exactly like the synchronous committer. This drives the full write-worker path +/// (which the `FinalizedState`-only equivalence test bypasses), where the pipeline +/// actually activates. +#[tokio::test(flavor = "multi_thread")] +async fn pipelined_state_service_commits_chain() -> Result<()> { + let _init_guard = zebra_test::init(); + let network = Network::Mainnet; + + let blocks: Vec> = zebra_test::vectors::MAINNET_BLOCKS + .range(0..=20) + .map(|(_, b)| b.zcash_deserialize_into::>().unwrap()) + .collect(); + let expected = blocks.len(); + + let mut config = Config::ephemeral(); + config.finalized_block_pipeline_depth = 2; + + let (state, _read, _latest, _change) = + StateService::new(config, &network, Height::MAX, 0).await; + let mut state = Buffer::new(BoxService::new(state), 10); + + // Queue all the commits (so several are in flight, exercising the pipeline depth), + // then await each response in order. + let mut rsps = Vec::new(); + for block in blocks { + let rsp = state + .ready() + .await + .expect("state service ready") + .call(Request::CommitCheckpointVerifiedBlock(block.into())); + rsps.push(rsp); + } + let mut committed = 0; + for (i, rsp) in rsps.into_iter().enumerate() { + rsp.await + .unwrap_or_else(|e| panic!("pipelined commit of block {i} failed: {e:?}")); + committed += 1; + } + + assert_eq!( + committed, expected, + "every queued block should commit through the pipeline" + ); + assert!( + expected >= 2, + "the test chain must have at least two blocks" + ); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread")] async fn empty_state_still_responds_to_requests() -> Result<()> { let _init_guard = zebra_test::init(); diff --git a/zebra-state/src/service/write.rs b/zebra-state/src/service/write.rs index 26110518a37..e519eb1eddb 100644 --- a/zebra-state/src/service/write.rs +++ b/zebra-state/src/service/write.rs @@ -583,9 +583,18 @@ impl WriteBlockWorkerTask { // So if there has been a block commit error, // we need to drop all the descendants of that block, // until we receive a block at the required next height. - let next_valid_height = finalized_state - .db - .finalized_tip_height() + // The next block must be the child of the current tip. In the run-ahead + // pipeline the assembler runs *ahead* of the durable disk tip (ack-after- + // flush), so the next valid height follows the in-memory assembled tip + // (`pipeline.tip()`), not the on-disk tip — otherwise a block whose parent + // is assembled-but-not-yet-flushed is wrongly dropped as "wrong height". + // `pipeline.tip()` is `None` until the first block is assembled (seeded), + // so the first block correctly falls back to the durable tip. + let next_valid_height = pipeline_state + .as_ref() + .and_then(|pipeline| pipeline.tip()) + .map(|(height, _hash)| height) + .or_else(|| finalized_state.db.finalized_tip_height()) .map(|height| (height + 1).expect("committed heights are valid")) .unwrap_or(Height(0)); From b07565c72f796b89640e6d3e20ee546589d66d74 Mon Sep 17 00:00:00 2001 From: Roman Date: Mon, 29 Jun 2026 21:50:32 +0000 Subject: [PATCH 04/17] feat(state): instrument commit pipeline and add offline header-store seeding Add per-phase commit timing for offline profiling: VCT commitment-root verify (fold), assemble-self, queue-wait and batch-commit on the commit-pressure trace, plus disk-writer service/recv-wait and run-ahead assembler park histograms. Add per-block checkpoint verify timing (pow/precompute/merkle) in zebra-consensus. Add ZebraDb::seed_zakura_headers_from_blocks, which seeds the Zakura header store from cached blocks so the header-authenticated checkpoint fast path can run against an offline snapshot with no live header sync. Ungate DiskWriteBatch::size_in_bytes: it is read by the commit-pressure trace independently of the commit-metrics feature, so the workspace did not build without that feature. --- zebra-consensus/src/checkpoint.rs | 18 +- zebra-consensus/src/lib.rs | 1 + zebra-consensus/src/verify_timing.rs | 79 +++++++++ zebra-state/src/service/finalized_state.rs | 27 ++- .../finalized_state/commit_pressure.rs | 157 ++++++++++++++++++ .../src/service/finalized_state/disk_db.rs | 74 ++++++++- .../src/service/finalized_state/tests/prop.rs | 6 +- .../service/finalized_state/zebra_db/block.rs | 95 ++++++++++- zebra-state/src/service/tests.rs | 5 +- zebra-state/src/service/write.rs | 63 +++++-- 10 files changed, 497 insertions(+), 28 deletions(-) create mode 100644 zebra-consensus/src/verify_timing.rs create mode 100644 zebra-state/src/service/finalized_state/commit_pressure.rs diff --git a/zebra-consensus/src/checkpoint.rs b/zebra-consensus/src/checkpoint.rs index 8f643c538a3..784bcd1464f 100644 --- a/zebra-consensus/src/checkpoint.rs +++ b/zebra-consensus/src/checkpoint.rs @@ -603,14 +603,24 @@ where // Cheap proof-of-work checks run *before* the expensive precomputation, // so a flood of invalid-PoW blocks can't make us do per-transaction work. + let pow_start = std::time::Instant::now(); self.check_proof_of_work(&block.header, height, hash)?; + let pow = pow_start.elapsed(); // Precompute the per-transaction hashes and auth data root, which scale // with block weight. (The precomputed path does this concurrently in the // caller and skips it here.) + let precompute_start = std::time::Instant::now(); let block = CheckpointVerifiedBlock::with_hash(block, hash); + let precompute = precompute_start.elapsed(); + metrics::histogram!("zebra.consensus.checkpoint.precompute.duration_seconds") + .record(precompute.as_secs_f64()); - self.finish_validation(block) + let merkle_start = std::time::Instant::now(); + let block = self.finish_validation(block)?; + crate::verify_timing::record(height.0, pow, precompute, merkle_start.elapsed()); + + Ok(block) } /// Check a [`CheckpointVerifiedBlock`] whose precomputation (txids, auth data @@ -636,6 +646,7 @@ where height: block::Height, hash: block::Hash, ) -> Result<(), VerifyCheckpointError> { + let pow_start = std::time::Instant::now(); if self.network.disable_pow() { crate::block::check::difficulty_threshold_is_valid( header, @@ -647,6 +658,8 @@ where crate::block::check::difficulty_is_valid(header, &self.network, &height, &hash)?; crate::block::check::equihash_solution_is_valid(header)?; } + metrics::histogram!("zebra.consensus.checkpoint.pow.duration_seconds") + .record(pow_start.elapsed().as_secs_f64()); Ok(()) } @@ -671,11 +684,14 @@ where .map(DeferredPoolBalanceChange::new), ); + let merkle_start = std::time::Instant::now(); crate::block::check::merkle_root_validity( &self.network, &block.block, &block.transaction_hashes, )?; + metrics::histogram!("zebra.consensus.checkpoint.merkle.duration_seconds") + .record(merkle_start.elapsed().as_secs_f64()); Ok(block) } diff --git a/zebra-consensus/src/lib.rs b/zebra-consensus/src/lib.rs index 30a81414975..858e288a3c5 100644 --- a/zebra-consensus/src/lib.rs +++ b/zebra-consensus/src/lib.rs @@ -38,6 +38,7 @@ mod block; mod checkpoint; mod primitives; mod script; +mod verify_timing; pub mod config; pub mod error; diff --git a/zebra-consensus/src/verify_timing.rs b/zebra-consensus/src/verify_timing.rs new file mode 100644 index 00000000000..92a63f1c655 --- /dev/null +++ b/zebra-consensus/src/verify_timing.rs @@ -0,0 +1,79 @@ +//! Optional per-block checkpoint-verify timing trace (off by default). +//! +//! When `ZEBRA_VERIFY_TIMING_TRACE=` is set, the checkpoint verifier appends a +//! JSON row per block (or every `ZEBRA_VERIFY_TIMING_EVERY` blocks) with the verify +//! sub-phase timings — proof-of-work (difficulty + equihash), precompute (per-tx txids +//! + auth digest), and Merkle root — keyed by height, so an offline analysis can build +//! a by-height verify breakdown and join it with the committer's per-block trace. +//! +//! The hot path is a single `OnceLock` load when the env var is unset (the production +//! default), so this stays inert unless explicitly enabled (e.g. by the replay bench). + +use std::{ + fs::File, + io::{BufWriter, Write}, + sync::{ + atomic::{AtomicU64, Ordering}, + Mutex, OnceLock, + }, + time::{Duration, Instant}, +}; + +struct VerifyTimingTrace { + file: Mutex>, + every: u64, + count: AtomicU64, + start: Instant, +} + +static TRACE: OnceLock> = OnceLock::new(); + +fn trace() -> Option<&'static VerifyTimingTrace> { + TRACE + .get_or_init(|| { + let path = std::env::var_os("ZEBRA_VERIFY_TIMING_TRACE")?; + let every = std::env::var("ZEBRA_VERIFY_TIMING_EVERY") + .ok() + .and_then(|s| s.parse::().ok()) + .unwrap_or(1) + .max(1); + match File::create(&path) { + Ok(file) => { + tracing::info!(?path, every, "verify-timing trace enabled"); + Some(VerifyTimingTrace { + file: Mutex::new(BufWriter::new(file)), + every, + count: AtomicU64::new(0), + start: Instant::now(), + }) + } + Err(error) => { + tracing::warn!(?path, %error, "could not open verify-timing trace"); + None + } + } + }) + .as_ref() +} + +/// Records one verify-timing row (every `every` blocks). Cheap no-op (one `OnceLock` +/// load) when `ZEBRA_VERIFY_TIMING_TRACE` is unset. +pub(crate) fn record(height: u32, pow: Duration, precompute: Duration, merkle: Duration) { + let Some(trace) = trace() else { return }; + let n = trace.count.fetch_add(1, Ordering::Relaxed); + if n % trace.every != 0 { + return; + } + let line = format!( + r#"{{"ts_us":{},"height":{},"pow_ms":{:.3},"precompute_ms":{:.3},"merkle_ms":{:.3}}}"#, + trace.start.elapsed().as_micros(), + height, + pow.as_secs_f64() * 1000.0, + precompute.as_secs_f64() * 1000.0, + merkle.as_secs_f64() * 1000.0, + ); + if let Ok(mut file) = trace.file.lock() { + let _ = writeln!(file, "{line}"); + let _ = file.flush(); + } +} diff --git a/zebra-state/src/service/finalized_state.rs b/zebra-state/src/service/finalized_state.rs index 01f2d2fba7d..4d99af2a431 100644 --- a/zebra-state/src/service/finalized_state.rs +++ b/zebra-state/src/service/finalized_state.rs @@ -134,6 +134,8 @@ pub struct AssembledCommit { note_commitment_trees: NoteCommitmentTrees, /// The retention plan, needed for post-flush archive-backlog bookkeeping. retention: RetentionPlan, + /// Commit-pressure trace metadata prepared during batch assembly. + commit_trace: PreparedCommitTrace, /// This block's run-ahead pipeline contribution, captured only when the /// committer is running ahead (`None` on the synchronous path). The committer /// records it into the [`FinalizedPipeline`](pipeline::FinalizedPipeline) @@ -164,10 +166,14 @@ pub(crate) struct PipelineAssembled { pub height: block::Height, /// The note-commitment trees after this block, threaded to the next block. pub note_commitment_trees: NoteCommitmentTrees, + /// Commit-pressure trace metadata prepared during batch assembly. + pub commit_trace: PreparedCommitTrace, } pub mod column_family; +mod commit_pressure; +pub(crate) use commit_pressure::PreparedCommitTrace; mod commitment_aux; mod commitment_aux_verify; mod disk_db; @@ -918,6 +924,7 @@ impl FinalizedState { height, note_commitment_trees, retention, + commit_trace, pipeline_contribution, // Elasticsearch indexing is skipped on the run-ahead path. .. @@ -937,6 +944,7 @@ impl FinalizedState { hash, height, note_commitment_trees, + commit_trace, }) } @@ -1012,6 +1020,11 @@ impl FinalizedState { overlay: Option<&pipeline::FinalizedPipeline>, source: &str, ) -> Result { + // VCT commitment-root verification time (the `verify_commitment_roots` fold), + // set inside the checkpoint match arm below and stamped onto the commit trace + // after assembly. It runs before `assemble_block_batch`'s `write_start`, so it + // is CPU not otherwise counted in `commit_total`. `ZERO` for legacy commits. + let mut fold_dur = std::time::Duration::ZERO; let ( height, hash, @@ -1121,6 +1134,7 @@ impl FinalizedState { // Verifies this block's own header, folds its supplied roots into // the candidate tree, and when buffered checks the successor header // against that candidate (the one-block lag). + let fold_start = std::time::Instant::now(); let candidate = COMMIT_COMPUTE_POOL .install(|| { commitment_aux_verify::verify_commitment_roots( @@ -1133,6 +1147,10 @@ impl FinalizedState { self.vct_prevalidated_next = None; self.vct_reject_supplied_root(height, error) })?; + fold_dur = fold_start.elapsed(); + #[cfg(feature = "commit-metrics")] + metrics::histogram!("zebra.state.write.vct_verify.duration_seconds") + .record(fold_dur.as_secs_f64()); if let Some((next_block, _next_auth)) = &next_checkpoint { self.vct_prevalidated_next = Some(( @@ -1399,7 +1417,7 @@ impl FinalizedState { let network = self.network(); // `assemble_block_batch` also returns `finalized.hash`, which equals the // `hash` already destructured above; keep the outer binding and ignore it. - let (batch, _hash, batch_contribution) = self.db.assemble_block_batch( + let (batch, _hash, batch_contribution, mut commit_trace) = self.db.assemble_block_batch( finalized, prev_note_commitment_trees, &network, @@ -1408,6 +1426,9 @@ impl FinalizedState { fast_sync_below, overlay, )?; + // The fold runs before `assemble_block_batch`'s `write_start`, so record it on + // the trace here (it is CPU not otherwise visible in `commit_total`). + commit_trace.fold = fold_dur; // When running ahead, fold this block's batch contribution together with its // tip cursor and trees into the contribution the committer records into the @@ -1429,6 +1450,7 @@ impl FinalizedState { height, note_commitment_trees, retention, + commit_trace, pipeline_contribution, #[cfg(feature = "elasticsearch")] finalized_block: finalized_inner_block, @@ -1454,6 +1476,7 @@ impl FinalizedState { height, note_commitment_trees, retention, + commit_trace, // Recorded into the pipeline by the committer before the flush, so the // disk-writer side ignores it here. pipeline_contribution: _, @@ -1461,7 +1484,7 @@ impl FinalizedState { finalized_block, } = assembled; - self.db.flush_block_batch(batch, source); + self.db.flush_block_batch(batch, commit_trace, source); self.pipeline_post_assemble(height, retention); diff --git a/zebra-state/src/service/finalized_state/commit_pressure.rs b/zebra-state/src/service/finalized_state/commit_pressure.rs new file mode 100644 index 00000000000..921c6fade37 --- /dev/null +++ b/zebra-state/src/service/finalized_state/commit_pressure.rs @@ -0,0 +1,157 @@ +//! Optional per-commit RocksDB pressure trace (off by default). +//! +//! When `ZEBRA_COMMIT_PRESSURE_TRACE=` is set, every block commit whose +//! `batch_commit` write takes at least `ZEBRA_COMMIT_PRESSURE_MS` (default 50 ms) +//! appends one JSON row to ``, pairing the commit latency with the RocksDB +//! internal pressure sampled right after the write (L0 files, pending-compaction +//! bytes, running compactions/flushes, memtable + SST bytes). This lets an offline +//! analysis line up commit-latency spikes with compaction/flush pressure. +//! +//! The hot path is a single `OnceLock` load when the env var is unset (the +//! production default), so this stays inert unless explicitly enabled — e.g. by +//! the offline replay bench, which sets the env var for a run. + +use std::{ + fs::File, + io::{BufWriter, Write}, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, Mutex, OnceLock, + }, + time::{Duration, Instant}, +}; + +use zebra_chain::{block::Block, serialization::ZcashSerialize}; + +use super::disk_db::DiskDb; + +struct CommitPressureTrace { + file: Mutex>, + threshold: Duration, + /// Also emit a baseline row every `every` commits (0 = only slow commits), so the + /// trace carries a continuous compaction-pressure curve, not just the spikes. + every: u64, + count: AtomicU64, + start: Instant, +} + +static TRACE: OnceLock> = OnceLock::new(); + +fn trace() -> Option<&'static CommitPressureTrace> { + TRACE + .get_or_init(|| { + let path = std::env::var_os("ZEBRA_COMMIT_PRESSURE_TRACE")?; + let threshold_ms = std::env::var("ZEBRA_COMMIT_PRESSURE_MS") + .ok() + .and_then(|s| s.parse::().ok()) + .unwrap_or(50); + let every = std::env::var("ZEBRA_COMMIT_PRESSURE_EVERY") + .ok() + .and_then(|s| s.parse::().ok()) + .unwrap_or(0); + match File::create(&path) { + Ok(file) => { + tracing::info!(?path, threshold_ms, every, "commit-pressure trace enabled"); + Some(CommitPressureTrace { + file: Mutex::new(BufWriter::new(file)), + threshold: Duration::from_millis(threshold_ms), + every, + count: AtomicU64::new(0), + start: Instant::now(), + }) + } + Err(error) => { + tracing::warn!(?path, %error, "could not open commit-pressure trace"); + None + } + } + }) + .as_ref() +} + +/// Per-block timings of the committer's instrumented phases, for the trace row. +pub(crate) struct PreparedCommitTrace { + pub(crate) height: u32, + pub(crate) block: Arc, + pub(crate) tx_count: usize, + pub(crate) output_count: usize, + pub(crate) batch_keys: usize, + pub(crate) batch_bytes: usize, + pub(crate) reads: Duration, + pub(crate) address_reads: Duration, + pub(crate) batch_assembly: Duration, + /// VCT commitment-root verification (`verify_commitment_roots`) — CPU, runs + /// before `write_start`, so it is *not* part of `commit_total`. Set by the + /// caller after assembly. `ZERO` for legacy (non-VCT-fast) commits. + pub(crate) fold: Duration, + /// Self-time of `assemble_block_batch` (`write_start` → trace build): the + /// read/compute half's CPU, excluding the pipeline queue-wait and disk flush. + pub(crate) assemble_self: Duration, + pub(crate) write_start: Instant, +} + +/// Records one commit-pressure row, when the trace is enabled and the commit was +/// at least the configured threshold (or on the periodic baseline tick). +/// +/// A cheap no-op (one `OnceLock` load) when `ZEBRA_COMMIT_PRESSURE_TRACE` is unset. +pub(super) fn record_commit(db: &DiskDb, prepared: PreparedCommitTrace, batch_commit: Duration) { + let Some(trace) = trace() else { return }; + let n = trace.count.fetch_add(1, Ordering::Relaxed); + let slow = batch_commit >= trace.threshold; + let periodic = trace.every > 0 && n % trace.every == 0; + if !slow && !periodic { + return; + } + + // Sample RocksDB pressure right after the write, and the block size (re-serialized + // here; only on the slow-commit / periodic-sample path, so it stays off the hot path). + let p = db.pressure_snapshot(); + let block_bytes = prepared.block.zcash_serialized_size(); + // Full per-block decomposition. `commit_total` spans `write_start` (assemble + // start) → here (after the disk flush), so it = assemble_self + queue_wait + + // batch_commit. `fold` is the VCT commitment-root verify, which runs *before* + // `write_start` and is therefore additional CPU not counted in `commit_total`. + let commit_total = prepared.write_start.elapsed(); + let assemble_self = prepared.assemble_self; + let queue_wait = commit_total + .saturating_sub(assemble_self) + .saturating_sub(batch_commit); + let line = format!( + concat!( + r#"{{"ts_us":{},"height":{},"slow":{},"fold_ms":{:.3},"reads_ms":{:.3},"address_reads_ms":{:.3},"#, + r#""batch_assembly_ms":{:.3},"assemble_self_ms":{:.3},"queue_wait_ms":{:.3},"batch_commit_ms":{:.3},"commit_total_ms":{:.3},"#, + r#""block_bytes":{},"tx_count":{},"output_count":{},"batch_keys":{},"batch_bytes":{},"#, + r#""l0_files":{},"pending_compaction_bytes":{},"running_compactions":{},"#, + r#""running_flushes":{},"memtable_bytes":{},"total_sst_bytes":{},"live_data_bytes":{}}}"# + ), + trace.start.elapsed().as_micros(), + prepared.height, + slow, + prepared.fold.as_secs_f64() * 1000.0, + prepared.reads.as_secs_f64() * 1000.0, + prepared.address_reads.as_secs_f64() * 1000.0, + prepared.batch_assembly.as_secs_f64() * 1000.0, + assemble_self.as_secs_f64() * 1000.0, + queue_wait.as_secs_f64() * 1000.0, + batch_commit.as_secs_f64() * 1000.0, + commit_total.as_secs_f64() * 1000.0, + block_bytes, + prepared.tx_count, + prepared.output_count, + prepared.batch_keys, + prepared.batch_bytes, + p.l0_files, + p.pending_compaction_bytes, + p.running_compactions, + p.running_flushes, + p.memtable_bytes, + p.total_sst_bytes, + p.live_data_bytes, + ); + + if let Ok(mut file) = trace.file.lock() { + // Flush each row so a killed/partial run is still inspectable. + let _ = writeln!(file, "{line}"); + let _ = file.flush(); + } +} diff --git a/zebra-state/src/service/finalized_state/disk_db.rs b/zebra-state/src/service/finalized_state/disk_db.rs index 49261aa4e93..b2d6e24e029 100644 --- a/zebra-state/src/service/finalized_state/disk_db.rs +++ b/zebra-state/src/service/finalized_state/disk_db.rs @@ -124,9 +124,10 @@ pub struct DiskWriteBatch { batch: rocksdb::WriteBatch, } -#[cfg(feature = "commit-metrics")] impl DiskWriteBatch { /// Returns the serialized size of this pending RocksDB write batch. + // Always compiled: the commit-pressure trace (`PreparedCommitTrace`) reads it + // independently of the `commit-metrics` feature. pub(crate) fn size_in_bytes(&self) -> usize { self.batch.size_in_bytes() } @@ -545,6 +546,25 @@ impl ReadDisk for DiskDb { } } +/// A point-in-time snapshot of RocksDB compaction/flush pressure, summed across +/// column families, sampled right after a commit for [`super::commit_pressure`]. +pub(super) struct PressureSnapshot { + /// Total SST files sitting at L0 across all CFs (the write-stall trigger). + pub(super) l0_files: u64, + /// Estimated bytes of pending compaction work across all CFs. + pub(super) pending_compaction_bytes: u64, + /// Compactions currently running (database-wide). + pub(super) running_compactions: u64, + /// Memtable flushes currently running (database-wide). + pub(super) running_flushes: u64, + /// Total memtable (mutable + immutable) bytes across all CFs. + pub(super) memtable_bytes: u64, + /// Total on-disk SST bytes across all CFs. + pub(super) total_sst_bytes: u64, + /// Estimated live (post-compaction) data bytes across all CFs. + pub(super) live_data_bytes: u64, +} + impl DiskWriteBatch { /// Creates and returns a new transactional batch write. /// @@ -558,6 +578,16 @@ impl DiskWriteBatch { batch: rocksdb::WriteBatch::default(), } } + + /// Returns the number of operations (puts + deletes) queued in this batch. + pub fn len(&self) -> usize { + self.batch.len() + } + + /// Returns whether this batch has no queued operations. + pub fn is_empty(&self) -> bool { + self.batch.is_empty() + } } impl DiskDb { @@ -687,6 +717,48 @@ impl DiskDb { } } + /// Samples RocksDB compaction/flush pressure across all column families, for the + /// optional per-commit pressure trace ([`super::commit_pressure`]). Reads only + /// in-memory RocksDB properties (no disk I/O); called on the rare slow-commit path. + pub(super) fn pressure_snapshot(&self) -> PressureSnapshot { + let db: &Arc = &self.db; + let db_options = DiskDb::options(); + + let mut l0_files = 0; + let mut pending_compaction_bytes = 0; + let mut memtable_bytes = 0; + let mut total_sst_bytes = 0; + let mut live_data_bytes = 0; + + for cf_descriptor in DiskDb::construct_column_families(db_options, db.path(), []) { + let Some(cf_handle) = db.cf_handle(cf_descriptor.name()) else { + continue; + }; + let cf_u64 = |prop: &str| { + db.property_int_value_cf(cf_handle, prop) + .ok() + .flatten() + .unwrap_or(0) + }; + l0_files += cf_u64("rocksdb.num-files-at-level0"); + pending_compaction_bytes += cf_u64("rocksdb.estimate-pending-compaction-bytes"); + memtable_bytes += cf_u64("rocksdb.size-all-mem-tables"); + total_sst_bytes += cf_u64("rocksdb.total-sst-files-size"); + live_data_bytes += cf_u64("rocksdb.estimate-live-data-size"); + } + + let db_u64 = |prop: &str| db.property_int_value(prop).ok().flatten().unwrap_or(0); + PressureSnapshot { + l0_files, + pending_compaction_bytes, + running_compactions: db_u64("rocksdb.num-running-compactions"), + running_flushes: db_u64("rocksdb.num-running-flushes"), + memtable_bytes, + total_sst_bytes, + live_data_bytes, + } + } + /// Returns the estimated total disk space usage of the database. pub fn size(&self) -> u64 { let db: &Arc = &self.db; diff --git a/zebra-state/src/service/finalized_state/tests/prop.rs b/zebra-state/src/service/finalized_state/tests/prop.rs index 4be16ce508f..b7fd0987cc1 100644 --- a/zebra-state/src/service/finalized_state/tests/prop.rs +++ b/zebra-state/src/service/finalized_state/tests/prop.rs @@ -245,7 +245,11 @@ fn pipelined_commit_matches_synchronous() -> Result<()> { let height = assembled.height; pipe_state .db - .flush_block_batch(assembled.batch, "pipelined_commit_matches_synchronous flush"); + .flush_block_batch( + assembled.batch, + assembled.commit_trace, + "pipelined_commit_matches_synchronous flush", + ); pipeline.retire_through(height); } diff --git a/zebra-state/src/service/finalized_state/zebra_db/block.rs b/zebra-state/src/service/finalized_state/zebra_db/block.rs index f2b80bb5058..e61933e3c61 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block.rs @@ -1041,7 +1041,7 @@ impl ZebraDb { vct_anchor_roots: Option<(sapling::tree::Root, orchard::tree::Root)>, vct_sync_below: Option, ) -> Result { - let (batch, hash, _contribution) = self.assemble_block_batch( + let (batch, hash, _contribution, commit_trace) = self.assemble_block_batch( finalized, prev_note_commitment_trees, network, @@ -1050,7 +1050,7 @@ impl ZebraDb { vct_sync_below, None, )?; - self.flush_block_batch(batch, source); + self.flush_block_batch(batch, commit_trace, source); Ok(hash) } @@ -1091,9 +1091,11 @@ impl ZebraDb { DiskWriteBatch, block::Hash, Option, + super::super::commit_pressure::PreparedCommitTrace, ), CommitCheckpointVerifiedError, > { + let write_start = std::time::Instant::now(); let tx_hash_indexes: HashMap = finalized .transaction_hashes .iter() @@ -1151,7 +1153,6 @@ impl ZebraDb { &finalized.new_outputs, ) }; - #[cfg(feature = "commit-metrics")] let spent_reads_start = std::time::Instant::now(); let (spent_utxos, precomputed_raw_txs): ( Vec<(transparent::OutPoint, OutputLocation, transparent::Utxo)>, @@ -1181,9 +1182,10 @@ impl ZebraDb { } }, ); + let reads_dur = spent_reads_start.elapsed(); #[cfg(feature = "commit-metrics")] metrics::histogram!("zebra.state.write.spent_utxo_reads.duration_seconds") - .record(spent_reads_start.elapsed().as_secs_f64()); + .record(reads_dur.as_secs_f64()); let spent_utxos_by_outpoint: HashMap = spent_utxos @@ -1261,7 +1263,6 @@ impl ZebraDb { } self.address_balance_location(addr) }; - #[cfg(feature = "commit-metrics")] let address_reads_start = std::time::Instant::now(); let address_balances: AddressBalanceLocationUpdates = if self.finished_format_upgrades() { AddressBalanceLocationUpdates::Insert(read_addr_locs(changed_addresses, |addr| { @@ -1272,9 +1273,10 @@ impl ZebraDb { Some(lookup_balance(addr)?.into_new_change()) })) }; + let address_reads_dur = address_reads_start.elapsed(); #[cfg(feature = "commit-metrics")] metrics::histogram!("zebra.state.write.address_reads.duration_seconds") - .record(address_reads_start.elapsed().as_secs_f64()); + .record(address_reads_dur.as_secs_f64()); // The value pool and `vct_upgrade_height` marker are threaded forward in // memory by the run-ahead pipeline; with `overlay = None` they come from @@ -1288,6 +1290,7 @@ impl ZebraDb { .unwrap_or(false); let capture_pipeline_outputs = overlay.is_some(); + let assembly_start = std::time::Instant::now(); let mut batch = DiskWriteBatch::new(); // In case of errors, propagate and do not write the batch. @@ -1320,6 +1323,7 @@ impl ZebraDb { // block commit path. In archive mode the plan is always `Store`, so this // is a no-op. retention.prepare_prune(&mut batch, self, &finalized); + let batch_assembly_dur = assembly_start.elapsed(); #[cfg(feature = "commit-metrics")] { metrics::histogram!("zebra.state.write.batch_prep.duration_seconds") @@ -1360,7 +1364,26 @@ impl ZebraDb { } }); - Ok((batch, finalized.hash, contribution)) + let commit_trace = super::super::commit_pressure::PreparedCommitTrace { + height: finalized.height.0, + block: finalized.block.clone(), + tx_count: finalized.transaction_hashes.len(), + output_count: finalized.new_outputs.len(), + batch_keys: batch.len(), + batch_bytes: batch.size_in_bytes(), + reads: reads_dur, + address_reads: address_reads_dur, + batch_assembly: batch_assembly_dur, + // The VCT fold runs before `assemble_block_batch`; the caller + // (`assemble_finalized_direct`) sets this after it returns. + fold: std::time::Duration::ZERO, + // Self-time of the read/compute half (excludes the queue-wait + flush + // that `commit_total` also captures). + assemble_self: write_start.elapsed(), + write_start, + }; + + Ok((batch, finalized.hash, contribution, commit_trace)) } /// Flush a previously [`assemble_block_batch`](Self::assemble_block_batch)d @@ -1370,14 +1393,32 @@ impl ZebraDb { /// on the dedicated disk-writer thread so it overlaps the next block's /// assembly; in tip mode it runs inline. A rocksdb write failure is fatal, as /// before. - pub(crate) fn flush_block_batch(&self, batch: DiskWriteBatch, source: &str) { + pub(crate) fn flush_block_batch( + &self, + batch: DiskWriteBatch, + commit_trace: super::super::commit_pressure::PreparedCommitTrace, + source: &str, + ) { // Track batch commit latency for observability let batch_start = std::time::Instant::now(); self.db .write(batch) .expect("unexpected rocksdb error while writing block"); + let batch_commit = batch_start.elapsed(); metrics::histogram!("zebra.state.rocksdb.batch_commit.duration_seconds") - .record(batch_start.elapsed().as_secs_f64()); + .record(batch_commit.as_secs_f64()); + let total_dur = commit_trace.write_start.elapsed(); + // Optional per-slow-commit pressure row (inert unless ZEBRA_COMMIT_PRESSURE_TRACE + // is set): pairs this commit's latency with RocksDB L0/compaction/flush state. + // Timed separately because, when enabled, it re-serializes the block and samples + // RocksDB stats on the disk-writer's critical path (a Heisenberg cost to subtract). + let record_start = std::time::Instant::now(); + super::super::commit_pressure::record_commit(&self.db, commit_trace, batch_commit); + metrics::histogram!("zebra.state.write.commit_trace_record.duration_seconds") + .record(record_start.elapsed().as_secs_f64()); + + metrics::histogram!("zebra.state.write.total.duration_seconds") + .record(total_dur.as_secs_f64()); tracing::trace!(?source, "committed block from"); } @@ -1420,6 +1461,42 @@ impl ZebraDb { error: error.to_string(), }) } + + /// Seeds the Zakura header store with frontier header rows for a contiguous run + /// of blocks whose bodies have **not** been committed (heights strictly above the + /// body tip), writing in chunked batches. + /// + /// Offline-bench only: this stands in for what header sync would persist, so the + /// header-authenticated checkpoint fast path (`authenticated_checkpoint_hash`) can + /// run against a snapshot with no live header sync. Each row is independent and the + /// insert is idempotent, so callers may seed in any order. + pub fn seed_zakura_headers_from_blocks( + &self, + blocks: impl IntoIterator)>, + ) -> Result<(), CommitHeaderRangeError> { + let write = |batch: DiskWriteBatch| { + self.db + .write(batch) + .map_err(|error| CommitHeaderRangeError::StorageWriteError { + error: error.to_string(), + }) + }; + + let mut batch = DiskWriteBatch::new(); + let mut pending = 0usize; + for (height, block) in blocks { + batch.prepare_zakura_header_from_committed_block(&self.db, height, &block)?; + pending += 1; + if pending >= 2000 { + write(std::mem::take(&mut batch))?; + pending = 0; + } + } + if pending > 0 { + write(batch)?; + } + Ok(()) + } } /// Read a spent transparent UTXO and its output location before deleting it from the database. diff --git a/zebra-state/src/service/tests.rs b/zebra-state/src/service/tests.rs index f17d585961b..bf0a2a230c6 100644 --- a/zebra-state/src/service/tests.rs +++ b/zebra-state/src/service/tests.rs @@ -284,10 +284,7 @@ async fn pipelined_state_service_commits_chain() -> Result<()> { committed, expected, "every queued block should commit through the pipeline" ); - assert!( - expected >= 2, - "the test chain must have at least two blocks" - ); + assert!(expected >= 2, "the test chain must have at least two blocks"); Ok(()) } diff --git a/zebra-state/src/service/write.rs b/zebra-state/src/service/write.rs index e519eb1eddb..5e7939bd88a 100644 --- a/zebra-state/src/service/write.rs +++ b/zebra-state/src/service/write.rs @@ -30,7 +30,8 @@ use crate::{ service::{ check, finalized_state::{ - spawn_note_precompute, DiskWriteBatch, FinalizedPipeline, FinalizedState, ZebraDb, + spawn_note_precompute, DiskWriteBatch, FinalizedPipeline, FinalizedState, + PreparedCommitTrace, ZebraDb, }, non_finalized_state::NonFinalizedState, queued_blocks::{QueuedCheckpointVerified, QueuedSemanticallyVerified}, @@ -65,6 +66,7 @@ type PendingPrecompute = ( /// the overlay. struct PipelineFlush { batch: DiskWriteBatch, + commit_trace: PreparedCommitTrace, hash: block::Hash, height: Height, rsp_tx: oneshot::Sender>, @@ -81,21 +83,42 @@ fn run_finalized_writer( batch_receiver: std::sync::mpsc::Receiver, ack_sender: std::sync::mpsc::Sender, ) { - while let Ok(PipelineFlush { - batch, - hash, - height, - rsp_tx, - }) = batch_receiver.recv() - { - db.flush_block_batch(batch, "commit checkpoint-verified request (pipelined)"); + loop { + // Time blocked waiting for the next batch: ~0 means the disk-writer is the + // bottleneck (a full queue feeding it), high means it is starved (the + // assembler upstream is the bottleneck). + let recv_start = std::time::Instant::now(); + let Ok(PipelineFlush { + batch, + commit_trace, + hash, + height, + rsp_tx, + }) = batch_receiver.recv() + else { + break; + }; + metrics::histogram!("zebra.state.write.disk_writer_recv_wait.duration_seconds") + .record(recv_start.elapsed().as_secs_f64()); + + // Full per-block service time on the disk-writer (flush + acks): the inverse + // of this is the disk-writer's max throughput. + let service_start = std::time::Instant::now(); + db.flush_block_batch( + batch, + commit_trace, + "commit checkpoint-verified request (pipelined)", + ); // Ack-after-flush: the commit response only resolves once the block is // durable on disk. let _ = rsp_tx.send(Ok(hash)); // If the assembler has gone away (shutdown), stop. - if ack_sender.send(height).is_err() { + let stop = ack_sender.send(height).is_err(); + metrics::histogram!("zebra.state.write.disk_writer_service.duration_seconds") + .record(service_start.elapsed().as_secs_f64()); + if stop { break; } } @@ -563,7 +586,15 @@ impl WriteBlockWorkerTask { None => match finalized_block_write_receiver.try_recv() { Ok(block) => block, Err(TryRecvError::Empty) => { + // Starved: no finalized block available. Time the park so we can + // tell "assembler is upstream-starved (verify/driver/feed slow)" + // apart from "assembler is the bottleneck". + let park = std::time::Instant::now(); std::thread::park_timeout(Duration::from_millis(10)); + metrics::histogram!( + "zebra.state.write.assembler_empty_park.duration_seconds" + ) + .record(park.elapsed().as_secs_f64()); continue; } Err(TryRecvError::Disconnected) => break, @@ -583,6 +614,7 @@ impl WriteBlockWorkerTask { // So if there has been a block commit error, // we need to drop all the descendants of that block, // until we receive a block at the required next height. + // // The next block must be the child of the current tip. In the run-ahead // pipeline the assembler runs *ahead* of the durable disk tip (ack-after- // flush), so the next valid height follows the in-memory assembled tip @@ -641,7 +673,17 @@ impl WriteBlockWorkerTask { "VCT: deferring fast checkpoint commit until successor is buffered" ); retry_finalized_block = Some(ordered_block); + // VCT one-block look-ahead stall: this fast block can't commit until its + // successor is buffered (needed to authenticate its supplied roots). If + // this dominates, the 10ms poll granularity (not real work) is the gate. + let park = std::time::Instant::now(); std::thread::park_timeout(Duration::from_millis(10)); + metrics::counter!("zebra.state.write.assembler_successor_defer.count") + .increment(1); + metrics::histogram!( + "zebra.state.write.assembler_successor_park.duration_seconds" + ) + .record(park.elapsed().as_secs_f64()); continue; } @@ -720,6 +762,7 @@ impl WriteBlockWorkerTask { .expect("flush sender is present while the pipeline is active") .send(PipelineFlush { batch: assembled.batch, + commit_trace: assembled.commit_trace, hash: assembled.hash, height: assembled.height, rsp_tx, From 1154fd47b625717b78302cbce3fc15f61ad7df20 Mon Sep 17 00:00:00 2001 From: Roman Date: Mon, 29 Jun 2026 21:50:49 +0000 Subject: [PATCH 05/17] feat(replay-bench): drive apply-sequencer through the production block-sync driver Replace the hand-rolled verify/commit loop in apply-sequencer with the real Zakura block-sync apply driver (drive_block_sync_actions), exposed from zebrad behind a new internal-bench feature via zebrad::bench_api. The bench feeds cached bodies into the real sequencer; the production driver applies them through the consensus router into state using the header-authenticated checkpoint fast path, and the bench gates on the sequencer's verified tip. - zebra-network: BenchSequencerHandle::into_driver_parts + BenchDriverParts (inert BlockSyncHandle + BlockApplyFinished->ApplyFinished shim) and BenchCommitter::wait_for_verified_tip, behind internal-bench. - zebrad: internal-bench feature; bench_api re-exports the (unmodified) driver and BlocksyncThroughputProbe; mod zakura widened to pub(crate). - zebra-replay-bench: rewire apply-sequencer; add a seed-headers subcommand (one-time Zakura header-store seeding of the base); depend on zebrad. - Harness/docs: point at the cleanly-pruned 1.85M base + 1849958..1899957 window; add the commit/verify-timing plot script. --- .cursor/skills/zakura-trace-plots/SKILL.md | 14 + .../scripts/plot_commit_verify_timing.py | 285 +++++++++++++++++ Cargo.lock | 2 + Makefile | 2 +- deploy/runner/replay_run.sh | 21 +- make/perf.mk | 1 + zebra-network/src/zakura/block_sync/bench.rs | 107 ++++++- zebra-network/src/zakura/block_sync/mod.rs | 4 +- zebra-replay-bench/Cargo.toml | 4 + zebra-replay-bench/README.md | 20 +- zebra-replay-bench/RESULTS.md | 13 +- zebra-replay-bench/src/apply_sequencer.rs | 300 +++++++++++------- zebra-replay-bench/src/config.rs | 10 +- zebra-replay-bench/src/main.rs | 33 +- zebra-replay-bench/src/prefetch.rs | 51 +++ zebra-replay-bench/src/seed_headers.rs | 100 ++++++ zebrad/Cargo.toml | 5 + zebrad/src/commands/start.rs | 4 +- .../start/zakura/block_sync_driver.rs | 10 +- zebrad/src/commands/start/zakura/mod.rs | 2 +- .../commands/start/zakura/throughput_probe.rs | 2 +- zebrad/src/lib.rs | 9 + 22 files changed, 849 insertions(+), 150 deletions(-) create mode 100644 .cursor/skills/zakura-trace-plots/scripts/plot_commit_verify_timing.py create mode 100644 zebra-replay-bench/src/seed_headers.rs diff --git a/.cursor/skills/zakura-trace-plots/SKILL.md b/.cursor/skills/zakura-trace-plots/SKILL.md index 24d1e9fa1c0..b4c51defea6 100644 --- a/.cursor/skills/zakura-trace-plots/SKILL.md +++ b/.cursor/skills/zakura-trace-plots/SKILL.md @@ -29,6 +29,20 @@ Default outputs: Use the time plot for stall diagnosis. Height plots collapse zero-progress stalls onto one x-position. +When the user is analyzing the verify -> commit pipeline and provides per-block +timing traces such as `seq50k-commit-timing.jsonl` and +`seq50k-verify-timing.jsonl`, also generate the by-height timing plot: + +```bash +python3 .cursor/skills/zakura-trace-plots/scripts/plot_commit_verify_timing.py \ + COMMIT_TIMING.jsonl VERIFY_TIMING.jsonl --out-dir perf-artifacts +``` + +This graph is useful for separating verifier cost from committer cost. It plots +commit phases (`spent_utxo_reads`, `address_reads`, `batch_assembly`, +`batch_commit`), commit total latency, verifier phases (`pow`, `precompute`, +`merkle`), and rolling implied throughput from each timing stream. + ## Metrics Awareness Use `block_sync.jsonl` as the source of truth for: diff --git a/.cursor/skills/zakura-trace-plots/scripts/plot_commit_verify_timing.py b/.cursor/skills/zakura-trace-plots/scripts/plot_commit_verify_timing.py new file mode 100644 index 00000000000..d0867999d9c --- /dev/null +++ b/.cursor/skills/zakura-trace-plots/scripts/plot_commit_verify_timing.py @@ -0,0 +1,285 @@ +#!/usr/bin/env python3 +"""Plot Zebra replay-bench commit + verify timing traces by height.""" + +from __future__ import annotations + +import argparse +import html +import json +import math +from pathlib import Path + + +def load_jsonl(path: Path) -> list[dict[str, object]]: + return [json.loads(line) for line in path.open() if line.strip()] + + +def number(row: dict[str, object], key: str) -> float: + try: + return float(row.get(key, 0) or 0) + except (TypeError, ValueError): + return 0.0 + + +def height(row: dict[str, object]) -> int: + return int(row["height"]) + + +def percentile(values: list[float], percent: float) -> float: + values = sorted(value for value in values if math.isfinite(value)) + if not values: + return 1.0 + + index = (len(values) - 1) * percent / 100 + low = math.floor(index) + high = math.ceil(index) + if low == high: + return values[low] + + return values[low] * (high - index) + values[high] * (index - low) + + +def nice_max(values: list[float]) -> float: + maximum = max([value for value in values if math.isfinite(value)] or [1.0]) + if maximum <= 0: + return 1.0 + + exponent = math.floor(math.log10(maximum)) + base = 10**exponent + for multiplier in (1, 2, 5, 10): + if maximum <= multiplier * base: + return multiplier * base + + return 10 * base + + +def rolling_blocks_per_second( + rows: list[dict[str, object]], + window: int, +) -> tuple[list[int], list[float]]: + xs: list[int] = [] + ys: list[float] = [] + + for index in range(0, len(rows) - window): + start = rows[index] + end = rows[index + window] + height_delta = height(end) - height(start) + time_delta = (number(end, "ts_us") - number(start, "ts_us")) / 1_000_000 + + if height_delta > 0 and time_delta > 0: + xs.append(height(end)) + ys.append(height_delta / time_delta) + + return xs, ys + + +def polyline( + xs: list[int], + ys: list[float], + *, + min_height: int, + max_height: int, + max_value: float, + left: int, + width: int, + right: int, + panel_top: int, + panel_height: int, +) -> str: + plot_height = panel_height - 45 + points = [] + + for x_value, y_value in zip(xs, ys): + clamped = min(max(y_value, 0.0), max_value) + x = left + (x_value - min_height) / (max_height - min_height) * (width - left - right) + y = panel_top + 25 + plot_height - (clamped / max_value) * plot_height + points.append(f"{x:.1f},{y:.1f}") + + return " ".join(points) + + +def write_svg( + commit_rows: list[dict[str, object]], + verify_rows: list[dict[str, object]], + out_path: Path, + *, + title: str, + window: int, +) -> None: + commit_heights = [height(row) for row in commit_rows] + verify_heights = [height(row) for row in verify_rows] + min_height = min(commit_heights + verify_heights) + max_height = max(commit_heights + verify_heights) + if min_height == max_height: + max_height += 1 + + commit_phase_heights = commit_heights + verify_phase_heights = verify_heights + commit_stream_heights, commit_bps = rolling_blocks_per_second(commit_rows, window) + verify_stream_heights, verify_bps = rolling_blocks_per_second(verify_rows, window) + + panels = [ + ( + "Commit phases (ms, y capped at p99.5)", + 99.5, + [ + ("spent_utxo_reads", commit_phase_heights, [number(row, "reads_ms") for row in commit_rows], "#1f77b4"), + ("address_reads", commit_phase_heights, [number(row, "address_reads_ms") for row in commit_rows], "#ff7f0e"), + ("batch_assembly", commit_phase_heights, [number(row, "batch_assembly_ms") for row in commit_rows], "#2ca02c"), + ("batch_commit", commit_phase_heights, [number(row, "batch_commit_ms") for row in commit_rows], "#d62728"), + ], + ), + ( + "Commit total (ms, y capped at p99.5)", + 99.5, + [ + ("commit_total", commit_phase_heights, [number(row, "commit_total_ms") for row in commit_rows], "#111111"), + ], + ), + ( + "Verify phases (ms, y capped at p99.5)", + 99.5, + [ + ("pow", verify_phase_heights, [number(row, "pow_ms") for row in verify_rows], "#9467bd"), + ("precompute", verify_phase_heights, [number(row, "precompute_ms") for row in verify_rows], "#8c564b"), + ("merkle", verify_phase_heights, [number(row, "merkle_ms") for row in verify_rows], "#17becf"), + ], + ), + ( + "Rolling throughput from timing timestamps (blk/s)", + 100.0, + [ + ("commit stream", commit_stream_heights, commit_bps, "#111111"), + ("verify stream", verify_stream_heights, verify_bps, "#2ca02c"), + ], + ), + ] + + width = 1500 + panel_height = 230 + left = 80 + right = 30 + top = 58 + gap = 52 + bottom = 45 + svg_height = top + len(panels) * panel_height + (len(panels) - 1) * gap + bottom + + svg: list[str] = [ + f'', + '', + '', + f'{html.escape(title)}', + ] + + for panel_index, (panel_title, cap_percentile, series) in enumerate(panels): + panel_top = top + panel_index * (panel_height + gap) + values = [value for _, _, ys, _ in series for value in ys] + max_value = nice_max([percentile(values, cap_percentile)]) + plot_height = panel_height - 45 + + svg.append( + f'' + f"{html.escape(panel_title)}; y_max={max_value:g}" + ) + + for fraction in (0.0, 0.25, 0.5, 0.75, 1.0): + y = panel_top + 25 + plot_height - fraction * plot_height + label = fraction * max_value + svg.append( + f'' + ) + svg.append(f'{label:.3g}') + + svg.append( + f'' + ) + svg.append( + f'' + ) + + legend_x = width - right - 260 + legend_y = panel_top + 15 + for series_index, (name, xs, ys, color) in enumerate(series): + if xs and ys: + points = polyline( + xs, + ys, + min_height=min_height, + max_height=max_height, + max_value=max_value, + left=left, + width=width, + right=right, + panel_top=panel_top, + panel_height=panel_height, + ) + svg.append(f'') + + y = legend_y + series_index * 18 + svg.append(f'') + svg.append(f'{html.escape(name)}') + + for fraction in (0.0, 0.25, 0.5, 0.75, 1.0): + x = left + fraction * (width - left - right) + value = int(min_height + fraction * (max_height - min_height)) + svg.append(f'{value}') + + svg.append(f'height') + svg.append("") + + out_path.write_text("\n".join(svg)) + + +def write_summary( + commit_rows: list[dict[str, object]], + verify_rows: list[dict[str, object]], + out_path: Path, + plot_path: Path, +) -> None: + out_path.write_text( + f"commit_samples: {len(commit_rows)}\n" + f"verify_samples: {len(verify_rows)}\n" + f"commit_height_range: {height(commit_rows[0])}-{height(commit_rows[-1])}\n" + f"verify_height_range: {height(verify_rows[0])}-{height(verify_rows[-1])}\n" + f"plot: {plot_path}\n" + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("commit_timing", type=Path, help="commit timing JSONL path") + parser.add_argument("verify_timing", type=Path, help="verify timing JSONL path") + parser.add_argument("--out-dir", type=Path, default=Path("perf-artifacts")) + parser.add_argument("--label", help="output file label; defaults to commit timing stem") + parser.add_argument("--window", type=int, default=50, help="sample window for rolling throughput") + args = parser.parse_args() + + commit_rows = load_jsonl(args.commit_timing) + verify_rows = load_jsonl(args.verify_timing) + if not commit_rows: + raise SystemExit(f"no rows in {args.commit_timing}") + if not verify_rows: + raise SystemExit(f"no rows in {args.verify_timing}") + + args.out_dir.mkdir(parents=True, exist_ok=True) + label = args.label or args.commit_timing.stem.removesuffix("-commit-timing") + plot_path = args.out_dir / f"{label}-commit-verify-by-height.svg" + summary_path = args.out_dir / f"{label}-commit-verify-summary.txt" + + write_svg( + commit_rows, + verify_rows, + plot_path, + title=f"{label} commit/verify timing by height", + window=max(args.window, 1), + ) + write_summary(commit_rows, verify_rows, summary_path, plot_path) + + print(plot_path) + print(summary_path) + + +if __name__ == "__main__": + main() diff --git a/Cargo.lock b/Cargo.lock index 0c25e01c9f1..00d62c2d75e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9324,7 +9324,9 @@ dependencies = [ "zebra-chain", "zebra-consensus", "zebra-network", + "zebra-node-services", "zebra-state", + "zebrad", ] [[package]] diff --git a/Makefile b/Makefile index 3d4fc069d46..d5f10b8d913 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ help: @echo " perf-replay-worker Replay through the real write worker (one rung up)" @echo " perf-replay-verifier Replay through the checkpoint verifier (adds PoW + Merkle)" @echo " perf-replay-sequencer Replay through the Zakura block-sync Sequencer (VCT-only," - @echo " pruned by default; REPLAY_ARCHIVE=1 for archive, REPLAY_TRACE_DIR= for traces)" + @echo " pruned by default; REPLAY_ARCHIVE=1 archive, REPLAY_STOP_HEIGHT= sub-range, REPLAY_TRACE_DIR= traces)" @echo " perf-analyze Bottleneck attribution over the CSV window (PERF_LABEL/PERF_LO/PERF_HI)" @echo " perf-dashboard Live metrics dashboard for the running bench node" @echo " perf-verify-isolation Confirm the bench sees only the two cohort peers" diff --git a/deploy/runner/replay_run.sh b/deploy/runner/replay_run.sh index 546ee9d9565..f624aa00100 100755 --- a/deploy/runner/replay_run.sh +++ b/deploy/runner/replay_run.sh @@ -28,11 +28,16 @@ RUNNER_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" DBREL="${BENCH_DB_REL:-state/v27/mainnet}" SRC="${REPLAY_SRC:-/mnt/roman-dev-2-data/zebra-cache}" -BASE_SRC="${REPLAY_BASE_SRC:-/mnt/roman-dev-2-data/zebra-ckpt-1800000-warm}" -CACHE="${REPLAY_CACHE:-/mnt/roman-dev-2-data/win-fwd.zrb}" -SIDECAR="${REPLAY_SIDECAR:-/mnt/roman-dev-2-data/win-fwd.vct}" -START="${REPLAY_START:-1802001}" -END="${REPLAY_END:-1832000}" +# Base = the cleanly-pruned snapshot at tip 1849957 (START-1). It was produced by +# replaying the older 1800000 pruned base forward past 1849957 so the checkpoint +# raw-tx archive backlog is already drained: forking it for a `run` does NOT +# trigger a startup DrainBacklog (the earlier 1800000-warm-pruned base did, which +# polluted the first ~10K blocks of every run with delete/compaction churn). +BASE_SRC="${REPLAY_BASE_SRC:-/mnt/roman-dev-2-data/zebra-ckpt-1850000-warm-pruned}" +CACHE="${REPLAY_CACHE:-/mnt/roman-dev-2-data/win-1850k.zrb}" +SIDECAR="${REPLAY_SIDECAR:-/mnt/roman-dev-2-data/win-1850k.vct}" +START="${REPLAY_START:-1849958}" +END="${REPLAY_END:-1899957}" FORK_DIR="${BENCH_FORK_DIR:-/mnt/roman-dev-2-data}" BIN_DEFAULT="${REPLAY_BIN:-/root/wal-bench/zebra-replay-bench}" @@ -158,6 +163,12 @@ cmd_run_sequencer() { else note "storage mode: pruned (default)" fi + # Clamp the committed window to a sub-range of the cache (stops at the last + # checkpoint <= REPLAY_STOP_HEIGHT). Lets a smaller window reuse a larger cache. + if [ -n "${REPLAY_STOP_HEIGHT:-}" ]; then + note "stop height: $REPLAY_STOP_HEIGHT" + args+=(--stop-height "$REPLAY_STOP_HEIGHT") + fi # Structured Zakura JSONL traces, like perf-run-mainnet's [network.zakura] trace_dir. if [ -n "${REPLAY_TRACE_DIR:-}" ]; then note "Zakura JSONL traces -> $REPLAY_TRACE_DIR" diff --git a/make/perf.mk b/make/perf.mk index f55226e626c..194a3ff0a34 100644 --- a/make/perf.mk +++ b/make/perf.mk @@ -114,6 +114,7 @@ perf-replay-verifier: # requires REPLAY_VCT_SIDECAR. Commits in Pruned storage mode by default (BASE_SRC must # be a pruned snapshot), matching the production mainnet config. Optional knobs: # REPLAY_ARCHIVE=1 commit in Archive storage mode instead (needs an archive base) +# REPLAY_STOP_HEIGHT= stop at the last checkpoint <= h (bench a sub-range of the cache) # REPLAY_TRACE_DIR= write structured Zakura JSONL traces (block_sync.jsonl), # the same tables perf-run-mainnet emits via trace_dir; plot # with .cursor/skills/zakura-trace-plots diff --git a/zebra-network/src/zakura/block_sync/bench.rs b/zebra-network/src/zakura/block_sync/bench.rs index ff191e41f23..47959004700 100644 --- a/zebra-network/src/zakura/block_sync/bench.rs +++ b/zebra-network/src/zakura/block_sync/bench.rs @@ -37,18 +37,19 @@ use zebra_jsonl_trace::{JsonlTraceGuard, JsonlTracer}; use crate::zakura::{ trace::{block_sync_trace as bs_trace, BLOCK_SYNC_TABLE}, transport::ByteBudget, - ZakuraPeerId, ZakuraTrace, + ServicePeerSnapshot, ZakuraBlockSyncCandidateState, ZakuraPeerId, ZakuraTrace, }; use super::{ - events::{BlockApplyResult, BlockApplyToken, BlockSyncAction}, + config::ZakuraBlockSyncConfig, + events::{BlockApplyResult, BlockApplyToken, BlockSyncAction, BlockSyncEvent}, reactor::{bs_insert_height, bs_insert_u64}, reorder::BufferedBlockBody, sequencer::Sequencer, sequencer_task::{ initial_view, SequencedBody, SequencerControlInput, SequencerTask, SequencerView, }, - state::{BlockSyncFrontiers, ThroughputMeter}, + state::{BlockSyncFrontiers, BlockSyncHandle, ThroughputMeter}, work_queue::WorkQueue, }; @@ -71,6 +72,8 @@ pub struct BenchSubmit { pub struct SequencerProgress { /// Verified block tip (last committed height the sequencer knows about). pub verified_tip: block::Height, + /// Hash of the verified block tip (the committed hash reported for `verified_tip`). + pub verified_hash: block::Hash, /// Bodies buffered out-of-order in the reorder queue. pub reorder_len: u64, /// Bodies drained into the contiguous `applying` set. @@ -207,6 +210,89 @@ impl BenchSequencerHandle { pub fn into_parts(self) -> (BenchBodyFeeder, BenchSubmissions, BenchCommitter) { (self.feeder, self.submissions, self.committer) } + + /// Production-driver split: returns the raw `BlockSyncAction` stream and an inert + /// [`BlockSyncHandle`] so the bench can drive the **real** block-sync apply driver + /// (`zebrad`'s `drive_block_sync_actions`) directly, instead of a hand-rolled + /// verify/commit loop. The driver reports completions through + /// [`BlockSyncHandle::send_control`]; with no reactor present, a spawned translator + /// forwards each `BlockApplyFinished` into the sequencer's `ApplyFinished` control + /// input — the exact hop `reactor::handle_block_apply_finished` performs in + /// production. Every other handle channel is inert: the bench feeds bodies directly, + /// so the driver never emits peer queries or reads peer/status/candidate state. + pub fn into_driver_parts(self) -> BenchDriverParts { + let BenchSequencerHandle { + feeder, + submissions, + committer, + } = self; + + let control = committer.control.clone(); + let (lifecycle_tx, mut lifecycle_rx) = mpsc::unbounded_channel::(); + tokio::spawn(async move { + while let Some(event) = lifecycle_rx.recv().await { + if let BlockSyncEvent::BlockApplyFinished { + token, + height, + hash, + result, + local_frontier, + } = event + { + if control + .send(SequencerControlInput::ApplyFinished { + token, + height, + hash, + result, + local_frontier, + }) + .is_err() + { + break; + } + } + } + }); + + let config = ZakuraBlockSyncConfig::default(); + let (events, _events_rx) = mpsc::channel(1); + let (_peers_tx, peers) = watch::channel(ServicePeerSnapshot::default()); + let (_status_tx, status) = watch::channel(config.initial_status()); + let (_candidates_tx, candidates) = watch::channel(ZakuraBlockSyncCandidateState::default()); + let block_sync = BlockSyncHandle { + events, + lifecycle: lifecycle_tx, + peers, + status, + candidates, + routine_wiring: None, + }; + + BenchDriverParts { + feeder, + actions: submissions.actions, + block_sync, + committer, + } + } +} + +/// The driver-shaped split returned by [`BenchSequencerHandle::into_driver_parts`]. +/// +/// Hand `actions` + `block_sync` to the production `drive_block_sync_actions`, feed +/// cached bodies through `feeder`, and use `committer` for progress/trace snapshots +/// (it also keeps the sequencer task alive). +pub struct BenchDriverParts { + /// Feeds cached bodies into the sequencer's reorder queue. + pub feeder: BenchBodyFeeder, + /// The ordered `SubmitBlock` (and other) actions the sequencer emits. + pub actions: mpsc::Receiver, + /// Inert handle the production driver reports completions through; its + /// `BlockApplyFinished` events are forwarded to the sequencer. + pub block_sync: BlockSyncHandle, + /// Progress/trace snapshots; retained to keep the sequencer task alive. + pub committer: BenchCommitter, } impl BenchBodyFeeder { @@ -315,11 +401,26 @@ impl BenchCommitter { } } + /// Awaits sequencer progress until the verified tip reaches `target` (or the + /// sequencer task ends). Used to detect completion when the production driver owns + /// the apply loop and the bench no longer sees individual commits. + pub async fn wait_for_verified_tip(&mut self, target: block::Height) { + loop { + if self.view.borrow().verified_tip >= target { + return; + } + if self.view.changed().await.is_err() { + return; + } + } + } + /// Latest progress snapshot from the sequencer view. pub fn progress(&self) -> SequencerProgress { let view = *self.view.borrow(); SequencerProgress { verified_tip: view.verified_tip, + verified_hash: view.verified_hash, reorder_len: view.reorder_len, applying_len: view.applying_len, committed_blocks_per_sec: view.committed_blocks_per_sec, diff --git a/zebra-network/src/zakura/block_sync/mod.rs b/zebra-network/src/zakura/block_sync/mod.rs index 30dc4eecf53..4f4241e61e3 100644 --- a/zebra-network/src/zakura/block_sync/mod.rs +++ b/zebra-network/src/zakura/block_sync/mod.rs @@ -54,8 +54,8 @@ mod work_queue; #[cfg(feature = "internal-bench")] pub use bench::{ - spawn_bench_sequencer, BenchBodyFeeder, BenchCommitter, BenchSequencerHandle, BenchSubmissions, - BenchSubmit, SequencerProgress, + spawn_bench_sequencer, BenchBodyFeeder, BenchCommitter, BenchDriverParts, BenchSequencerHandle, + BenchSubmissions, BenchSubmit, SequencerProgress, }; pub use config::{ BlockSyncStatus, CwndUnit, ZakuraBlockSyncConfig, DEFAULT_BS_MAX_SUBMITTED_BLOCK_APPLIES, diff --git a/zebra-replay-bench/Cargo.toml b/zebra-replay-bench/Cargo.toml index 8e59ea5ca14..f0046387919 100644 --- a/zebra-replay-bench/Cargo.toml +++ b/zebra-replay-bench/Cargo.toml @@ -24,6 +24,10 @@ zebra-chain = { path = "../zebra-chain", version = "9.0.0" } zebra-state = { path = "../zebra-state", version = "8.0.0" } zebra-consensus = { path = "../zebra-consensus", version = "8.0.0" } zebra-network = { path = "../zebra-network", version = "8.0.0", features = ["internal-bench"] } +zebra-node-services = { path = "../zebra-node-services", version = "7.0.0" } +# The production Zakura block-sync apply driver, exposed via `zebrad::bench_api` +# behind `internal-bench`, so `apply-sequencer` can drive the real apply pipeline. +zebrad = { path = "../zebrad", version = "5.0.0-rc.3", default-features = false, features = ["internal-bench"] } clap = { workspace = true, features = ["derive"] } color-eyre.workspace = true diff --git a/zebra-replay-bench/README.md b/zebra-replay-bench/README.md index b41efd2fb96..f94ab5f5d15 100644 --- a/zebra-replay-bench/README.md +++ b/zebra-replay-bench/README.md @@ -26,22 +26,25 @@ window warm in page cache first. # What height is a snapshot at? zebra-replay-bench info --src /path/to/snapshot -# Phase 1: extract a 30k window into a cache (source opened read-only) +# Phase 1: extract a 50k window into a cache (source opened read-only) zebra-replay-bench index \ --src /path/to/high-snapshot \ - --cache /tmp/win.zrb --start 1800001 --end 1830000 + --cache /tmp/win.zrb --start 1849958 --end 1899957 -# Phase 2: replay onto a fork whose tip == 1800000 -zebra-replay-bench apply --base /path/to/fork-at-1800000 --cache /tmp/win.zrb +# Phase 2: replay onto a fork whose tip == 1849957 +# The base must be a cleanly-pruned snapshot at START-1 (1849957) — one produced by +# replaying forward past the checkpoint raw-tx archive backlog, so a run does not +# re-trigger a DrainBacklog delete storm at startup. +zebra-replay-bench apply --base /path/to/fork-at-1849957 --cache /tmp/win.zrb # One altitude up: replay through the real zebra-state write worker -zebra-replay-bench apply-worker --base /path/to/fork-at-1800000 --cache /tmp/win.zrb +zebra-replay-bench apply-worker --base /path/to/fork-at-1849957 --cache /tmp/win.zrb # Two altitudes up: replay through the real zebra-consensus checkpoint verifier -zebra-replay-bench apply-verifier --base /path/to/fork-at-1800000 --cache /tmp/win.zrb +zebra-replay-bench apply-verifier --base /path/to/fork-at-1849957 --cache /tmp/win.zrb # Three altitudes up: replay through the real Zakura block-sync Sequencer (VCT-only) -zebra-replay-bench apply-sequencer --base /path/to/fork-at-1800000 --cache /tmp/win.zrb --vct-sidecar /tmp/win.vct +zebra-replay-bench apply-sequencer --base /path/to/fork-at-1849957 --cache /tmp/win.zrb --vct-sidecar /tmp/win.vct ``` ## Third altitude: `apply-verifier` @@ -96,6 +99,9 @@ Two optional flags: writer is flushed at end-of-run. Without it the tracer is `noop()` (zero overhead). Via the harness: `REPLAY_TRACE_DIR= make perf-replay-sequencer` (add `REPLAY_ARCHIVE=1` for archive mode). +- `--stop-height ` clamps the committed window to a sub-range of the cache (stops at + the last checkpoint `<= h`), so a larger cache can be reused for a shorter run. + Harness: `REPLAY_STOP_HEIGHT=`. ## Two altitudes: `apply` vs `apply-worker` diff --git a/zebra-replay-bench/RESULTS.md b/zebra-replay-bench/RESULTS.md index 578eac88c14..f2c6a51ed22 100644 --- a/zebra-replay-bench/RESULTS.md +++ b/zebra-replay-bench/RESULTS.md @@ -24,11 +24,14 @@ and find the next optimization lever. ## Environment - 8 cores (DO Premium Intel), 31 GiB RAM, Linux. -- Snapshots under `/mnt/roman-dev-2-data`: base = `zebra-ckpt-1800000-warm` - (archive 27.3.0, tip 1,802,000); block + root source = `zebra-cache` - (tip 3,376,789). -- Window: heights **1,802,001–1,832,000** (30,000 blocks, ~19.1 GiB of bodies); - VCT roots sidecar derived from the same source over the same window. +- Snapshots under `/mnt/roman-dev-2-data`: base = `zebra-ckpt-1850000-warm-pruned` + (pruned 27.3.0, tip 1,849,957). It is a *cleanly-pruned* base — produced by + replaying the older `zebra-ckpt-1800000-warm-pruned` forward past the checkpoint + raw-tx archive backlog, so forking it does not trigger a startup `DrainBacklog` + delete storm (the older base did, polluting the first ~10K blocks of every run). + Block + root source = `zebra-cache` (tip 3,376,789). +- Window: heights **1,849,958–1,899,957** (50,000 blocks); cache `win-1850k.zrb`, + VCT roots sidecar `win-1850k.vct` derived from the same source over the same window. - Forward replay, no rollback. Each run executes on a fresh hard-link fork of the base; the final tip-hash gate (byte-identical to the source snapshot tip 1,832,000) passes on every run. diff --git a/zebra-replay-bench/src/apply_sequencer.rs b/zebra-replay-bench/src/apply_sequencer.rs index 9b60ea63045..f229c0745df 100644 --- a/zebra-replay-bench/src/apply_sequencer.rs +++ b/zebra-replay-bench/src/apply_sequencer.rs @@ -4,9 +4,16 @@ //! The sequencer is the body reorder + ordered-submit pipeline: bodies are fed into //! its reorder queue (here in height order, sequentially — random/out-of-order arrival //! is a future knob), it drains the contiguous prefix into `applying` and emits -//! `SubmitBlock`s, and a thin driver here commits each through the same real -//! `CheckpointVerifier` → `StateService` the verifier rung uses, reporting the commit -//! back so the sequencer frontier advances and releases the next blocks. +//! `SubmitBlock`s. +//! +//! Those actions are driven by the **production** Zakura block-sync apply driver +//! (`zebrad::bench_api::drive_block_sync_actions`, unmodified): it applies each body +//! through the real consensus router into the real `StateService` and reports the +//! commit back to the sequencer so the frontier advances. Checkpoint-class blocks take +//! the header-authenticated fast path (`Request::CommitCheckpointAuthenticated`), which +//! requires the base's Zakura header store to be seeded first +//! (`zebra-replay-bench seed-headers`). The bench only feeds bodies and watches the +//! sequencer view for the gate checkpoint — there is no hand-rolled apply loop. //! //! VCT mode only. Checkpoint batching is the same as `apply_verifier`: feed up to the //! last checkpoint `<= end` so the last range delivers the successors the worker's VCT @@ -18,18 +25,18 @@ use std::{ atomic::{AtomicU64, Ordering}, Arc, }, - time::Instant, + time::{Duration, Instant}, }; use color_eyre::eyre::{bail, eyre, Result}; -use futures::stream::{FuturesOrdered, StreamExt}; -use tower::{buffer::Buffer, Service, ServiceExt}; -use zebra_chain::{ - block::{self, Height}, - parameters::Network, +use tokio::sync::oneshot; +use tower::{buffer::Buffer, util::BoxService}; +use zebra_chain::{block::Height, chain_tip::NoChainTip, parameters::Network}; +use zebra_consensus::{router, BoxError, Config as ConsensusConfig}; +use zebra_network::zakura::{ + spawn_bench_sequencer, BenchDriverParts, ZakuraSupervisorHandle, ZakuraTrace, }; -use zebra_consensus::CheckpointVerifier; -use zebra_network::zakura::{spawn_bench_sequencer, BenchSubmit}; +use zebra_node_services::mempool; use zebra_state::{FinalizedState, PruningConfig, StorageMode}; use crate::{ @@ -57,6 +64,7 @@ pub fn run( vct_sidecar: Option<&Path>, network: Network, archive: bool, + stop_height: Option, trace_dir: Option<&Path>, ) -> Result { let reader = CacheReader::open(cache_path)?; @@ -72,7 +80,18 @@ pub fn run( ); } let start = header.start_height; - let end = start + header.count - 1; + let cache_end = start + header.count - 1; + // Optionally bench a sub-range of a larger cache: clamp the window's effective end + // to `--stop-height` (the checkpoint logic below still feeds to the last checkpoint + // <= end and gates to the second-to-last, so the prefetch stops early). + let end = match stop_height { + Some(h) if h < start => bail!("--stop-height {h} is below the window start {start}"), + Some(h) if h > cache_end => { + bail!("--stop-height {h} exceeds the cache end {cache_end}") + } + Some(h) => h, + None => cache_end, + }; let expected_parent = start.checked_sub(1).ok_or_else(|| { eyre!("cache starts at genesis (height 0); apply needs a base at start-1") })?; @@ -182,7 +201,7 @@ pub fn run( let stats = runtime.block_on(async move { // Real buffered StateService + checkpoint verifier on the base fork. - let (state, _read, _latest, _change) = zebra_state::init( + let (state, read_state, _latest, _change) = zebra_state::init( config, &network, max_checkpoint_height, @@ -190,21 +209,38 @@ pub fn run( ) .await; let state = Buffer::new(state, STATE_BUFFER_BOUND); - let mut verifier = CheckpointVerifier::new( + // Drive through the production block-verifier router (`zebra_consensus::router::init`), + // not the bare `CheckpointVerifier`, so the bench exercises the real Zakura apply + // entry point (`Request::Commit` → checkpoint verifier → state). The router is + // already buffered, so its synchronous verify work runs on the router's worker, + // off this single driver loop. It seeds its checkpoint verifier from the state + // tip (the forked base at `expected_parent`), so no explicit initial tip is + // needed. The mempool input is never sent (block verification does not use it). + let consensus_config = ConsensusConfig { + checkpoint_sync: true, + vct_fast_sync: true, + }; + let (verifier, _tx_verifier, _bg_handles, _max_ckpt) = router::init( + consensus_config, &network, - Some((Height(expected_parent), parent_hash)), state.clone(), - ); - - // Per-height serialized byte length, written by the feed task and read by the - // driver (the SubmitBlock action does not carry the size). One slot per fed - // block; small (8 bytes each). - let lens: Arc> = Arc::new( - (0..feed_target).map(|_| AtomicU64::new(0)).collect(), - ); + oneshot::channel::< + Buffer, mempool::Request>, + >() + .1, + ) + .await; - // Spawn the real block-sync Sequencer starting from the base tip. - let (feeder, mut submissions, mut committer) = spawn_bench_sequencer( + // Spawn the real block-sync Sequencer from the base tip, split into + // production-driver parts: the raw `BlockSyncAction` stream + an inert + // `BlockSyncHandle` whose `BlockApplyFinished` feedback is forwarded into the + // sequencer's `ApplyFinished` input (the hop the production reactor performs). + let BenchDriverParts { + feeder, + actions, + block_sync, + mut committer, + } = spawn_bench_sequencer( Height(expected_parent), Height(expected_parent), parent_hash, @@ -212,32 +248,57 @@ pub fn run( SEQUENCER_MAX_INFLIGHT_BYTES, trace_dir.clone(), ) - .into_parts(); + .into_driver_parts(); - let (_producer, rx) = prefetch::spawn(reader, in_flight); + // Spawn the *production* Zakura block-sync apply driver against the bench's + // action stream, state, and consensus router. It owns verify+commit+report: + // checkpoint-class blocks take the header-authenticated fast path + // (`Request::CommitCheckpointAuthenticated`), so the base must be header-seeded + // (`zebra-replay-bench seed-headers`). Applies drain through `FuturesUnordered` + // with the production checkpoint/full/combined limits — no bench apply loop. + let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel::<()>(); + let driver = tokio::spawn(zebrad::bench_api::drive_block_sync_actions( + actions, + ZakuraSupervisorHandle::new(1), + None, + block_sync, + NoChainTip, + read_state.clone(), + verifier, + max_checkpoint_height, + in_flight, + in_flight, + in_flight, + ZakuraTrace::noop(), + None, + async move { + let _ = shutdown_rx.await; + }, + )); + + let (_producer, rx) = prefetch::spawn_raw(reader, in_flight); - // Feed task: stream prepared blocks into the reorder queue, in height order, - // up to the last checkpoint (`feed_target`). Records each block's size in `lens`. - let feed_lens = lens.clone(); + // Feed task: stream raw bodies into the sequencer's reorder queue in height + // order, up to the last checkpoint (`feed_target`). Accumulates fed bytes for + // the throughput report. + let fed_bytes = Arc::new(AtomicU64::new(0)); + let feed_bytes = fed_bytes.clone(); let feed = tokio::spawn(async move { let mut fed = 0u32; while fed < feed_target { // The prefetch producer is a std thread; do the blocking recv off the // async executor. let item = tokio::task::block_in_place(|| rx.recv()); - let prepared = match item { + let raw = match item { Ok(Ok(p)) => p, Ok(Err(e)) => return Err(e), Err(_) => break, // producer exhausted before feed_target }; - let height = prepared.height; - let hash = prepared.block.hash(); - let len = prepared.len as u64; - feed_lens[(height - start) as usize].store(len, Ordering::Relaxed); - if !feeder - .feed_body(Height(height), hash, prepared.block, len) - .await - { + let height = raw.height; + let hash = raw.block.hash(); + let len = raw.len as u64; + feed_bytes.fetch_add(len, Ordering::Relaxed); + if !feeder.feed_body(Height(height), hash, raw.block, len).await { break; // sequencer gone } fed += 1; @@ -245,107 +306,106 @@ pub fn run( Ok::<(), color_eyre::Report>(()) }); - let mut stats = Stats::default(); - let mut verifies = FuturesOrdered::new(); - let mut submitted = 0u32; - let mut done = 0u32; - - let wall_start = Instant::now(); - let mut last = wall_start; - - // Cadence for the `block_sync_state` trace snapshots (only when tracing is on). - // Dense enough for a smooth throughput/applying curve; the plotter downsamples. let tracing_on = trace_dir.is_some(); if tracing_on { committer.emit_state_snapshot(); } - // Drive: pull ordered submissions and verify+commit them concurrently; report - // each commit back so the sequencer frontier advances. The sequencer's submit - // limit bounds the in-flight set. - while done < target { - tokio::select! { - biased; - // A verify+commit completed: report it back and record it. - Some(item) = verifies.next(), if !verifies.is_empty() => { - let (token, height, committed_hash, len): (_, Height, block::Hash, u64) = item?; - if height == last_checkpoint && committed_hash != expected_tip_hash { - bail!( - "committed hash at the second-to-last checkpoint {} does not match the embedded checkpoint hash", - last_checkpoint.0 - ); - } - committer.apply_committed(token, height, committed_hash); - done += 1; - let now = Instant::now(); - stats.record(len as usize, now - last); - last = now; - if tracing_on && (done.is_multiple_of(20) || done == target) { - committer.emit_state_snapshot(); - } - if done.is_multiple_of(5000) || done == target { - let p = committer.progress(); - tracing::info!( - done, submitted, - verified_tip = p.verified_tip.0, - reorder = p.reorder_len, - applying = p.applying_len, - "sequencer-progress" - ); - } - } - // The next ordered submission from the sequencer: start its verify. - // The next ordered submission from the sequencer: start its verify. - // `None` (action channel closed) just leaves the drain arm to finish. - maybe = submissions.next_submit(), if submitted < feed_target => { - if let Some(BenchSubmit { token, block }) = maybe { - let height = block - .coinbase_height() - .expect("submitted checkpoint block has a coinbase height"); - let len = lens[(height.0 - start) as usize].load(Ordering::Relaxed); - let fut = verifier - .ready() - .await - .map_err(|e| eyre!("verifier not ready: {e}"))? - .call(block); - verifies.push_back(async move { - let committed = fut.await.map_err(|e| { - eyre!("verify/commit failed at height {}: {e}", height.0) - })?; - Ok::<(_, Height, block::Hash, u64), color_eyre::Report>(( - token, height, committed, len, - )) - }); - submitted += 1; - } - } - else => break, + // The production driver owns verify+commit+report; the bench just feeds bodies + // and waits for the verified tip to reach the gate checkpoint, emitting periodic + // progress + `block_sync_state` snapshots from the sequencer view as it advances. + let target_height = last_checkpoint; + let wall_start = Instant::now(); + loop { + let reached = tokio::time::timeout( + Duration::from_secs(5), + committer.wait_for_verified_tip(target_height), + ) + .await + .is_ok(); + let p = committer.progress(); + if tracing_on { + committer.emit_state_snapshot(); + } + tracing::info!( + verified_tip = p.verified_tip.0, + reorder = p.reorder_len, + applying = p.applying_len, + bps = p.committed_blocks_per_sec, + "sequencer-progress (prod driver)" + ); + if p.verified_tip >= target_height { + break; + } + // `wait_*` returned without reaching the target => the sequencer task ended + // (driver gone); or the feed drained with nothing left in flight. Either way + // stop and let the gate below surface the failure. + if reached || (feed.is_finished() && p.reorder_len == 0 && p.applying_len == 0) { + break; } } let wall = wall_start.elapsed(); - if done < target { - // Drive couldn't reach the target: surface a feed error if there was one. - match feed.await { - Ok(Ok(())) => bail!("sequencer stalled at {done}/{target} committed (no feed error)"), - Ok(Err(e)) => return Err(e), - Err(e) => bail!("feed task panicked: {e}"), + // Gate via the sequencer view (the driver's `ApplyFinished` feedback), not a + // read-state query: reaching the gate is itself the correctness proof. The + // header-authenticated fast path refuses any block whose hash doesn't match the + // hardcoded checkpoint, and a refused block never advances `verified_tip` — so + // `verified_tip >= target` means every block up to it committed against the + // authenticated checkpoint chain. When the tip lands exactly on the gate (the + // common case), additionally assert the verified hash matches. + let gate = committer.progress(); + if gate.verified_tip < target_height { + let _ = shutdown_tx.send(()); + if let Ok(Err(e)) = feed.await { + return Err(e); } + bail!( + "driver stalled at verified tip {} before reaching the gate checkpoint {} (a non-authenticated block would refuse to commit)", + gate.verified_tip.0, + target_height.0 + ); + } + if gate.verified_tip == target_height && gate.verified_hash != expected_tip_hash { + let _ = shutdown_tx.send(()); + bail!( + "verified hash at the second-to-last checkpoint {} ({}) does not match the embedded checkpoint hash {}", + target_height.0, + gate.verified_hash, + expected_tip_hash + ); + } + + // Stop the driver and drain the feed. + let _ = shutdown_tx.send(()); + let _ = driver.await; + match feed.await { + Ok(Ok(())) => {} + Ok(Err(e)) => return Err(e), + Err(e) => bail!("feed task panicked: {e}"), } - // Drain + flush the JSONL trace writer so the tables are complete on disk. committer.flush_trace().await; + let committed = (target_height.0 - start + 1) as usize; + let total_bytes = fed_bytes.load(Ordering::Relaxed); + let mut stats = Stats::default(); + stats.record(total_bytes as usize, wall); + tracing::info!( - committed = done, + committed, last_checkpoint = last_checkpoint.0, - "replay verified (sequencer, vct): committed through the second-to-last checkpoint; hash matches" + "replay verified (sequencer, vct, prod driver): committed through the second-to-last checkpoint; hash matches" ); - println!("mode=sequencer (vct, {storage_label})"); + println!("mode=sequencer-prod (vct, {storage_label})"); if let Some(dir) = trace_dir.as_deref() { println!("zakura-traces={}", dir.display()); } - println!("{}", stats.report(wall)); + let secs = wall.as_secs_f64().max(f64::MIN_POSITIVE); + println!( + "throughput: {:.1} blk/s {:.2} MiB/s", + committed as f64 / secs, + total_bytes as f64 / secs / (1024.0 * 1024.0) + ); Ok::(stats) })?; diff --git a/zebra-replay-bench/src/config.rs b/zebra-replay-bench/src/config.rs index 3becdb2c817..873674c9936 100644 --- a/zebra-replay-bench/src/config.rs +++ b/zebra-replay-bench/src/config.rs @@ -15,12 +15,18 @@ use zebra_state::{Config, StorageMode}; /// work this benchmark exists to measure. Left on (the default), Mainnet would /// select the VCT peer-source fast path and skip the recompute. pub fn state_config(cache_dir: PathBuf, force_legacy: bool) -> Config { - Config { + let mut config = Config { cache_dir, ephemeral: false, checkpoint_sync: true, vct_fast_sync: !force_legacy, storage_mode: StorageMode::Archive, ..Config::default() - } + }; + // Run-ahead finalized-commit pipeline depth (PR #309). 0 = synchronous (default). + config.finalized_block_pipeline_depth = std::env::var("ZRB_PIPELINE_DEPTH") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(0); + config } diff --git a/zebra-replay-bench/src/main.rs b/zebra-replay-bench/src/main.rs index 7e63d77ff00..cc8d72289a1 100644 --- a/zebra-replay-bench/src/main.rs +++ b/zebra-replay-bench/src/main.rs @@ -22,6 +22,7 @@ mod index; mod prefetch; mod rollback; mod roots_cache; +mod seed_headers; mod stats; use std::path::PathBuf; @@ -136,11 +137,29 @@ enum Cmd { /// Pruned (the base must already be a pruned snapshot; pruning is one-way). #[arg(long)] archive: bool, + /// Stop after committing up to this height, clamping the cache window (to bench + /// a sub-range of a larger cache). Defaults to the full cache end. + #[arg(long)] + stop_height: Option, /// Write structured Zakura JSONL trace tables to this directory (the same /// tables `perf-run-mainnet` produces via `[network.zakura] trace_dir`). #[arg(long)] trace_dir: Option, }, + /// Seed the Zakura header store on a base snapshot for every cached block height, + /// so the production block-sync driver's header-authenticated checkpoint fast path + /// can run offline. Run once on the base (tip must be `start-1`); forks inherit it. + SeedHeaders { + /// Base fork root (opened writable, mutated in place; must be at start-1). + #[arg(long)] + base: PathBuf, + /// Cache file produced by `index`. + #[arg(long)] + cache: PathBuf, + /// Seed against an Archive-mode base. Default mirrors the run (Pruned). + #[arg(long)] + archive: bool, + }, /// Replay a cache through the real `zebra-consensus` checkpoint verifier, which /// commits to a real `StateService` (tip must be `start-1`). One altitude above /// `apply-worker`; adds PoW/equihash + Merkle verification and checkpoint @@ -208,6 +227,13 @@ fn main() -> Result<()> { Cmd::Rollback { base, target } => { rollback::run(&base, target, network)?; } + Cmd::SeedHeaders { + base, + cache, + archive, + } => { + seed_headers::run(&base, &cache, network, archive)?; + } Cmd::Apply { base, cache, @@ -252,6 +278,7 @@ fn main() -> Result<()> { cache, vct_sidecar, archive, + stop_height, trace_dir, } => { #[cfg(feature = "commit-metrics")] @@ -263,6 +290,7 @@ fn main() -> Result<()> { vct_sidecar.as_deref(), network, archive, + stop_height, trace_dir.as_deref(), )?; @@ -289,7 +317,10 @@ fn render_metrics(handle: metrics_exporter_prometheus::PrometheusHandle) { if line.starts_with('#') { continue; } - if line.starts_with("zebra_state_") || line.starts_with("state_vct_") { + if line.starts_with("zebra_state_") + || line.starts_with("state_vct_") + || line.starts_with("zebra_consensus_") + { println!("{line}"); } } diff --git a/zebra-replay-bench/src/prefetch.rs b/zebra-replay-bench/src/prefetch.rs index 9f0504d582d..7d44e0144e3 100644 --- a/zebra-replay-bench/src/prefetch.rs +++ b/zebra-replay-bench/src/prefetch.rs @@ -45,6 +45,16 @@ pub fn capacity() -> usize { .unwrap_or(DEFAULT_PREFETCH_CAPACITY) } +/// One raw block ready to feed into an upstream verifier or sequencer. +/// +/// This path avoids verifier-side precomputation when the consumer will submit the +/// raw block to the real verifier anyway. +pub struct RawBlock { + pub block: Arc, + pub len: usize, + pub height: u32, +} + /// One prepared block ready to commit: the verified block plus the per-block prep /// the committer's `next_checkpoint` needs (the block handle and its auth-data /// root), with its height and serialized byte length. @@ -104,3 +114,44 @@ pub fn spawn(reader: CacheReader, capacity: usize) -> (JoinHandle<()>, Receiver< (handle, rx) } + +/// Spawns a raw-block prefetch producer over `reader`. +/// +/// Use this for paths that feed raw blocks into the real verifier. Unlike +/// [`spawn`], this does not build [`CheckpointVerifiedBlock`] values that would +/// be discarded before the verifier rebuilds them. +pub fn spawn_raw( + reader: CacheReader, + capacity: usize, +) -> (JoinHandle<()>, Receiver>) { + let (tx, rx) = sync_channel(capacity); + let mut reader = reader; + let mut height = reader.header().start_height; + + let handle = thread::spawn(move || loop { + let bytes = match reader.next_block() { + Ok(Some(bytes)) => bytes, + Ok(None) => return, + Err(e) => { + let _ = tx.send(Err(eyre!("reading block {height}: {e}"))); + return; + } + }; + let len = bytes.len(); + let block = match Block::zcash_deserialize(&bytes[..]) { + Ok(block) => Arc::new(block), + Err(e) => { + let _ = tx.send(Err(eyre!("deserializing block {height}: {e}"))); + return; + } + }; + + if tx.send(Ok(RawBlock { block, len, height })).is_err() { + // Consumer dropped the receiver: stop producing. + return; + } + height += 1; + }); + + (handle, rx) +} diff --git a/zebra-replay-bench/src/seed_headers.rs b/zebra-replay-bench/src/seed_headers.rs new file mode 100644 index 00000000000..51a10ad0534 --- /dev/null +++ b/zebra-replay-bench/src/seed_headers.rs @@ -0,0 +1,100 @@ +//! One-time seeding of the Zakura header store on a base snapshot. +//! +//! The production block-sync apply driver commits checkpoint-class blocks through the +//! **header-authenticated** fast path (`Request::CommitCheckpointAuthenticated`), and +//! refuses any block whose height is not header-authenticated. Authentication reads +//! strictly from the Zakura header store (`zakura_header_hash`), which live header sync +//! populates. An offline base snapshot has no header sync, so this command writes the +//! frontier header rows (hash + header) for every cached block height — exactly what +//! header sync would persist — directly into the base. +//! +//! Run it **once** on the base: forks inherit the rows via the hard-link clone, and the +//! driver releases each row when it commits that height, so a fresh fork always presents +//! the full frontier above the body tip. The rows are independent of `tx_by_loc` +//! pruning, so seeding does not disturb the base's cleanly-pruned state. + +use std::{path::Path, sync::Arc}; + +use color_eyre::eyre::{bail, eyre, Result}; +use zebra_chain::{ + block::{Block, Height}, + parameters::Network, + serialization::ZcashDeserialize, +}; +use zebra_state::{FinalizedState, PruningConfig, StorageMode}; + +use crate::{cache::CacheReader, config::state_config}; + +/// Seeds the Zakura header store on `base` for every block in `cache_path`. +/// +/// `base` must be at `cache.start_height - 1` (the frontier sits strictly above the +/// body tip). `archive` mirrors the run's storage mode (default Pruned). +pub fn run(base: &Path, cache_path: &Path, network: Network, archive: bool) -> Result<()> { + let mut reader = CacheReader::open(cache_path)?; + let header = reader.header(); + let start = header.start_height; + let expected_parent = start.checked_sub(1).ok_or_else(|| { + eyre!("cache starts at genesis (height 0); seeding needs a base at start-1") + })?; + + let mut config = state_config(base.to_path_buf(), /* force_legacy */ false); + if !archive { + config.storage_mode = StorageMode::Pruned(PruningConfig::default()); + } + + let state = FinalizedState::new_writable(&config, &network); + match state.db.finalized_tip_height() { + Some(tip) if tip.0 == expected_parent => {} + Some(tip) => bail!( + "base tip is {} but cache window starts at {start}; base must be at height {expected_parent}", + tip.0 + ), + None => bail!("base has no finalized tip; expected height {expected_parent}"), + } + + let mut height = start; + let mut chunk: Vec<(Height, Arc)> = Vec::with_capacity(2000); + let mut seeded = 0u64; + + let flush = |state: &FinalizedState, chunk: &mut Vec<(Height, Arc)>| -> Result { + let n = chunk.len() as u64; + state + .db + .seed_zakura_headers_from_blocks(chunk.drain(..)) + .map_err(|e| eyre!("seeding zakura header rows: {e}"))?; + Ok(n) + }; + + loop { + let bytes = match reader + .next_block() + .map_err(|e| eyre!("reading cache block {height}: {e}"))? + { + Some(b) => b, + None => break, + }; + let block = Arc::new( + Block::zcash_deserialize(&bytes[..]) + .map_err(|e| eyre!("deserializing block {height}: {e}"))?, + ); + chunk.push((Height(height), block)); + height += 1; + if chunk.len() >= 2000 { + seeded += flush(&state, &mut chunk)?; + if seeded.is_multiple_of(20000) { + tracing::info!(seeded, "seeding zakura header store"); + } + } + } + if !chunk.is_empty() { + seeded += flush(&state, &mut chunk)?; + } + + let end = height.saturating_sub(1); + tracing::info!(seeded, start, end, "zakura header store seeded"); + println!( + "seeded {seeded} zakura header rows ({start}..={end}) into {}", + base.display() + ); + Ok(()) +} diff --git a/zebrad/Cargo.toml b/zebrad/Cargo.toml index aca6b4647cf..d77338ec796 100644 --- a/zebrad/Cargo.toml +++ b/zebrad/Cargo.toml @@ -65,6 +65,11 @@ indexer = ["zebra-state/indexer"] # Per-block commit-phase timing histograms (zebra.state.write.*), for perf profiling commit-metrics = ["zebra-state/commit-metrics"] +# Exposes the production Zakura block-sync apply driver (`drive_block_sync_actions`) +# from the crate root so the offline `zebra-replay-bench` can drive the real apply +# pipeline. Not part of any release; no production behavior change. +internal-bench = ["zebra-network/internal-bench"] + # Experimental internal miner support internal-miner = [ "thread-priority", diff --git a/zebrad/src/commands/start.rs b/zebrad/src/commands/start.rs index 98dec3a2af5..f3eb05c553b 100644 --- a/zebrad/src/commands/start.rs +++ b/zebrad/src/commands/start.rs @@ -73,7 +73,9 @@ //! //! Some of the diagnostic features are optional, and need to be enabled at compile-time. -mod zakura; +// `pub(crate)` (not private) so the crate root can re-export the block-sync apply +// driver under the `internal-bench` feature for `zebra-replay-bench`. +pub(crate) mod zakura; use std::{net::SocketAddr, path::Path, sync::Arc}; diff --git a/zebrad/src/commands/start/zakura/block_sync_driver.rs b/zebrad/src/commands/start/zakura/block_sync_driver.rs index 85171855b7f..6ff01b797e7 100644 --- a/zebrad/src/commands/start/zakura/block_sync_driver.rs +++ b/zebrad/src/commands/start/zakura/block_sync_driver.rs @@ -91,8 +91,16 @@ impl CheckpointFrontierRefresh { } } +/// Drives the Zakura block-sync apply pipeline: consumes `BlockSyncAction`s from the +/// sequencer, applies submitted bodies through the consensus router into state +/// (checkpoint-class blocks take the header-authenticated fast path), drains applies +/// concurrently under the checkpoint/full/combined limits, and reports completions +/// back to the sequencer via `block_sync`. Runs until `shutdown` resolves. +/// +/// This is the production sync driver; it is also reused by the offline +/// `zebra-replay-bench` (via `zebrad::bench_api`) to benchmark the real apply path. #[allow(clippy::too_many_arguments)] -pub(crate) async fn drive_block_sync_actions( +pub async fn drive_block_sync_actions( mut actions: mpsc::Receiver, // Retained so the disconnect capability stays wired into the driver, even // though peer scoring no longer drives disconnects (misbehavior is record-only). diff --git a/zebrad/src/commands/start/zakura/mod.rs b/zebrad/src/commands/start/zakura/mod.rs index bff8f791fd8..82a5028c592 100644 --- a/zebrad/src/commands/start/zakura/mod.rs +++ b/zebrad/src/commands/start/zakura/mod.rs @@ -12,7 +12,7 @@ pub(crate) mod frontier; pub(crate) mod header_sync_driver; pub(crate) mod throughput_probe; -pub(crate) use block_sync_driver::drive_block_sync_actions; +pub use block_sync_driver::drive_block_sync_actions; #[cfg(test)] pub(crate) use block_sync_driver::{ apply_block_sync_body, block_apply_class, block_sync_missing_body_window, diff --git a/zebrad/src/commands/start/zakura/throughput_probe.rs b/zebrad/src/commands/start/zakura/throughput_probe.rs index 6ddf5325e5e..989828711c8 100644 --- a/zebrad/src/commands/start/zakura/throughput_probe.rs +++ b/zebrad/src/commands/start/zakura/throughput_probe.rs @@ -24,7 +24,7 @@ use zebra_network::zakura::{BlockApplyResult, BlockSyncFrontiers}; /// expected block at the next height; the probe only confirms contiguity and /// takes the body's own hash as the new tip. #[derive(Clone, Debug)] -pub(crate) struct BlocksyncThroughputProbe { +pub struct BlocksyncThroughputProbe { inner: Arc>, } diff --git a/zebrad/src/lib.rs b/zebrad/src/lib.rs index fe7efa59bcf..e1af8d4067b 100644 --- a/zebrad/src/lib.rs +++ b/zebrad/src/lib.rs @@ -150,5 +150,14 @@ pub mod components; pub mod config; pub mod prelude; +/// Narrow internal API for the offline `zebra-replay-bench`: the production Zakura +/// block-sync apply driver, so the bench can drive the real apply/commit/report loop +/// instead of a hand-rolled one. Feature-gated; not part of any release. +#[cfg(feature = "internal-bench")] +pub mod bench_api { + pub use crate::commands::start::zakura::drive_block_sync_actions; + pub use crate::commands::start::zakura::throughput_probe::BlocksyncThroughputProbe; +} + #[cfg(feature = "sentry")] pub(crate) mod sentry; From 087cb72be2839858b0109ccefec8060bae2f9aaf Mon Sep 17 00:00:00 2001 From: Roman Date: Tue, 30 Jun 2026 05:31:42 +0000 Subject: [PATCH 06/17] perf(replay-bench): fix prefetch OOM and add commit-pipeline profiling Fix a memory blow-up in the offline apply-sequencer bench and add the instrumentation used to localize it. All instrumentation is env-gated and inert in production. Prefetch OOM: - The parallel deserialize path used an unbounded worker->reorder channel, so when the consumer backpressured (the read-bound sandblast region ~1.86M) the reader raced ahead and deserialized a large swath of the window into RAM -- each block's Orchard Halo2 proof is large -- exhausting memory. Bound read-ahead with a permit channel: at most `capacity` blocks may be read but not yet emitted, while the worker->reorder channel stays unbounded so the in-order reorder thread can never deadlock waiting for the next block. - apply-sequencer: feed backpressure on the committed tip (ZRB_MAX_FEED_AHEAD) bounds the sequencer applying set; bench.rs exposes committed_tip. Profiling instrumentation: - stage_timing: add enabled() so callers can skip trace-only aggregates. - commit_pressure: record process RSS per traced row (separates RocksDB-internal growth from total process growth). - block.rs: per-category commit-batch key attribution (shielded/transparent/trees/...) and spent-UTXO create->spend distance buckets (cache-locality measurement). - disk_db: env-tunable RocksDB knobs (background jobs, subcompactions, memtable budget) for compaction experiments; default behavior unchanged. Tooling: - by-height latency/throughput/keys-by-category plot script for a trace dir. --- .../plot_latency_throughput_by_height.py | 274 ++++++++++++++++++ Cargo.lock | 1 + zebra-chain/src/lib.rs | 1 + zebra-chain/src/stage_timing.rs | 79 +++++ zebra-consensus/src/checkpoint.rs | 5 + zebra-network/src/zakura/block_sync/bench.rs | 11 + zebra-replay-bench/Cargo.toml | 1 + zebra-replay-bench/src/apply_sequencer.rs | 54 +++- zebra-replay-bench/src/prefetch.rs | 152 +++++++++- zebra-state/src/service.rs | 1 + .../finalized_state/commit_pressure.rs | 15 +- .../src/service/finalized_state/disk_db.rs | 29 +- .../service/finalized_state/zebra_db/block.rs | 47 +++ zebra-state/src/service/tests.rs | 5 +- zebra-state/src/service/write.rs | 22 +- .../start/zakura/block_sync_driver.rs | 19 ++ 16 files changed, 699 insertions(+), 17 deletions(-) create mode 100644 .cursor/skills/zakura-trace-plots/scripts/plot_latency_throughput_by_height.py create mode 100644 zebra-chain/src/stage_timing.rs diff --git a/.cursor/skills/zakura-trace-plots/scripts/plot_latency_throughput_by_height.py b/.cursor/skills/zakura-trace-plots/scripts/plot_latency_throughput_by_height.py new file mode 100644 index 00000000000..b3c5c338ab8 --- /dev/null +++ b/.cursor/skills/zakura-trace-plots/scripts/plot_latency_throughput_by_height.py @@ -0,0 +1,274 @@ +#!/usr/bin/env python3 +"""Plot per-block metric percentiles, throughput, and key composition by height. + +Companion to plot_commit_verify_timing.py: a percentile-by-height view from a +commit-timing.jsonl stream, optionally with a stage-timing.jsonl stream for the +per-category key breakdown. + +Panels (top -> bottom), all sharing the block-height x-axis: + 1. p50/p90 of a chosen metric (default block size, KB) + 2. throughput (blk/s) derived from ts_us deltas + 3. (if --stage-file given) commit batch keys/block, stacked by category + (shielded nullifiers vs transparent UTXO+index vs other) + +Usage: + python3 plot_latency_throughput_by_height.py COMMIT_TIMING.jsonl \ + --stage-file STAGE.jsonl --out-dir perf-artifacts [--bins 150] +""" +import argparse +import json +import math +from pathlib import Path + +BLUE, RED, GREEN = "#1f77b4", "#d62728", "#2ca02c" +# stacked key categories: (label, color, stage names summed into it) +KEY_CATS = [ + ("shielded (nullifiers)", "#9467bd", ["keys_shielded"]), + ("transparent (UTXO+index)", "#ff7f0e", ["keys_transparent"]), + ("other (header/trees/pool)", "#9aa0a6", + ["keys_header_tx", "keys_trees", "keys_valuepool", "keys_vct_evict"]), +] + + +def load(path): + rows = [] + for line in open(path): + line = line.strip() + if line: + try: + rows.append(json.loads(line)) + except json.JSONDecodeError: + pass + return rows + + +def percentile(values, pct): + if not values: + return 0.0 + s = sorted(values) + k = (len(s) - 1) * pct / 100.0 + lo, hi = math.floor(k), math.ceil(k) + return s[int(k)] if lo == hi else s[lo] * (hi - k) + s[hi] * (k - lo) + + +def nice_max(v): + if v <= 0: + return 1.0 + mag = 10 ** math.floor(math.log10(v)) + for m in (1, 1.5, 2, 2.5, 3, 4, 5, 6, 8, 10): + if m * mag >= v: + return m * mag + return 10 * mag + + +def x_map(h, h0, h1, left, w): + return left + (h - h0) / (h1 - h0) * w + + +def lin_y(v, vmax, top, h): + return top + h - min(max(v, 0.0), vmax) / vmax * h + + +def polyline(pts, color, wid=2.0): + if not pts: + return "" + d = " ".join(f"{x:.1f},{y:.1f}" for x, y in pts) + return f'' + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("commit_timing") + ap.add_argument("--stage-file", default=None) + ap.add_argument("--out-dir", default="perf-artifacts") + ap.add_argument("--bins", type=int, default=150) + ap.add_argument("--metric", default="block_bytes") + ap.add_argument("--scale", default="kb", choices=["kb", "mb", "raw", "ms"]) + ap.add_argument("--name", default=None) + args = ap.parse_args() + + sf = {"kb": 1 / 1024, "mb": 1 / 1048576, "raw": 1.0, "ms": 1.0}[args.scale] + unit = {"kb": "KB", "mb": "MB", "raw": "", "ms": "ms"}[args.scale] + + rows = sorted(load(args.commit_timing), key=lambda r: r["height"]) + if not rows: + raise SystemExit("no rows") + h0, h1 = rows[0]["height"], rows[-1]["height"] + width = max(1, math.ceil((h1 - h0 + 1) / args.bins)) + + def bin_by(items, key): + b = {} + for it in items: + b.setdefault((key(it) - h0) // width, []).append(it) + return b + + # commit-timing bins: metric p50/p90 + cb = bin_by(rows, lambda r: r["height"]) + mbins = [] + for k in sorted(cb): + rs = cb[k] + vals = [float(r.get(args.metric, 0.0)) * sf for r in rs] + mbins.append({"h": h0 + k * width + width // 2, + "p50": percentile(vals, 50), "p90": percentile(vals, 90)}) + + # stage-timing: per-category mean keys/block + per-height commit timestamps. + # Throughput is taken from the stage stream when available (every block, so far + # finer than the sampled commit-timing, and unskewed by the commit-trace cost). + kbins = None + ts_pairs = [(r["height"], r["ts_us"]) for r in rows] # fallback: sampled commit-timing + if args.stage_file and Path(args.stage_file).exists(): + per_height = {} + per_height_ts = {} + for r in load(args.stage_file): + s = r.get("stage", "") + h = r["height"] + if s.startswith("keys_"): + per_height.setdefault(h, {})[s] = r.get("val", 0) + ts = r.get("ts_us") + if ts is not None: + per_height_ts[h] = max(per_height_ts.get(h, 0), ts) + if per_height_ts: + ts_pairs = sorted(per_height_ts.items()) + items = [{"h": h, **v} for h, v in per_height.items()] + sb = bin_by(items, lambda it: it["h"]) + kbins = [] + for k in sorted(sb): + rs = sb[k] + row = {"h": h0 + k * width + width // 2} + for label, _c, stages in KEY_CATS: + row[label] = sum(sum(it.get(s, 0) for s in stages) for it in rs) / len(rs) + kbins.append(row) + + # throughput bins from the chosen timestamp stream + tb = bin_by([{"h": h, "ts": ts} for h, ts in ts_pairs], lambda it: it["h"]) + tbins = [] + for k in sorted(tb): + rs = sorted(tb[k], key=lambda it: it["h"]) + bps = None + if len(rs) >= 2: + dt = (rs[-1]["ts"] - rs[0]["ts"]) / 1e6 + dh = rs[-1]["h"] - rs[0]["h"] + if dt > 0 and dh > 0: + bps = dh / dt + tbins.append({"h": h0 + k * width + width // 2, "bps": bps}) + + # ---- layout ---- + W, left, right = 1240, 96, 36 + plot_w = W - left - right + panel_h, gap = 280, 86 + n_panels = 3 if kbins else 2 + tops = [104 + i * (panel_h + gap) for i in range(n_panels)] + H = tops[-1] + panel_h + 64 + + m_max = nice_max(max([b["p90"] for b in mbins] + [1.0])) + bps_max = nice_max(max([b["bps"] for b in tbins if b["bps"] is not None] + [1.0]) * 1.05) + keys_max = nice_max(max([sum(b[l] for l, _c, _s in KEY_CATS) for b in kbins] + [1.0])) if kbins else 1.0 + + svg = [ + f'', + f'', + ] + title = args.name or Path(args.commit_timing).parent.name + svg.append(f'' + f"Commit profile by height — {title}") + svg.append(f'{len(rows):,} sampled blocks, ' + f"heights {h0:,}–{h1:,}, {len(mbins)} bins") + + def grid(top, vmax, fmt): + for frac in (0, 0.25, 0.5, 0.75, 1.0): + v = vmax * frac + y = lin_y(v, vmax, top, panel_h) + svg.append(f'') + svg.append(f'{fmt(v)}') + + def frame(top, segs): + svg.append(f'') + x = left + for text, color, weight in segs: + svg.append(f'{text}') + x += int(8.6 * len(text)) + + # Panel 1: metric p50/p90 + mlabel = args.metric.replace("_", " ") + frame(tops[0], [(f"{mlabel} ({unit}) — ", "#222", "600"), ("p50", BLUE, "700"), + (" / ", "#888", "400"), ("p90", RED, "700")]) + grid(tops[0], m_max, lambda v: f"{v:.0f} {unit}") + svg.append(polyline([(x_map(b["h"], h0, h1, left, plot_w), lin_y(b["p90"], m_max, tops[0], panel_h)) for b in mbins], RED)) + svg.append(polyline([(x_map(b["h"], h0, h1, left, plot_w), lin_y(b["p50"], m_max, tops[0], panel_h)) for b in mbins], BLUE)) + + # Panel 2: throughput (instantaneous per-bin) + overall sustained reference line + overall_bps = None + if len(ts_pairs) >= 2 and ts_pairs[-1][1] > ts_pairs[0][1]: + overall_bps = (ts_pairs[-1][0] - ts_pairs[0][0]) / ((ts_pairs[-1][1] - ts_pairs[0][1]) / 1e6) + frame(tops[1], [("throughput (blk/s) — ", "#222", "600"), + ("per-bin instantaneous", GREEN, "700"), + (f" overall sustained = {overall_bps:.0f}" if overall_bps else "", "#666", "400")]) + grid(tops[1], bps_max, lambda v: f"{v:.0f}") + if overall_bps: + y = lin_y(overall_bps, bps_max, tops[1], panel_h) + svg.append(f'') + svg.append(polyline([(x_map(b["h"], h0, h1, left, plot_w), lin_y(b["bps"], bps_max, tops[1], panel_h)) + for b in tbins if b["bps"] is not None], GREEN)) + + # Panel 3: stacked keys by category + if kbins: + segs = [("commit batch keys/block, stacked: ", "#222", "600")] + for label, color, _s in KEY_CATS: + segs.append((label, color, "700")) + segs.append((" ", "#888", "400")) + frame(tops[2], segs) + grid(tops[2], keys_max, lambda v: f"{v:.0f}") + cum = [0.0] * len(kbins) + for label, color, _s in KEY_CATS: + top_edge, bot_edge = [], [] + for i, b in enumerate(kbins): + x = x_map(b["h"], h0, h1, left, plot_w) + lo = cum[i] + hi = cum[i] + b[label] + cum[i] = hi + top_edge.append((x, lin_y(hi, keys_max, tops[2], panel_h))) + bot_edge.append((x, lin_y(lo, keys_max, tops[2], panel_h))) + pts = top_edge + bot_edge[::-1] + d = " ".join(f"{x:.1f},{y:.1f}" for x, y in pts) + svg.append(f'') + + # shared x ticks + for i in range(6): + h = h0 + (h1 - h0) * i / 5 + x = x_map(h, h0, h1, left, plot_w) + for top in tops: + svg.append(f'') + svg.append(f'{int(h):,}') + svg.append(f'block height') + svg.append("") + + out_dir = Path(args.out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + out = out_dir / f"{title}-commit-profile-by-height.svg" + out.write_text("\n".join(svg)) + print(f"wrote {out}") + # time-weighted regional throughput (blocks / wall-time), NOT mean of bin rates + def region_rate(lo, hi): + sel = [(h, t) for h, t in ts_pairs if lo <= h < hi] + if len(sel) < 2 or sel[-1][1] <= sel[0][1]: + return None + return (sel[-1][0] - sel[0][0]) / ((sel[-1][1] - sel[0][1]) / 1e6) + print(" throughput (time-weighted, blk/s):") + for name, lo, hi in [("overall", h0, h1 + 1), ("light <1862900", h0, 1862900), + ("sandblast 1862.9-1872k", 1862900, 1872000), + ("recovery 1872k-end", 1872000, h1 + 1)]: + r = region_rate(lo, hi) + print(f" {name:26} {r:7.1f}" if r else f" {name:26} n/a") + if kbins: + def avg(rs, l): return sum(b[l] for b in rs) / len(rs) if rs else 0 + light = [b for b in kbins if b["h"] < 1862900] + sand = [b for b in kbins if 1862900 <= b["h"] <= 1872000] + print(" keys/block (mean):") + for l, _c, _s in KEY_CATS: + print(f" {l:28} light~{avg(light,l):7.0f} sandblast~{avg(sand,l):7.0f}") + + +if __name__ == "__main__": + main() diff --git a/Cargo.lock b/Cargo.lock index 00d62c2d75e..67edea8ecb5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9314,6 +9314,7 @@ version = "0.1.0" dependencies = [ "clap 4.6.1", "color-eyre", + "crossbeam-channel", "futures", "metrics-exporter-prometheus", "tempfile", diff --git a/zebra-chain/src/lib.rs b/zebra-chain/src/lib.rs index 9721ea7df19..d84eb49704c 100644 --- a/zebra-chain/src/lib.rs +++ b/zebra-chain/src/lib.rs @@ -38,6 +38,7 @@ pub mod sapling; pub mod serialization; pub mod shutdown; pub mod sprout; +pub mod stage_timing; pub mod subtree; pub mod transaction; pub mod transparent; diff --git a/zebra-chain/src/stage_timing.rs b/zebra-chain/src/stage_timing.rs new file mode 100644 index 00000000000..23b71601141 --- /dev/null +++ b/zebra-chain/src/stage_timing.rs @@ -0,0 +1,79 @@ +//! Optional per-height pipeline-stage timing trace for offline profiling. +//! +//! When `ZEBRA_STAGE_TIMING_TRACE=` is set, [`record`] appends one JSON row +//! per call, pairing a block `height` with a named pipeline `stage` and a wall-clock +//! timestamp (microseconds since the Unix epoch). Because the timestamp is wall-clock +//! and the facility is process-global, callers in different crates and threads (the +//! apply driver, the consensus verifier, the state service, the write worker) all +//! land on the same timeline, so an offline join by `height` localizes where a block +//! spends its time between stages. +//! +//! This lives in `zebra-chain` only because it is the common crate every stage +//! depends on; it has no consensus role. The hot path is a single `OnceLock` load +//! when the env var is unset (the default), so it stays inert in production. + +use std::{ + fs::File, + io::{BufWriter, Write}, + sync::{Mutex, OnceLock}, + time::{SystemTime, UNIX_EPOCH}, +}; + +static TRACE: OnceLock>>> = OnceLock::new(); + +fn writer() -> Option<&'static Mutex>> { + TRACE + .get_or_init(|| { + let path = std::env::var_os("ZEBRA_STAGE_TIMING_TRACE")?; + match File::create(&path) { + Ok(file) => Some(Mutex::new(BufWriter::new(file))), + Err(_) => None, + } + }) + .as_ref() +} + +/// Whether stage timing is active (`ZEBRA_STAGE_TIMING_TRACE` is set). Lets callers +/// skip building trace-only aggregates on the hot path when tracing is off. +pub fn enabled() -> bool { + writer().is_some() +} + +/// Records that block `height` reached pipeline `stage`, with a wall-clock +/// (epoch-microsecond) timestamp, when `ZEBRA_STAGE_TIMING_TRACE` is set. +/// +/// A cheap no-op (one `OnceLock` load) when the env var is unset. The row is small +/// and carries no re-serialized block data, so the per-call cost is a timestamp read +/// plus a buffered write — negligible relative to the stages it measures. +pub fn record(height: u32, stage: &str) { + write_row(height, stage, None); +} + +/// Like [`record`], but also tags the row with a numeric `value` (e.g. an in-flight +/// queue depth at this stage), under the JSON key `"val"`. +pub fn record_val(height: u32, stage: &str, value: u64) { + write_row(height, stage, Some(value)); +} + +fn write_row(height: u32, stage: &str, value: Option) { + let Some(writer) = writer() else { return }; + let ts_us = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_micros()) + .unwrap_or(0); + if let Ok(mut file) = writer.lock() { + // Flush each row so a killed/partial run is still inspectable; the write is a + // tiny line (no block re-serialization), so this stays off the critical path. + let _ = match value { + Some(v) => writeln!( + file, + r#"{{"height":{height},"stage":"{stage}","ts_us":{ts_us},"val":{v}}}"# + ), + None => writeln!( + file, + r#"{{"height":{height},"stage":"{stage}","ts_us":{ts_us}}}"# + ), + }; + let _ = file.flush(); + } +} diff --git a/zebra-consensus/src/checkpoint.rs b/zebra-consensus/src/checkpoint.rs index 784bcd1464f..ac6c7700a83 100644 --- a/zebra-consensus/src/checkpoint.rs +++ b/zebra-consensus/src/checkpoint.rs @@ -934,12 +934,17 @@ where expected_hash: AuthenticatedCheckpointHash, ) -> Pin> + Send + 'static>> { + zebra_chain::stage_timing::record( + block.coinbase_height().map_or(0, |h| h.0), + "verifier_call_start", + ); // Per-block validity checks (height, proof of work, Merkle root), the same checks the // accumulation path runs per block — defense in depth on top of the authenticated header. let block = match self.check_block(block) { Ok(block) => block, Err(e) => return async move { Err(e) }.boxed(), }; + zebra_chain::stage_timing::record(block.height.0, "verifier_checked"); // Bind the body to the authenticated header. A mismatch means the downloaded body and the // checkpoint-authenticated header disagree: a hard invariant violation, never recoverable. diff --git a/zebra-network/src/zakura/block_sync/bench.rs b/zebra-network/src/zakura/block_sync/bench.rs index 47959004700..e4044e49dd5 100644 --- a/zebra-network/src/zakura/block_sync/bench.rs +++ b/zebra-network/src/zakura/block_sync/bench.rs @@ -227,6 +227,12 @@ impl BenchSequencerHandle { committer, } = self; + // Highest committed height the driver has reported, so the feed can backpressure + // on the *committed* tip (the submit window caps submission, but the sequencer's + // `applying` set drains all contiguous fed bodies unbounded — a fast feed would + // otherwise pile up the whole window in memory and OOM). + let committed_tip = Arc::new(AtomicU64::new(0)); + let committed_tip_shim = committed_tip.clone(); let control = committer.control.clone(); let (lifecycle_tx, mut lifecycle_rx) = mpsc::unbounded_channel::(); tokio::spawn(async move { @@ -239,6 +245,7 @@ impl BenchSequencerHandle { local_frontier, } = event { + committed_tip_shim.fetch_max(u64::from(height.0), Ordering::Relaxed); if control .send(SequencerControlInput::ApplyFinished { token, @@ -274,6 +281,7 @@ impl BenchSequencerHandle { actions: submissions.actions, block_sync, committer, + committed_tip, } } } @@ -293,6 +301,9 @@ pub struct BenchDriverParts { pub block_sync: BlockSyncHandle, /// Progress/trace snapshots; retained to keep the sequencer task alive. pub committer: BenchCommitter, + /// Highest committed height reported by the driver. The feed reads this to cap how + /// far ahead of the committed tip it runs, bounding the in-flight body backlog. + pub committed_tip: Arc, } impl BenchBodyFeeder { diff --git a/zebra-replay-bench/Cargo.toml b/zebra-replay-bench/Cargo.toml index f0046387919..abd900adb76 100644 --- a/zebra-replay-bench/Cargo.toml +++ b/zebra-replay-bench/Cargo.toml @@ -31,6 +31,7 @@ zebrad = { path = "../zebrad", version = "5.0.0-rc.3", default-features = false, clap = { workspace = true, features = ["derive"] } color-eyre.workspace = true +crossbeam-channel = { workspace = true } futures.workspace = true tower = { workspace = true, features = ["buffer", "util"] } tokio = { workspace = true, features = ["sync", "rt-multi-thread", "macros", "time"] } diff --git a/zebra-replay-bench/src/apply_sequencer.rs b/zebra-replay-bench/src/apply_sequencer.rs index f229c0745df..37d0eaa4fc5 100644 --- a/zebra-replay-bench/src/apply_sequencer.rs +++ b/zebra-replay-bench/src/apply_sequencer.rs @@ -240,6 +240,7 @@ pub fn run( actions, block_sync, mut committer, + committed_tip, } = spawn_bench_sequencer( Height(expected_parent), Height(expected_parent), @@ -276,19 +277,51 @@ pub fn run( }, )); - let (_producer, rx) = prefetch::spawn_raw(reader, in_flight); + // Block deserialization is the single-threaded feed's dominant cost, which + // starves the apply pipeline. Deserialize on `ZRB_PREFETCH_WORKERS` threads + // (default 4) so the feed runs deep ahead and the real commit/verify ceiling + // shows. The sequencer reorders, so out-of-order producer output is fine. + let prefetch_workers = std::env::var("ZRB_PREFETCH_WORKERS") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(4); + let (_producers, rx) = prefetch::spawn_raw_parallel(reader, in_flight, prefetch_workers); - // Feed task: stream raw bodies into the sequencer's reorder queue in height - // order, up to the last checkpoint (`feed_target`). Accumulates fed bytes for - // the throughput report. + // Feed backpressure: cap how far ahead of the committed tip the feed runs, so a + // fast (parallel) feed can't pile the whole window into the sequencer's unbounded + // `applying` set and OOM. Count-based (not per-height) so it's deadlock-safe with + // the out-of-order parallel producer. Seed the committed tip to the base. + committed_tip.store(u64::from(expected_parent), Ordering::Relaxed); + let max_feed_ahead: u64 = std::env::var("ZRB_MAX_FEED_AHEAD") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(2000); + + // Feed task: stream raw bodies into the sequencer's reorder queue, up to the last + // checkpoint (`feed_target`). Accumulates fed bytes for the throughput report. let fed_bytes = Arc::new(AtomicU64::new(0)); let feed_bytes = fed_bytes.clone(); let feed = tokio::spawn(async move { let mut fed = 0u32; while fed < feed_target { + // Backpressure: hold the feed once it is `max_feed_ahead` blocks past the + // committed tip, so the in-flight body backlog stays bounded. Count-based + // (`start + fed` vs committed), so a high out-of-order block never blocks a + // lower one the commit needs. Leaving blocks in the prefetch ring stalls + // the producers, propagating the bound upstream. + while (u64::from(start) + u64::from(fed)) + .saturating_sub(committed_tip.load(Ordering::Relaxed)) + > max_feed_ahead + { + tokio::time::sleep(Duration::from_millis(1)).await; + } // The prefetch producer is a std thread; do the blocking recv off the - // async executor. + // async executor. Time it: a long recv wait = the bench's deserialize + // supply is the gate; a long feed_body = the sequencer is backpressuring + // (the real apply pipeline downstream is the gate). + let recv_start = Instant::now(); let item = tokio::task::block_in_place(|| rx.recv()); + let recv_us = recv_start.elapsed().as_micros() as u64; let raw = match item { Ok(Ok(p)) => p, Ok(Err(e)) => return Err(e), @@ -298,7 +331,16 @@ pub fn run( let hash = raw.block.hash(); let len = raw.len as u64; feed_bytes.fetch_add(len, Ordering::Relaxed); - if !feeder.feed_body(Height(height), hash, raw.block, len).await { + zebra_chain::stage_timing::record_val(height, "prefetch_recv_us", recv_us); + zebra_chain::stage_timing::record(height, "body_fed"); + let feed_start = Instant::now(); + let accepted = feeder.feed_body(Height(height), hash, raw.block, len).await; + zebra_chain::stage_timing::record_val( + height, + "feed_body_us", + feed_start.elapsed().as_micros() as u64, + ); + if !accepted { break; // sequencer gone } fed += 1; diff --git a/zebra-replay-bench/src/prefetch.rs b/zebra-replay-bench/src/prefetch.rs index 7d44e0144e3..76536ab6044 100644 --- a/zebra-replay-bench/src/prefetch.rs +++ b/zebra-replay-bench/src/prefetch.rs @@ -17,6 +17,7 @@ use std::{ Arc, }, thread::{self, JoinHandle}, + time::Instant, }; use color_eyre::eyre::{eyre, Result}; @@ -129,6 +130,9 @@ pub fn spawn_raw( let mut height = reader.header().start_height; let handle = thread::spawn(move || loop { + // Split the single-threaded producer's per-block cost: disk read vs shielded + // deserialize (point decompression) vs send backpressure (consumer slow). + let read_start = Instant::now(); let bytes = match reader.next_block() { Ok(Some(bytes)) => bytes, Ok(None) => return, @@ -137,7 +141,9 @@ pub fn spawn_raw( return; } }; + let read_us = read_start.elapsed().as_micros() as u64; let len = bytes.len(); + let deser_start = Instant::now(); let block = match Block::zcash_deserialize(&bytes[..]) { Ok(block) => Arc::new(block), Err(e) => { @@ -145,8 +151,21 @@ pub fn spawn_raw( return; } }; + zebra_chain::stage_timing::record_val(height, "prefetch_read_us", read_us); + zebra_chain::stage_timing::record_val( + height, + "prefetch_deser_us", + deser_start.elapsed().as_micros() as u64, + ); - if tx.send(Ok(RawBlock { block, len, height })).is_err() { + let send_start = Instant::now(); + let sent = tx.send(Ok(RawBlock { block, len, height })); + zebra_chain::stage_timing::record_val( + height, + "prefetch_send_us", + send_start.elapsed().as_micros() as u64, + ); + if sent.is_err() { // Consumer dropped the receiver: stop producing. return; } @@ -155,3 +174,134 @@ pub fn spawn_raw( (handle, rx) } + +/// Like [`spawn_raw`], but deserializes on `workers` threads in parallel. +/// +/// Block deserialization (shielded point decompression) is the single-threaded +/// producer's dominant cost, so this splits it: one reader thread streams raw bytes +/// off the cache (cheap, mostly page-cached) into an MPMC work queue, and `workers` +/// threads deserialize concurrently. Output arrives in *roughly* — not strictly — +/// height order (workers finish out of order); the Zakura sequencer reorders, so the +/// caller may feed bodies as received. `workers <= 1` falls back to [`spawn_raw`]. +pub fn spawn_raw_parallel( + reader: CacheReader, + capacity: usize, + workers: usize, +) -> (Vec>, Receiver>) { + let workers = workers.max(1); + if workers == 1 { + let (handle, rx) = spawn_raw(reader, capacity); + return (vec![handle], rx); + } + + let start_height = reader.header().start_height; + let (out_tx, out_rx) = sync_channel(capacity); + let (work_tx, work_rx) = crossbeam_channel::bounded::<(u32, Vec)>(capacity); + // Workers emit (height, result) out of order; the reorder thread serializes them. + // The worker->reorder channel is intentionally unbounded so the reorder thread can + // always accept the (out-of-order) block it is waiting for. Read-ahead is instead + // bounded by `permit`: at most `capacity` blocks may be read but not yet emitted. + // Without this bound, a backpressured consumer (e.g. a slow sandblast region) lets + // the reader race ahead and deserialize the whole window into RAM (each block's + // Orchard Halo2 proof is large), exhausting memory. + let (result_tx, result_rx) = std::sync::mpsc::channel::<(u32, Result)>(); + let (permit_tx, permit_rx) = crossbeam_channel::bounded::<()>(capacity); + for _ in 0..capacity { + let _ = permit_tx.send(()); + } + let mut handles = Vec::with_capacity(workers + 2); + + // Reader thread: sequential disk read -> work queue (errors go to the reorder). + let mut reader = reader; + let mut height = start_height; + let reader_result = result_tx.clone(); + handles.push(thread::spawn(move || loop { + // Acquire a read-ahead permit (released by the reorder thread once a block is + // emitted downstream). Closed channel => consumer is gone, so stop reading. + if permit_rx.recv().is_err() { + return; + } + let read_start = Instant::now(); + let bytes = match reader.next_block() { + Ok(Some(bytes)) => bytes, + Ok(None) => return, + Err(e) => { + let _ = reader_result.send((height, Err(eyre!("reading block {height}: {e}")))); + return; + } + }; + zebra_chain::stage_timing::record_val( + height, + "prefetch_read_us", + read_start.elapsed().as_micros() as u64, + ); + if work_tx.send((height, bytes)).is_err() { + return; + } + height += 1; + })); + + // Deserialize workers: pull bytes, deserialize (the expensive part), emit (height, result). + for _ in 0..workers { + let work_rx = work_rx.clone(); + let result_tx = result_tx.clone(); + handles.push(thread::spawn(move || { + while let Ok((height, bytes)) = work_rx.recv() { + let len = bytes.len(); + let deser_start = Instant::now(); + let item = match Block::zcash_deserialize(&bytes[..]) { + Ok(block) => Ok(RawBlock { + block: Arc::new(block), + len, + height, + }), + Err(e) => Err(eyre!("deserializing block {height}: {e}")), + }; + let is_err = item.is_err(); + zebra_chain::stage_timing::record_val( + height, + "prefetch_deser_us", + deser_start.elapsed().as_micros() as u64, + ); + if result_tx.send((height, item)).is_err() || is_err { + return; + } + } + })); + } + drop(result_tx); // result_rx closes once the reader + all workers finish + + // Reorder thread: emit blocks in strict contiguous height order, so the consumer's + // committed-tip backpressure is deadlock-safe (the next block the committer needs is + // always produced before any block beyond it). Pending is bounded by the read-ahead + // permit capacity, so it can't grow unbounded. + handles.push(thread::spawn(move || { + let mut pending: std::collections::BTreeMap> = + std::collections::BTreeMap::new(); + let mut next = start_height; + while let Ok((height, item)) = result_rx.recv() { + pending.insert(height, item); + while let Some(item) = pending.remove(&next) { + let stop = item.is_err(); + if out_tx.send(item).is_err() { + return; + } + // Release one read-ahead permit now that this block has left the buffer. + let _ = permit_tx.send(()); + next += 1; + if stop { + return; + } + } + } + // Producers done (normal completion drains fully above; this only runs if a gap + // remains, e.g. after an error elsewhere — emit what's left so the consumer ends). + while let Some((_, item)) = pending.pop_first() { + if out_tx.send(item).is_err() { + return; + } + } + })); + + (handles, out_rx) +} diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs index 72332c952a4..9e304a55139 100644 --- a/zebra-state/src/service.rs +++ b/zebra-state/src/service.rs @@ -617,6 +617,7 @@ impl StateService { // If we've finished sending finalized blocks, ignore any repeated blocks. // (Blocks can be repeated after a syncer reset.) if let Some(finalized_block_write_sender) = &self.block_write_sender.finalized { + zebra_chain::stage_timing::record(queued_block.0.height.0, "state_to_worker"); let send_result = finalized_block_write_sender.send(queued_block); // If the receiver is closed, we can't send any more blocks. diff --git a/zebra-state/src/service/finalized_state/commit_pressure.rs b/zebra-state/src/service/finalized_state/commit_pressure.rs index 921c6fade37..966450c6584 100644 --- a/zebra-state/src/service/finalized_state/commit_pressure.rs +++ b/zebra-state/src/service/finalized_state/commit_pressure.rs @@ -94,6 +94,18 @@ pub(crate) struct PreparedCommitTrace { /// at least the configured threshold (or on the periodic baseline tick). /// /// A cheap no-op (one `OnceLock` load) when `ZEBRA_COMMIT_PRESSURE_TRACE` is unset. +/// Resident set size of this process in MiB, read from `/proc/self/statm` +/// (field 2 = resident pages × 4 KiB page). Returns 0 if unavailable. Used to +/// distinguish RocksDB-internal growth from total-process growth in the trace. +fn process_rss_mb() -> u64 { + std::fs::read_to_string("/proc/self/statm") + .ok() + .and_then(|s| s.split_whitespace().nth(1).map(str::to_string)) + .and_then(|pages| pages.parse::().ok()) + .map(|pages| pages * 4096 / (1024 * 1024)) + .unwrap_or(0) +} + pub(super) fn record_commit(db: &DiskDb, prepared: PreparedCommitTrace, batch_commit: Duration) { let Some(trace) = trace() else { return }; let n = trace.count.fetch_add(1, Ordering::Relaxed); @@ -122,7 +134,7 @@ pub(super) fn record_commit(db: &DiskDb, prepared: PreparedCommitTrace, batch_co r#""batch_assembly_ms":{:.3},"assemble_self_ms":{:.3},"queue_wait_ms":{:.3},"batch_commit_ms":{:.3},"commit_total_ms":{:.3},"#, r#""block_bytes":{},"tx_count":{},"output_count":{},"batch_keys":{},"batch_bytes":{},"#, r#""l0_files":{},"pending_compaction_bytes":{},"running_compactions":{},"#, - r#""running_flushes":{},"memtable_bytes":{},"total_sst_bytes":{},"live_data_bytes":{}}}"# + r#""running_flushes":{},"memtable_bytes":{},"total_sst_bytes":{},"live_data_bytes":{},"rss_mb":{}}}"# ), trace.start.elapsed().as_micros(), prepared.height, @@ -147,6 +159,7 @@ pub(super) fn record_commit(db: &DiskDb, prepared: PreparedCommitTrace, batch_co p.memtable_bytes, p.total_sst_bytes, p.live_data_bytes, + process_rss_mb(), ); if let Ok(mut file) = trace.file.lock() { diff --git a/zebra-state/src/service/finalized_state/disk_db.rs b/zebra-state/src/service/finalized_state/disk_db.rs index b2d6e24e029..0a7513cc719 100644 --- a/zebra-state/src/service/finalized_state/disk_db.rs +++ b/zebra-state/src/service/finalized_state/disk_db.rs @@ -1363,7 +1363,34 @@ impl DiskDb { // Tune level-style database file compaction. // // This improves Zebra's initial sync speed slightly, as of April 2022. - opts.optimize_level_style_compaction(Self::MEMTABLE_RAM_CACHE_MEGABYTES * ONE_MEGABYTE); + // + // The memtable budget is env-tunable for empirical benchmarking; it defaults + // to the production constant. `optimize_level_style_compaction` derives + // write_buffer_size (budget/4), max_write_buffer_number (6), the L0 trigger, + // and the level sizes from this budget. + let memtable_mb = std::env::var("ZEBRA_ROCKSDB_MEMTABLE_MB") + .ok() + .and_then(|v| v.parse::().ok()) + .unwrap_or(Self::MEMTABLE_RAM_CACHE_MEGABYTES); + opts.optimize_level_style_compaction(memtable_mb * ONE_MEGABYTE); + + // Compaction/flush parallelism. RocksDB defaults to `max_background_jobs=2` + // (effectively one flush + one compaction) and `max_subcompactions=1`. With + // random-keyed nullifier/UTXO inserts a single compaction thread cannot keep + // up at high write rates, so L0 backs up. Env-gated for empirical sweeps; + // default behavior is unchanged when unset. + if let Some(jobs) = std::env::var("ZEBRA_ROCKSDB_BG_JOBS") + .ok() + .and_then(|v| v.parse::().ok()) + { + opts.set_max_background_jobs(jobs); + } + if let Some(subc) = std::env::var("ZEBRA_ROCKSDB_SUBCOMPACTIONS") + .ok() + .and_then(|v| v.parse::().ok()) + { + opts.set_max_subcompactions(subc); + } // Increase the process open file limit if needed, // then use it to set RocksDB's limit. diff --git a/zebra-state/src/service/finalized_state/zebra_db/block.rs b/zebra-state/src/service/finalized_state/zebra_db/block.rs index e61933e3c61..b75afa8fd82 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block.rs @@ -1187,6 +1187,35 @@ impl ZebraDb { metrics::histogram!("zebra.state.write.spent_utxo_reads.duration_seconds") .record(reads_dur.as_secs_f64()); + // Spend-distance instrumentation (trace-only): how many blocks back was each + // spent UTXO created? The cumulative buckets give the hit rate an in-memory + // UTXO cache of a given size would achieve, since the run-ahead overlay already + // serves only the pipeline-depth window. Built only when stage timing is on. + if zebra_chain::stage_timing::enabled() { + let spend_h = finalized.height.0; + let (mut le64, mut le256, mut le1024, mut le4096) = (0u64, 0u64, 0u64, 0u64); + for (_op, out_loc, _utxo) in &spent_utxos { + let d = spend_h.saturating_sub(out_loc.height().0); + if d <= 64 { + le64 += 1; + } + if d <= 256 { + le256 += 1; + } + if d <= 1024 { + le1024 += 1; + } + if d <= 4096 { + le4096 += 1; + } + } + zebra_chain::stage_timing::record_val(spend_h, "spent_total", spent_utxos.len() as u64); + zebra_chain::stage_timing::record_val(spend_h, "spent_le64", le64); + zebra_chain::stage_timing::record_val(spend_h, "spent_le256", le256); + zebra_chain::stage_timing::record_val(spend_h, "spent_le1024", le1024); + zebra_chain::stage_timing::record_val(spend_h, "spent_le4096", le4096); + } + let spent_utxos_by_outpoint: HashMap = spent_utxos .iter() @@ -1794,6 +1823,12 @@ impl DiskWriteBatch { // absolute address balances for the run-ahead pipeline overlay. capture_pipeline_outputs: bool, ) -> Result { + // Per-category key-count attribution (env-gated; no-op when stage timing + // is off). Each step's `batch.len()` delta is how many keys/deletes that + // category contributes to the commit batch (the compaction input). + let kh = finalized.height.0; + let mut kp = self.len(); + // Commit block, transaction, and note commitment tree data. self.prepare_block_header_and_transaction_data_batch( zebra_db, @@ -1801,11 +1836,16 @@ impl DiskWriteBatch { store_raw_transactions, precomputed_raw_txs, )?; + zebra_chain::stage_timing::record_val(kh, "keys_header_tx", (self.len() - kp) as u64); + kp = self.len(); + let zakura_header_commitment_roots_by_height = zebra_db .db .cf_handle(ZAKURA_HEADER_COMMITMENT_ROOTS_BY_HEIGHT) .unwrap(); self.zs_delete(&zakura_header_commitment_roots_by_height, finalized.height); + zebra_chain::stage_timing::record_val(kh, "keys_vct_evict", (self.len() - kp) as u64); + kp = self.len(); // The consensus rules are silent on shielded transactions in the genesis block, // because there aren't any in the mainnet or testnet genesis blocks. @@ -1814,6 +1854,8 @@ impl DiskWriteBatch { // // In Zebra we include the nullifiers and note commitments in the genesis block because it simplifies our code. self.prepare_shielded_transaction_batch(zebra_db, finalized); + zebra_chain::stage_timing::record_val(kh, "keys_shielded", (self.len() - kp) as u64); + kp = self.len(); // The `vct_upgrade_height` marker is written once, by the first block this // binary commits. In the run-ahead pipeline an earlier not-yet-flushed block @@ -1830,6 +1872,8 @@ impl DiskWriteBatch { vct_sync_below, wrote_vct_upgrade_marker, ); + zebra_chain::stage_timing::record_val(kh, "keys_trees", (self.len() - kp) as u64); + kp = self.len(); // # Consensus // @@ -1857,6 +1901,8 @@ impl DiskWriteBatch { capture_pipeline_outputs, ); } + zebra_chain::stage_timing::record_val(kh, "keys_transparent", (self.len() - kp) as u64); + kp = self.len(); // Commit UTXOs and value pools. This runs for every height, including // genesis (which writes the initial pool and block info), matching the @@ -1867,6 +1913,7 @@ impl DiskWriteBatch { spent_utxos_by_outpoint, value_pool, )?; + zebra_chain::stage_timing::record_val(kh, "keys_valuepool", (self.len() - kp) as u64); // The block has passed contextual validation, so update the metrics block_precommit_metrics(&finalized.block, finalized.hash, finalized.height); diff --git a/zebra-state/src/service/tests.rs b/zebra-state/src/service/tests.rs index bf0a2a230c6..f17d585961b 100644 --- a/zebra-state/src/service/tests.rs +++ b/zebra-state/src/service/tests.rs @@ -284,7 +284,10 @@ async fn pipelined_state_service_commits_chain() -> Result<()> { committed, expected, "every queued block should commit through the pipeline" ); - assert!(expected >= 2, "the test chain must have at least two blocks"); + assert!( + expected >= 2, + "the test chain must have at least two blocks" + ); Ok(()) } diff --git a/zebra-state/src/service/write.rs b/zebra-state/src/service/write.rs index 5e7939bd88a..006028e923e 100644 --- a/zebra-state/src/service/write.rs +++ b/zebra-state/src/service/write.rs @@ -584,7 +584,13 @@ impl WriteBlockWorkerTask { { Some(block) => block, None => match finalized_block_write_receiver.try_recv() { - Ok(block) => block, + Ok(block) => { + zebra_chain::stage_timing::record( + block.0.height.0, + "write_worker_received", + ); + block + } Err(TryRecvError::Empty) => { // Starved: no finalized block available. Time the park so we can // tell "assembler is upstream-starved (verify/driver/feed slow)" @@ -656,6 +662,7 @@ impl WriteBlockWorkerTask { // sizes plus this block's note counts (the sizes after this block). if finalized_lookahead.is_empty() { if let Ok(next) = finalized_block_write_receiver.try_recv() { + zebra_chain::stage_timing::record(next.0.height.0, "write_worker_received"); finalized_lookahead.push_back(next); } } @@ -672,18 +679,19 @@ impl WriteBlockWorkerTask { hash = ?ordered_block.0.hash, "VCT: deferring fast checkpoint commit until successor is buffered" ); + // First time this iteration we discover N's successor isn't buffered: + // anchors "worker started waiting for N+1" so an offline join can compare + // it against when N+1 was actually sent to the worker (state_to_worker). + zebra_chain::stage_timing::record(ordered_block.0.height.0, "successor_wait_start"); retry_finalized_block = Some(ordered_block); // VCT one-block look-ahead stall: this fast block can't commit until its // successor is buffered (needed to authenticate its supplied roots). If // this dominates, the 10ms poll granularity (not real work) is the gate. let park = std::time::Instant::now(); std::thread::park_timeout(Duration::from_millis(10)); - metrics::counter!("zebra.state.write.assembler_successor_defer.count") - .increment(1); - metrics::histogram!( - "zebra.state.write.assembler_successor_park.duration_seconds" - ) - .record(park.elapsed().as_secs_f64()); + metrics::counter!("zebra.state.write.assembler_successor_defer.count").increment(1); + metrics::histogram!("zebra.state.write.assembler_successor_park.duration_seconds") + .record(park.elapsed().as_secs_f64()); continue; } diff --git a/zebrad/src/commands/start/zakura/block_sync_driver.rs b/zebrad/src/commands/start/zakura/block_sync_driver.rs index 6ff01b797e7..0aa2ff69df9 100644 --- a/zebrad/src/commands/start/zakura/block_sync_driver.rs +++ b/zebrad/src/commands/start/zakura/block_sync_driver.rs @@ -477,6 +477,14 @@ pub async fn drive_block_sync_actions( BlockSyncAction::SubmitBlock { token, block } => { let class = block_apply_class(block.as_ref(), max_checkpoint_height); let height = block.coinbase_height(); + // The pending-apply backlog when this SubmitBlock arrives: a large + // backlog means the sequencer is emitting well ahead and the driver's + // dispatch is the limiter; ~0 means the sequencer feeds just-in-time. + zebra_chain::stage_timing::record_val( + height.map_or(0, |h| h.0), + "submit_received", + pending_applies.len() as u64, + ); emit_commit_state( &trace, cs_trace::BLOCK_SUBMIT_QUEUED, @@ -762,6 +770,15 @@ fn drain_pending_block_applies( } } + // How many applies are concurrently in flight at the moment this one is + // dispatched: ~1-2 means the driver is paced one-at-a-time (verify can't run + // ahead); a large depth means the driver is sprinting and the gate is elsewhere. + zebra_chain::stage_timing::record_val( + pending.block.coinbase_height().map_or(0, |h| h.0), + "apply_dispatch", + (*checkpoint_in_flight + *full_in_flight) as u64, + ); + let class = pending.class; in_flight_applies.push( apply_block_sync_body( @@ -942,6 +959,7 @@ where }; }; + zebra_chain::stage_timing::record(height.0, "driver_commit_start"); emit_commit_state(&trace, cs_trace::COMMIT_START, "block_sync_driver", |row| { insert_cs_u64(row, cs_trace::APPLY_TOKEN, token); insert_cs_str(row, cs_trace::APPLY_CLASS, block_apply_class_label(class)); @@ -963,6 +981,7 @@ where } else { None }; + zebra_chain::stage_timing::record(height.0, "auth_hash_done"); if class == BlockApplyClass::Checkpoint && checkpoint_auth_hash.is_none() { // Hard sync-invariant violation: there is no fallback into range accumulation. We From d59811dc24689ef0fa3a8b1620c4a7791162357d Mon Sep 17 00:00:00 2001 From: Roman Date: Tue, 30 Jun 2026 18:33:15 +0000 Subject: [PATCH 07/17] perf(state): skip transparent address index in pruned checkpoint-sync The transparent address index (balances, address->utxo, address->tx) is RPC-only state, not consensus. A node that is both pruned and checkpoint-syncing now omits it -- just as pruned mode already drops raw-transaction storage -- removing the per-block address-balance reads and the address-index writes. In high address-churn regions (e.g. the 2022 sandblast consolidation) the index was ~74% of the transparent write volume, so dropping it is a large commit speedup; the UTXO set, value pool, and nullifiers are byte-identical. - Gate: `Config::skip_address_index()` = `Pruned && checkpoint_sync`. An archive node keeps the index; so does any node with checkpoint sync disabled (full semantic verification), even when pruned. Unit test covers all three branches. - Commit path: `block.rs`/`transparent.rs` elide the address-balance reads and the three address index column families when skipping; the UTXO-set writes/deletes still run. - Query path: the `getaddressbalance`/`getaddressutxos`/`getaddresstxids` read handlers return a clear "address index disabled in pruned storage mode" error rather than wrong (empty) results. Also refines the offline spend-distance instrumentation buckets and adds an A/B throughput/keys-by-height plot script. --- .../scripts/plot_ab_throughput_keys.py | 174 ++++++++++++++++++ zebra-state/src/config.rs | 50 +++++ zebra-state/src/service.rs | 56 ++++-- .../service/finalized_state/zebra_db/block.rs | 102 +++++----- .../finalized_state/zebra_db/transparent.rs | 67 ++++--- 5 files changed, 363 insertions(+), 86 deletions(-) create mode 100644 .cursor/skills/zakura-trace-plots/scripts/plot_ab_throughput_keys.py diff --git a/.cursor/skills/zakura-trace-plots/scripts/plot_ab_throughput_keys.py b/.cursor/skills/zakura-trace-plots/scripts/plot_ab_throughput_keys.py new file mode 100644 index 00000000000..85b93f5cedd --- /dev/null +++ b/.cursor/skills/zakura-trace-plots/scripts/plot_ab_throughput_keys.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +"""Overlay two stage-timing runs: throughput and transparent keys/block by height. + +For A/B comparisons where each run has a stage-timing.jsonl (per-block keys_* and +ts_us). Throughput comes from the stage timestamps, so it is NOT skewed by any +commit-timing trace overhead — both runs are measured the same, lightweight way. + +Usage: + python3 plot_ab_throughput_keys.py LABEL_A=stageA.jsonl LABEL_B=stageB.jsonl \ + --out-dir perf-artifacts --name skip-ab [--bins 90] +""" +import argparse +import json +import math +from pathlib import Path + +COLORS = ["#d62728", "#2ca02c", "#1f77b4", "#ff7f0e"] + + +def load(path): + ts, ktrans = {}, {} + for line in open(path): + line = line.strip() + if not line: + continue + try: + r = json.loads(line) + except json.JSONDecodeError: + continue + s, h = r.get("stage", ""), r.get("height") + if h is None: + continue + if s.startswith("keys_"): + if s == "keys_transparent": + ktrans[h] = r.get("val", 0) + ts[h] = max(ts.get(h, 0), r.get("ts_us", 0)) + return ts, ktrans + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("runs", nargs="+", help="LABEL=stage.jsonl") + ap.add_argument("--out-dir", default="perf-artifacts") + ap.add_argument("--name", default="ab") + ap.add_argument("--bins", type=int, default=90) + args = ap.parse_args() + + runs = [] + for spec in args.runs: + label, path = spec.split("=", 1) + ts, kt = load(path) + runs.append((label, ts, kt)) + + h0 = min(min(ts) for _, ts, _ in runs) + h1 = max(max(ts) for _, ts, _ in runs) + width = max(1, math.ceil((h1 - h0 + 1) / args.bins)) + + def throughput_bins(ts): + buckets = {} + for h, t in ts.items(): + buckets.setdefault((h - h0) // width, []).append((h, t)) + out = {} + for k, pts in buckets.items(): + pts.sort() + if len(pts) >= 2 and pts[-1][1] > pts[0][1]: + out[k] = (pts[-1][0] - pts[0][0]) / ((pts[-1][1] - pts[0][1]) / 1e6) + return out + + def key_bins(kt): + buckets = {} + for h, v in kt.items(): + buckets.setdefault((h - h0) // width, []).append(v) + return {k: sum(v) / len(v) for k, v in buckets.items()} + + series = [] + for i, (label, ts, kt) in enumerate(runs): + series.append((label, COLORS[i % len(COLORS)], throughput_bins(ts), key_bins(kt), + (h1 - h0) / ((max(ts.values()) - min(ts.values())) / 1e6) if len(ts) > 1 else 0)) + + W, left, right = 1240, 96, 36 + plot_w = W - left - right + panel_h, gap = 290, 92 + top1, top2 = 104, 104 + 290 + 92 + H = top2 + panel_h + 60 + + bps_max = max(max(s[2].values(), default=1) for s in series) * 1.05 + key_max = max(max(s[3].values(), default=1) for s in series) * 1.05 + + def xm(k): + h = h0 + k * width + width // 2 + return left + (h - h0) / (h1 - h0) * plot_w + + def ymap(v, vmax, top): + return top + panel_h - min(max(v, 0), vmax) / vmax * panel_h + + svg = [ + f'', + f'', + f'A/B by height — {args.name}', + ] + + # sandblast region band + per-run time-weighted rates for annotation + SB_LO, SB_HI = 1862900, 1872000 + + def region_rate(ts, lo, hi): + sel = sorted((h, t) for h, t in ts.items() if lo <= h < hi) + return (sel[-1][0] - sel[0][0]) / ((sel[-1][1] - sel[0][1]) / 1e6) if len(sel) > 1 and sel[-1][1] > sel[0][1] else 0 + sb_rates = [(label, color, region_rate(ts, SB_LO, SB_HI)) for (label, color, _, _, _), (_, ts, _) in zip(series, runs)] + sb_x0 = left + (SB_LO - h0) / (h1 - h0) * plot_w + sb_x1 = left + (SB_HI - h0) / (h1 - h0) * plot_w + + def panel(top, title, get, vmax, fmt, dashed=None, annotate_rates=False): + svg.append(f'') + # shade the sandblast region + svg.append(f'') + svg.append(f'sandblast') + # title + inline legend + x = left + svg.append(f'{title}: ') + x += int(8.6 * (len(title) + 3)) + for label, color, *_ in series: + svg.append(f'{label}') + x += int(8.6 * len(label)) + 14 + for frac in (0, 0.25, 0.5, 0.75, 1.0): + v = vmax * frac + y = top + panel_h - frac * panel_h + svg.append(f'') + svg.append(f'{fmt(v)}') + for label, color, tb, kb, overall in series: + data = get(tb, kb) + pts = " ".join(f"{xm(k):.1f},{ymap(v, vmax, top):.1f}" for k, v in sorted(data.items())) + svg.append(f'') + if dashed: + y = ymap(overall, vmax, top) + svg.append(f'') + svg.append(f'overall {overall:.0f}') + # annotate the per-run sandblast time-weighted rate inside the band + if annotate_rates: + for j, (label, color, r) in enumerate(sb_rates): + ry = ymap(r, vmax, top) + svg.append(f'') + svg.append(f'{label} {r:.0f} blk/s') + + panel(top1, "throughput (blk/s) — solid=per-bin, dashed=overall", lambda tb, kb: tb, bps_max, lambda v: f"{v:.0f}", dashed=True, annotate_rates=True) + panel(top2, "transparent keys/block (UTXO set + address index)", lambda tb, kb: kb, key_max, lambda v: f"{v:.0f}") + + for i in range(6): + h = h0 + (h1 - h0) * i / 5 + x = left + (h - h0) / (h1 - h0) * plot_w + for top in (top1, top2): + svg.append(f'') + svg.append(f'{int(h):,}') + svg.append(f'block height') + svg.append("") + + out = Path(args.out_dir) + out.mkdir(parents=True, exist_ok=True) + p = out / f"{args.name}-throughput-keys-ab.svg" + p.write_text("\n".join(svg)) + print(f"wrote {p}") + # time-weighted regional throughput + def rate(ts, lo, hi): + sel = sorted((h, t) for h, t in ts.items() if lo <= h < hi) + return (sel[-1][0] - sel[0][0]) / ((sel[-1][1] - sel[0][1]) / 1e6) if len(sel) > 1 and sel[-1][1] > sel[0][1] else None + print(f" {'region':>22} " + " ".join(f"{lbl:>10}" for lbl, *_ in series)) + for name, lo, hi in [("light <1862900", h0, 1862900), ("sandblast 1862.9-1872k", 1862900, 1872000), ("recovery 1872k+", 1872000, h1 + 1)]: + vals = [rate(ts, lo, hi) for _, ts, _ in runs] + if all(v is not None for v in vals): + print(f" {name:>22} " + " ".join(f"{v:>10.0f}" for v in vals)) + + +if __name__ == "__main__": + main() diff --git a/zebra-state/src/config.rs b/zebra-state/src/config.rs index 0b7c241d181..bdba227e2d3 100644 --- a/zebra-state/src/config.rs +++ b/zebra-state/src/config.rs @@ -291,6 +291,23 @@ impl Config { } } + /// Whether to omit the auxiliary transparent **address index** (balances, + /// address→utxo, address→tx). + /// + /// The address index is RPC-only state, not consensus. It is skipped only for the + /// minimal fast-validator configuration: a node that is both [`StorageMode::Pruned`] + /// **and** checkpoint-syncing ([`checkpoint_sync`](Config::checkpoint_sync)). That + /// drops the per-block address-balance reads and index writes, just as pruned mode + /// drops raw-transaction storage. + /// + /// An archive node keeps the index; so does a node with checkpoint sync disabled + /// (full semantic verification), even when pruned. Address-lookup RPCs + /// (`getaddressbalance`/`getaddressutxos`/`getaddresstxids`) return an error when + /// the index is skipped, rather than wrong (empty) results. + pub fn skip_address_index(&self) -> bool { + matches!(self.storage_mode, StorageMode::Pruned(_)) && self.checkpoint_sync + } + /// Validates the configured [`StorageMode`]. /// /// This must be called before opening the database, so that a misconfigured @@ -469,6 +486,39 @@ impl Default for Config { mod tests { use super::*; + #[test] + fn skip_address_index_only_when_pruned_and_checkpoint_syncing() { + let pruned_checkpoint = Config { + storage_mode: StorageMode::Pruned(PruningConfig::default()), + checkpoint_sync: true, + ..Default::default() + }; + assert!( + pruned_checkpoint.skip_address_index(), + "pruned + checkpoint-sync skips the transparent address index" + ); + + let archive = Config { + storage_mode: StorageMode::Archive, + checkpoint_sync: true, + ..Default::default() + }; + assert!( + !archive.skip_address_index(), + "archive mode keeps the address index" + ); + + let pruned_legacy = Config { + storage_mode: StorageMode::Pruned(PruningConfig::default()), + checkpoint_sync: false, + ..Default::default() + }; + assert!( + !pruned_legacy.skip_address_index(), + "pruned with checkpoint sync disabled keeps the address index" + ); + } + #[test] fn storage_mode_deserializes_from_documented_toml() { assert!( diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs index 9e304a55139..ddd2374408f 100644 --- a/zebra-state/src/service.rs +++ b/zebra-state/src/service.rs @@ -96,6 +96,12 @@ const FAST_SYNCED_TREE_UNAVAILABLE_ERROR: &str = "note commitment treestate is unavailable below the checkpoint on a fast-synced node; \ historical treestate queries require an archive node"; +/// Returned for transparent address-lookup requests when the address index was +/// not built (pruned storage mode). The index is RPC-only, not consensus. +const ADDRESS_INDEX_DISABLED: &str = + "the transparent address index is disabled in pruned storage mode; \ + getaddressbalance, getaddressutxos, and getaddresstxids require an archive node"; + /// A read-write service for Zebra's cached blockchain state. /// /// This service modifies and provides access to: @@ -1994,31 +2000,47 @@ impl Service for ReadStateService { // For the get_address_balance RPC. ReadRequest::AddressBalance(addresses) => { - let (balance, received) = - read::transparent_balance(state.latest_best_chain(), &state.db, addresses)?; - Ok(ReadResponse::AddressBalance { balance, received }) + if state.db.config().skip_address_index() { + Err(ADDRESS_INDEX_DISABLED.into()) + } else { + let (balance, received) = + read::transparent_balance(state.latest_best_chain(), &state.db, addresses)?; + Ok(ReadResponse::AddressBalance { balance, received }) + } } // For the get_address_tx_ids RPC. ReadRequest::TransactionIdsByAddresses { addresses, height_range, - } => read::transparent_tx_ids( - state.latest_best_chain(), - &state.db, - addresses, - height_range, - ) - .map(ReadResponse::AddressesTransactionIds), + } => { + if state.db.config().skip_address_index() { + Err(ADDRESS_INDEX_DISABLED.into()) + } else { + read::transparent_tx_ids( + state.latest_best_chain(), + &state.db, + addresses, + height_range, + ) + .map(ReadResponse::AddressesTransactionIds) + } + } // For the get_address_utxos RPC. - ReadRequest::UtxosByAddresses(addresses) => read::address_utxos( - &state.network, - state.latest_best_chain(), - &state.db, - addresses, - ) - .map(ReadResponse::AddressUtxos), + ReadRequest::UtxosByAddresses(addresses) => { + if state.db.config().skip_address_index() { + Err(ADDRESS_INDEX_DISABLED.into()) + } else { + read::address_utxos( + &state.network, + state.latest_best_chain(), + &state.db, + addresses, + ) + .map(ReadResponse::AddressUtxos) + } + } ReadRequest::CheckBestChainTipNullifiersAndAnchors(unmined_tx) => { let latest_non_finalized_best_chain = state.latest_best_chain(); diff --git a/zebra-state/src/service/finalized_state/zebra_db/block.rs b/zebra-state/src/service/finalized_state/zebra_db/block.rs index b75afa8fd82..14e57c024ea 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block.rs @@ -1193,27 +1193,32 @@ impl ZebraDb { // serves only the pipeline-depth window. Built only when stage timing is on. if zebra_chain::stage_timing::enabled() { let spend_h = finalized.height.0; - let (mut le64, mut le256, mut le1024, mut le4096) = (0u64, 0u64, 0u64, 0u64); + let (mut le4k, mut le16k, mut le65k, mut le262k, mut le1m) = + (0u64, 0u64, 0u64, 0u64, 0u64); for (_op, out_loc, _utxo) in &spent_utxos { let d = spend_h.saturating_sub(out_loc.height().0); - if d <= 64 { - le64 += 1; + if d <= 4_096 { + le4k += 1; } - if d <= 256 { - le256 += 1; + if d <= 16_384 { + le16k += 1; } - if d <= 1024 { - le1024 += 1; + if d <= 65_536 { + le65k += 1; } - if d <= 4096 { - le4096 += 1; + if d <= 262_144 { + le262k += 1; + } + if d <= 1_048_576 { + le1m += 1; } } zebra_chain::stage_timing::record_val(spend_h, "spent_total", spent_utxos.len() as u64); - zebra_chain::stage_timing::record_val(spend_h, "spent_le64", le64); - zebra_chain::stage_timing::record_val(spend_h, "spent_le256", le256); - zebra_chain::stage_timing::record_val(spend_h, "spent_le1024", le1024); - zebra_chain::stage_timing::record_val(spend_h, "spent_le4096", le4096); + zebra_chain::stage_timing::record_val(spend_h, "spent_le4k", le4k); + zebra_chain::stage_timing::record_val(spend_h, "spent_le16k", le16k); + zebra_chain::stage_timing::record_val(spend_h, "spent_le65k", le65k); + zebra_chain::stage_timing::record_val(spend_h, "spent_le262k", le262k); + zebra_chain::stage_timing::record_val(spend_h, "spent_le1m", le1m); } let spent_utxos_by_outpoint: HashMap = @@ -1233,21 +1238,6 @@ impl ZebraDb { .map(|(_outpoint, out_loc, utxo)| (out_loc, utxo)) .collect(); - // Get the transparent addresses with changed balances/UTXOs - let changed_addresses: HashSet = spent_utxos_by_out_loc - .values() - .chain( - finalized - .new_outputs - .values() - .map(|ordered_utxo| &ordered_utxo.utxo), - ) - .filter_map(|utxo| utxo.output.address(network)) - .unique() - .collect(); - - // Get the current address balances, before the transactions in this block - // Like the spent-UTXO reads above, the per-address balance lookups are // cache-served but serial. Fan them across the rayon pool once a block // touches enough addresses to amortize the fork-join cost. @@ -1281,26 +1271,46 @@ impl ZebraDb { // reading all of the pending merge operands (potentially hundreds), and applying pending merge operands to the // fully-merged value such that it's much faster to read entries that have been updated with insertions than it // is to read entries that have been updated with merge operations. - // Resolve an address balance from the run-ahead overlay first (it may have - // been updated by a not-yet-flushed block), then from disk. With - // `overlay = None` this is exactly the disk read. - let lookup_balance = |addr: &transparent::Address| { - if let Some(overlay) = overlay { - if let Some(balance) = overlay.address_balance_override(addr) { - return Some(balance); - } - } - self.address_balance_location(addr) - }; + // + // When the address index is skipped (fast-sync), none of the per-address + // balance reads happen and `address_balances` is left empty; the gated + // transparent index passes below then write no address entries. let address_reads_start = std::time::Instant::now(); - let address_balances: AddressBalanceLocationUpdates = if self.finished_format_upgrades() { - AddressBalanceLocationUpdates::Insert(read_addr_locs(changed_addresses, |addr| { - lookup_balance(addr) - })) + let address_balances: AddressBalanceLocationUpdates = if self.config().skip_address_index() + { + AddressBalanceLocationUpdates::Insert(HashMap::new()) } else { - AddressBalanceLocationUpdates::Merge(read_addr_locs(changed_addresses, |addr| { - Some(lookup_balance(addr)?.into_new_change()) - })) + // Transparent addresses with changed balances/UTXOs in this block. + let changed_addresses: HashSet = spent_utxos_by_out_loc + .values() + .chain( + finalized + .new_outputs + .values() + .map(|ordered_utxo| &ordered_utxo.utxo), + ) + .filter_map(|utxo| utxo.output.address(network)) + .unique() + .collect(); + // Resolve an address balance from the run-ahead overlay first (it may + // have been updated by a not-yet-flushed block), then from disk. + let lookup_balance = |addr: &transparent::Address| { + if let Some(overlay) = overlay { + if let Some(balance) = overlay.address_balance_override(addr) { + return Some(balance); + } + } + self.address_balance_location(addr) + }; + if self.finished_format_upgrades() { + AddressBalanceLocationUpdates::Insert(read_addr_locs(changed_addresses, |addr| { + lookup_balance(addr) + })) + } else { + AddressBalanceLocationUpdates::Merge(read_addr_locs(changed_addresses, |addr| { + Some(lookup_balance(addr)?.into_new_change()) + })) + } }; let address_reads_dur = address_reads_start.elapsed(); #[cfg(feature = "commit-metrics")] diff --git a/zebra-state/src/service/finalized_state/zebra_db/transparent.rs b/zebra-state/src/service/finalized_state/zebra_db/transparent.rs index d7ffa2c60e3..58cfb69e42f 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/transparent.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/transparent.rs @@ -439,18 +439,27 @@ impl DiskWriteBatch { let db = &zebra_db.db; let FinalizedBlock { block, height, .. } = finalized; + // A pruned, checkpoint-syncing node skips the auxiliary transparent address + // index entirely. The UTXO-set passes still run (they write/delete + // `utxo_by_out_loc`), but the address-balance update, the per-tx address + // indexing, and the address balance CF are all elided. The UTXO/value-pool/ + // nullifier state is unchanged. + let skip_index = zebra_db.config().skip_address_index(); + // Update the in-memory `address_balances` transaction-by-transaction, debiting inputs // before crediting outputs within each transaction. This ordering keeps every // intermediate per-address balance within the consensus range, even when the block // contains a same-address transparent self-spend chain whose batch credit-first // intermediate balance would otherwise exceed MAX_MONEY. - Self::prepare_transparent_address_balance_updates( - network, - *height, - &block.transactions, - spent_utxos_by_outpoint, - &mut address_balances, - ); + if !skip_index { + Self::prepare_transparent_address_balance_updates( + network, + *height, + &block.transactions, + spent_utxos_by_outpoint, + &mut address_balances, + ); + } // Write the new and spent transparent output index entries. These passes no longer // touch `address_balances`; they only read each entry's `address_location()`. @@ -459,35 +468,41 @@ impl DiskWriteBatch { network, new_outputs_by_out_loc, &address_balances, + skip_index, ); self.prepare_spent_transparent_outputs_batch( db, network, spent_utxos_by_out_loc, &address_balances, + skip_index, ); // Index the transparent addresses that spent in each transaction - for (tx_index, transaction) in block.transactions.iter().enumerate() { - let spending_tx_location = TransactionLocation::from_usize(*height, tx_index); - - self.prepare_spending_transparent_tx_ids_batch( - zebra_db, - network, - spending_tx_location, - transaction, - spent_utxos_by_outpoint, - #[cfg(feature = "indexer")] - out_loc_by_outpoint, - &address_balances, - ); + if !skip_index { + for (tx_index, transaction) in block.transactions.iter().enumerate() { + let spending_tx_location = TransactionLocation::from_usize(*height, tx_index); + + self.prepare_spending_transparent_tx_ids_batch( + zebra_db, + network, + spending_tx_location, + transaction, + spent_utxos_by_outpoint, + #[cfg(feature = "indexer")] + out_loc_by_outpoint, + &address_balances, + ); + } } // Capture the post-block absolute balances for the pipeline overlay before // they are moved into the write batch (cheap clone, only when running ahead). let captured = capture_balances.then(|| address_balances.clone()); - self.prepare_transparent_balances_batch(db, address_balances); + if !skip_index { + self.prepare_transparent_balances_batch(db, address_balances); + } captured } @@ -602,6 +617,7 @@ impl DiskWriteBatch { network: &Network, new_outputs_by_out_loc: &BTreeMap, address_balances: &AddressBalanceLocationUpdates, + skip_index: bool, ) { let utxo_by_out_loc = db.cf_handle("utxo_by_out_loc").unwrap(); let utxo_loc_by_transparent_addr_loc = @@ -612,7 +628,9 @@ impl DiskWriteBatch { // Index all new transparent outputs for (new_output_location, utxo) in new_outputs_by_out_loc { let unspent_output = &utxo.output; - let receiving_address = unspent_output.address(network); + let receiving_address = (!skip_index) + .then(|| unspent_output.address(network)) + .flatten(); if let Some(receiving_address) = receiving_address { let receiving_address_location = match address_balances { @@ -674,6 +692,7 @@ impl DiskWriteBatch { network: &Network, spent_utxos_by_out_loc: &BTreeMap, address_balances: &AddressBalanceLocationUpdates, + skip_index: bool, ) { let utxo_by_out_loc = db.cf_handle("utxo_by_out_loc").unwrap(); let utxo_loc_by_transparent_addr_loc = @@ -684,7 +703,9 @@ impl DiskWriteBatch { // Coinbase inputs represent new coins, so there are no UTXOs to mark as spent. for (spent_output_location, utxo) in spent_utxos_by_out_loc { let spent_output = &utxo.output; - let sending_address = spent_output.address(network); + let sending_address = (!skip_index) + .then(|| spent_output.address(network)) + .flatten(); // Fetch the link from the address to the AddressLocation, from memory. if let Some(sending_address) = sending_address { From 0bf6fee8bdd00b9d90b7c131ccdb08661be3e87d Mon Sep 17 00:00:00 2001 From: Roman Date: Tue, 30 Jun 2026 19:31:59 +0000 Subject: [PATCH 08/17] test(replay-bench): run the production pruned + checkpoint-sync config Flip the offline replay bench from Archive to StorageMode::Pruned with checkpoint sync, the real fast-sync configuration. This exercises the pruned-only paths in the bench: raw-transaction pruning and the transparent address-index skip (now gated on pruned + checkpoint_sync, with the env override removed). The per-run fork is a throwaway copy, so online pruning never touches the base snapshot. Verified: the snapshot opens in pruned mode without tripping the one-way pruned reopen guard, the run commits through the gate (hash matches), and the address index is skipped at runtime (sandblast keys_transparent drops to the UTXO-set floor) without any env override. --- zebra-replay-bench/src/config.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/zebra-replay-bench/src/config.rs b/zebra-replay-bench/src/config.rs index 873674c9936..219b0c579f1 100644 --- a/zebra-replay-bench/src/config.rs +++ b/zebra-replay-bench/src/config.rs @@ -2,13 +2,15 @@ use std::path::PathBuf; -use zebra_state::{Config, StorageMode}; +use zebra_state::{Config, PruningConfig, StorageMode}; /// Builds a [`zebra_state::Config`] pointing at `cache_dir`. /// /// `cache_dir` is the snapshot/fork root that contains `state/vN/`. The -/// benchmark only ever opens archive snapshots (full block bodies + per-height -/// trees), so storage mode is fixed to [`StorageMode::Archive`]. +/// benchmark runs the production fast-sync configuration: [`StorageMode::Pruned`] +/// with checkpoint sync, so it measures exactly what a real pruned validator does +/// (including skipping the transparent address index). The per-run fork is a +/// throwaway copy, so online pruning never touches the base snapshot. /// /// `force_legacy` clears `vct_fast_sync`, which forces the committer onto the full /// per-block note-commitment recompute path — the write-assembler + disk-writer @@ -20,7 +22,7 @@ pub fn state_config(cache_dir: PathBuf, force_legacy: bool) -> Config { ephemeral: false, checkpoint_sync: true, vct_fast_sync: !force_legacy, - storage_mode: StorageMode::Archive, + storage_mode: StorageMode::Pruned(PruningConfig::default()), ..Config::default() }; // Run-ahead finalized-commit pipeline depth (PR #309). 0 = synchronous (default). From deb3d7ca60aed80c697f3b4fd4d19d4ff232fb59 Mon Sep 17 00:00:00 2001 From: Roman Date: Tue, 30 Jun 2026 23:37:29 +0000 Subject: [PATCH 09/17] perf(state): deferred transparent reconcile prototype (default-off) Defer the per-block spent-UTXO resolution (utxo_by_out_loc deletes + the transparent value-pool debit) off the finalized commit critical path and reconcile it in a batched pass on a dedicated worker thread, in the checkpoint-trusted fast-sync range. In the sandblast spam range the per-block spent-UTXO reads (~439 cold random reads/block) are the dominant commit cost; a ceiling probe showed removing them is +209% sandblast throughput. The reads can't be cached (97% of spends are >4096 blocks old) but can be deferred and batched: - per block: record spent outpoints into an in-memory window, write UTXO creates + headers + nullifiers + the VCT root fold inline, skip the spent-side value pool; - every `defer_reconcile_interval` blocks: a reconcile resolves the window's spends from disk (sorted/deduped), recomputes the value pool, and writes the deletes + value pool (chain_value_pools tip + per-height BlockInfo) atomically; - v2 runs the reconcile on a worker thread (capacity-1 handoff) so it overlaps assembly: the worker owns the value pool, processes windows FIFO, and writes keys disjoint from the assembler's creates (no shared-state races). Result (offline replay bench, mainnet sandblast range): the deferred run is byte-identical to the non-deferred run (value pool, ~20M-entry UTXO set, and per-height BlockInfo digests all match), and v2 lifts sandblast throughput +73% (overall +43%). All env/config-gated and default-off; no production path is enabled. This is a measurement PROTOTYPE: see docs/design/deferred-transparent-reconcile.md for the correctness invariants, results, and the production-readiness gaps (handoff drain barrier, crash recovery, RPC guards) that must land before it can be turned on. Adds a cf-dump bench subcommand + column-family verification digests for the byte-match. --- docs/design/deferred-transparent-reconcile.md | 108 +++++++++++ zebra-replay-bench/src/apply_sequencer.rs | 17 ++ zebra-replay-bench/src/config.rs | 25 +++ zebra-replay-bench/src/main.rs | 42 ++++ zebra-state/src/config.rs | 41 ++++ zebra-state/src/service/finalized_state.rs | 5 +- .../src/service/finalized_state/pipeline.rs | 181 +++++++++++++++++- .../src/service/finalized_state/zebra_db.rs | 32 ++++ .../service/finalized_state/zebra_db/block.rs | 60 +++++- .../service/finalized_state/zebra_db/chain.rs | 131 ++++++++++++- .../finalized_state/zebra_db/transparent.rs | 56 +++++- zebra-state/src/service/write.rs | 171 ++++++++++++++++- 12 files changed, 850 insertions(+), 19 deletions(-) create mode 100644 docs/design/deferred-transparent-reconcile.md diff --git a/docs/design/deferred-transparent-reconcile.md b/docs/design/deferred-transparent-reconcile.md new file mode 100644 index 00000000000..0c1b01f5608 --- /dev/null +++ b/docs/design/deferred-transparent-reconcile.md @@ -0,0 +1,108 @@ +# Deferred transparent reconcile (prototype) + +Status: **prototype, benchmark-gated, default-off. NOT safe to enable on a real node yet** +(see "Production-readiness gaps"). + +## Problem + +In the checkpoint-trusted fast-sync range, the finalized committer's dominant cost in +high transparent-churn regions (the 2022 "sandblast" consolidation spam) is **spent-UTXO +resolution**: for each spent transparent input the committer reads `tx_loc_by_hash` +(txid → location) and `utxo_by_out_loc` (location → value), then deletes the entry and +debits the value pool. In the sandblast range this is ~439 cold, random reads per block, +all on the per-block commit critical path. + +These reads are not cacheable: ~97% of spends in that range reference UTXOs created +>4096 blocks earlier, so there is no temporal locality. A ceiling probe (skip the spend +work entirely) measured **sandblast throughput 124 → 383 blk/s (+209%)** — i.e. the spend +resolution *is* the entire sandblast penalty; with it removed a sandblast block runs at +the normal (light-region) rate. + +## Design + +The reads can't be removed (consensus needs the spent value + location), but they can be +**deferred off the per-block path and batched on a separate worker**: + +- **Per block** (in the deferred range): the committer does not resolve spent UTXOs. It + records the spent outpoints + the block into an in-memory window, writes the UTXO + *creates* + headers + nullifiers + the VCT root fold inline, and skips the spent-side of + the value pool. (The auxiliary address index is already skipped in pruned mode.) +- **Every N blocks** (`defer_reconcile_interval`, not per checkpoint — mainnet checkpoints + are only ~30–40 blocks apart, too frequent to amortize): a **reconcile** resolves the + window's spent outpoints from disk (sorted/deduped), recomputes the value-pool delta, and + writes the deletes + value pool (`chain_value_pools` tip + per-height `BlockInfo`) in one + atomic batch. +- **v2** runs the reconcile on a dedicated **worker thread** with a capacity-1 handoff + channel, so the (CPU-bound) resolution + value-pool recompute + write overlap with + continued block assembly. The worker owns the running value pool exclusively (the + assembler defers it), processes windows FIFO (so the value-pool chaining stays ordered), + and writes only keys disjoint from the assembler's (deletes + value pool vs. creates), so + there are no shared-state races or key conflicts. + +### Correctness invariants + +1. **Older spends are durable at reconcile time.** A UTXO is deleted only when its + spender's window reconciles; each UTXO is spent once, so it is never deleted earlier, and + by reconcile time all the window's blocks are flushed. +2. **Lazy deletes are safe.** Between reconciles `utxo_by_out_loc` is a transient *superset*; + the only reader is the next reconcile, which knows the entries are spent. A stale entry + would only be re-read by a double-spend, impossible in a checkpoint-valid chain. +3. **Deferring the value pool is safe in the checkpoint range.** It is consensus state but + only *checked* above `max_checkpoint_height` (the semantic verifier). The per-block + value-pool change is additive, so the interval delta is the sum of per-block deltas. + +## Results + +Benchmarked offline via `zebra-replay-bench` on a mainnet 1.85M–1.9M pruned snapshot +(includes the sandblast region), run-ahead pipeline depth 8, interval 2000. + +- **Byte-match: identical.** The deferred run (both the v1 inline path and the v2 worker) + produces a **byte-identical value pool, UTXO set (`utxo_by_out_loc`, ~20.0M entries), and + per-height `BlockInfo`** as the non-deferred run, verified with the `cf-dump` digests at a + deterministic checkpoint stop. The block-hash checkpoint gate also passes. +- **Throughput (sandblast region):** + | variant | sandblast | overall | + |---|---|---| + | non-deferred baseline | 122 | 213 blk/s | + | v1 per-checkpoint (~38-block) | −51% | −43% | + | v1 interval=2000 inline | +2% | +3% | + | **v2 interval=2000 worker** | **+73%** | **+43%** | + + v2 recovers ~34% of the +209% ceiling. The remaining gap is that the single-threaded + worker is now the bottleneck (it re-traverses every transaction of each block for the + value-pool recompute), so sandblast stays below the light-region rate. + +## Gating + +Env-gated for benchmarking, default off; no production code path is enabled: + +- `zebra-state` `Config`: `defer_transparent_reconcile` (off), `defer_reconcile_interval` + (0 = per checkpoint), `defer_reconcile_inline` (force v1). All `#[serde(skip)]`. +- `Config::defers_transparent_spends()` currently requires `defer_transparent_reconcile && + skip_address_index()` (pruned + checkpoint-sync). It does **not** yet bound the height to + `max_checkpoint_height`. +- `zebra-replay-bench`: `ZRB_DEFER_TRANSPARENT`, `ZRB_RECONCILE_INTERVAL`, + `ZRB_RECONCILE_INLINE`, `ZRB_STOP_AT_HEIGHT`. + +## Production-readiness gaps (why it is not safe to enable yet) + +The reconcile *logic* is byte-match-verified, but the surrounding lifecycle is not: + +1. **No handoff drain barrier.** The deferral is only valid below `max_checkpoint_height`. + At the handoff to semantic verification the value pool + UTXO set must be fully + reconciled first (the semantic verifier reads them to validate spends). The prototype + omits the height bound and the drain, so enabling it on a node that reaches the handoff + would feed the semantic verifier a stale/superset UTXO set + lagging value pool — a + consensus failure. +2. **No crash recovery.** The reconciled (durable transparent) tip trails the committed + tip. A crash mid-window leaves `utxo_by_out_loc` a superset and `BlockInfo`/value pool + lagging; restart must re-derive the pending window. Not implemented. +3. **No RPC guards.** Mid-window the UTXO set is a superset and the value pool lags, so + address/utxo/value RPCs would return wrong results. Not guarded. +4. **Background format check.** `check_new_blocks` trips on the transient `BlockInfo` lag; + the prototype sidesteps it only via the deterministic-stop bench harness. A production + path must skip/relax it in the deferred range. +5. **Shutdown + error handling.** The worker is fatal-on-error (panic), and clean shutdown + requires draining the worker before exit (the bench parks its main task to do this). + +Until 1–5 are implemented and tested, this is a measurement prototype only. diff --git a/zebra-replay-bench/src/apply_sequencer.rs b/zebra-replay-bench/src/apply_sequencer.rs index 37d0eaa4fc5..c90122ff37c 100644 --- a/zebra-replay-bench/src/apply_sequencer.rs +++ b/zebra-replay-bench/src/apply_sequencer.rs @@ -199,6 +199,13 @@ pub fn run( .build() .map_err(|e| eyre!("building tokio runtime: {e}"))?; + // When a deterministic stop height is configured, the state's committer thread + // exits the process itself (after flushing the deferred-reconcile worker). The + // bench's main task must not return first and exit out from under that flush, so + // it parks after reaching the gate and lets the committer be the authoritative + // terminator. + let debug_stop_at_height = config.debug_stop_at_height; + let stats = runtime.block_on(async move { // Real buffered StateService + checkpoint verifier on the base fork. let (state, read_state, _latest, _change) = zebra_state::init( @@ -448,6 +455,16 @@ pub fn run( committed as f64 / secs, total_bytes as f64 / secs / (1024.0 * 1024.0) ); + + // With a configured stop height, the state committer thread flushes the + // deferred-reconcile worker and exits the process once it commits the stop + // block. That flush can outlast this task's gate wait, so park here and let + // the committer's `process::exit` terminate the run, ensuring every reconcile + // is durable first. The bounded timeout is a safety net against a missed exit. + if debug_stop_at_height.is_some() { + tokio::time::sleep(Duration::from_secs(900)).await; + } + Ok::(stats) })?; diff --git a/zebra-replay-bench/src/config.rs b/zebra-replay-bench/src/config.rs index 219b0c579f1..b7dcdb41f75 100644 --- a/zebra-replay-bench/src/config.rs +++ b/zebra-replay-bench/src/config.rs @@ -30,5 +30,30 @@ pub fn state_config(cache_dir: PathBuf, force_legacy: bool) -> Config { .ok() .and_then(|s| s.parse().ok()) .unwrap_or(0); + // Per-checkpoint transparent reconcile prototype: defer the per-block spent-UTXO + // resolution off the commit path and reconcile it in a batch at each checkpoint. + // Requires the run-ahead pipeline (ZRB_PIPELINE_DEPTH > 0) for correctness. + config.defer_transparent_reconcile = std::env::var("ZRB_DEFER_TRANSPARENT") + .map(|v| v == "1") + .unwrap_or(false); + // Blocks between deferred reconciles (0 = per checkpoint). Mainnet checkpoints are + // ~30-40 blocks apart here, too frequent to amortize the reconcile; a larger fixed + // interval batches a bigger window. + config.defer_reconcile_interval = std::env::var("ZRB_RECONCILE_INTERVAL") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(0); + // Force the v1 inline reconcile (on the assembler thread) instead of the default + // v2 worker thread; mainly to A/B the off-critical-path parallelism win. + config.defer_reconcile_inline = std::env::var("ZRB_RECONCILE_INLINE") + .map(|v| v == "1") + .unwrap_or(false); + // Deterministic stop for byte-match verification: when set, the committer flushes + // and exits the process at exactly this height (and the per-checkpoint reconcile + // fires for the final window via the stop-height hook), so two runs land on an + // identical tip. Leave unset for throughput runs. + config.debug_stop_at_height = std::env::var("ZRB_STOP_AT_HEIGHT") + .ok() + .and_then(|s| s.parse().ok()); config } diff --git a/zebra-replay-bench/src/main.rs b/zebra-replay-bench/src/main.rs index cc8d72289a1..dedd2134baa 100644 --- a/zebra-replay-bench/src/main.rs +++ b/zebra-replay-bench/src/main.rs @@ -52,6 +52,13 @@ enum Cmd { #[arg(long)] src: PathBuf, }, + /// Print a snapshot's finalized value pool and a stable digest of the + /// `utxo_by_out_loc` column family (read-only), for byte-match verification. + CfDump { + /// Snapshot root containing `state/vN/`. + #[arg(long)] + src: PathBuf, + }, /// Read blocks `start..=end` from a snapshot into a flat cache file. Index { /// Snapshot root to read from (opened read-only). @@ -208,6 +215,41 @@ fn main() -> Result<()> { has_body ); } + Cmd::CfDump { src } => { + // Open with the VCT fast path enabled (force_legacy = false): the + // benchmark forks are interrupted VCT snapshots below the handoff, which + // refuse to open read-only with vct_fast_sync off. + let config = state_config(src.clone(), false); + let state = FinalizedState::new_read_only(&config, &network); + let tip = state + .db + .tip() + .ok_or_else(|| eyre!("snapshot has no finalized tip"))?; + let value_pool = state.db.finalized_value_pool(); + let (count, key_sum, key_xor, value_sum, value_xor) = + state.db.utxo_by_out_loc_verification_digest(); + let lo = zebra_chain::block::Height(tip.0 .0.saturating_sub(14_500)); + let (bi_count, bi_first_missing, bi_missing) = state.db.block_info_coverage(lo, tip.0); + let bi_digest = state.db.block_info_verification_digest(); + println!( + "block_info[{}..={}]: count={} missing={} first_missing={:?}", + lo.0, tip.0 .0, bi_count, bi_missing, bi_first_missing + ); + println!("block_info_digest={bi_digest:?}"); + println!( + "src={}\ntip_height={}\ntip_hash={}\nvalue_pool={:?}\n\ + utxo_count={}\nutxo_key_sum={}\nutxo_key_xor={}\nutxo_value_sum={}\nutxo_value_xor={}", + src.display(), + tip.0 .0, + tip.1, + value_pool, + count, + key_sum, + key_xor, + value_sum, + value_xor, + ); + } Cmd::Index { src, cache, diff --git a/zebra-state/src/config.rs b/zebra-state/src/config.rs index bdba227e2d3..ef5b4ca924b 100644 --- a/zebra-state/src/config.rs +++ b/zebra-state/src/config.rs @@ -153,6 +153,43 @@ pub struct Config { /// overlay of not-yet-flushed blocks. pub finalized_block_pipeline_depth: usize, + /// Prototype: defer the transparent spend resolution (UTXO deletes + value-pool + /// debit) off the per-block commit path and reconcile it in a batched pass at + /// each checkpoint boundary, in the checkpoint-trusted range. + /// + /// The per-block committer records spent outpoints into an in-memory window + /// instead of resolving them; at each checkpoint the window is batch-resolved + /// (sorted/deduped disk reads), the value pool is recomputed, and the deletes + + /// pool are written in one atomic batch. The auxiliary address index must also be + /// off ([`skip_address_index`](Config::skip_address_index)). + /// + /// Not exposed in serde; set by the benchmark for measurement. Production wiring + /// (auto-enable under pruned + checkpoint sync, the handoff drain barrier, RPC + /// guards, crash recovery) is not yet implemented, so this defaults to `false`. + #[serde(skip)] + pub defer_transparent_reconcile: bool, + + /// Prototype: how many blocks between deferred-transparent reconciles, when + /// [`defer_transparent_reconcile`](Config::defer_transparent_reconcile) is on. + /// + /// `0` (the default) reconciles at every checkpoint boundary — but mainnet + /// checkpoints are only ~30-40 blocks apart in dense regions, so the reconcile + /// fires far too often to amortize its fixed cost. A larger fixed interval + /// (e.g. ~2000) reconciles a bigger batched window. Correctness only requires a + /// bounded window with the window's spent UTXOs durable by reconcile time, not + /// checkpoint alignment. + #[serde(skip)] + pub defer_reconcile_interval: usize, + + /// Prototype: run the deferred-transparent reconcile inline on the assembler + /// thread (the v1 path) instead of on the dedicated reconcile worker thread (v2). + /// + /// The default (`false`) hands each window to a worker thread so the reconcile's + /// disk reads and value-pool recompute overlap continued block assembly. Set to + /// `true` to force the inline path, mainly to A/B the parallelism win. + #[serde(skip)] + pub defer_reconcile_inline: bool, + /// Whether to delete the old database directories when present. /// /// Set to `true` by default. If this is set to `false`, @@ -467,6 +504,10 @@ impl Default for Config { // Off by default: the committer is synchronous (assemble then flush) // until run-ahead is explicitly enabled and validated. finalized_block_pipeline_depth: 0, + // Off by default: prototype, benchmark-set only. + defer_transparent_reconcile: false, + defer_reconcile_interval: 0, + defer_reconcile_inline: false, delete_old_database: true, storage_mode: StorageMode::default(), debug_stop_at_height: None, diff --git a/zebra-state/src/service/finalized_state.rs b/zebra-state/src/service/finalized_state.rs index 4d99af2a431..0c259a41d47 100644 --- a/zebra-state/src/service/finalized_state.rs +++ b/zebra-state/src/service/finalized_state.rs @@ -185,7 +185,7 @@ mod zebra_db; use vct::VctState; use pipeline::BlockPipelineContribution; -pub(crate) use pipeline::FinalizedPipeline; +pub(crate) use pipeline::{reconcile_window, FinalizedPipeline, ReconcileBlock}; /// The verified-commitment-trees `tree_aux` serving read path (design §9): the per-block /// commitment roots for a height range, derived from the per-height trees. @@ -1442,6 +1442,9 @@ impl FinalizedState { wrote_vct_upgrade_marker: contribution.wrote_vct_upgrade_marker, created_outputs: contribution.created_outputs, updated_balances: contribution.updated_balances, + deferred_spent: contribution.deferred_spent, + deferred_block: contribution.deferred_block, + deferred_pool_change: contribution.deferred_pool_change, }); Ok(AssembledCommit { diff --git a/zebra-state/src/service/finalized_state/pipeline.rs b/zebra-state/src/service/finalized_state/pipeline.rs index c8172909357..4575830a7fe 100644 --- a/zebra-state/src/service/finalized_state/pipeline.rs +++ b/zebra-state/src/service/finalized_state/pipeline.rs @@ -36,17 +36,20 @@ use std::{collections::HashMap, sync::Arc}; use zebra_chain::{ - amount::NonNegative, - block::{self, Height}, + amount::{DeferredPoolBalanceChange, NonNegative}, + block::{self, Block, Height}, history_tree::HistoryTree, transparent, value_balance::ValueBalance, }; -use crate::service::finalized_state::{ - disk_format::transparent::{AddressBalanceLocation, AddressBalanceLocationUpdates}, - disk_format::OutputLocation, - ZebraDb, +use crate::{ + service::finalized_state::{ + disk_format::transparent::{AddressBalanceLocation, AddressBalanceLocationUpdates}, + disk_format::OutputLocation, + FinalizedState, ZebraDb, + }, + BoxError, }; use super::NoteCommitmentTrees; @@ -80,6 +83,14 @@ pub(crate) struct PipelineBatchContribution { pub created_outputs: Vec<(transparent::OutPoint, OutputLocation, transparent::Utxo)>, /// The absolute address balances this block updated. pub updated_balances: Vec<(transparent::Address, AddressBalanceLocation)>, + /// Per-checkpoint reconcile: the spent outpoints this block deferred (empty + /// unless [`ZebraDb::defers_transparent_spends`] is on). + pub deferred_spent: Vec, + /// The deferring block, held for the value-pool recompute at the checkpoint. + pub deferred_block: Option>, + /// The block's deferred-pool balance change, needed by the reconcile's + /// value-pool recompute (`Some` only after the deferred-pool activation). + pub deferred_pool_change: Option, } /// A transparent output created by a not-yet-flushed block. @@ -120,6 +131,37 @@ pub(crate) struct BlockPipelineContribution { pub created_outputs: Vec<(transparent::OutPoint, OutputLocation, transparent::Utxo)>, /// The absolute address balances this block updated. pub updated_balances: Vec<(transparent::Address, AddressBalanceLocation)>, + /// Per-checkpoint reconcile: the spent outpoints this block deferred (empty + /// unless [`ZebraDb::defers_transparent_spends`] is on). + pub deferred_spent: Vec, + /// The deferring block, held for the value-pool recompute at the checkpoint. + pub deferred_block: Option>, + /// The block's deferred-pool balance change, needed by the reconcile's + /// value-pool recompute. + pub deferred_pool_change: Option, +} + +/// One block held in the [`DeferralWindow`] for the per-checkpoint transparent +/// reconcile: its height, the block (for the value-pool recompute), the spent +/// outpoints to resolve and delete, and its deferred-pool change. +#[derive(Clone, Debug)] +pub(crate) struct ReconcileBlock { + /// The deferring block's height. + pub height: Height, + /// The deferring block, for the value-pool recompute. + pub block: Arc, + /// The transparent outpoints this block spent, resolved + deleted at the reconcile. + pub spent: Vec, + /// The block's deferred-pool balance change, passed to `chain_value_pool_change`. + pub deferred_pool_change: Option, +} + +/// The in-memory window of blocks whose transparent spend resolution has been +/// deferred to the next checkpoint reconcile, in increasing height order. +#[derive(Debug, Default)] +struct DeferralWindow { + /// The deferred blocks, pushed in height order by [`FinalizedPipeline::record_block`]. + blocks: Vec, } /// The in-memory tip state and read-through overlay for the run-ahead committer. @@ -145,6 +187,9 @@ pub(crate) struct FinalizedPipeline { utxos: HashMap, /// Absolute address balances updated by not-yet-flushed blocks. address_balances: HashMap, + /// Blocks whose transparent spend resolution is deferred to the next + /// checkpoint reconcile (empty unless deferral is enabled). + deferral: DeferralWindow, } impl FinalizedPipeline { @@ -160,6 +205,7 @@ impl FinalizedPipeline { vct_upgrade_marker_set: false, utxos: HashMap::new(), address_balances: HashMap::new(), + deferral: DeferralWindow::default(), } } @@ -243,6 +289,9 @@ impl FinalizedPipeline { wrote_vct_upgrade_marker, created_outputs, updated_balances, + deferred_spent, + deferred_block, + deferred_pool_change, } = contribution; let (height, _hash) = tip; @@ -253,6 +302,18 @@ impl FinalizedPipeline { self.value_pool = value_pool; self.vct_upgrade_marker_set |= wrote_vct_upgrade_marker; + // Per-checkpoint reconcile: record this block's deferred spends + block for + // the batched resolve/delete/value-pool pass at the next checkpoint. Only + // populated when deferral is on (`deferred_block` is `Some`). + if let Some(block) = deferred_block { + self.deferral.blocks.push(ReconcileBlock { + height, + block, + spent: deferred_spent, + deferred_pool_change, + }); + } + for (outpoint, out_loc, utxo) in created_outputs { self.utxos.insert( outpoint, @@ -281,4 +342,112 @@ impl FinalizedPipeline { self.address_balances .retain(|_, entry| entry.height > flushed_height); } + + /// Removes and returns the deferral-window prefix at or below `boundary_height`, + /// in height order, leaving any later (next-interval) blocks in the window. + /// + /// Acks drain lazily, so by the time a boundary becomes durable the window may + /// already hold blocks past it; those belong to the next interval and are kept. + pub(crate) fn take_reconcile_prefix(&mut self, boundary_height: Height) -> Vec { + let split = self + .deferral + .blocks + .iter() + .position(|block| block.height > boundary_height) + .unwrap_or(self.deferral.blocks.len()); + self.deferral.blocks.drain(..split).collect() + } + + /// Reconcile the deferred transparent spends for the interval ending at + /// `height`, inline on the caller's thread (the v1 path). + /// + /// Resolves every window spend from disk, deletes the spent `utxo_by_out_loc` + /// entries, recomputes the value pool, and writes the deletes + value pool + /// (tip + per-height `BlockInfo`) in one atomic batch. Only the window prefix at + /// or below `height` is reconciled. + /// + /// v2 moves this work to a dedicated worker thread (see + /// [`reconcile_window`])); this method remains for the inline fallback and is + /// byte-identical to it. + pub(crate) fn reconcile_checkpoint( + &mut self, + finalized_state: &mut FinalizedState, + height: Height, + ) -> Result<(), BoxError> { + let records = self.take_reconcile_prefix(height); + if records.is_empty() { + return Ok(()); + } + + let db = &finalized_state.db; + let network = db.network(); + let new_value_pool = reconcile_window(db, &network, &records, self.value_pool)?; + self.value_pool = new_value_pool; + + Ok(()) + } +} + +/// Resolve, delete, and re-debit a window of deferred transparent spends, writing +/// the result in one atomic batch, and return the value pool after the window. +/// +/// This is the shared core of the per-interval reconcile, called either inline +/// ([`FinalizedPipeline::reconcile_checkpoint`]) or off the assembler thread by the +/// dedicated reconcile worker. `start_value_pool` is the running pool before the +/// first block in `records`; the caller chains the returned pool into the next +/// window so the value pool stays exactly the sum of the per-block deltas. +/// +/// # Correctness +/// +/// Every spent UTXO is durable here: each window block is flushed before its +/// boundary becomes durable (pipeline depth is far below the interval), so even +/// same-interval creates are on disk. The deletes (old spent outputs) are disjoint +/// from the creates the assembler writes inline, and the assembler does not write +/// the value pool when deferring, so this runs concurrently with assembly without +/// key conflicts. The resulting value pool and UTXO set are byte-identical to the +/// non-deferred per-block path. +pub(crate) fn reconcile_window( + db: &ZebraDb, + network: &zebra_chain::parameters::Network, + records: &[ReconcileBlock], + start_value_pool: ValueBalance, +) -> Result, BoxError> { + if records.is_empty() { + return Ok(start_value_pool); + } + + // Batch-resolve every spent outpoint from disk. Sort by txid (so repeated txids + // dedup and the `output_location`/`tx_loc_by_hash` reads are grouped), then sort + // the resolved locations so the `utxo_by_out_loc` value reads are sequential-ish + // rather than random per block. + let mut outpoints: Vec = records + .iter() + .flat_map(|r| r.spent.iter().copied()) + .collect(); + outpoints.sort_unstable_by_key(|outpoint| (outpoint.hash.0, outpoint.index)); + outpoints.dedup(); + + let mut located: Vec<(transparent::OutPoint, OutputLocation)> = + Vec::with_capacity(outpoints.len()); + for outpoint in outpoints { + let out_loc = db.output_location(&outpoint).ok_or_else(|| { + format!("deferred spent outpoint missing from state at reconcile: {outpoint:?}") + })?; + located.push((outpoint, out_loc)); + } + located.sort_unstable_by_key(|(_outpoint, out_loc)| *out_loc); + + let mut resolved: HashMap = + HashMap::with_capacity(located.len()); + for (outpoint, out_loc) in located { + let utxo = db + .utxo_by_location(out_loc) + .ok_or_else(|| { + format!("deferred spent UTXO missing from state at reconcile: {outpoint:?}") + })? + .utxo; + resolved.insert(outpoint, (out_loc, utxo)); + } + + db.commit_checkpoint_reconcile(network, start_value_pool, records, &resolved) } diff --git a/zebra-state/src/service/finalized_state/zebra_db.rs b/zebra-state/src/service/finalized_state/zebra_db.rs index be72207dfca..e333eb735c1 100644 --- a/zebra-state/src/service/finalized_state/zebra_db.rs +++ b/zebra-state/src/service/finalized_state/zebra_db.rs @@ -63,6 +63,22 @@ pub mod transparent; // TODO: when the database is split out of zebra-state, always expose these methods. pub mod arbitrary; +/// Benchmark-only throughput-ceiling probe (`ZEBRA_BENCH_SKIP_TRANSPARENT_READS=1`). +/// +/// When set, the committer skips the per-block spent-UTXO resolution entirely, to +/// measure the upper bound of deferring that work off the commit critical path (the +/// per-checkpoint transparent reconcile). This deliberately produces an incorrect +/// value pool and UTXO set, so it is a measurement tool only — never a shipped path. +pub(crate) fn bench_skip_transparent_reads() -> bool { + static PROBE: std::sync::OnceLock = std::sync::OnceLock::new(); + *PROBE.get_or_init(|| { + matches!( + std::env::var("ZEBRA_BENCH_SKIP_TRANSPARENT_READS").as_deref(), + Ok("1") | Ok("true") + ) + }) +} + /// Wrapper struct to ensure high-level `zebra-state` database access goes through the correct API. /// /// `rocksdb` allows concurrent writes through a shared reference, @@ -234,6 +250,22 @@ impl ZebraDb { &self.config } + /// Whether the per-block committer defers transparent spend resolution (the + /// `utxo_by_out_loc` deletes and the transparent value-pool debit) off the + /// commit critical path, to a batched reconcile at each checkpoint boundary. + /// + /// The auxiliary address index must be off ([`Config::skip_address_index`]), + /// because the deferred path skips the per-block spent-UTXO reads that the + /// address-balance update depends on. + /// + /// v1 (prototype) omits the `height <= max_checkpoint_height` check: the + /// benchmark runs entirely inside the checkpoint range, and the handoff drain + /// barrier (so no semantic-verified block reads a mid-reconcile value pool or + /// UTXO set) is a later production stage. + pub(crate) fn defers_transparent_spends(&self) -> bool { + self.config().defer_transparent_reconcile && self.config().skip_address_index() + } + /// Returns the configured database kind for this database. pub fn db_kind(&self) -> String { self.db.db_kind() diff --git a/zebra-state/src/service/finalized_state/zebra_db/block.rs b/zebra-state/src/service/finalized_state/zebra_db/block.rs index 14e57c024ea..b2a0d5788d3 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block.rs @@ -1120,13 +1120,52 @@ impl ZebraDb { .collect(); // Get a list of the spent UTXOs, before we delete any from the database. - let outpoints: Vec = finalized - .block - .transactions - .iter() - .flat_map(|tx| tx.inputs().iter()) - .flat_map(|input| input.outpoint()) - .collect(); + // + // Per-checkpoint transparent reconcile (`defer_transparent_reconcile`): in the + // deferred range, record this block's spent outpoints into the reconcile window + // and pass NO outpoints to the read loop, so the per-block spent-UTXO resolution, + // the `utxo_by_out_loc` deletes, and the transparent value-pool debit are all + // skipped here. Unlike the probe below, this is correct: the checkpoint reconcile + // resolves, deletes, and debits in one batched pass. The recorded outpoints (and + // the block) are threaded into the pipeline contribution below; deferral only + // takes effect on the run-ahead pipeline path, which the bench always uses. + // + // Benchmark-only ceiling probe (`ZEBRA_BENCH_SKIP_TRANSPARENT_READS=1`): drop the + // spent outpoints with NO reconcile, so the spend work is skipped entirely. This + // measures the throughput ceiling of deferring that work off the commit critical + // path. It produces an INCORRECT value pool and UTXO set, so it is never a shipped + // path — only a measurement of the upper bound. + let defer_spends = self.defers_transparent_spends(); + // Deferral records into the run-ahead pipeline's reconcile window, so it is + // only correct when an overlay is present. Without one there is nowhere to + // record the spends and the checkpoint reconcile would never run, silently + // corrupting the value pool / UTXO set. The bench always runs pipelined. + assert!( + !defer_spends || overlay.is_some(), + "defer_transparent_reconcile requires the run-ahead pipeline \ + (finalized_block_pipeline_depth > 0)" + ); + let mut deferred_spent: Vec = Vec::new(); + let outpoints: Vec = if defer_spends { + deferred_spent = finalized + .block + .transactions + .iter() + .flat_map(|tx| tx.inputs().iter()) + .flat_map(|input| input.outpoint()) + .collect(); + Vec::new() + } else if super::bench_skip_transparent_reads() { + Vec::new() + } else { + finalized + .block + .transactions + .iter() + .flat_map(|tx| tx.inputs().iter()) + .flat_map(|input| input.outpoint()) + .collect() + }; // Serialize the raw transaction bytes for `tx_by_loc` concurrently with the // spent-UTXO reads. Serialization is CPU-bound while the reads wait on disk, @@ -1400,6 +1439,13 @@ impl ZebraDb { wrote_vct_upgrade_marker: block_outputs.wrote_vct_upgrade_marker, created_outputs, updated_balances, + // Per-checkpoint reconcile: when deferring, hand the spent outpoints + // + block + deferred-pool change to the pipeline window so the + // checkpoint reconcile can resolve, delete, and re-debit them. Empty + // / `None` when not deferring (no behavior change). + deferred_spent, + deferred_block: defer_spends.then(|| finalized.block.clone()), + deferred_pool_change: finalized.deferred_pool_balance_change, } }); diff --git a/zebra-state/src/service/finalized_state/zebra_db/chain.rs b/zebra-state/src/service/finalized_state/zebra_db/chain.rs index 9b869ba33c1..e3b353e8227 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/chain.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/chain.rs @@ -21,6 +21,7 @@ use zebra_chain::{ block::Height, block_info::BlockInfo, history_tree::HistoryTree, + parameters::Network, serialization::{CompactSizeMessage, ZcashSerialize as _}, transparent, value_balance::ValueBalance, @@ -30,11 +31,13 @@ use crate::{ request::FinalizedBlock, service::finalized_state::{ disk_db::{DiskWriteBatch, ReadDisk}, - disk_format::{chain::HistoryTreeParts, RawBytes}, + disk_format::transparent::AddressBalanceLocationUpdates, + disk_format::{chain::HistoryTreeParts, OutputLocation, RawBytes}, + pipeline::ReconcileBlock, zebra_db::{metrics::value_pool_metrics, ZebraDb}, TypedColumnFamily, }, - HashOrHeight, ValidateContextError, + BoxError, HashOrHeight, ValidateContextError, }; /// The name of the History Tree column family. @@ -219,6 +222,26 @@ impl ZebraDb { .unwrap_or_else(ValueBalance::zero) } + /// Verification helper (offline tools): the count of `block_info` entries and + /// the lowest/highest heights missing a `BlockInfo` in `[lo, hi]`. + pub fn block_info_coverage(&self, lo: Height, hi: Height) -> (u64, Option, u64) { + let cf = self.block_info_cf(); + let mut count = 0u64; + let mut first_missing = None; + let mut missing = 0u64; + for height in lo.0..=hi.0 { + if cf.zs_get(&Height(height)).is_some() { + count += 1; + } else { + if first_missing.is_none() { + first_missing = Some(Height(height)); + } + missing += 1; + } + } + (count, first_missing, missing) + } + /// Returns the stored `BlockInfo` for the given block. pub fn block_info(&self, hash_or_height: HashOrHeight) -> Option { let height = hash_or_height.height_or_else(|hash| self.height(hash))?; @@ -227,6 +250,100 @@ impl ZebraDb { block_info_cf.zs_get(&height) } + + /// Per-checkpoint transparent reconcile: write, in one atomic batch, the + /// `utxo_by_out_loc` deletes for every deferred spend in `records` and the + /// recomputed chain value pool (the tip pool plus per-height `BlockInfo`). + /// Returns the value pool after the last block in `records`. + /// + /// `start_value_pool` is the pool before the first block in `records`, + /// `resolved` maps each spent outpoint to its on-disk location and UTXO. + /// + /// This reproduces, in a batched pass, exactly what the per-block + /// [`prepare_spent_transparent_outputs_batch`](DiskWriteBatch::prepare_spent_transparent_outputs_batch) + /// (with the address index off) and + /// [`prepare_chain_value_pools_batch`](DiskWriteBatch::prepare_chain_value_pools_batch) + /// would have written inline, so the resulting state is byte-identical. + #[allow(clippy::unwrap_in_result)] + pub(crate) fn commit_checkpoint_reconcile( + &self, + network: &Network, + start_value_pool: ValueBalance, + records: &[ReconcileBlock], + resolved: &HashMap, + ) -> Result, BoxError> { + let mut batch = DiskWriteBatch::new(); + + // The deletes: every resolved spend, keyed by output location. The address + // index is off in the deferred range, so `skip_index = true` makes this + // delete only the `utxo_by_out_loc` entries (no address-link deletes), and + // the empty `address_balances` is never consulted. + let spent_utxos_by_out_loc: BTreeMap = resolved + .values() + .map(|(out_loc, utxo)| (*out_loc, utxo.clone())) + .collect(); + batch.prepare_spent_transparent_outputs_batch( + &self.db, + network, + &spent_utxos_by_out_loc, + &AddressBalanceLocationUpdates::Insert(HashMap::new()), + true, + ); + + // Recompute the value pool sequentially in block order, writing each block's + // running pool into `BlockInfo` (exactly as the inline per-block path does), + // and the final pool into the single-key tip CF once at the end. + let mut value_pool = start_value_pool; + for record in records { + let spent_by_block: HashMap = record + .spent + .iter() + .map(|outpoint| { + let (_out_loc, utxo) = resolved + .get(outpoint) + .expect("every deferred spend was resolved above"); + (*outpoint, utxo.clone()) + }) + .collect(); + + let block_value_pool_change = record + .block + .chain_value_pool_change(&spent_by_block, record.deferred_pool_change)?; + value_pool = value_pool.add_chain_value_pool_change(block_value_pool_change)?; + value_pool_metrics(&value_pool); + + // Block size, summed per-transaction (byte-identical to serializing the + // whole block), as in `prepare_chain_value_pools_batch`. + let block_size = { + let transactions = &record.block.transactions; + let transactions_size: usize = transactions + .iter() + .map(|transaction| transaction.zcash_serialized_size()) + .sum(); + let tx_count_size = CompactSizeMessage::try_from(transactions.len()) + .expect("block must have a valid transaction count") + .zcash_serialized_size(); + record.block.header.zcash_serialized_size() + tx_count_size + transactions_size + }; + + let _ = self + .block_info_cf() + .with_batch_for_writing(&mut batch) + .zs_insert( + &record.height, + &BlockInfo::new(value_pool, block_size as u32), + ); + } + + let _ = self + .chain_value_pools_cf() + .with_batch_for_writing(&mut batch) + .zs_insert(&(), &value_pool); + + self.write_batch(batch)?; + + Ok(value_pool) + } } impl DiskWriteBatch { @@ -295,6 +412,16 @@ impl DiskWriteBatch { utxos_spent_by_block: HashMap, value_pool: ValueBalance, ) -> Result, ValidateContextError> { + // Per-checkpoint reconcile / ceiling probe: the value-pool change needs the spent + // values. When deferring (`defers_transparent_spends`), the spent reads are skipped + // here and recomputed in the batched checkpoint reconcile, so leave the pool + // unchanged (it is threaded forward and corrected at the next checkpoint). The + // benchmark probe (`bench_skip_transparent_reads`) skips it with no reconcile, + // leaving the pool permanently stale (measurement only, never shipped). + if db.defers_transparent_spends() || super::bench_skip_transparent_reads() { + return Ok(value_pool); + } + let block_value_pool_change = finalized .block .chain_value_pool_change( diff --git a/zebra-state/src/service/finalized_state/zebra_db/transparent.rs b/zebra-state/src/service/finalized_state/zebra_db/transparent.rs index 58cfb69e42f..6c10a727012 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/transparent.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/transparent.rs @@ -37,7 +37,7 @@ use crate::{ AddressBalanceLocationUpdates, AddressLocation, AddressTransaction, AddressUnspentOutput, OutputLocation, }, - TransactionLocation, + RawBytes, TransactionLocation, }, zebra_db::ZebraDb, }, @@ -83,6 +83,60 @@ pub type TransactionLocationBySpentOutputLocationCf<'cf> = impl ZebraDb { // Column family convenience methods + /// Verification helper (offline tools): a stable digest of the `utxo_by_out_loc` + /// column family — the entry count plus order-independent rolling hashes of the + /// raw key and value bytes — for comparing two databases' UTXO sets without + /// dumping every entry. + /// + /// Returns `(count, key_sum, key_xor, value_sum, value_xor)`. Two databases + /// have an identical UTXO set iff all five fields match. Used by the + /// per-checkpoint reconcile byte-match verification. + pub fn utxo_by_out_loc_verification_digest(&self) -> (u64, u64, u64, u64, u64) { + self.cf_verification_digest("utxo_by_out_loc") + } + + /// Verification helper (offline tools): the same stable digest as + /// [`Self::utxo_by_out_loc_verification_digest`], over the `block_info` column + /// family — so two databases' per-height value pools can be compared. + pub fn block_info_verification_digest(&self) -> (u64, u64, u64, u64, u64) { + self.cf_verification_digest("block_info") + } + + /// Order-independent rolling digest of a column family's raw key/value bytes: + /// `(count, key_sum, key_xor, value_sum, value_xor)`. + fn cf_verification_digest(&self, cf_name: &str) -> (u64, u64, u64, u64, u64) { + // FNV-1a over a byte slice. + fn fnv1a(bytes: &[u8]) -> u64 { + let mut hash: u64 = 0xcbf2_9ce4_8422_2325; + for &byte in bytes { + hash ^= u64::from(byte); + hash = hash.wrapping_mul(0x0000_0100_0000_01b3); + } + hash + } + + let cf = self + .db + .cf_handle(cf_name) + .expect("column family was created when database was created"); + + let (mut count, mut key_sum, mut key_xor, mut value_sum, mut value_xor) = (0, 0, 0, 0, 0); + for (key, value) in self + .db + .zs_forward_range_iter::<_, RawBytes, RawBytes, _>(&cf, ..) + { + let key_hash = fnv1a(key.raw_bytes()); + let value_hash = fnv1a(value.raw_bytes()); + count += 1; + key_sum = u64::wrapping_add(key_sum, key_hash); + key_xor ^= key_hash; + value_sum = u64::wrapping_add(value_sum, value_hash); + value_xor ^= value_hash; + } + + (count, key_sum, key_xor, value_sum, value_xor) + } + /// Returns a typed handle to the transaction location by spent output location column family. pub(crate) fn tx_loc_by_spent_output_loc_cf( &self, diff --git a/zebra-state/src/service/write.rs b/zebra-state/src/service/write.rs index 006028e923e..0ba5ef7293c 100644 --- a/zebra-state/src/service/write.rs +++ b/zebra-state/src/service/write.rs @@ -17,7 +17,9 @@ use tokio::sync::{ }; use tracing::Span; +use zebra_chain::amount::NonNegative; use zebra_chain::block::{self, Height}; +use zebra_chain::value_balance::ValueBalance; use zebra_chain::parallel::{ commitment_aux::BlockCommitmentRoots, @@ -30,8 +32,8 @@ use crate::{ service::{ check, finalized_state::{ - spawn_note_precompute, DiskWriteBatch, FinalizedPipeline, FinalizedState, - PreparedCommitTrace, ZebraDb, + reconcile_window, spawn_note_precompute, DiskWriteBatch, FinalizedPipeline, + FinalizedState, PreparedCommitTrace, ReconcileBlock, ZebraDb, }, non_finalized_state::NonFinalizedState, queued_blocks::{QueuedCheckpointVerified, QueuedSemanticallyVerified}, @@ -124,6 +126,75 @@ fn run_finalized_writer( } } +/// A window of deferred transparent spends handed to the reconcile worker thread. +/// +/// The window blocks (height-ordered, all at or below the boundary) are resolved, +/// deleted, and re-debited off the assembler thread. Empty jobs are never sent. +struct ReconcileJob { + records: Vec, +} + +/// The deferred-transparent reconcile worker thread loop (v2). +/// +/// Owns the running chain value pool (seeded once from disk by the spawner) and is +/// the **sole** writer of `chain_value_pools`, `block_info`, and the spent +/// `utxo_by_out_loc` deletes in the deferred range — the assembler and disk writer +/// write only the disjoint UTXO creates / headers / nullifiers / trees, and skip +/// the value pool entirely when deferring. So this runs concurrently with assembly +/// without shared-state races or key conflicts. +/// +/// Jobs are processed FIFO, so the value-pool chaining stays ordered. A reconcile +/// failure is fatal, as a rocksdb write failure is on the disk writer. +fn run_reconcile_worker( + db: ZebraDb, + mut value_pool: ValueBalance, + job_receiver: std::sync::mpsc::Receiver, +) { + let network = db.network(); + while let Ok(ReconcileJob { records }) = job_receiver.recv() { + match reconcile_window(&db, &network, &records, value_pool) { + Ok(new_pool) => value_pool = new_pool, + Err(error) => panic!("deferred transparent reconcile worker failed: {error}"), + } + } +} + +/// Handle to the reconcile worker thread and its bounded job channel. +/// +/// The channel is capacity-1: handing off the next window blocks the assembler only +/// while the worker is still busy with the previous one, bounding the in-flight +/// `Arc` to ~two windows. +struct ReconcileWorker { + sender: Option>, + handle: Option>, +} + +impl ReconcileWorker { + /// Hand a window prefix to the worker, applying backpressure if it is still busy. + /// Empty windows are ignored. + fn send(&self, records: Vec) { + if records.is_empty() { + return; + } + if let Some(sender) = self.sender.as_ref() { + // The worker only stops on channel close, so a send error means it + // panicked mid-reconcile; that is fatal (inconsistent value pool / UTXOs). + sender + .send(ReconcileJob { records }) + .expect("reconcile worker thread has gone away"); + } + } + + /// Drop the job channel and wait for the worker to finish every queued window, so + /// all reconcile writes are durable. Idempotent. + fn flush_and_join(&mut self) { + drop(self.sender.take()); + if let Some(handle) = self.handle.take() { + handle.join().expect("reconcile worker thread panicked"); + } + } +} + /// Process the disk-writer thread's flush acknowledgements: for each acked height, /// advance the finalized chain tip, retire the now-durable overlay entries, and run /// the configured stop-height check (which exits the process if matched). @@ -137,6 +208,7 @@ fn drain_finalized_acks( ack_receiver: &std::sync::mpsc::Receiver, in_flight: &mut VecDeque<(Height, block::Hash, ChainTipBlock)>, chain_tip_sender: &mut ChainTipSender, + mut reconcile_worker: Option<&mut ReconcileWorker>, block_until_empty: bool, ) { while !in_flight.is_empty() { @@ -161,6 +233,46 @@ fn drain_finalized_acks( chain_tip_sender.set_finalized_tip(tip_block); pipeline.retire_through(height); + // Deferred transparent reconcile: when deferral is on, reconcile the window + // prefix at or below this now-durable height at each reconcile boundary and + // at the stop height (so the final partial window is reconciled before the + // process exits). The boundary is a fixed block interval when + // `defer_reconcile_interval > 0` (mainnet checkpoints are only ~30-40 blocks + // apart here, too frequent to amortize the reconcile cost), otherwise each + // checkpoint. + // + // v2: hand the window to the dedicated reconcile worker thread so its disk + // reads + value-pool recompute overlap continued assembly. v1 fallback (no + // worker): reconcile inline. Fatal on error either way: a failed reconcile + // would leave the value pool / UTXO set inconsistent. + if finalized_state.db.defers_transparent_spends() { + let interval = finalized_state.db.config().defer_reconcile_interval; + let is_boundary = if interval > 0 { + height.0 % (interval as u32) == 0 + } else { + finalized_state + .db + .network() + .checkpoint_list() + .contains(height) + }; + let at_stop = finalized_state.is_at_stop_height(height); + if is_boundary || at_stop { + if let Some(worker) = reconcile_worker.as_deref_mut() { + worker.send(pipeline.take_reconcile_prefix(height)); + // The process is about to exit at the stop height; flush the + // worker so every queued reconcile is durable first. + if at_stop { + worker.flush_and_join(); + } + } else { + pipeline + .reconcile_checkpoint(finalized_state, height) + .expect("deferred transparent reconcile failed"); + } + } + } + // The block is now durable, so the stop-height check (which exits the // process) is safe to run here. finalized_state.finalized_stop_at_height_if_configured( @@ -552,6 +664,36 @@ impl WriteBlockWorkerTask { pipeline_writer_handle = Some(handle); } + // Deferred-transparent reconcile worker (v2). When deferral is on (and not + // forced inline), spawn a dedicated thread that resolves/deletes/re-debits + // each window off the assembler thread. It owns the running value pool, + // seeded once here from disk (the worker is the sole writer of the value pool + // in the deferred range, so this stays exclusively owned with no races). + let mut reconcile_worker: Option = None; + if pipeline_active + && finalized_state.db.defers_transparent_spends() + && !finalized_state.db.config().defer_reconcile_inline + { + // Capacity 1: hand-off blocks the assembler only while the worker is + // still busy with the previous window (~two windows of blocks in flight). + let (job_sender, job_receiver) = std::sync::mpsc::sync_channel::(1); + let worker_db = finalized_state.db.clone(); + let start_value_pool = finalized_state.db.finalized_value_pool(); + let worker_span = Span::current(); + let handle = std::thread::Builder::new() + .name("zebra-reconcile-worker".to_string()) + .spawn(move || { + worker_span.in_scope(|| { + run_reconcile_worker(worker_db, start_value_pool, job_receiver) + }) + }) + .expect("failed to spawn the reconcile worker thread"); + reconcile_worker = Some(ReconcileWorker { + sender: Some(job_sender), + handle: Some(handle), + }); + } + // Write all the finalized blocks sent by the state, // until the state closes the finalized block channel's sender. loop { @@ -834,6 +976,7 @@ impl WriteBlockWorkerTask { ack_receiver, &mut pipeline_in_flight, chain_tip_sender, + reconcile_worker.as_mut(), false, ); if finalized_state.is_at_stop_height(committed_height) { @@ -843,6 +986,7 @@ impl WriteBlockWorkerTask { ack_receiver, &mut pipeline_in_flight, chain_tip_sender, + reconcile_worker.as_mut(), true, ); } @@ -957,8 +1101,31 @@ impl WriteBlockWorkerTask { ack_receiver, &mut pipeline_in_flight, chain_tip_sender, + reconcile_worker.as_mut(), true, ); + + // Deferred reconcile: the trailing blocks committed past the last + // boundary form a partial window that nothing reconciled. Reconcile it + // now (through the durable tip) so the value pool and UTXO set are + // consistent before the handoff to the non-finalized state. (The + // production handoff drain barrier is a later stage; this keeps the + // prototype correct at the channel close.) + if finalized_state.db.defers_transparent_spends() { + let tip_height = finalized_state.db.tip().map(|(height, _)| height); + if let Some(tip_height) = tip_height { + if let Some(worker) = reconcile_worker.as_mut() { + // v2: send the final window, then wait for the worker to + // make every queued reconcile durable. + worker.send(pipeline.take_reconcile_prefix(tip_height)); + worker.flush_and_join(); + } else { + pipeline + .reconcile_checkpoint(finalized_state, tip_height) + .expect("final deferred transparent reconcile failed"); + } + } + } } if let Some(handle) = pipeline_writer_handle.take() { let _ = handle.join(); From 89d1038bd2eb37098105d19ab753765b39d28d48 Mon Sep 17 00:00:00 2001 From: roman Date: Tue, 30 Jun 2026 18:09:35 -0600 Subject: [PATCH 10/17] perf(block-sync): throttle sequencer budget audit off the hot path `SequencerTask::publish_view` ran on every body and control event and each time recomputed the reserved-byte total via `WorkQueue::reserved_bytes()`, an O(pending + in_flight) scan of the whole work queue, purely to feed the `sync.block.budget.audit_drift` consistency metric. As the apply/commit backlog grew to thousands of blocks during checkpoint sync, that per-event scan became quadratic and saturated the single sequencer task, starving the commit-compute threads and freezing body commits for tens of seconds (observed ~18-30s stalls around the peak-backlog heights via mid-stall thread dumps). The published `SequencerView` already uses the budget's O(1) running counter, so the full scan is only a drift check. Sample it at most once per `BUDGET_AUDIT_INTERVAL` (1s) instead of on every event, keeping the audit while removing it from the hot path. --- CHANGELOG.md | 11 +++++ CHANGELOG_PARAMS.md | 1 + .../src/zakura/block_sync/sequencer_task.rs | 48 ++++++++++++++----- 3 files changed, 48 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8699a0c55d3..ad2a0129560 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,17 @@ and this project adheres to [Semantic Versioning](https://semver.org). ### Performance +- Fix a quadratic slowdown in the Zakura block-sync sequencer that stalled the + commit pipeline for tens of seconds during checkpoint sync. `publish_view` ran + on every body and control event and each time re-derived the reserved-byte + total with an O(pending + in_flight) scan of the work queue, purely to feed a + budget drift-audit metric. As the apply/commit backlog grew to thousands of + blocks, that per-event scan became quadratic and saturated the single sequencer + task, starving the commit-compute threads (observed as ~18–30s body-commit + freezes around the peak-backlog heights). The published view already uses the + budget's O(1) running counter, so the audit's full scan is now sampled at most + once per `BUDGET_AUDIT_INTERVAL` (1s) instead of on every event, keeping the + drift check while removing it from the hot path. - Compute the v5 ZIP-244 txid and authorizing-data digest natively. Both previously routed through `Transaction::to_librustzcash`, which re-serializes and reparses the whole transaction — decompressing every Jubjub and Pallas diff --git a/CHANGELOG_PARAMS.md b/CHANGELOG_PARAMS.md index c7695aa028a..d98d917ff76 100644 --- a/CHANGELOG_PARAMS.md +++ b/CHANGELOG_PARAMS.md @@ -28,5 +28,6 @@ Keep entries **newest-first**. Each row records: | Parameter | Location | Old → New | PR | Why | | --- | --- | --- | --- | --- | +| `BUDGET_AUDIT_INTERVAL` | `zebra-network/src/zakura/block_sync/sequencer_task.rs` | _(new)_ → `1s` | _(this PR)_ | Throttle the sequencer's byte-budget drift audit to at most once per interval. The audit re-derives the reserved total via an O(pending + in_flight) work-queue scan; running it on every body/control event (as `publish_view` did) is quadratic in the backlog and stalls the commit pipeline. The published view uses the budget's O(1) running counter, so the scan is only a drift metric. | | `Config::finalized_block_pipeline_depth` | `zebra-state/src/config.rs` | _(new)_ → `0` | _(this PR)_ | Run-ahead finalized-commit pipeline depth (blocks the assembler may build ahead of the durable disk write). Defaults to `0` = synchronous (original behavior); `> 0` overlaps the next block's assembly with the current block's flush, bounded so the in-memory overlay stays small. | | `OUTBOUND_WINDOW_FLOOR_TIMEOUTS_BEFORE_DISCONNECT` | `zebra-network/src/zakura/block_sync/state.rs` | `3` → `2 * OUTBOUND_WINDOW_REDUCTION_EPOCH_TIMEOUTS` (`32`) | [#303](https://github.com/valargroup/zebra/pull/303) | Tolerate two full reduction epochs (~256s at the 8s request timeout) of floor-pinned timeouts before disconnecting a block-sync peer, instead of ~24s, so briefly-congested peers are not churned. Any successful response resets the streak. | diff --git a/zebra-network/src/zakura/block_sync/sequencer_task.rs b/zebra-network/src/zakura/block_sync/sequencer_task.rs index 6c1963706ef..0185859426e 100644 --- a/zebra-network/src/zakura/block_sync/sequencer_task.rs +++ b/zebra-network/src/zakura/block_sync/sequencer_task.rs @@ -196,6 +196,17 @@ pub(super) fn initial_view(frontiers: BlockSyncFrontiers) -> SequencerView { } } +/// How often [`SequencerTask::publish_view`] re-audits the byte budget against a +/// full work-queue scan. +/// +/// The audit only feeds the `sync.block.budget.audit_drift` metric — the budget +/// already keeps an O(1) running counter that the published view uses directly. +/// `publish_view` runs on every body and control event, and the audit's scan is +/// O(pending + in_flight), so scanning on each call is quadratic in the backlog +/// and stalls the commit pipeline once it grows. Sampling on an interval keeps +/// the drift check while bounding its cost to one scan per interval. +const BUDGET_AUDIT_INTERVAL: Duration = Duration::from_secs(1); + /// The serial commit-pipeline task. Owns the `Sequencer` (moved out of state), a /// `ByteBudget` clone, an `Arc` clone, an action sender clone, and the /// committed throughput meter. Releases bytes directly and emits `SubmitBlock` / @@ -206,6 +217,9 @@ pub(super) struct SequencerTask { work: Arc, actions: mpsc::Sender, committed_throughput: ThroughputMeter, + /// Next time [`Self::publish_view`] may run the O(n) budget audit; see + /// [`BUDGET_AUDIT_INTERVAL`]. + next_budget_audit: Instant, /// Tracks the finalized height so the published view carries it forward; the /// reactor folds it into its `finalized_height` mirror with a `max`. finalized_height: block::Height, @@ -242,6 +256,7 @@ impl SequencerTask { work, actions, committed_throughput, + next_budget_audit: Instant::now(), finalized_height: frontiers.finalized_height, verified_block_hash: frontiers.verified_block_hash, reset_epoch: 0, @@ -732,20 +747,29 @@ impl SequencerTask { } fn publish_view(&mut self) { - self.committed_throughput.sample(Instant::now()); + let now = Instant::now(); + self.committed_throughput.sample(now); let reorder_buffered_bytes = self.sequencer.reorder_buffered_bytes(); let applying_buffered_bytes = self.sequencer.applying_buffered_bytes(); - let body_input_bytes = self - .body_input_bytes - .load(std::sync::atomic::Ordering::Relaxed); - let expected_budget = self - .work - .reserved_bytes() - .saturating_add(reorder_buffered_bytes) - .saturating_add(applying_buffered_bytes) - .saturating_add(body_input_bytes); - self.budget - .audit(expected_budget, "block-sync sequencer view"); + // The budget audit recomputes the reserved total by scanning the whole + // work queue (O(pending + in_flight)). Since `publish_view` runs on every + // body/control event, keep that scan off the hot path and sample it on an + // interval instead (see `BUDGET_AUDIT_INTERVAL`); the published view below + // does not depend on it. + if now >= self.next_budget_audit { + let body_input_bytes = self + .body_input_bytes + .load(std::sync::atomic::Ordering::Relaxed); + let expected_budget = self + .work + .reserved_bytes() + .saturating_add(reorder_buffered_bytes) + .saturating_add(applying_buffered_bytes) + .saturating_add(body_input_bytes); + self.budget + .audit(expected_budget, "block-sync sequencer view"); + self.next_budget_audit = now + BUDGET_AUDIT_INTERVAL; + } let _ = self.view_tx.send_replace(SequencerView { verified_tip: self.sequencer.verified_tip(), verified_hash: self.verified_block_hash, From 9fef29c2c19b7f0af8c170322770832e23aebbea Mon Sep 17 00:00:00 2001 From: Roman Date: Wed, 1 Jul 2026 00:14:57 +0000 Subject: [PATCH 11/17] feat(state): lifecycle guards for deferred-transparent-reconcile Make the deferred transparent reconcile prototype safe to benchmark on a real, disposable below-checkpoint node. The reconcile logic is byte-match-verified; these are lifecycle guards around it, all no-ops when the feature is off. - Reachability: expose defer_transparent_reconcile + defer_reconcile_interval as [state] config fields (default off), so a node can opt in from zebrad.toml. - Height-bound gate: defers_transparent_spends_at requires height <= max_checkpoint_height, so every above-checkpoint block commits inline (non-deferred). Unit-tested. - Handoff drain barrier (consensus-critical): before the first above-checkpoint block, flush in-flight blocks, drain the reconcile window (flush_and_join the worker), and reseed the pipeline value pool from disk, so the semantic verifier never reads a stale/superset UTXO set or a lagging value pool. The common handoff (checkpoint-verified channel close) drains the same way. - Format check + clean shutdown: check_new_blocks skips while deferral is configured and the tip is in the deferred range (the BlockInfo/value-pool lag is expected there); the reconcile worker is flush_and_join'd at the handoff and via a Drop safety net. Crash recovery + RPC guards remain out of scope (disposable-node model, documented in docs/design/deferred-transparent-reconcile.md). Byte-match preserved: value pool + UTXO set + per-height BlockInfo identical to a non-deferred run. --- docs/design/deferred-transparent-reconcile.md | 88 +++++++++------ zebra-state/src/config.rs | 100 +++++++++++++++--- .../block_info_and_address_received.rs | 11 ++ .../src/service/finalized_state/pipeline.rs | 11 ++ .../src/service/finalized_state/zebra_db.rs | 30 ++++-- .../service/finalized_state/zebra_db/block.rs | 2 +- .../service/finalized_state/zebra_db/chain.rs | 2 +- zebra-state/src/service/write.rs | 65 ++++++++++-- 8 files changed, 243 insertions(+), 66 deletions(-) diff --git a/docs/design/deferred-transparent-reconcile.md b/docs/design/deferred-transparent-reconcile.md index 0c1b01f5608..9f98638a65e 100644 --- a/docs/design/deferred-transparent-reconcile.md +++ b/docs/design/deferred-transparent-reconcile.md @@ -1,7 +1,9 @@ # Deferred transparent reconcile (prototype) -Status: **prototype, benchmark-gated, default-off. NOT safe to enable on a real node yet** -(see "Production-readiness gaps"). +Status: **prototype, default-off, opt-in.** Byte-match-verified with lifecycle guards +(height gate, handoff drain barrier, format-check + clean-shutdown). Benchmark-only: safe on +a **disposable** below-checkpoint node; crash recovery + RPC guards are still missing (see +"Production-readiness gaps (remaining)"). ## Problem @@ -74,35 +76,61 @@ Benchmarked offline via `zebra-replay-bench` on a mainnet 1.85M–1.9M pruned sn ## Gating -Env-gated for benchmarking, default off; no production code path is enabled: +Default off. Opt-in, reachable by a real node for benchmarking: -- `zebra-state` `Config`: `defer_transparent_reconcile` (off), `defer_reconcile_interval` - (0 = per checkpoint), `defer_reconcile_inline` (force v1). All `#[serde(skip)]`. -- `Config::defers_transparent_spends()` currently requires `defer_transparent_reconcile && - skip_address_index()` (pruned + checkpoint-sync). It does **not** yet bound the height to - `max_checkpoint_height`. +- `zebra-state` `Config` (settable under `[state]` in `zebrad.toml`): + `defer_transparent_reconcile` (off), `defer_reconcile_interval` (0 = per checkpoint). + `defer_reconcile_inline` (force the v1 inline path) stays `#[serde(skip)]` (bench A/B only). +- `Config::defers_transparent_spends_at(network, height)` requires + `defer_transparent_reconcile && skip_address_index()` (pruned + checkpoint-sync) **and** + `height <= max_checkpoint_height`. `Config::defer_reconcile_configured()` is the + height-independent lifecycle predicate (worker spawn / drain triggers). - `zebra-replay-bench`: `ZRB_DEFER_TRANSPARENT`, `ZRB_RECONCILE_INTERVAL`, `ZRB_RECONCILE_INLINE`, `ZRB_STOP_AT_HEIGHT`. -## Production-readiness gaps (why it is not safe to enable yet) - -The reconcile *logic* is byte-match-verified, but the surrounding lifecycle is not: - -1. **No handoff drain barrier.** The deferral is only valid below `max_checkpoint_height`. - At the handoff to semantic verification the value pool + UTXO set must be fully - reconciled first (the semantic verifier reads them to validate spends). The prototype - omits the height bound and the drain, so enabling it on a node that reaches the handoff - would feed the semantic verifier a stale/superset UTXO set + lagging value pool — a - consensus failure. -2. **No crash recovery.** The reconciled (durable transparent) tip trails the committed - tip. A crash mid-window leaves `utxo_by_out_loc` a superset and `BlockInfo`/value pool - lagging; restart must re-derive the pending window. Not implemented. -3. **No RPC guards.** Mid-window the UTXO set is a superset and the value pool lags, so - address/utxo/value RPCs would return wrong results. Not guarded. -4. **Background format check.** `check_new_blocks` trips on the transient `BlockInfo` lag; - the prototype sidesteps it only via the deterministic-stop bench harness. A production - path must skip/relax it in the deferred range. -5. **Shutdown + error handling.** The worker is fatal-on-error (panic), and clean shutdown - requires draining the worker before exit (the bench parks its main task to do this). - -Until 1–5 are implemented and tested, this is a measurement prototype only. +## Lifecycle guards (implemented) + +The reconcile *logic* is byte-match-verified; these lifecycle guards make it safe to +benchmark on a real below-checkpoint node: + +1. **Reachability.** `defer_transparent_reconcile` + `defer_reconcile_interval` are serde + fields on the state config, so a node can opt in from `[state]`. Default off. +2. **Height-bound gate.** `defers_transparent_spends_at` returns false above + `max_checkpoint_height`, so every above-checkpoint block commits inline (non-deferred). + Unit-tested (`defers_transparent_spends_only_in_checkpoint_range`). +3. **Handoff drain barrier (consensus-critical).** Before the first block above + `max_checkpoint_height` commits, the finalized committer flushes every in-flight block to + disk, drains the pending reconcile window (`flush_and_join` the worker), and refreshes the + pipeline's threaded value pool from the now-current disk pool — so the semantic verifier + never reads a stale/superset UTXO set or lagging value pool. The common handoff (the + checkpoint→non-finalized channel close) drains the same way. +4. **Format check + clean shutdown.** The background `check_new_blocks` skips the new-blocks + validation while deferral is configured and the tip is in the deferred range (the + `BlockInfo`/value-pool lag there is expected, and the address index is off). The reconcile + worker is `flush_and_join`'d at the handoff and again via a `Drop` safety net, so a clean + stop finishes any queued reconcile before teardown. + +## Production-readiness gaps (remaining) + +1. **No crash recovery.** The reconciled (durable transparent) tip trails the committed tip. + A crash mid-window leaves `utxo_by_out_loc` a superset and `BlockInfo`/value pool lagging; + restart would need to re-derive the pending window. Not implemented — a node enabling this + must be treated as **disposable / re-snapshottable**. +2. **No RPC guards.** Mid-window the UTXO set is a superset and the value pool lags, so + address/utxo/value RPCs would return wrong results in the deferred range. Not guarded. + +Until crash recovery + RPC guards are implemented, this remains a benchmarking feature for a +disposable below-checkpoint node, not a general-purpose production mode. + +## Benchmarking on a real below-checkpoint node + +1. Snapshot a pruned + checkpoint-sync node whose tip is well below `max_checkpoint_height` + (mainnet ~3.36M) — the node must be **disposable** (re-snapshot per run; no crash + recovery). +2. In `zebrad.toml` `[state]`: set `defer_transparent_reconcile = true` and + `defer_reconcile_interval = 2000` (pruned storage + checkpoint sync are required for the + address index to be off, which the gate needs). +3. Set `debug_stop_at_height` to a height still below `max_checkpoint_height` so the run + stops inside the deferred range (the handoff barrier is exercised by code/tests, not by + the bench range). +4. Avoid address/utxo/value RPCs against the node while it is in the deferred range. diff --git a/zebra-state/src/config.rs b/zebra-state/src/config.rs index ef5b4ca924b..7fd580b7df1 100644 --- a/zebra-state/src/config.rs +++ b/zebra-state/src/config.rs @@ -16,7 +16,7 @@ use serde::{ use tokio::task::{spawn_blocking, JoinHandle}; use tracing::Span; -use zebra_chain::{common::default_cache_dir, parameters::Network}; +use zebra_chain::{block, common::default_cache_dir, parameters::Network}; use crate::{ constants::{ @@ -153,23 +153,25 @@ pub struct Config { /// overlay of not-yet-flushed blocks. pub finalized_block_pipeline_depth: usize, - /// Prototype: defer the transparent spend resolution (UTXO deletes + value-pool - /// debit) off the per-block commit path and reconcile it in a batched pass at - /// each checkpoint boundary, in the checkpoint-trusted range. + /// **Experimental.** Defer the transparent spend resolution (UTXO deletes + + /// value-pool debit) off the per-block commit path and reconcile it in a batched + /// pass on a dedicated worker thread, in the checkpoint-trusted range. /// /// The per-block committer records spent outpoints into an in-memory window - /// instead of resolving them; at each checkpoint the window is batch-resolved - /// (sorted/deduped disk reads), the value pool is recomputed, and the deletes + - /// pool are written in one atomic batch. The auxiliary address index must also be - /// off ([`skip_address_index`](Config::skip_address_index)). - /// - /// Not exposed in serde; set by the benchmark for measurement. Production wiring - /// (auto-enable under pruned + checkpoint sync, the handoff drain barrier, RPC - /// guards, crash recovery) is not yet implemented, so this defaults to `false`. - #[serde(skip)] + /// instead of resolving them; every [`defer_reconcile_interval`] blocks the window + /// is batch-resolved (sorted/deduped disk reads), the value pool is recomputed, and + /// the deletes + pool are written in one atomic batch. Only takes effect when the + /// auxiliary address index is off ([`skip_address_index`](Config::skip_address_index), + /// i.e. pruned + checkpoint sync) and only below the last checkpoint; at the + /// checkpoint→semantic handoff the window is drained so the value pool / UTXO set + /// are current before semantic verification reads them. + /// + /// Default `false`. Opt-in for performance benchmarking on a disposable + /// below-checkpoint node; crash recovery and RPC mid-reconcile guards are not yet + /// implemented, so a node enabling this should be treated as re-snapshottable. pub defer_transparent_reconcile: bool, - /// Prototype: how many blocks between deferred-transparent reconciles, when + /// **Experimental.** How many blocks between deferred-transparent reconciles, when /// [`defer_transparent_reconcile`](Config::defer_transparent_reconcile) is on. /// /// `0` (the default) reconciles at every checkpoint boundary — but mainnet @@ -178,7 +180,6 @@ pub struct Config { /// (e.g. ~2000) reconciles a bigger batched window. Correctness only requires a /// bounded window with the window's spent UTXOs durable by reconcile time, not /// checkpoint alignment. - #[serde(skip)] pub defer_reconcile_interval: usize, /// Prototype: run the deferred-transparent reconcile inline on the assembler @@ -345,6 +346,29 @@ impl Config { matches!(self.storage_mode, StorageMode::Pruned(_)) && self.checkpoint_sync } + /// Whether the deferred-transparent reconcile is configured for this node: the + /// experimental [`defer_transparent_reconcile`](Config::defer_transparent_reconcile) + /// opt-in is on **and** the auxiliary address index is off (the deferred path skips + /// the per-block spent-UTXO reads the address index depends on). + /// + /// This is the lifecycle predicate (worker spawn, drain triggers); the per-block + /// decision is the height-bound [`defers_transparent_spends_at`](Config::defers_transparent_spends_at). + pub fn defer_reconcile_configured(&self) -> bool { + self.defer_transparent_reconcile && self.skip_address_index() + } + + /// Whether the per-block committer defers transparent spend resolution for a block + /// at `height` on `network`. + /// + /// Deferral is only valid in the checkpoint-trusted range: above the last + /// checkpoint the semantic verifier validates each block's spends against the UTXO + /// set and value pool, so those must be current (non-deferred). The handoff drain + /// barrier flushes any pending window before the first above-checkpoint block, and + /// this gate keeps every above-checkpoint block on the inline (non-deferred) path. + pub fn defers_transparent_spends_at(&self, network: &Network, height: block::Height) -> bool { + self.defer_reconcile_configured() && height <= network.checkpoint_list().max_height() + } + /// Validates the configured [`StorageMode`]. /// /// This must be called before opening the database, so that a misconfigured @@ -560,6 +584,52 @@ mod tests { ); } + #[test] + fn defers_transparent_spends_only_in_checkpoint_range() { + let network = Network::Mainnet; + let max_checkpoint = network.checkpoint_list().max_height(); + let below = block::Height(max_checkpoint.0 - 1); + let above = block::Height(max_checkpoint.0 + 1); + + // Configured: pruned + checkpoint sync (so the address index is off) + opt-in. + let deferring = Config { + storage_mode: StorageMode::Pruned(PruningConfig::default()), + checkpoint_sync: true, + defer_transparent_reconcile: true, + ..Default::default() + }; + assert!( + deferring.defers_transparent_spends_at(&network, below), + "defers below the last checkpoint" + ); + assert!( + deferring.defers_transparent_spends_at(&network, max_checkpoint), + "defers at the last checkpoint height" + ); + assert!( + !deferring.defers_transparent_spends_at(&network, above), + "must NOT defer above the last checkpoint (the semantic verifier reads the live UTXO set / value pool there)" + ); + + // Opt-in off: never defers, at any height. + let off = Config { + storage_mode: StorageMode::Pruned(PruningConfig::default()), + checkpoint_sync: true, + defer_transparent_reconcile: false, + ..Default::default() + }; + assert!(!off.defers_transparent_spends_at(&network, below)); + + // Address index on (archive): the deferred path is unsafe, never defers. + let archive = Config { + storage_mode: StorageMode::Archive, + checkpoint_sync: true, + defer_transparent_reconcile: true, + ..Default::default() + }; + assert!(!archive.defers_transparent_spends_at(&network, below)); + } + #[test] fn storage_mode_deserializes_from_documented_toml() { assert!( diff --git a/zebra-state/src/service/finalized_state/disk_format/upgrade/block_info_and_address_received.rs b/zebra-state/src/service/finalized_state/disk_format/upgrade/block_info_and_address_received.rs index ad059d7d844..8c6770a9100 100644 --- a/zebra-state/src/service/finalized_state/disk_format/upgrade/block_info_and_address_received.rs +++ b/zebra-state/src/service/finalized_state/disk_format/upgrade/block_info_and_address_received.rs @@ -250,6 +250,17 @@ impl DiskFormatUpgrade for Upgrade { return Ok(Ok(())); }; + // Deferred-transparent reconcile: in the checkpoint-trusted range the per-height + // `BlockInfo` (and the value pool) are written by the reconcile worker a window + // behind the committed tip, so a transient gap below the tip is expected and not + // corruption. The auxiliary address index is also off in this mode, so the + // received-balance check below does not apply either. Skip the new-blocks check + // while deferral is active and the tip is still inside the deferred range; above + // the last checkpoint blocks commit inline and the check resumes normally. + if db.defer_reconcile_configured() && tip_height <= network.checkpoint_list().max_height() { + return Ok(Ok(())); + } + // Check any outputs in the last 1000 blocks. let start_height = (tip_height - 1_000).unwrap_or(Height::MIN); diff --git a/zebra-state/src/service/finalized_state/pipeline.rs b/zebra-state/src/service/finalized_state/pipeline.rs index 4575830a7fe..d079245db06 100644 --- a/zebra-state/src/service/finalized_state/pipeline.rs +++ b/zebra-state/src/service/finalized_state/pipeline.rs @@ -250,6 +250,17 @@ impl FinalizedPipeline { self.value_pool } + /// Refresh the threaded value pool from the durable database. + /// + /// In the deferred range the reconcile worker is the sole writer of the value + /// pool, so the pipeline's threaded copy stays at its seed (deferred blocks pass + /// it through unchanged). After the handoff drain barrier the worker has written + /// the current pool to disk; this adopts it so the inline (non-deferred) commits + /// above the last checkpoint start from the correct pool. + pub(crate) fn reseed_value_pool(&mut self, db: &ZebraDb) { + self.value_pool = db.finalized_value_pool(); + } + /// Whether a not-yet-flushed block has already written the set-once /// `vct_upgrade_height` marker, so the next block must not write it again. pub(crate) fn vct_upgrade_marker_set(&self) -> bool { diff --git a/zebra-state/src/service/finalized_state/zebra_db.rs b/zebra-state/src/service/finalized_state/zebra_db.rs index e333eb735c1..57de6b6002c 100644 --- a/zebra-state/src/service/finalized_state/zebra_db.rs +++ b/zebra-state/src/service/finalized_state/zebra_db.rs @@ -251,19 +251,27 @@ impl ZebraDb { } /// Whether the per-block committer defers transparent spend resolution (the - /// `utxo_by_out_loc` deletes and the transparent value-pool debit) off the - /// commit critical path, to a batched reconcile at each checkpoint boundary. + /// `utxo_by_out_loc` deletes and the transparent value-pool debit) off the commit + /// critical path, to a batched reconcile on the worker thread, for the block at + /// `height`. /// - /// The auxiliary address index must be off ([`Config::skip_address_index`]), - /// because the deferred path skips the per-block spent-UTXO reads that the + /// Only defers in the checkpoint-trusted range (`height <= max_checkpoint_height`): + /// above the last checkpoint the semantic verifier validates spends against the + /// live UTXO set and value pool, so those must stay current (non-deferred). The + /// auxiliary address index must also be off ([`Config::skip_address_index`]), + /// because the deferred path skips the per-block spent-UTXO reads the /// address-balance update depends on. - /// - /// v1 (prototype) omits the `height <= max_checkpoint_height` check: the - /// benchmark runs entirely inside the checkpoint range, and the handoff drain - /// barrier (so no semantic-verified block reads a mid-reconcile value pool or - /// UTXO set) is a later production stage. - pub(crate) fn defers_transparent_spends(&self) -> bool { - self.config().defer_transparent_reconcile && self.config().skip_address_index() + pub(crate) fn defers_transparent_spends(&self, height: Height) -> bool { + self.config() + .defers_transparent_spends_at(&self.network(), height) + } + + /// Whether the deferred-transparent reconcile is configured for this node (the + /// height-independent lifecycle predicate: worker spawn, drain triggers, the + /// handoff barrier). The per-block decision is the height-bound + /// [`defers_transparent_spends`](Self::defers_transparent_spends). + pub(crate) fn defer_reconcile_configured(&self) -> bool { + self.config().defer_reconcile_configured() } /// Returns the configured database kind for this database. diff --git a/zebra-state/src/service/finalized_state/zebra_db/block.rs b/zebra-state/src/service/finalized_state/zebra_db/block.rs index b2a0d5788d3..2c26b99c961 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block.rs @@ -1135,7 +1135,7 @@ impl ZebraDb { // measures the throughput ceiling of deferring that work off the commit critical // path. It produces an INCORRECT value pool and UTXO set, so it is never a shipped // path — only a measurement of the upper bound. - let defer_spends = self.defers_transparent_spends(); + let defer_spends = self.defers_transparent_spends(finalized.height); // Deferral records into the run-ahead pipeline's reconcile window, so it is // only correct when an overlay is present. Without one there is nowhere to // record the spends and the checkpoint reconcile would never run, silently diff --git a/zebra-state/src/service/finalized_state/zebra_db/chain.rs b/zebra-state/src/service/finalized_state/zebra_db/chain.rs index e3b353e8227..68758ce661b 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/chain.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/chain.rs @@ -418,7 +418,7 @@ impl DiskWriteBatch { // unchanged (it is threaded forward and corrected at the next checkpoint). The // benchmark probe (`bench_skip_transparent_reads`) skips it with no reconcile, // leaving the pool permanently stale (measurement only, never shipped). - if db.defers_transparent_spends() || super::bench_skip_transparent_reads() { + if db.defers_transparent_spends(finalized.height) || super::bench_skip_transparent_reads() { return Ok(value_pool); } diff --git a/zebra-state/src/service/write.rs b/zebra-state/src/service/write.rs index 0ba5ef7293c..f7c6b70fd1e 100644 --- a/zebra-state/src/service/write.rs +++ b/zebra-state/src/service/write.rs @@ -195,6 +195,21 @@ impl ReconcileWorker { } } +impl Drop for ReconcileWorker { + /// Clean-shutdown safety net: if the worker was not already drained at the + /// checkpoint handoff (the common path), close the job channel and join the + /// thread so a clean stop finishes any queued reconcile before the state service + /// is torn down. Idempotent — a no-op once `flush_and_join` has run. Join errors + /// are swallowed here to avoid panicking during unwinding; crash recovery from a + /// mid-window stop is out of scope (the node is treated as re-snapshottable). + fn drop(&mut self) { + drop(self.sender.take()); + if let Some(handle) = self.handle.take() { + let _ = handle.join(); + } + } +} + /// Process the disk-writer thread's flush acknowledgements: for each acked height, /// advance the finalized chain tip, retire the now-durable overlay entries, and run /// the configured stop-height check (which exits the process if matched). @@ -245,7 +260,7 @@ fn drain_finalized_acks( // reads + value-pool recompute overlap continued assembly. v1 fallback (no // worker): reconcile inline. Fatal on error either way: a failed reconcile // would leave the value pool / UTXO set inconsistent. - if finalized_state.db.defers_transparent_spends() { + if finalized_state.db.defer_reconcile_configured() { let interval = finalized_state.db.config().defer_reconcile_interval; let is_boundary = if interval > 0 { height.0 % (interval as u32) == 0 @@ -671,7 +686,7 @@ impl WriteBlockWorkerTask { // in the deferred range, so this stays exclusively owned with no races). let mut reconcile_worker: Option = None; if pipeline_active - && finalized_state.db.defers_transparent_spends() + && finalized_state.db.defer_reconcile_configured() && !finalized_state.db.config().defer_reconcile_inline { // Capacity 1: hand-off blocks the assembler only while the worker is @@ -884,6 +899,40 @@ impl WriteBlockWorkerTask { let prev_note_commitment_trees_for_retry = prev_note_commitment_trees.clone(); let committed_height = ordered_block.0.height; + + // Handoff drain barrier (consensus-critical). Deferral is only valid below + // the last checkpoint (guard 2); above it the semantic verifier validates + // each block's spends against the live UTXO set and value pool, so those + // must be current. Before the first above-checkpoint block commits, flush + // every in-flight block to disk (so the window's spent UTXOs are durable), + // drain the pending reconcile window, and refresh the pipeline's threaded + // value pool from the now-current disk pool for the inline commits that + // follow. The finalized committer normally only sees checkpoint-verified + // blocks, so this rarely fires here — the common handoff is the + // channel-close drain below — but it guards the boundary itself. + if reconcile_worker.is_some() + && committed_height > finalized_state.db.network().checkpoint_list().max_height() + { + if let (Some(pipeline), Some(ack_receiver)) = + (pipeline_state.as_mut(), pipeline_ack_receiver.as_ref()) + { + drain_finalized_acks( + finalized_state, + pipeline, + ack_receiver, + &mut pipeline_in_flight, + chain_tip_sender, + reconcile_worker.as_mut(), + true, + ); + if let Some(mut worker) = reconcile_worker.take() { + worker.send(pipeline.take_reconcile_prefix(committed_height)); + worker.flush_and_join(); + } + pipeline.reseed_value_pool(&finalized_state.db); + } + } + let next_block_took_vct_path = finalized_state.vct_fast_will_apply(committed_height); // Commit the block. When the run-ahead pipeline is active, assemble the @@ -1106,12 +1155,12 @@ impl WriteBlockWorkerTask { ); // Deferred reconcile: the trailing blocks committed past the last - // boundary form a partial window that nothing reconciled. Reconcile it - // now (through the durable tip) so the value pool and UTXO set are - // consistent before the handoff to the non-finalized state. (The - // production handoff drain barrier is a later stage; this keeps the - // prototype correct at the channel close.) - if finalized_state.db.defers_transparent_spends() { + // boundary form a partial window that nothing reconciled. This is the + // common checkpoint→non-finalized handoff: drain the window (through the + // durable tip) so the value pool and UTXO set are current before the + // semantic verifier reads them. (The in-loop barrier above guards the + // rarer case of an above-checkpoint block arriving on this channel.) + if finalized_state.db.defer_reconcile_configured() { let tip_height = finalized_state.db.tip().map(|(height, _)| height); if let Some(tip_height) = tip_height { if let Some(worker) = reconcile_worker.as_mut() { From b8abb5f2d2b282c111218632b5f937bd949321f4 Mon Sep 17 00:00:00 2001 From: Roman Date: Wed, 1 Jul 2026 00:33:50 +0000 Subject: [PATCH 12/17] perf(state): parallelize the deferred-transparent reconcile worker The v2 reconcile worker was single-threaded while the committer's other cores sat idle, so it capped the deferral win at +73% sandblast. Parallelize its two heavy per-interval passes on the rayon pool: - spent-UTXO resolution (output_location + utxo_by_location per outpoint) is CPU-bound page-cache reads, embarrassingly parallel over the window's deduped outpoints; - the per-block value-pool change (chain_value_pool_change re-folds every tx's value balance) is independent per block, so compute the (delta, block_size) per block in parallel, then apply the deltas in a sequential running-sum for the per-height BlockInfo + tip pool (chaining stays ordered, so the arithmetic is byte-identical). Uses the global rayon pool, not the committer's treestate pool, so it doesn't steal from the committer's critical path (which is light in the deferred range). Byte-match preserved (value pool + UTXO set + per-height BlockInfo identical to a non-deferred run). Offline replay bench, mainnet sandblast range: sandblast throughput +73% -> +114% vs baseline (210 -> 284 blk/s), ~60% of the ceiling headroom. Remaining serial cost is the per-interval delete-batch build + write (a follow-up). --- .../src/service/finalized_state/pipeline.rs | 62 ++++++++------ .../service/finalized_state/zebra_db/chain.rs | 84 +++++++++++-------- 2 files changed, 87 insertions(+), 59 deletions(-) diff --git a/zebra-state/src/service/finalized_state/pipeline.rs b/zebra-state/src/service/finalized_state/pipeline.rs index d079245db06..c8daf5d271c 100644 --- a/zebra-state/src/service/finalized_state/pipeline.rs +++ b/zebra-state/src/service/finalized_state/pipeline.rs @@ -427,10 +427,9 @@ pub(crate) fn reconcile_window( return Ok(start_value_pool); } - // Batch-resolve every spent outpoint from disk. Sort by txid (so repeated txids - // dedup and the `output_location`/`tx_loc_by_hash` reads are grouped), then sort - // the resolved locations so the `utxo_by_out_loc` value reads are sequential-ish - // rather than random per block. + // Collect the window's spent outpoints, deduped (each UTXO is spent once, so the + // dedup is defensive). Sorting by txid also groups repeated txids for the + // `output_location`/`tx_loc_by_hash` reads. let mut outpoints: Vec = records .iter() .flat_map(|r| r.spent.iter().copied()) @@ -438,27 +437,40 @@ pub(crate) fn reconcile_window( outpoints.sort_unstable_by_key(|outpoint| (outpoint.hash.0, outpoint.index)); outpoints.dedup(); - let mut located: Vec<(transparent::OutPoint, OutputLocation)> = - Vec::with_capacity(outpoints.len()); - for outpoint in outpoints { - let out_loc = db.output_location(&outpoint).ok_or_else(|| { - format!("deferred spent outpoint missing from state at reconcile: {outpoint:?}") - })?; - located.push((outpoint, out_loc)); - } - located.sort_unstable_by_key(|(_outpoint, out_loc)| *out_loc); - - let mut resolved: HashMap = - HashMap::with_capacity(located.len()); - for (outpoint, out_loc) in located { - let utxo = db - .utxo_by_location(out_loc) - .ok_or_else(|| { - format!("deferred spent UTXO missing from state at reconcile: {outpoint:?}") - })? - .utxo; - resolved.insert(outpoint, (out_loc, utxo)); - } + // Batch-resolve every spent outpoint from disk in parallel. Each resolve is + // `output_location` (txid → location, `tx_loc_by_hash`) then `utxo_by_location` + // (location → value, `utxo_by_out_loc`); both are CPU-bound (page-cache-resident + // decode + index traversal) and independent across outpoints, and RocksDB reads + // are thread-safe through the cloneable `DiskDb`. This runs off the assembler + // thread on the worker, spreading the resolution across otherwise-idle cores. + use rayon::prelude::*; + let resolved: HashMap = + outpoints + .par_iter() + .map( + |outpoint| -> Result< + (transparent::OutPoint, (OutputLocation, transparent::Utxo)), + BoxError, + > { + let out_loc = db.output_location(outpoint).ok_or_else(|| -> BoxError { + format!( + "deferred spent outpoint missing from state at reconcile: {outpoint:?}" + ) + .into() + })?; + let utxo = db + .utxo_by_location(out_loc) + .ok_or_else(|| -> BoxError { + format!( + "deferred spent UTXO missing from state at reconcile: {outpoint:?}" + ) + .into() + })? + .utxo; + Ok((*outpoint, (out_loc, utxo))) + }, + ) + .collect::, BoxError>>()?; db.commit_checkpoint_reconcile(network, start_value_pool, records, &resolved) } diff --git a/zebra-state/src/service/finalized_state/zebra_db/chain.rs b/zebra-state/src/service/finalized_state/zebra_db/chain.rs index 68758ce661b..5dde4ce35db 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/chain.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/chain.rs @@ -17,7 +17,7 @@ use std::{ }; use zebra_chain::{ - amount::NonNegative, + amount::{NegativeAllowed, NonNegative}, block::Height, block_info::BlockInfo, history_tree::HistoryTree, @@ -290,42 +290,58 @@ impl ZebraDb { true, ); - // Recompute the value pool sequentially in block order, writing each block's - // running pool into `BlockInfo` (exactly as the inline per-block path does), - // and the final pool into the single-key tip CF once at the end. + // Compute each block's independent value-pool delta and serialized size in + // parallel — `chain_value_pool_change` re-folds every transaction's value + // balance and the size sums every transaction's serialized length, so this is + // the CPU-heavy pass, and it is independent across the window's blocks. Then + // apply the deltas in block order sequentially (cheap integer adds) to get the + // per-height running pool for `BlockInfo`. Splitting the parallel per-block + // work from the ordered running-sum keeps the result byte-identical to the + // inline per-block path (same deltas, same application order). + use rayon::prelude::*; + let per_block: Vec<(ValueBalance, usize)> = records + .par_iter() + .map( + |record| -> Result<(ValueBalance, usize), BoxError> { + let spent_by_block: HashMap = record + .spent + .iter() + .map(|outpoint| { + let (_out_loc, utxo) = resolved + .get(outpoint) + .expect("every deferred spend was resolved above"); + (*outpoint, utxo.clone()) + }) + .collect(); + + let delta = record + .block + .chain_value_pool_change(&spent_by_block, record.deferred_pool_change)?; + + // Block size, summed per-transaction (byte-identical to serializing the + // whole block), as in `prepare_chain_value_pools_batch`. + let transactions = &record.block.transactions; + let transactions_size: usize = transactions + .iter() + .map(|transaction| transaction.zcash_serialized_size()) + .sum(); + let tx_count_size = CompactSizeMessage::try_from(transactions.len()) + .expect("block must have a valid transaction count") + .zcash_serialized_size(); + let block_size = record.block.header.zcash_serialized_size() + + tx_count_size + + transactions_size; + + Ok((delta, block_size)) + }, + ) + .collect::, BoxError>>()?; + let mut value_pool = start_value_pool; - for record in records { - let spent_by_block: HashMap = record - .spent - .iter() - .map(|outpoint| { - let (_out_loc, utxo) = resolved - .get(outpoint) - .expect("every deferred spend was resolved above"); - (*outpoint, utxo.clone()) - }) - .collect(); - - let block_value_pool_change = record - .block - .chain_value_pool_change(&spent_by_block, record.deferred_pool_change)?; - value_pool = value_pool.add_chain_value_pool_change(block_value_pool_change)?; + for (record, (delta, block_size)) in records.iter().zip(per_block) { + value_pool = value_pool.add_chain_value_pool_change(delta)?; value_pool_metrics(&value_pool); - // Block size, summed per-transaction (byte-identical to serializing the - // whole block), as in `prepare_chain_value_pools_batch`. - let block_size = { - let transactions = &record.block.transactions; - let transactions_size: usize = transactions - .iter() - .map(|transaction| transaction.zcash_serialized_size()) - .sum(); - let tx_count_size = CompactSizeMessage::try_from(transactions.len()) - .expect("block must have a valid transaction count") - .zcash_serialized_size(); - record.block.header.zcash_serialized_size() + tx_count_size + transactions_size - }; - let _ = self .block_info_cf() .with_batch_for_writing(&mut batch) From bf0a1cde1d7d8ac8608b78d405ec54b2403e1397 Mon Sep 17 00:00:00 2001 From: roman Date: Tue, 30 Jun 2026 18:40:53 -0600 Subject: [PATCH 13/17] perf(block-sync): make work-queue reserved_bytes O(1) and GC the floor incrementally MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the earlier throttle band-aid with the real fix. When headers race far ahead of the body tip, the sequencer's WorkQueue holds the whole lag (100k+ pending heights) in mutex-guarded BTreeMaps, and two O(n) operations ran under that lock on the hot path: - `reserved_bytes()` re-summed reserved request bytes across pending + in_flight on every `publish_view` (i.e. every body/control event); and - `advance_floor()` ran a full-map `retain` to drop committed heights on every floor advance. As the backlog grew both went quadratic, saturating the single sequencer task and serializing the work-queue lock so commit and download stalled together (mid-stall dumps showed the sequencer in `retain`/`reserved_bytes` with peers blocked on the lock). Fixes: - `WorkQueueInner` maintains a `reserved_bytes` running counter, updated at every ledger transition; `reserved_bytes()` returns it in O(1). The per-event budget audit cross-checks it against the independently-maintained ByteBudget, and a new unit test asserts the counter never drifts from a full scan across every transition. - `advance_floor`/`reset_above` pop only the committed prefix/suffix (O(removed · log n)) instead of a full-map scan. Reverts the `BUDGET_AUDIT_INTERVAL` throttle from the previous commit: with `reserved_bytes` O(1), the audit runs every event again with no work-queue scan. --- CHANGELOG.md | 25 ++-- CHANGELOG_PARAMS.md | 1 - .../src/zakura/block_sync/sequencer_task.rs | 51 +++----- zebra-network/src/zakura/block_sync/tests.rs | 96 +++++++++++++++ .../src/zakura/block_sync/work_queue.rs | 109 ++++++++++++++---- 5 files changed, 214 insertions(+), 68 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ad2a0129560..af61554997e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,17 +9,20 @@ and this project adheres to [Semantic Versioning](https://semver.org). ### Performance -- Fix a quadratic slowdown in the Zakura block-sync sequencer that stalled the - commit pipeline for tens of seconds during checkpoint sync. `publish_view` ran - on every body and control event and each time re-derived the reserved-byte - total with an O(pending + in_flight) scan of the work queue, purely to feed a - budget drift-audit metric. As the apply/commit backlog grew to thousands of - blocks, that per-event scan became quadratic and saturated the single sequencer - task, starving the commit-compute threads (observed as ~18–30s body-commit - freezes around the peak-backlog heights). The published view already uses the - budget's O(1) running counter, so the audit's full scan is now sampled at most - once per `BUDGET_AUDIT_INTERVAL` (1s) instead of on every event, keeping the - drift check while removing it from the hot path. +- Fix work-queue scans that stalled the Zakura block-sync commit pipeline for + tens of seconds during checkpoint sync. When headers race far ahead of the body + tip, the sequencer's `WorkQueue` holds the entire lag (100k+ pending heights) in + mutex-guarded `BTreeMap`s, and two O(n) operations ran under that lock on the + hot path: `reserved_bytes()` (a full scan re-summing reserved request bytes on + every `publish_view`, i.e. every body/control event) and `advance_floor()` (a + full-map `retain` garbage-collecting committed heights on every floor advance). + As the backlog grew these became quadratic, saturating the single sequencer task + and serializing the work-queue lock so both commit *and* download stalled + (observed as ~12–30s body-commit freezes with peers blocked on the lock). Both + are now bounded: `reserved_bytes` is an O(1) incrementally-maintained counter + (cross-checked against the independent byte budget by the existing audit), and + `advance_floor`/`reset_above` pop only the committed prefix/suffix + (O(removed · log n)) instead of scanning the whole map. - Compute the v5 ZIP-244 txid and authorizing-data digest natively. Both previously routed through `Transaction::to_librustzcash`, which re-serializes and reparses the whole transaction — decompressing every Jubjub and Pallas diff --git a/CHANGELOG_PARAMS.md b/CHANGELOG_PARAMS.md index d98d917ff76..c7695aa028a 100644 --- a/CHANGELOG_PARAMS.md +++ b/CHANGELOG_PARAMS.md @@ -28,6 +28,5 @@ Keep entries **newest-first**. Each row records: | Parameter | Location | Old → New | PR | Why | | --- | --- | --- | --- | --- | -| `BUDGET_AUDIT_INTERVAL` | `zebra-network/src/zakura/block_sync/sequencer_task.rs` | _(new)_ → `1s` | _(this PR)_ | Throttle the sequencer's byte-budget drift audit to at most once per interval. The audit re-derives the reserved total via an O(pending + in_flight) work-queue scan; running it on every body/control event (as `publish_view` did) is quadratic in the backlog and stalls the commit pipeline. The published view uses the budget's O(1) running counter, so the scan is only a drift metric. | | `Config::finalized_block_pipeline_depth` | `zebra-state/src/config.rs` | _(new)_ → `0` | _(this PR)_ | Run-ahead finalized-commit pipeline depth (blocks the assembler may build ahead of the durable disk write). Defaults to `0` = synchronous (original behavior); `> 0` overlaps the next block's assembly with the current block's flush, bounded so the in-memory overlay stays small. | | `OUTBOUND_WINDOW_FLOOR_TIMEOUTS_BEFORE_DISCONNECT` | `zebra-network/src/zakura/block_sync/state.rs` | `3` → `2 * OUTBOUND_WINDOW_REDUCTION_EPOCH_TIMEOUTS` (`32`) | [#303](https://github.com/valargroup/zebra/pull/303) | Tolerate two full reduction epochs (~256s at the 8s request timeout) of floor-pinned timeouts before disconnecting a block-sync peer, instead of ~24s, so briefly-congested peers are not churned. Any successful response resets the streak. | diff --git a/zebra-network/src/zakura/block_sync/sequencer_task.rs b/zebra-network/src/zakura/block_sync/sequencer_task.rs index 0185859426e..9bb71d2cdf1 100644 --- a/zebra-network/src/zakura/block_sync/sequencer_task.rs +++ b/zebra-network/src/zakura/block_sync/sequencer_task.rs @@ -196,17 +196,6 @@ pub(super) fn initial_view(frontiers: BlockSyncFrontiers) -> SequencerView { } } -/// How often [`SequencerTask::publish_view`] re-audits the byte budget against a -/// full work-queue scan. -/// -/// The audit only feeds the `sync.block.budget.audit_drift` metric — the budget -/// already keeps an O(1) running counter that the published view uses directly. -/// `publish_view` runs on every body and control event, and the audit's scan is -/// O(pending + in_flight), so scanning on each call is quadratic in the backlog -/// and stalls the commit pipeline once it grows. Sampling on an interval keeps -/// the drift check while bounding its cost to one scan per interval. -const BUDGET_AUDIT_INTERVAL: Duration = Duration::from_secs(1); - /// The serial commit-pipeline task. Owns the `Sequencer` (moved out of state), a /// `ByteBudget` clone, an `Arc` clone, an action sender clone, and the /// committed throughput meter. Releases bytes directly and emits `SubmitBlock` / @@ -217,9 +206,6 @@ pub(super) struct SequencerTask { work: Arc, actions: mpsc::Sender, committed_throughput: ThroughputMeter, - /// Next time [`Self::publish_view`] may run the O(n) budget audit; see - /// [`BUDGET_AUDIT_INTERVAL`]. - next_budget_audit: Instant, /// Tracks the finalized height so the published view carries it forward; the /// reactor folds it into its `finalized_height` mirror with a `max`. finalized_height: block::Height, @@ -256,7 +242,6 @@ impl SequencerTask { work, actions, committed_throughput, - next_budget_audit: Instant::now(), finalized_height: frontiers.finalized_height, verified_block_hash: frontiers.verified_block_hash, reset_epoch: 0, @@ -747,29 +732,23 @@ impl SequencerTask { } fn publish_view(&mut self) { - let now = Instant::now(); - self.committed_throughput.sample(now); + self.committed_throughput.sample(Instant::now()); let reorder_buffered_bytes = self.sequencer.reorder_buffered_bytes(); let applying_buffered_bytes = self.sequencer.applying_buffered_bytes(); - // The budget audit recomputes the reserved total by scanning the whole - // work queue (O(pending + in_flight)). Since `publish_view` runs on every - // body/control event, keep that scan off the hot path and sample it on an - // interval instead (see `BUDGET_AUDIT_INTERVAL`); the published view below - // does not depend on it. - if now >= self.next_budget_audit { - let body_input_bytes = self - .body_input_bytes - .load(std::sync::atomic::Ordering::Relaxed); - let expected_budget = self - .work - .reserved_bytes() - .saturating_add(reorder_buffered_bytes) - .saturating_add(applying_buffered_bytes) - .saturating_add(body_input_bytes); - self.budget - .audit(expected_budget, "block-sync sequencer view"); - self.next_budget_audit = now + BUDGET_AUDIT_INTERVAL; - } + let body_input_bytes = self + .body_input_bytes + .load(std::sync::atomic::Ordering::Relaxed); + // Cross-layer drift check: the independently-maintained `ByteBudget` total + // must equal the sum of the component counters. `work.reserved_bytes()` is + // now an O(1) counter, so this runs on every event without a work-queue scan. + let expected_budget = self + .work + .reserved_bytes() + .saturating_add(reorder_buffered_bytes) + .saturating_add(applying_buffered_bytes) + .saturating_add(body_input_bytes); + self.budget + .audit(expected_budget, "block-sync sequencer view"); let _ = self.view_tx.send_replace(SequencerView { verified_tip: self.sequencer.verified_tip(), verified_hash: self.verified_block_hash, diff --git a/zebra-network/src/zakura/block_sync/tests.rs b/zebra-network/src/zakura/block_sync/tests.rs index d0213c1f849..cd1f8068cd7 100644 --- a/zebra-network/src/zakura/block_sync/tests.rs +++ b/zebra-network/src/zakura/block_sync/tests.rs @@ -1447,6 +1447,102 @@ fn work_queue_force_cancel_and_owner_timeout_release_once() { assert!(queue.pending_contains(block::Height(1))); } +#[test] +fn work_queue_reserved_bytes_counter_matches_scan_across_transitions() { + // Exercise every ledger transition and assert the O(1) `reserved_bytes` counter + // never drifts from the O(n) ground-truth scan. + let queue = work_queue_with( + 0, + (1..=6).map(|h| needed(h, BlockSizeEstimate::Advertised(100))), + ); + let check = |label: &str| { + assert_eq!( + queue.reserved_bytes(), + queue.reserved_bytes_scanned(), + "reserved_bytes counter drifted from scan after {label}" + ); + }; + // Nothing reserved yet: all pending items are `Released`. + assert_eq!(queue.reserved_bytes(), 0); + check("seed"); + + // take + mark_reserved: Released -> Reserved. + let taken = queue.take_in_range(block::Height(1), block::Height(6), 6); + assert_eq!(taken.len(), 6); + assert_eq!(queue.mark_reserved((1..=6).map(block::Height)), 600); + assert_eq!(queue.reserved_bytes(), 600); + check("mark_reserved"); + + // settle: Reserved -> Held drops the reservation for height 1. + queue + .settle_active_reserved_height(block::Height(1), 80) + .expect("height 1 is reserved"); + assert_eq!(queue.reserved_bytes(), 500); + check("settle"); + + // mark_held_direct: Reserved -> Held drops height 2's reservation. + queue.mark_held_direct(block::Height(2), 90); + assert_eq!(queue.reserved_bytes(), 400); + check("mark_held_direct"); + + // release_heights: Reserved -> Released for height 3. + queue.release_heights([block::Height(3)]); + assert_eq!(queue.reserved_bytes(), 300); + check("release_heights"); + + // release_reserved_and_return_items: only the still-reserved height 4 releases. + let released = queue.release_reserved_and_return_items([block::Height(4)]); + assert_eq!(released, 100); + assert_eq!(queue.reserved_bytes(), 200); + check("release_reserved_and_return_items"); + + // advance_floor drops the committed `<= floor` prefix. Heights 1 (Held) and 2 + // (Held) contribute no reservation; height 3 is already Released. Only heights + // 5 and 6 remain reserved above the new floor. + let released = queue.advance_floor(block::Height(4)); + assert_eq!(released, 0, "heights <= 4 owned no live reservation"); + assert!(!queue.pending_contains(block::Height(1))); + assert!(!queue.in_flight_contains(block::Height(2))); + assert_eq!(queue.reserved_bytes(), 200); + check("advance_floor"); + + // reset_above drops the `> floor` suffix (heights 5 and 6), releasing their + // reservations and zeroing the counter. + let released = queue.reset_above(block::Height(4)); + assert_eq!(released, 200); + assert_eq!(queue.reserved_bytes(), 0); + assert!(!queue.in_flight_contains(block::Height(5))); + assert!(!queue.in_flight_contains(block::Height(6))); + check("reset_above"); +} + +#[test] +fn work_queue_advance_floor_drops_only_committed_prefix() { + let queue = work_queue_with( + 0, + (1..=10).map(|h| needed(h, BlockSizeEstimate::Advertised(100))), + ); + // Reserve a contiguous run so we can see the reservation accounting on GC. + let _ = queue.take_in_range(block::Height(1), block::Height(10), 10); + assert_eq!(queue.mark_reserved((1..=10).map(block::Height)), 1000); + + // advance_floor to 4: drops heights 1..=4 (still reserved -> released), keeps 5..=10. + let released = queue.advance_floor(block::Height(4)); + assert_eq!(released, 400, "reserved bytes for the dropped 1..=4 prefix"); + for h in 1..=4 { + assert!(!queue.in_flight_contains(block::Height(h))); + } + for h in 5..=10 { + assert!(queue.in_flight_contains(block::Height(h))); + } + assert_eq!(queue.reserved_bytes(), 600); + assert_eq!(queue.reserved_bytes(), queue.reserved_bytes_scanned()); + + // A lower floor is a no-op (floor only advances). + assert_eq!(queue.advance_floor(block::Height(2)), 0); + assert_eq!(queue.reserved_bytes(), 600); +} + #[test] fn watchdog_after_held_settle_releases_once() { let queue = work_queue_with(0, [needed(1, BlockSizeEstimate::Advertised(100))]); diff --git a/zebra-network/src/zakura/block_sync/work_queue.rs b/zebra-network/src/zakura/block_sync/work_queue.rs index e4058d1f824..b79616d5f7a 100644 --- a/zebra-network/src/zakura/block_sync/work_queue.rs +++ b/zebra-network/src/zakura/block_sync/work_queue.rs @@ -56,6 +56,12 @@ struct WorkQueueInner { floor: block::Height, /// Floor clamp for size estimates (overridable for tests). floor_estimate_bytes: u64, + /// Running sum of `reserved_charge()` across every `pending` + `in_flight` + /// item, maintained incrementally at each ledger transition so + /// [`WorkQueue::reserved_bytes`] is O(1) instead of an O(pending + in_flight) + /// scan on the sequencer's hot path. `debug_assert`s and the throttled budget + /// audit cross-check it against the map contents. + reserved_bytes: u64, } impl WorkQueueInner { @@ -91,6 +97,7 @@ impl WorkQueue { in_flight: std::collections::BTreeMap::new(), floor, floor_estimate_bytes: DEFAULT_BS_SIZE_FLOOR_BYTES, + reserved_bytes: 0, }), available: Notify::new(), } @@ -300,6 +307,9 @@ impl WorkQueue { item.budget = BlockBudgetLedger::reserved(item.estimated_bytes); marked = marked.saturating_add(item.estimated_bytes); } + // Released (0) -> Reserved(estimate): the reserved total grows by exactly + // the bytes just marked. + inner.reserved_bytes = inner.reserved_bytes.saturating_add(marked); marked } @@ -314,10 +324,16 @@ impl WorkQueue { actual: u64, ) -> Option { let mut inner = self.lock(); - let item = inner.in_flight.get_mut(&height)?; - item.budget - .is_reserved() - .then(|| item.budget.settle(actual)) + let (reserved_before, delta) = { + let item = inner.in_flight.get_mut(&height)?; + if !item.budget.is_reserved() { + return None; + } + // Reserved(reserved) -> Held(actual): the reserved charge drops to 0. + (item.budget.reserved_charge(), item.budget.settle(actual)) + }; + inner.reserved_bytes = inner.reserved_bytes.saturating_sub(reserved_before); + Some(delta) } /// Mark a height as directly held after the caller admitted `actual` bytes. @@ -327,14 +343,19 @@ impl WorkQueue { pub(super) fn mark_held_direct(&self, height: block::Height, actual: u64) -> u64 { let mut inner = self.lock(); if let Some(item) = inner.in_flight.get_mut(&height) { + // X -> Held(actual): any reserved charge this item still owned is gone. + let reserved_before = item.budget.reserved_charge(); let previous_charge = item.budget.release(); item.budget = BlockBudgetLedger::Held(actual); + inner.reserved_bytes = inner.reserved_bytes.saturating_sub(reserved_before); return previous_charge; } if let Some(mut item) = inner.pending.remove(&height) { + let reserved_before = item.budget.reserved_charge(); let previous_charge = item.budget.release(); item.budget = BlockBudgetLedger::Held(actual); inner.in_flight.insert(height, item); + inner.reserved_bytes = inner.reserved_bytes.saturating_sub(reserved_before); return previous_charge; } 0 @@ -343,14 +364,18 @@ impl WorkQueue { /// Release any live charges for `heights`, exactly once. pub(super) fn release_heights(&self, heights: impl IntoIterator) -> u64 { let mut released = 0u64; + let mut reserved_removed = 0u64; let mut inner = self.lock(); for height in heights { if let Some(item) = inner.in_flight.get_mut(&height) { + reserved_removed = reserved_removed.saturating_add(item.budget.reserved_charge()); released = released.saturating_add(item.budget.release()); } else if let Some(item) = inner.pending.get_mut(&height) { + reserved_removed = reserved_removed.saturating_add(item.budget.reserved_charge()); released = released.saturating_add(item.budget.release()); } } + inner.reserved_bytes = inner.reserved_bytes.saturating_sub(reserved_removed); released } @@ -361,15 +386,19 @@ impl WorkQueue { ) -> u64 { let mut moved = false; let mut released = 0u64; + let mut reserved_removed = 0u64; { let mut inner = self.lock(); for height in heights { if let Some(mut item) = inner.in_flight.remove(&height) { + reserved_removed = + reserved_removed.saturating_add(item.budget.reserved_charge()); released = released.saturating_add(item.budget.release()); inner.pending.insert(height, item); moved = true; } } + inner.reserved_bytes = inner.reserved_bytes.saturating_sub(reserved_removed); } if moved { self.available.notify_waiters(); @@ -401,10 +430,13 @@ impl WorkQueue { .in_flight .remove(&height) .expect("reserved item exists because it was just checked"); + // Only reserved items reach here, so the released bytes are exactly + // the reserved charge leaving the queue. released = released.saturating_add(item.budget.release()); inner.pending.insert(height, item); moved = true; } + inner.reserved_bytes = inner.reserved_bytes.saturating_sub(released); } if moved { self.available.notify_waiters(); @@ -423,15 +455,32 @@ impl WorkQueue { let mut inner = self.lock(); inner.floor = inner.floor.max(floor); let floor = inner.floor; + // Pop only the committed `<= floor` prefix from each map. `pending` can hold + // the entire header-ahead lag (100k+ heights), so a `retain` over the whole + // map on every floor advance is O(total) and serializes the work-queue lock; + // popping the prefix is O(removed · log n). let mut released = 0u64; - for item in inner.pending.range_mut(..=floor).map(|(_, item)| item) { + while let Some((&height, _)) = inner.pending.first_key_value() { + if height > floor { + break; + } + let (_, mut item) = inner + .pending + .pop_first() + .expect("first_key_value returned Some"); released = released.saturating_add(item.budget.release_reserved()); } - for item in inner.in_flight.range_mut(..=floor).map(|(_, item)| item) { + while let Some((&height, _)) = inner.in_flight.first_key_value() { + if height > floor { + break; + } + let (_, mut item) = inner + .in_flight + .pop_first() + .expect("first_key_value returned Some"); released = released.saturating_add(item.budget.release_reserved()); } - inner.pending.retain(|height, _| *height > floor); - inner.in_flight.retain(|height, _| *height > floor); + inner.reserved_bytes = inner.reserved_bytes.saturating_sub(released); released } @@ -444,23 +493,30 @@ impl WorkQueue { pub(super) fn reset_above(&self, floor: block::Height) -> u64 { let mut inner = self.lock(); inner.floor = floor; + // Pop only the `> floor` suffix from each map (O(removed · log n)); see the + // note in `advance_floor` on why a full-map `retain` is too expensive here. let mut released = 0u64; - for item in inner - .pending - .range_mut((std::ops::Bound::Excluded(floor), std::ops::Bound::Unbounded)) - .map(|(_, item)| item) - { + while let Some((&height, _)) = inner.pending.last_key_value() { + if height <= floor { + break; + } + let (_, mut item) = inner + .pending + .pop_last() + .expect("last_key_value returned Some"); released = released.saturating_add(item.budget.release_reserved()); } - for item in inner - .in_flight - .range_mut((std::ops::Bound::Excluded(floor), std::ops::Bound::Unbounded)) - .map(|(_, item)| item) - { + while let Some((&height, _)) = inner.in_flight.last_key_value() { + if height <= floor { + break; + } + let (_, mut item) = inner + .in_flight + .pop_last() + .expect("last_key_value returned Some"); released = released.saturating_add(item.budget.release_reserved()); } - inner.pending.retain(|height, _| *height <= floor); - inner.in_flight.retain(|height, _| *height <= floor); + inner.reserved_bytes = inner.reserved_bytes.saturating_sub(released); released } @@ -495,7 +551,20 @@ impl WorkQueue { }) } + /// Sum of reserved request-estimate bytes across `pending` + `in_flight`. + /// + /// O(1): returns the incrementally-maintained counter (see + /// [`WorkQueueInner::reserved_bytes`]). This is on the sequencer's hot path via + /// `publish_view`, so it must not scan the maps. [`reserved_bytes_scanned`] is + /// the O(n) ground-truth recomputation used by the audit / tests to catch drift. pub(super) fn reserved_bytes(&self) -> u64 { + self.lock().reserved_bytes + } + + /// Ground-truth O(pending + in_flight) recomputation of [`reserved_bytes`], + /// used by tests to assert the maintained counter never drifts. + #[cfg(test)] + pub(super) fn reserved_bytes_scanned(&self) -> u64 { let inner = self.lock(); inner .pending From 10caf8ffbcbbbef64fa691548fe280a7008b571b Mon Sep 17 00:00:00 2001 From: roman Date: Tue, 30 Jun 2026 18:51:55 -0600 Subject: [PATCH 14/17] perf(block-sync): make applying-map view counters O(1) `publish_view` builds the `SequencerView` on every body/control event, and four of its fields each scanned the whole `applying` map (the apply backlog, thousands of entries under stall): `applying_buffered_bytes`, `submitted_applying_bytes`, `submitted_applying_count`, and `unsubmitted_applying_count`. Same O(n)-per-event anti-pattern as the work-queue scans just fixed. Maintain `applying_buffered_bytes`, `submitted_applying_count`, and `submitted_applying_bytes` incrementally on the Sequencer at every applying insert/remove and submit-flag flip (`bytes` is immutable after insert); derive `unsubmitted_applying_count` as `applying.len() - submitted_applying_count`. All four accessors are now O(1). A new unit test asserts the counters never drift from a full scan across insert/submit/unsubmit/remove/commit-release/reset. --- CHANGELOG.md | 6 +- .../src/zakura/block_sync/sequencer.rs | 99 +++++++++++++++---- zebra-network/src/zakura/block_sync/tests.rs | 87 ++++++++++++++++ 3 files changed, 173 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index af61554997e..eb457ee9693 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,7 +22,11 @@ and this project adheres to [Semantic Versioning](https://semver.org). are now bounded: `reserved_bytes` is an O(1) incrementally-maintained counter (cross-checked against the independent byte budget by the existing audit), and `advance_floor`/`reset_above` pop only the committed prefix/suffix - (O(removed · log n)) instead of scanning the whole map. + (O(removed · log n)) instead of scanning the whole map. The `publish_view` + scans over the `applying` map (`applying_buffered_bytes`, + `submitted_applying_count`/`_bytes`, and the derived `unsubmitted_applying_count`) + are likewise now O(1) incrementally-maintained counters instead of a scan of the + apply backlog on every event. - Compute the v5 ZIP-244 txid and authorizing-data digest natively. Both previously routed through `Transaction::to_librustzcash`, which re-serializes and reparses the whole transaction — decompressing every Jubjub and Pallas diff --git a/zebra-network/src/zakura/block_sync/sequencer.rs b/zebra-network/src/zakura/block_sync/sequencer.rs index cf2f1e22ada..e510ff25a0d 100644 --- a/zebra-network/src/zakura/block_sync/sequencer.rs +++ b/zebra-network/src/zakura/block_sync/sequencer.rs @@ -72,6 +72,16 @@ pub(super) struct Sequencer { submitted_applies: BTreeMap>, next_apply_token: BlockApplyToken, + /// Running totals over `applying`, maintained incrementally at every + /// insert/remove/submit-flag change so `publish_view` reads them in O(1) + /// instead of scanning `applying` (which holds the whole apply backlog) on + /// every body/control event. `unsubmitted` is derived as + /// `applying.len() - submitted_applying_count`. Tests assert these never drift + /// from a full scan. + applying_buffered_bytes: u64, + submitted_applying_count: usize, + submitted_applying_bytes: u64, + // The highest block height whose body has already been accepted into the contiguous // download-apply pipeline. body_download_floor: block::Height, @@ -86,6 +96,9 @@ impl Sequencer { applying: BTreeMap::new(), submitted_applies: BTreeMap::new(), next_apply_token: 1, + applying_buffered_bytes: 0, + submitted_applying_count: 0, + submitted_applying_bytes: 0, body_download_floor: verified_block_tip, verified_block_tip, submitted_apply_limit, @@ -126,6 +139,13 @@ impl Sequencer { } pub(super) fn applying_buffered_bytes(&self) -> u64 { + self.applying_buffered_bytes + } + + /// Ground-truth recomputation of [`applying_buffered_bytes`], used by tests to + /// assert the maintained counter never drifts. + #[cfg(test)] + pub(super) fn applying_buffered_bytes_scanned(&self) -> u64 { self.applying .values() .map(|applying| applying.bytes) @@ -142,27 +162,38 @@ impl Sequencer { } pub(super) fn unsubmitted_applying_count(&self) -> usize { + // Derived: every applying body is either submitted or not. self.applying - .values() - .filter(|applying| !applying.submitted) - .count() + .len() + .saturating_sub(self.submitted_applying_count) } pub(super) fn submitted_applying_bytes(&self) -> u64 { - self.applying - .values() - .filter_map(|applying| applying.submitted.then_some(applying.bytes)) - .fold(0u64, u64::saturating_add) + self.submitted_applying_bytes } /// Number of `applying` bodies already submitted to the verifier. pub(super) fn submitted_applying_count(&self) -> usize { + self.submitted_applying_count + } + + /// Ground-truth recomputations of the submitted-apply counters, for tests. + #[cfg(test)] + pub(super) fn submitted_applying_count_scanned(&self) -> usize { self.applying .values() .filter(|applying| applying.submitted) .count() } + #[cfg(test)] + pub(super) fn submitted_applying_bytes_scanned(&self) -> u64 { + self.applying + .values() + .filter_map(|applying| applying.submitted.then_some(applying.bytes)) + .fold(0u64, u64::saturating_add) + } + pub(super) fn has_submitted_apply(&self, height: block::Height, hash: block::Hash) -> bool { self.submitted_applies .get(&height) @@ -296,6 +327,8 @@ impl Sequencer { source_peer, }, ); + // New bodies enter `applying` unsubmitted, so only the total grows. + self.applying_buffered_bytes = self.applying_buffered_bytes.saturating_add(bytes); } covered } @@ -327,12 +360,20 @@ impl Sequencer { .get(&height) .map(|applying| applying.block.clone())?; let token = self.next_apply_token(); - let applying = self.applying.get_mut(&height)?; - applying.token = token; - applying.submitted = true; + let (hash, bytes, was_submitted) = { + let applying = self.applying.get_mut(&height)?; + applying.token = token; + let was_submitted = applying.submitted; + applying.submitted = true; + (applying.hash, applying.bytes, was_submitted) + }; + if !was_submitted { + self.submitted_applying_count = self.submitted_applying_count.saturating_add(1); + self.submitted_applying_bytes = self.submitted_applying_bytes.saturating_add(bytes); + } Some(SubmitItem { height, - hash: applying.hash, + hash, token, block, }) @@ -341,11 +382,23 @@ impl Sequencer { /// Roll back a submit whose dispatch failed (only if the token still matches, /// so a stale rollback cannot clobber a newer submission). pub(super) fn unsubmit(&mut self, height: block::Height, token: BlockApplyToken) { - if let Some(applying) = self.applying.get_mut(&height) { - if applying.token == token { - applying.token = 0; - applying.submitted = false; + let unsubmitted_bytes = { + let Some(applying) = self.applying.get_mut(&height) else { + return; + }; + if applying.token != token { + return; } + // Only a currently-submitted body affects the submitted counters; if the + // matched token was already rolled back, just clear it. + let was_submitted = applying.submitted; + applying.token = 0; + applying.submitted = false; + was_submitted.then_some(applying.bytes) + }; + if let Some(bytes) = unsubmitted_bytes { + self.submitted_applying_count = self.submitted_applying_count.saturating_sub(1); + self.submitted_applying_bytes = self.submitted_applying_bytes.saturating_sub(bytes); } } @@ -422,7 +475,14 @@ impl Sequencer { } pub(super) fn remove_applying(&mut self, height: block::Height) -> Option { - self.applying.remove(&height) + let removed = self.applying.remove(&height)?; + self.applying_buffered_bytes = self.applying_buffered_bytes.saturating_sub(removed.bytes); + if removed.submitted { + self.submitted_applying_count = self.submitted_applying_count.saturating_sub(1); + self.submitted_applying_bytes = + self.submitted_applying_bytes.saturating_sub(removed.bytes); + } + Some(removed) } /// After a rejected/timed-out apply at `height`, roll the download floor back @@ -448,7 +508,7 @@ impl Sequencer { .collect(); let mut released = 0u64; for height in heights { - if let Some(applying) = self.applying.remove(&height) { + if let Some(applying) = self.remove_applying(height) { released = released.saturating_add(applying.bytes); } } @@ -466,7 +526,7 @@ impl Sequencer { self.clear_submitted_applies_through(tip); let mut released = 0u64; for height in applied { - if let Some(applying) = self.applying.remove(&height) { + if let Some(applying) = self.remove_applying(height) { released = released.saturating_add(applying.bytes); } } @@ -521,6 +581,9 @@ impl Sequencer { self.submitted_applies.clear(); } self.applying.clear(); + self.applying_buffered_bytes = 0; + self.submitted_applying_count = 0; + self.submitted_applying_bytes = 0; released } } diff --git a/zebra-network/src/zakura/block_sync/tests.rs b/zebra-network/src/zakura/block_sync/tests.rs index cd1f8068cd7..d28566f91eb 100644 --- a/zebra-network/src/zakura/block_sync/tests.rs +++ b/zebra-network/src/zakura/block_sync/tests.rs @@ -2851,6 +2851,93 @@ fn sequencer_retains_raw_bytes_for_non_contiguous_backlog() { ); } +#[test] +fn sequencer_applying_counters_match_scan_across_transitions() { + // Assert the O(1) applying counters (buffered bytes, submitted count/bytes) + // never drift from a full scan across insert / submit / unsubmit / remove / + // commit-release / reset. + let mut seq = test_sequencer(0, 8); + let blocks = mainnet_blocks_1_to_3(); + let check = |seq: &Sequencer, label: &str| { + assert_eq!( + seq.applying_buffered_bytes(), + seq.applying_buffered_bytes_scanned(), + "applying_buffered_bytes drifted after {label}" + ); + assert_eq!( + seq.submitted_applying_count(), + seq.submitted_applying_count_scanned(), + "submitted_applying_count drifted after {label}" + ); + assert_eq!( + seq.submitted_applying_bytes(), + seq.submitted_applying_bytes_scanned(), + "submitted_applying_bytes drifted after {label}" + ); + assert_eq!( + seq.unsubmitted_applying_count(), + seq.applying_len() - seq.submitted_applying_count(), + "unsubmitted derivation wrong after {label}" + ); + }; + + // Buffer heights 1..=3 with distinct sizes and drain them into `applying`. + for (i, block) in blocks.iter().enumerate() { + let height = (i + 1) as u32; + seq.accept_body( + block::Height(height), + block.hash(), + block.clone(), + 100 * u64::from(height), + peer(0), + ); + } + assert_eq!( + seq.drain_ready_into_applying(), + vec![block::Height(1), block::Height(2), block::Height(3)] + ); + assert_eq!(seq.applying_buffered_bytes(), 600); + assert_eq!(seq.submitted_applying_count(), 0); + check(&seq, "drain"); + + // Submit heights 1 and 2. + let item1 = seq + .prepare_submit(block::Height(1)) + .expect("height 1 applying"); + let _ = seq + .prepare_submit(block::Height(2)) + .expect("height 2 applying"); + assert_eq!(seq.submitted_applying_count(), 2); + assert_eq!(seq.submitted_applying_bytes(), 300); + assert_eq!(seq.unsubmitted_applying_count(), 1); + check(&seq, "submit 1,2"); + + // Roll back the submit for height 1. + seq.unsubmit(block::Height(1), item1.token); + assert_eq!(seq.submitted_applying_count(), 1); + assert_eq!(seq.submitted_applying_bytes(), 200); + check(&seq, "unsubmit 1"); + + // Remove the still-submitted height 2 directly. + seq.remove_applying(block::Height(2)); + assert_eq!(seq.submitted_applying_count(), 0); + assert_eq!(seq.submitted_applying_bytes(), 0); + assert_eq!(seq.applying_buffered_bytes(), 400); + check(&seq, "remove submitted 2"); + + // Commit through height 1: releases the applied body (height 1) from `applying`. + seq.advance_verified_tip(block::Height(1), true); + assert_eq!(seq.applying_buffered_bytes(), 300); + check(&seq, "advance_verified_tip"); + + // Reset drops all applying state and zeroes the counters. + seq.reset_to(block::Height(0), false); + assert_eq!(seq.applying_buffered_bytes(), 0); + assert_eq!(seq.submitted_applying_count(), 0); + assert_eq!(seq.submitted_applying_bytes(), 0); + check(&seq, "reset"); +} + #[test] fn sequencer_accept_body_rejects_at_or_below_floor() { let mut seq = test_sequencer(5, 4); From 556cb271c58fd8bfb7aace9ad4393b9f8099865e Mon Sep 17 00:00:00 2001 From: Roman Date: Wed, 1 Jul 2026 01:00:45 +0000 Subject: [PATCH 15/17] perf(state): isolate the reconcile worker on a dedicated rayon pool Per-phase timing (env-gated ZRB_RECONCILE_DEBUG) showed the reconcile worker is no longer the bottleneck after A1 (assembler backpressure = 0, worker 32-56% idle), but its resolution par_iter (~900k outpoints) shared the global rayon pool with the assembler's raw-transaction serialization par_iters, so the worker's large task starved the assembler's on the shared queue. Run the worker's per-interval reconcile on a dedicated RECONCILE_POOL so it no longer contends with the committer's global-pool work. Byte-match preserved (a scheduling-only change: value pool + UTXO set + per-height BlockInfo identical to a non-deferred run). Offline replay bench, mainnet sandblast range: sandblast throughput +114% -> +143% (284 -> 330 blk/s, ~86% of the +209% ceiling); overall +68%. The remaining sandblast gap (330 vs the ~470 light-region rate) is now assembler-side (the per-block commit of large blocks: creates, tx serialization, nullifiers, flush), not reconcile-side. --- zebra-state/src/service/finalized_state.rs | 21 ++++++++++++ .../src/service/finalized_state/pipeline.rs | 10 ++++++ .../service/finalized_state/zebra_db/chain.rs | 25 ++++++++++++++ zebra-state/src/service/write.rs | 34 +++++++++++++++++-- 4 files changed, 88 insertions(+), 2 deletions(-) diff --git a/zebra-state/src/service/finalized_state.rs b/zebra-state/src/service/finalized_state.rs index 0c259a41d47..4ab4e4e32de 100644 --- a/zebra-state/src/service/finalized_state.rs +++ b/zebra-state/src/service/finalized_state.rs @@ -76,6 +76,27 @@ static COMMIT_COMPUTE_POOL: LazyLock = LazyLock::new(|| { .expect("rayon thread pool configuration is valid") }); +/// A dedicated rayon thread pool for the deferred-transparent reconcile worker's +/// per-interval parallel passes (spent-UTXO resolution + per-block value-pool +/// deltas). +/// +/// Isolating the worker's large `par_iter`s (a heavy interval resolves ~900k +/// outpoints) onto their own pool keeps them out of the global pool's shared queue, +/// so the concurrently-running assembler's smaller `par_iter`s (raw-tx +/// serialization) are not stuck behind them. The worker has spare time per interval, +/// so a separate queue trades a little worker parallelism for steadier assembler +/// progress. +pub(crate) static RECONCILE_POOL: LazyLock = LazyLock::new(|| { + let threads = std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(4); + rayon::ThreadPoolBuilder::new() + .num_threads(threads) + .thread_name(|i| format!("reconcile-{i}")) + .build() + .expect("rayon thread pool configuration is valid") +}); + /// Spawns the note-commitment tree per-leaf hashing for `block` onto the /// commit-compute pool, returning a receiver for the result and a cancellation /// flag. diff --git a/zebra-state/src/service/finalized_state/pipeline.rs b/zebra-state/src/service/finalized_state/pipeline.rs index c8daf5d271c..4b3afc7a082 100644 --- a/zebra-state/src/service/finalized_state/pipeline.rs +++ b/zebra-state/src/service/finalized_state/pipeline.rs @@ -443,6 +443,7 @@ pub(crate) fn reconcile_window( // decode + index traversal) and independent across outpoints, and RocksDB reads // are thread-safe through the cloneable `DiskDb`. This runs off the assembler // thread on the worker, spreading the resolution across otherwise-idle cores. + let resolve_start = std::time::Instant::now(); use rayon::prelude::*; let resolved: HashMap = outpoints @@ -472,5 +473,14 @@ pub(crate) fn reconcile_window( ) .collect::, BoxError>>()?; + if std::env::var("ZRB_RECONCILE_DEBUG").is_ok() { + let last_h = records.last().map(|r| r.height.0).unwrap_or(0); + eprintln!( + "[recon-resolve] last_h={last_h} n_outpoints={} resolve_ms={:.1}", + outpoints.len(), + resolve_start.elapsed().as_secs_f64() * 1e3, + ); + } + db.commit_checkpoint_reconcile(network, start_value_pool, records, &resolved) } diff --git a/zebra-state/src/service/finalized_state/zebra_db/chain.rs b/zebra-state/src/service/finalized_state/zebra_db/chain.rs index 5dde4ce35db..0b16a171261 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/chain.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/chain.rs @@ -272,12 +272,17 @@ impl ZebraDb { records: &[ReconcileBlock], resolved: &HashMap, ) -> Result, BoxError> { + // Env-gated per-phase timing for the A2 bottleneck study. + let debug = std::env::var("ZRB_RECONCILE_DEBUG").is_ok(); + let last_h = records.last().map(|r| r.height.0).unwrap_or(0); + let mut batch = DiskWriteBatch::new(); // The deletes: every resolved spend, keyed by output location. The address // index is off in the deferred range, so `skip_index = true` makes this // delete only the `utxo_by_out_loc` entries (no address-link deletes), and // the empty `address_balances` is never consulted. + let deletes_start = std::time::Instant::now(); let spent_utxos_by_out_loc: BTreeMap = resolved .values() .map(|(out_loc, utxo)| (*out_loc, utxo.clone())) @@ -289,6 +294,8 @@ impl ZebraDb { &AddressBalanceLocationUpdates::Insert(HashMap::new()), true, ); + let deletes_dur = deletes_start.elapsed(); + let n_deletes = spent_utxos_by_out_loc.len(); // Compute each block's independent value-pool delta and serialized size in // parallel — `chain_value_pool_change` re-folds every transaction's value @@ -298,6 +305,7 @@ impl ZebraDb { // per-height running pool for `BlockInfo`. Splitting the parallel per-block // work from the ordered running-sum keeps the result byte-identical to the // inline per-block path (same deltas, same application order). + let vp_start = std::time::Instant::now(); use rayon::prelude::*; let per_block: Vec<(ValueBalance, usize)> = records .par_iter() @@ -336,7 +344,9 @@ impl ZebraDb { }, ) .collect::, BoxError>>()?; + let vp_dur = vp_start.elapsed(); + let bi_start = std::time::Instant::now(); let mut value_pool = start_value_pool; for (record, (delta, block_size)) in records.iter().zip(per_block) { value_pool = value_pool.add_chain_value_pool_change(delta)?; @@ -355,8 +365,23 @@ impl ZebraDb { .chain_value_pools_cf() .with_batch_for_writing(&mut batch) .zs_insert(&(), &value_pool); + let bi_dur = bi_start.elapsed(); + let write_start = std::time::Instant::now(); self.write_batch(batch)?; + let write_dur = write_start.elapsed(); + + if debug { + eprintln!( + "[recon-commit] last_h={last_h} n_blocks={} n_deletes={n_deletes} \ + deletes_ms={:.1} vp_ms={:.1} blockinfo_ms={:.1} write_ms={:.1}", + records.len(), + deletes_dur.as_secs_f64() * 1e3, + vp_dur.as_secs_f64() * 1e3, + bi_dur.as_secs_f64() * 1e3, + write_dur.as_secs_f64() * 1e3, + ); + } Ok(value_pool) } diff --git a/zebra-state/src/service/write.rs b/zebra-state/src/service/write.rs index f7c6b70fd1e..256c79d273b 100644 --- a/zebra-state/src/service/write.rs +++ b/zebra-state/src/service/write.rs @@ -151,8 +151,26 @@ fn run_reconcile_worker( job_receiver: std::sync::mpsc::Receiver, ) { let network = db.network(); - while let Ok(ReconcileJob { records }) = job_receiver.recv() { - match reconcile_window(&db, &network, &records, value_pool) { + let debug = std::env::var("ZRB_RECONCILE_DEBUG").is_ok(); + loop { + // Env-gated: time how long the worker sits idle waiting for the next window. A + // large wait means the worker is faster than the assembler produces intervals. + let recv_start = std::time::Instant::now(); + let Ok(ReconcileJob { records }) = job_receiver.recv() else { + break; + }; + if debug { + let last_h = records.last().map(|r| r.height.0).unwrap_or(0); + eprintln!( + "[recon-recv] last_h={last_h} recv_wait_ms={:.1}", + recv_start.elapsed().as_secs_f64() * 1e3, + ); + } + // Run the interval's parallel passes on the dedicated reconcile pool so they + // do not share the global pool's queue with the assembler's `par_iter`s. + let result = crate::service::finalized_state::RECONCILE_POOL + .install(|| reconcile_window(&db, &network, &records, value_pool)); + match result { Ok(new_pool) => value_pool = new_pool, Err(error) => panic!("deferred transparent reconcile worker failed: {error}"), } @@ -177,11 +195,23 @@ impl ReconcileWorker { return; } if let Some(sender) = self.sender.as_ref() { + // Env-gated: time how long the bounded send blocks. A large wait means the + // worker is behind (the capacity-1 channel is full), i.e. the reconcile + // gates the assembler. + let debug = std::env::var("ZRB_RECONCILE_DEBUG").is_ok(); + let last_h = records.last().map(|r| r.height.0).unwrap_or(0); + let send_start = std::time::Instant::now(); // The worker only stops on channel close, so a send error means it // panicked mid-reconcile; that is fatal (inconsistent value pool / UTXOs). sender .send(ReconcileJob { records }) .expect("reconcile worker thread has gone away"); + if debug { + eprintln!( + "[recon-send] last_h={last_h} backpressure_ms={:.1}", + send_start.elapsed().as_secs_f64() * 1e3, + ); + } } } From a9636bb182a979ded260df75a09303fdb43f9361 Mon Sep 17 00:00:00 2001 From: roman Date: Tue, 30 Jun 2026 20:05:03 -0600 Subject: [PATCH 16/17] test --- zebra-network/src/zakura/block_sync/tests.rs | 169 +++++++++++++++++++ 1 file changed, 169 insertions(+) diff --git a/zebra-network/src/zakura/block_sync/tests.rs b/zebra-network/src/zakura/block_sync/tests.rs index d28566f91eb..e3da6e222b6 100644 --- a/zebra-network/src/zakura/block_sync/tests.rs +++ b/zebra-network/src/zakura/block_sync/tests.rs @@ -3137,6 +3137,175 @@ fn sequencer_reject_drops_successors_and_rolls_floor_back() { assert_eq!(seq.applying_len(), 0); } +#[test] +fn sequencer_release_applying_blocks_from_keeps_submitted_counters_consistent() { + // `release_applying_blocks_from` removes each height through `remove_applying`, + // which must decrement the O(1) submitted counters for any *submitted* body it + // drops. Existing reject coverage only releases unsubmitted bodies, so this + // exercises the submitted-counter branch of that path. + let mut seq = test_sequencer(0, 8); + let blocks = mainnet_blocks_1_to_3(); + for (index, block) in blocks.iter().enumerate() { + let height = block::Height(index as u32 + 1); + seq.accept_body( + height, + block.hash(), + block.clone(), + 100 * (index as u64 + 1), + peer(0), + ); + } + seq.drain_ready_into_applying(); + // Submit heights 2 and 3 so the released prefix (>= 2) is all submitted work. + let _ = seq + .prepare_submit(block::Height(2)) + .expect("height 2 applying"); + let _ = seq + .prepare_submit(block::Height(3)) + .expect("height 3 applying"); + assert_eq!(seq.submitted_applying_count(), 2); + assert_eq!(seq.submitted_applying_bytes(), 200 + 300); + + let released = seq.release_applying_blocks_from(block::Height(2)); + assert_eq!(released, 500); + assert_eq!(seq.applying_len(), 1); + // Only the unsubmitted height 1 (100 bytes) survives; the submitted counters + // shed exactly the released bodies' contribution. + assert_eq!(seq.applying_buffered_bytes(), 100); + assert_eq!(seq.submitted_applying_count(), 0); + assert_eq!(seq.submitted_applying_bytes(), 0); + // Every maintained counter still agrees with a full scan. + assert_eq!( + seq.applying_buffered_bytes(), + seq.applying_buffered_bytes_scanned() + ); + assert_eq!( + seq.submitted_applying_count(), + seq.submitted_applying_count_scanned() + ); + assert_eq!( + seq.submitted_applying_bytes(), + seq.submitted_applying_bytes_scanned() + ); +} + +#[test] +fn sequencer_unsubmit_ignores_stale_or_mismatched_token() { + // `unsubmit` only rolls back (and decrements the submitted counters) when the + // token still matches the live submission, so a stale rollback cannot clobber a + // newer one or double-decrement the O(1) counters. + let mut seq = test_sequencer(0, 4); + let block = mainnet_block(&BLOCK_MAINNET_1_BYTES); + seq.accept_body(block::Height(1), block.hash(), block.clone(), 100, peer(0)); + seq.drain_ready_into_applying(); + let item = seq + .prepare_submit(block::Height(1)) + .expect("height 1 applying"); + assert_eq!(seq.submitted_applying_count(), 1); + assert_eq!(seq.submitted_applying_bytes(), 100); + + // A rollback carrying a non-matching token is ignored: the submission and the + // counters are untouched. + seq.unsubmit(block::Height(1), item.token + 1); + assert_eq!(seq.submitted_applying_count(), 1); + assert_eq!(seq.submitted_applying_bytes(), 100); + assert_eq!( + seq.submitted_applying_count(), + seq.submitted_applying_count_scanned() + ); + assert_eq!( + seq.submitted_applying_bytes(), + seq.submitted_applying_bytes_scanned() + ); + + // The matching rollback frees the slot exactly once. + seq.unsubmit(block::Height(1), item.token); + assert_eq!(seq.submitted_applying_count(), 0); + assert_eq!(seq.submitted_applying_bytes(), 0); + + // Replaying the now-stale token must not decrement a second time. + seq.unsubmit(block::Height(1), item.token); + assert_eq!(seq.submitted_applying_count(), 0); + assert_eq!(seq.submitted_applying_bytes(), 0); + assert_eq!( + seq.submitted_applying_count(), + seq.submitted_applying_count_scanned() + ); + assert_eq!( + seq.submitted_applying_bytes(), + seq.submitted_applying_bytes_scanned() + ); +} + +#[test] +fn sequencer_reorder_max_height_reports_highest_buffered() { + let mut seq = test_sequencer(0, 8); + let blocks = mainnet_blocks_1_to_3(); + // Empty reorder buffer has no top. + assert_eq!(seq.reorder_max_height(), None); + // Buffer heights 1 and 3, leaving a gap at 2; the top is the highest buffered. + seq.accept_body( + block::Height(1), + blocks[0].hash(), + blocks[0].clone(), + 100, + peer(0), + ); + seq.accept_body( + block::Height(3), + blocks[2].hash(), + blocks[2].clone(), + 300, + peer(0), + ); + assert_eq!(seq.reorder_max_height(), Some(block::Height(3))); + // Draining the contiguous prefix removes height 1 but leaves the top (3). + seq.drain_ready_into_applying(); + assert_eq!(seq.reorder_max_height(), Some(block::Height(3))); + // Filling the gap drains 2 and 3 out, emptying the reorder buffer. + seq.accept_body( + block::Height(2), + blocks[1].hash(), + blocks[1].clone(), + 200, + peer(0), + ); + seq.drain_ready_into_applying(); + assert_eq!(seq.reorder_max_height(), None); +} + +#[test] +fn sequencer_keeps_whole_body_for_contiguous_height() { + // The retain-for-backlog trim only applies to *non-contiguous* bodies. A body + // arriving at the next contiguous height above the floor is kept whole, so its + // decoded block — not a re-decode of the raw payload — drains into `applying`. + // This is the mirror of `sequencer_retains_raw_bytes_for_non_contiguous_backlog`. + let mut seq = test_sequencer(0, 4); + let block1 = mainnet_block(&BLOCK_MAINNET_1_BYTES); + // A decoded block distinguishable from its own raw payload lets us observe which + // half of `DecodedWithRawFramePayload` survived acceptance. + let distinguishable_decoded_block1 = forked_block(&block1, 99); + assert_ne!(distinguishable_decoded_block1.hash(), block1.hash()); + + let body = BufferedBlockBody::from_decoded_block( + distinguishable_decoded_block1.clone(), + Some(raw_block_payload(&block1)), + ); + // Height 1 is the next contiguous height above the floor (0). + assert_eq!( + seq.accept_buffered_body(block::Height(1), block1.hash(), body, 100, peer(0)), + AcceptOutcome::Buffered { + covered: block::Height(1) + } + ); + assert_eq!(seq.drain_ready_into_applying(), vec![block::Height(1)]); + // The kept decoded block drained in, not a re-decode of the raw payload. + assert_eq!( + seq.applying_hash(block::Height(1)), + Some(distinguishable_decoded_block1.hash()) + ); +} + #[test] fn reorder_fuzzes_arrival_order_as_parent_first() { let orders = [ From eade83305798d47b5ca2a9be698815323f6490d2 Mon Sep 17 00:00:00 2001 From: roman Date: Tue, 30 Jun 2026 20:15:10 -0600 Subject: [PATCH 17/17] feat(bench-dashboard): add peer-churn and download-supply panels MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add panels to the checkpoint-sync dashboard/recorder to distinguish peer/download problems from commit-side stalls: - Download: blocks received / s (rate of `sync.block.body.received`), and the floor-gap available / servable peer gauges (`sync.block.floor_gap.*`) — the head-of-line-on-the-floor signature (available drops to 0 while servable > 0). - Peers: connections accepted / s and closed / s (`zakura.p2p.conn.accepted`, `.conn.closed.neutral`) to surface churn, plus dial failures / s (`zakura.p2p.discovery.dial.failed`) for discovery/connectivity trouble. Recorded into per-run samples.jsonl (via PANEL_KEYS) so past runs can be compared. --- scripts/zebra-metrics-dashboard.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/scripts/zebra-metrics-dashboard.py b/scripts/zebra-metrics-dashboard.py index 2b4fb17a310..963d479bc29 100644 --- a/scripts/zebra-metrics-dashboard.py +++ b/scripts/zebra-metrics-dashboard.py @@ -103,6 +103,15 @@ def quantile(m, name, q): ("Zakura", "zk_peers", "Cohort peers (active)", "", "gauge"), ("Zakura", "zk_qdepth", "Zakura queue depth", "", "gauge"), ("Zakura", "zk_block_sync", "block_sync streams accepted", "", "gauge"), + # Download supply: received-body rate + whether a peer can serve the floor. + ("Download", "recv_s", "Blocks received / s", "blk/s", "rate"), + ("Download", "floor_avail_peers", "Floor-gap available peers", "", "gauge"), + ("Download", "floor_servable_peers","Floor-gap servable peers", "", "gauge"), + # Peer churn / connectivity: high accepted+closed with flat active = churn; + # dial failures = discovery/connectivity trouble. + ("Peers", "conn_new_s", "Connections accepted / s", "/s", "rate"), + ("Peers", "conn_closed_s", "Connections closed / s", "/s", "rate"), + ("Peers", "dial_fail_s", "Dial failures / s", "/s", "rate"), # Apply-queue depth + floor-gap attribution: separates HOL download stalls # from the sequencer→committer handoff ("glue") from peer-supply starvation. ("Apply queue","applied_s", "Submitted blocks / s", "/s", "rate"), @@ -360,6 +369,12 @@ def _tick(self): _reorder_bytes = bare(m, "sync_block_reorder_buffered_bytes") d["reorder_mb"] = (_reorder_bytes / 1e6) if _reorder_bytes is not None else None d["outstanding"] = bare(m, "sync_block_outstanding") + # Download-supply health at the sync floor: whether any peer can currently + # serve the next-needed block. `available` dropping to 0 while `servable` + # is non-zero is the head-of-line-on-the-floor signature (peers exist but + # none is dispatchable — the stall we saw at ~9 blk/s). + d["floor_avail_peers"] = bare(m, "sync_block_floor_gap_available_peers") + d["floor_servable_peers"] = bare(m, "sync_block_floor_gap_servable_peers") cur = { "h": bare(m, "state_finalized_block_height"), @@ -371,6 +386,12 @@ def _tick(self): "txid_s": bare(m, "zebra_state_prepare_txid_auth_digest_duration_seconds_sum"), "bc_s": bare(m, "zebra_state_rocksdb_batch_commit_duration_seconds_sum"), "bc_c": bare(m, "zebra_state_rocksdb_batch_commit_duration_seconds_count"), + # Download supply + peer-churn counters (rates computed below). `total` + # so a labeled series still aggregates. + "recv": total(m, "sync_block_body_received"), + "conn_acc": total(m, "zakura_p2p_conn_accepted"), + "conn_closed": total(m, "zakura_p2p_conn_closed_neutral"), + "dial_fail": total(m, "zakura_p2p_discovery_dial_failed"), } # Maintain the trailing height window for smoothed throughput. if cur["h"] is not None: @@ -413,6 +434,12 @@ def busy_pct(*names): d["vct_fast_s"] = rate(p["vf"], cur["vf"], dt) d["vct_prevalid_s"] = rate(p["vp"], cur["vp"], dt) d["applied_s"] = rate(p["sub"], cur["sub"], dt) + # Download supply rate + peer churn (connections opened/closed per sec, + # dial failures per sec) — to tell "slow/few peers" from a commit stall. + d["recv_s"] = rate(p["recv"], cur["recv"], dt) + d["conn_new_s"] = rate(p["conn_acc"], cur["conn_acc"], dt) + d["conn_closed_s"] = rate(p["conn_closed"], cur["conn_closed"], dt) + d["dial_fail_s"] = rate(p["dial_fail"], cur["dial_fail"], dt) d["p_auth_root"] = ms_per_block(p["aroot_s"], cur["aroot_s"], p["h"], cur["h"]) d["p_ordered_out"] = ms_per_block(p["ordered_s"], cur["ordered_s"], p["h"], cur["h"]) d["p_txid_digest"] = ms_per_block(p["txid_s"], cur["txid_s"], p["h"], cur["h"])