From 4df65164064625db60e5717f3dfa31d46123b78d Mon Sep 17 00:00:00 2001 From: roman Date: Thu, 2 Jul 2026 19:00:31 -0600 Subject: [PATCH 1/3] feat(state): activate VCT fast checkpoint sync in the committer --- docs/design/verified-commitment-trees.md | 56 +- zebra-state/CHANGELOG.md | 15 +- .../src/service/check/tests/nullifier.rs | 21 +- zebra-state/src/service/check/tests/utxo.rs | 8 +- .../src/service/check/tests/vectors.rs | 2 +- zebra-state/src/service/finalized_state.rs | 622 +++++- .../finalized_state/disk_format/chain.rs | 7 - .../disk_format/tests/snapshot.rs | 2 +- .../tests/snapshots/column_family_names.snap | 1 - .../empty_column_families@mainnet_0.snap | 1 - .../empty_column_families@mainnet_1.snap | 1 - .../empty_column_families@mainnet_2.snap | 1 - .../empty_column_families@no_blocks.snap | 1 - .../empty_column_families@testnet_0.snap | 1 - .../empty_column_families@testnet_1.snap | 1 - .../empty_column_families@testnet_2.snap | 1 - .../src/service/finalized_state/tests/prop.rs | 1868 ++++++++++++++++- .../service/finalized_state/tests/rollback.rs | 2 +- .../src/service/finalized_state/vct.rs | 123 +- .../service/finalized_state/zebra_db/block.rs | 96 +- .../zebra_db/block/tests/prune.rs | 48 +- .../zebra_db/block/tests/snapshot.rs | 2 +- .../zebra_db/block/tests/vectors.rs | 91 +- .../service/finalized_state/zebra_db/prune.rs | 2 +- .../finalized_state/zebra_db/rollback.rs | 43 +- .../finalized_state/zebra_db/shielded.rs | 23 +- zebra-state/src/service/write.rs | 127 +- zebra-state/src/service/write/vct_write.rs | 205 ++ .../src/service/write/vct_write/tests.rs | 280 +++ zebra-state/src/tests/setup.rs | 2 +- 30 files changed, 3350 insertions(+), 303 deletions(-) create mode 100644 zebra-state/src/service/write/vct_write.rs create mode 100644 zebra-state/src/service/write/vct_write/tests.rs diff --git a/docs/design/verified-commitment-trees.md b/docs/design/verified-commitment-trees.md index 4601dd6d13d..cbef9f24f93 100644 --- a/docs/design/verified-commitment-trees.md +++ b/docs/design/verified-commitment-trees.md @@ -28,7 +28,7 @@ header-sync reactor (zebra-network): validate root count + per-height alignment; │ unrequested or non-finalized roots as MalformedMessage (§8.1) ▼ CommitHeaderRange (zebra-state): persist provisional roots into - │ zakura_header_commitment_roots_by_height, ahead of body commit (§4.2) + │ commitment_roots_by_height, ahead of body commit (§4.2) ▼ PeerSource (DB-backed reader) ── vct_root(height) ──▶ finalized committer │ @@ -42,7 +42,7 @@ finalized committer: verify-before-commit (§6) ──fold roots, skip recompute ```text peer GetHeaders { want_tree_aux_roots } ─▶ header-sync reactor ─▶ header-sync driver (zebrad) ─▶ ReadRequest::BlockRoots ─▶ committed commitment_roots_by_height index, then provisional - zakura_header_commitment_roots_by_height for header-ahead heights (all-or-nothing; §9) + entries in the same index for header-ahead heights (all-or-nothing; §9) ``` **Lifecycle of one fast sync.** @@ -65,7 +65,7 @@ the direct below-Heartwood/below-NU5 checks); fold it in; freeze the frontier ( | **Frozen frontier** | During VCT fast sync below the last checkpoint, Zebra folds verified roots into the root indexes but does not advance the full on-disk note-commitment trees for every block. If a required root is missing, the committer must stop and retry later, because recomputing from the stale frontier would write invalid state (§8). | | **Verify-before-commit** | Authenticating each root against the node's header commitments (ZIP-221 MMR one-block-lag + direct sub-Heartwood/sub-NU5 checks) before it affects state (§6). | | **Fail closed** | Stop and retry without writing state when a required root is missing or invalid (§8). | -| **Provisional roots** | Peer-supplied roots carried in the header-sync `Headers` message and persisted to `zakura_header_commitment_roots_by_height` ahead of body commit. Advisory until verify-before-commit authenticates them (§4.2, §6). | +| **Provisional roots** | Peer-supplied roots carried in the header-sync `Headers` message and persisted to `commitment_roots_by_height` ahead of body commit. Advisory until verify-before-commit authenticates them (§4.2, §6). | | **All-or-nothing** | A `Headers` message carries roots for _every_ header in the range or none; a partial root set is rejected on the wire and never served (§5.4). | | **Kill switch** | `consensus.vct_fast_sync = false`: keep checkpoint sync but force the legacy committer (§4.4). | @@ -151,7 +151,7 @@ the new field. Roots are requested and accepted **only for finalized (checkpoint-verified) header ranges** — the reactor rejects roots on a non-finalized range, and rejects roots a request opted out of, as `MalformedMessage` (§8.1). When a finalized header range commits via `CommitHeaderRange`, its -roots are **persisted into the `zakura_header_commitment_roots_by_height` column family ahead of +roots are **persisted into the `commitment_roots_by_height` column family ahead of body commit** (§5.3). The committer then reads them per height through the `PeerSource` seam. The same header commit stores non-zero advertised body-size hints in `zakura_header_body_size_by_height`, so block sync can later request realistic ranges even @@ -168,10 +168,10 @@ Because roots ride the header-sync `Headers` message, they are fetched exactly w already is — for the finalized ranges between the verified tip and the last checkpoint height — with no separate fetch cursor, fetch-ahead cap, or eviction watermark to manage. The committer only ever looks up a root for a block it is about to commit, and persisted provisional roots are -naturally bounded above by the header tip and cleaned up below it: each provisional root is -**deleted from `zakura_header_commitment_roots_by_height` when its block body commits** (so the -column family does not grow without bound), and header-store rollback also trims provisional -roots above the rollback target (§5.3). Advertised body-size hints follow the same header-store +naturally bounded above by the header tip and settled below it: each provisional root is +**replaced by the verified serving-index row when its block body commits** (the same atomic +batch deletes the provisional entry and writes the committed row), and header-store rollback +also trims provisional roots above the rollback target. Advertised body-size hints follow the same header-store lifecycle: header reorgs and rollbacks drop stale hints, and committed block sizes take precedence once the corresponding body is durable. @@ -262,15 +262,15 @@ fn evict_committed_through(&self, height); // drop roots for already-committed h Implementations: -- `PeerSource` — the production default, a **DB-backed reader** (`PeerSource::new_with_db`). Each - `vct_root(height)` reads the provisional root for that height from the - `zakura_header_commitment_roots_by_height` column family that header sync persisted (§4.2). The +- `PeerSource` — the production default, a **DB-backed reader** (`PeerSource::new(db, + frontiers)`). Each `vct_root(height)` reads the provisional root for that height from the + `commitment_roots_by_height` column family that header sync persisted (§4.2). The last checkpoint height frontier is held immutably from the embedded constant, so only roots come from the network. `invalidate` **deletes** a rejected root from that column family so the next read misses and header sync can re-deliver a verifiable replacement from another peer (the key - to not letting one malicious peer wedge a bad root in place — §8, §11). An in-memory cache - variant (`PeerSource::new`, paired with a `PeerSourceWriter`) remains as **test-only** - scaffolding for proptests that fill roots without a database. + to not letting one malicious peer wedge a bad root in place — §8, §11). The earlier in-memory + cache variant and its `PeerSourceWriter` are removed; proptests fill roots by writing to an + ephemeral database through the same header-sync persistence path production uses. - `FixtureSource` — a crate-local `#[cfg(test)]` source over the same height→roots map, used only to isolate committer behavior and DB-produced payload round trips without networking. @@ -314,7 +314,8 @@ Wire and DoS bounds: - The `body_sizes` count must exactly match the header count (`BodySizeCountMismatch`); there is no independent untrusted body-size length to preallocate from. - The byte budget that bounds a `Headers` message accounts for the per-header root - (`HEADER_SYNC_BLOCK_COMMITMENT_ROOTS_BYTES = 4 + 32 + 32`), and the static + (`HEADER_SYNC_BLOCK_COMMITMENT_ROOTS_BYTES = 4 + 32·3 + 8·3 + 32` — height, the three + note-commitment roots, the three shielded tx-counts, and the auth-data root), and the static range-fits-budget assertion includes it, so requesting roots reduces the per-message header count accordingly (`inbound_get_headers_count_limit(.., want_tree_aux_roots)`). - Decoding validates: the `has_roots` marker must be 0 or 1 (`InvalidBoolMarker`); roots are @@ -324,7 +325,8 @@ Wire and DoS bounds: - The reactor additionally checks each root's height is `start_height + offset` (`TreeAuxRootHeightMismatch` / `validate_tree_aux_root_heights`) and rejects any roots on a non-finalized range, before the roots reach state. State re-checks both invariants in - `CommitHeaderRange` (`prepare_header_range_batch_with_roots`) as defense in depth. + `CommitHeaderRange` (`prepare_header_range_batch_with_roots`) as defense in depth, and never + writes peer-supplied roots for a height whose body is already committed — a re-delivered header range over committed heights cannot overwrite the verified serving-index rows. `BlockCommitmentRoots` still carries no trust: a recipient re-verifies every root against its own checkpoint-committed headers (§6) before folding it in, so a forwarding/serving node is @@ -498,7 +500,7 @@ provenance/cooldown/demotion/hedging policy. Bad roots are handled in two layers reach state. - **At verify-before-commit**, a well-formed but _wrong_ root fails authentication against the header commitment (§6). The committer evicts it (`PeerSource::invalidate` **deletes** it from - `zakura_header_commitment_roots_by_height`) and refuses the commit with the retryable + `commitment_roots_by_height`) and refuses the commit with the retryable `VctSuppliedRootUnavailable` error (§8). Header sync then re-requests that finalized range and delivers a replacement root from whichever peer answers; the block commits in place once a verifiable root arrives, without resetting the block queue. @@ -522,7 +524,7 @@ A node serves roots from local state via `ReadRequest::BlockRoots { start_height - serves **committed** verified roots first, from the compact `commitment_roots_by_height` index (so a fast-synced node lacking historical per-height trees can still serve), falling back to `produce_block_roots` over per-height trees only on a pre-index archive database; -- then appends **provisional** header-ahead roots from `zakura_header_commitment_roots_by_height` +- then appends **provisional** header-ahead roots from `commitment_roots_by_height` for the contiguous heights that have headers but no committed body yet — committed roots win on any overlap because they are already verified; - returns an empty vec for out-of-range/empty requests. @@ -542,8 +544,9 @@ freeze or retries the finalized range over header sync in the frozen window; it state. Two mechanisms address it, in order of cost: - **Roots-index CF (lightweight, preferred).** A fast node already verified every root it - folded in. Persisting them into a compact column family (~68 bytes/block, ~200 MB for all of - Mainnet) lets it serve them without per-height trees, at near-zero extra cost. A background + folded in. Persisting them into a compact column family (~160 bytes/block, ~550 MB for all of + Mainnet before compression) lets it serve them without per-height trees, at near-zero extra + cost. A background task can backfill missing lower ranges by fetching _roots_ (not bodies), so even a snapshot-started node becomes a full-range roots server cheaply. This is the targeted fix for the §10 serving-availability gap. @@ -594,7 +597,7 @@ commitment before it influences the anchor set or the history MMR.** Consequence - **Increment 6c — fold roots into header sync (current).** The standalone `tree_aux` stream, its driver, in-memory cache writer, and bespoke peer policy are **removed**. Roots now ride the header-sync `Headers` message as all-or-nothing finalized-range metadata (§4.2, §5.4), are - persisted provisionally to `zakura_header_commitment_roots_by_height` ahead of body commit, and + persisted provisionally to `commitment_roots_by_height` ahead of body commit, and are read back by a DB-backed `PeerSource`. Recovery from a bad/missing root is an in-place commit retry fed by header sync re-delivery; peer accountability rides header sync's existing misbehavior scoring (§8.1). @@ -665,9 +668,12 @@ asserts to prove roots actually came over the wire rather than a silent legacy s all-or-nothing serving helper (roots attached only on complete coverage, otherwise rootless headers) and routing received roots into `CommitHeaderRange`. - **State persistence:** `CommitHeaderRange` persists provisional roots into - `zakura_header_commitment_roots_by_height`, rejects count/height mismatches, deletes a - provisional root when its body commits, and trims provisional roots above a header-store - rollback target. + `commitment_roots_by_height`, rejects count/height mismatches, refuses to overwrite the + verified row of an already-committed height + (`header_range_roots_do_not_overwrite_committed_serving_index_rows`), replaces a provisional + root with the verified row when its body commits + (`write_block_replaces_matching_provisional_zakura_roots_with_verified_row`), and trims + provisional roots above a header-store rollback target. - **Real-data manual runs (`#[ignore]`, env-gated):** `verifies_real_nu5_range_over_synced_forks` verifies the real NU5/V2 range against synced archive forks (corrupted root rejected at H+1). - **Headline end-to-end (manual, follow-up):** a fresh node fast-syncing @@ -688,7 +694,7 @@ asserts to prove roots actually came over the wire rather than a silent legacy s | Embedded Mainnet frontier | `zebra-state/src/service/finalized_state/vct/mainnet-frontier.bin` | | Commit-path hook, last checkpoint height, frozen-frontier policy | `zebra-state/src/service/finalized_state.rs` | | `BlockRoots` serving read (committed + provisional) | `zebra-state/src/service.rs` | -| Provisional roots CF (`zakura_header_commitment_roots_by_height`), persistence, body-commit/rollback cleanup | `zebra-state/src/service/finalized_state/zebra_db/block.rs`, `.../rollback.rs` | +| Provisional roots in `commitment_roots_by_height`, persistence, body-commit/rollback cleanup | `zebra-state/src/service/finalized_state/zebra_db/block.rs`, `.../rollback.rs` | | `CommitHeaderRange` with roots, fast-path hit/miss metrics | `zebra-state/src/service/write.rs` | | Header-sync wire (`GetHeaders`/`Headers` roots, markers, byte budget) | `zebra-network/src/zakura/header_sync/wire.rs` | | Header-sync root validation (count, height alignment, markers) | `zebra-network/src/zakura/header_sync/validation.rs`, `.../error.rs` | diff --git a/zebra-state/CHANGELOG.md b/zebra-state/CHANGELOG.md index 70f0709aa77..c18c5498b88 100644 --- a/zebra-state/CHANGELOG.md +++ b/zebra-state/CHANGELOG.md @@ -26,8 +26,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 handoff height, and the retryable `ValidateContextError` variants `VctSuppliedRootUnavailable` and `VctSuppliedRootAwaitingSuccessor` (with `vct_retryable_height` / `vct_supplied_root_unavailable_height` accessors on - the commit error types). Nothing returns these errors yet; the committer fast - path that raises them lands in a follow-up increment. + the commit error types). +- Verified-commitment-trees fast checkpoint sync is now active in the state + layer: below the checkpoint, the committer folds header-sync-supplied + Sapling/Orchard roots (verified against the node's own header commitments) + into the anchor set and history tree, skipping the per-block note-commitment + frontier recompute. At the checkpoint handoff the embedded final frontier is + verified against that block's proven root and written as the tip treestate. A + root that cannot be obtained or verified defers the commit (retryable) rather + than recomputing against the frozen frontier, and a fast-synced database + fails closed on reopen if its frozen frontier would be misused. The resulting + consensus state is byte-identical to the legacy recompute. Default-on for + Mainnet under checkpoint sync; the `consensus.vct_fast_sync` opt-out is wired + through zebrad in a follow-up increment. ## [8.0.0] - 2026-06-02 diff --git a/zebra-state/src/service/check/tests/nullifier.rs b/zebra-state/src/service/check/tests/nullifier.rs index e634f485e59..420a4ee2be7 100644 --- a/zebra-state/src/service/check/tests/nullifier.rs +++ b/zebra-state/src/service/check/tests/nullifier.rs @@ -89,7 +89,7 @@ proptest! { // randomly choose to commit the block to the finalized or non-finalized state if use_finalized_state { let block1 = CheckpointVerifiedBlock::from(Arc::new(block1)); - let commit_result = finalized_state.commit_finalized_direct(block1.clone().into(), None, "test"); + let commit_result = finalized_state.commit_finalized_direct(block1.clone().into(), None, None, "test"); // the block was committed prop_assert_eq!(Some((Height(1), block1.hash)), read::best_tip(&non_finalized_state, &finalized_state.db)); @@ -353,7 +353,7 @@ proptest! { // randomly choose to commit the next block to the finalized or non-finalized state if duplicate_in_finalized_state { let block1 = CheckpointVerifiedBlock::from(Arc::new(block1)); - let commit_result = finalized_state.commit_finalized_direct(block1.clone().into(), None, "test"); + let commit_result = finalized_state.commit_finalized_direct(block1.clone().into(), None, None, "test"); prop_assert_eq!(Some((Height(1), block1.hash)), read::best_tip(&non_finalized_state, &finalized_state.db)); prop_assert!(commit_result.is_ok()); @@ -452,7 +452,7 @@ proptest! { // randomly choose to commit the block to the finalized or non-finalized state if use_finalized_state { let block1 = CheckpointVerifiedBlock::from(Arc::new(block1)); - let commit_result = finalized_state.commit_finalized_direct(block1.clone().into(),None, "test"); + let commit_result = finalized_state.commit_finalized_direct(block1.clone().into(), None, None, "test"); prop_assert_eq!(Some((Height(1), block1.hash)), read::best_tip(&non_finalized_state, &finalized_state.db)); prop_assert!(commit_result.is_ok()); @@ -632,7 +632,7 @@ proptest! { // randomly choose to commit the next block to the finalized or non-finalized state if duplicate_in_finalized_state { let block1 = CheckpointVerifiedBlock::from(Arc::new(block1)); - let commit_result = finalized_state.commit_finalized_direct(block1.clone().into(),None, "test"); + let commit_result = finalized_state.commit_finalized_direct(block1.clone().into(), None, None, "test"); prop_assert_eq!(Some((Height(1), block1.hash)), read::best_tip(&non_finalized_state, &finalized_state.db)); prop_assert!(commit_result.is_ok()); @@ -729,7 +729,7 @@ proptest! { // randomly choose to commit the block to the finalized or non-finalized state if use_finalized_state { let block1 = CheckpointVerifiedBlock::from(Arc::new(block1)); - let commit_result = finalized_state.commit_finalized_direct(block1.clone().into(), None, "test"); + let commit_result = finalized_state.commit_finalized_direct(block1.clone().into(), None, None, "test"); prop_assert_eq!(Some((Height(1), block1.hash)), read::best_tip(&non_finalized_state, &finalized_state.db)); prop_assert!(commit_result.is_ok()); @@ -918,7 +918,7 @@ proptest! { // randomly choose to commit the next block to the finalized or non-finalized state if duplicate_in_finalized_state { let block1 = CheckpointVerifiedBlock::from(Arc::new(block1)); - let commit_result = finalized_state.commit_finalized_direct(block1.clone().into(), None, "test"); + let commit_result = finalized_state.commit_finalized_direct(block1.clone().into(), None, None, "test"); prop_assert_eq!(Some((Height(1), block1.hash)), read::best_tip(&non_finalized_state, &finalized_state.db)); prop_assert!(commit_result.is_ok()); @@ -1025,7 +1025,8 @@ proptest! { let block1_hash; if duplicate_in_finalized_state { let block1 = CheckpointVerifiedBlock::from(Arc::new(block1)); - let commit_result = finalized_state.commit_finalized_direct(block1.clone().into(), None, "test"); + let commit_result = + finalized_state.commit_finalized_direct(block1.clone().into(), None, None, "test"); prop_assert_eq!(Some((Height(1), block1.hash)), read::best_tip(&non_finalized_state, &finalized_state.db)); prop_assert!(commit_result.is_ok()); @@ -1118,7 +1119,7 @@ proptest! { finalized_state.populate_with_anchors(&block2); let block1 = CheckpointVerifiedBlock::from(Arc::new(block1)); - let commit_result = finalized_state.commit_finalized_direct(block1.into(), None, "test"); + let commit_result = finalized_state.commit_finalized_direct(block1.into(), None, None, "test"); prop_assert!(commit_result.is_ok()); let block2 = Arc::new(block2).prepare(); @@ -1172,7 +1173,7 @@ proptest! { finalized_state.populate_with_anchors(&block2); let block1 = CheckpointVerifiedBlock::from(Arc::new(block1)); - let commit_result = finalized_state.commit_finalized_direct(block1.into(), None, "test"); + let commit_result = finalized_state.commit_finalized_direct(block1.into(), None, None, "test"); prop_assert!(commit_result.is_ok()); let block2 = Arc::new(block2).prepare(); @@ -1226,7 +1227,7 @@ proptest! { finalized_state.populate_with_anchors(&block2); let block1 = CheckpointVerifiedBlock::from(Arc::new(block1)); - let commit_result = finalized_state.commit_finalized_direct(block1.into(), None, "test"); + let commit_result = finalized_state.commit_finalized_direct(block1.into(), None, None, "test"); prop_assert!(commit_result.is_ok()); let block2 = Arc::new(block2).prepare(); diff --git a/zebra-state/src/service/check/tests/utxo.rs b/zebra-state/src/service/check/tests/utxo.rs index dd9017bea20..69bfe446f69 100644 --- a/zebra-state/src/service/check/tests/utxo.rs +++ b/zebra-state/src/service/check/tests/utxo.rs @@ -185,7 +185,7 @@ proptest! { // randomly choose to commit the block to the finalized or non-finalized state if use_finalized_state { let block1 = CheckpointVerifiedBlock::from(Arc::new(block1)); - let commit_result = finalized_state.commit_finalized_direct(block1.clone().into(), None, "test"); + let commit_result = finalized_state.commit_finalized_direct(block1.clone().into(), None, None, "test"); // the block was committed prop_assert_eq!(Some((Height(1), block1.hash)), read::best_tip(&non_finalized_state, &finalized_state.db)); @@ -273,7 +273,7 @@ proptest! { if use_finalized_state_spend { let block2 = CheckpointVerifiedBlock::from(Arc::new(block2)); - let commit_result = finalized_state.commit_finalized_direct(block2.clone().into(),None, "test"); + let commit_result = finalized_state.commit_finalized_direct(block2.clone().into(), None, None, "test"); // the block was committed prop_assert_eq!(Some((Height(2), block2.hash)), read::best_tip(&non_finalized_state, &finalized_state.db)); @@ -609,7 +609,7 @@ proptest! { if use_finalized_state_spend { let block2 = CheckpointVerifiedBlock::from(block2.clone()); - let commit_result = finalized_state.commit_finalized_direct(block2.clone().into(), None, "test"); + let commit_result = finalized_state.commit_finalized_direct(block2.clone().into(), None, None, "test"); // the block was committed prop_assert_eq!(Some((Height(2), block2.hash)), read::best_tip(&non_finalized_state, &finalized_state.db)); @@ -878,7 +878,7 @@ fn new_state_with_mainnet_transparent_data( if use_finalized_state { let block1 = CheckpointVerifiedBlock::from(block1.clone()); let commit_result = - finalized_state.commit_finalized_direct(block1.clone().into(), None, "test"); + finalized_state.commit_finalized_direct(block1.clone().into(), None, None, "test"); // the block was committed assert_eq!( diff --git a/zebra-state/src/service/check/tests/vectors.rs b/zebra-state/src/service/check/tests/vectors.rs index 2e07f5bfb0e..315ef98e885 100644 --- a/zebra-state/src/service/check/tests/vectors.rs +++ b/zebra-state/src/service/check/tests/vectors.rs @@ -136,7 +136,7 @@ fn block_commitment_uses_the_precomputed_auth_data_root() { assert!(matches!( error, ValidateContextError::InvalidBlockCommitment( - CommitmentError::InvalidChainHistoryBlockTxAuthCommitment { actual, expected }, + CommitmentError::InvalidChainHistoryBlockTxAuthCommitment { actual, expected } ) if actual == block_commitment && expected == <[u8; 32]>::from(forged_hash_block_commitments) )); diff --git a/zebra-state/src/service/finalized_state.rs b/zebra-state/src/service/finalized_state.rs index 7024ebbd5c8..a1a0a730b4d 100644 --- a/zebra-state/src/service/finalized_state.rs +++ b/zebra-state/src/service/finalized_state.rs @@ -22,11 +22,11 @@ use std::{ }, }; -use zebra_chain::{block, parallel::tree::NoteCommitmentTrees, parameters::Network}; +use zebra_chain::{ + block, ironwood, orchard, parallel::tree::NoteCommitmentTrees, parameters::Network, sapling, +}; use zebra_db::{ - block::{ - RetentionPlan, ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT, ZAKURA_HEADER_COMMITMENT_ROOTS_BY_HEIGHT, - }, + block::{RetentionPlan, ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT}, chain::BLOCK_INFO, transparent::{BALANCE_BY_TRANSPARENT_ADDR, TX_LOC_BY_SPENT_OUT_LOC}, }; @@ -78,12 +78,11 @@ pub(crate) mod commitment_aux; pub(crate) mod commitment_aux_verify; mod disk_db; mod disk_format; -// The committer fast path (next increment) constructs and consumes `VctState`; until it -// lands, the runtime is exercised only by its tests and the frontier-bytes validators. -#[allow(dead_code)] mod vct; mod zebra_db; +use vct::{VctCommitState, VctState, VctWriteData}; + #[cfg(any(test, feature = "proptest-impl"))] mod arbitrary; @@ -101,7 +100,7 @@ pub use disk_format::{ FromDisk, IntoDisk, OutputLocation, RawBytes, TransactionIndex, TransactionLocation, MAX_ON_DISK_HEIGHT, }; -pub use vct::{validate_final_frontiers_bytes, FinalFrontiersValidationError}; +pub use vct::{validate_final_frontiers_bytes, FinalFrontiersValidationError, NextVctBlock}; pub use zebra_db::ZebraDb; #[cfg(any(test, feature = "proptest-impl"))] @@ -130,7 +129,6 @@ pub const STATE_COLUMN_FAMILIES_IN_CODE: &[&str] = &[ "zakura_header_height_by_hash", "zakura_header_by_height", ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT, - ZAKURA_HEADER_COMMITMENT_ROOTS_BY_HEIGHT, // Transactions "tx_by_loc", "hash_by_tx_loc", @@ -250,6 +248,9 @@ pub struct FinalizedState { #[cfg(feature = "elasticsearch")] /// A collection of blocks to be sent to elasticsearch as a bulk. pub elastic_blocks: Vec, + + /// Commit-time verified-commitment-trees state. + vct: VctCommitState, } impl FinalizedState { @@ -270,6 +271,23 @@ impl FinalizedState { ) } + /// Opens (or creates) the on-disk finalized state database read-write, for + /// offline tooling (e.g. the replay benchmark). + /// + /// Equivalent to [`FinalizedState::new`] but without the `elasticsearch` + /// feature's `enable_elastic_db` parameter, so callers compile unchanged + /// regardless of feature flags (elasticsearch is never enabled here). + pub fn new_writable(config: &Config, network: &Network) -> Self { + Self::new_with_debug( + config, + network, + false, + #[cfg(feature = "elasticsearch")] + false, + false, + ) + } + /// Returns an on-disk database instance with the supplied production and debug settings. /// If there is no existing database, creates a new database on disk. /// @@ -289,6 +307,7 @@ impl FinalizedState { enable_elastic_db, read_only, true, + true, ) } @@ -312,6 +331,7 @@ impl FinalizedState { enable_elastic_db, read_only, false, + true, ) } @@ -322,6 +342,7 @@ impl FinalizedState { #[cfg(feature = "elasticsearch")] enable_elastic_db: bool, read_only: bool, validate_storage_mode: bool, + enforce_resume_guard: bool, ) -> Self { // Fail fast on an invalid storage configuration, before opening the database. if validate_storage_mode { @@ -370,6 +391,21 @@ impl FinalizedState { read_only, ); + let vct = VctState::from_config( + config.checkpoint_sync, + config.vct_fast_sync, + network, + db.clone(), + ); + + // Re-derive this flag from the durable fast-sync marker, so reopening + // before the checkpoint handoff still refuses roots below the last + // checkpoint. The checkpoint height itself has the real frontier. + let is_vct_sync_below_last_checkpoint = db + .vct_synced_below() + .zip(db.finalized_tip_height()) + .is_some_and(|(last_checkpoint_height, tip)| tip < last_checkpoint_height); + #[cfg(feature = "elasticsearch")] let new_state = Self { debug_stop_at_height: config.debug_stop_at_height.map(block::Height), @@ -378,6 +414,7 @@ impl FinalizedState { db, elastic_db, elastic_blocks: vec![], + vct: VctCommitState::new(vct, is_vct_sync_below_last_checkpoint), }; #[cfg(not(feature = "elasticsearch"))] @@ -386,6 +423,7 @@ impl FinalizedState { checkpoint_raw_tx_retention_start: None, checkpoint_raw_tx_archive_backlog: Arc::new(AtomicBool::new(false)), db, + vct: VctCommitState::new(vct, is_vct_sync_below_last_checkpoint), }; // Pruning is a one-way storage mode. Refuse to open a database that has @@ -399,6 +437,22 @@ impl FinalizedState { ); } + // Interrupted VCT syncs below the checkpoint handoff need the VCT root + // source to resume. Without it, the legacy committer would refuse every + // remaining checkpoint block. + if enforce_resume_guard + && new_state.vct.is_below_last_checkpoint() + && new_state.vct.source().is_none() + { + panic!( + "this database was previously synced in verified commitment tree mode that was \ + interrupted below the last checkpoint height. the fast path that supplies \ + the verified roots needed to resume the VCT sync is disabled. Set \ + `consensus.checkpoint_sync = true` and `consensus.vct_fast_sync = true` to \ + finish the VCT sync, or delete the cache directory and re-sync from genesis" + ); + } + // TODO: move debug_stop_at_height into a task in the start command (#3442) if let Some(tip_height) = new_state.db.finalized_tip_height() { if new_state.is_at_stop_height(tip_height) { @@ -560,11 +614,16 @@ impl FinalizedState { &mut self, ordered_block: QueuedCheckpointVerified, prev_note_commitment_trees: Option, - ) -> Result<(CheckpointVerifiedBlock, NoteCommitmentTrees), CommitCheckpointVerifiedError> { + next_vct_block: Option, + ) -> Result< + (CheckpointVerifiedBlock, NoteCommitmentTrees), + (QueuedCheckpointVerified, CommitCheckpointVerifiedError), + > { let (checkpoint_verified, rsp_tx) = ordered_block; let result = self.commit_finalized_direct( checkpoint_verified.clone().into(), prev_note_commitment_trees, + next_vct_block, "commit checkpoint-verified request", ); @@ -585,9 +644,13 @@ impl FinalizedState { .set(checkpoint_verified.height.0 as f64); }; - let _ = rsp_tx.send(result.clone().map(|(hash, _)| hash)); - - result.map(|(_hash, note_commitment_trees)| (checkpoint_verified, note_commitment_trees)) + match result { + Ok((hash, note_commitment_trees)) => { + let _ = rsp_tx.send(Ok(hash)); + Ok((checkpoint_verified, note_commitment_trees)) + } + Err(error) => Err(((checkpoint_verified, rsp_tx), error)), + } } /// Immediately commit a `finalized` block to the finalized state. @@ -608,9 +671,10 @@ impl FinalizedState { &mut self, finalizable_block: FinalizableBlock, prev_note_commitment_trees: Option, + next_vct_block: Option, source: &str, ) -> Result<(block::Hash, NoteCommitmentTrees), CommitCheckpointVerifiedError> { - let (height, hash, finalized, prev_note_commitment_trees, retention) = + let (height, hash, finalized, prev_note_commitment_trees, retention, fast_write) = match finalizable_block { FinalizableBlock::Checkpoint { checkpoint_verified, @@ -629,90 +693,288 @@ impl FinalizedState { let mut note_commitment_trees = prev_note_commitment_trees.clone(); let network = self.network(); + let height = checkpoint_verified.height; - // Run two independent CPU-intensive crypto operations concurrently - // on the rayon pool (Part 1 of the checkpoint-commit parallelization): - // - // - updating the note commitment trees, and - // - checking this block's commitment against the *parent* history tree. - // - // These are independent: the commitment check reads only the parent - // history tree (not this block's note commitment trees), and the - // history tree push below depends on both, so it runs after the join. - // - // The commitment check is done here (and not during semantic - // validation) because it needs the history tree root, and the - // checkpoint verifier doesn't run contextual validation. For - // Nu5-onward the block hash commits only to non-authorizing data - // (ZIP-244), so this verifies the authorizing-data commitment. - #[cfg(feature = "commit-metrics")] - metrics::histogram!("zebra.state.write.block_tx_count") - .record(block.transactions.len() as f64); - #[cfg(feature = "commit-metrics")] - let _ckpt_compute = std::time::Instant::now(); - let mut commitment_result = None; - // Run the two CPU-intensive operations inside the dedicated - // commit-compute pool so their nested rayon work uses isolated workers instead of - // contending with the verifier on the global pool. - let tree_result = COMMIT_COMPUTE_POOL.install(|| { - rayon::in_place_scope_fifo(|scope| { - scope.spawn_fifo(|_scope| { - commitment_result = Some(timed_commit_phase!( - "zebra.state.write.commitment_check.duration_seconds", - check::block_commitment_is_valid_for_chain_history( - block.clone(), - &network, - &history_tree, - precomputed_auth_data_root, - ) - )); - }); - - timed_commit_phase!( - "zebra.state.write.update_trees.duration_seconds", - note_commitment_trees.update_trees_parallel(&block) - ) - }) + // The last checkpoint height (boundary below which the vct + // path skips per-height trees). + let vct_last_checkpoint_height = self + .vct + .source() + .map(|v| v.vct_sync_last_checkpoint_height()); + + // In vct mode, if the source has this height's roots at or below the + // last checkpoint height, we skip the per-block note-commitment frontier recompute + // (`update_trees_parallel`). Instead, we validate the peer-supplied roots + // against the successor block's header/MMR. + let vct_roots = self.vct.source().and_then(|v| { + if vct_last_checkpoint_height + .is_some_and(|last_checkpoint_height| height > last_checkpoint_height) + { + None + } else { + v.vct_roots_at_height(height) + } }); - // Surface the tree-update error first, preserving the error - // precedence of the previous sequential code. - tree_result.map_err(ValidateContextError::from)?; - // `rayon::in_place_scope_fifo` guarantees all spawned tasks - // complete before the scope returns, so `commitment_result` is - // always `Some` here: the spawned closure wrote to it before - // the scope exited. - commitment_result.expect("scope has already finished")?; - - // Update the history tree (depends on both operations above). - let history_tree_mut = Arc::make_mut(&mut history_tree); - let sapling_root = note_commitment_trees.sapling.root(); - let orchard_root = note_commitment_trees.orchard.root(); - let ironwood_root = note_commitment_trees.ironwood.root(); - history_tree_mut - .push( - &network, - block.clone(), - &sapling_root, - &orchard_root, - &ironwood_root, + let mut vct_write = VctWriteData::default(); + + if let Some((sapling_root, orchard_root)) = vct_roots { + // The last checkpoint frontiers are the only non-successor authority that + // can authenticate this block's own supplied roots before they are + // persisted. + let last_checkpoint_frontiers = self + .vct + .source() + .and_then(|v| v.final_frontiers_for_last_checkpoint(height)); + + // This block's own commitment check is identical to the + // previous vct block's look-ahead. When that look-ahead + // already validated this exact header, skip the duplicate. + let block_hash = block.hash(); + + // Defense in depth: only a witness that links to this block can + // authenticate its roots — a non-successor's commitment binds a + // different parent tree, so verifying against it would fail and + // wrongly evict a good supplied root. Treat a non-linking witness + // as absent, so the await-successor deferral below handles it. The + // write worker only buffers direct successors, so this should + // never fire. + let next_vct_block = next_vct_block.filter(|next_vct_block| { + let links = + next_vct_block.block.header.previous_block_hash == block_hash; + if !links { + tracing::warn!( + ?height, + witness_parent = ?next_vct_block.block.header.previous_block_hash, + expected_parent = ?block_hash, + "VCT: ignoring a successor witness that does not link \ + to the block being committed" + ); + } + links + }); + + let is_prevalidated = + self.vct.prevalidated_next() == Some((height, block_hash)); + if is_prevalidated { + if let Some(v) = self.vct.source() { + v.record_prevalidated(); + } + // Observability: the previous fast block's look-ahead already + // validated this header, so its commitment check was skipped (the + // dedup). A subset of `state.vct.fast.block.count`. + metrics::counter!("state.vct.prevalidated.block.count").increment(1); + } + + let mut verification_items = vec![ + commitment_aux_verify::CommitmentRootVerification::with_roots( + block.clone(), + sapling_root, + orchard_root, + precomputed_auth_data_root, + is_prevalidated, + ), + ]; + + // If a buffered VCT successor block is available, we verify the current block's + // supplied roots against the successor block's header/MMR. + if let Some(next_vct_block) = &next_vct_block { + verification_items.push( + commitment_aux_verify::CommitmentRootVerification::header_only( + next_vct_block.block.clone(), + next_vct_block.auth_data_root, + ), + ); + } + + // Verifies this block's own header, folds its supplied roots into + // the candidate tree, and when buffered checks the successor header + // against that candidate (the one-block lag). + let candidate = COMMIT_COMPUTE_POOL + .install(|| { + commitment_aux_verify::verify_commitment_roots( + &network, + (*history_tree).clone(), + verification_items, + ) + }) + .map_err(|(_fail_height, error)| { + self.vct.clear_prevalidated_next(); + self.vct_reject_supplied_root(height, error) + })?; + + if let Some(next_vct_block) = &next_vct_block { + self.vct.mark_prevalidated( + (height + 1).expect("checkpoint block heights are valid"), + next_vct_block.block.hash(), + ); + } else if self + .vct + .source() + .is_some_and(|v| v.vct_root_needs_successor(height, &network)) + { + // Untrusted root at/above Heartwood, no successor to confirm it, + // not the last checkpoint: defer rather than persist it unverified. Leaves + // the database untouched; the block re-commits once the successor + // is buffered. + metrics::counter!("state.vct.root.await_successor.count").increment(1); + return Err(ValidateContextError::VctSuppliedRootAwaitingSuccessor { + height, + } + .into()); + } else { + self.vct.clear_prevalidated_next(); + } + + history_tree = Arc::new(candidate); + if let Some(v) = self.vct.source() { + v.record_fast_block(); + } + // Observability: this block folded supplied roots and skipped the + // note-commitment frontier recompute (the verified-commitment-trees + // fast path). Paired with `state.vct.legacy.block.count` below, this + // gives a live fast-vs-legacy ratio. + metrics::counter!("state.vct.fast.block.count").increment(1); + + // When final frontiers are loaded, this is a persistent fast + // sync: mark the database fast-synced (per-height trees absent + // below the handoff height). + vct_write.sync_below = vct_last_checkpoint_height; + + if let Some((sapling_frontier, orchard_frontier, sprout_frontier)) = + last_checkpoint_frontiers + { + // Last checkpoint verification: verify the supplied frontiers against + // this block's verified roots. + self.vct_verify_handoff_frontier_roots( + height, + &sapling_frontier, + &orchard_frontier, + &sapling_root, + &orchard_root, + )?; + + // Subtree tips are left `None`: the resuming chain recomputes + // them from the frontier position. + note_commitment_trees = NoteCommitmentTrees { + sprout: sprout_frontier, + sapling: sapling_frontier, + sapling_subtree: None, + orchard: orchard_frontier, + orchard_subtree: None, + ironwood: Arc::::default(), + ironwood_subtree: None, + }; + + // The handoff writes the real final frontier as the tip + // treestate, so the frontier is no longer frozen: heights at and + // above the handoff resume legacy recompute from a correct frontier. + self.vct.stop_vct_sync_at_last_checkpoint(); + } else { + vct_write.anchor_roots = Some((sapling_root, orchard_root)); + + // A non-handoff fast block leaves the note-commitment frontier + // frozen (it folds roots instead of advancing the trees), so a + // later height with no valid supplied root must not legacy-recompute + // against this stale frontier (see the `else` branch below). + self.vct.start_vct_sync_below_last_checkpoint(); + } + } else if self.vct.is_below_last_checkpoint() { + // Frozen-frontier safety: a fast sync has already frozen the + // note-commitment frontier, but this height has no valid supplied root + // (never fetched, or evicted after failing verification). Recomputing + // here would fold a wrong root into the history MMR and corrupt state, + // so refuse with a retryable error and leave the database untouched — + // the block is committed once a verifiable root is fetched from a peer. + metrics::counter!("state.vct.root.unavailable.count").increment(1); + tracing::warn!( + ?height, + "VCT: no verifiable supplied root for a frozen-frontier height; \ + refusing to recompute (retryable)" + ); + return Err( + ValidateContextError::VctSuppliedRootUnavailable { height }.into() + ); + } else { + // Not a fast block: any cached pre-validation does not apply to + // the next fast block (its parent frontier differs), so clear it. + self.vct.clear_prevalidated_next(); + + // Observability: this block recomputed the note-commitment frontier + // (the legacy path) — either VCT is off, or the fast path's roots were + // unavailable for this height and it safely fell back. + metrics::counter!("state.vct.legacy.block.count").increment(1); + + // Legacy / capture path: recompute the note-commitment frontier. + // + // Run two independent CPU-intensive crypto operations concurrently + // on the rayon pool: updating the note commitment trees, and + // checking this block's commitment against the *parent* history + // tree. They are independent; the history push below joins them. + #[cfg(feature = "commit-metrics")] + metrics::histogram!("zebra.state.write.block_tx_count") + .record(block.transactions.len() as f64); + #[cfg(feature = "commit-metrics")] + let _ckpt_compute = std::time::Instant::now(); + let mut commitment_result = None; + // Run the two CPU-intensive operations inside the dedicated + // commit-compute pool so their nested rayon work uses isolated workers instead of + // contending with the verifier on the global pool. + let tree_result = COMMIT_COMPUTE_POOL.install(|| { + rayon::in_place_scope_fifo(|scope| { + scope.spawn_fifo(|_scope| { + commitment_result = Some(timed_commit_phase!( + "zebra.state.write.commitment_check.duration_seconds", + check::block_commitment_is_valid_for_chain_history( + block.clone(), + &network, + &history_tree, + precomputed_auth_data_root, + ) + )); + }); + + timed_commit_phase!( + "zebra.state.write.update_trees.duration_seconds", + note_commitment_trees.update_trees_parallel(&block) + ) + }) + }); + + // Surface the tree-update error first, preserving the error + // precedence of the previous sequential code. + tree_result.map_err(ValidateContextError::from)?; + // `in_place_scope_fifo` joins all spawned tasks, so this is `Some`. + commitment_result.expect("scope has already finished")?; + + // Update the history tree (depends on both operations above). + let history_tree_mut = Arc::make_mut(&mut history_tree); + let sapling_root = note_commitment_trees.sapling.root(); + let orchard_root = note_commitment_trees.orchard.root(); + let ironwood_root = note_commitment_trees.ironwood.root(); + history_tree_mut + .push( + &network, + block.clone(), + &sapling_root, + &orchard_root, + &ironwood_root, + ) + .map_err(Arc::new) + .map_err(ValidateContextError::from)?; + + #[cfg(feature = "commit-metrics")] + metrics::histogram!( + "zebra.state.write.checkpoint_compute.duration_seconds" ) - .map_err(Arc::new) - .map_err(ValidateContextError::from)?; - - // Total serial wall time of the checkpoint compute phase (note tree - // update + commitment check, then history push). Compared against the - // summed phase times, this shows the overlap win. - #[cfg(feature = "commit-metrics")] - metrics::histogram!("zebra.state.write.checkpoint_compute.duration_seconds") .record(_ckpt_compute.elapsed().as_secs_f64()); + } let treestate = Treestate { note_commitment_trees, history_tree, }; - let height = checkpoint_verified.height; let hash = checkpoint_verified.hash; ( @@ -721,6 +983,7 @@ impl FinalizedState { FinalizedBlock::from_checkpoint_verified(checkpoint_verified, treestate), Some(prev_note_commitment_trees), self.retention_plan(height, true), + vct_write, ) } FinalizableBlock::Contextual { @@ -738,6 +1001,7 @@ impl FinalizedState { ), prev_note_commitment_trees, self.retention_plan(height, false), + VctWriteData::default(), ) } }; @@ -773,22 +1037,25 @@ impl FinalizedState { let finalized_inner_block = finalized.block.clone(); let note_commitment_trees = finalized.treestate.note_commitment_trees.clone(); - // Build and write the block's RocksDB batch inside the dedicated - // commit-compute pool. The par-iter calls inside write_block end up scheduled - // on a separate pool from global (which is used by download/verify pipeline). - // This leads to less contention and more throughput, as benchmarked over the - // sand-blasting region. + // Run `write_block` directly on the committer thread rather than entering the + // dedicated commit-compute pool via `install()`. + // + // The committer is not a member of `COMMIT_COMPUTE_POOL`, so `install()` is a + // synchronous cross-thread handoff: the committer parks until a pool worker + // picks up the job, runs it, and signals back. That wait can dominate the + // isolation it was meant to provide for `write_block`'s internal rayon + // (`join`/`par_iter`). Running `write_block` here removes the per-block + // round-trip; its internal rayon uses the global pool instead. Measured net + // win on the sandblast region (see PR). let network = self.network(); - let result = COMMIT_COMPUTE_POOL.install(|| { - self.db.write_block( - finalized, - prev_note_commitment_trees, - &network, - source, - retention, - None, - ) - }); + let result = self.db.write_block( + finalized, + prev_note_commitment_trees, + &network, + source, + retention, + fast_write, + ); if result.is_ok() { if retention.clears_archive_backlog() { @@ -809,6 +1076,9 @@ impl FinalizedState { "stopping at configured height, flushing database to disk" ); + // POC: emit the equivalence digest + fast-path summary before exit. + self.vct_log_equivalence_digest(); + // We're just about to do a forced exit, so it's ok to do a forced db shutdown self.db.shutdown(true); @@ -824,6 +1094,154 @@ impl FinalizedState { result.map(|hash| (hash, note_commitment_trees)) } + /// POC: `true` when the verified-commitment-trees fast (skip-recompute) path will + /// apply to `height` — i.e. fast mode is active *and* the source already holds this + /// height's roots, so the committer will fold them in and skip the frontier recompute. + pub(crate) fn vct_fast_will_apply(&self, height: block::Height) -> bool { + self.vct + .source() + .is_some_and(|v| v.is_enabled() && v.vct_roots_at_height(height).is_some()) + } + + /// Clears any cached successor prevalidation. + /// + /// The finalized write loop calls this when it discards checkpoint queue state, so a + /// look-ahead header that no longer corresponds to the next committed block cannot + /// authorize a later fast-path skip. + pub(crate) fn clear_vct_prevalidated_next(&mut self) { + self.vct.clear_prevalidated_next(); + } + + /// `true` when committing `height` on the fast path needs a buffered successor before + /// it can safely persist this block's supplied roots. + /// + /// Only untrusted peer-supplied roots at or above Heartwood require this. The + /// checkpoint handoff is exempt because its embedded final frontiers are verified + /// against this block's roots before the real tip treestate is written; trusted + /// local fixtures can commit their tip root on the in-arrears check. + pub(crate) fn vct_fast_needs_successor(&self, height: block::Height) -> bool { + self.vct + .source() + .is_some_and(|v| v.vct_root_needs_successor(height, &self.network())) + } + + /// Verify checkpoint handoff frontiers against this block's supplied roots. + fn vct_verify_handoff_frontier_roots( + &mut self, + height: block::Height, + sapling_frontier: &sapling::tree::NoteCommitmentTree, + orchard_frontier: &orchard::tree::NoteCommitmentTree, + sapling_root: &sapling::tree::Root, + orchard_root: &orchard::tree::Root, + ) -> Result<(), CommitCheckpointVerifiedError> { + if sapling_frontier.root() != *sapling_root || orchard_frontier.root() != *orchard_root { + self.vct.clear_prevalidated_next(); + return Err(self.vct_reject_supplied_root( + height, + ValidateContextError::VctSuppliedRootUnavailable { height }, + )); + } + + Ok(()) + } + + /// Reject a supplied fast-path root that failed verification for `height`. + /// + /// Evicts the bad root from the source so a re-fetch can replace it with a verifiable + /// one from a different peer, and returns a typed, retryable error. In fast mode the + /// note-commitment frontier is frozen, so the committer cannot recompute the root + /// locally (that would fold a wrong root into the history MMR); it must refuse and + /// leave the database untouched rather than persist or corrupt state. This is what + /// keeps a single malicious peer from halting the sync: the bad root is dropped, not + /// retried forever, and any honest peer's root verifies. + fn vct_reject_supplied_root( + &self, + height: block::Height, + error: ValidateContextError, + ) -> CommitCheckpointVerifiedError { + if let Some(v) = self.vct.source() { + v.invalidate_fast_root(height); + } + metrics::counter!("state.vct.root.rejected.count").increment(1); + tracing::warn!( + ?height, + ?error, + "VCT: supplied commitment root failed verification; evicted for re-fetch" + ); + ValidateContextError::VctSuppliedRootUnavailable { height }.into() + } + + /// Test-only: enable fast mode reading roots/frontiers from an arbitrary + /// [`commitment_aux::CommitmentRootSource`] (e.g. a payload produced from a + /// database via [`commitment_aux::produce_block_roots`]), so the producer→consumer + /// round-trip can be exercised in-process. `requires_verified_successor` marks + /// whether the installed source is untrusted and must defer tip roots until their + /// successor is buffered. + #[cfg(test)] + pub(in crate::service::finalized_state) fn enable_vct_fast_source( + &mut self, + source: Box, + requires_verified_successor: bool, + ) { + self.vct + .install_test_source(source, requires_verified_successor); + } + + /// Test-only: the fast-sync handoff height recorded in the database marker, if any. + #[cfg(test)] + pub(crate) fn vct_fast_synced_below(&self) -> Option { + self.db.vct_synced_below() + } + + /// Test-only: number of blocks that took the fast (skip-recompute) path so far. + #[cfg(test)] + pub(crate) fn vct_fast_count(&self) -> u64 { + self.vct.source().map(|v| v.vct_count()).unwrap_or(0) + } + + /// Test-only: number of fast blocks whose own commitment check was skipped by + /// the dedup (the previous block's look-ahead already validated them). + #[cfg(test)] + pub(crate) fn vct_prevalidated_count(&self) -> u64 { + self.vct + .source() + .map(|v| v.prevalidated_count()) + .unwrap_or(0) + } + + /// POC: log the consensus-equivalence digest (anchor sets + history root) and + /// the fast-path block count at the stop height, so a legacy run and a fast run + /// can be compared. Gated by `VCT_DIGEST` so normal runs pay nothing. + fn vct_log_equivalence_digest(&self) { + if std::env::var_os("VCT_DIGEST").is_none() { + return; + } + + let fast_count = if let Some(v) = self.vct.source() { + v.vct_count() + } else { + 0 + }; + + let ( + sapling_anchor_count, + sapling_anchor_digest, + orchard_anchor_count, + orchard_anchor_digest, + ) = self.db.vct_anchor_digest(); + let history_root = self.db.history_tree().hash(); + + tracing::info!( + sapling_anchor_count, + sapling_anchor_digest, + orchard_anchor_count, + orchard_anchor_digest, + ?history_root, + vct_fast_blocks = fast_count, + "VCT-DIGEST" + ); + } + #[cfg(feature = "elasticsearch")] /// Store finalized blocks into an elasticsearch database. /// diff --git a/zebra-state/src/service/finalized_state/disk_format/chain.rs b/zebra-state/src/service/finalized_state/disk_format/chain.rs index ae86b025eb5..39e5e0e136e 100644 --- a/zebra-state/src/service/finalized_state/disk_format/chain.rs +++ b/zebra-state/src/service/finalized_state/disk_format/chain.rs @@ -133,13 +133,6 @@ impl FromDisk for HistoryTreeParts { let bytes = bytes.as_ref(); let options = bincode::DefaultOptions::new(); - // Try the current entry width first. Databases written before NU6.3 widened - // `zcash_history::Entry` store narrower entries that fail to parse at the current width, - // so fall back to the legacy width and zero-pad each entry up to the current width. - // - // Legacy-width rows can fail the current-width decoder with errors other than - // `UnexpectedEof`, because the wider entry can read into the next legacy entry and - // interpret arbitrary entry bytes as bincode control bytes. options .deserialize::(bytes) .or_else(|_| { diff --git a/zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs b/zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs index fe1c390a76f..6b3436fc2ef 100644 --- a/zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs +++ b/zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs @@ -96,7 +96,7 @@ fn test_raw_rocksdb_column_families_with_network(network: Network) { .expect("test data deserializes"); state - .commit_finalized_direct(block.into(), None, "snapshot tests") + .commit_finalized_direct(block.into(), None, None, "snapshot tests") .expect("test block is valid"); let mut settings = insta::Settings::clone_current(); diff --git a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/column_family_names.snap b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/column_family_names.snap index 800761c0823..7ed109ef870 100644 --- a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/column_family_names.snap +++ b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/column_family_names.snap @@ -40,7 +40,6 @@ expression: cf_names "vct_upgrade_metadata", "zakura_header_body_size_by_height", "zakura_header_by_height", - "zakura_header_commitment_roots_by_height", "zakura_header_hash_by_height", "zakura_header_height_by_hash", ] diff --git a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@mainnet_0.snap b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@mainnet_0.snap index 83108b6f06b..28dcbb6cc7d 100644 --- a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@mainnet_0.snap +++ b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@mainnet_0.snap @@ -20,7 +20,6 @@ expression: empty_column_families "vct_sync_metadata: no entries", "zakura_header_body_size_by_height: no entries", "zakura_header_by_height: no entries", - "zakura_header_commitment_roots_by_height: no entries", "zakura_header_hash_by_height: no entries", "zakura_header_height_by_hash: no entries", ] diff --git a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@mainnet_1.snap b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@mainnet_1.snap index 305e16179ba..025934c3d39 100644 --- a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@mainnet_1.snap +++ b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@mainnet_1.snap @@ -16,7 +16,6 @@ expression: empty_column_families "vct_sync_metadata: no entries", "zakura_header_body_size_by_height: no entries", "zakura_header_by_height: no entries", - "zakura_header_commitment_roots_by_height: no entries", "zakura_header_hash_by_height: no entries", "zakura_header_height_by_hash: no entries", ] diff --git a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@mainnet_2.snap b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@mainnet_2.snap index 305e16179ba..025934c3d39 100644 --- a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@mainnet_2.snap +++ b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@mainnet_2.snap @@ -16,7 +16,6 @@ expression: empty_column_families "vct_sync_metadata: no entries", "zakura_header_body_size_by_height: no entries", "zakura_header_by_height: no entries", - "zakura_header_commitment_roots_by_height: no entries", "zakura_header_hash_by_height: no entries", "zakura_header_height_by_hash: no entries", ] diff --git a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@no_blocks.snap b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@no_blocks.snap index 9a2a2e6bffb..92dd3a3983d 100644 --- a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@no_blocks.snap +++ b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@no_blocks.snap @@ -38,7 +38,6 @@ expression: empty_column_families "vct_upgrade_metadata: no entries", "zakura_header_body_size_by_height: no entries", "zakura_header_by_height: no entries", - "zakura_header_commitment_roots_by_height: no entries", "zakura_header_hash_by_height: no entries", "zakura_header_height_by_hash: no entries", ] diff --git a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@testnet_0.snap b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@testnet_0.snap index 83108b6f06b..28dcbb6cc7d 100644 --- a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@testnet_0.snap +++ b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@testnet_0.snap @@ -20,7 +20,6 @@ expression: empty_column_families "vct_sync_metadata: no entries", "zakura_header_body_size_by_height: no entries", "zakura_header_by_height: no entries", - "zakura_header_commitment_roots_by_height: no entries", "zakura_header_hash_by_height: no entries", "zakura_header_height_by_hash: no entries", ] diff --git a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@testnet_1.snap b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@testnet_1.snap index 305e16179ba..025934c3d39 100644 --- a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@testnet_1.snap +++ b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@testnet_1.snap @@ -16,7 +16,6 @@ expression: empty_column_families "vct_sync_metadata: no entries", "zakura_header_body_size_by_height: no entries", "zakura_header_by_height: no entries", - "zakura_header_commitment_roots_by_height: no entries", "zakura_header_hash_by_height: no entries", "zakura_header_height_by_hash: no entries", ] diff --git a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@testnet_2.snap b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@testnet_2.snap index 305e16179ba..025934c3d39 100644 --- a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@testnet_2.snap +++ b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@testnet_2.snap @@ -16,7 +16,6 @@ expression: empty_column_families "vct_sync_metadata: no entries", "zakura_header_body_size_by_height: no entries", "zakura_header_by_height: no entries", - "zakura_header_commitment_roots_by_height: no entries", "zakura_header_hash_by_height: no entries", "zakura_header_height_by_hash: no entries", ] diff --git a/zebra-state/src/service/finalized_state/tests/prop.rs b/zebra-state/src/service/finalized_state/tests/prop.rs index 930014d4a3c..5a6a39d5f69 100644 --- a/zebra-state/src/service/finalized_state/tests/prop.rs +++ b/zebra-state/src/service/finalized_state/tests/prop.rs @@ -1,9 +1,12 @@ //! Randomised property tests for the finalized state. -use std::{env, error::Error}; +use std::{collections::HashMap, env, error::Error, fs, sync::Arc}; + +use tempfile::TempDir; +use tokio::sync::oneshot; use zebra_chain::{ - block::Height, + block::{Block, Height}, parameters::{ testnet::{ConfiguredActivationHeights, ParametersBuilder}, NetworkUpgrade, @@ -13,16 +16,160 @@ use zebra_chain::{ use zebra_test::prelude::*; use crate::{ - config::Config, - service::{ - arbitrary::PreparedChain, - finalized_state::{CheckpointVerifiedBlock, FinalizedState}, - }, - tests::FakeChainHelper, + config::Config, service::arbitrary::PreparedChain, tests::FakeChainHelper, HashOrHeight, +}; + +use super::super::{ + commitment_aux, serve_block_roots, vct::validate_final_frontiers_bytes, + CheckpointVerifiedBlock, DiskWriteBatch, FinalizedState, NextVctBlock, }; const DEFAULT_PARTIAL_CHAIN_PROPTEST_CASES: u32 = 1; +type TestRootMap = HashMap< + u32, + ( + zebra_chain::sapling::tree::Root, + zebra_chain::orchard::tree::Root, + ), +>; +type SaplingTree = Arc; +type OrchardTree = Arc; +type SproutTree = Arc; + +fn next_vct_block(block: Arc) -> Option { + Some(NextVctBlock { + block, + auth_data_root: None, + }) +} + +/// A handoff frontier over empty trees at `height`, for sources whose test does not +/// exercise the handoff itself. The frontier is mandatory on every source; placing it +/// above every height a test commits keeps all roots fast-path eligible and never +/// engages the handoff behaviors (bounding, treestate write, successor exemption). +fn test_handoff_frontiers(height: Height) -> commitment_aux::FinalFrontiers { + commitment_aux::FinalFrontiers { + height, + sapling: Arc::new(Default::default()), + orchard: Arc::new(Default::default()), + sprout: Arc::new(Default::default()), + } +} + +fn enable_vct_test_fixture_source(state: &mut FinalizedState, roots: TestRootMap) { + state.enable_vct_fast_source( + Box::new(commitment_aux::FixtureSource::new( + roots, + test_handoff_frontiers(Height::MAX), + )), + false, + ); +} + +fn enable_vct_test_fixture_source_with_handoff( + state: &mut FinalizedState, + roots: TestRootMap, + handoff_height: Height, + sapling: SaplingTree, + orchard: OrchardTree, + sprout: SproutTree, +) { + state.enable_vct_fast_source( + Box::new(commitment_aux::FixtureSource::new( + roots, + commitment_aux::FinalFrontiers { + height: handoff_height, + sapling, + orchard, + sprout, + }, + )), + false, + ); +} + +#[test] +fn vct_generated_final_frontier_bytes_are_node_loader_compatible() -> Result<()> { + let _init_guard = zebra_test::init(); + + let network = ParametersBuilder::default() + .with_activation_heights(ConfiguredActivationHeights { + before_overwinter: Some(1), + overwinter: Some(10), + sapling: Some(15), + blossom: Some(20), + heartwood: Some(25), + canopy: Some(30), + nu5: Some(35), + nu6: Some(40), + nu6_1: Some(45), + nu6_2: Some(47), + nu6_3: Some(48), + nu7: Some(50), + }) + .expect("failed to set activation heights") + .extend_funding_streams() + .to_network() + .expect("failed to build configured network"); + let ledger_strategy = + LedgerState::genesis_strategy(Some(network), None::, None, false); + + proptest!(ProptestConfig::with_cases(1), + |((chain, _count, network, _history_tree) in PreparedChain::default().with_ledger_strategy(ledger_strategy.clone()).with_valid_commitments().no_shrink())| { + let blocks: Vec<_> = chain.iter().collect(); + let nu5 = NetworkUpgrade::Nu5.activation_height(&network).unwrap().0; + let last = (nu5 + 3) as usize; + prop_assert!(blocks.len() > last, "generated chain unexpectedly short"); + let height = Height(last as u32); + + let mut legacy = FinalizedState::new(&Config::ephemeral(), &network, #[cfg(feature = "elasticsearch")] false); + for block in blocks.iter().take(last + 1) { + let cv = CheckpointVerifiedBlock::from(block.block.clone()); + legacy + .commit_finalized_direct(cv.into(), None, None, "vct frontier bytes legacy") + .unwrap(); + } + + let bytes = commitment_aux::produce_final_frontiers_bytes(&legacy.db, height) + .expect("legacy DB has final frontiers at the requested height"); + let temp_dir = TempDir::new().expect("temp dir"); + let path = temp_dir.path().join("frontier.bin"); + fs::write(&path, &bytes).expect("frontier bytes write to temp file"); + + let bytes_from_file = fs::read(&path).expect("frontier bytes read from temp file"); + validate_final_frontiers_bytes(&bytes_from_file, height) + .expect("generated frontier bytes pass node loader validation"); + + let parsed = commitment_aux::FinalFrontiers::from_bytes(&bytes_from_file) + .expect("validated bytes parse as final frontiers"); + prop_assert_eq!(parsed.height, height, "frontier height round-trips"); + prop_assert_eq!( + parsed.sapling.root(), + legacy.db.sapling_tree_by_height(&height).unwrap().root(), + "parsed Sapling frontier matches the DB tree at the requested height" + ); + prop_assert_eq!( + parsed.orchard.root(), + legacy.db.orchard_tree_by_height(&height).unwrap().root(), + "parsed Orchard frontier matches the DB tree at the requested height" + ); + prop_assert_eq!( + parsed.sprout.root(), + legacy.db.sprout_tree_for_tip().root(), + "parsed Sprout frontier matches the DB tip tree" + ); + + let wrong_height = Height(height.0.checked_add(1).expect("test height is in range")); + prop_assert!( + validate_final_frontiers_bytes(&bytes_from_file, wrong_height).is_err(), + "node loader validation rejects a frontier whose height does not match the checkpoint" + ); + }); + + Ok(()) +} + #[test] fn blocks_with_v5_transactions() -> Result<()> { let _init_guard = zebra_test::init(); @@ -39,6 +186,7 @@ fn blocks_with_v5_transactions() -> Result<()> { let (hash, _) = state.commit_finalized_direct( checkpoint_verified.into(), None, + None, "blocks_with_v5_transactions test" ).unwrap(); prop_assert_eq!(Some(height), state.finalized_tip_height()); @@ -115,6 +263,7 @@ fn all_upgrades_and_wrong_commitments_with_fake_activation_heights() -> Result<( state.commit_finalized_direct( checkpoint_verified.into(), None, + None, "all_upgrades test" ).expect_err("Must fail commitment check"); failure_count += 1; @@ -124,10 +273,11 @@ fn all_upgrades_and_wrong_commitments_with_fake_activation_heights() -> Result<( if current_height == nu5_height_plus1 { let mut checkpoint_verified = CheckpointVerifiedBlock::from(block.block.clone()); - checkpoint_verified.auth_data_root = Some([0x42; 32].into()); + checkpoint_verified.0.auth_data_root = Some([0x42; 32].into()); let err = state.commit_finalized_direct( checkpoint_verified.into(), None, + None, "all_upgrades bad auth root test" ).expect_err("Must fail when the supplied auth data root is incorrect"); let commit_error = err @@ -155,6 +305,7 @@ fn all_upgrades_and_wrong_commitments_with_fake_activation_heights() -> Result<( let (hash, _) = state.commit_finalized_direct( checkpoint_verified.into(), None, + None, "all_upgrades test" ).unwrap(); prop_assert_eq!(Some(height), state.finalized_tip_height()); @@ -168,3 +319,1702 @@ fn all_upgrades_and_wrong_commitments_with_fake_activation_heights() -> Result<( Ok(()) } + +/// Verified-commitment-trees fast path (`commit_finalized_direct` Checkpoint arm): +/// committing with correct fixture roots produces the same consensus state (anchor +/// sets + history root) as the legacy recompute path across all upgrade boundaries, +/// and a wrong fixture root is rejected (verify-before-commit) rather than persisted. +/// Exercises: a below-Heartwood seed, history-tree creation at Heartwood, the NU5 +/// V1->V2 transition, verify-ahead against the buffered successor, trusted fixture tip +/// commits without a successor, and rejection of a corrupted root. +#[test] +#[allow(clippy::needless_range_loop)] // the loops index blocks[i+1] and the fixture by height +fn vct_fast_path_matches_legacy_and_rejects_wrong_roots() -> Result<()> { + let _init_guard = zebra_test::init(); + + let network = ParametersBuilder::default() + .with_activation_heights(ConfiguredActivationHeights { + before_overwinter: Some(1), + overwinter: Some(10), + sapling: Some(15), + blossom: Some(20), + heartwood: Some(25), + canopy: Some(30), + nu5: Some(35), + nu6: Some(40), + nu6_1: Some(45), + nu6_2: Some(47), + nu6_3: Some(48), + nu7: Some(50), + }) + .expect("failed to set activation heights") + .extend_funding_streams() + .to_network() + .expect("failed to build configured network"); + let ledger_strategy = + LedgerState::genesis_strategy(Some(network), None::, None, false); + + proptest!(ProptestConfig::with_cases(env::var("PROPTEST_CASES") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(DEFAULT_PARTIAL_CHAIN_PROPTEST_CASES)), + |((chain, _count, network, _history_tree) in PreparedChain::default().with_ledger_strategy(ledger_strategy.clone()).with_valid_commitments().no_shrink())| { + + let blocks: Vec<_> = chain.iter().collect(); + let nu5 = NetworkUpgrade::Nu5.activation_height(&network).unwrap().0; + let heartwood = NetworkUpgrade::Heartwood.activation_height(&network).unwrap().0; + + // Process a bounded prefix [0, last] spanning the Heartwood (history-tree + // creation) and NU5 (V1->V2) boundaries plus a couple of V2 blocks; `last` is + // the tip we compare at. Chains are far longer than this + // (MAX_PARTIAL_CHAIN_BLOCKS), so this is a plain assertion, not a discard. + let last = (nu5 + 3) as usize; + prop_assert!(blocks.len() > last + 1, "generated chain unexpectedly short"); + + // The fast path runs below the checkpoint, seeded from an already-committed + // tip. Seed just before Heartwood so the fast range creates the history tree + // (Heartwood) and crosses NU5 (V1->V2). + let seed = (heartwood - 1) as usize; + + // Legacy pass over [0, last]: record per-block roots for the fast range as + // the fixture, and the golden consensus state at the tip. + let mut legacy = FinalizedState::new(&Config::ephemeral(), &network, #[cfg(feature = "elasticsearch")] false); + let mut fixture = std::collections::HashMap::new(); + for i in 0..=last { + let cv = CheckpointVerifiedBlock::from(blocks[i].block.clone()); + let (_h, trees) = legacy + .commit_finalized_direct(cv.into(), None, None, "vct legacy") + .unwrap(); + if i > seed { + fixture.insert(i as u32, (trees.sapling.root(), trees.orchard.root())); + } + } + let golden_anchors = legacy.db.vct_anchor_digest(); + let golden_history = legacy.db.history_tree().hash(); + + // Fast pass over [0, last] with the correct fixture: genesis..=seed recompute + // (no fixture entry); seed+1..=last verify-ahead against their buffered + // successor. Every fast-eligible block takes the fast path, and the result + // equals legacy. + let mut fast = FinalizedState::new(&Config::ephemeral(), &network, #[cfg(feature = "elasticsearch")] false); + enable_vct_test_fixture_source(&mut fast, fixture.clone()); + for i in 0..=last { + let cv = CheckpointVerifiedBlock::from(blocks[i].block.clone()); + let next = next_vct_block(blocks[i + 1].block.clone()); + fast.commit_finalized_direct(cv.into(), None, next, "vct fast") + .expect("verified fast commit succeeds"); + } + prop_assert_eq!(fast.db.vct_anchor_digest(), golden_anchors, "fast anchors must match legacy"); + prop_assert_eq!(fast.db.history_tree().hash(), golden_history, "fast history must match legacy"); + prop_assert_eq!(fast.vct_fast_count(), (last - seed) as u64, "every fast-eligible block took the fast path"); + // The dedup: each header commitment is checked once, not twice. Only the + // first fast block runs its own commitment check; every later fast block + // was already validated by its predecessor's look-ahead, so it is skipped. + prop_assert_eq!(fast.vct_prevalidated_count(), (last - seed - 1) as u64, "every fast block after the first skips its redundant own commitment check"); + + // A trusted local fixture may commit its tip root without a successor: it is + // not adversarial and the root is checked in arrears when a successor arrives. + let mut no_successor = FinalizedState::new(&Config::ephemeral(), &network, #[cfg(feature = "elasticsearch")] false); + enable_vct_test_fixture_source(&mut no_successor, fixture.clone()); + for i in 0..last { + let cv = CheckpointVerifiedBlock::from(blocks[i].block.clone()); + let next = next_vct_block(blocks[i + 1].block.clone()); + no_successor + .commit_finalized_direct(cv.into(), None, next, "vct no-successor seed") + .expect("verified fast commit succeeds with successor"); + } + prop_assert!(!no_successor.vct_fast_needs_successor(Height(last as u32)), "a trusted fixture tip can commit without a successor"); + let cv = CheckpointVerifiedBlock::from(blocks[last].block.clone()); + no_successor + .commit_finalized_direct(cv.into(), None, None, "vct trusted fixture no successor") + .expect("trusted fixture tip commits without a successor"); + prop_assert_eq!( + no_successor.db.finalized_tip_height(), + Some(Height(last as u32)), + "the trusted fixture tip committed" + ); + + // Negative: corrupt the fixture Sapling root at a V2 (post-NU5) height with a + // distinct value (the empty root; a V2 block has a non-empty Sapling tree). + // Fast mode cannot recompute a bad root away (the frontier is frozen), so the + // wrong root must be *rejected* by the next block's commitment (verify-before- + // commit) — the commit at that height fails rather than persisting it. + let bad_height = (nu5 + 1) as usize; + let mut bad_fixture = fixture.clone(); + let bad_entry = bad_fixture.get_mut(&(bad_height as u32)).unwrap(); + prop_assert_ne!(bad_entry.0, Default::default(), "a V2 block must have a non-empty Sapling root"); + bad_entry.0 = Default::default(); + + let mut bad = FinalizedState::new(&Config::ephemeral(), &network, #[cfg(feature = "elasticsearch")] false); + enable_vct_test_fixture_source(&mut bad, bad_fixture); + let mut error_height = None; + for i in 0..=last { + let cv = CheckpointVerifiedBlock::from(blocks[i].block.clone()); + let next = next_vct_block(blocks[i + 1].block.clone()); + if bad.commit_finalized_direct(cv.into(), None, next, "vct bad").is_err() { + error_height = Some(i); + break; + } + } + prop_assert_eq!(error_height, Some(bad_height), "a wrong fixture root is rejected at its own commit"); + + // Negative (Orchard, below NU5): no header commits to an Orchard root below + // NU5 (V1 history leaves ignore it; no MMR below Heartwood), so the fast path + // pins it to the empty-tree root. Corrupt a below-NU5 fixture Orchard root to + // a non-empty value. Unlike the Sapling MMR path (one-block lag), this is a + // direct check, so it is rejected at the block's *own* commit — closing the + // hole where an untrusted source injects a spurious Orchard anchor. + let bad_orchard_height = (nu5 - 1) as usize; + prop_assert!(bad_orchard_height > seed, "the corrupted height must be in the fast range"); + let empty_orchard = zebra_chain::orchard::tree::NoteCommitmentTree::default().root(); + let wrong_orchard = zebra_chain::orchard::tree::Root::try_from([0u8; 32]) + .expect("zero is a valid pallas base field element"); + prop_assert_ne!(wrong_orchard, empty_orchard, "the wrong root must differ from the empty-tree root"); + + let mut bad_orchard_fixture = fixture.clone(); + let bad_orchard_entry = bad_orchard_fixture.get_mut(&(bad_orchard_height as u32)).unwrap(); + prop_assert_eq!(bad_orchard_entry.1, empty_orchard, "a below-NU5 block has the empty Orchard root"); + bad_orchard_entry.1 = wrong_orchard; + + let mut bad_orchard = FinalizedState::new(&Config::ephemeral(), &network, #[cfg(feature = "elasticsearch")] false); + enable_vct_test_fixture_source(&mut bad_orchard, bad_orchard_fixture); + let mut orchard_error_height = None; + for i in 0..=last { + let cv = CheckpointVerifiedBlock::from(blocks[i].block.clone()); + let next = next_vct_block(blocks[i + 1].block.clone()); + if bad_orchard.commit_finalized_direct(cv.into(), None, next, "vct bad orchard").is_err() { + orchard_error_height = Some(i); + break; + } + } + prop_assert_eq!(orchard_error_height, Some(bad_orchard_height), "a wrong below-NU5 orchard root is rejected at its own commit"); + }); + + Ok(()) +} + +/// A verified-commitment-trees fast sync must never legacy-recompute a height whose +/// supplied root is missing once the note-commitment frontier is frozen: the running +/// frontier is no longer the real one, so recomputing would fold a wrong root into the +/// history MMR and silently corrupt consensus state (a peer that omits a height — see the +/// driver's gap handling — could trigger this). Instead the committer must refuse with the +/// retryable `VctSuppliedRootUnavailable` error and leave the database untouched, so the +/// block can be committed later from a fetched root. This guards the liveness/no-corruption +/// half of the peer-source fast path (the bad-root rejection half is covered by +/// `vct_fast_path_matches_legacy_and_rejects_wrong_roots`). +#[test] +#[allow(clippy::needless_range_loop)] // the loop indexes blocks[i+1] and the fixture by height +fn vct_frozen_frontier_hole_refuses_instead_of_recomputing() -> Result<()> { + let _init_guard = zebra_test::init(); + + let network = ParametersBuilder::default() + .with_activation_heights(ConfiguredActivationHeights { + before_overwinter: Some(1), + overwinter: Some(10), + sapling: Some(15), + blossom: Some(20), + heartwood: Some(25), + canopy: Some(30), + nu5: Some(35), + nu6: Some(40), + nu6_1: Some(45), + nu6_2: Some(47), + nu6_3: Some(48), + nu7: Some(50), + }) + .expect("failed to set activation heights") + .extend_funding_streams() + .to_network() + .expect("failed to build configured network"); + let ledger_strategy = + LedgerState::genesis_strategy(Some(network), None::, None, false); + + proptest!(ProptestConfig::with_cases(1), + |((chain, _count, network, _history_tree) in PreparedChain::default().with_ledger_strategy(ledger_strategy.clone()).with_valid_commitments().no_shrink())| { + + let blocks: Vec<_> = chain.iter().collect(); + let nu5 = NetworkUpgrade::Nu5.activation_height(&network).unwrap().0; + let heartwood = NetworkUpgrade::Heartwood.activation_height(&network).unwrap().0; + let last = (nu5 + 3) as usize; + prop_assert!(blocks.len() > last + 1, "generated chain unexpectedly short"); + let seed = (heartwood - 1) as usize; + + // Record the per-block roots for the fast range as the fixture. + let mut legacy = FinalizedState::new(&Config::ephemeral(), &network, #[cfg(feature = "elasticsearch")] false); + let mut fixture = std::collections::HashMap::new(); + for i in 0..=last { + let cv = CheckpointVerifiedBlock::from(blocks[i].block.clone()); + let (_h, trees) = legacy + .commit_finalized_direct(cv.into(), None, None, "vct hole legacy") + .unwrap(); + if i > seed { + fixture.insert(i as u32, (trees.sapling.root(), trees.orchard.root())); + } + } + + // Punch a hole: drop a post-NU5 height's root from the fixture, simulating a + // peer that omitted it (or a root evicted after failing verification). Earlier + // fast blocks freeze the frontier, so this height has no real frontier to + // recompute against. + let hole = (nu5 + 1) as usize; + prop_assert!(hole > seed && hole < last, "the hole must be inside the fast range"); + fixture.remove(&(hole as u32)); + + let mut fast = FinalizedState::new(&Config::ephemeral(), &network, #[cfg(feature = "elasticsearch")] false); + enable_vct_test_fixture_source(&mut fast, fixture); + + let mut error_height = None; + for i in 0..=last { + let cv = CheckpointVerifiedBlock::from(blocks[i].block.clone()); + let next = (i < last).then(|| NextVctBlock { + block: blocks[i + 1].block.clone(), + auth_data_root: None, + }); + match fast.commit_finalized_direct(cv.into(), None, next, "vct hole fast") { + Ok(_) => {} + Err(error) => { + // The refusal is the typed, retryable error — not a generic + // invalid-block error and not silent corruption. + prop_assert!( + format!("{error:?}").contains("VctSuppliedRootUnavailable"), + "a frozen-frontier hole returns the retryable VctSuppliedRootUnavailable error, got: {error:?}" + ); + error_height = Some(i); + break; + } + } + } + + prop_assert_eq!(error_height, Some(hole), "the commit refuses at the hole height, not before or after"); + // Nothing at or past the hole was persisted: the tip is the last block before + // the hole, so no corrupt MMR leaf was written. + prop_assert_eq!( + fast.db.finalized_tip_height(), + Some(Height((hole - 1) as u32)), + "the database tip stays just below the hole — the refused block left state untouched" + ); + }); + + Ok(()) +} + +/// Retryable VCT root misses must stay internal to the finalized write loop: the +/// public checkpoint commit wrapper returns the queued block and error to the caller +/// that can retry, rather than completing the block's response channel with a +/// transient error. +#[test] +#[allow(clippy::needless_range_loop)] // the loop indexes blocks[i+1] and the fixture by height +fn vct_retryable_root_miss_keeps_checkpoint_response_pending() -> Result<()> { + let _init_guard = zebra_test::init(); + + let network = ParametersBuilder::default() + .with_activation_heights(ConfiguredActivationHeights { + before_overwinter: Some(1), + overwinter: Some(10), + sapling: Some(15), + blossom: Some(20), + heartwood: Some(25), + canopy: Some(30), + nu5: Some(35), + nu6: Some(40), + nu6_1: Some(45), + nu6_2: Some(47), + nu6_3: Some(48), + nu7: Some(50), + }) + .expect("failed to set activation heights") + .extend_funding_streams() + .to_network() + .expect("failed to build configured network"); + let ledger_strategy = + LedgerState::genesis_strategy(Some(network), None::, None, false); + + proptest!(ProptestConfig::with_cases(1), + |((chain, _count, network, _history_tree) in PreparedChain::default().with_ledger_strategy(ledger_strategy.clone()).with_valid_commitments().no_shrink())| { + + let blocks: Vec<_> = chain.iter().collect(); + let nu5 = NetworkUpgrade::Nu5.activation_height(&network).unwrap().0; + let heartwood = NetworkUpgrade::Heartwood.activation_height(&network).unwrap().0; + let last = (nu5 + 3) as usize; + prop_assert!(blocks.len() > last, "generated chain unexpectedly short"); + let seed = (heartwood - 1) as usize; + + let mut legacy = FinalizedState::new(&Config::ephemeral(), &network, #[cfg(feature = "elasticsearch")] false); + let mut fixture = std::collections::HashMap::new(); + for i in 0..=last { + let cv = CheckpointVerifiedBlock::from(blocks[i].block.clone()); + let (_h, trees) = legacy + .commit_finalized_direct(cv.into(), None, None, "vct response legacy") + .unwrap(); + if i > seed { + fixture.insert(i as u32, (trees.sapling.root(), trees.orchard.root())); + } + } + + let hole = (nu5 + 1) as usize; + fixture.remove(&(hole as u32)); + + let mut fast = FinalizedState::new(&Config::ephemeral(), &network, #[cfg(feature = "elasticsearch")] false); + enable_vct_test_fixture_source(&mut fast, fixture); + + for i in 0..hole { + let cv = CheckpointVerifiedBlock::from(blocks[i].block.clone()); + let next = next_vct_block(blocks[i + 1].block.clone()); + fast.commit_finalized_direct(cv.into(), None, next, "vct response fast") + .expect("pre-hole fast commits succeed"); + } + + let cv = CheckpointVerifiedBlock::from(blocks[hole].block.clone()); + let (rsp_tx, mut rsp_rx) = oneshot::channel(); + let next = next_vct_block(blocks[hole + 1].block.clone()); + let result = fast.commit_finalized((cv, rsp_tx), None, next); + let Err((returned_block, error)) = result else { + panic!("missing frozen-frontier root should return the queued block for retry"); + }; + + prop_assert_eq!(returned_block.0.height, Height(hole as u32)); + prop_assert!( + error.vct_supplied_root_unavailable_height().is_some(), + "the returned error is the typed retryable VCT root miss" + ); + prop_assert!( + matches!(rsp_rx.try_recv(), Err(oneshot::error::TryRecvError::Empty)), + "the checkpoint response stays pending so the write loop can retry internally" + ); + }); + + Ok(()) +} + +/// An *untrusted* (peer) source must never commit a fast block whose own supplied root has +/// no buffered successor to confirm it against the header chain. A block's roots are only +/// committed by the next block's header (the one-block lag), so committing at the sync tip +/// would persist a root checked only one block later — irreversibly, once on disk. A wrong +/// tip root would then wedge the sync with no recovery (the failure surfaces at the next +/// block and is mis-attributed to *its* root). So the committer defers: it refuses the tip +/// block with the retryable `VctSuppliedRootAwaitingSuccessor`, leaves the database +/// untouched, and commits the same height once a successor is buffered. A trusted local +/// fixture is exempt (covered by `vct_fast_path_matches_legacy_and_rejects_wrong_roots`, +/// whose tip commits on the in-arrears check); this guards the peer path specifically. +#[test] +#[allow(clippy::needless_range_loop)] // the loop indexes blocks[i+1] and inserts roots by height +fn vct_peer_source_defers_unverifiable_tip_root_until_successor() -> Result<()> { + use crate::service::finalized_state::commitment_aux::PeerSource; + use zebra_chain::parallel::commitment_aux::BlockCommitmentRoots; + + let _init_guard = zebra_test::init(); + + let network = ParametersBuilder::default() + .with_activation_heights(ConfiguredActivationHeights { + before_overwinter: Some(1), + overwinter: Some(10), + sapling: Some(15), + blossom: Some(20), + heartwood: Some(25), + canopy: Some(30), + nu5: Some(35), + nu6: Some(40), + nu6_1: Some(45), + nu6_2: Some(47), + nu6_3: Some(48), + nu7: Some(50), + }) + .expect("failed to set activation heights") + .extend_funding_streams() + .to_network() + .expect("failed to build configured network"); + let ledger_strategy = + LedgerState::genesis_strategy(Some(network), None::, None, false); + + proptest!(ProptestConfig::with_cases(1), + |((chain, _count, network, _history_tree) in PreparedChain::default().with_ledger_strategy(ledger_strategy.clone()).with_valid_commitments().no_shrink())| { + + let blocks: Vec<_> = chain.iter().collect(); + let nu5 = NetworkUpgrade::Nu5.activation_height(&network).unwrap().0; + let heartwood = NetworkUpgrade::Heartwood.activation_height(&network).unwrap().0; + // The deferral target: a post-NU5 (real MMR root) height, so it sits above + // Heartwood where the root needs a successor to be confirmed. + let tip_target = (nu5 + 1) as usize; + prop_assert!(blocks.len() > tip_target + 1, "generated chain unexpectedly short"); + let seed = (heartwood - 1) as usize; + + // Legacy golden pass to source the correct per-block roots for the fast range. + let mut legacy = FinalizedState::new(&Config::ephemeral(), &network, #[cfg(feature = "elasticsearch")] false); + let mut peer_roots = Vec::new(); + for i in 0..=tip_target { + let cv = CheckpointVerifiedBlock::from(blocks[i].block.clone()); + let (_h, trees) = legacy + .commit_finalized_direct(cv.into(), None, None, "vct defer legacy") + .unwrap(); + if i > seed { + peer_roots.push(BlockCommitmentRoots { + height: Height(i as u32), + sapling_root: trees.sapling.root(), + orchard_root: trees.orchard.root(), + ironwood_root: zebra_chain::ironwood::tree::NoteCommitmentTree::default().root(), + sapling_tx: 0, + orchard_tx: 0, + ironwood_tx: 0, + auth_data_root: blocks[i].block.auth_data_root(), + }); + } + } + + // An untrusted peer source pre-filled with the *correct* roots: the deferral is + // about the missing successor, not a bad root. The roots are persisted into the + // fast state's own database through the same header-sync write path production + // uses, and the peer source reads them back from that database. + let mut fast = FinalizedState::new(&Config::ephemeral(), &network, #[cfg(feature = "elasticsearch")] false); + fast.db + .insert_zakura_header_commitment_roots(peer_roots) + .expect("writing header-sync roots to an ephemeral database succeeds"); + let source = PeerSource::new(fast.db.clone(), test_handoff_frontiers(Height::MAX)); + fast.enable_vct_fast_source(Box::new(source), true); + + // Commit up to (but not including) the tip target, each with its successor. + for i in 0..tip_target { + let cv = CheckpointVerifiedBlock::from(blocks[i].block.clone()); + let next = next_vct_block(blocks[i + 1].block.clone()); + fast.commit_finalized_direct(cv.into(), None, next, "vct defer pre-tip") + .expect("pre-tip fast commits succeed"); + } + prop_assert_eq!(fast.db.finalized_tip_height(), Some(Height((tip_target - 1) as u32))); + + // The tip target with no buffered successor must defer, not commit: its own + // (correct) root is not yet confirmed, and the peer source is untrusted. + prop_assert!( + fast.vct_fast_needs_successor(Height(tip_target as u32)), + "an untrusted peer tip root needs successor verification" + ); + let pre_deferral_prevalidated = fast.vct_prevalidated_count(); + let cv = CheckpointVerifiedBlock::from(blocks[tip_target].block.clone()); + let error = fast + .commit_finalized_direct(cv.into(), None, None, "vct defer tip no successor") + .expect_err("an untrusted tip root with no successor must defer, not commit"); + prop_assert!( + error.vct_supplied_root_unavailable_height().is_none(), + "deferral is not a refetch case (the root is present): {error:?}" + ); + prop_assert!( + format!("{error:?}").contains("VctSuppliedRootAwaitingSuccessor"), + "the tip defers with the await-successor error, got: {error:?}" + ); + prop_assert_eq!( + fast.db.finalized_tip_height(), + Some(Height((tip_target - 1) as u32)), + "the deferred block left the database untouched" + ); + let after_deferral_prevalidated = fast.vct_prevalidated_count(); + prop_assert_eq!( + after_deferral_prevalidated, + pre_deferral_prevalidated + 1, + "the deferred attempt uses the predecessor look-ahead" + ); + + // Defense in depth: a witness that does not link to the block being committed + // (here, the block itself — its parent is the previous height) must be ignored + // and deferred exactly like a missing successor. It must *not* be treated as a + // verification failure: that would evict the correct root and, because the write + // loop's parked retry is taken before the look-ahead, wedge the retry loop. + let cv = CheckpointVerifiedBlock::from(blocks[tip_target].block.clone()); + let forged_witness = next_vct_block(blocks[tip_target].block.clone()); + let error = fast + .commit_finalized_direct(cv.into(), None, forged_witness, "vct defer tip forged witness") + .expect_err("a non-linking witness must defer, not commit or evict"); + prop_assert!( + format!("{error:?}").contains("VctSuppliedRootAwaitingSuccessor"), + "a non-linking witness defers with the await-successor error, got: {error:?}" + ); + prop_assert!( + error.vct_supplied_root_unavailable_height().is_none(), + "a non-linking witness is not a root failure — the correct root stays cached: {error:?}" + ); + prop_assert_eq!( + fast.db.finalized_tip_height(), + Some(Height((tip_target - 1) as u32)), + "the forged-witness attempt left the database untouched" + ); + let after_forged_prevalidated = fast.vct_prevalidated_count(); + prop_assert_eq!( + after_forged_prevalidated, + after_deferral_prevalidated + 1, + "the forged-witness attempt still uses the predecessor look-ahead" + ); + + // Once a successor is buffered, the very same height commits and the tip advances: + // the deferral was a wait, not a permanent stall — and the root survived the + // forged-witness attempt (it was never evicted). + let cv = CheckpointVerifiedBlock::from(blocks[tip_target].block.clone()); + let next = next_vct_block(blocks[tip_target + 1].block.clone()); + fast.commit_finalized_direct(cv.into(), None, next, "vct defer tip with successor") + .expect("the deferred height commits once its successor is buffered"); + prop_assert_eq!( + fast.vct_prevalidated_count(), + after_forged_prevalidated + 1, + "the retry reuses the preserved predecessor look-ahead" + ); + prop_assert_eq!( + fast.db.finalized_tip_height(), + Some(Height(tip_target as u32)), + "the tip advances once the successor confirms the root" + ); + }); + + Ok(()) +} + +/// A wrong peer-supplied root must be recoverable at the same height: the committer rejects and +/// evicts the bad cached value, leaves the database parked below the height, then commits the +/// same block once the `tree_aux` driver refills that height with a verifiable root. +#[test] +#[allow(clippy::needless_range_loop)] // the loop indexes blocks[i+1] and inserts roots by height +fn vct_peer_source_bad_root_refill_commits_same_height() -> Result<()> { + use crate::service::finalized_state::commitment_aux::PeerSource; + use zebra_chain::parallel::commitment_aux::BlockCommitmentRoots; + + let _init_guard = zebra_test::init(); + + let network = ParametersBuilder::default() + .with_activation_heights(ConfiguredActivationHeights { + before_overwinter: Some(1), + overwinter: Some(10), + sapling: Some(15), + blossom: Some(20), + heartwood: Some(25), + canopy: Some(30), + nu5: Some(35), + nu6: Some(40), + nu6_1: Some(45), + nu6_2: Some(47), + nu6_3: Some(48), + nu7: Some(50), + }) + .expect("failed to set activation heights") + .extend_funding_streams() + .to_network() + .expect("failed to build configured network"); + let ledger_strategy = + LedgerState::genesis_strategy(Some(network), None::, None, false); + + proptest!(ProptestConfig::with_cases(1), + |((chain, _count, network, _history_tree) in PreparedChain::default().with_ledger_strategy(ledger_strategy.clone()).with_valid_commitments().no_shrink())| { + + let blocks: Vec<_> = chain.iter().collect(); + let nu5 = NetworkUpgrade::Nu5.activation_height(&network).unwrap().0; + let heartwood = NetworkUpgrade::Heartwood.activation_height(&network).unwrap().0; + let target = (nu5 + 1) as usize; + prop_assert!(blocks.len() > target + 1, "generated chain unexpectedly short"); + let seed = (heartwood - 1) as usize; + + // Source the true roots from a legacy pass, then poison the target height exactly + // as a malicious peer would. Earlier roots are correct so the frontier freezes + // before the bad root is encountered. + let mut legacy = FinalizedState::new(&Config::ephemeral(), &network, #[cfg(feature = "elasticsearch")] false); + let mut peer_roots = Vec::new(); + let mut correct_target_root = None; + for i in 0..=target { + let cv = CheckpointVerifiedBlock::from(blocks[i].block.clone()); + let (_h, trees) = legacy + .commit_finalized_direct(cv.into(), None, None, "vct refill legacy") + .unwrap(); + if i > seed { + let root = BlockCommitmentRoots { + height: Height(i as u32), + sapling_root: trees.sapling.root(), + orchard_root: trees.orchard.root(), + ironwood_root: zebra_chain::ironwood::tree::NoteCommitmentTree::default().root(), + sapling_tx: 0, + orchard_tx: 0, + ironwood_tx: 0, + auth_data_root: blocks[i].block.auth_data_root(), + }; + if i == target { + correct_target_root = Some(root.clone()); + let mut poisoned = root; + prop_assert_ne!( + poisoned.sapling_root, + Default::default(), + "a V2 target block must have a non-empty Sapling root" + ); + poisoned.sapling_root = Default::default(); + peer_roots.push(poisoned); + } else { + peer_roots.push(root); + } + } + } + let correct_target_root = correct_target_root.expect("target root was produced"); + + let mut fast = FinalizedState::new(&Config::ephemeral(), &network, #[cfg(feature = "elasticsearch")] false); + fast.db + .insert_zakura_header_commitment_roots(peer_roots) + .expect("writing header-sync roots to an ephemeral database succeeds"); + let source = PeerSource::new(fast.db.clone(), test_handoff_frontiers(Height::MAX)); + fast.enable_vct_fast_source(Box::new(source), true); + + for i in 0..target { + let cv = CheckpointVerifiedBlock::from(blocks[i].block.clone()); + let next = next_vct_block(blocks[i + 1].block.clone()); + fast.commit_finalized_direct(cv.into(), None, next, "vct refill pre-target") + .expect("pre-target fast commits succeed"); + } + prop_assert_eq!(fast.db.finalized_tip_height(), Some(Height((target - 1) as u32))); + + let cv = CheckpointVerifiedBlock::from(blocks[target].block.clone()); + let next = next_vct_block(blocks[target + 1].block.clone()); + let error = fast + .commit_finalized_direct(cv.into(), None, next.clone(), "vct poisoned target") + .expect_err("the poisoned peer root must be rejected before commit"); + prop_assert_eq!( + error.vct_supplied_root_unavailable_height(), + Some(Height(target as u32)), + "the bad root is exposed as a retryable refetch for its own height" + ); + prop_assert_eq!( + fast.db.finalized_tip_height(), + Some(Height((target - 1) as u32)), + "the rejected root left the database parked below the target" + ); + + // Simulate the `tree_aux` driver refilling the evicted height from another peer: + // header sync persists the replacement through the same database write path. + fast.db + .insert_zakura_header_commitment_roots([correct_target_root]) + .expect("refilling the evicted height succeeds"); + + let cv = CheckpointVerifiedBlock::from(blocks[target].block.clone()); + fast.commit_finalized_direct(cv.into(), None, next, "vct refilled target") + .expect("the same height commits once the peer cache is refilled"); + prop_assert_eq!( + fast.db.finalized_tip_height(), + Some(Height(target as u32)), + "the refilled root unblocks the parked height" + ); + }); + + Ok(()) +} + +/// The frozen-frontier guard must survive a restart. A fast sync interrupted before the +/// checkpoint handoff leaves the stale frozen frontier persisted (fast commits never write +/// per-height trees) with the tip still below the handoff, but the in-memory `frozen` flag +/// is rebuilt from scratch on open. If it came back `false`, the first post-restart height +/// with no supplied root would legacy-recompute against the stale on-disk frontier and +/// corrupt the history MMR — the exact hazard the in-session guard prevents +/// (`vct_frozen_frontier_hole_refuses_instead_of_recomputing`). So `FinalizedState::new` +/// re-derives the flag from the durable fast-sync marker. This reopens the database between +/// freezing and the hole, and asserts that the very first commit of the new session (no +/// prior fast block to re-arm the flag in-session) still refuses with the retryable +/// `VctSuppliedRootUnavailable`, leaves state untouched, and commits once the root arrives. +#[test] +#[allow(clippy::needless_range_loop)] // the loop indexes blocks[i+1] and the fixture by height +fn vct_frozen_frontier_survives_reopen() -> Result<()> { + let _init_guard = zebra_test::init(); + + let network = ParametersBuilder::default() + .with_activation_heights(ConfiguredActivationHeights { + before_overwinter: Some(1), + overwinter: Some(10), + sapling: Some(15), + blossom: Some(20), + heartwood: Some(25), + canopy: Some(30), + nu5: Some(35), + nu6: Some(40), + nu6_1: Some(45), + nu6_2: Some(47), + nu6_3: Some(48), + nu7: Some(50), + }) + .expect("failed to set activation heights") + .extend_funding_streams() + .to_network() + .expect("failed to build configured network"); + let ledger_strategy = + LedgerState::genesis_strategy(Some(network), None::, None, false); + + proptest!(ProptestConfig::with_cases(1), + |((chain, _count, network, _history_tree) in PreparedChain::default().with_ledger_strategy(ledger_strategy.clone()).with_valid_commitments().no_shrink())| { + + let blocks: Vec<_> = chain.iter().collect(); + let nu5 = NetworkUpgrade::Nu5.activation_height(&network).unwrap().0; + let heartwood = NetworkUpgrade::Heartwood.activation_height(&network).unwrap().0; + let handoff_height = nu5 + 3; + let last = handoff_height as usize; + prop_assert!(blocks.len() > last, "generated chain unexpectedly short"); + let seed = (heartwood - 1) as usize; + + // Stop the fast sync two blocks below the handoff, so the tip is inside the + // frozen region and there is room for the hole at `stop + 1` (still below the + // handoff, where the real frontier would have been written). + let stop = (handoff_height - 2) as usize; + let hole = stop + 1; + prop_assert!(seed < stop && hole < last, "the hole must sit inside the frozen fast range"); + + // Legacy golden pass over [0, last]: the per-block fixture for the fast range + // and the real final frontiers at the handoff (needed to configure fast mode). + let mut legacy = FinalizedState::new(&Config::ephemeral(), &network, #[cfg(feature = "elasticsearch")] false); + let mut fixture = std::collections::HashMap::new(); + let mut handoff_trees = None; + for i in 0..=last { + let cv = CheckpointVerifiedBlock::from(blocks[i].block.clone()); + let (_h, trees) = legacy + .commit_finalized_direct(cv.into(), None, None, "vct reopen legacy") + .unwrap(); + if i > seed { + fixture.insert(i as u32, (trees.sapling.root(), trees.orchard.root())); + } + if i == last { + handoff_trees = Some(trees); + } + } + let handoff_trees = handoff_trees.expect("committed the handoff block"); + + // A persistent database so the syncing handle can be dropped and reopened by + // path, modelling a node restart. Archive storage mode (the default): fast sync + // is the default under checkpoint sync, and a fast-synced database reopens fine + // in archive mode, exactly as in production. + let dir = TempDir::new().expect("temp dir"); + let config = Config { + cache_dir: dir.path().to_path_buf(), + ephemeral: false, + ..Config::default() + }; + + // Session 1: a genesis-start fast sync interrupted at `stop`, two blocks below + // the handoff. The fast commits write the fast-sync marker but no per-height + // trees, so the on-disk frontier is frozen and the tip is below the handoff. + { + let mut fast = FinalizedState::new(&config, &network, #[cfg(feature = "elasticsearch")] false); + enable_vct_test_fixture_source_with_handoff( + &mut fast, + fixture.clone(), + Height(handoff_height), + handoff_trees.sapling.clone(), + handoff_trees.orchard.clone(), + handoff_trees.sprout.clone(), + ); + for i in 0..=stop { + let cv = CheckpointVerifiedBlock::from(blocks[i].block.clone()); + let next = next_vct_block(blocks[i + 1].block.clone()); + fast.commit_finalized_direct(cv.into(), None, next, "vct reopen fast") + .expect("verified fast commit succeeds"); + } + prop_assert_eq!(fast.vct_fast_synced_below(), Some(Height(handoff_height)), "the interrupted sync left the fast-sync marker"); + prop_assert_eq!(fast.db.finalized_tip_height(), Some(Height(stop as u32)), "the tip is parked below the handoff"); + // Drop releases the database lock for the reopen below. + } + + // Session 2 (restart): reopen the same database, then punch a hole at the next + // height (a peer that omitted it, or a root evicted after failing verification). + // Skip the constructor-time interrupted-fast-sync resume guard: this configured + // network has no embedded frontiers, so `from_config` yields no source, but the + // test attaches a fixture source below the way a real (Mainnet) node's configured + // source is already present at open time. + let mut reopened = FinalizedState::new_with_debug_and_storage_validation( + &config, + &network, + false, + #[cfg(feature = "elasticsearch")] + false, + false, + true, + false, + ); + prop_assert_eq!(reopened.vct_fast_synced_below(), Some(Height(handoff_height)), "the marker is still durable after reopen"); + + let mut holed = fixture.clone(); + holed.remove(&(hole as u32)); + enable_vct_test_fixture_source_with_handoff( + &mut reopened, + holed, + Height(handoff_height), + handoff_trees.sapling.clone(), + handoff_trees.orchard.clone(), + handoff_trees.sprout.clone(), + ); + + // The very first commit of the new session is the hole. No fast block has run + // since the reopen, so the only thing that can arm the guard is the flag seeded + // from the durable marker. Before the fix it came back `false` and this would + // legacy-recompute against the stale frontier; now it refuses. + let cv = CheckpointVerifiedBlock::from(blocks[hole].block.clone()); + let next = next_vct_block(blocks[hole + 1].block.clone()); + let error = reopened + .commit_finalized_direct(cv.into(), None, next, "vct reopen hole") + .expect_err("a frozen-frontier hole must refuse after reopen, not recompute"); + prop_assert!( + format!("{error:?}").contains("VctSuppliedRootUnavailable"), + "the reopened committer returns the retryable VctSuppliedRootUnavailable, got: {error:?}" + ); + prop_assert_eq!(reopened.db.finalized_tip_height(), Some(Height(stop as u32)), "the refused block left the reopened state untouched"); + + // Retryable: once a verifiable root for the hole is supplied, the same height + // commits and the tip advances — the refusal was a stall, not a permanent wedge. + enable_vct_test_fixture_source_with_handoff( + &mut reopened, + fixture.clone(), + Height(handoff_height), + handoff_trees.sapling.clone(), + handoff_trees.orchard.clone(), + handoff_trees.sprout.clone(), + ); + let cv = CheckpointVerifiedBlock::from(blocks[hole].block.clone()); + let next = next_vct_block(blocks[hole + 1].block.clone()); + reopened + .commit_finalized_direct(cv.into(), None, next, "vct reopen refill") + .expect("the height commits once its root is fetched"); + prop_assert_eq!(reopened.db.finalized_tip_height(), Some(Height(hole as u32)), "the tip advances past the former hole once the root arrives"); + }); + + Ok(()) +} + +/// Verified-commitment-trees checkpoint handoff (merged increments 4+5): a +/// genesis-start fast sync writes the verified final frontier at the handoff +/// height, marks the database fast-synced, guards historical per-height tree reads +/// below the handoff, and leaves the tip treestate (which post-checkpoint semantic +/// verification resumes from) byte-identical to the legacy recompute. +#[test] +#[allow(clippy::needless_range_loop)] // the loops index blocks[i+1] and the fixture by height +fn vct_fast_sync_handoff_marks_database_and_resumes() -> Result<()> { + let _init_guard = zebra_test::init(); + + let network = ParametersBuilder::default() + .with_activation_heights(ConfiguredActivationHeights { + before_overwinter: Some(1), + overwinter: Some(10), + sapling: Some(15), + blossom: Some(20), + heartwood: Some(25), + canopy: Some(30), + nu5: Some(35), + nu6: Some(40), + nu6_1: Some(45), + nu6_2: Some(47), + nu6_3: Some(48), + nu7: Some(50), + }) + .expect("failed to set activation heights") + .extend_funding_streams() + .to_network() + .expect("failed to build configured network"); + let ledger_strategy = + LedgerState::genesis_strategy(Some(network), None::, None, false); + + proptest!(ProptestConfig::with_cases(env::var("PROPTEST_CASES") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(DEFAULT_PARTIAL_CHAIN_PROPTEST_CASES)), + |((chain, _count, network, _history_tree) in PreparedChain::default().with_ledger_strategy(ledger_strategy.clone()).with_valid_commitments().no_shrink())| { + + let blocks: Vec<_> = chain.iter().collect(); + let nu5 = NetworkUpgrade::Nu5.activation_height(&network).unwrap().0; + let heartwood = NetworkUpgrade::Heartwood.activation_height(&network).unwrap().0; + let last = (nu5 + 3) as usize; + prop_assert!(blocks.len() > last, "generated chain unexpectedly short"); + let handoff = Height(last as u32); + + // The fast range is seeded just below Heartwood, so it is authenticated by + // the ZIP-221 MMR (the synthetic chain's pre-Heartwood `FinalSaplingRoot` + // headers are not consistent with the computed trees, so the Sapling-era + // direct-header path can't be exercised here — that rides with the real + // synced node). The handoff is at the tip. + let seed = (heartwood - 1) as usize; + + // Legacy pass over [0, last]: the per-block fixture for the fast range, the + // golden consensus state, and the real final frontiers at the handoff. + let mut legacy = FinalizedState::new(&Config::ephemeral(), &network, #[cfg(feature = "elasticsearch")] false); + let mut fixture = std::collections::HashMap::new(); + let mut handoff_trees = None; + for i in 0..=last { + let cv = CheckpointVerifiedBlock::from(blocks[i].block.clone()); + let (_h, trees) = legacy + .commit_finalized_direct(cv.into(), None, None, "vct legacy") + .unwrap(); + if i > seed { + fixture.insert(i as u32, (trees.sapling.root(), trees.orchard.root())); + } + if i == last { + handoff_trees = Some(trees); + } + } + let golden_anchors = legacy.db.vct_anchor_digest(); + let golden_history = legacy.db.history_tree().hash(); + let golden_tip = legacy.db.note_commitment_trees_for_tip(); + let handoff_trees = handoff_trees.expect("committed the handoff block"); + + // Fast genesis-start pass over [0, last], supplying the verified frontiers + // for the handoff at `last`. + let mut fast = FinalizedState::new(&Config::ephemeral(), &network, #[cfg(feature = "elasticsearch")] false); + enable_vct_test_fixture_source_with_handoff( + &mut fast, + fixture.clone(), + handoff, + handoff_trees.sapling.clone(), + handoff_trees.orchard.clone(), + handoff_trees.sprout.clone(), + ); + prop_assert!(!fast.vct_fast_needs_successor(handoff), "the trusted handoff frontier authenticates the handoff root without a successor"); + for i in 0..=last { + let cv = CheckpointVerifiedBlock::from(blocks[i].block.clone()); + let next = (i < last).then(|| NextVctBlock { + block: blocks[i + 1].block.clone(), + auth_data_root: None, + }); + fast.commit_finalized_direct(cv.into(), None, next, "vct fast handoff") + .expect("verified fast commit succeeds"); + } + + // The database is marked fast-synced at the handoff height, and the upgrade height is + // genesis: a node that fast-syncs from genesis records `U = 0`, so its whole `[0, H)` + // range is the absent band and every request is served from the index. + prop_assert_eq!(fast.vct_fast_synced_below(), Some(handoff), "fast-sync marker is set to the handoff height"); + prop_assert_eq!(fast.db.vct_upgrade_height(), Some(Height(0)), "genesis fast sync records the upgrade height at genesis"); + + // Consensus state (anchor sets + history root) matches the legacy recompute. + prop_assert_eq!(fast.db.vct_anchor_digest(), golden_anchors, "fast anchors must match legacy"); + prop_assert_eq!(fast.db.history_tree().hash(), golden_history, "fast history must match legacy"); + + // The handoff wrote the real frontier at the checkpoint, so the tip + // treestate that semantic verification resumes from matches legacy. + let fast_tip = fast.db.note_commitment_trees_for_tip(); + prop_assert_eq!(fast_tip.sapling.root(), golden_tip.sapling.root(), "tip sapling frontier must match legacy"); + prop_assert_eq!(fast_tip.orchard.root(), golden_tip.orchard.root(), "tip orchard frontier must match legacy"); + prop_assert_eq!(fast_tip.sprout.root(), golden_tip.sprout.root(), "tip sprout frontier must match legacy"); + + // Historical per-height tree reads below the handoff are unavailable + // (guarded, no panic), while the handoff height itself is present. + prop_assert!(fast.db.sapling_tree_by_height(&Height(last as u32 - 1)).is_none(), "below-handoff sapling tree read is guarded"); + prop_assert!(fast.db.orchard_tree_by_height(&Height(last as u32 - 1)).is_none(), "below-handoff orchard tree read is guarded"); + prop_assert!(fast.db.sapling_tree_by_height(&handoff).is_some(), "handoff sapling tree is present"); + prop_assert!(fast.db.orchard_tree_by_height(&handoff).is_some(), "handoff orchard tree is present"); + + // Root-serving index (design §4): the fast-synced node holds no per-height trees + // below the handoff (asserted just above), yet it must still serve `tree_aux` + // roots for that range so the root-serving fleet does not collapse as nodes + // fast-sync. Those roots come from the compact `commitment_roots_by_height` index + // the fast path persists per block, and they match exactly the roots the + // legacy/archive node derives from its per-height trees. + let below_handoff = Height((seed + 1) as u32)..=Height(last as u32 - 1); + let served = fast.db.commitment_roots_by_height_range(below_handoff.clone()); + let expected = commitment_aux::produce_block_roots(&legacy.db, below_handoff.clone()); + prop_assert!(!served.is_empty(), "a fast-synced node serves below-handoff roots from the index"); + prop_assert_eq!(served, expected.clone(), "index-served roots match the legacy per-height-tree roots"); + + // The same range goes through `serve_block_roots`: with `U = 0` the request starts at + // or above the upgrade height, so it is served entirely from the index — no per-height + // trees (which the fast-synced node lacks below the handoff) are consulted. + prop_assert_eq!(serve_block_roots(&fast.db, below_handoff), expected, "serve_block_roots serves the fast-synced range from the index"); + + // The `z_gettreestate` RPC gate predicate matches the read guard: a + // below-handoff height is unavailable (typed archive-mode error), while the + // handoff height itself is available. + prop_assert!(fast.db.vct_historical_tree_unavailable(HashOrHeight::Height(Height(last as u32 - 1))), "RPC gate: below-handoff treestate is unavailable"); + prop_assert!(!fast.db.vct_historical_tree_unavailable(HashOrHeight::Height(handoff)), "RPC gate: handoff treestate is available"); + + // Negative: a peer can supply a wrong root exactly at the handoff height, + // where there is no buffered checkpoint successor to authenticate it. The + // final embedded frontier still binds the expected root, so the committer + // must reject and retry instead of panicking or writing a bad handoff. + let mut bad_handoff_fixture = fixture.clone(); + let bad_handoff_entry = bad_handoff_fixture + .get_mut(&(last as u32)) + .expect("fixture contains the handoff root"); + prop_assert_ne!(bad_handoff_entry.0, Default::default(), "a post-NU5 handoff block must have a non-empty Sapling root"); + bad_handoff_entry.0 = Default::default(); + + let mut bad_handoff = FinalizedState::new(&Config::ephemeral(), &network, #[cfg(feature = "elasticsearch")] false); + enable_vct_test_fixture_source_with_handoff( + &mut bad_handoff, + bad_handoff_fixture, + handoff, + handoff_trees.sapling.clone(), + handoff_trees.orchard.clone(), + handoff_trees.sprout.clone(), + ); + + let mut error_height = None; + let mut handoff_error = None; + for i in 0..=last { + let cv = CheckpointVerifiedBlock::from(blocks[i].block.clone()); + let next = (i < last).then(|| NextVctBlock { + block: blocks[i + 1].block.clone(), + auth_data_root: None, + }); + match bad_handoff.commit_finalized_direct(cv.into(), None, next, "vct bad handoff") { + Ok(_) => {} + Err(error) => { + error_height = Some(i); + handoff_error = Some(error); + break; + } + } + } + prop_assert_eq!(error_height, Some(last), "the bad handoff root is rejected at the handoff height"); + let handoff_error = handoff_error.expect("the bad handoff root failed"); + prop_assert!( + format!("{handoff_error:?}").contains("VctSuppliedRootUnavailable"), + "a bad handoff root returns the retryable VctSuppliedRootUnavailable error, got: {handoff_error:?}" + ); + prop_assert_eq!( + bad_handoff.db.finalized_tip_height(), + Some(Height(last as u32 - 1)), + "the refused handoff block left state untouched" + ); + }); + + Ok(()) +} + +/// Switching between the rollout fast path and the manual recompute path is safe at the +/// committed-state boundaries: after the handoff writes the real frontier, legacy recompute can +/// resume from that frontier; before any fast commit has frozen the frontier, a later fast sync +/// can consume verified roots for future heights. +#[test] +#[allow(clippy::needless_range_loop)] // the loops index blocks[i+1] and the fixture by height +fn vct_mode_switches_continue_from_safe_boundaries() -> Result<()> { + let _init_guard = zebra_test::init(); + + let network = ParametersBuilder::default() + .with_activation_heights(ConfiguredActivationHeights { + before_overwinter: Some(1), + overwinter: Some(10), + sapling: Some(15), + blossom: Some(20), + heartwood: Some(25), + canopy: Some(30), + nu5: Some(35), + nu6: Some(40), + nu6_1: Some(45), + nu6_2: Some(47), + nu6_3: Some(48), + nu7: Some(50), + }) + .expect("failed to set activation heights") + .extend_funding_streams() + .to_network() + .expect("failed to build configured network"); + let ledger_strategy = + LedgerState::genesis_strategy(Some(network), None::, None, false); + + proptest!(ProptestConfig::with_cases(1), + |((chain, _count, network, _history_tree) in PreparedChain::default().with_ledger_strategy(ledger_strategy.clone()).with_valid_commitments().no_shrink())| { + let blocks: Vec<_> = chain.iter().collect(); + let nu5 = NetworkUpgrade::Nu5.activation_height(&network).unwrap().0; + let heartwood = NetworkUpgrade::Heartwood.activation_height(&network).unwrap().0; + let handoff_index = (nu5 + 3) as usize; + let post_handoff_tip = handoff_index + 2; + prop_assert!(blocks.len() > post_handoff_tip, "generated chain unexpectedly short"); + let handoff = Height(handoff_index as u32); + let seed = (heartwood - 1) as usize; + + // Legacy golden pass over the full range: source fast roots and final frontiers, then + // compare both switching scenarios against this byte-identical manual recompute. + let mut legacy = FinalizedState::new(&Config::ephemeral(), &network, #[cfg(feature = "elasticsearch")] false); + let mut fixture = std::collections::HashMap::new(); + let mut handoff_trees = None; + let mut post_handoff_roots = None; + for i in 0..=post_handoff_tip { + let cv = CheckpointVerifiedBlock::from(blocks[i].block.clone()); + let (_h, trees) = legacy + .commit_finalized_direct(cv.into(), None, None, "vct switch legacy") + .unwrap(); + if i > seed && i <= handoff_index { + fixture.insert(i as u32, (trees.sapling.root(), trees.orchard.root())); + } + if i == handoff_index { + handoff_trees = Some(trees); + } else if i == handoff_index + 1 { + post_handoff_roots = Some((trees.sapling.root(), trees.orchard.root())); + } + } + let golden_anchors = legacy.db.vct_anchor_digest(); + let golden_history = legacy.db.history_tree().hash(); + let golden_tip = legacy.db.note_commitment_trees_for_tip(); + let handoff_trees = handoff_trees.expect("committed the handoff block"); + let post_handoff_roots = post_handoff_roots.expect("committed a post-handoff block"); + + // Fast -> manual: complete the fast handoff, reopen with the force-disable knob, and + // keep checkpoint sync enabled while post-handoff blocks recompute from the real + // frontier written at the handoff. + let fast_to_manual_dir = TempDir::new().expect("temp dir"); + let fast_config = Config { + cache_dir: fast_to_manual_dir.path().to_path_buf(), + ephemeral: false, + ..Config::default() + }; + { + let mut fast = FinalizedState::new(&fast_config, &network, #[cfg(feature = "elasticsearch")] false); + enable_vct_test_fixture_source_with_handoff( + &mut fast, + fixture.clone(), + handoff, + handoff_trees.sapling.clone(), + handoff_trees.orchard.clone(), + handoff_trees.sprout.clone(), + ); + for i in 0..=handoff_index { + let cv = CheckpointVerifiedBlock::from(blocks[i].block.clone()); + let next = (i < handoff_index).then(|| NextVctBlock { + block: blocks[i + 1].block.clone(), + auth_data_root: None, + }); + fast.commit_finalized_direct(cv.into(), None, next, "vct switch fast prefix") + .expect("verified fast prefix commits"); + } + prop_assert_eq!(fast.vct_fast_synced_below(), Some(handoff), "fast sync reached the handoff before the switch"); + } + + let manual_config = Config { + vct_fast_sync: false, + ..fast_config + }; + let mut manual = FinalizedState::new(&manual_config, &network, #[cfg(feature = "elasticsearch")] false); + for i in (handoff_index + 1)..=post_handoff_tip { + let cv = CheckpointVerifiedBlock::from(blocks[i].block.clone()); + manual + .commit_finalized_direct(cv.into(), None, None, "vct switch manual suffix") + .expect("manual suffix commits after fast handoff"); + } + let manual_tip = manual.db.note_commitment_trees_for_tip(); + prop_assert_eq!(manual.db.vct_anchor_digest(), golden_anchors, "fast-to-manual anchors match legacy"); + prop_assert_eq!(manual.db.history_tree().hash(), golden_history, "fast-to-manual history matches legacy"); + prop_assert_eq!(manual_tip.sapling.root(), golden_tip.sapling.root(), "fast-to-manual sapling tip matches legacy"); + prop_assert_eq!(manual_tip.orchard.root(), golden_tip.orchard.root(), "fast-to-manual orchard tip matches legacy"); + prop_assert_eq!(manual_tip.sprout.root(), golden_tip.sprout.root(), "fast-to-manual sprout tip matches legacy"); + + // Manual -> fast: commit a prefix with the force-disable knob before any fast block + // can freeze the frontier, then reopen and consume verified roots through the handoff. + let manual_to_fast_dir = TempDir::new().expect("temp dir"); + let manual_prefix_config = Config { + cache_dir: manual_to_fast_dir.path().to_path_buf(), + ephemeral: false, + vct_fast_sync: false, + ..Config::default() + }; + { + let mut manual_prefix = FinalizedState::new(&manual_prefix_config, &network, #[cfg(feature = "elasticsearch")] false); + for i in 0..=seed { + let cv = CheckpointVerifiedBlock::from(blocks[i].block.clone()); + manual_prefix + .commit_finalized_direct(cv.into(), None, None, "vct switch manual prefix") + .expect("manual prefix commits"); + } + } + + let fast_suffix_config = Config { + vct_fast_sync: true, + ..manual_prefix_config + }; + let mut fast_suffix = FinalizedState::new(&fast_suffix_config, &network, #[cfg(feature = "elasticsearch")] false); + let mut guarded_fixture = fixture; + // A stale or over-eager peer cache entry above the handoff must be ignored so + // the committer resumes legacy recompute from the real handoff frontier. + prop_assert_ne!( + post_handoff_roots.0, + Default::default(), + "a post-NU5 post-handoff block must have a non-empty Sapling root", + ); + guarded_fixture.insert( + (handoff_index + 1) as u32, + (Default::default(), post_handoff_roots.1), + ); + enable_vct_test_fixture_source_with_handoff( + &mut fast_suffix, + guarded_fixture, + handoff, + handoff_trees.sapling.clone(), + handoff_trees.orchard.clone(), + handoff_trees.sprout.clone(), + ); + for i in (seed + 1)..=post_handoff_tip { + let cv = CheckpointVerifiedBlock::from(blocks[i].block.clone()); + let next = (i < post_handoff_tip).then(|| NextVctBlock { + block: blocks[i + 1].block.clone(), + auth_data_root: None, + }); + fast_suffix + .commit_finalized_direct(cv.into(), None, next, "vct switch fast suffix") + .expect("fast suffix commits after manual prefix"); + } + prop_assert_eq!( + fast_suffix.vct_fast_count(), + (handoff_index - seed) as u64, + "an above-handoff cached root must not keep the committer on the fast path", + ); + let fast_suffix_tip = fast_suffix.db.note_commitment_trees_for_tip(); + prop_assert_eq!(fast_suffix.db.vct_anchor_digest(), golden_anchors, "manual-to-fast anchors match legacy"); + prop_assert_eq!(fast_suffix.db.history_tree().hash(), golden_history, "manual-to-fast history matches legacy"); + prop_assert_eq!(fast_suffix_tip.sapling.root(), golden_tip.sapling.root(), "manual-to-fast sapling tip matches legacy"); + prop_assert_eq!(fast_suffix_tip.orchard.root(), golden_tip.orchard.root(), "manual-to-fast orchard tip matches legacy"); + prop_assert_eq!(fast_suffix_tip.sprout.root(), golden_tip.sprout.root(), "manual-to-fast sprout tip matches legacy"); + }); + + Ok(()) +} + +/// Standalone test isolating the verify-before-commit **dedup**: each header +/// commitment is checked once, not twice. +/// +/// - **Skip:** the first fast block runs its own commitment check; the next one +/// is skipped, because the first block's look-ahead already validated it. +/// - **Stale-cache guard:** a cache entry with the right height but the *wrong* +/// hash must not trigger a skip — the guard forces the own check to run, so a +/// stale or mismatched entry can never let an unverified block through. +/// - **Wrapper-hash guard:** a public `CheckpointVerifiedBlock::with_hash` caller +/// cannot replay a stale cached successor hash onto a different block. +#[test] +fn vct_dedup_skips_redundant_check_and_guards_stale_cache() -> Result<()> { + let _init_guard = zebra_test::init(); + + let network = ParametersBuilder::default() + .with_activation_heights(ConfiguredActivationHeights { + before_overwinter: Some(1), + overwinter: Some(10), + sapling: Some(15), + blossom: Some(20), + heartwood: Some(25), + canopy: Some(30), + nu5: Some(35), + nu6: Some(40), + nu6_1: Some(45), + nu6_2: Some(47), + nu6_3: Some(48), + nu7: Some(50), + }) + .expect("failed to set activation heights") + .extend_funding_streams() + .to_network() + .expect("failed to build configured network"); + let ledger_strategy = + LedgerState::genesis_strategy(Some(network), None::, None, false); + + proptest!(ProptestConfig::with_cases(1), + |((chain, _count, network, _history_tree) in PreparedChain::default().with_ledger_strategy(ledger_strategy.clone()).with_valid_commitments().no_shrink())| { + + let blocks: Vec<_> = chain.iter().collect(); + let nu5 = NetworkUpgrade::Nu5.activation_height(&network).unwrap().0 as usize; + + // Seed just before NU5, then operate on four consecutive fast blocks so + // the forged-wrapper regression exercises `hashBlockCommitments`. + let seed = nu5 - 2; + let last = seed + 4; + prop_assert!(blocks.len() > last + 1, "generated chain unexpectedly short"); + + // Legacy pass to record the correct per-block roots as the fixture. + let mut legacy = FinalizedState::new(&Config::ephemeral(), &network, #[cfg(feature = "elasticsearch")] false); + let mut fixture = std::collections::HashMap::new(); + for (i, prepared) in blocks.iter().take(last + 1).enumerate() { + let cv = CheckpointVerifiedBlock::from(prepared.block.clone()); + let (_h, trees) = legacy + .commit_finalized_direct(cv.into(), None, None, "vct dedup legacy") + .unwrap(); + if i > seed { + fixture.insert(i as u32, (trees.sapling.root(), trees.orchard.root())); + } + } + + let mut fast = FinalizedState::new(&Config::ephemeral(), &network, #[cfg(feature = "elasticsearch")] false); + enable_vct_test_fixture_source(&mut fast, fixture); + + // Commit block `i` with its real successor as the one-block look-ahead. + let commit = |fast: &mut FinalizedState, i: usize| { + let cv = CheckpointVerifiedBlock::from(blocks[i].block.clone()); + let next = next_vct_block(blocks[i + 1].block.clone()); + fast.commit_finalized_direct(cv.into(), None, next, "vct dedup fast") + .expect("verified fast commit succeeds"); + }; + + // genesis..=seed take the recompute path (no fixture entries), so the dedup + // never engages here. + for i in 0..=seed { + commit(&mut fast, i); + } + prop_assert_eq!(fast.vct_prevalidated_count(), 0, "no fast blocks committed yet"); + + // First fast block: no cached predecessor, so it runs its own check. + commit(&mut fast, seed + 1); + prop_assert_eq!(fast.vct_prevalidated_count(), 0, "the first fast block runs its own commitment check"); + + // Second fast block: its predecessor's look-ahead already validated it, + // so the own check is skipped — the dedup engages. + commit(&mut fast, seed + 2); + prop_assert_eq!(fast.vct_prevalidated_count(), 1, "the second fast block skips its redundant own commitment check"); + + // Stale-cache guard: overwrite the cache with the correct height but the + // hash of a *different* block. The next commit must NOT skip. + let stale_hash = blocks[seed + 1].hash; + prop_assert_ne!(stale_hash, blocks[seed + 3].hash, "stale hash must differ from the real block"); + fast.vct + .set_prevalidated_next(Some((Height((seed + 3) as u32), stale_hash))); + commit(&mut fast, seed + 3); + prop_assert_eq!(fast.vct_prevalidated_count(), 1, "a stale cache entry (wrong hash) must not cause a false skip"); + + // Public wrapper-hash guard: the stale cache records a real look-ahead + // hash, but a caller-controlled checkpoint wrapper tries to replay that + // hash onto a different block whose own NU5 header commitment is invalid. + // The skip must compare the cache against the wrapped block's real hash, + // not the wrapper hash, so the bad commitment is checked and rejected. + let forged_wrapper_hash = blocks[seed + 2].hash; + let bad_block = blocks[seed + 4].block.clone().set_block_commitment([0x42; 32]); + let bad_block_hash = bad_block.hash(); + prop_assert_ne!( + forged_wrapper_hash, + bad_block_hash, + "the forged wrapper hash must differ from the bad block's real hash", + ); + fast.vct + .set_prevalidated_next(Some((Height((seed + 4) as u32), forged_wrapper_hash))); + let forged = CheckpointVerifiedBlock::with_hash(bad_block, forged_wrapper_hash); + let error = fast + .commit_finalized_direct(forged.into(), None, None, "vct forged wrapper hash") + .expect_err("a forged wrapper hash must not skip the bad block's own commitment check"); + prop_assert!( + format!("{error:?}").contains("VctSuppliedRootUnavailable"), + "the forged wrapper hash path must reject the bad commitment, got: {error:?}", + ); + prop_assert_eq!( + fast.vct_prevalidated_count(), + 1, + "the forged wrapper hash must not increment the prevalidated count", + ); + prop_assert_eq!( + fast.db.finalized_tip_height(), + Some(Height((seed + 3) as u32)), + "the rejected forged block must leave finalized state untouched", + ); + }); + + Ok(()) +} + +/// Clearing a cached VCT successor prevalidation must disarm exactly one possible +/// skip without disabling the normal dedup optimization for future contiguous fast +/// blocks. This covers the write-loop reset/drop behavior indirectly: those paths +/// call `clear_vct_prevalidated_next()` when buffered checkpoint state is discarded. +#[test] +fn vct_clear_prevalidation_cache_disarms_skip_then_dedup_resumes() -> Result<()> { + let _init_guard = zebra_test::init(); + + let network = ParametersBuilder::default() + .with_activation_heights(ConfiguredActivationHeights { + before_overwinter: Some(1), + overwinter: Some(10), + sapling: Some(15), + blossom: Some(20), + heartwood: Some(25), + canopy: Some(30), + nu5: Some(35), + nu6: Some(40), + nu6_1: Some(45), + nu6_2: Some(47), + nu6_3: Some(48), + nu7: Some(50), + }) + .expect("failed to set activation heights") + .extend_funding_streams() + .to_network() + .expect("failed to build configured network"); + let ledger_strategy = + LedgerState::genesis_strategy(Some(network), None::, None, false); + + proptest!(ProptestConfig::with_cases(1), + |((chain, _count, network, _history_tree) in PreparedChain::default().with_ledger_strategy(ledger_strategy.clone()).with_valid_commitments().no_shrink())| { + + let blocks: Vec<_> = chain.iter().collect(); + let nu5 = NetworkUpgrade::Nu5.activation_height(&network).unwrap().0 as usize; + let seed = nu5 - 2; + let last = seed + 5; + prop_assert!(blocks.len() > last + 1, "generated chain unexpectedly short"); + + let mut legacy = FinalizedState::new(&Config::ephemeral(), &network, #[cfg(feature = "elasticsearch")] false); + let mut fixture = std::collections::HashMap::new(); + for (i, prepared) in blocks.iter().take(last + 1).enumerate() { + let cv = CheckpointVerifiedBlock::from(prepared.block.clone()); + let (_h, trees) = legacy + .commit_finalized_direct(cv.into(), None, None, "vct clear legacy") + .unwrap(); + if i > seed { + fixture.insert(i as u32, (trees.sapling.root(), trees.orchard.root())); + } + } + + let mut fast = FinalizedState::new(&Config::ephemeral(), &network, #[cfg(feature = "elasticsearch")] false); + enable_vct_test_fixture_source(&mut fast, fixture); + + let commit = |fast: &mut FinalizedState, i: usize| { + let cv = CheckpointVerifiedBlock::from(blocks[i].block.clone()); + let next = next_vct_block(blocks[i + 1].block.clone()); + fast.commit_finalized_direct(cv.into(), None, next, "vct clear fast") + .expect("verified fast commit succeeds"); + }; + + for i in 0..=seed { + commit(&mut fast, i); + } + commit(&mut fast, seed + 1); + prop_assert_eq!(fast.vct_prevalidated_count(), 0, "first fast block runs its own check"); + + commit(&mut fast, seed + 2); + prop_assert_eq!(fast.vct_prevalidated_count(), 1, "second fast block uses predecessor look-ahead"); + + fast.clear_vct_prevalidated_next(); + commit(&mut fast, seed + 3); + prop_assert_eq!( + fast.vct_prevalidated_count(), + 1, + "clearing the cache forces the next fast block to run its own check", + ); + + commit(&mut fast, seed + 4); + prop_assert_eq!( + fast.vct_prevalidated_count(), + 2, + "normal successor dedup resumes after the cleared block commits", + ); + }); + + Ok(()) +} + +/// Increment-3 contract proof: a roots/frontier payload **produced from a database** +/// (the serving read path) can replace the fixture and drives the fast path to +/// byte-identical consensus state. +/// +/// Builds an archive/legacy state over a generated valid-commitment chain (crossing +/// Heartwood and NU5), produces the per-block roots and final frontier from that DB +/// via [`commitment_aux::produce_block_roots`] / [`commitment_aux::produce_final_frontiers`], +/// then drives a fresh fast-sync state that consumes the produced payload through the +/// test-only [`commitment_aux::FixtureSource`]. Asserts the fast anchors + history-tree hash are +/// byte-identical to the legacy build, and that the produced final frontier agrees with +/// the legacy tip frontier and the produced root at the handoff height. +/// +/// This is coverage the existing equivalence test lacks: there the roots are captured +/// from the committer's inline-returned trees, here they come from the **DB read path** +/// a serving node runs. No networking and no DB-format change. +#[test] +#[allow(clippy::needless_range_loop)] // the loops index blocks[i+1] (the look-ahead) and by height +fn vct_db_produced_payload_round_trips_to_byte_identical_state() -> Result<()> { + let _init_guard = zebra_test::init(); + + let network = ParametersBuilder::default() + .with_activation_heights(ConfiguredActivationHeights { + before_overwinter: Some(1), + overwinter: Some(10), + sapling: Some(15), + blossom: Some(20), + heartwood: Some(25), + canopy: Some(30), + nu5: Some(35), + nu6: Some(40), + nu6_1: Some(45), + nu6_2: Some(47), + nu6_3: Some(48), + nu7: Some(50), + }) + .expect("failed to set activation heights") + .extend_funding_streams() + .to_network() + .expect("failed to build configured network"); + let ledger_strategy = + LedgerState::genesis_strategy(Some(network), None::, None, false); + + proptest!(ProptestConfig::with_cases(1), + |((chain, _count, network, _history_tree) in PreparedChain::default().with_ledger_strategy(ledger_strategy.clone()).with_valid_commitments().no_shrink())| { + + let blocks: Vec<_> = chain.iter().collect(); + let nu5 = NetworkUpgrade::Nu5.activation_height(&network).unwrap().0; + let heartwood = NetworkUpgrade::Heartwood.activation_height(&network).unwrap().0; + let last = (nu5 + 3) as usize; + prop_assert!(blocks.len() > last + 1, "generated chain unexpectedly short"); + // Seed below Heartwood so the fast range creates the history tree and + // crosses the NU5 V1->V2 boundary, matching the equivalence test. + let seed = (heartwood - 1) as usize; + + // Legacy/archive pass: a real DB with per-height trees, plus the golden state. + let mut legacy = FinalizedState::new(&Config::ephemeral(), &network, #[cfg(feature = "elasticsearch")] false); + for block in blocks.iter().take(last + 1) { + let cv = CheckpointVerifiedBlock::from(block.block.clone()); + legacy + .commit_finalized_direct(cv.into(), None, None, "vct round-trip legacy") + .unwrap(); + } + let golden_anchors = legacy.db.vct_anchor_digest(); + let golden_history = legacy.db.history_tree().hash(); + + // Produce the payload from the legacy DB's per-height trees (the serving read path). + let last_height = Height(last as u32); + let produced_roots = commitment_aux::produce_block_roots( + &legacy.db, + Height((seed + 1) as u32)..=last_height, + ); + let produced_frontiers = commitment_aux::produce_final_frontiers(&legacy.db, last_height) + .expect("legacy DB has the tip frontier"); + + // The produced final frontier agrees with the legacy tip frontier and with the + // produced root at the handoff height (the two producer outputs are consistent). + let handoff = produced_roots.last().expect("produced a non-empty range"); + prop_assert_eq!(produced_frontiers.sapling.root(), handoff.sapling_root, "produced sapling frontier matches the produced root at handoff"); + prop_assert_eq!(produced_frontiers.orchard.root(), handoff.orchard_root, "produced orchard frontier matches the produced root at handoff"); + prop_assert_eq!(produced_frontiers.sapling.root(), legacy.db.sapling_tree_by_height(&last_height).unwrap().root(), "produced sapling frontier matches legacy tip"); + prop_assert_eq!(produced_frontiers.sprout.root(), legacy.db.sprout_tree_for_tip().root(), "produced sprout frontier matches legacy tip"); + + // Consume the DB-produced roots in a fresh fast-sync state. + let mut fast = FinalizedState::new(&Config::ephemeral(), &network, #[cfg(feature = "elasticsearch")] false); + let produced_roots = produced_roots + .into_iter() + .map(|root| (root.height.0, (root.sapling_root, root.orchard_root))) + .collect(); + fast.enable_vct_fast_source( + Box::new(commitment_aux::FixtureSource::new( + produced_roots, + test_handoff_frontiers(Height::MAX), + )), + false, + ); + for i in 0..=last { + let cv = CheckpointVerifiedBlock::from(blocks[i].block.clone()); + let next = next_vct_block(blocks[i + 1].block.clone()); + fast.commit_finalized_direct(cv.into(), None, next, "vct round-trip fast") + .expect("verified fast commit from DB-produced roots succeeds"); + } + + prop_assert_eq!(fast.db.vct_anchor_digest(), golden_anchors, "fast anchors from DB-produced roots match legacy"); + prop_assert_eq!(fast.db.history_tree().hash(), golden_history, "fast history from DB-produced roots match legacy"); + + // Serving stitch across the upgrade height `U`. Simulate a node that upgraded + // mid-chain: it keeps the full per-height trees (written before the upgrade) but only + // has the serving index from `U` upward. `serve_block_roots` must still return the + // whole requested range as one contiguous run — trees fill `[start, U)`, the index + // fills `[U, end]` — matching the all-trees reference, with no short batch at the + // boundary that would stall the client's minimum-progress check. + let serve_range = Height((seed + 1) as u32)..=last_height; + let all_trees_reference = + commitment_aux::produce_block_roots(&legacy.db, serve_range.clone()); + let upgrade = Height(((seed + 1 + last) / 2) as u32); + prop_assert!( + serve_range.start() < &upgrade && upgrade <= last_height, + "the chosen upgrade height splits the served range" + ); + let mut batch = DiskWriteBatch::new(); + batch.delete_range_commitment_roots_by_height(&legacy.db, &Height(0), &upgrade); + batch.update_vct_upgrade_marker(&legacy.db, upgrade); + legacy + .db + .write_batch(batch) + .expect("simulating a mid-chain upgrade succeeds"); + prop_assert!( + legacy + .db + .commitment_roots_by_height_range(Height(0)..=Height(upgrade.0 - 1)) + .is_empty(), + "the serving index is dropped below the upgrade height" + ); + let stitched = serve_block_roots(&legacy.db, serve_range); + prop_assert_eq!( + stitched, + all_trees_reference, + "serve_block_roots stitches the trees below U with the index at/above U into one gap-free run" + ); + }); + + Ok(()) +} + +/// Verified-commitment-trees consumer half of the peer source: a +/// [`commitment_aux::PeerSource`] whose database is **filled incrementally** (as header +/// sync persists provisional root ranges when they arrive from peers) drives the fast +/// path to byte-identical consensus state. Same harness as the DB-produced round-trip, +/// but the produced roots are written into `commitment_roots_by_height` in two chunks +/// via [`ZebraDb::insert_zakura_header_commitment_roots`] — proving the DB-backed, +/// header-sync-fed source is a drop-in for the fixture. +#[test] +#[allow(clippy::needless_range_loop)] // the loops index blocks[i+1] (the look-ahead) and by height +fn vct_peer_source_filled_incrementally_drives_byte_identical_state() -> Result<()> { + let _init_guard = zebra_test::init(); + + let network = ParametersBuilder::default() + .with_activation_heights(ConfiguredActivationHeights { + before_overwinter: Some(1), + overwinter: Some(10), + sapling: Some(15), + blossom: Some(20), + heartwood: Some(25), + canopy: Some(30), + nu5: Some(35), + nu6: Some(40), + nu6_1: Some(45), + nu6_2: Some(47), + nu6_3: Some(48), + nu7: Some(50), + }) + .expect("failed to set activation heights") + .extend_funding_streams() + .to_network() + .expect("failed to build configured network"); + let ledger_strategy = + LedgerState::genesis_strategy(Some(network), None::, None, false); + + proptest!(ProptestConfig::with_cases(1), + |((chain, _count, network, _history_tree) in PreparedChain::default().with_ledger_strategy(ledger_strategy.clone()).with_valid_commitments().no_shrink())| { + + let blocks: Vec<_> = chain.iter().collect(); + let nu5 = NetworkUpgrade::Nu5.activation_height(&network).unwrap().0; + let heartwood = NetworkUpgrade::Heartwood.activation_height(&network).unwrap().0; + // The untrusted peer source defers any fast block whose own root has no buffered + // successor, so every committed fast block needs `blocks[i + 1]`. Keep `last` one + // below the chain tip so the deepest commit still has a successor witness. + let last = ((nu5 + 3) as usize).min(blocks.len().saturating_sub(2)); + prop_assert!(last > (nu5 as usize), "generated chain unexpectedly short"); + let seed = (heartwood - 1) as usize; + + // Legacy/archive pass: a real DB with per-height trees, plus the golden state. + let mut legacy = FinalizedState::new(&Config::ephemeral(), &network, #[cfg(feature = "elasticsearch")] false); + for block in blocks.iter().take(last + 1) { + let cv = CheckpointVerifiedBlock::from(block.block.clone()); + legacy + .commit_finalized_direct(cv.into(), None, None, "vct peer-source legacy") + .unwrap(); + } + let golden_anchors = legacy.db.vct_anchor_digest(); + let golden_history = legacy.db.history_tree().hash(); + + // Produce the payload from the legacy DB (the serving read path). + let produced_roots = commitment_aux::produce_block_roots( + &legacy.db, + Height((seed + 1) as u32)..=Height(last as u32), + ); + + // Consume the peer-source-supplied roots in a fresh fast-sync state. Each fast + // block is committed with its successor buffered, as the write loop does — the + // untrusted source defers a tip commit with no successor (covered by + // `vct_peer_source_defers_unverifiable_tip_root_until_successor`). + let mut fast = FinalizedState::new(&Config::ephemeral(), &network, #[cfg(feature = "elasticsearch")] false); + + // Fill the fast state's database incrementally, in two chunks, as header sync + // would when successive root ranges arrive from a peer; the peer source reads + // them back from that database. + let split = produced_roots.len() / 2; + fast.db + .insert_zakura_header_commitment_roots(produced_roots[..split].iter().cloned()) + .expect("writing the first header-sync root chunk succeeds"); + fast.db + .insert_zakura_header_commitment_roots(produced_roots[split..].iter().cloned()) + .expect("writing the second header-sync root chunk succeeds"); + let peer_source = + commitment_aux::PeerSource::new(fast.db.clone(), test_handoff_frontiers(Height::MAX)); + fast.enable_vct_fast_source(Box::new(peer_source), true); + for i in 0..=last { + let cv = CheckpointVerifiedBlock::from(blocks[i].block.clone()); + let next = next_vct_block(blocks[i + 1].block.clone()); + fast.commit_finalized_direct(cv.into(), None, next, "vct peer-source fast") + .expect("verified fast commit from peer-source roots succeeds"); + } + + prop_assert_eq!(fast.db.vct_anchor_digest(), golden_anchors, "fast anchors from peer-source roots match legacy"); + prop_assert_eq!(fast.db.history_tree().hash(), golden_history, "fast history from peer-source roots match legacy"); + }); + + Ok(()) +} diff --git a/zebra-state/src/service/finalized_state/tests/rollback.rs b/zebra-state/src/service/finalized_state/tests/rollback.rs index 73740a32698..336f05b1d6c 100644 --- a/zebra-state/src/service/finalized_state/tests/rollback.rs +++ b/zebra-state/src/service/finalized_state/tests/rollback.rs @@ -80,7 +80,7 @@ fn sync_to(config: &Config, network: &Network, blocks: &[SemanticallyVerifiedBlo for block in blocks { let checkpoint_verified = CheckpointVerifiedBlock::from(block.block.clone()); state - .commit_finalized_direct(checkpoint_verified.into(), None, "rollback test") + .commit_finalized_direct(checkpoint_verified.into(), None, None, "rollback test") .expect("committing a generated block to a fresh state succeeds"); } } diff --git a/zebra-state/src/service/finalized_state/vct.rs b/zebra-state/src/service/finalized_state/vct.rs index 5559e7d4e15..487fdfcaab9 100644 --- a/zebra-state/src/service/finalized_state/vct.rs +++ b/zebra-state/src/service/finalized_state/vct.rs @@ -14,7 +14,8 @@ use thiserror::Error; #[cfg(test)] use zebra_chain::parallel::tree::NoteCommitmentTrees; use zebra_chain::{ - block, orchard, + block::{self, merkle::AuthDataRoot, Block}, + orchard, parameters::{Network, NetworkUpgrade}, sapling, sprout, }; @@ -24,6 +25,16 @@ use super::{ ZebraDb, }; +/// A buffered VCT successor block used to authenticate the current block's +/// supplied note-commitment roots. +#[derive(Clone, Debug)] +pub struct NextVctBlock { + /// The successor block whose header commits to the current block's VCT roots. + pub(crate) block: Arc, + /// The successor block's precomputed ZIP-244 auth-data root, if available. + pub(crate) auth_data_root: Option, +} + /// Embedded verified final note-commitment frontiers for Mainnet. const MAINNET_FINAL_FRONTIERS: &[u8] = include_bytes!("vct/mainnet-frontier.bin"); @@ -262,6 +273,116 @@ impl VctState { } } +/// Commit-time vct state carried by [`super::FinalizedState`]: the configured +/// root source plus the commit-loop dedup and below-last-checkpoint state its +/// fast path depends on, grouped so their invariants live next to the data they guard. +#[derive(Clone, Debug)] +pub(crate) struct VctCommitState { + /// The root source (peer/fixture/capture mode), or `None` for legacy recompute. + source: Option>, + + /// `(height, hash)` of the next block already validated by the previous fast + /// commit's look-ahead, so its own commitment check can be skipped. Guarded by + /// hash identity, so a stale or cloned value can't cause an incorrect skip. + prevalidated_next: Option<(block::Height, block::Hash)>, + + /// `true` while a vct sync is in-progress below the last checkpoint height. + /// During this time, we do not reconstruct per-height note-commitment trees. + /// As a result, the frontier is unknown. + is_vct_sync_below_last_checkpoint: bool, +} + +impl VctCommitState { + /// Builds the commit state from a resolved `source` and an + /// `is_vct_sync_below_last_checkpoint` flag re-derived from durable state on open. + pub(super) fn new( + source: Option>, + is_vct_sync_below_last_checkpoint: bool, + ) -> Self { + VctCommitState { + source, + prevalidated_next: None, + is_vct_sync_below_last_checkpoint, + } + } + + /// The configured root source, or `None` for legacy recompute. + pub(super) fn source(&self) -> Option<&Arc> { + self.source.as_ref() + } + + /// `true` while the note-commitment frontier is below the last checkpoint height. + pub(super) fn is_below_last_checkpoint(&self) -> bool { + self.is_vct_sync_below_last_checkpoint + } + + /// The cached successor prevalidation, if any. + pub(super) fn prevalidated_next(&self) -> Option<(block::Height, block::Hash)> { + self.prevalidated_next + } + + /// Caches the next block's `(height, hash)` as already validated by this + /// fast commit's look-ahead. + pub(super) fn mark_prevalidated(&mut self, height: block::Height, hash: block::Hash) { + self.prevalidated_next = Some((height, hash)); + } + + /// Clears any cached successor prevalidation. + pub(super) fn clear_prevalidated_next(&mut self) { + self.prevalidated_next = None; + } + + /// Test-only: overwrites the cached successor prevalidation, so tests can + /// install a stale or forged entry to exercise the dedup's guard checks. + #[cfg(test)] + pub(super) fn set_prevalidated_next(&mut self, next: Option<(block::Height, block::Hash)>) { + self.prevalidated_next = next; + } + + /// Starts a VCT sync below the last checkpoint height: below the last checkpoint height, + /// the frontier is unknown as we are not reconstructing the trees every height. + pub(super) fn start_vct_sync_below_last_checkpoint(&mut self) { + self.is_vct_sync_below_last_checkpoint = true; + } + + /// Stops a VCT sync at the last checkpoint height: the last checkpoint wrote the + /// real final frontier as the tip treestate. + pub(super) fn stop_vct_sync_at_last_checkpoint(&mut self) { + self.is_vct_sync_below_last_checkpoint = false; + } + + /// Test-only: installs an arbitrary [`CommitmentRootSource`] as fast-mode + /// state, so the producer→consumer round-trip can be exercised in-process. + /// `requires_verified_successor` marks an untrusted source that must defer + /// tip roots until their successor is buffered. + #[cfg(test)] + pub(super) fn install_test_source( + &mut self, + source: Box, + requires_verified_successor: bool, + ) { + self.source = Some(VctState::test_with_source( + source, + requires_verified_successor, + )); + } +} + +/// Fast-path (vct) outputs for the block being committed, passed as one +/// parameter from the committer down through +/// [`super::ZebraDb::write_block`] to [`super::ZebraDb::prepare_trees_batch`]. +/// +/// The fields are independent: a checkpoint-handoff block sets `sync_below` +/// but leaves `anchor_roots` `None` (it writes the real frontier via the +/// legacy path instead), while a non-handoff fast block sets both. +#[derive(Clone, Copy, Debug, Default)] +pub struct VctWriteData { + /// When `Some`, skip per-height tree writes and fold these roots into the anchor set. + pub anchor_roots: Option<(sapling::tree::Root, orchard::tree::Root)>, + /// When `Some(height)`, mark the database as vct-synced below `height`. + pub sync_below: Option, +} + /// The verified final frontiers embedded for `network`, if supported. /// /// Mainnet uses the constant embedded in the binary. Regtest has no fixed checkpoint — diff --git a/zebra-state/src/service/finalized_state/zebra_db/block.rs b/zebra-state/src/service/finalized_state/zebra_db/block.rs index 3177bee75ae..f22037ad2dd 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block.rs @@ -46,8 +46,10 @@ use crate::{ shielded::CommitmentRootsByHeight, transparent::{AddressBalanceLocationUpdates, OutputLocation}, }, + vct::VctWriteData, zebra_db::{metrics::block_precommit_metrics, ZebraDb}, - FromDisk, IntoDisk, RawBytes, PRUNING_METADATA, VCT_SYNC_METADATA, VCT_UPGRADE_METADATA, + FromDisk, IntoDisk, RawBytes, COMMITMENT_ROOTS_BY_HEIGHT, PRUNING_METADATA, + VCT_SYNC_METADATA, VCT_UPGRADE_METADATA, }, HashOrHeight, }; @@ -62,26 +64,10 @@ const ZAKURA_HEADER_HASH_BY_HEIGHT: &str = "zakura_header_hash_by_height"; const ZAKURA_HEADER_HEIGHT_BY_HASH: &str = "zakura_header_height_by_hash"; const ZAKURA_HEADER_BY_HEIGHT: &str = "zakura_header_by_height"; pub const ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT: &str = "zakura_header_body_size_by_height"; -pub const ZAKURA_HEADER_COMMITMENT_ROOTS_BY_HEIGHT: &str = - "zakura_header_commitment_roots_by_height"; #[derive(Copy, Clone, Debug, Eq, PartialEq)] struct AdvertisedBodySize(u32); -/// Verified-commitment-trees data used only while checkpoint fast-sync skips -/// per-height Sapling and Orchard tree writes. -/// -/// Live semantic sync must pass `None` for this data so it keeps writing the -/// full per-height trees. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub struct VctData { - /// Roots to insert into the anchor set instead of writing per-height trees. - pub(in super::super) anchor_roots: (sapling::tree::Root, orchard::tree::Root), - - /// Height below which per-height trees are absent in a VCT-synced database. - pub(in super::super) sync_below: Height, -} - impl AdvertisedBodySize { fn new(size: u32) -> Option { (size != 0).then_some(Self(size)) @@ -205,10 +191,7 @@ impl ZebraDb { &self, range: impl RangeBounds, ) -> Vec { - let roots_by_height = self - .db - .cf_handle(ZAKURA_HEADER_COMMITMENT_ROOTS_BY_HEIGHT) - .unwrap(); + let roots_by_height = self.db.cf_handle(COMMITMENT_ROOTS_BY_HEIGHT).unwrap(); self.db .zs_forward_range_iter::<_, block::Height, CommitmentRootsByHeight, _>( @@ -640,10 +623,7 @@ impl ZebraDb { &self, roots: impl IntoIterator, ) -> Result<(), rocksdb::Error> { - let cf = self - .db - .cf_handle(ZAKURA_HEADER_COMMITMENT_ROOTS_BY_HEIGHT) - .unwrap(); + let cf = self.db.cf_handle(COMMITMENT_ROOTS_BY_HEIGHT).unwrap(); let mut batch = DiskWriteBatch::new(); for roots in roots { batch.zs_insert( @@ -668,10 +648,7 @@ impl ZebraDb { &self, heights: impl IntoIterator, ) -> Result<(), rocksdb::Error> { - let cf = self - .db - .cf_handle(ZAKURA_HEADER_COMMITMENT_ROOTS_BY_HEIGHT) - .unwrap(); + let cf = self.db.cf_handle(COMMITMENT_ROOTS_BY_HEIGHT).unwrap(); let mut batch = DiskWriteBatch::new(); for height in heights { batch.zs_delete(&cf, height); @@ -1033,6 +1010,7 @@ impl ZebraDb { /// - Propagates any errors from computing the block's chain value balance change or /// from applying the change to the chain value balance #[allow(clippy::unwrap_in_result)] + #[allow(clippy::too_many_arguments)] pub(in super::super) fn write_block( &mut self, finalized: FinalizedBlock, @@ -1040,7 +1018,7 @@ impl ZebraDb { network: &Network, source: &str, retention: RetentionPlan, - vct_data: Option, + vct_data: VctWriteData, ) -> Result { let tx_hash_indexes: HashMap = finalized .transaction_hashes @@ -1582,7 +1560,7 @@ impl DiskWriteBatch { prev_note_commitment_trees: Option, store_raw_transactions: bool, precomputed_raw_txs: Option>, - vct_data: Option, + vct_data: VctWriteData, ) -> Result<(), CommitCheckpointVerifiedError> { // Commit block, transaction, and note commitment tree data. self.prepare_block_header_and_transaction_data_batch( @@ -1591,10 +1569,8 @@ impl DiskWriteBatch { store_raw_transactions, precomputed_raw_txs, )?; - let zakura_header_commitment_roots_by_height = zebra_db - .db - .cf_handle(ZAKURA_HEADER_COMMITMENT_ROOTS_BY_HEIGHT) - .unwrap(); + let zakura_header_commitment_roots_by_height = + zebra_db.db.cf_handle(COMMITMENT_ROOTS_BY_HEIGHT).unwrap(); self.zs_delete(&zakura_header_commitment_roots_by_height, finalized.height); // The consensus rules are silent on shielded transactions in the genesis block, @@ -1825,9 +1801,7 @@ impl DiskWriteBatch { let zakura_hash_by_height = db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); let zakura_height_by_hash = db.cf_handle(ZAKURA_HEADER_HEIGHT_BY_HASH).unwrap(); let zakura_body_size_by_height = db.cf_handle(ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT).unwrap(); - let zakura_roots_by_height = db - .cf_handle(ZAKURA_HEADER_COMMITMENT_ROOTS_BY_HEIGHT) - .unwrap(); + let zakura_roots_by_height = db.cf_handle(COMMITMENT_ROOTS_BY_HEIGHT).unwrap(); let tx_by_loc = db.cf_handle("tx_by_loc").unwrap(); let hash = block.hash(); @@ -1910,9 +1884,7 @@ impl DiskWriteBatch { let zakura_hash_by_height = db.cf_handle(ZAKURA_HEADER_HASH_BY_HEIGHT).unwrap(); let zakura_height_by_hash = db.cf_handle(ZAKURA_HEADER_HEIGHT_BY_HASH).unwrap(); let zakura_body_size_by_height = db.cf_handle(ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT).unwrap(); - let zakura_roots_by_height = db - .cf_handle(ZAKURA_HEADER_COMMITMENT_ROOTS_BY_HEIGHT) - .unwrap(); + let zakura_roots_by_height = db.cf_handle(COMMITMENT_ROOTS_BY_HEIGHT).unwrap(); let tx_by_loc = db.cf_handle("tx_by_loc").unwrap(); let existing_zakura_header: Option> = @@ -2024,10 +1996,7 @@ impl DiskWriteBatch { .db .cf_handle(ZAKURA_HEADER_BODY_SIZE_BY_HEIGHT) .unwrap(); - let roots_by_height = zebra_db - .db - .cf_handle(ZAKURA_HEADER_COMMITMENT_ROOTS_BY_HEIGHT) - .unwrap(); + let roots_by_height = zebra_db.db.cf_handle(COMMITMENT_ROOTS_BY_HEIGHT).unwrap(); let anchor_height = zebra_db .header_height(anchor) @@ -2204,20 +2173,29 @@ impl DiskWriteBatch { } else { self.zs_delete(&body_size_by_height, height); } - let roots = &tree_aux_roots[index]; - self.zs_insert( - &roots_by_height, - height, - CommitmentRootsByHeight { - sapling: roots.sapling_root, - orchard: roots.orchard_root, - ironwood: roots.ironwood_root, - sapling_tx: roots.sapling_tx, - orchard_tx: roots.orchard_tx, - ironwood_tx: roots.ironwood_tx, - auth_data_root: roots.auth_data_root, - }, - ); + // A height with a committed body already has a *verified* row in the + // serving index, written by the body-commit batch. Peer-supplied roots + // are unauthenticated until verify-before-commit, and that verification + // only ever runs above the body tip — so a header range re-delivered + // over committed heights (a header store behind the body store, or a + // late range response racing body sync) must never overwrite the + // verified row: committed roots win on any overlap (design §9). + if !zebra_db.contains_body_at_height(height) { + let roots = &tree_aux_roots[index]; + self.zs_insert( + &roots_by_height, + height, + CommitmentRootsByHeight { + sapling: roots.sapling_root, + orchard: roots.orchard_root, + ironwood: roots.ironwood_root, + sapling_tx: roots.sapling_tx, + orchard_tx: roots.orchard_tx, + ironwood_tx: roots.ironwood_tx, + auth_data_root: roots.auth_data_root, + }, + ); + } } Ok(block::Hash::from( diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/prune.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/prune.rs index 6c24d19b301..c2b8a1c493b 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/prune.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/prune.rs @@ -49,7 +49,7 @@ fn new_state_with_blocks(config: &Config, network: &Network) -> FinalizedState { .expect("test data deserializes"); state - .commit_finalized_direct(block.into(), None, "prune tests") + .commit_finalized_direct(block.into(), None, None, "prune tests") .expect("test block is valid"); } @@ -80,7 +80,7 @@ fn new_state_with_checkpoint_retention( .expect("test data deserializes"); state - .commit_finalized_direct(block.into(), None, "checkpoint retention tests") + .commit_finalized_direct(block.into(), None, None, "checkpoint retention tests") .expect("test block is valid"); } @@ -351,7 +351,7 @@ fn checkpoint_retention_hands_off_to_online_pruning_at_start() { .expect("test data deserializes"); state - .commit_finalized_direct(block.into(), None, "checkpoint handoff tests") + .commit_finalized_direct(block.into(), None, None, "checkpoint handoff tests") .expect("test block is valid"); } @@ -386,7 +386,7 @@ fn checkpoint_retention_hands_off_to_online_pruning_at_start() { .expect("test data deserializes"); state - .commit_finalized_direct(block.into(), None, "checkpoint handoff tests") + .commit_finalized_direct(block.into(), None, None, "checkpoint handoff tests") .expect("handoff block is valid"); let online_prune_until = @@ -630,7 +630,7 @@ fn archive_to_pruned_checkpoint_sync_drains_archive_raw_transactions_before_skip .expect("test data deserializes"); archive_state - .commit_finalized_direct(block.into(), None, "archive phase") + .commit_finalized_direct(block.into(), None, None, "archive phase") .expect("archive block is valid"); } @@ -670,7 +670,7 @@ fn archive_to_pruned_checkpoint_sync_drains_archive_raw_transactions_before_skip .expect("test data deserializes"); pruned_state - .commit_finalized_direct(block.into(), None, "archive to pruned checkpoint") + .commit_finalized_direct(block.into(), None, None, "archive to pruned checkpoint") .expect("checkpoint block is valid"); assert_eq!( @@ -728,7 +728,7 @@ fn archive_backlog_flag_is_recomputed_when_reopening_a_pruned_database() { .expect("test data deserializes"); archive_state - .commit_finalized_direct(block.into(), None, "archive phase") + .commit_finalized_direct(block.into(), None, None, "archive phase") .expect("archive block is valid"); } std::mem::drop(archive_state); @@ -761,7 +761,7 @@ fn archive_backlog_flag_is_recomputed_when_reopening_a_pruned_database() { .zcash_deserialize_into() .expect("test data deserializes"); pruned_state - .commit_finalized_direct(block.into(), None, "archive to pruned checkpoint") + .commit_finalized_direct(block.into(), None, None, "archive to pruned checkpoint") .expect("checkpoint block is valid"); assert_eq!( pruned_state.db.lowest_retained_height(), @@ -842,7 +842,7 @@ fn contextual_commits_keep_raw_transactions_before_checkpoint_retention_start() .zcash_deserialize_into() .expect("genesis test data deserializes"); state - .commit_finalized_direct(genesis.into(), None, "contextual retention tests") + .commit_finalized_direct(genesis.into(), None, None, "contextual retention tests") .expect("genesis block is valid"); let block: Arc = blocks @@ -858,7 +858,7 @@ fn contextual_commits_keep_raw_transactions_before_checkpoint_retention_start() let finalizable = FinalizableBlock::new(contextually_verified, Treestate::default()); state - .commit_finalized_direct(finalizable, None, "contextual retention tests") + .commit_finalized_direct(finalizable, None, None, "contextual retention tests") .expect("contextual block is valid"); assert!( @@ -1197,7 +1197,8 @@ fn reopening_fast_synced_database_in_pruned_mode_with_vct_disabled_succeeds() { } #[test] -fn reopening_interrupted_fast_sync_without_a_root_source_succeeds() { +#[should_panic(expected = "interrupted below the last checkpoint height")] +fn reopening_interrupted_fast_sync_without_a_root_source_panics() { let _init_guard = zebra_test::init(); let network = Mainnet; @@ -1220,23 +1221,19 @@ fn reopening_interrupted_fast_sync_without_a_root_source_succeeds() { state.db.write_batch(batch).expect("marker batch writes"); } - // This storage-only PR does not enable the fast path or its resume policy. Reopening preserves - // the marker so later fast-sync wiring can decide how to resume or repair. - let reopened = FinalizedState::new( + // Reopening with the fast path disabled must refuse: the on-disk frontier is stale and no + // root source exists, so the committer would otherwise stall on every below-handoff block. + let _state = FinalizedState::new( &config, &network, #[cfg(feature = "elasticsearch")] false, ); - assert_eq!( - reopened.db.vct_synced_below(), - Some(Height(100)), - "the interrupted fast-sync marker is preserved across reopen" - ); } #[test] -fn reopening_interrupted_fast_sync_with_vct_disabled_succeeds() { +#[should_panic(expected = "interrupted below the last checkpoint height")] +fn reopening_interrupted_fast_sync_with_vct_disabled_panics() { let _init_guard = zebra_test::init(); let network = Mainnet; @@ -1259,19 +1256,14 @@ fn reopening_interrupted_fast_sync_with_vct_disabled_succeeds() { state.db.write_batch(batch).expect("marker batch writes"); } - // The force-disable knob only affects the future fast-sync source selection. The database still - // opens and keeps the marker available for follow-up resume or repair logic. - let reopened = FinalizedState::new( + // Reopening with the VCT force-disable knob must refuse: the on-disk frontier is stale and + // no root source exists, so the committer would otherwise stall on every below-handoff block. + let _state = FinalizedState::new( &config, &network, #[cfg(feature = "elasticsearch")] false, ); - assert_eq!( - reopened.db.vct_synced_below(), - Some(Height(100)), - "the interrupted fast-sync marker is preserved across reopen" - ); } #[test] diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs index 82a214099c6..b875ed9903d 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs @@ -195,7 +195,7 @@ fn test_block_and_transaction_data_with_network(network: Network) { .expect("test data deserializes"); state - .commit_finalized_direct(block.into(), None, "snapshot tests") + .commit_finalized_direct(block.into(), None, None, "snapshot tests") .expect("test block is valid"); let mut settings = insta::Settings::clone_current(); diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs index e72f7bb74c5..92230f2e8d1 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs @@ -543,7 +543,7 @@ fn committed_body_releases_only_its_height_and_keeps_the_frontier() { } #[test] -fn write_block_deletes_matching_provisional_zakura_roots() { +fn write_block_replaces_matching_provisional_zakura_roots_with_verified_row() { let _init_guard = zebra_test::init(); let genesis = zebra_test::vectors::BLOCK_MAINNET_GENESIS_BYTES .zcash_deserialize_into::>() @@ -562,6 +562,8 @@ fn write_block_deletes_matching_provisional_zakura_roots() { .map(ToString::to_string), false, ); + // Provisional rows are distinguishable from the verified row the body commit + // writes: `root_at` uses a zeroed auth-data root, the commit stores the real one. let roots = [root_at(Height(1)), root_at(Height(2))]; write_full_block(&mut state, genesis); @@ -573,17 +575,94 @@ fn write_block_deletes_matching_provisional_zakura_roots() { roots.to_vec() ); - write_full_block(&mut state, block1); + write_full_block(&mut state, block1.clone()); - assert!(state - .zakura_header_commitment_roots_by_height_range(Height(1)..=Height(1)) - .is_empty()); + // The body commit replaces the provisional row at its height with the verified + // row derived from the committed treestate, and leaves higher provisional rows + // untouched. + let verified_row = BlockCommitmentRoots { + height: Height(1), + sapling_root: sapling::tree::NoteCommitmentTree::default().root(), + orchard_root: orchard::tree::NoteCommitmentTree::default().root(), + ironwood_root: zebra_chain::ironwood::tree::NoteCommitmentTree::default().root(), + sapling_tx: 0, + orchard_tx: 0, + ironwood_tx: 0, + auth_data_root: block1.auth_data_root(), + }; + assert_eq!( + state.zakura_header_commitment_roots_by_height_range(Height(1)..=Height(1)), + vec![verified_row] + ); assert_eq!( state.zakura_header_commitment_roots_by_height_range(Height(2)..=Height(2)), vec![root_at(Height(2))] ); } +/// A header range re-delivered over a height whose body is already committed (a +/// header store behind the body store, or a late range response racing body sync) +/// must not overwrite the verified serving-index row with peer-supplied roots: +/// committed roots win on any overlap (design §9). +#[test] +fn header_range_roots_do_not_overwrite_committed_serving_index_rows() { + let _init_guard = zebra_test::init(); + let genesis = zebra_test::vectors::BLOCK_MAINNET_GENESIS_BYTES + .zcash_deserialize_into::>() + .expect("genesis block deserializes"); + let block1 = zebra_test::vectors::BLOCK_MAINNET_1_BYTES + .zcash_deserialize_into::>() + .expect("block 1 deserializes"); + let mut state = ZebraDb::new( + &Config::ephemeral(), + STATE_DATABASE_KIND, + &state_database_format_version_in_code(), + &Mainnet, + true, + STATE_COLUMN_FAMILIES_IN_CODE + .iter() + .map(ToString::to_string), + false, + ); + + write_full_block(&mut state, genesis.clone()); + write_full_block(&mut state, block1.clone()); + + let verified_rows = state.zakura_header_commitment_roots_by_height_range(Height(1)..=Height(1)); + assert_eq!( + verified_rows.len(), + 1, + "the body commit writes the verified serving-index row" + ); + + // Re-deliver the real header for the committed height, but with garbage roots — + // exactly what a malicious serving peer can put in a `Headers` response, since + // root bytes are unauthenticated at header-commit time. + let mut poisoned = root_at(Height(1)); + poisoned.sapling_tx = 99; + poisoned.auth_data_root = zebra_chain::block::merkle::AuthDataRoot::from([0xAA; 32]); + + let mut batch = DiskWriteBatch::new(); + batch + .prepare_header_range_batch_with_roots( + &state, + genesis.hash(), + std::slice::from_ref(&block1.header), + &[0], + &[poisoned], + ) + .expect("re-delivering the same header over a committed height is accepted"); + state + .write_batch(batch) + .expect("header range batch writes successfully"); + + assert_eq!( + state.zakura_header_commitment_roots_by_height_range(Height(1)..=Height(1)), + verified_rows, + "peer-supplied roots must not overwrite the verified committed row" + ); +} + /// Pruning-readiness guard: a committed height whose body is removed (as online /// pruning deletes `tx_by_loc` rows) keeps its header readable from the retained /// consensus `block_header_by_height`, because the header readers are not gated @@ -1310,7 +1389,7 @@ fn write_full_block(state: &mut ZebraDb, block: Arc) { &Mainnet, "test", RetentionPlan::Store, - None, + Default::default(), ) .expect("block commit succeeds"); } diff --git a/zebra-state/src/service/finalized_state/zebra_db/prune.rs b/zebra-state/src/service/finalized_state/zebra_db/prune.rs index 42813e35378..e32c54f7988 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/prune.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/prune.rs @@ -355,7 +355,7 @@ mod tests { .expect("test data deserializes"); state - .commit_finalized_direct(block.into(), None, "offline prune tests") + .commit_finalized_direct(block.into(), None, None, "offline prune tests") .expect("test block is valid"); } diff --git a/zebra-state/src/service/finalized_state/zebra_db/rollback.rs b/zebra-state/src/service/finalized_state/zebra_db/rollback.rs index 5cce82d505f..66d53149090 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/rollback.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/rollback.rs @@ -44,7 +44,6 @@ use crate::{ ZebraDb, }, COMMITMENT_ROOTS_BY_HEIGHT, STATE_COLUMN_FAMILIES_IN_CODE, - ZAKURA_HEADER_COMMITMENT_ROOTS_BY_HEIGHT, }, non_finalized_state::write_semantically_verified_backup_block, }, @@ -906,10 +905,7 @@ fn delete_zakura_headers_above(db: &ZebraDb, batch: &mut DiskWriteBatch, target_ .db .cf_handle("zakura_header_body_size_by_height") .unwrap(); - let roots_by_height = db - .db - .cf_handle(ZAKURA_HEADER_COMMITMENT_ROOTS_BY_HEIGHT) - .unwrap(); + let roots_by_height = db.db.cf_handle(COMMITMENT_ROOTS_BY_HEIGHT).unwrap(); let Some((tip_height, _tip_hash)) = db .db @@ -1368,6 +1364,43 @@ mod tests { } } + /// `serve_block_roots` reads a request that starts at or above the upgrade height `U` straight + /// from the serving index, without touching the per-height trees. + #[test] + fn serve_block_roots_serves_at_or_above_upgrade_from_index() { + let _init_guard = zebra_test::init(); + let db = ephemeral_mainnet_db(); + + // Index covers [4, 6]; the upgrade height is U = 4. + let mut batch = DiskWriteBatch::new(); + batch.update_vct_upgrade_marker(&db, Height(4)); + for height in 4u32..=6 { + batch.insert_commitment_roots_by_height( + &db, + Height(height), + &sapling_root(height.into()), + &orchard_root(height.into()), + &zebra_chain::ironwood::tree::NoteCommitmentTree::default().root(), + 0, + 0, + 0, + &zebra_chain::block::merkle::AuthDataRoot::from([0u8; 32]), + ); + } + db.write_batch(batch) + .expect("seeding the serving index succeeds"); + + let served = crate::service::finalized_state::serve_block_roots(&db, Height(4)..=Height(6)); + assert_eq!( + served + .into_iter() + .map(|root| root.height) + .collect::>(), + vec![Height(4), Height(5), Height(6)], + "a request at or above U is served from the index" + ); + } + /// `delete_zakura_headers_above` must truncate every Zakura header CF above the target, /// including the hash→height index, while leaving rows at or below the target intact. This /// is the consistency guarantee that lets a rolled-back snapshot re-sync bodies from its tip diff --git a/zebra-state/src/service/finalized_state/zebra_db/shielded.rs b/zebra-state/src/service/finalized_state/zebra_db/shielded.rs index 8422c7ac935..4fb99bb2f0f 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/shielded.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/shielded.rs @@ -33,7 +33,8 @@ use crate::{ service::finalized_state::{ disk_db::{DiskWriteBatch, ReadDisk, WriteDisk}, disk_format::{shielded::CommitmentRootsByHeight, RawBytes}, - zebra_db::{block::VctData, ZebraDb}, + vct::VctWriteData, + zebra_db::ZebraDb, COMMITMENT_ROOTS_BY_HEIGHT, }, TransactionLocation, @@ -775,8 +776,12 @@ impl DiskWriteBatch { zebra_db: &ZebraDb, finalized: &FinalizedBlock, prev_note_commitment_trees: Option, - vct_data: Option, + fast_write: VctWriteData, ) { + let VctWriteData { + anchor_roots: vct_anchor_roots, + sync_below: vct_sync_below, + } = fast_write; let FinalizedBlock { height, treestate: @@ -818,8 +823,8 @@ impl DiskWriteBatch { // below the checkpoint handoff height). Written in the same atomic batch as // every vct commit, so a vct-synced database always carries the marker and // the read/validity guards never see absent trees without it. - if let Some(VctData { sync_below, .. }) = vct_data { - self.update_vct_sync_marker(zebra_db, sync_below); + if let Some(handoff) = vct_sync_below { + self.update_vct_sync_marker(zebra_db, handoff); } // POC (verified-commitment-trees) vct path: the committer skipped the @@ -829,15 +834,7 @@ impl DiskWriteBatch { // tree CFs and subtrees entirely. The Sprout tree is unchanged below any // modern checkpoint, so it is correctly left untouched here. // See docs/design/verified-commitment-trees.md. - if let Some(VctData { - anchor_roots: (sapling_root, orchard_root), - sync_below, - }) = vct_data - { - // Mark the database as vct-synced in the same atomic batch as every - // fast commit, so the read/validity guards never see absent trees - // without the handoff marker. - self.update_vct_sync_marker(zebra_db, sync_below); + if let Some((sapling_root, orchard_root)) = vct_anchor_roots { self.insert_sapling_anchor(zebra_db, &sapling_root); self.insert_orchard_anchor(zebra_db, &orchard_root); // Persist the per-height roots into the serving index even though no per-height diff --git a/zebra-state/src/service/write.rs b/zebra-state/src/service/write.rs index fceafbf42d7..6ac39f9ed58 100644 --- a/zebra-state/src/service/write.rs +++ b/zebra-state/src/service/write.rs @@ -16,7 +16,7 @@ use tokio::sync::{ use tracing::Span; use zebra_chain::{ block::{self, Height}, - parallel::commitment_aux::BlockCommitmentRoots, + parallel::{commitment_aux::BlockCommitmentRoots, tree::NoteCommitmentTrees}, }; use crate::{ @@ -39,6 +39,10 @@ use crate::service::{ non_finalized_state::Chain, }; +mod vct_write; + +use vct_write::VctWriteManager; + /// The maximum size of the parent error map. /// /// We allow enough space for multiple concurrent chain forks with errors. @@ -220,7 +224,7 @@ impl From for NonFinalizedWriteMessage { /// `finalized_state` or `non_finalized_state` and channels for sending /// it blocks. #[derive(Clone, Debug)] -pub(super) struct BlockWriteSender { +pub struct BlockWriteSender { /// A channel to send blocks to the `block_write_task`, /// so they can be written to the [`NonFinalizedState`]. pub non_finalized: Option>, @@ -328,9 +332,13 @@ impl WriteBlockWorkerTask { backup_dir_path, } = &mut self; - let mut prev_finalized_note_commitment_trees = None; + let mut prev_finalized_note_commitment_trees: Option = None; let mut deferred_non_finalized_messages = VecDeque::new(); + // Look-ahead buffering and root-stall tracking for the VCT fast-sync + // checkpoint path. See [`VctWriteManager`]. + let mut vct_write_manager = VctWriteManager::default(); + // Write all the finalized blocks sent by the state, // until the state closes the finalized block channel's sender. loop { @@ -357,13 +365,16 @@ impl WriteBlockWorkerTask { Err(TryRecvError::Disconnected) => {} } - let ordered_block = match finalized_block_write_receiver.try_recv() { - Ok(block) => block, - Err(TryRecvError::Empty) => { - std::thread::park_timeout(Duration::from_millis(10)); - continue; - } - Err(TryRecvError::Disconnected) => break, + let ordered_block = match vct_write_manager.take_ready() { + Some(block) => block, + None => match finalized_block_write_receiver.try_recv() { + Ok(block) => block, + Err(TryRecvError::Empty) => { + std::thread::park_timeout(Duration::from_millis(10)); + continue; + } + Err(TryRecvError::Disconnected) => break, + }, }; // TODO: split these checks into separate functions @@ -394,22 +405,95 @@ impl WriteBlockWorkerTask { Assuming a parent block failed, and dropping this block", ); + // The pipeline is broken; drop any look-ahead so commit resumes + // from the real finalized tip. + vct_write_manager.reset(finalized_state); + // We don't want to send a reset here, because it could overwrite a valid sent hash std::mem::drop(ordered_block); continue; } - // Try committing the block - match finalized_state - .commit_finalized(ordered_block, prev_finalized_note_commitment_trees.take()) + // Peek the next block so VCT fast commits can verify the current + // block's supplied roots against the successor's header. + vct_write_manager.fill_successor(finalized_block_write_receiver, &ordered_block); + + // A non-handoff VCT fast block's supplied roots are authenticated by + // its successor's header. If the successor is not buffered yet, keep + // this block local and wait instead of surfacing a checkpoint commit + // error through the invalid-block reset path. + if vct_write_manager.is_lookahead_empty() + && finalized_state.vct_fast_needs_successor(ordered_block.0.height) { + tracing::trace!( + height = ?ordered_block.0.height, + hash = ?ordered_block.0.hash, + "VCT: deferring fast checkpoint commit until successor is buffered" + ); + vct_write_manager.defer(ordered_block); + std::thread::park_timeout(Duration::from_millis(10)); + continue; + } + + // The buffered VCT successor (if any) lets the committer verify this block's + // verified-commitment-trees fixture roots before trusting them: a block's + // roots are only committed by the next block's header. Its auth data root + // is already precomputed by the checkpoint verifier. + let next_vct_block = vct_write_manager.next_vct_block(); + let prev_note_commitment_trees = prev_finalized_note_commitment_trees.take(); + let prev_note_commitment_trees_for_retry = prev_note_commitment_trees.clone(); + + let next_block_took_vct_path = + finalized_state.vct_fast_will_apply(ordered_block.0.height); + + // Try committing the block + match finalized_state.commit_finalized( + ordered_block, + prev_note_commitment_trees, + next_vct_block, + ) { Ok((finalized, note_commitment_trees)) => { + // Whether this successful commit consumed header-carried + // tree-aux roots to skip the note-commitment frontier rebuild. + if next_block_took_vct_path { + metrics::counter!("state.vct.fast_path.hit").increment(1); + } else { + metrics::counter!("state.vct.fast_path.miss").increment(1); + } + + // A successful commit clears any VCT root stall: log recovery and reset + // the stalled-height gauge if it had been raised. + vct_write_manager.on_commit_success(); + let tip_block = ChainTipBlock::from(finalized); prev_finalized_note_commitment_trees = Some(note_commitment_trees); chain_tip_sender.set_finalized_tip(tip_block); } - Err(error) => { + Err((ordered_block, error)) => { + // Retryable VCT root stalls (an absent/evicted root, or one not yet + // verifiable for lack of a buffered successor) park-and-retry the same + // block in place rather than resetting the queue. An absent root waits + // for header sync to deliver it; an await-successor stall just waits for + // the next block to be downloaded into the look-ahead, so it polls faster. + if let Some(height) = error.vct_retryable_height() { + let needs_refetch = error.vct_supplied_root_unavailable_height(); + + prev_finalized_note_commitment_trees = prev_note_commitment_trees_for_retry; + let wait = vct_write_manager.on_retryable_error( + height, + needs_refetch.is_some(), + ordered_block, + ); + std::thread::park_timeout(wait); + continue; + } + let finalized_tip = finalized_state.db.tip(); + let _ = ordered_block.1.send(Err(error.clone())); + + // The commit failed and the queue is being reset, so clear + // any buffered look-ahead block. + vct_write_manager.reset(finalized_state); // The last block in the queue failed, so we can't commit the next block. // Instead, we need to reset the state queue, @@ -581,10 +665,17 @@ impl WriteBlockWorkerTask { tracing::trace!("finalizing block past the reorg limit"); let contextually_verified_with_trees = non_finalized_state.finalize(); prev_finalized_note_commitment_trees = finalized_state - .commit_finalized_direct(contextually_verified_with_trees, prev_finalized_note_commitment_trees.take(), "commit contextually-verified request") - .expect( - "unexpected finalized block commit error: note commitment and history trees were already checked by the non-finalized state", - ).1.into(); + .commit_finalized_direct( + contextually_verified_with_trees, + prev_finalized_note_commitment_trees.take(), + None, + "commit contextually-verified request", + ) + .expect( + "unexpected finalized block commit error: note commitment and history trees were already checked by the non-finalized state", + ) + .1 + .into(); } // Update the metrics if semantic and contextual validation passes diff --git a/zebra-state/src/service/write/vct_write.rs b/zebra-state/src/service/write/vct_write.rs new file mode 100644 index 00000000000..42d9711e416 --- /dev/null +++ b/zebra-state/src/service/write/vct_write.rs @@ -0,0 +1,205 @@ +//! Look-ahead buffering and root-stall tracking for the checkpoint write loop's +//! verified-commitment-trees (vct) fast path. + +use std::{ + collections::VecDeque, + time::{Duration, Instant}, +}; + +use tokio::sync::mpsc::UnboundedReceiver; +use tracing::info; +use zebra_chain::block::Height; + +use crate::service::{ + finalized_state::{FinalizedState, NextVctBlock}, + queued_blocks::QueuedCheckpointVerified, +}; + +/// Delay between retryable VCT root-miss commit attempts while the peer cache refills. +const VCT_ROOT_RETRY_WAIT: Duration = Duration::from_millis(500); + +/// Delay between retryable VCT await-successor commit attempts. Shorter than +/// [`VCT_ROOT_RETRY_WAIT`]: the root is already cached and only the next block needs to be +/// downloaded into the look-ahead, so a tighter poll keeps the one-block commit lag small. +const VCT_AWAIT_SUCCESSOR_WAIT: Duration = Duration::from_millis(20); + +/// How long a single checkpoint height may stay stuck on a retryable VCT root stall before +/// the committer escalates to an error-level log and a `state.vct.root.stalled.height` gauge. +/// Transient waits (a successor still downloading, a root still in flight) clear well within +/// this; staying stuck past it means no peer can serve a root the frozen frontier requires, +/// and — by design — the committer will not recompute against the stale frontier, so the node +/// cannot advance until a peer supplies it. Surfacing that loudly is the operator's only signal. +const VCT_ROOT_STALL_WARN_AFTER: Duration = Duration::from_secs(30); + +/// Look-ahead buffering and root-stall tracking for the checkpoint write +/// loop's verified-commitment-trees (vct) fast path. Bundles the state the +/// loop needs to authenticate a fast block's supplied roots against its +/// successor and to retry/escalate a stuck height, so their invariants +/// (single log per stall, look-ahead cleared on reset) live next to the data +/// they guard. +#[derive(Default)] +pub(super) struct VctWriteManager { + /// One-block look-ahead: the current block's supplied roots are + /// authenticated by the successor's header commitment. + lookahead: VecDeque, + /// A block parked for retry (awaiting a successor, or a re-fetched root) + /// instead of going through the invalid-block reset path. + retry: Option, + /// `(height, first-seen)` of the height currently stuck retrying, if any. + stall: Option<(Height, Instant)>, + /// Whether the current stall has already been escalated to an + /// error-level log and gauge. + stall_logged: bool, +} + +impl VctWriteManager { + /// Takes the next locally-buffered block ready to commit (a parked retry, + /// then the look-ahead), if any. + pub(super) fn take_ready(&mut self) -> Option { + self.retry.take().or_else(|| self.lookahead.pop_front()) + } + + /// Clears the look-ahead and any cached successor prevalidation, for a + /// queue reset (wrong-height block, or a hard commit failure). + pub(super) fn reset(&mut self, finalized_state: &mut FinalizedState) { + self.lookahead.clear(); + finalized_state.clear_vct_prevalidated_next(); + } + + /// Buffers the direct successor of `current` into the look-ahead, if available. + /// + /// Discards any buffered block that does not extend `current`: it cannot + /// witness this commit or be committed next. Since retries are taken before + /// the look-ahead, leaving a non-successor parked there could wedge the retry + /// loop against the wrong witness. Dropping its response sender lets upstream + /// redeliver the block. + pub(super) fn fill_successor( + &mut self, + receiver: &mut UnboundedReceiver, + current: &QueuedCheckpointVerified, + ) { + loop { + let front_links = self + .lookahead + .front() + .map(|next| next.0.block.header.previous_block_hash == current.0.hash); + + match front_links { + Some(true) => break, + Some(false) => { + let dropped = self + .lookahead + .pop_front() + .expect("the front entry was just inspected"); + tracing::debug!( + current_height = ?current.0.height, + current_hash = ?current.0.hash, + dropped_height = ?dropped.0.height, + dropped_hash = ?dropped.0.hash, + "dropping a buffered block that does not extend the block being \ + committed. Assuming a parent block failed, and dropping this block", + ); + } + None => match receiver.try_recv() { + Ok(next) => self.lookahead.push_back(next), + Err(_) => break, + }, + } + } + } + + /// `true` when no block is buffered as the look-ahead successor. + pub(super) fn is_lookahead_empty(&self) -> bool { + self.lookahead.is_empty() + } + + /// The buffered successor block's header data, used to verify the + /// current block's supplied vct roots before trusting them. + pub(super) fn next_vct_block(&self) -> Option { + self.lookahead.front().map(|next| NextVctBlock { + block: next.0.block.clone(), + auth_data_root: next.0.auth_data_root, + }) + } + + /// Parks `block` for retry instead of committing it now. + pub(super) fn defer(&mut self, block: QueuedCheckpointVerified) { + self.retry = Some(block); + } + + /// A successful commit clears any vct root stall: logs recovery and + /// resets the stalled-height gauge if the stall had been escalated. + pub(super) fn on_commit_success(&mut self) { + if self.stall.is_some() { + if self.stall_logged { + info!( + stalled_height = ?self.stall.map(|(h, _)| h), + "VCT: checkpoint commit recovered; the stalled height now has a verifiable supplied root" + ); + metrics::gauge!("state.vct.root.stalled.height").set(0.0); + } + self.stall = None; + self.stall_logged = false; + } + } + + /// Tracks and, past the warn threshold, escalates a retryable vct root + /// stall at `height`, parks `block` for retry, and returns how long the + /// caller should park before retrying. + pub(super) fn on_retryable_error( + &mut self, + height: Height, + needs_refetch: bool, + block: QueuedCheckpointVerified, + ) -> Duration { + metrics::counter!("state.vct.root.retry.count").increment(1); + + // Escalate a stall that persists on the same height past the warn + // threshold: a transient wait resolves in a few polls and stays + // quiet, but a height stuck longer means no peer can serve a root the + // frozen frontier requires — the node will not advance (it will not, + // by design, recompute against the stale frontier). Surface it loudly. + match self.stall { + Some((stuck, _)) if stuck == height => {} + _ => { + self.stall = Some((height, Instant::now())); + self.stall_logged = false; + } + } + if !self.stall_logged + && self + .stall + .is_some_and(|(_, since)| since.elapsed() >= VCT_ROOT_STALL_WARN_AFTER) + { + tracing::error!( + ?height, + awaiting_refetch = needs_refetch, + stalled_for = ?VCT_ROOT_STALL_WARN_AFTER, + "VCT: checkpoint commit stalled with no verifiable supplied root; \ + the node cannot advance until a peer serves this height (it will \ + not recompute against the frozen frontier)" + ); + metrics::gauge!("state.vct.root.stalled.height").set(f64::from(height.0)); + self.stall_logged = true; + } else { + tracing::warn!( + ?height, + block_height = ?block.0.height, + block_hash = ?block.0.hash, + awaiting_refetch = needs_refetch, + "VCT: supplied root not yet verifiable; retrying checkpoint commit in place" + ); + } + + self.retry = Some(block); + + if needs_refetch { + VCT_ROOT_RETRY_WAIT + } else { + VCT_AWAIT_SUCCESSOR_WAIT + } + } +} + +#[cfg(test)] +mod tests; diff --git a/zebra-state/src/service/write/vct_write/tests.rs b/zebra-state/src/service/write/vct_write/tests.rs new file mode 100644 index 00000000000..92a26f01754 --- /dev/null +++ b/zebra-state/src/service/write/vct_write/tests.rs @@ -0,0 +1,280 @@ +use std::{sync::Arc, time::Duration, time::Instant}; + +use tokio::sync::{mpsc, oneshot}; +use zebra_chain::{block::Height, serialization::ZcashDeserializeInto}; + +use super::{VctWriteManager, VCT_AWAIT_SUCCESSOR_WAIT, VCT_ROOT_RETRY_WAIT}; +use crate::{ + request::CheckpointVerifiedBlock, service::queued_blocks::QueuedCheckpointVerified, + tests::FakeChainHelper, +}; + +/// Builds a distinct [`QueuedCheckpointVerified`] with a discarded response channel, so +/// tests can tell blocks apart by hash without caring about the response side. +fn queued_block(seed: u128) -> QueuedCheckpointVerified { + let genesis = zebra_test::vectors::BLOCK_MAINNET_GENESIS_BYTES + .zcash_deserialize_into::>() + .expect("genesis block deserializes"); + let block = genesis.make_fake_child().set_work(seed); + let (rsp_tx, _rsp_rx) = oneshot::channel(); + (CheckpointVerifiedBlock::from(block), rsp_tx) +} + +/// Builds a queued block that extends `parent`, so it passes the +/// successor-linkage check in [`VctWriteManager::fill_successor`]. +fn queued_child_of(parent: &QueuedCheckpointVerified, seed: u128) -> QueuedCheckpointVerified { + let block = parent.0.block.clone().make_fake_child().set_work(seed); + let (rsp_tx, _rsp_rx) = oneshot::channel(); + (CheckpointVerifiedBlock::from(block), rsp_tx) +} + +#[test] +fn take_ready_returns_none_when_empty() { + let mut manager = VctWriteManager::default(); + assert!(manager.take_ready().is_none()); +} + +#[test] +fn take_ready_prefers_retry_over_lookahead() { + let mut manager = VctWriteManager::default(); + let retry_block = queued_block(1); + let retry_hash = retry_block.0.hash; + let lookahead_block = queued_block(2); + let lookahead_hash = lookahead_block.0.hash; + + manager.lookahead.push_back(lookahead_block); + manager.retry = Some(retry_block); + + let first = manager.take_ready().expect("retry block is ready"); + assert_eq!( + first.0.hash, retry_hash, + "retry must be taken before lookahead" + ); + + let second = manager.take_ready().expect("lookahead block is ready"); + assert_eq!(second.0.hash, lookahead_hash); + + assert!(manager.take_ready().is_none()); +} + +#[test] +fn defer_makes_block_the_next_ready_block() { + let mut manager = VctWriteManager::default(); + let block = queued_block(1); + let hash = block.0.hash; + + manager.defer(block); + + let ready = manager.take_ready().expect("deferred block is ready"); + assert_eq!(ready.0.hash, hash); +} + +#[test] +fn fill_successor_only_buffers_one_linking_block_at_a_time() { + let mut manager = VctWriteManager::default(); + let (tx, mut rx) = mpsc::unbounded_channel(); + let current = queued_block(1); + let first = queued_child_of(¤t, 2); + let first_hash = first.0.hash; + let second = queued_child_of(&first, 3); + let second_hash = second.0.hash; + tx.send(first).expect("channel is open"); + tx.send(second).expect("channel is open"); + + // First fill: lookahead was empty, so it pulls exactly one (linking) block. + manager.fill_successor(&mut rx, ¤t); + assert_eq!(manager.lookahead.len(), 1); + assert_eq!(manager.lookahead.front().unwrap().0.hash, first_hash); + + // Second fill: the front already links to `current`, so it's a no-op — the + // second block stays buffered in the channel, not in the look-ahead. + manager.fill_successor(&mut rx, ¤t); + assert_eq!(manager.lookahead.len(), 1); + assert_eq!(manager.lookahead.front().unwrap().0.hash, first_hash); + + // Once the successor commits it becomes the current block; draining the + // look-ahead lets the next fill pull that block's own successor. + let committed = manager.take_ready().expect("look-ahead block is ready"); + manager.fill_successor(&mut rx, &committed); + assert_eq!(manager.lookahead.front().unwrap().0.hash, second_hash); +} + +/// A buffered block that does not extend the block being committed is discarded: +/// it can't witness the commit, and — because a parked retry is always taken +/// before the look-ahead — it would otherwise never be popped, wedging the retry +/// loop against a bogus witness that spuriously evicts good roots. +#[test] +fn fill_successor_discards_non_successors_and_keeps_the_linking_block() { + let mut manager = VctWriteManager::default(); + let (tx, mut rx) = mpsc::unbounded_channel(); + let current = queued_block(1); + // A sibling of `current` (another child of genesis): buffered, but not linking. + let non_successor = queued_block(2); + let successor = queued_child_of(¤t, 3); + let successor_hash = successor.0.hash; + + manager.lookahead.push_back(non_successor); + tx.send(successor).expect("channel is open"); + + manager.fill_successor(&mut rx, ¤t); + + assert_eq!(manager.lookahead.len(), 1); + assert_eq!( + manager.lookahead.front().unwrap().0.hash, + successor_hash, + "the non-linking block is dropped and the true successor takes its place" + ); + let witness = manager + .next_vct_block() + .expect("successor is buffered") + .block; + assert_eq!(witness.header.previous_block_hash, current.0.hash); +} + +/// With no linking block buffered anywhere, the look-ahead ends up empty, so the +/// write loop's await-successor deferral engages instead of a bogus witness. +#[test] +fn fill_successor_empties_the_lookahead_when_no_successor_exists() { + let mut manager = VctWriteManager::default(); + let (_tx, mut rx) = mpsc::unbounded_channel(); + let current = queued_block(1); + manager.lookahead.push_back(queued_block(2)); + + manager.fill_successor(&mut rx, ¤t); + + assert!(manager.is_lookahead_empty()); + assert!(manager.next_vct_block().is_none()); +} + +#[test] +fn next_vct_block_reflects_the_lookahead_front() { + let mut manager = VctWriteManager::default(); + assert!(manager.next_vct_block().is_none()); + + let block = queued_block(1); + let expected_block = block.0.block.clone(); + let expected_auth_data_root = block.0.auth_data_root; + manager.lookahead.push_back(block); + + let next_vct_block = manager.next_vct_block().expect("look-ahead has a block"); + assert_eq!(next_vct_block.block, expected_block); + assert_eq!(next_vct_block.auth_data_root, expected_auth_data_root); +} + +#[test] +fn reset_clears_the_lookahead() { + let mut manager = VctWriteManager::default(); + manager.lookahead.push_back(queued_block(1)); + manager.lookahead.push_back(queued_block(2)); + + let network = zebra_chain::parameters::Network::Mainnet; + let config = crate::Config::ephemeral(); + let mut finalized_state = crate::service::finalized_state::FinalizedState::new( + &config, + &network, + #[cfg(feature = "elasticsearch")] + false, + ); + + manager.reset(&mut finalized_state); + + assert!(manager.is_lookahead_empty()); +} + +#[test] +fn on_commit_success_is_a_no_op_without_a_stall() { + let mut manager = VctWriteManager::default(); + // Must not panic, and must leave the (already-clear) stall state alone. + manager.on_commit_success(); + assert!(manager.stall.is_none()); + assert!(!manager.stall_logged); +} + +#[test] +fn on_commit_success_clears_an_escalated_stall() { + let mut manager = VctWriteManager::default(); + let height = Height(1); + + // Force the stall past the warn threshold so it gets escalated (logged). + manager.stall = Some((height, Instant::now() - Duration::from_secs(31))); + manager.on_retryable_error(height, true, queued_block(1)); + assert!(manager.stall_logged, "the stall should have been escalated"); + + manager.on_commit_success(); + + assert!(manager.stall.is_none()); + assert!(!manager.stall_logged); +} + +#[test] +fn on_retryable_error_keeps_the_same_stall_start_for_a_repeated_height() { + let mut manager = VctWriteManager::default(); + let height = Height(5); + + manager.on_retryable_error(height, true, queued_block(1)); + let first_seen = manager.stall.expect("a stall is now tracked").1; + + manager.on_retryable_error(height, true, queued_block(2)); + let still_first_seen = manager.stall.expect("the stall is still tracked").1; + + assert_eq!( + first_seen, still_first_seen, + "retrying the same height must not reset the stall's start time" + ); +} + +#[test] +fn on_retryable_error_resets_the_stall_for_a_different_height() { + let mut manager = VctWriteManager::default(); + + manager.on_retryable_error(Height(1), true, queued_block(1)); + manager.stall_logged = true; // simulate an already-escalated stall + + manager.on_retryable_error(Height(2), true, queued_block(2)); + + assert_eq!(manager.stall.map(|(h, _)| h), Some(Height(2))); + assert!( + !manager.stall_logged, + "a new height starts a fresh, unescalated stall" + ); +} + +#[test] +fn on_retryable_error_escalates_past_the_warn_threshold() { + let mut manager = VctWriteManager::default(); + let height = Height(7); + + // Below the threshold: not escalated yet. + manager.on_retryable_error(height, true, queued_block(1)); + assert!(!manager.stall_logged); + + // Backdate the stall past the warn threshold and retry the same height. + manager.stall = Some((height, Instant::now() - Duration::from_secs(31))); + manager.on_retryable_error(height, true, queued_block(2)); + assert!(manager.stall_logged); +} + +#[test] +fn on_retryable_error_parks_the_block_for_retry() { + let mut manager = VctWriteManager::default(); + let block = queued_block(1); + let hash = block.0.hash; + + manager.on_retryable_error(Height(1), true, block); + + let ready = manager + .take_ready() + .expect("the block was parked for retry"); + assert_eq!(ready.0.hash, hash); +} + +#[test] +fn on_retryable_error_wait_depends_on_needs_refetch() { + let mut manager = VctWriteManager::default(); + + let refetch_wait = manager.on_retryable_error(Height(1), true, queued_block(1)); + assert_eq!(refetch_wait, VCT_ROOT_RETRY_WAIT); + + let successor_wait = manager.on_retryable_error(Height(2), false, queued_block(2)); + assert_eq!(successor_wait, VCT_AWAIT_SUCCESSOR_WAIT); +} diff --git a/zebra-state/src/tests/setup.rs b/zebra-state/src/tests/setup.rs index 7c4c4a0bd6c..34b1785a84d 100644 --- a/zebra-state/src/tests/setup.rs +++ b/zebra-state/src/tests/setup.rs @@ -113,7 +113,7 @@ pub(crate) fn new_state_with_mainnet_genesis( let genesis = CheckpointVerifiedBlock::from(genesis); finalized_state - .commit_finalized_direct(genesis.clone().into(), None, "test") + .commit_finalized_direct(genesis.clone().into(), None, None, "test") .expect("unexpected invalid genesis block test vector"); assert_eq!( From 6b619e49324435dc323b7b8af2d835746f2a6346 Mon Sep 17 00:00:00 2001 From: Dev Ojha Date: Thu, 2 Jul 2026 23:04:25 -0500 Subject: [PATCH 2/3] Minor comment resolution --- .../src/service/finalized_state/disk_format/chain.rs | 7 +++++++ zebra-state/src/service/finalized_state/vct.rs | 5 ++++- .../src/service/finalized_state/zebra_db/shielded.rs | 8 ++------ 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/zebra-state/src/service/finalized_state/disk_format/chain.rs b/zebra-state/src/service/finalized_state/disk_format/chain.rs index 39e5e0e136e..ae86b025eb5 100644 --- a/zebra-state/src/service/finalized_state/disk_format/chain.rs +++ b/zebra-state/src/service/finalized_state/disk_format/chain.rs @@ -133,6 +133,13 @@ impl FromDisk for HistoryTreeParts { let bytes = bytes.as_ref(); let options = bincode::DefaultOptions::new(); + // Try the current entry width first. Databases written before NU6.3 widened + // `zcash_history::Entry` store narrower entries that fail to parse at the current width, + // so fall back to the legacy width and zero-pad each entry up to the current width. + // + // Legacy-width rows can fail the current-width decoder with errors other than + // `UnexpectedEof`, because the wider entry can read into the next legacy entry and + // interpret arbitrary entry bytes as bincode control bytes. options .deserialize::(bytes) .or_else(|_| { diff --git a/zebra-state/src/service/finalized_state/vct.rs b/zebra-state/src/service/finalized_state/vct.rs index 487fdfcaab9..6e31e0b39aa 100644 --- a/zebra-state/src/service/finalized_state/vct.rs +++ b/zebra-state/src/service/finalized_state/vct.rs @@ -278,7 +278,10 @@ impl VctState { /// fast path depends on, grouped so their invariants live next to the data they guard. #[derive(Clone, Debug)] pub(crate) struct VctCommitState { - /// The root source (peer/fixture/capture mode), or `None` for legacy recompute. + /// The root source (peer/fixture/capture mode), or `None` for any of: + /// - checkpoint sync is disabled + /// - vct fast sync is disabled + /// - legacy Zebra checkpoint sync source: Option>, /// `(height, hash)` of the next block already validated by the previous fast diff --git a/zebra-state/src/service/finalized_state/zebra_db/shielded.rs b/zebra-state/src/service/finalized_state/zebra_db/shielded.rs index 4fb99bb2f0f..d22c0c208c6 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/shielded.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/shielded.rs @@ -778,10 +778,6 @@ impl DiskWriteBatch { prev_note_commitment_trees: Option, fast_write: VctWriteData, ) { - let VctWriteData { - anchor_roots: vct_anchor_roots, - sync_below: vct_sync_below, - } = fast_write; let FinalizedBlock { height, treestate: @@ -823,7 +819,7 @@ impl DiskWriteBatch { // below the checkpoint handoff height). Written in the same atomic batch as // every vct commit, so a vct-synced database always carries the marker and // the read/validity guards never see absent trees without it. - if let Some(handoff) = vct_sync_below { + if let Some(handoff) = fast_write.sync_below { self.update_vct_sync_marker(zebra_db, handoff); } @@ -834,7 +830,7 @@ impl DiskWriteBatch { // tree CFs and subtrees entirely. The Sprout tree is unchanged below any // modern checkpoint, so it is correctly left untouched here. // See docs/design/verified-commitment-trees.md. - if let Some((sapling_root, orchard_root)) = vct_anchor_roots { + if let Some((sapling_root, orchard_root)) = fast_write.anchor_roots { self.insert_sapling_anchor(zebra_db, &sapling_root); self.insert_orchard_anchor(zebra_db, &orchard_root); // Persist the per-height roots into the serving index even though no per-height From 0868fa6fd4a431d720e7d844663e4a447dbd231d Mon Sep 17 00:00:00 2001 From: roman Date: Fri, 3 Jul 2026 10:21:10 -0600 Subject: [PATCH 3/3] rename --- zebra-state/src/service/finalized_state.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zebra-state/src/service/finalized_state.rs b/zebra-state/src/service/finalized_state.rs index a1a0a730b4d..0b967d2b181 100644 --- a/zebra-state/src/service/finalized_state.rs +++ b/zebra-state/src/service/finalized_state.rs @@ -846,7 +846,7 @@ impl FinalizedState { { // Last checkpoint verification: verify the supplied frontiers against // this block's verified roots. - self.vct_verify_handoff_frontier_roots( + self.vct_verify_last_checkpoint_frontier_roots( height, &sapling_frontier, &orchard_frontier, @@ -1126,7 +1126,7 @@ impl FinalizedState { } /// Verify checkpoint handoff frontiers against this block's supplied roots. - fn vct_verify_handoff_frontier_roots( + fn vct_verify_last_checkpoint_frontier_roots( &mut self, height: block::Height, sapling_frontier: &sapling::tree::NoteCommitmentTree,