From 456f7b5903f43381c07c9b01375f62ec8a30ebba Mon Sep 17 00:00:00 2001 From: zancas Date: Wed, 1 Jul 2026 10:15:57 -0700 Subject: [PATCH 001/134] Harden init-podman-volumes against stale volume records `podman volume inspect` reads podman's database, not the filesystem, so a named volume can exist in the DB while its backing directory is gone (after a partial ~/.local/share/containers cleanup or an interrupted `podman system reset`). The old inspect-only guard then skipped `podman volume create`, and the later container-test mount failed with "failed to validate if host path is dangerous: lstat .../volumes/zaino-container-target: no such file or directory", producing no nextest summary for the live partitions. Guard on the actual Mountpoint directory instead: when the DB record is missing or its backing dir is gone, `podman volume rm --force` drops the dangling record before recreating it, keeping the DB and on-disk storage in sync. Recovers automatically on the next `makers test` rather than requiring a full `podman system reset`. Co-Authored-By: Claude Opus 4.8 (1M context) --- tools/scripts/init-podman-volumes.sh | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tools/scripts/init-podman-volumes.sh b/tools/scripts/init-podman-volumes.sh index cd9d6f92e..8ae163ded 100755 --- a/tools/scripts/init-podman-volumes.sh +++ b/tools/scripts/init-podman-volumes.sh @@ -1,8 +1,19 @@ #!/usr/bin/env bash # Initialize named podman volumes for container builds. +# `podman volume inspect` reads podman's database, not the filesystem, so a +# volume can "exist" in the DB while its backing directory is gone (e.g. after +# a partial ~/.local/share/containers cleanup or an interrupted `podman system +# reset`). Podman then refuses to mount it with "failed to validate if host +# path is dangerous: lstat .../volumes/: no such file or directory". Guard +# on the actual Mountpoint directory, and force-recreate a stale record so the +# DB and on-disk storage stay in sync. for vol in zaino-container-target zaino-cargo-git zaino-cargo-registry; do - if ! podman volume inspect "$vol" >/dev/null 2>&1; then + mountpoint="$(podman volume inspect --format '{{.Mountpoint}}' "$vol" 2>/dev/null)" + if [[ -z "$mountpoint" || ! -d "$mountpoint" ]]; then + # -z: no DB record. `! -d`: DB record present but backing dir gone; + # rm --force drops the dangling record before we recreate it. + podman volume rm --force "$vol" >/dev/null 2>&1 || true podman volume create "$vol" echo "Created podman volume: $vol" fi From 6d87cb0340785ee682a498925fd1851a431cc26a Mon Sep 17 00:00:00 2001 From: zancas Date: Wed, 1 Jul 2026 13:36:25 -0700 Subject: [PATCH 002/134] Derive seam depth from NON_FINALIZED_DEPTH, not stale literal 100/99 Two production sites hard-coded the finalised/non-finalised seam depth as the pre-zebra-10 value. zebra-10 raised MAX_BLOCK_REORG_HEIGHT 99->1000, so the real NON_FINALIZED_DEPTH is 1001 and both were wrong: - non_finalised_state.rs: the reorg-detection recursion bound used `height + 100`, aborting on any reorg deeper than 100 blocks; now uses MAX_NFS_DEPTH, matching its sibling bounds at 553/863. - chain_index.rs get_compact_block_range: the cache/finalised handoff used `tip - 99`, routing non-finalised blocks (tip-NON_FINALIZED_DEPTH .. tip-99) to the finalised stream that does not hold them; now derives from finalized_height_floor(tip). Unchanged for chains shorter than the depth (tip-99 == floor+1 at depth 100). Also update production doc comments describing the seam as a literal "100 blocks" to reference NON_FINALIZED_DEPTH / MAX_BLOCK_REORG_HEIGHT. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/zaino-state/src/chain_index.rs | 14 ++++++++------ .../zaino-state/src/chain_index/finalised_state.rs | 4 ++-- .../src/chain_index/non_finalised_state.rs | 4 ++-- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/packages/zaino-state/src/chain_index.rs b/packages/zaino-state/src/chain_index.rs index 182fd6107..84e377347 100644 --- a/packages/zaino-state/src/chain_index.rs +++ b/packages/zaino-state/src/chain_index.rs @@ -2,7 +2,7 @@ //! //! Components: //! - Mempool: Holds mempool transactions -//! - NonFinalisedState: Holds block data for the top 100 blocks of all chains. +//! - NonFinalisedState: Holds block data for the top `NON_FINALIZED_DEPTH` blocks of all chains. //! - FinalisedState: Holds block data for the remainder of the best chain. //! //! - Chain: Holds chain / block structs used internally by the ChainIndex. @@ -196,7 +196,7 @@ fn branch_len_to_active_chain( /// The interface to the chain index. /// /// `ChainIndex` provides a unified interface for querying blockchain data from different -/// backend sources. It combines access to both finalized state (older than 100 blocks) and +/// backend sources. It combines access to both finalized state (older than `NON_FINALIZED_DEPTH` blocks) and /// non-finalized state (recent blocks that may still be reorganized). /// /// # Implementation @@ -1621,7 +1621,7 @@ impl ChainIndex for NodeBackedChainIndexSubscriber ChainIndex for NodeBackedChainIndexSubscriber Result, Self::Error> { let chain_tip_height = self.best_chaintip(nonfinalized_snapshot).await?.height; - // The nonfinalized cache holds the tip block plus the previous 99 blocks (100 total), - // so the lowest possible cached height is `tip - 99` (saturating at 0). - let lowest_nonfinalized_height = types::Height(chain_tip_height.0.saturating_sub(99)); + // The finalised state serves heights up to `finalized_height_floor(tip)` + // (= `tip - NON_FINALIZED_DEPTH`, saturating at genesis); the non-finalised cache serves + // everything above it, so the lowest non-finalised height is one past the finalised floor. + let lowest_nonfinalized_height = + types::Height(finalized_height_floor(chain_tip_height.0).0 + 1); let is_ascending = start_height <= end_height; diff --git a/packages/zaino-state/src/chain_index/finalised_state.rs b/packages/zaino-state/src/chain_index/finalised_state.rs index 456df8ddf..15d249eed 100644 --- a/packages/zaino-state/src/chain_index/finalised_state.rs +++ b/packages/zaino-state/src/chain_index/finalised_state.rs @@ -2,8 +2,8 @@ //! //! This module provides `FinalisedState`, the *finalised* portion of the chain index. //! -//! “Finalised” in this context means: All but the top 100 blocks in the blockchain. This follows -//! Zebra's model where a reorg of depth greater than 100 would require a complete network restart. +//! “Finalised” in this context means: All but the top `NON_FINALIZED_DEPTH` blocks in the blockchain. This +//! follows Zebra's model where a reorg deeper than `MAX_BLOCK_REORG_HEIGHT` would require a complete network restart. //! //! `FinalisedState` is a facade over a `FinalisedSource` — the //! backing implementation that actually serves finalised data. That backing is **not necessarily a diff --git a/packages/zaino-state/src/chain_index/non_finalised_state.rs b/packages/zaino-state/src/chain_index/non_finalised_state.rs index 5e473e977..beca24fef 100644 --- a/packages/zaino-state/src/chain_index/non_finalised_state.rs +++ b/packages/zaino-state/src/chain_index/non_finalised_state.rs @@ -99,7 +99,7 @@ impl ChainIndexSnapshot { #[derive(Debug, Clone)] /// A snapshot of the nonfinalized state as it existed when this was created. pub(crate) struct NonfinalizedBlockCacheSnapshot { - /// the set of all known blocks < 100 blocks old + /// the set of all known blocks less than `NON_FINALIZED_DEPTH` blocks old /// this includes all blocks on-chain, as well as /// all blocks known to have been on-chain before being /// removed by a reorg. Blocks reorged away have no height. @@ -606,7 +606,7 @@ impl NonFinalizedState { height_to_recurse_to: Option, ) -> Result<(), SyncError> { if height_to_recurse_to - .is_some_and(|height| height + 100 < working_snapshot.best_tip.height) + .is_some_and(|height| height + MAX_NFS_DEPTH < working_snapshot.best_tip.height) { return Err(SyncError::ReorgFailure( "reorg detection recursed beyond reason".to_string(), From 81e93dc7355da20aa2a8376a0b9f450a709d9c90 Mon Sep 17 00:00:00 2001 From: zancas Date: Wed, 1 Jul 2026 13:55:50 -0700 Subject: [PATCH 003/134] Introduce zaino_common::consensus; single-source seam depth & coinbase maturity Add a consensus module holding the workspace's authoritative, upstream-derived constants, so nothing hard-codes them: - MAX_NONFINALISED_DEPTH = zebra_chain MAX_BLOCK_REORG_HEIGHT + 1 - COINBASE_MATURITY = zebra_chain MIN_TRANSPARENT_COINBASE_MATURITY - FAST_TEST_MAX_NONFINALISED_DEPTH = MAX_NONFINALISED_DEPTH / 10 (tractable test seam) Both consts live in zaino-common, which already depends on zebra-chain, so this adds no new workspace dependency. Re-source zaino-state's NON_FINALIZED_DEPTH from the mod (production = real MAX_NONFINALISED_DEPTH; in-crate tests select the derived FAST_TEST value so short mock fixtures still exercise a moving seam), and the tx_out_set_accumulator maturity split from COINBASE_MATURITY. Behaviour-preserving: in-crate test depth stays 100, production stays 1001; the values are now derived from a single upstream source rather than literals. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/zaino-common/src/consensus.rs | 25 +++++++++++++ packages/zaino-common/src/lib.rs | 1 + packages/zaino-state/src/chain_index.rs | 35 +++++++++---------- .../v1/tx_out_set_accumulator.rs | 20 ++++++----- 4 files changed, 54 insertions(+), 27 deletions(-) create mode 100644 packages/zaino-common/src/consensus.rs diff --git a/packages/zaino-common/src/consensus.rs b/packages/zaino-common/src/consensus.rs new file mode 100644 index 000000000..1f02dcab1 --- /dev/null +++ b/packages/zaino-common/src/consensus.rs @@ -0,0 +1,25 @@ +//! Consensus-derived constants with a single source of truth. +//! +//! Each value derives from zebra's authoritative upstream constant; nothing else in +//! the workspace should hard-code these — reference this module instead. + +/// Number of confirmations before a coinbase output becomes spendable. +/// +/// Single source of truth, from zebra's transparent-coinbase maturity rule. +pub const COINBASE_MATURITY: u32 = zebra_chain::transparent::MIN_TRANSPARENT_COINBASE_MATURITY; + +/// Distance below the best-chain tip of the finalised / non-finalised seam: a block +/// buried deeper than this is finalised (reorg-stable). +/// +/// Derived from zebra's protocol reorg limit (`MAX_BLOCK_REORG_HEIGHT`). The `+ 1` +/// accounts for the tip block itself, preserving the historical seam semantics. +pub const MAX_NONFINALISED_DEPTH: u32 = + zebra_chain::parameters::constants::MAX_BLOCK_REORG_HEIGHT + 1; + +/// A tractable one-tenth of [`MAX_NONFINALISED_DEPTH`], for fast tests that need a +/// finalised seam without building a full ~[`MAX_NONFINALISED_DEPTH`]-block chain. +/// +/// Integer division, so this is `100` when the real depth is `1001`. Test-only: it +/// lets in-crate tests select a shallow seam that still derives from the single +/// source of truth rather than a hard-coded literal. +pub const FAST_TEST_MAX_NONFINALISED_DEPTH: u32 = MAX_NONFINALISED_DEPTH / 10; diff --git a/packages/zaino-common/src/lib.rs b/packages/zaino-common/src/lib.rs index 3e46a792c..e3f62182c 100644 --- a/packages/zaino-common/src/lib.rs +++ b/packages/zaino-common/src/lib.rs @@ -4,6 +4,7 @@ //! and common utilities used across the Zaino blockchain indexer ecosystem. pub mod config; +pub mod consensus; pub mod logging; pub mod net; pub mod probing; diff --git a/packages/zaino-state/src/chain_index.rs b/packages/zaino-state/src/chain_index.rs index 84e377347..b15e4dc89 100644 --- a/packages/zaino-state/src/chain_index.rs +++ b/packages/zaino-state/src/chain_index.rs @@ -68,28 +68,25 @@ pub mod types; #[cfg(test)] mod tests; -/// Distance (in blocks) between the best-known chain tip and the -/// highest block that zaino treats as part of the finalized DB. +/// Distance (in blocks) between the best-known chain tip and the highest block that +/// zaino treats as part of the finalised DB — the finalised / non-finalised seam. /// -/// Sourced from Zebra's protocol-derived reorg bound. The `+ 1` -/// preserves the original literal-`100` behavior; deriving the -/// depth from an explicit wider-consensus reference is tracked in -/// zingolabs/zaino#1130. -#[cfg(not(test))] -pub(crate) const NON_FINALIZED_DEPTH: u32 = zebra_state::MAX_BLOCK_REORG_HEIGHT + 1; - -/// In-crate unit tests pin the depth at the pre-zebra-10 value (`100`). +/// Sourced from the workspace's single source of truth, +/// [`zaino_common::consensus`]. Production uses the real +/// [`MAX_NONFINALISED_DEPTH`]. In-crate unit tests select the tractable +/// [`FAST_TEST_MAX_NONFINALISED_DEPTH`] (= depth / 10) so the short mock-chain +/// fixtures still exercise a *moving* finalised seam — at the real depth +/// `finalized_height_floor` would saturate to genesis for the whole fixture and the +/// eviction/seam invariants become untestable (see zingolabs/zaino#1288). Both arms +/// derive from the same upstream reorg bound, so neither is a hard-coded literal. /// -/// Zebra 10 raised `MAX_BLOCK_REORG_HEIGHT` from 99 to 1000, so the -/// production depth is now 1001. The 201-block mock-chain test vector is -/// far shorter than that, so at the production depth `finalized_height_floor` -/// saturates to genesis for the whole fixture: the finalized seam never moves -/// off block 0 and the eviction/seam invariants become untestable (see -/// zingolabs/zaino#1288). The eviction and seam invariants are scale-free, so -/// exercising them at a tractable depth is sound; the production depth is -/// covered by the clientless suite, which reaches real chain heights. +/// [`MAX_NONFINALISED_DEPTH`]: zaino_common::consensus::MAX_NONFINALISED_DEPTH +/// [`FAST_TEST_MAX_NONFINALISED_DEPTH`]: zaino_common::consensus::FAST_TEST_MAX_NONFINALISED_DEPTH +#[cfg(not(test))] +pub(crate) const NON_FINALIZED_DEPTH: u32 = zaino_common::consensus::MAX_NONFINALISED_DEPTH; #[cfg(test)] -pub(crate) const NON_FINALIZED_DEPTH: u32 = 100; +pub(crate) const NON_FINALIZED_DEPTH: u32 = + zaino_common::consensus::FAST_TEST_MAX_NONFINALISED_DEPTH; /// Lower bound on zaino's finalized-DB tip, derived from the current /// best-known chain tip. diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/tx_out_set_accumulator.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/tx_out_set_accumulator.rs index e826d2631..45e3f74ac 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/tx_out_set_accumulator.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/tx_out_set_accumulator.rs @@ -2161,7 +2161,7 @@ mod tests { /// by a small range (`write_blocks_to_height`'s steady-state branch) — must produce exactly the /// accumulator a full from-genesis rebuild produces at the same tip, for all five fields. This is /// the correctness gate for `update_tx_out_set_accumulator_for_range`: with regtest coinbase - /// maturity of 100, splitting the sync at height 100 guarantees the second segment spends outputs + /// maturity `COINBASE_MATURITY`, splitting the sync at that height guarantees the second segment spends outputs /// created in the first (exercising the `transactions` "Set B" decrement) as well as outputs both /// created and spent within the range (the XOR-cancel case). #[tokio::test(flavor = "multi_thread")] @@ -2171,6 +2171,7 @@ mod tests { use crate::chain_index::finalised_state::capability::{ CapabilityRequest, DbRead, TransparentHistExt, }; + use zaino_common::consensus::COINBASE_MATURITY; let blocks = load_test_vectors().unwrap().blocks; let source = build_mockchain_source(blocks); @@ -2190,9 +2191,12 @@ mod tests { let zaino_db = FinalisedState::spawn(config, source.clone()).await.unwrap(); - // First segment builds the accumulator to height 100 (no watermark yet => full rebuild), - // the second advances it by a 100-block range => the incremental update path under test. - zaino_db.sync_to_height(Height(100), &source).await.unwrap(); + // First segment builds the accumulator to `COINBASE_MATURITY` (no watermark yet => full + // rebuild), the second advances it to the fixture tip => the incremental update path under test. + zaino_db + .sync_to_height(Height(COINBASE_MATURITY), &source) + .await + .unwrap(); // Background catch-up (>LONG_RUNNING_SYNC_THRESHOLD); wait for the persistent build + watermark. zaino_db.wait_until_synced().await; @@ -2200,15 +2204,15 @@ mod tests { .backend_for_cap(CapabilityRequest::WriteCore) .unwrap(); - // The watermark must sit at 100 here: that (together with gap 100 <= the incremental cap) - // pins the next sync to the incremental branch rather than a silent rebuild fallback that - // would make the comparison below trivial. + // The watermark must sit at `COINBASE_MATURITY` here: that (together with the gap to the tip + // <= the incremental cap) pins the next sync to the incremental branch rather than a silent + // rebuild fallback that would make the comparison below trivial. assert_eq!( backend .read_tx_out_set_accumulator_built_height() .await .unwrap(), - Some(Height(100)), + Some(Height(COINBASE_MATURITY)), "first segment must leave the accumulator watermark at the synced tip" ); From 961410de6a02f95d97cb589199c39741fe0bc545 Mon Sep 17 00:00:00 2001 From: zancas Date: Wed, 1 Jul 2026 14:17:15 -0700 Subject: [PATCH 004/134] Add fast-test-seam feature so cross-crate tests can use the tractable seam The FAST_TEST_MAX_NONFINALISED_DEPTH shrink was only reachable by zaino-state's own cfg(test) build. Cross-crate live tests (e2e, clientless) drive a real zaino-state instance whose seam depth is a compile-time constant, so the only way for them to use the tractable seam is a build feature. Add fast-test-seam to zaino-state: NON_FINALIZED_DEPTH now selects FAST_TEST under cfg(test) OR the feature, else the real MAX_NONFINALISED_DEPTH. e2e and clientless enable it via [dev-dependencies] only; resolver = "2" keeps it out of production graphs (verified: zainod's zaino-state does not carry the feature) and default-members excludes the live-test crates from production builds. This un-vacuums the clientless chain_cache eviction/finalisation tests: at depth 100 their ~150-block chain has a real finalised region instead of saturating to genesis at the production depth of 1001. Co-Authored-By: Claude Opus 4.8 (1M context) --- live-tests/clientless/Cargo.toml | 4 ++++ live-tests/e2e/Cargo.toml | 6 ++++++ packages/zaino-state/Cargo.toml | 8 ++++++++ packages/zaino-state/src/chain_index.rs | 17 +++++++++-------- 4 files changed, 27 insertions(+), 8 deletions(-) diff --git a/live-tests/clientless/Cargo.toml b/live-tests/clientless/Cargo.toml index b7af22cdb..ba912e690 100644 --- a/live-tests/clientless/Cargo.toml +++ b/live-tests/clientless/Cargo.toml @@ -40,6 +40,10 @@ tracing.workspace = true [dev-dependencies] anyhow = { workspace = true } +# Enable the tractable seam depth (fast-test-seam) so the chain_cache tests exercise +# finalisation/eviction at small chain heights. Dev-dependency only — the feature +# must never reach a shipped build (see the zaino-state feature comment). +zaino-state = { workspace = true, features = ["fast-test-seam"] } # Test utility zainod = { workspace = true } diff --git a/live-tests/e2e/Cargo.toml b/live-tests/e2e/Cargo.toml index 2d07ff170..2e6c53f4d 100644 --- a/live-tests/e2e/Cargo.toml +++ b/live-tests/e2e/Cargo.toml @@ -57,3 +57,9 @@ hex = { workspace = true } corez = { workspace = true } tower = { workspace = true } serde_json = { workspace = true } + +[dev-dependencies] +# Enable the tractable seam depth (fast-test-seam) so this crate's live tests can +# exercise finalisation/eviction at small chain heights. Dev-dependency only — the +# feature must never reach a shipped build (see the zaino-state feature comment). +zaino-state = { workspace = true, features = ["fast-test-seam"] } diff --git a/packages/zaino-state/Cargo.toml b/packages/zaino-state/Cargo.toml index 7ea77da65..4f96dbe9f 100644 --- a/packages/zaino-state/Cargo.toml +++ b/packages/zaino-state/Cargo.toml @@ -11,6 +11,14 @@ version = "0.3.1" [features] default = [] +# TEST-ONLY: shrink the finalised/non-finalised seam to the tractable +# `FAST_TEST_MAX_NONFINALISED_DEPTH` so cross-crate live tests can exercise +# finalisation/eviction with small chains. MUST appear only in `[dev-dependencies]` +# of the live-test crates — never a normal/shipped dependency: resolver = "2" keeps +# dev-dependency features out of production dependency graphs, and `default-members` +# excludes the live-test crates from production builds. +fast-test-seam = [] + # Support for connecting to a zcashd validator node. Opt-in (default off, docs/adr/0005); being # deprecated. Forwards to zaino-fetch. See # docs/adr/0001-zcashd-support-feature-gate.md. diff --git a/packages/zaino-state/src/chain_index.rs b/packages/zaino-state/src/chain_index.rs index b15e4dc89..727e9e81b 100644 --- a/packages/zaino-state/src/chain_index.rs +++ b/packages/zaino-state/src/chain_index.rs @@ -73,18 +73,19 @@ mod tests; /// /// Sourced from the workspace's single source of truth, /// [`zaino_common::consensus`]. Production uses the real -/// [`MAX_NONFINALISED_DEPTH`]. In-crate unit tests select the tractable -/// [`FAST_TEST_MAX_NONFINALISED_DEPTH`] (= depth / 10) so the short mock-chain -/// fixtures still exercise a *moving* finalised seam — at the real depth -/// `finalized_height_floor` would saturate to genesis for the whole fixture and the -/// eviction/seam invariants become untestable (see zingolabs/zaino#1288). Both arms -/// derive from the same upstream reorg bound, so neither is a hard-coded literal. +/// [`MAX_NONFINALISED_DEPTH`]. The tractable [`FAST_TEST_MAX_NONFINALISED_DEPTH`] +/// (= depth / 10) is selected for in-crate unit tests (`cfg(test)`) *and* for +/// cross-crate live tests that enable the `fast-test-seam` feature — so short mock +/// fixtures and small live chains still exercise a *moving* finalised seam. At the +/// real depth `finalized_height_floor` saturates to genesis for those fixtures and +/// the eviction/seam invariants become untestable (see zingolabs/zaino#1288). Both +/// arms derive from the same upstream reorg bound, so neither is a hard-coded literal. /// /// [`MAX_NONFINALISED_DEPTH`]: zaino_common::consensus::MAX_NONFINALISED_DEPTH /// [`FAST_TEST_MAX_NONFINALISED_DEPTH`]: zaino_common::consensus::FAST_TEST_MAX_NONFINALISED_DEPTH -#[cfg(not(test))] +#[cfg(not(any(test, feature = "fast-test-seam")))] pub(crate) const NON_FINALIZED_DEPTH: u32 = zaino_common::consensus::MAX_NONFINALISED_DEPTH; -#[cfg(test)] +#[cfg(any(test, feature = "fast-test-seam"))] pub(crate) const NON_FINALIZED_DEPTH: u32 = zaino_common::consensus::FAST_TEST_MAX_NONFINALISED_DEPTH; From fdf9bc9cd1e4fae4e1d87dcbeeefe9e6aeeb2c0c Mon Sep 17 00:00:00 2001 From: zancas Date: Wed, 1 Jul 2026 14:37:40 -0700 Subject: [PATCH 005/134] Rename NON_FINALIZED_DEPTH -> OPERATIONAL_NFS_DEPTH in zaino-state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The zaino-state-local const is a cfg/feature selector — it resolves to the authoritative MAX_NONFINALISED_DEPTH in production and to FAST_TEST_MAX_NONFINALISED_DEPTH under cfg(test)/fast-test-seam — not a source of truth. The old name read like a source and diverged in spelling from the finalised_state module (FINALIZED vs FINALISED). OPERATIONAL_NFS_DEPTH names it as the effective/selected depth the code operates on. Pure rename, compiler-verified (cargo check -p zaino-state --tests); behaviour unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/zaino-state/src/chain_index.rs | 20 +++++++++---------- .../src/chain_index/finalised_state.rs | 2 +- .../src/chain_index/non_finalised_state.rs | 14 ++++++------- .../chain_index/tests/proptest_blockgen.rs | 4 ++-- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/packages/zaino-state/src/chain_index.rs b/packages/zaino-state/src/chain_index.rs index 727e9e81b..87e0d7f68 100644 --- a/packages/zaino-state/src/chain_index.rs +++ b/packages/zaino-state/src/chain_index.rs @@ -2,7 +2,7 @@ //! //! Components: //! - Mempool: Holds mempool transactions -//! - NonFinalisedState: Holds block data for the top `NON_FINALIZED_DEPTH` blocks of all chains. +//! - NonFinalisedState: Holds block data for the top `OPERATIONAL_NFS_DEPTH` blocks of all chains. //! - FinalisedState: Holds block data for the remainder of the best chain. //! //! - Chain: Holds chain / block structs used internally by the ChainIndex. @@ -53,11 +53,11 @@ use zebra_rpc::{ use zebra_state::HashOrHeight; pub mod encoding; -/// All state below [`NON_FINALIZED_DEPTH`] blocks of the best-known chain tip. +/// All state below [`OPERATIONAL_NFS_DEPTH`] blocks of the best-known chain tip. pub mod finalised_state; /// State in the mempool, not yet on-chain pub mod mempool; -/// State within [`NON_FINALIZED_DEPTH`] blocks of the best-known chain tip; +/// State within [`OPERATIONAL_NFS_DEPTH`] blocks of the best-known chain tip; /// stored separately as it may be reorged. pub mod non_finalised_state; /// BlockchainSource @@ -84,9 +84,9 @@ mod tests; /// [`MAX_NONFINALISED_DEPTH`]: zaino_common::consensus::MAX_NONFINALISED_DEPTH /// [`FAST_TEST_MAX_NONFINALISED_DEPTH`]: zaino_common::consensus::FAST_TEST_MAX_NONFINALISED_DEPTH #[cfg(not(any(test, feature = "fast-test-seam")))] -pub(crate) const NON_FINALIZED_DEPTH: u32 = zaino_common::consensus::MAX_NONFINALISED_DEPTH; +pub(crate) const OPERATIONAL_NFS_DEPTH: u32 = zaino_common::consensus::MAX_NONFINALISED_DEPTH; #[cfg(any(test, feature = "fast-test-seam"))] -pub(crate) const NON_FINALIZED_DEPTH: u32 = +pub(crate) const OPERATIONAL_NFS_DEPTH: u32 = zaino_common::consensus::FAST_TEST_MAX_NONFINALISED_DEPTH; /// Lower bound on zaino's finalized-DB tip, derived from the current @@ -98,7 +98,7 @@ pub(crate) const NON_FINALIZED_DEPTH: u32 = /// `finalized_height` should account for the asymmetry /// (see zingolabs/zaino#1128). pub(crate) fn finalized_height_floor(chain_tip: u32) -> crate::Height { - crate::Height(chain_tip.saturating_sub(NON_FINALIZED_DEPTH)) + crate::Height(chain_tip.saturating_sub(OPERATIONAL_NFS_DEPTH)) } /// Current wall-clock time as a Unix timestamp in fractional seconds, for @@ -194,7 +194,7 @@ fn branch_len_to_active_chain( /// The interface to the chain index. /// /// `ChainIndex` provides a unified interface for querying blockchain data from different -/// backend sources. It combines access to both finalized state (older than `NON_FINALIZED_DEPTH` blocks) and +/// backend sources. It combines access to both finalized state (older than `OPERATIONAL_NFS_DEPTH` blocks) and /// non-finalized state (recent blocks that may still be reorganized). /// /// # Implementation @@ -937,7 +937,7 @@ impl NodeBackedChainIndex { Some(ref nfs) => nfs, None => { // Anchor the non-finalised state at `finalised_height` - // (= chain tip − NON_FINALIZED_DEPTH), never at genesis: a missing + // (= chain tip − OPERATIONAL_NFS_DEPTH), never at genesis: a missing // anchor used to fall through to genesis and then re-anchor up to the // lagging finalised tip, grinding millions of blocks one at a time // (#1261). `resolve_anchor_block` serves the anchor from the finalised @@ -1619,7 +1619,7 @@ impl ChainIndex for NodeBackedChainIndexSubscriber ChainIndex for NodeBackedChainIndexSubscriber NonFinalizedState { ) -> Result<(), SyncError> { let mut initial_state = self.get_snapshot(); let local_finalized_tip = finalized_db.to_reader().db_height().await?; - // Anchor floor: the non-finalised state must never start more than `NON_FINALIZED_DEPTH` + // Anchor floor: the non-finalised state must never start more than `OPERATIONAL_NFS_DEPTH` // blocks below the chain tip, even when the finalised DB tip lags far behind during // background catch-up. Without this floor a freshly-initialised (or genesis-fallback) // snapshot would try to bridge the entire gap from the finalised tip up to the chain tip one @@ -444,7 +444,7 @@ impl NonFinalizedState { local_finalized_tip .map(|height| height.0) .unwrap_or(0) - .max(u32::from(chain_height).saturating_sub(NON_FINALIZED_DEPTH)), + .max(u32::from(chain_height).saturating_sub(OPERATIONAL_NFS_DEPTH)), ); if initial_state.best_tip.height.0 < anchor_height.0 { let anchor_block = Self::resolve_anchor_block( @@ -520,7 +520,7 @@ impl NonFinalizedState { // we need to work backwards from it and update heights_to_hashes // with it and all its parents. } - if initial_state.best_tip.height + NON_FINALIZED_DEPTH + if initial_state.best_tip.height + OPERATIONAL_NFS_DEPTH < working_snapshot.best_tip.height { self.update(finalized_db.clone(), initial_state, working_snapshot) diff --git a/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs b/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs index 8dde1281c..aacc06536 100644 --- a/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs +++ b/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs @@ -31,7 +31,7 @@ use crate::{ source::{BlockchainSourceResult, GetTransactionLocation}, tests::{init_tracing, poll::poll_until, proptest_blockgen::proptest_helpers::add_segment}, types::BestChainLocation, - NonFinalizedSnapshot, NON_FINALIZED_DEPTH, + NonFinalizedSnapshot, OPERATIONAL_NFS_DEPTH, }, BlockHash, BlockchainSource, ChainIndex, ChainIndexConfig, NodeBackedChainIndex, NodeBackedChainIndexSubscriber, TransactionHash, @@ -52,7 +52,7 @@ fn passthrough_test( init_tracing(); let network = Network::Regtest(ActivationHeights::default()); // Long enough to have some finalized blocks to play with - let segment_length = NON_FINALIZED_DEPTH as usize + 20; + let segment_length = OPERATIONAL_NFS_DEPTH as usize + 20; // No need to worry about non-best chains for this test let branch_count = 1; From 3e30d477b73e584fce96b35661b3f83f6d163f06 Mon Sep 17 00:00:00 2001 From: zancas Date: Wed, 1 Jul 2026 14:57:13 -0700 Subject: [PATCH 006/134] Single-source cross-crate test-side seam references e2e and clientless live tests run zaino-state under fast-test-seam, so their seam-relative boundaries and prose must reference the mod's FAST_TEST_MAX_NONFINALISED_DEPTH (the depth those instances actually use), not hard-coded literals or the retired NON_FINALIZED_DEPTH name. - e2e get_outpoint_spenders: FINALITY_DEPTH = FAST_TEST_MAX_NONFINALISED_DEPTH + 5. - e2e prose (send_to_transparent_finalization, get_outpoint_spenders ignore reason): retarget the stale NON_FINALIZED_DEPTH mentions to the seam const. - clientless chain_cache: express the finalised-floor / retention boundaries as FAST_TEST_MAX_NONFINALISED_DEPTH +/- test margins (values preserved). MAX_NFS_DEPTH stays a zaino-state internal (retention window = seam + margin). Value-preserving and cargo-check clean; no NON_FINALIZED_DEPTH remains in live-tests. Co-Authored-By: Claude Opus 4.8 (1M context) --- live-tests/clientless/tests/chain_cache.rs | 30 +++++++++++++--------- live-tests/e2e/tests/devtool.rs | 20 +++++++++------ 2 files changed, 30 insertions(+), 20 deletions(-) diff --git a/live-tests/clientless/tests/chain_cache.rs b/live-tests/clientless/tests/chain_cache.rs index 8f724bd3c..78b33ceee 100644 --- a/live-tests/clientless/tests/chain_cache.rs +++ b/live-tests/clientless/tests/chain_cache.rs @@ -285,8 +285,8 @@ mod chain_query_interface { /// validator via the ephemeral passthrough. /// /// In ephemeral mode `db_height` is `0`, so the non-finalised cache retains - /// blocks down to `tip - MAX_NFS_DEPTH` (110). We therefore generate well - /// past that depth and query a height below `tip - 110`, so the reads are + /// blocks only down to `tip - MAX_NFS_DEPTH` (a small margin past the seam). We + /// therefore generate well past that and query a height below it, so the reads are /// genuinely served by the ephemeral *finalised* passthrough rather than the /// non-finalised cache. The test then: /// - fetches a finalised chain (indexed) block by height, re-fetches it by @@ -306,18 +306,22 @@ mod chain_query_interface { create_test_manager_and_chain_index::(validator, None, false, false, true) .await; - // Generate well past MAX_NFS_DEPTH (110) so low heights are evicted from - // the non-finalised cache and served by the ephemeral finalised passthrough. + // The finalised floor sits at `tip - seam`; the non-finalised cache retains a + // little past that. Generate well beyond it so low heights are evicted from the + // cache and served by the ephemeral finalised passthrough. `fast-test-seam` + // shrinks the seam to `FAST_TEST_MAX_NONFINALISED_DEPTH`, so a small chain suffices. + let seam = zaino_common::consensus::FAST_TEST_MAX_NONFINALISED_DEPTH; test_manager - .generate_blocks_and_wait_for_tip(150, &indexer) + .generate_blocks_and_wait_for_tip(seam + 50, &indexer) .await; let snapshot = indexer.snapshot_nonfinalized_state().await.unwrap(); let chain_height: u32 = json_service.get_blockchain_info().await.unwrap().blocks.0; - // `start_height` is below `tip - 110` (evicted from the NFS cache, served - // by the passthrough); `end_height` is above `tip - 100` (non-finalised). - let start_height: u32 = chain_height - 120; - let end_height: u32 = chain_height - 40; + // `start_height` is comfortably below the retention window → evicted from the NFS + // cache, served by the passthrough; `end_height` is above the finalised floor + // (`tip - seam`) → non-finalised. + let start_height: u32 = chain_height - (seam + 20); + let end_height: u32 = chain_height - seam / 2; let finalised_height = Height::try_from(start_height).unwrap(); // --- chain (indexed) block: fetch by height, then by its hash --- @@ -416,9 +420,11 @@ mod chain_query_interface { let snapshot = indexer.snapshot_nonfinalized_state().await.unwrap(); let chain_height = json_service.get_blockchain_info().await.unwrap().blocks.0; - let finalised_start = Height::try_from(chain_height - 150).unwrap(); - let finalised_tip = Height::try_from(chain_height - 100).unwrap(); - let end = Height::try_from(chain_height - 50).unwrap(); + // Finalised floor is `tip - seam`; pick a range straddling it. + let seam = zaino_common::consensus::FAST_TEST_MAX_NONFINALISED_DEPTH; + let finalised_start = Height::try_from(chain_height - (seam + 50)).unwrap(); + let finalised_tip = Height::try_from(chain_height - seam).unwrap(); + let end = Height::try_from(chain_height - seam / 2).unwrap(); let finalized_blocks = indexer .get_block_range(&snapshot, finalised_start, Some(finalised_tip)) diff --git a/live-tests/e2e/tests/devtool.rs b/live-tests/e2e/tests/devtool.rs index 5e14d4443..1bfa4da1c 100644 --- a/live-tests/e2e/tests/devtool.rs +++ b/live-tests/e2e/tests/devtool.rs @@ -1580,7 +1580,7 @@ async fn get_block_range_out_of_range_lower_bound() { /// zebrad): a transparent send to the recipient returns the same address txids /// whether served from the non-finalized chain (just mined) or after the /// 99-block advance pushes the send past the finalization boundary -/// (NON_FINALIZED_DEPTH = 100). +/// (the seam depth `FAST_TEST_MAX_NONFINALISED_DEPTH` under `fast-test-seam`). /// /// `#[ignore]`d (gated): the load-bearing 99-block advance mines orchard /// coinbase here — the devtool faucet funds via orchard, since the original's @@ -1611,8 +1611,9 @@ where .await .unwrap(); - // The load-bearing advance: 99 blocks push the send below NON_FINALIZED_DEPTH - // into the finalized DB. Orchard coinbase here (see #[ignore] rationale). + // The load-bearing advance: these blocks push the send below the seam + // (`FAST_TEST_MAX_NONFINALISED_DEPTH`) into the finalized DB. Orchard coinbase + // here (see #[ignore] rationale). test_manager .generate_blocks_bulk_and_wait_for_tips( 99, @@ -2066,10 +2067,13 @@ async fn get_outpoint_spenders_fetch_vs_state() { use zaino_state::chain_index::types::ChainScope; use zaino_state::ChainIndex as _; - // `NON_FINALIZED_DEPTH` is 100; mining this many blocks past a spend buries - // it under the finalised floor so `ChainScope::Finalised` can resolve it. - // A small margin above 100 keeps the boundary unambiguous. - const FINALITY_DEPTH: u32 = 105; + // Bury a spend this many blocks past its block to push it below the finalised + // floor (tip − seam depth) so `ChainScope::Finalised` can resolve it. This crate + // enables `fast-test-seam`, so the live zaino-state uses the tractable + // `FAST_TEST_MAX_NONFINALISED_DEPTH`; a small margin above it keeps the boundary + // unambiguous. (Without the feature the real seam is ~1000, impractical to mine + // here — see zingolabs/zaino#1352.) + const FINALITY_DEPTH: u32 = zaino_common::consensus::FAST_TEST_MAX_NONFINALISED_DEPTH + 5; const FUNDING_AMOUNT: u64 = 250_000; let mut svc = zaino_testutils::launch_state_and_fetch_services_mining_to::( @@ -2518,7 +2522,7 @@ mod zebrad { #[tokio::test(flavor = "multi_thread")] #[cfg_attr( not(feature = "devtool-incompatible"), - ignore = "heavy: mines ~110 orchard-coinbase blocks (~110 halo2 proofs) to bury a finalised spend below NON_FINALIZED_DEPTH; un-ignore for manual / dedicated CI" + ignore = "heavy: mines ~105 orchard-coinbase blocks (~105 halo2 proofs) to bury a finalised spend below the seam (FAST_TEST_MAX_NONFINALISED_DEPTH); un-ignore for manual / dedicated CI" )] async fn get_outpoint_spenders_fetch_vs_state() { crate::get_outpoint_spenders_fetch_vs_state().await; From c29b082c1bad5d9db55e2073fdd4379f799c1729 Mon Sep 17 00:00:00 2001 From: zancas Date: Wed, 1 Jul 2026 15:04:42 -0700 Subject: [PATCH 007/134] Fix send_to_transparent_finalization off-by-one; advance past the seam The load-bearing advance mined 99 blocks to push a send into the finalised DB, but a send at height H finalises only once the tip reaches H + seam. At the fast-test-seam depth (100) a 99-block advance falls one block short, so the test would still query the non-finalised chain if un-ignored. Advance FAST_TEST_MAX_NONFINALISED_DEPTH + 5 blocks (single-sourced, crosses the seam with margin), and make the surrounding prose seam-relative so it can't drift. Still #[ignore]d (waits on cheap transparent filler, round-3 P2). Co-Authored-By: Claude Opus 4.8 (1M context) --- live-tests/e2e/tests/devtool.rs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/live-tests/e2e/tests/devtool.rs b/live-tests/e2e/tests/devtool.rs index 1bfa4da1c..5efdab32a 100644 --- a/live-tests/e2e/tests/devtool.rs +++ b/live-tests/e2e/tests/devtool.rs @@ -21,7 +21,7 @@ //! //! Deferred, with the capability each waits on: //! - `send_to_transparent` (heavy / finalization) — runnable now via orchard -//! funding, but the load-bearing 99-block advance across the finalized / +//! funding, but the load-bearing seam-deep advance across the finalized / //! non-finalized boundary costs a halo2 proof per block; waits on cheap //! filler-block mining (round-3 spec P2). //! - `monitor_unverified_mempool` — unconfirmed (mempool) wallet balances; @@ -1579,13 +1579,13 @@ async fn get_block_range_out_of_range_lower_bound() { /// Port of `send_to_transparent` (wallet_to_validator, heavy/finalization, /// zebrad): a transparent send to the recipient returns the same address txids /// whether served from the non-finalized chain (just mined) or after the -/// 99-block advance pushes the send past the finalization boundary +/// seam-deep advance pushes the send past the finalization boundary /// (the seam depth `FAST_TEST_MAX_NONFINALISED_DEPTH` under `fast-test-seam`). /// -/// `#[ignore]`d (gated): the load-bearing 99-block advance mines orchard +/// `#[ignore]`d (gated): the load-bearing seam-deep advance mines orchard /// coinbase here — the devtool faucet funds via orchard, since the original's /// transparent-mine + shield path needs devtool transparent-coinbase shielding -/// (round-2 P1) — so it costs ~99 halo2 proofs, against the net-speedup +/// (round-2 P1) — so it costs ~100 halo2 proofs, against the net-speedup /// criterion. The miner pool is fixed at launch and `generate_blocks` has no /// per-call override, so the advance can't be cheap transparent filler until /// per-call filler mining (round-3 P2) lands; un-ignore and mine the advance to @@ -1616,7 +1616,9 @@ where // here (see #[ignore] rationale). test_manager .generate_blocks_bulk_and_wait_for_tips( - 99, + // Advance past the seam so the send crosses the finalised floor + // (`tip - seam`); a small margin above it keeps the boundary unambiguous. + zaino_common::consensus::FAST_TEST_MAX_NONFINALISED_DEPTH + 5, test_manager.subscriber(), test_manager.subscriber(), ) @@ -2332,7 +2334,7 @@ mod zebrad { #[tokio::test(flavor = "multi_thread")] #[cfg_attr( not(feature = "devtool-incompatible"), - ignore = "heavy: 99-block orchard advance (~99 halo2 proofs); un-ignore + transparent filler when round-3 P2 lands" + ignore = "heavy: seam-deep orchard advance (~100 halo2 proofs); un-ignore + transparent filler when round-3 P2 lands" )] async fn send_to_transparent_finalization() { crate::send_to_transparent_finalization::().await; @@ -2585,7 +2587,7 @@ mod zebrad { #[tokio::test(flavor = "multi_thread")] #[cfg_attr( not(feature = "devtool-incompatible"), - ignore = "heavy: 99-block orchard advance (~99 halo2 proofs); un-ignore + transparent filler when round-3 P2 lands" + ignore = "heavy: seam-deep orchard advance (~100 halo2 proofs); un-ignore + transparent filler when round-3 P2 lands" )] async fn send_to_transparent_finalization() { crate::send_to_transparent_finalization::().await; From 908f84418c0fab6de592a099d79cb5c9bd1c8283 Mon Sep 17 00:00:00 2001 From: zancas Date: Wed, 1 Jul 2026 15:11:13 -0700 Subject: [PATCH 008/134] Fix zcashd finalization off-by-one; seam-derive both advances MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The zcashd matrix (devtool_zcashd.rs, cfg feature zcashd_support) carries twins of the zebrad finalization tests, and cargo check without that feature never compiled them — so the send_to_transparent_finalization 99-block advance had the same off-by-one, and send_to_all_finalization used a magic 100-block advance. Both now advance FAST_TEST_MAX_NONFINALISED_DEPTH + 5 (single-sourced, crosses the seam), with the prose made seam-relative. Verified with cargo check -p e2e --tests --features zcashd_support. Co-Authored-By: Claude Opus 4.8 (1M context) --- live-tests/e2e/tests/devtool_zcashd.rs | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/live-tests/e2e/tests/devtool_zcashd.rs b/live-tests/e2e/tests/devtool_zcashd.rs index bcc002bd1..0f142eb9d 100644 --- a/live-tests/e2e/tests/devtool_zcashd.rs +++ b/live-tests/e2e/tests/devtool_zcashd.rs @@ -483,7 +483,7 @@ async fn shield_for_validator() { /// Devtool ports of `wallet_to_validator`'s `mod zcashd` send/shield/get-info /// column. Deferred: the heavy finalization send (`sent_to::transparent`'s -/// 99-block mine, round-3 P2), `sent_to::all`, and `monitor_unverified_mempool` +/// seam-deep mine, round-3 P2), `sent_to::all`, and `monitor_unverified_mempool` /// (round-3 P3). `send_to_transparent` here is the light send, matching the /// zebrad devtool port. mod wallet_to_validator { @@ -519,13 +519,13 @@ mod wallet_to_validator { /// zcashd analogue of devtool.rs's gated `send_to_transparent_finalization`: /// a transparent send returns the same address txids from the non-finalized - /// chain and after the 99-block advance into the finalized DB. `#[ignore]`d - /// for the same reason — the advance mines orchard coinbase (~99 halo2 + /// chain and after the seam-deep advance into the finalized DB. `#[ignore]`d + /// for the same reason — the advance mines orchard coinbase (~100 halo2 /// proofs) until per-call cheap filler mining (round-3 P2) lands. #[tokio::test(flavor = "multi_thread")] #[cfg_attr( not(feature = "devtool-incompatible"), - ignore = "heavy: 99-block orchard advance (~99 halo2 proofs); un-ignore + transparent filler when round-3 P2 lands" + ignore = "heavy: seam-deep orchard advance (~100 halo2 proofs); un-ignore + transparent filler when round-3 P2 lands" )] async fn send_to_transparent_finalization() { let (mut test_manager, mut clients) = launch_and_fund_zcashd_faucet(1).await; @@ -545,7 +545,9 @@ mod wallet_to_validator { test_manager .generate_blocks_bulk_and_wait_for_tips( - 99, + // Advance past the seam so the send crosses the finalised floor + // (`tip - seam`); a small margin above it keeps the boundary unambiguous. + zaino_common::consensus::FAST_TEST_MAX_NONFINALISED_DEPTH + 5, test_manager.subscriber(), test_manager.subscriber(), ) @@ -567,8 +569,8 @@ mod wallet_to_validator { } /// zcashd port of `sent_to::all` (heavy): one faucet funds a send to all - /// three pools, then a 100-block advance, and each recipient pool reports - /// 250_000. `#[ignore]`d: the 100-block advance mines orchard coinbase + /// three pools, then a seam-deep advance, and each recipient pool reports + /// 250_000. `#[ignore]`d: the seam-deep advance mines orchard coinbase /// (~100 halo2 proofs). The advance is faithful to the original but not /// load-bearing for the per-pool balance asserts (the sends confirm in one /// block), so this could be re-ported light (like the zebrad `send_to_all`) @@ -576,7 +578,7 @@ mod wallet_to_validator { #[tokio::test(flavor = "multi_thread")] #[cfg_attr( not(feature = "devtool-incompatible"), - ignore = "heavy: 100-block orchard advance (~100 halo2 proofs); re-port light or un-ignore with transparent filler (round-3 P2)" + ignore = "heavy: seam-deep orchard advance (~100 halo2 proofs); re-port light or un-ignore with transparent filler (round-3 P2)" )] async fn send_to_all() { let (mut test_manager, mut clients) = launch_and_fund_zcashd_faucet(3).await; @@ -590,7 +592,8 @@ mod wallet_to_validator { test_manager .generate_blocks_bulk_and_wait_for_tips( - 100, + // Advance past the seam so all three pool sends cross the finalised floor. + zaino_common::consensus::FAST_TEST_MAX_NONFINALISED_DEPTH + 5, test_manager.subscriber(), test_manager.subscriber(), ) From 1f3d56d0fe52bb416955b889345d41ffd19e6eef Mon Sep 17 00:00:00 2001 From: zancas Date: Wed, 1 Jul 2026 15:17:13 -0700 Subject: [PATCH 009/134] Remove internal "round-N PX" spec jargon from live-test docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The round-2 P1 / round-3 P2 / round-3 P3 / round-2 P0 tags reference an internal roadmap document and are meaningless in the source. Replace each with the plain capability it named — devtool transparent-coinbase shielding, cheap transparent filler-block mining, unconfirmed (mempool) wallet balances, zcashd activation-height alignment — or drop the tag where the surrounding text already states the fact. Compiles default and with --features zcashd_support; fmt clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- live-tests/e2e/tests/devtool.rs | 31 ++++++++++++-------------- live-tests/e2e/tests/devtool_zcashd.rs | 15 +++++++------ live-tests/e2e/tests/test_vectors.rs | 4 ++-- 3 files changed, 24 insertions(+), 26 deletions(-) diff --git a/live-tests/e2e/tests/devtool.rs b/live-tests/e2e/tests/devtool.rs index 5efdab32a..be20e3d08 100644 --- a/live-tests/e2e/tests/devtool.rs +++ b/live-tests/e2e/tests/devtool.rs @@ -23,16 +23,15 @@ //! - `send_to_transparent` (heavy / finalization) — runnable now via orchard //! funding, but the load-bearing seam-deep advance across the finalized / //! non-finalized boundary costs a halo2 proof per block; waits on cheap -//! filler-block mining (round-3 spec P2). +//! transparent filler-block mining. //! - `monitor_unverified_mempool` — unconfirmed (mempool) wallet balances; -//! devtool sync is block-based (round-3 spec P3 — likely stays on zingolib, -//! the indexer-side mempool views above already cover the surface). +//! devtool sync is block-based (likely stays on zingolib, the indexer-side +//! mempool views above already cover the surface). //! - the zcashd matrix (`json_server`, zcashd send/query) — the devtool wallet -//! rejects zcashd's default regtest activation heights at construction -//! (round-2 spec P0); `json_server` is additionally zcashd-bound (its -//! reference subscriber *is* zcashd). -//! - the `test_vectors` chain builder — transparent-coinbase shielding -//! (round-2 spec P1). +//! rejects zcashd's default regtest activation heights at construction; +//! `json_server` is additionally zcashd-bound (its reference subscriber *is* +//! zcashd). +//! - the `test_vectors` chain builder — devtool transparent-coinbase shielding. //! - `get_mempool_info` — recomputes expected sizes from //! `FetchServiceSubscriber` internals; low value over the mempool surfaces //! already covered. @@ -1321,8 +1320,7 @@ async fn get_address_balance_fetch_vs_state() { /// `launch_and_build_faucet_request`'s preamble for the faucet-taddr query /// tests. The faucet taddr is the abandon-art transparent receiver, which is /// also the zebrad miner address under transparent mining; the non-vacuity -/// probes in the callers verify that equality empirically (devtool round-2 -/// spec P1(1)). +/// probes in the callers verify that equality empirically. async fn launch_transparent_and_faucet_taddr( blocks: u32, ) -> (zaino_testutils::StateAndFetchServices, String) { @@ -1356,7 +1354,7 @@ async fn launch_transparent_and_faucet_taddr( /// the fetch and state indexers agree on `get_address_tx_ids` over the faucet's /// coinbase taddr. The non-vacuity probe (`!txids.is_empty()`) guards against a /// silent empty==empty pass and confirms the devtool faucet's transparent -/// receiver equals the zebrad miner address (devtool round-2 spec P1(1)). +/// receiver equals the zebrad miner address. async fn get_taddress_txids_faucet_fetch_vs_state() { let (mut svc, faucet_taddr) = launch_transparent_and_faucet_taddr(100).await; @@ -1384,8 +1382,7 @@ async fn get_taddress_txids_faucet_fetch_vs_state() { /// Port of `state_service_…::get_taddress_balance` (zebrad, faucet-taddr /// cluster): the fetch and state indexers agree on `get_taddress_balance` over /// the faucet's coinbase taddr. The non-vacuity probe (`value_zat > 0`) guards -/// against a silent 0==0 pass and confirms the address equality of round-2 -/// spec P1(1). +/// against a silent 0==0 pass and confirms the address equality. async fn get_taddress_balance_faucet_fetch_vs_state() { let (mut svc, faucet_taddr) = launch_transparent_and_faucet_taddr(5).await; @@ -1585,10 +1582,10 @@ async fn get_block_range_out_of_range_lower_bound() { /// `#[ignore]`d (gated): the load-bearing seam-deep advance mines orchard /// coinbase here — the devtool faucet funds via orchard, since the original's /// transparent-mine + shield path needs devtool transparent-coinbase shielding -/// (round-2 P1) — so it costs ~100 halo2 proofs, against the net-speedup +/// — so it costs ~100 halo2 proofs, against the net-speedup /// criterion. The miner pool is fixed at launch and `generate_blocks` has no /// per-call override, so the advance can't be cheap transparent filler until -/// per-call filler mining (round-3 P2) lands; un-ignore and mine the advance to +/// per-call filler mining lands; un-ignore and mine the advance to /// a transparent throwaway then. async fn send_to_transparent_finalization() where @@ -2334,7 +2331,7 @@ mod zebrad { #[tokio::test(flavor = "multi_thread")] #[cfg_attr( not(feature = "devtool-incompatible"), - ignore = "heavy: seam-deep orchard advance (~100 halo2 proofs); un-ignore + transparent filler when round-3 P2 lands" + ignore = "heavy: seam-deep orchard advance (~100 halo2 proofs); un-ignore + transparent filler when cheap filler mining lands" )] async fn send_to_transparent_finalization() { crate::send_to_transparent_finalization::().await; @@ -2587,7 +2584,7 @@ mod zebrad { #[tokio::test(flavor = "multi_thread")] #[cfg_attr( not(feature = "devtool-incompatible"), - ignore = "heavy: seam-deep orchard advance (~100 halo2 proofs); un-ignore + transparent filler when round-3 P2 lands" + ignore = "heavy: seam-deep orchard advance (~100 halo2 proofs); un-ignore + transparent filler when cheap filler mining lands" )] async fn send_to_transparent_finalization() { crate::send_to_transparent_finalization::().await; diff --git a/live-tests/e2e/tests/devtool_zcashd.rs b/live-tests/e2e/tests/devtool_zcashd.rs index 0f142eb9d..4405ff344 100644 --- a/live-tests/e2e/tests/devtool_zcashd.rs +++ b/live-tests/e2e/tests/devtool_zcashd.rs @@ -10,8 +10,8 @@ //! //! zcashd mines ORCHARD coinbase to `REG_O_ADDR_FROM_ABANDONART` (the abandon-art //! orchard address the devtool faucet owns), so the faucet is funded with no -//! transparent-coinbase shielding — the zcashd matrix is NOT gated on round-2 -//! P1, only on this heights alignment. +//! transparent-coinbase shielding — the zcashd matrix is NOT gated on devtool +//! transparent-coinbase shielding, only on this heights alignment. //! //! If this passes, the `json_server` tests port by swapping their zingolib //! funding for `DevtoolClients` while keeping the zaino-vs-zcashd oracle @@ -483,8 +483,9 @@ async fn shield_for_validator() { /// Devtool ports of `wallet_to_validator`'s `mod zcashd` send/shield/get-info /// column. Deferred: the heavy finalization send (`sent_to::transparent`'s -/// seam-deep mine, round-3 P2), `sent_to::all`, and `monitor_unverified_mempool` -/// (round-3 P3). `send_to_transparent` here is the light send, matching the +/// seam-deep mine, waits on cheap filler mining), `sent_to::all`, and +/// `monitor_unverified_mempool` (unconfirmed mempool balances). +/// `send_to_transparent` here is the light send, matching the /// zebrad devtool port. mod wallet_to_validator { use super::*; @@ -521,11 +522,11 @@ mod wallet_to_validator { /// a transparent send returns the same address txids from the non-finalized /// chain and after the seam-deep advance into the finalized DB. `#[ignore]`d /// for the same reason — the advance mines orchard coinbase (~100 halo2 - /// proofs) until per-call cheap filler mining (round-3 P2) lands. + /// proofs) until per-call cheap filler mining lands. #[tokio::test(flavor = "multi_thread")] #[cfg_attr( not(feature = "devtool-incompatible"), - ignore = "heavy: seam-deep orchard advance (~100 halo2 proofs); un-ignore + transparent filler when round-3 P2 lands" + ignore = "heavy: seam-deep orchard advance (~100 halo2 proofs); un-ignore + transparent filler when cheap filler mining lands" )] async fn send_to_transparent_finalization() { let (mut test_manager, mut clients) = launch_and_fund_zcashd_faucet(1).await; @@ -578,7 +579,7 @@ mod wallet_to_validator { #[tokio::test(flavor = "multi_thread")] #[cfg_attr( not(feature = "devtool-incompatible"), - ignore = "heavy: seam-deep orchard advance (~100 halo2 proofs); re-port light or un-ignore with transparent filler (round-3 P2)" + ignore = "heavy: seam-deep orchard advance (~100 halo2 proofs); re-port light or un-ignore with transparent filler" )] async fn send_to_all() { let (mut test_manager, mut clients) = launch_and_fund_zcashd_faucet(3).await; diff --git a/live-tests/e2e/tests/test_vectors.rs b/live-tests/e2e/tests/test_vectors.rs index 83dc4ac47..642d1d55a 100644 --- a/live-tests/e2e/tests/test_vectors.rs +++ b/live-tests/e2e/tests/test_vectors.rs @@ -40,7 +40,7 @@ macro_rules! expected_read_response { #[tokio::test(flavor = "multi_thread")] #[cfg_attr( not(feature = "devtool-incompatible"), - ignore = "Not a test: builds test-vector data for zaino_state::chain_index unit tests. Also funds via transparent-coinbase shielding (round-2 P1) — un-ignore to regenerate vectors once devtool can shield its own transparent coinbase (tracked by tests/devtool.rs's address_deltas)." + ignore = "Not a test: builds test-vector data for zaino_state::chain_index unit tests. Also funds via transparent-coinbase shielding — un-ignore to regenerate vectors once devtool can shield its own transparent coinbase (tracked by tests/devtool.rs's address_deltas)." )] #[allow(deprecated)] async fn create_200_block_regtest_chain_vectors() { @@ -81,7 +81,7 @@ async fn create_200_block_regtest_chain_vectors() { // *** Mine past coinbase maturity, shield the first reward, and mine it in *** // Mature the faucet's transparent coinbase (100-block maturity) and shield // it. Devtool analogue of `shield_faucet_rounds`; requires the devtool wallet - // to spend its own transparent coinbase (round-2 P1, see #[ignore]). Mine + // to spend its own transparent coinbase (see #[ignore]). Mine // generously (150) so the earliest coinbase — a few blocks past genesis — is // comfortably mature before the shield, regardless of startup height. test_manager From 8780e94aec38a3de8ea0249ab7fe5afcc301c1ac Mon Sep 17 00:00:00 2001 From: zancas Date: Wed, 1 Jul 2026 21:13:33 -0700 Subject: [PATCH 010/134] Rename `makers test` set `package` -> `packages` The front-door test set is named for the `packages/` directory it runs, so the plural is the faithful name. Hard-cut: `makers test package` now errors with `unknown set` rather than aliasing. Updates every reference site: Makefile.toml (default, case matcher, dispatch, descriptions, comment block), help.sh, docs/testing.md, README.md, and the live-tests/CONTEXT.md glossary term (now records the retired singular under _Avoid_). ADR-0004 gets a one-line supersession footer; its historical text is preserved. Co-Authored-By: Claude Opus 4.8 (1M context) --- Makefile.toml | 32 +++++++++---------- README.md | 2 +- ...ame-integration-partition-to-clientless.md | 3 ++ docs/testing.md | 6 ++-- live-tests/CONTEXT.md | 12 ++++--- tools/scripts/help.sh | 8 ++--- 6 files changed, 34 insertions(+), 29 deletions(-) diff --git a/Makefile.toml b/Makefile.toml index f38a60907..3595ca15a 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -93,7 +93,7 @@ script.main = 'source "./tools/scripts/push-image.sh"' [tasks.container-test] clear = true -description = "Engine: run nextest in the local CI container, forwarding args (zcashd_support OFF by default; --with-zcashd adds --features zcashd_support). Devs use `makers test [package|e2e|clientless|live|all]`; live-clientless / live-e2e and the `test` front door call this." +description = "Engine: run nextest in the local CI container, forwarding args (zcashd_support OFF by default; --with-zcashd adds --features zcashd_support). Devs use `makers test [packages|e2e|clientless|live|all]`; live-clientless / live-e2e and the `test` front door call this." # This task runs tests inside the container built from live-tests/test_environment. # The entrypoint.sh script in the container sets up test binaries (zcashd, zebrad, zcash-cli) # by creating symlinks from /home/container_user/artifacts to the expected live-tests/test_binaries/bins location. @@ -132,16 +132,16 @@ script.main = 'source "./tools/scripts/container-nextest-workspace.sh"' # =================================================================== # Developer-facing test front door: a single `makers test [SET]` task. # -# makers test packaged (the default) -# makers test package packages/* production-crate tests (no validator) +# makers test packages (the default) +# makers test packages packages/* production-crate tests (no validator) # makers test e2e the e2e live partition # makers test clientless the clientless live partition # makers test live both live partitions + combined summary -# makers test all package then live (everything) +# makers test all packages then live (everything) # -# SET names mirror the crate/dir names (`e2e`, `clientless`); `package`, -# `live`, and `all` are the groupings. `live` = all non-package tests; `all` -# = `package` + `live` (see live-tests/CONTEXT.md). Each set delegates to an +# SET names mirror the crate/dir names (`e2e`, `clientless`); `packages`, +# `live`, and `all` are the groupings. `live` = all non-packages tests; `all` +# = `packages` + `live` (see live-tests/CONTEXT.md). Each set delegates to an # existing engine rather than re-implementing a run. `--with-zcashd` (handled # by the engines) adds the zcashd-backed tests to any set. # =================================================================== @@ -150,33 +150,33 @@ script.main = 'source "./tools/scripts/container-nextest-workspace.sh"' # clear = true fully replaces cargo-make's built-in `test` task (which runs # `cargo test`); without it our `script` would merge with the builtin `command`. clear = true -description = "Run a test set: `makers test [package|e2e|clientless|live|all]` (default: package). Pass --with-zcashd for the zcashd-backed tests." +description = "Run a test set: `makers test [packages|e2e|clientless|live|all]` (default: packages). Pass --with-zcashd for the zcashd-backed tests." script_runner = "bash" script = ''' set -uo pipefail usage() { - echo "usage: makers test [package|e2e|clientless|live|all] [-- nextest args]" >&2 - echo " package (default) packages/* tests, no live validator" >&2 + echo "usage: makers test [packages|e2e|clientless|live|all] [-- nextest args]" >&2 + echo " packages (default) packages/* tests, no live validator" >&2 echo " e2e | clientless that single live partition" >&2 echo " live both live partitions + combined summary" >&2 - echo " all package then live (everything)" >&2 + echo " all packages then live (everything)" >&2 } # A non-flag first arg selects the set and must be a known name; a flag first -# arg (or none) keeps the default `package` and is forwarded to nextest. So -# `makers test --with-zcashd` runs package with the flag, while a mistyped set +# arg (or none) keeps the default `packages` and is forwarded to nextest. So +# `makers test --with-zcashd` runs packages with the flag, while a mistyped set # name errors instead of silently running the default. -set="package" +set="packages" if [ "$#" -gt 0 ] && [ "${1#-}" = "$1" ]; then case "$1" in - package|e2e|clientless|live|all) set="$1"; shift ;; + packages|e2e|clientless|live|all) set="$1"; shift ;; *) echo "makers test: unknown set '$1'" >&2; usage; exit 2 ;; esac fi case "$set" in - package) exec makers container-test "$@" ;; + packages) exec makers container-test "$@" ;; e2e) exec makers live-e2e "$@" ;; clientless) exec makers live-clientless "$@" ;; live) exec cargo run -q --manifest-path tools/test-runner/Cargo.toml --bin live-summary -- "$@" ;; diff --git a/README.md b/README.md index a72a9a495..0b8ca92f4 100644 --- a/README.md +++ b/README.md @@ -106,7 +106,7 @@ The test suites run inside a **podman** container via `makers` (cargo-make): ```sh makers test # packages/* tests that need no live validator (default) makers test live # both live partitions (clientless + e2e) + combined summary -makers test all # everything: package then live +makers test all # everything: packages then live ``` zcashd-backed tests are **off by default**; add `--with-zcashd` to include them diff --git a/docs/adr/0004-rename-integration-partition-to-clientless.md b/docs/adr/0004-rename-integration-partition-to-clientless.md index c98de87d4..cb80fbfa7 100644 --- a/docs/adr/0004-rename-integration-partition-to-clientless.md +++ b/docs/adr/0004-rename-integration-partition-to-clientless.md @@ -50,3 +50,6 @@ unchanged. "clientless" is no longer avoided. - ADR-0003's naming bullet and its three-front-door task surface are superseded; its two-crate split and the `live` umbrella decision still stand. + +> **Later note:** the `package` set was renamed `packages` (plural, matching the +> `packages/` dir); `makers test package` now errors. See `live-tests/CONTEXT.md`. diff --git a/docs/testing.md b/docs/testing.md index f5c6d8f10..9bfd24575 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -26,14 +26,14 @@ The `makers` tasks below build and run the test suites inside a **podman** container, so you don't need the validator binaries on your host `$PATH`. The container image is built or pulled automatically on first run. -The single front door is `makers test [SET]`, where `SET` defaults to `package`: +The single front door is `makers test [SET]`, where `SET` defaults to `packages`: -- `makers test` (or `makers test package`) — the **package** set: the +- `makers test` (or `makers test packages`) — the **packages** set: the `packages/*` production-crate tests that need no live validator. - `makers test e2e` / `makers test clientless` — one live partition. - `makers test live` — both live partitions (`clientless` then `e2e`) against a live validator, with a combined pass/fail summary. -- `makers test all` — the whole suite: package then live. +- `makers test all` — the whole suite: packages then live. (`container-test`, `live-clientless`, and `live-e2e` are the internal engines the `test` front door delegates to; invoke them directly only when you need to diff --git a/live-tests/CONTEXT.md b/live-tests/CONTEXT.md index d00e0f064..1bef4076e 100644 --- a/live-tests/CONTEXT.md +++ b/live-tests/CONTEXT.md @@ -40,15 +40,17 @@ owns shared fixtures and the feature-forwarding surface (`zcashd_support`, `transparent_address_history_experimental`). _Avoid_: test helpers, testlib. -**package (test set)**: +**packages (test set)**: The tests of the `packages/*` production crates — the workspace `default-members`, exercised by bare `cargo nextest run` and by `makers test` -(the default with no argument, equivalently `makers test package`). Named for +(the default with no argument, equivalently `makers test packages`). Named for where the crates live (`packages/`); the functional reason they run apart from the live partitions is that they need **no** live validator. They are *not* network-free, though: e.g. zaino-serve's gRPC `spawn` regression test binds a loopback socket and stands up a tonic server. The absent ingredient is a validator, not the network. -_Avoid_: offline (overclaims "no network" — these tests bind loopback sockets); -unit test (the set includes crate-level integration tests); container test -(names the run mechanism, which the live suite shares). +_Avoid_: package (singular — the set is plural, matching the `packages/` dir; +`makers test package` now errors with `unknown set`); offline (overclaims "no +network" — these tests bind loopback sockets); unit test (the set includes +crate-level integration tests); container test (names the run mechanism, which +the live suite shares). diff --git a/tools/scripts/help.sh b/tools/scripts/help.sh index 524c80b6c..7f962451c 100755 --- a/tools/scripts/help.sh +++ b/tools/scripts/help.sh @@ -12,7 +12,7 @@ echo "" echo "Common usage:" echo " makers test # packages/* tests, no live validator (default)" echo " makers test live # both live partitions + combined summary" -echo " makers test all # everything: package then live" +echo " makers test all # everything: packages then live" echo "" echo "If you modify '.env.testing-artifacts', the test command will \ automatically:" @@ -21,15 +21,15 @@ echo " - Build a new local container image if needed" echo "" echo "Available commands:" echo "" -echo " test [SET] Front door. SET = package (default) | e2e | \ +echo " test [SET] Front door. SET = packages (default) | e2e | \ clientless | live | all" -echo " package packages/* tests, no live \ +echo " packages packages/* tests, no live \ validator" echo " e2e the e2e live partition" echo " clientless the clientless live partition" echo " live both live partitions + \ combined summary" -echo " all package then live (everything)" +echo " all packages then live (everything)" echo " container-test Engine: run nextest in the container \ (used by the front door; invoke directly to forward engine flags)" echo " live-clientless Engine: run the clientless live-test \ From 9923aaa0680b3c1ef72c9ccaabcc43df79cd627a Mon Sep 17 00:00:00 2001 From: idky137 Date: Thu, 2 Jul 2026 15:34:12 +0100 Subject: [PATCH 011/134] added ChainIndexRpcExt trait to split functionality not required by zallet --- packages/zaino-state/src/backends/fetch.rs | 2 +- packages/zaino-state/src/backends/state.rs | 2 +- packages/zaino-state/src/chain_index.rs | 1569 +++++++++-------- .../src/chain_index/tests/mockchain_tests.rs | 2 +- packages/zaino-state/src/lib.rs | 4 +- 5 files changed, 804 insertions(+), 775 deletions(-) diff --git a/packages/zaino-state/src/backends/fetch.rs b/packages/zaino-state/src/backends/fetch.rs index 058004802..74f63dfcc 100644 --- a/packages/zaino-state/src/backends/fetch.rs +++ b/packages/zaino-state/src/backends/fetch.rs @@ -76,7 +76,7 @@ use crate::{ }; use crate::{ chain_index::{non_finalised_state::ChainIndexSnapshot, NonFinalizedSnapshot}, - ChainIndex, NodeBackedChainIndex, NodeBackedChainIndexSubscriber, + ChainIndex, ChainIndexRpcExt, NodeBackedChainIndex, NodeBackedChainIndexSubscriber, }; /// Chain fetch service backed by Zcashd's JsonRPC engine. diff --git a/packages/zaino-state/src/backends/state.rs b/packages/zaino-state/src/backends/state.rs index 62945c1df..5617ce9e2 100644 --- a/packages/zaino-state/src/backends/state.rs +++ b/packages/zaino-state/src/backends/state.rs @@ -5,7 +5,7 @@ use crate::{ chain_index::{ mempool::{Mempool, MempoolSubscriber}, source::ValidatorConnector, - types as chain_types, ChainIndex, + types as chain_types, ChainIndex, ChainIndexRpcExt, }, config::{DonationAddress, StateServiceConfig}, error::{BlockCacheError, StateServiceError}, diff --git a/packages/zaino-state/src/chain_index.rs b/packages/zaino-state/src/chain_index.rs index 182fd6107..f5a005b76 100644 --- a/packages/zaino-state/src/chain_index.rs +++ b/packages/zaino-state/src/chain_index.rs @@ -315,6 +315,14 @@ fn branch_len_to_active_chain( /// /// When a call asks for info (e.g. a block), Zaino selects sources in this order: #[doc = simple_mermaid::mermaid!("chain_index_passthrough.mmd")] +/// +/// This trait holds the core methods required by the embedded wallet consumer +/// (zallet). RPC-server-only methods live on the [`ChainIndexRpcExt`] extension +/// trait. +/// +/// TODO: The core/extension split is a provisional first pass. It should be refined +/// into finer capability-based traits (zallet / lwd / block-explorer) in a follow-up +/// PR. pub trait ChainIndex { /// A snapshot of the nonfinalized state, needed for atomic access type Snapshot: NonFinalizedSnapshot; @@ -387,50 +395,6 @@ pub trait ChainIndex { end: Option, ) -> Option, Self::Error>>>; - /// Returns the *compact* block for the given height. - /// - /// Returns `None` if the specified `height` is greater than the snapshot's tip. - /// - /// ## Pool filtering - /// - /// - `pool_types` controls which per-transaction components are populated. - /// - Transactions that contain no elements in any requested pool are omitted from `vtx`. - /// The original transaction index is preserved in `CompactTx.index`. - /// - `PoolTypeFilter::default()` preserves the legacy behaviour (only Sapling and Orchard - /// components are populated). - #[allow(clippy::type_complexity)] - fn get_compact_block( - &self, - nonfinalized_snapshot: &Self::Snapshot, - height: types::Height, - pool_types: PoolTypeFilter, - ) -> impl std::future::Future< - Output = Result, Self::Error>, - >; - - /// Streams *compact* blocks for an inclusive height range. - /// - /// Returns `None` if the requested range is entirely above the snapshot's tip. - /// - /// - The stream covers `[start_height, end_height]` (inclusive). - /// - If `start_height <= end_height` the stream is ascending; otherwise it is descending. - /// - /// ## Pool filtering - /// - /// - `pool_types` controls which per-transaction components are populated. - /// - Transactions that contain no elements in any requested pool are omitted from `vtx`. - /// The original transaction index is preserved in `CompactTx.index`. - /// - `PoolTypeFilter::default()` preserves the legacy behaviour (only Sapling and Orchard - /// components are populated). - #[allow(clippy::type_complexity)] - fn get_compact_block_stream( - &self, - nonfinalized_snapshot: &Self::Snapshot, - start_height: types::Height, - end_height: types::Height, - pool_types: PoolTypeFilter, - ) -> impl std::future::Future, Self::Error>>; - // ********** Transaction methods ********** /// given a transaction id, returns the transaction, along with @@ -515,12 +479,6 @@ pub trait ChainIndex { // ********** Transparent address history methods ********** - /// Returns all changes for the given transparent addresses. - fn get_address_deltas( - &self, - params: GetAddressDeltasParams, - ) -> impl std::future::Future>; - /// Returns the total transparent balance for the given addresses. fn get_address_balance( &self, @@ -556,6 +514,73 @@ pub trait ChainIndex { outpoints: Vec, scope: types::ChainScope, ) -> impl std::future::Future>, Self::Error>>; +} + +/// RPC-server extension methods layered on top of [`ChainIndex`]. +/// +/// The core [`ChainIndex`] trait holds the subset required by the embedded wallet +/// consumer (zallet). This extension holds the additional functionality required by +/// the gRPC (lightwalletd) and JSON-RPC servers: compact-block serving, mempool +/// metadata, address deltas, and the block-explorer / node-passthrough RPCs. +/// +/// TODO: This two-way core/extension split is a provisional first pass. It should be +/// refined into finer capability-based traits (zallet / lwd / block-explorer) in a +/// follow-up PR, at which point methods will be redistributed to their narrowest +/// capability. +pub trait ChainIndexRpcExt: ChainIndex { + // ********** Block methods ********** + + /// Returns the *compact* block for the given height. + /// + /// Returns `None` if the specified `height` is greater than the snapshot's tip. + /// + /// ## Pool filtering + /// + /// - `pool_types` controls which per-transaction components are populated. + /// - Transactions that contain no elements in any requested pool are omitted from `vtx`. + /// The original transaction index is preserved in `CompactTx.index`. + /// - `PoolTypeFilter::default()` preserves the legacy behaviour (only Sapling and Orchard + /// components are populated). + #[allow(clippy::type_complexity)] + fn get_compact_block( + &self, + nonfinalized_snapshot: &Self::Snapshot, + height: types::Height, + pool_types: PoolTypeFilter, + ) -> impl std::future::Future< + Output = Result, Self::Error>, + >; + + /// Streams *compact* blocks for an inclusive height range. + /// + /// Returns `None` if the requested range is entirely above the snapshot's tip. + /// + /// - The stream covers `[start_height, end_height]` (inclusive). + /// - If `start_height <= end_height` the stream is ascending; otherwise it is descending. + /// + /// ## Pool filtering + /// + /// - `pool_types` controls which per-transaction components are populated. + /// - Transactions that contain no elements in any requested pool are omitted from `vtx`. + /// The original transaction index is preserved in `CompactTx.index`. + /// - `PoolTypeFilter::default()` preserves the legacy behaviour (only Sapling and Orchard + /// components are populated). + #[allow(clippy::type_complexity)] + fn get_compact_block_stream( + &self, + nonfinalized_snapshot: &Self::Snapshot, + start_height: types::Height, + end_height: types::Height, + pool_types: PoolTypeFilter, + ) -> impl std::future::Future, Self::Error>>; + + // ********** Transparent address history methods ********** + + /// Returns all changes for the given transparent addresses. + fn get_address_deltas( + &self, + params: GetAddressDeltasParams, + ) -> impl std::future::Future>; // ********** Metadata methods ********** @@ -1684,378 +1709,82 @@ impl ChainIndex for NodeBackedChainIndexSubscriber Result, Self::Error> { - match snapshot { - ChainIndexSnapshot::NonFinalizedStateExists { - non_finalized_snapshot, - } => { - if height <= non_finalized_snapshot.best_tip.height { - Ok(Some(match snapshot.get_chainblock_by_height(&height) { - Some(block) => compact_block_with_pool_types( - block.to_compact_block(), - &pool_types.to_pool_types_vector(), - ), - None => { - match self - .finalized_state - .get_compact_block(height, pool_types.clone()) - .await - { - Ok(block) => block, - Err(_) => self - .get_compact_block_from_node(height, &pool_types) - .await? - .ok_or(ChainIndexError::database_hole(height, None))?, - } - } - })) - } else { - Ok(None) + txid: &types::TransactionHash, + ) -> Result, Option)>, Self::Error> { + // ChainIndex step 1 + if let Some(mempool_tx) = self + .mempool + .get_transaction(&mempool::MempoolKey { + txid: txid.to_rpc_hex(), + }) + .await + { + let bytes = mempool_tx.serialized_tx.as_ref().as_ref().to_vec(); + let mempool_branch_id = self.mempool_branch_id(snapshot); + + return Ok(Some((bytes, mempool_branch_id))); + } + + let Some((transaction, location)) = self + .source() + .get_transaction(*txid) + .await + .map_err(ChainIndexError::backing_validator)? + else { + return Ok(None); + }; + // as the reorg process cannot modify a transaction + // it's safe to serve nonfinalized state directly here + let height = match location { + GetTransactionLocation::BestChain(height) => height, + GetTransactionLocation::NonbestChain => { + // if the tranasction isn't on the best chain + // check our indexes. We need to find out the height from our index + // to determine the consensus branch ID + let Some(non_finalized_snapshot) = snapshot.get_nfs_snapshot() else { + // If we don't have a block containing the transaction + // locally and the transaction's not on the validator's + // best chain, we can't determine its consensus branch ID + return Ok(None); + }; + + match self + .blocks_containing_transaction(non_finalized_snapshot, txid.0) + .await? + .next() + { + Some(block) => block.context.index.height.into(), + // As above Ok(None) + None => return Ok(None), } } + // We've already checked the mempool. Should be unreachable? + // todo: error here? + GetTransactionLocation::Mempool => return Ok(None), + }; - ChainIndexSnapshot::StillSyncingFinalizedState { - validator_finalized_height: _, - //TODO: Once we make chainwork an option field we should be able to - // support passthrougth for this - } => Ok(None), - } + Ok(Some(( + zebra_chain::transaction::SerializedTransaction::from(transaction) + .as_ref() + .to_vec(), + ConsensusBranchId::current(&self.network, height).map(u32::from), + ))) } - /// Streams *compact* blocks for an inclusive height range. - /// - /// Returns `Ok(None)` if the request is descending and `start_height` exceeds the chain tip. - /// For ascending requests that exceed the tip, returns a stream that ends with an - /// `out_of_range` error after all available blocks have been sent. - /// - /// - The stream covers `[start_height, end_height]` (inclusive). - /// - If `start_height <= end_height` the stream is ascending; otherwise it is descending. - /// - /// ## Pool filtering - /// - /// - `pool_types` controls which per-transaction components are populated. - /// - Transactions that contain no elements in any requested pool are omitted from `vtx`. - /// The original transaction index is preserved in `CompactTx.index`. - /// - `PoolTypeFilter::default()` preserves the legacy behaviour (only Sapling and Orchard - /// components are populated). + /// Given a transaction ID, returns all known blocks containing this transaction /// - /// **NOTE: This Method is currently not "passthrough aware", this should be added by - /// fetching block data from the backing validator when not locally available.** - #[allow(clippy::type_complexity)] - async fn get_compact_block_stream( - &self, - nonfinalized_snapshot: &Self::Snapshot, - start_height: types::Height, - end_height: types::Height, - pool_types: PoolTypeFilter, - ) -> Result, Self::Error> { - let chain_tip_height = self.best_chaintip(nonfinalized_snapshot).await?.height; - - // The nonfinalized cache holds the tip block plus the previous 99 blocks (100 total), - // so the lowest possible cached height is `tip - 99` (saturating at 0). - let lowest_nonfinalized_height = types::Height(chain_tip_height.0.saturating_sub(99)); - - let is_ascending = start_height <= end_height; - - // Descending: the first block we'd try to return is already above the tip → error immediately. - if !is_ascending && start_height > chain_tip_height { - return Ok(None); - } - - let pool_types_vector = pool_types.to_pool_types_vector(); - - // For ascending requests that extend past the tip: cap the streaming range at the tip, - // then append a trailing out_of_range error after all valid blocks have been sent. - let needs_out_of_range = is_ascending && end_height > chain_tip_height; - let capped_end_height = if needs_out_of_range { - chain_tip_height - } else { - end_height - }; - - // Pre-create any finalized-state stream(s) we will need so that errors are returned - // from this method (not deferred into the spawned task). - let finalized_stream: Option = if is_ascending { - if start_height < lowest_nonfinalized_height { - let finalized_end_height = types::Height(std::cmp::min( - capped_end_height.0, - lowest_nonfinalized_height.0.saturating_sub(1), - )); - - if start_height <= finalized_end_height { - Some( - self.finalized_state - .get_compact_block_stream( - start_height, - finalized_end_height, - pool_types.clone(), - ) - .await - .map_err(ChainIndexError::from)?, - ) - } else { - None - } - } else { - None - } - // Serve in reverse order. - } else if end_height < lowest_nonfinalized_height { - let finalized_start_height = if start_height < lowest_nonfinalized_height { - start_height - } else { - types::Height(lowest_nonfinalized_height.0.saturating_sub(1)) - }; - - Some( - self.finalized_state - .get_compact_block_stream( - finalized_start_height, - end_height, - pool_types.clone(), - ) - .await - .map_err(ChainIndexError::from)?, - ) - } else { - None - }; - - let nonfinalized_snapshot = nonfinalized_snapshot.clone(); - let source = self.source.clone(); - let network = self.network.clone(); - let pool_types_for_node = pool_types.clone(); - // TODO: Investigate whether channel size should be changed, added to config, or set dynamically based on resources. - let (channel_sender, channel_receiver) = tokio::sync::mpsc::channel(128); - - tokio::spawn(async move { - if is_ascending { - // 1) Finalized segment (if any), ascending. - if let Some(mut finalized_stream) = finalized_stream { - while let Some(stream_item) = finalized_stream.next().await { - if channel_sender.send(stream_item).await.is_err() { - return; - } - } - } - - // 2) Nonfinalized segment, ascending. - let nonfinalized_start_height = - types::Height(std::cmp::max(start_height.0, lowest_nonfinalized_height.0)); - - for height_value in nonfinalized_start_height.0..=capped_end_height.0 { - let Some(indexed_block) = nonfinalized_snapshot - .get_chainblock_by_height(&types::Height(height_value)) - else { - match compact_block_from_source( - &source, - network.clone(), - types::Height(height_value), - &pool_types_for_node, - ) - .await - { - Ok(Some(compact_block)) => { - if channel_sender.send(Ok(compact_block)).await.is_err() { - return; - } - continue; - } - Ok(None) => { - let _ = channel_sender - .send(Err(tonic::Status::internal(format!( - "Internal error, missing nonfinalized block at height [{height_value}].", - )))) - .await; - return; - } - Err(error) => { - let _ = channel_sender - .send(Err(tonic::Status::internal(error.to_string()))) - .await; - return; - } - } - }; - let compact_block = compact_block_with_pool_types( - indexed_block.to_compact_block(), - &pool_types_vector, - ); - if channel_sender.send(Ok(compact_block)).await.is_err() { - return; - } - } - // If the original end_height was above the tip, signal out_of_range after all valid blocks. - if needs_out_of_range { - let _ = channel_sender - .send(Err(tonic::Status::out_of_range(format!( - "Error: Height out of range [{}]. Height requested is greater than the best chain tip [{}].", - end_height.0, chain_tip_height.0, - )))) - .await; - } - } else { - // 1) Nonfinalized segment, descending. - if start_height >= lowest_nonfinalized_height { - let nonfinalized_end_height = - types::Height(std::cmp::max(end_height.0, lowest_nonfinalized_height.0)); - - for height_value in (nonfinalized_end_height.0..=start_height.0).rev() { - let Some(indexed_block) = nonfinalized_snapshot - .get_chainblock_by_height(&types::Height(height_value)) - else { - match compact_block_from_source( - &source, - network.clone(), - types::Height(height_value), - &pool_types_for_node, - ) - .await - { - Ok(Some(compact_block)) => { - if channel_sender.send(Ok(compact_block)).await.is_err() { - return; - } - continue; - } - Ok(None) => { - let _ = channel_sender - .send(Err(tonic::Status::internal(format!( - "Internal error, missing nonfinalized block at height [{height_value}].", - )))) - .await; - return; - } - Err(error) => { - let _ = channel_sender - .send(Err(tonic::Status::internal(error.to_string()))) - .await; - return; - } - } - }; - let compact_block = compact_block_with_pool_types( - indexed_block.to_compact_block(), - &pool_types_vector, - ); - if channel_sender.send(Ok(compact_block)).await.is_err() { - return; - } - } - } - - // 2) Finalized segment (if any), descending. - if let Some(mut finalized_stream) = finalized_stream { - while let Some(stream_item) = finalized_stream.next().await { - if channel_sender.send(stream_item).await.is_err() { - return; - } - } - } - } - }); - - Ok(Some(CompactBlockStream::new(channel_receiver))) - } - - // ********** Transaction methods ********** - - /// given a transaction id, returns the transaction - /// and the consensus branch ID for the block the transaction - /// is in - async fn get_raw_transaction( - &self, - snapshot: &Self::Snapshot, - txid: &types::TransactionHash, - ) -> Result, Option)>, Self::Error> { - // ChainIndex step 1 - if let Some(mempool_tx) = self - .mempool - .get_transaction(&mempool::MempoolKey { - txid: txid.to_rpc_hex(), - }) - .await - { - let bytes = mempool_tx.serialized_tx.as_ref().as_ref().to_vec(); - let mempool_branch_id = self.mempool_branch_id(snapshot); - - return Ok(Some((bytes, mempool_branch_id))); - } - - let Some((transaction, location)) = self - .source() - .get_transaction(*txid) - .await - .map_err(ChainIndexError::backing_validator)? - else { - return Ok(None); - }; - // as the reorg process cannot modify a transaction - // it's safe to serve nonfinalized state directly here - let height = match location { - GetTransactionLocation::BestChain(height) => height, - GetTransactionLocation::NonbestChain => { - // if the tranasction isn't on the best chain - // check our indexes. We need to find out the height from our index - // to determine the consensus branch ID - let Some(non_finalized_snapshot) = snapshot.get_nfs_snapshot() else { - // If we don't have a block containing the transaction - // locally and the transaction's not on the validator's - // best chain, we can't determine its consensus branch ID - return Ok(None); - }; - - match self - .blocks_containing_transaction(non_finalized_snapshot, txid.0) - .await? - .next() - { - Some(block) => block.context.index.height.into(), - // As above Ok(None) - None => return Ok(None), - } - } - // We've already checked the mempool. Should be unreachable? - // todo: error here? - GetTransactionLocation::Mempool => return Ok(None), - }; - - Ok(Some(( - zebra_chain::transaction::SerializedTransaction::from(transaction) - .as_ref() - .to_vec(), - ConsensusBranchId::current(&self.network, height).map(u32::from), - ))) - } - - /// Given a transaction ID, returns all known blocks containing this transaction - /// - /// If the transaction is in the mempool, it will be in the `BestChainLocation` - /// if the mempool and snapshot are up-to-date, and the `NonBestChainLocation` set - /// if the snapshot is out-of-date compared to the mempool - async fn get_transaction_status( + /// If the transaction is in the mempool, it will be in the `BestChainLocation` + /// if the mempool and snapshot are up-to-date, and the `NonBestChainLocation` set + /// if the snapshot is out-of-date compared to the mempool + async fn get_transaction_status( &self, snapshot: &Self::Snapshot, txid: &types::TransactionHash, @@ -2169,260 +1898,704 @@ impl ChainIndex for NodeBackedChainIndexSubscriber Result, Self::Error> { - self.mempool - .get_mempool() + /// Returns all txids currently in the mempool. + async fn get_mempool_txids(&self) -> Result, Self::Error> { + self.mempool + .get_mempool() + .await + .into_iter() + .map(|(txid_key, _)| { + TransactionHash::from_hex(&txid_key.txid) + .map_err(ChainIndexError::backing_validator) + }) + .collect::>() + } + + /// Returns all transactions currently in the mempool, filtered by `exclude_list`. + /// + /// The `exclude_list` may contain shortened transaction ID hex prefixes (client-endian). + /// The transaction IDs in the Exclude list can be shortened to any number of bytes to make the request + /// more bandwidth-efficient; if two or more transactions in the mempool + /// match a shortened txid, they are all sent (none is excluded). Transactions + /// in the exclude list that don't exist in the mempool are ignored. + async fn get_mempool_transactions( + &self, + exclude_list: Vec, + ) -> Result>, Self::Error> { + // Use the mempool's own filtering (it already handles client-endian shortened prefixes). + let pairs: Vec<(mempool::MempoolKey, mempool::MempoolValue)> = + self.mempool.get_filtered_mempool(exclude_list).await; + + // Transform to the Vec> that the trait requires. + let bytes: Vec> = pairs + .into_iter() + .map(|(_, v)| v.serialized_tx.as_ref().as_ref().to_vec()) + .collect(); + + Ok(bytes) + } + + /// Returns a stream of mempool transactions, ending the stream when the chain tip block hash + /// changes (a new block is mined or a reorg occurs). + /// + /// If a snapshot is given and the chain tip has changed from the given spanshot, returns None. + fn get_mempool_stream( + &self, + snapshot: Option<&Self::Snapshot>, + ) -> Option, Self::Error>>> { + let non_finalized_snapshot = match snapshot { + Some(s) => match s { + ChainIndexSnapshot::NonFinalizedStateExists { + non_finalized_snapshot, + } => Some(non_finalized_snapshot), + // If we're still syncing the finalized state, the chain tip + // is newer than the snapshot's tip. Return None. + ChainIndexSnapshot::StillSyncingFinalizedState { .. } => return None, + }, + None => None, + }; + let expected_chain_tip = non_finalized_snapshot.map(|snapshot| snapshot.best_tip.hash); + let mut subscriber = self.mempool.clone(); + + match subscriber + .get_mempool_stream(expected_chain_tip) + .now_or_never() + { + Some(Ok((in_rx, _handle))) => { + let (out_tx, out_rx) = + tokio::sync::mpsc::channel::, ChainIndexError>>(32); + + tokio::spawn(async move { + let mut in_stream = tokio_stream::wrappers::ReceiverStream::new(in_rx); + while let Some(item) = in_stream.next().await { + match item { + Ok((_key, value)) => { + let _ = out_tx + .send(Ok(value.serialized_tx.as_ref().as_ref().to_vec())) + .await; + } + Err(e) => { + let _ = out_tx + .send(Err(ChainIndexError::child_process_status_error( + "mempool", e, + ))) + .await; + break; + } + } + } + }); + + Some(tokio_stream::wrappers::ReceiverStream::new(out_rx)) + } + Some(Err(crate::error::MempoolError::IncorrectChainTip { .. })) => None, + Some(Err(e)) => { + let (out_tx, out_rx) = + tokio::sync::mpsc::channel::, ChainIndexError>>(1); + let _ = out_tx.try_send(Err(e.into())); + Some(tokio_stream::wrappers::ReceiverStream::new(out_rx)) + } + None => { + // Should not happen because the inner tip check is synchronous, but fail safe. + let (out_tx, out_rx) = + tokio::sync::mpsc::channel::, ChainIndexError>>(1); + let _ = out_tx.try_send(Err(ChainIndexError::child_process_status_error( + "mempool", + crate::error::StatusError { + server_status: crate::StatusType::RecoverableError, + }, + ))); + Some(tokio_stream::wrappers::ReceiverStream::new(out_rx)) + } + } + } + + // ********** Chain methods ********** + + /// For a given block, + /// find its newest main-chain ancestor, + /// or the block itself if it is on the main-chain. + /// Returns Ok(None) if no fork point found. This is not an error, + /// as zaino does not guarentee knowledge of all sidechain data. + async fn find_fork_point( + &self, + snapshot: &Self::Snapshot, + hash: &types::BlockHash, + ) -> Result, Self::Error> { + // ChainIndex step 1: Skip + // mempool blocks have no canon height, guaranteed to return None + // todo: possible efficiency boost by checking mempool for a negative? + + match snapshot { + ChainIndexSnapshot::NonFinalizedStateExists { + non_finalized_snapshot, + } => { + match non_finalized_snapshot.get_chainblock_by_hash(hash) { + Some(block) => { + // At this point, we know that + // The block is non-FINALIZED in the INDEXER + // ChainIndex step 3: + if non_finalized_snapshot + .heights_to_hashes + .get(&block.height()) + == Some(block.hash()) + { + // The block is in the best chain. + Ok(Some((*block.hash(), block.height()))) + } else { + // Otherwise, it's non-best chain! Grab its parent, and recurse + Box::pin(self.find_fork_point(snapshot, &block.context.parent_hash)) + .await + // gotta pin recursive async functions to prevent infinite-sized + // Future-implementing types + } + } + None => { + // At this point, we know that + // the block is NOT non-FINALIZED in the INDEXER. + // as the non finalzed state is known to be populated, + // we now check the finalized state + match self.finalized_state.get_block_height(*hash).await { + Ok(Some(height)) => { + // the block is FINALIZED in the INDEXER + Ok(Some((*hash, height))) + } + Err(e) => Err(ChainIndexError::database_hole(hash, Some(Box::new(e)))), + Ok(None) => Ok(None), + } + } + } + } + ChainIndexSnapshot::StillSyncingFinalizedState { + validator_finalized_height, + } => { + // We're not fully synced, so we pass through. + // Now, we ask the VALIDATOR. + // ChainIndex step 5 + match self + .source() + .get_block(HashOrHeight::Hash(zebra_chain::block::Hash::from(*hash))) + .await + { + Ok(Some(block)) => { + // At this point, we know that + // the block is in the VALIDATOR. + match block.coinbase_height() { + None => { + // the block is in the VALIDATOR. but doesnt have a height. That would imply a bug. + Err(ChainIndexError::validator_data_error_block_coinbase_height_missing()) + } + Some(height) => { + // The VALIDATOR returned a block with a height. + // However, there is as of yet no guaranteed the Block is FINALIZED + if height <= *validator_finalized_height { + Ok(Some(( + types::BlockHash::from(block.hash()), + types::Height::from(height), + ))) + } else { + // non-finalized block + // no passthrough + Ok(None) + } + } + } + } + + Ok(None) => { + // At this point, we know that + // the block is NOT FINALIZED in the VALIDATOR. + // Return Ok(None) = no block found. + Ok(None) + } + Err(e) => Err(ChainIndexError::backing_validator(e)), + } + } + } + } + + /// Returns the block commitment tree data by hash. + async fn get_treestate( + &self, + hash: &types::BlockHash, + ) -> Result<(Option>, Option>), Self::Error> { + let snapshot = self.snapshot_nonfinalized_state().await?; + if !self.block_hash_known_for_treestate(&snapshot, hash).await? { + return Err(ChainIndexError::internal(format!( + "block hash {hash} not found in local chain index" + ))); + } + + match self.source().get_treestate(*hash).await { + Ok(resp) => Ok(resp), + Err(e) => Err(ChainIndexError { + kind: ChainIndexErrorKind::InternalServerError, + message: "failed to fetch treestate from validator".to_string(), + source: Some(Box::new(e)), + }), + } + } + + /// Gets the subtree roots of a given pool and the end heights of each root, + /// starting at the provided index, up to an optional maximum number of roots. + async fn get_subtree_roots( + &self, + pool: ShieldedPool, + start_index: u16, + max_entries: Option, + ) -> Result, Self::Error> { + self.source() + .get_subtree_roots(pool, start_index, max_entries) + .await + .map_err(ChainIndexError::backing_validator) + } + + // ********** Transparent address history methods ********** + + /// Returns the total transparent balance for the given addresses. + async fn get_address_balance( + &self, + address_strings: GetAddressBalanceRequest, + ) -> Result { + self.source() + .get_address_balance(address_strings) .await - .into_iter() - .map(|(txid_key, _)| { - TransactionHash::from_hex(&txid_key.txid) - .map_err(ChainIndexError::backing_validator) - }) - .collect::>() + .map_err(ChainIndexError::backing_validator) } - /// Returns all transactions currently in the mempool, filtered by `exclude_list`. - /// - /// The `exclude_list` may contain shortened transaction ID hex prefixes (client-endian). - /// The transaction IDs in the Exclude list can be shortened to any number of bytes to make the request - /// more bandwidth-efficient; if two or more transactions in the mempool - /// match a shortened txid, they are all sent (none is excluded). Transactions - /// in the exclude list that don't exist in the mempool are ignored. - async fn get_mempool_transactions( + /// Returns the transaction ids made by the given transparent addresses. + async fn get_address_txids( &self, - exclude_list: Vec, - ) -> Result>, Self::Error> { - // Use the mempool's own filtering (it already handles client-endian shortened prefixes). - let pairs: Vec<(mempool::MempoolKey, mempool::MempoolValue)> = - self.mempool.get_filtered_mempool(exclude_list).await; - - // Transform to the Vec> that the trait requires. - let bytes: Vec> = pairs - .into_iter() - .map(|(_, v)| v.serialized_tx.as_ref().as_ref().to_vec()) - .collect(); + request: GetAddressTxIdsRequest, + ) -> Result, Self::Error> { + self.source() + .get_address_txids(request) + .await + .map_err(ChainIndexError::backing_validator) + } - Ok(bytes) + /// Returns all unspent transparent outputs for the given addresses. + async fn get_address_utxos( + &self, + address_strings: GetAddressBalanceRequest, + ) -> Result, Self::Error> { + self.source() + .get_address_utxos(address_strings) + .await + .map_err(ChainIndexError::backing_validator) } - /// Returns a stream of mempool transactions, ending the stream when the chain tip block hash - /// changes (a new block is mined or a reorg occurs). - /// - /// If a snapshot is given and the chain tip has changed from the given spanshot, returns None. - fn get_mempool_stream( + async fn get_outpoint_spenders( &self, - snapshot: Option<&Self::Snapshot>, - ) -> Option, Self::Error>>> { - let non_finalized_snapshot = match snapshot { - Some(s) => match s { - ChainIndexSnapshot::NonFinalizedStateExists { - non_finalized_snapshot, - } => Some(non_finalized_snapshot), - // If we're still syncing the finalized state, the chain tip - // is newer than the snapshot's tip. Return None. - ChainIndexSnapshot::StillSyncingFinalizedState { .. } => return None, - }, - None => None, - }; - let expected_chain_tip = non_finalized_snapshot.map(|snapshot| snapshot.best_tip.hash); - let mut subscriber = self.mempool.clone(); + snapshot: &Self::Snapshot, + outpoints: Vec, + scope: types::ChainScope, + ) -> Result>, Self::Error> { + use std::collections::HashMap; - match subscriber - .get_mempool_stream(expected_chain_tip) - .now_or_never() - { - Some(Ok((in_rx, _handle))) => { - let (out_tx, out_rx) = - tokio::sync::mpsc::channel::, ChainIndexError>>(32); + let mut result: Vec> = vec![None; outpoints.len()]; - tokio::spawn(async move { - let mut in_stream = tokio_stream::wrappers::ReceiverStream::new(in_rx); - while let Some(item) = in_stream.next().await { - match item { - Ok((_key, value)) => { - let _ = out_tx - .send(Ok(value.serialized_tx.as_ref().as_ref().to_vec())) - .await; - } - Err(e) => { - let _ = out_tx - .send(Err(ChainIndexError::child_process_status_error( - "mempool", e, - ))) - .await; - break; - } - } + // 1) Non-finalised best chain (FullChain scope only). Scan only the blocks reachable + // via `heights_to_hashes` (the `blocks` map also holds reorged-away blocks, which + // must not count). One pass builds an outpoint -> spending-txid map regardless of + // how many outpoints we look up. Under `Finalised` scope this is skipped so results + // are reorg-stable. + if let ( + types::ChainScope::FullChain, + ChainIndexSnapshot::NonFinalizedStateExists { + non_finalized_snapshot, + }, + ) = (scope, snapshot) + { + let mut nfs_spenders: HashMap = HashMap::new(); + for hash in non_finalized_snapshot.heights_to_hashes.values() { + let Some(block) = non_finalized_snapshot.blocks.get(hash) else { + continue; + }; + for tx in block.transactions() { + let txid = *tx.txid(); + // `spent_outpoints` already skips coinbase null prevouts and builds each + // `Outpoint`, keeping the construction in one place (see #1332). + for outpoint in tx.transparent().spent_outpoints() { + nfs_spenders.insert(outpoint, txid); } - }); - - Some(tokio_stream::wrappers::ReceiverStream::new(out_rx)) + } } - Some(Err(crate::error::MempoolError::IncorrectChainTip { .. })) => None, - Some(Err(e)) => { - let (out_tx, out_rx) = - tokio::sync::mpsc::channel::, ChainIndexError>>(1); - let _ = out_tx.try_send(Err(e.into())); - Some(tokio_stream::wrappers::ReceiverStream::new(out_rx)) + for (i, outpoint) in outpoints.iter().enumerate() { + if let Some(txid) = nfs_spenders.get(outpoint) { + result[i] = Some(*txid); + } } - None => { - // Should not happen because the inner tip check is synchronous, but fail safe. - let (out_tx, out_rx) = - tokio::sync::mpsc::channel::, ChainIndexError>>(1); - let _ = out_tx.try_send(Err(ChainIndexError::child_process_status_error( - "mempool", - crate::error::StatusError { - server_status: crate::StatusType::RecoverableError, - }, - ))); - Some(tokio_stream::wrappers::ReceiverStream::new(out_rx)) + } + + // 2) Finalised lookup for the still-unresolved outpoints, batched into one DB call. + let unresolved_indices: Vec = result + .iter() + .enumerate() + .filter_map(|(i, r)| r.is_none().then_some(i)) + .collect(); + if unresolved_indices.is_empty() { + return Ok(result); + } + let unresolved_outpoints: Vec = + unresolved_indices.iter().map(|&i| outpoints[i]).collect(); + let locations = self + .finalized_state + .get_outpoint_spenders(unresolved_outpoints) + .await?; + + // 3) Resolve each finalised `TxLocation` to a txid. Dedup identical locations so a + // block spending several queried outpoints is only fetched once. `get_txid` is a + // single keyed lookup, far cheaper than reconstructing the whole block. + let mut slots_by_location: HashMap> = HashMap::new(); + for (slot, location) in unresolved_indices.into_iter().zip(locations) { + if let Some(location) = location { + slots_by_location.entry(location).or_default().push(slot); } } + for (location, slots) in slots_by_location { + let txid = self.finalized_state.get_txid(location).await?; + for slot in slots { + result[slot] = Some(txid); + } + } + + Ok(result) } - // ********** Chain methods ********** + // ********** Metadata methods ********** - /// For a given block, - /// find its newest main-chain ancestor, - /// or the block itself if it is on the main-chain. - /// Returns Ok(None) if no fork point found. This is not an error, - /// as zaino does not guarentee knowledge of all sidechain data. - async fn find_fork_point( + async fn best_chaintip(&self, snapshot: &Self::Snapshot) -> Result { + Ok(match snapshot { + ChainIndexSnapshot::NonFinalizedStateExists { + non_finalized_snapshot, + } => non_finalized_snapshot.best_tip, + + ChainIndexSnapshot::StillSyncingFinalizedState { + validator_finalized_height, + } => { + BlockIndex { + height: *validator_finalized_height, + hash: self + .source() + // TODO: do something more efficient than getting the whole block + .get_block(HashOrHeight::Height((*validator_finalized_height).into())) + .await + .map_err(|e| { + ChainIndexError::database_hole( + validator_finalized_height, + Some(Box::new(e)), + ) + })? + .ok_or(ChainIndexError::database_hole( + validator_finalized_height, + None, + ))? + .hash() + .into(), + } + } + }) + } +} + +impl ChainIndexRpcExt for NodeBackedChainIndexSubscriber { + /// Returns the *compact* block for the given height. + /// + /// Returns `None` if the specified `height` is greater than the snapshot's tip. + /// + /// ## Pool filtering + /// + /// - `pool_types` controls which per-transaction components are populated. + /// - Transactions that contain no elements in any requested pool are omitted from `vtx`. + /// The original transaction index is preserved in `CompactTx.index`. + /// - `PoolTypeFilter::default()` preserves the legacy behaviour (only Sapling and Orchard + /// components are populated). + /// + /// Returns None if the specified height + /// is greater than the snapshot's tip + /// + /// **NOTE: This Method is currently not "passthrough aware", this should be added by + /// fetching block data from the backing validator when not locally available.** + async fn get_compact_block( &self, snapshot: &Self::Snapshot, - hash: &types::BlockHash, - ) -> Result, Self::Error> { - // ChainIndex step 1: Skip - // mempool blocks have no canon height, guaranteed to return None - // todo: possible efficiency boost by checking mempool for a negative? - + height: types::Height, + pool_types: PoolTypeFilter, + ) -> Result, Self::Error> { match snapshot { ChainIndexSnapshot::NonFinalizedStateExists { non_finalized_snapshot, } => { - match non_finalized_snapshot.get_chainblock_by_hash(hash) { - Some(block) => { - // At this point, we know that - // The block is non-FINALIZED in the INDEXER - // ChainIndex step 3: - if non_finalized_snapshot - .heights_to_hashes - .get(&block.height()) - == Some(block.hash()) - { - // The block is in the best chain. - Ok(Some((*block.hash(), block.height()))) - } else { - // Otherwise, it's non-best chain! Grab its parent, and recurse - Box::pin(self.find_fork_point(snapshot, &block.context.parent_hash)) + if height <= non_finalized_snapshot.best_tip.height { + Ok(Some(match snapshot.get_chainblock_by_height(&height) { + Some(block) => compact_block_with_pool_types( + block.to_compact_block(), + &pool_types.to_pool_types_vector(), + ), + None => { + match self + .finalized_state + .get_compact_block(height, pool_types.clone()) .await - // gotta pin recursive async functions to prevent infinite-sized - // Future-implementing types - } - } - None => { - // At this point, we know that - // the block is NOT non-FINALIZED in the INDEXER. - // as the non finalzed state is known to be populated, - // we now check the finalized state - match self.finalized_state.get_block_height(*hash).await { - Ok(Some(height)) => { - // the block is FINALIZED in the INDEXER - Ok(Some((*hash, height))) + { + Ok(block) => block, + Err(_) => self + .get_compact_block_from_node(height, &pool_types) + .await? + .ok_or(ChainIndexError::database_hole(height, None))?, } - Err(e) => Err(ChainIndexError::database_hole(hash, Some(Box::new(e)))), - Ok(None) => Ok(None), } - } + })) + } else { + Ok(None) } } + ChainIndexSnapshot::StillSyncingFinalizedState { - validator_finalized_height, - } => { - // We're not fully synced, so we pass through. - // Now, we ask the VALIDATOR. - // ChainIndex step 5 - match self - .source() - .get_block(HashOrHeight::Hash(zebra_chain::block::Hash::from(*hash))) + validator_finalized_height: _, + //TODO: Once we make chainwork an option field we should be able to + // support passthrougth for this + } => Ok(None), + } + } + + /// Streams *compact* blocks for an inclusive height range. + /// + /// Returns `Ok(None)` if the request is descending and `start_height` exceeds the chain tip. + /// For ascending requests that exceed the tip, returns a stream that ends with an + /// `out_of_range` error after all available blocks have been sent. + /// + /// - The stream covers `[start_height, end_height]` (inclusive). + /// - If `start_height <= end_height` the stream is ascending; otherwise it is descending. + /// + /// ## Pool filtering + /// + /// - `pool_types` controls which per-transaction components are populated. + /// - Transactions that contain no elements in any requested pool are omitted from `vtx`. + /// The original transaction index is preserved in `CompactTx.index`. + /// - `PoolTypeFilter::default()` preserves the legacy behaviour (only Sapling and Orchard + /// components are populated). + /// + /// **NOTE: This Method is currently not "passthrough aware", this should be added by + /// fetching block data from the backing validator when not locally available.** + #[allow(clippy::type_complexity)] + async fn get_compact_block_stream( + &self, + nonfinalized_snapshot: &Self::Snapshot, + start_height: types::Height, + end_height: types::Height, + pool_types: PoolTypeFilter, + ) -> Result, Self::Error> { + let chain_tip_height = self.best_chaintip(nonfinalized_snapshot).await?.height; + + // The nonfinalized cache holds the tip block plus the previous 99 blocks (100 total), + // so the lowest possible cached height is `tip - 99` (saturating at 0). + let lowest_nonfinalized_height = types::Height(chain_tip_height.0.saturating_sub(99)); + + let is_ascending = start_height <= end_height; + + // Descending: the first block we'd try to return is already above the tip → error immediately. + if !is_ascending && start_height > chain_tip_height { + return Ok(None); + } + + let pool_types_vector = pool_types.to_pool_types_vector(); + + // For ascending requests that extend past the tip: cap the streaming range at the tip, + // then append a trailing out_of_range error after all valid blocks have been sent. + let needs_out_of_range = is_ascending && end_height > chain_tip_height; + let capped_end_height = if needs_out_of_range { + chain_tip_height + } else { + end_height + }; + + // Pre-create any finalized-state stream(s) we will need so that errors are returned + // from this method (not deferred into the spawned task). + let finalized_stream: Option = if is_ascending { + if start_height < lowest_nonfinalized_height { + let finalized_end_height = types::Height(std::cmp::min( + capped_end_height.0, + lowest_nonfinalized_height.0.saturating_sub(1), + )); + + if start_height <= finalized_end_height { + Some( + self.finalized_state + .get_compact_block_stream( + start_height, + finalized_end_height, + pool_types.clone(), + ) + .await + .map_err(ChainIndexError::from)?, + ) + } else { + None + } + } else { + None + } + // Serve in reverse order. + } else if end_height < lowest_nonfinalized_height { + let finalized_start_height = if start_height < lowest_nonfinalized_height { + start_height + } else { + types::Height(lowest_nonfinalized_height.0.saturating_sub(1)) + }; + + Some( + self.finalized_state + .get_compact_block_stream( + finalized_start_height, + end_height, + pool_types.clone(), + ) .await - { - Ok(Some(block)) => { - // At this point, we know that - // the block is in the VALIDATOR. - match block.coinbase_height() { - None => { - // the block is in the VALIDATOR. but doesnt have a height. That would imply a bug. - Err(ChainIndexError::validator_data_error_block_coinbase_height_missing()) + .map_err(ChainIndexError::from)?, + ) + } else { + None + }; + + let nonfinalized_snapshot = nonfinalized_snapshot.clone(); + let source = self.source.clone(); + let network = self.network.clone(); + let pool_types_for_node = pool_types.clone(); + // TODO: Investigate whether channel size should be changed, added to config, or set dynamically based on resources. + let (channel_sender, channel_receiver) = tokio::sync::mpsc::channel(128); + + tokio::spawn(async move { + if is_ascending { + // 1) Finalized segment (if any), ascending. + if let Some(mut finalized_stream) = finalized_stream { + while let Some(stream_item) = finalized_stream.next().await { + if channel_sender.send(stream_item).await.is_err() { + return; + } + } + } + + // 2) Nonfinalized segment, ascending. + let nonfinalized_start_height = + types::Height(std::cmp::max(start_height.0, lowest_nonfinalized_height.0)); + + for height_value in nonfinalized_start_height.0..=capped_end_height.0 { + let Some(indexed_block) = nonfinalized_snapshot + .get_chainblock_by_height(&types::Height(height_value)) + else { + match compact_block_from_source( + &source, + network.clone(), + types::Height(height_value), + &pool_types_for_node, + ) + .await + { + Ok(Some(compact_block)) => { + if channel_sender.send(Ok(compact_block)).await.is_err() { + return; + } + continue; } - Some(height) => { - // The VALIDATOR returned a block with a height. - // However, there is as of yet no guaranteed the Block is FINALIZED - if height <= *validator_finalized_height { - Ok(Some(( - types::BlockHash::from(block.hash()), - types::Height::from(height), - ))) - } else { - // non-finalized block - // no passthrough - Ok(None) + Ok(None) => { + let _ = channel_sender + .send(Err(tonic::Status::internal(format!( + "Internal error, missing nonfinalized block at height [{height_value}].", + )))) + .await; + return; + } + Err(error) => { + let _ = channel_sender + .send(Err(tonic::Status::internal(error.to_string()))) + .await; + return; + } + } + }; + let compact_block = compact_block_with_pool_types( + indexed_block.to_compact_block(), + &pool_types_vector, + ); + if channel_sender.send(Ok(compact_block)).await.is_err() { + return; + } + } + // If the original end_height was above the tip, signal out_of_range after all valid blocks. + if needs_out_of_range { + let _ = channel_sender + .send(Err(tonic::Status::out_of_range(format!( + "Error: Height out of range [{}]. Height requested is greater than the best chain tip [{}].", + end_height.0, chain_tip_height.0, + )))) + .await; + } + } else { + // 1) Nonfinalized segment, descending. + if start_height >= lowest_nonfinalized_height { + let nonfinalized_end_height = + types::Height(std::cmp::max(end_height.0, lowest_nonfinalized_height.0)); + + for height_value in (nonfinalized_end_height.0..=start_height.0).rev() { + let Some(indexed_block) = nonfinalized_snapshot + .get_chainblock_by_height(&types::Height(height_value)) + else { + match compact_block_from_source( + &source, + network.clone(), + types::Height(height_value), + &pool_types_for_node, + ) + .await + { + Ok(Some(compact_block)) => { + if channel_sender.send(Ok(compact_block)).await.is_err() { + return; + } + continue; + } + Ok(None) => { + let _ = channel_sender + .send(Err(tonic::Status::internal(format!( + "Internal error, missing nonfinalized block at height [{height_value}].", + )))) + .await; + return; + } + Err(error) => { + let _ = channel_sender + .send(Err(tonic::Status::internal(error.to_string()))) + .await; + return; } } + }; + let compact_block = compact_block_with_pool_types( + indexed_block.to_compact_block(), + &pool_types_vector, + ); + if channel_sender.send(Ok(compact_block)).await.is_err() { + return; } } + } - Ok(None) => { - // At this point, we know that - // the block is NOT FINALIZED in the VALIDATOR. - // Return Ok(None) = no block found. - Ok(None) + // 2) Finalized segment (if any), descending. + if let Some(mut finalized_stream) = finalized_stream { + while let Some(stream_item) = finalized_stream.next().await { + if channel_sender.send(stream_item).await.is_err() { + return; + } } - Err(e) => Err(ChainIndexError::backing_validator(e)), } } - } - } - - /// Returns the block commitment tree data by hash. - async fn get_treestate( - &self, - hash: &types::BlockHash, - ) -> Result<(Option>, Option>), Self::Error> { - let snapshot = self.snapshot_nonfinalized_state().await?; - if !self.block_hash_known_for_treestate(&snapshot, hash).await? { - return Err(ChainIndexError::internal(format!( - "block hash {hash} not found in local chain index" - ))); - } - - match self.source().get_treestate(*hash).await { - Ok(resp) => Ok(resp), - Err(e) => Err(ChainIndexError { - kind: ChainIndexErrorKind::InternalServerError, - message: "failed to fetch treestate from validator".to_string(), - source: Some(Box::new(e)), - }), - } - } + }); - /// Gets the subtree roots of a given pool and the end heights of each root, - /// starting at the provided index, up to an optional maximum number of roots. - async fn get_subtree_roots( - &self, - pool: ShieldedPool, - start_index: u16, - max_entries: Option, - ) -> Result, Self::Error> { - self.source() - .get_subtree_roots(pool, start_index, max_entries) - .await - .map_err(ChainIndexError::backing_validator) + Ok(Some(CompactBlockStream::new(channel_receiver))) } - // ********** Transparent address history methods ********** - /// Returns all changes for the given transparent addresses. async fn get_address_deltas( &self, @@ -2434,119 +2607,6 @@ impl ChainIndex for NodeBackedChainIndexSubscriber Result { - self.source() - .get_address_balance(address_strings) - .await - .map_err(ChainIndexError::backing_validator) - } - - /// Returns the transaction ids made by the given transparent addresses. - async fn get_address_txids( - &self, - request: GetAddressTxIdsRequest, - ) -> Result, Self::Error> { - self.source() - .get_address_txids(request) - .await - .map_err(ChainIndexError::backing_validator) - } - - /// Returns all unspent transparent outputs for the given addresses. - async fn get_address_utxos( - &self, - address_strings: GetAddressBalanceRequest, - ) -> Result, Self::Error> { - self.source() - .get_address_utxos(address_strings) - .await - .map_err(ChainIndexError::backing_validator) - } - - async fn get_outpoint_spenders( - &self, - snapshot: &Self::Snapshot, - outpoints: Vec, - scope: types::ChainScope, - ) -> Result>, Self::Error> { - use std::collections::HashMap; - - let mut result: Vec> = vec![None; outpoints.len()]; - - // 1) Non-finalised best chain (FullChain scope only). Scan only the blocks reachable - // via `heights_to_hashes` (the `blocks` map also holds reorged-away blocks, which - // must not count). One pass builds an outpoint -> spending-txid map regardless of - // how many outpoints we look up. Under `Finalised` scope this is skipped so results - // are reorg-stable. - if let ( - types::ChainScope::FullChain, - ChainIndexSnapshot::NonFinalizedStateExists { - non_finalized_snapshot, - }, - ) = (scope, snapshot) - { - let mut nfs_spenders: HashMap = HashMap::new(); - for hash in non_finalized_snapshot.heights_to_hashes.values() { - let Some(block) = non_finalized_snapshot.blocks.get(hash) else { - continue; - }; - for tx in block.transactions() { - let txid = *tx.txid(); - // `spent_outpoints` already skips coinbase null prevouts and builds each - // `Outpoint`, keeping the construction in one place (see #1332). - for outpoint in tx.transparent().spent_outpoints() { - nfs_spenders.insert(outpoint, txid); - } - } - } - for (i, outpoint) in outpoints.iter().enumerate() { - if let Some(txid) = nfs_spenders.get(outpoint) { - result[i] = Some(*txid); - } - } - } - - // 2) Finalised lookup for the still-unresolved outpoints, batched into one DB call. - let unresolved_indices: Vec = result - .iter() - .enumerate() - .filter_map(|(i, r)| r.is_none().then_some(i)) - .collect(); - if unresolved_indices.is_empty() { - return Ok(result); - } - let unresolved_outpoints: Vec = - unresolved_indices.iter().map(|&i| outpoints[i]).collect(); - let locations = self - .finalized_state - .get_outpoint_spenders(unresolved_outpoints) - .await?; - - // 3) Resolve each finalised `TxLocation` to a txid. Dedup identical locations so a - // block spending several queried outpoints is only fetched once. `get_txid` is a - // single keyed lookup, far cheaper than reconstructing the whole block. - let mut slots_by_location: HashMap> = HashMap::new(); - for (slot, location) in unresolved_indices.into_iter().zip(locations) { - if let Some(location) = location { - slots_by_location.entry(location).or_default().push(slot); - } - } - for (location, slots) in slots_by_location { - let txid = self.finalized_state.get_txid(location).await?; - for slot in slots { - result[slot] = Some(txid); - } - } - - Ok(result) - } - - // ********** Metadata methods ********** - /// Returns Information about the mempool state: /// - size: Current tx count /// - bytes: Sum of all tx sizes @@ -2555,39 +2615,6 @@ impl ChainIndex for NodeBackedChainIndexSubscriber Result { - Ok(match snapshot { - ChainIndexSnapshot::NonFinalizedStateExists { - non_finalized_snapshot, - } => non_finalized_snapshot.best_tip, - - ChainIndexSnapshot::StillSyncingFinalizedState { - validator_finalized_height, - } => { - BlockIndex { - height: *validator_finalized_height, - hash: self - .source() - // TODO: do something more efficient than getting the whole block - .get_block(HashOrHeight::Height((*validator_finalized_height).into())) - .await - .map_err(|e| { - ChainIndexError::database_hole( - validator_finalized_height, - Some(Box::new(e)), - ) - })? - .ok_or(ChainIndexError::database_hole( - validator_finalized_height, - None, - ))? - .hash() - .into(), - } - } - }) - } - async fn get_tx_out_set_info(&self) -> Result { use crate::chain_index::types::db::metadata::{ is_unspendable_tx_out, ZAINO_TXOUTSET_ENTRY_LEN, diff --git a/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs b/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs index 31518411e..b7552f8d5 100644 --- a/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs +++ b/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs @@ -7,7 +7,7 @@ use crate::{ vectors::{indexed_block_chain, load_test_vectors, TestVectorBlockData}, }, types::{BestChainLocation, ChainScope, TransactionHash}, - ChainIndex, NodeBackedChainIndexSubscriber, + ChainIndex, ChainIndexRpcExt, NodeBackedChainIndexSubscriber, }, BlockchainSource as _, Outpoint, }; diff --git a/packages/zaino-state/src/lib.rs b/packages/zaino-state/src/lib.rs index 548c1da16..22cb0121a 100644 --- a/packages/zaino-state/src/lib.rs +++ b/packages/zaino-state/src/lib.rs @@ -67,7 +67,9 @@ pub use backends::{ pub mod chain_index; // Core ChainIndex trait and implementations -pub use chain_index::{ChainIndex, NodeBackedChainIndex, NodeBackedChainIndexSubscriber}; +pub use chain_index::{ + ChainIndex, ChainIndexRpcExt, NodeBackedChainIndex, NodeBackedChainIndexSubscriber, +}; // Source types for ChainIndex backends pub use chain_index::source::{BlockchainSource, State, ValidatorConnector}; // Supporting types From abbeb86aa40445c4702ba46234f447fffaa352fa Mon Sep 17 00:00:00 2001 From: idky137 Date: Thu, 2 Jul 2026 16:25:47 +0100 Subject: [PATCH 012/134] move methods into chain_index / blockchainsource pt 1 --- packages/zaino-state/src/backends/fetch.rs | 12 +++++--- packages/zaino-state/src/backends/state.rs | 32 ++++++---------------- 2 files changed, 16 insertions(+), 28 deletions(-) diff --git a/packages/zaino-state/src/backends/fetch.rs b/packages/zaino-state/src/backends/fetch.rs index 74f63dfcc..237204479 100644 --- a/packages/zaino-state/src/backends/fetch.rs +++ b/packages/zaino-state/src/backends/fetch.rs @@ -485,7 +485,9 @@ impl ZcashIndexer for FetchServiceSubscriber { /// [The function in rpc/blockchain.cpp](https://github.com/zcash/zcash/blob/654a8be2274aa98144c80c1ac459400eaf0eacbe/src/rpc/blockchain.cpp#L325) /// where `return chainActive.Tip()->GetBlockHash().GetHex();` is the [return expression](https://github.com/zcash/zcash/blob/654a8be2274aa98144c80c1ac459400eaf0eacbe/src/rpc/blockchain.cpp#L339)returning a `std::string` async fn get_best_blockhash(&self) -> Result { - Ok(self.fetcher.get_best_blockhash().await?.into()) + let snapshot = self.indexer.snapshot_nonfinalized_state().await?; + let tip = self.indexer.best_chaintip(&snapshot).await?; + Ok(GetBlockHashResponse::new(tip.hash.into())) } /// Returns the current block count in the best valid block chain. @@ -494,15 +496,17 @@ impl ZcashIndexer for FetchServiceSubscriber { /// method: post /// tags: blockchain async fn get_block_count(&self) -> Result { - Ok(self.fetcher.get_block_count().await?.into()) + let snapshot = self.indexer.snapshot_nonfinalized_state().await?; + let tip = self.indexer.best_chaintip(&snapshot).await?; + Ok(tip.height.into()) } + #[allow(deprecated)] async fn get_chain_tips(&self) -> Result { let snapshot = self.indexer.snapshot_nonfinalized_state().await?; let Some(non_finalized_snapshot) = snapshot.get_nfs_snapshot() else { - return Ok(self.fetcher.get_chain_tips().await?); + return Err(FetchServiceError::UnavailableNotSyncedEnough); }; - Ok(chain_tips_from_nonfinalized_snapshot( non_finalized_snapshot, )) diff --git a/packages/zaino-state/src/backends/state.rs b/packages/zaino-state/src/backends/state.rs index 5617ce9e2..e2fe4f08b 100644 --- a/packages/zaino-state/src/backends/state.rs +++ b/packages/zaino-state/src/backends/state.rs @@ -3,6 +3,7 @@ #[allow(deprecated)] use crate::{ chain_index::{ + chain_tips_from_nonfinalized_snapshot, mempool::{Mempool, MempoolSubscriber}, source::ValidatorConnector, types as chain_types, ChainIndex, ChainIndexRpcExt, @@ -1545,19 +1546,9 @@ impl ZcashIndexer for StateServiceSubscriber { /// [The function in rpc/blockchain.cpp](https://github.com/zcash/zcash/blob/654a8be2274aa98144c80c1ac459400eaf0eacbe/src/rpc/blockchain.cpp#L325) /// where `return chainActive.Tip()->GetBlockHash().GetHex();` is the [return expression](https://github.com/zcash/zcash/blob/654a8be2274aa98144c80c1ac459400eaf0eacbe/src/rpc/blockchain.cpp#L339)returning a `std::string` async fn get_best_blockhash(&self) -> Result { - // return should be valid hex encoded. - // Hash from zebra says: - // Return the hash bytes in big-endian byte-order suitable for printing out byte by byte. - // - // Zebra displays transaction and block hashes in big-endian byte-order, - // following the u256 convention set by Bitcoin and zcashd. - match self.read_state_service.best_tip() { - Some(x) => return Ok(GetBlockHash::new(x.1)), - None => { - // try RPC if state read fails: - Ok(self.rpc_client.get_best_blockhash().await?.into()) - } - } + let snapshot = self.indexer.snapshot_nonfinalized_state().await?; + let tip = self.indexer.best_chaintip(&snapshot).await?; + Ok(GetBlockHash::new(tip.hash.into())) } /// Returns the current block count in the best valid block chain. @@ -1567,23 +1558,16 @@ impl ZcashIndexer for StateServiceSubscriber { /// tags: blockchain async fn get_block_count(&self) -> Result { let snapshot = self.indexer.snapshot_nonfinalized_state().await?; - let Some(non_finalized_snapshot) = snapshot.get_nfs_snapshot() else { - // TODO: This probably shouldn't be an error. - // this is an improvement over previous behaviour of - // acting as if we are only synced to the genesis block - return Err(StateServiceError::UnavailableNotSyncedEnough); - }; - let h = non_finalized_snapshot.best_tip.height; - Ok(h.into()) + let tip = self.indexer.best_chaintip(&snapshot).await?; + Ok(tip.height.into()) } async fn get_chain_tips(&self) -> Result { let snapshot = self.indexer.snapshot_nonfinalized_state().await?; let Some(non_finalized_snapshot) = snapshot.get_nfs_snapshot() else { - return Ok(self.rpc_client.get_chain_tips().await?); + return Err(StateServiceError::UnavailableNotSyncedEnough); }; - - Ok(crate::chain_index::chain_tips_from_nonfinalized_snapshot( + Ok(chain_tips_from_nonfinalized_snapshot( non_finalized_snapshot, )) } From c7671b8f1418cf8c558fe0bb34c1257787942c99 Mon Sep 17 00:00:00 2001 From: idky137 Date: Thu, 2 Jul 2026 17:19:25 +0100 Subject: [PATCH 013/134] move methods into chain_index / blockchainsource pt 2 - get_block / get_block_header --- packages/zaino-state/src/backends/fetch.rs | 8 +- packages/zaino-state/src/backends/state.rs | 387 +------------- packages/zaino-state/src/chain_index.rs | 48 +- .../zaino-state/src/chain_index/source.rs | 23 +- .../chain_index/source/mockchain_source.rs | 151 +++++- .../chain_index/source/validator_connector.rs | 470 +++++++++++++++++- .../src/chain_index/tests/mockchain_tests.rs | 94 +++- .../chain_index/tests/proptest_blockgen.rs | 19 + 8 files changed, 809 insertions(+), 391 deletions(-) diff --git a/packages/zaino-state/src/backends/fetch.rs b/packages/zaino-state/src/backends/fetch.rs index 237204479..98f1b7b67 100644 --- a/packages/zaino-state/src/backends/fetch.rs +++ b/packages/zaino-state/src/backends/fetch.rs @@ -426,11 +426,7 @@ impl ZcashIndexer for FetchServiceSubscriber { hash_or_height: String, verbosity: Option, ) -> Result { - Ok(self - .fetcher - .get_block(hash_or_height, verbosity) - .await? - .try_into()?) + Ok(self.indexer.z_get_block(hash_or_height, verbosity).await?) } /// Returns information about the given block and its transactions. @@ -449,7 +445,7 @@ impl ZcashIndexer for FetchServiceSubscriber { hash: String, verbose: bool, ) -> Result { - Ok(self.fetcher.get_block_header(hash, verbose).await?) + Ok(self.indexer.get_block_header(hash, verbose).await?) } async fn get_mining_info(&self) -> Result { diff --git a/packages/zaino-state/src/backends/state.rs b/packages/zaino-state/src/backends/state.rs index e2fe4f08b..59b6e748e 100644 --- a/packages/zaino-state/src/backends/state.rs +++ b/packages/zaino-state/src/backends/state.rs @@ -66,22 +66,20 @@ use zcash_protocol::consensus::NetworkType; use zcash_transparent::address::TransparentAddress; use zebra_chain::{ amount::{Amount, NonNegative}, - block::{Header, Height, SerializedBlock}, + block::Height, chain_tip::NetworkChainTipHeightEstimator, - parameters::{ConsensusBranchId, Network, NetworkKind, NetworkUpgrade}, - serialization::{BytesInDisplayOrder as _, ZcashDeserialize as _, ZcashSerialize}, + parameters::{ConsensusBranchId, NetworkKind, NetworkUpgrade}, + serialization::ZcashDeserialize as _, subtree::NoteCommitmentSubtreeIndex, }; use zebra_rpc::{ client::{ - GetAddressBalanceRequest, GetBlockchainInfoBalance, GetSubtreesByIndexResponse, - GetTreestateResponse, HexData, Input, SubtreeRpcData, TransactionObject, - ValidateAddressResponse, + GetAddressBalanceRequest, GetSubtreesByIndexResponse, GetTreestateResponse, Input, + SubtreeRpcData, TransactionObject, ValidateAddressResponse, }, methods::{ chain_tip_difficulty, AddressBalance, ConsensusBranchIdHex, GetAddressTxIdsRequest, - GetAddressUtxos, GetBlock, GetBlockHash, GetBlockHeader as GetBlockHeaderZebra, - GetBlockHeaderObject, GetBlockTransaction, GetBlockTrees, GetBlockchainInfoResponse, + GetAddressUtxos, GetBlock, GetBlockHash, GetBlockTransaction, GetBlockchainInfoResponse, GetInfo, GetRawTransaction, NetworkUpgradeInfo, NetworkUpgradeStatus, SentTransactionHash, TipConsensusBranch, }, @@ -90,7 +88,7 @@ use zebra_rpc::{ }; use zebra_state::{HashOrHeight, ReadRequest, ReadResponse, ReadStateService}; -use chrono::{DateTime, Utc}; +use chrono::Utc; use futures::TryFutureExt as _; use hex::{FromHex as _, ToHex}; use indexmap::IndexMap; @@ -405,149 +403,6 @@ impl StateServiceSubscriber { monitor: self.chain_tip_change.clone(), } } - /// Returns the requested block header by hash or height, as a [`GetBlockHeader`] JSON string. - /// If the block is not in Zebra's state, - /// returns [error code `-8`.](https://github.com/zcash/zcash/issues/5758) - /// if a height was passed or -5 if a hash was passed. - /// - /// zcashd reference: [`getblockheader`](https://zcash.github.io/rpc/getblockheader.html) - /// method: post - /// tags: blockchain - /// - /// # Parameters - /// - /// - `hash_or_height`: (string, required, example="1") The hash or height - /// for the block to be returned. - /// - `verbose`: (bool, optional, default=false, example=true) false for hex encoded data, - /// true for a json object - /// - /// # Notes - /// - /// The undocumented `chainwork` field is not returned. - /// - /// This rpc is used by get_block(verbose), there is currently no - /// plan to offer this RPC publicly. - async fn get_block_header_inner( - state: &ReadStateService, - network: &Network, - hash_or_height: HashOrHeight, - verbose: Option, - ) -> Result { - let mut state = state.clone(); - let verbose = verbose.unwrap_or(true); - let network = network.clone(); - - let zebra_state::ReadResponse::BlockHeader { - header, - hash, - height, - next_block_hash, - } = state - .ready() - .and_then(|service| service.call(zebra_state::ReadRequest::BlockHeader(hash_or_height))) - .await - .map_err(|_| { - StateServiceError::RpcError(RpcError { - // Compatibility with zcashd. Note that since this function - // is reused by getblock(), we return the errors expected - // by it (they differ whether a hash or a height was passed) - code: LegacyCode::InvalidParameter as i64, - message: "block height not in best chain".to_string(), - data: None, - }) - })? - else { - return Err(StateServiceError::Custom( - "Unexpected response to BlockHeader request".to_string(), - )); - }; - - let response = if !verbose { - GetBlockHeaderZebra::Raw(HexData(header.zcash_serialize_to_vec()?)) - } else { - let zebra_state::ReadResponse::SaplingTree(sapling_tree) = state - .ready() - .and_then(|service| { - service.call(zebra_state::ReadRequest::SaplingTree(hash_or_height)) - }) - .await? - else { - return Err(StateServiceError::Custom( - "Unexpected response to SaplingTree request".to_string(), - )); - }; - // This could be `None` if there's a chain reorg between state queries. - let sapling_tree = sapling_tree.ok_or_else(|| { - StateServiceError::RpcError(zaino_fetch::jsonrpsee::connector::RpcError { - code: LegacyCode::InvalidParameter as i64, - message: "missing sapling tree for block".to_string(), - data: None, - }) - })?; - - let zebra_state::ReadResponse::Depth(depth) = state - .ready() - .and_then(|service| service.call(zebra_state::ReadRequest::Depth(hash))) - .await? - else { - return Err(StateServiceError::Custom( - "Unexpected response to Depth request".to_string(), - )); - }; - - // From - // TODO: Deduplicate const definition, consider - // refactoring this to avoid duplicate logic - const NOT_IN_BEST_CHAIN_CONFIRMATIONS: i64 = -1; - - // Confirmations are one more than the depth. - // Depth is limited by height, so it will never overflow an i64. - let confirmations = depth - .map(|depth| i64::from(depth) + 1) - .unwrap_or(NOT_IN_BEST_CHAIN_CONFIRMATIONS); - - let mut nonce = *header.nonce; - nonce.reverse(); - - let sapling_activation = NetworkUpgrade::Sapling.activation_height(&network); - let sapling_tree_size = sapling_tree.count(); - let final_sapling_root: [u8; 32] = - if sapling_activation.is_some() && height >= sapling_activation.unwrap() { - let mut root: [u8; 32] = sapling_tree.root().into(); - root.reverse(); - root - } else { - [0; 32] - }; - - let difficulty = header.difficulty_threshold.relative_to_network(&network); - let block_commitments = - header_to_block_commitments(&header, &network, height, final_sapling_root)?; - - let block_header = GetBlockHeaderObject::new( - hash, - confirmations, - height, - header.version, - header.merkle_root, - block_commitments, - final_sapling_root, - sapling_tree_size, - header.time.timestamp(), - nonce, - header.solution, - header.difficulty_threshold, - difficulty, - header.previous_block_hash, - next_block_hash, - ); - - GetBlockHeaderZebra::Object(Box::new(block_header)) - }; - - Ok(response) - } - /// Return a list of consecutive compact blocks. #[allow(dead_code, deprecated)] async fn get_block_range_inner( @@ -704,185 +559,6 @@ impl StateServiceSubscriber { }) } - pub(crate) async fn get_block_inner( - state: &ReadStateService, - network: &Network, - hash_or_height: HashOrHeight, - verbosity: Option, - ) -> Result { - let mut state_1 = state.clone(); - - let verbosity = verbosity.unwrap_or(1); - match verbosity { - 0 => { - let request = ReadRequest::Block(hash_or_height); - let response = state_1 - .ready() - .and_then(|service| service.call(request)) - .await?; - let block = expected_read_response!(response, Block); - block.map(SerializedBlock::from).map(GetBlock::Raw).ok_or( - StateServiceError::RpcError(RpcError::new_from_legacycode( - LegacyCode::InvalidParameter, - "block not found", - )), - ) - } - 1 | 2 => { - let state_2 = state.clone(); - let state_3 = state.clone(); - let state_4 = state.clone(); - - let blockandsize_future = { - let req = ReadRequest::BlockAndSize(hash_or_height); - async move { state_1.ready().and_then(|service| service.call(req)).await } - }; - let orchard_future = { - let req = ReadRequest::OrchardTree(hash_or_height); - async move { - state_2 - .clone() - .ready() - .and_then(|service| service.call(req)) - .await - } - }; - - let block_info_future = { - let req = ReadRequest::BlockInfo(hash_or_height); - async move { - state_4 - .clone() - .ready() - .and_then(|service| service.call(req)) - .await - } - }; - let (fullblock, orchard_tree_response, header, block_info) = futures::join!( - blockandsize_future, - orchard_future, - StateServiceSubscriber::get_block_header_inner( - &state_3, - network, - hash_or_height, - Some(true) - ), - block_info_future - ); - - let header_obj = match header? { - GetBlockHeaderZebra::Raw(_hex_data) => unreachable!( - "`true` was passed to get_block_header, an object should be returned" - ), - GetBlockHeaderZebra::Object(get_block_header_object) => get_block_header_object, - }; - - let (transactions_response, size, block_info): (Vec, _, _) = - match (fullblock, block_info) { - ( - Ok(ReadResponse::BlockAndSize(Some((block, size)))), - Ok(ReadResponse::BlockInfo(Some(block_info))), - ) => Ok(( - block - .transactions - .iter() - .map(|transaction| { - match verbosity { - 1 => GetBlockTransaction::Hash(transaction.hash()), - 2 => GetBlockTransaction::Object(Box::new( - TransactionObject::from_transaction( - transaction.clone(), - Some(header_obj.height()), - Some(header_obj.confirmations() as u32), - network, - DateTime::::from_timestamp( - header_obj.time(), - 0, - ), - Some(header_obj.hash()), - // block header has a non-optional height, which indicates - // a mainchain block. It is implied this method cannot return sidechain - // data, at least for now. This is subject to change: TODO - // return Some(true/false) after this assumption is resolved - None, - transaction.hash(), - ), - )), - _ => unreachable!("verbosity known to be 1 or 2"), - } - }) - .collect(), - size, - block_info, - )), - (Ok(ReadResponse::Block(None)), Ok(ReadResponse::BlockInfo(None))) => { - Err(StateServiceError::RpcError(RpcError::new_from_legacycode( - LegacyCode::InvalidParameter, - "block not found", - ))) - } - (Ok(unexpected), Ok(unexpected2)) => { - unreachable!("Unexpected responses from state service: {unexpected:?} {unexpected2:?}") - } - (Err(e), _) | (_, Err(e)) => Err(e.into()), - }?; - - let orchard_tree_response = orchard_tree_response?; - let orchard_tree = expected_read_response!(orchard_tree_response, OrchardTree) - .ok_or(StateServiceError::RpcError(RpcError::new_from_legacycode( - LegacyCode::Misc, - "missing orchard tree", - )))?; - - let final_orchard_root = match NetworkUpgrade::Nu5.activation_height(network) { - Some(activation_height) if header_obj.height() >= activation_height => { - Some(orchard_tree.root().into()) - } - _otherwise => None, - }; - - let trees = - GetBlockTrees::new(header_obj.sapling_tree_size(), orchard_tree.count()); - - let (chain_supply, value_pools) = ( - GetBlockchainInfoBalance::chain_supply(*block_info.value_pools()), - GetBlockchainInfoBalance::value_pools(*block_info.value_pools(), None), - ); - let transaction_count = transactions_response.len(); - - Ok(GetBlock::Object(Box::new( - zebra_rpc::client::BlockObject::new( - header_obj.hash(), - header_obj.confirmations(), - Some(size as i64), - Some(header_obj.height()), - Some(header_obj.version()), - Some(header_obj.merkle_root()), - Some(header_obj.block_commitments()), - Some(header_obj.final_sapling_root()), - final_orchard_root, - transaction_count, - transactions_response, - Some(header_obj.time()), - Some(header_obj.nonce()), - Some(header_obj.solution()), - Some(header_obj.bits()), - Some(header_obj.difficulty()), - Some(chain_supply), - Some(value_pools), - trees, - Some(header_obj.previous_block_hash()), - header_obj.next_block_hash(), - ), - ))) - } - more_than_two => Err(StateServiceError::RpcError(RpcError::new_from_legacycode( - LegacyCode::InvalidParameter, - format!("invalid verbosity of {more_than_two}"), - ))), - } - } - /// Returns the network type running. #[allow(deprecated)] pub fn network(&self) -> zaino_common::Network { @@ -1217,10 +893,7 @@ impl ZcashIndexer for StateServiceSubscriber { hash: String, verbose: bool, ) -> Result { - self.rpc_client - .get_block_header(hash, verbose) - .await - .map_err(|e| StateServiceError::Custom(e.to_string())) + Ok(self.indexer.get_block_header(hash, verbose).await?) } async fn z_get_block( @@ -1228,15 +901,10 @@ impl ZcashIndexer for StateServiceSubscriber { hash_or_height_string: String, verbosity: Option, ) -> Result { - let hash_or_height = HashOrHeight::from_str(&hash_or_height_string); - - StateServiceSubscriber::get_block_inner( - &self.read_state_service.clone(), - &self.data.network(), - hash_or_height?, - verbosity, - ) - .await + Ok(self + .indexer + .z_get_block(hash_or_height_string, verbosity) + .await?) } async fn get_block_deltas(&self, hash: String) -> Result { @@ -2718,37 +2386,6 @@ impl LightWalletIndexer for StateServiceSubscriber { } } -#[allow(clippy::result_large_err, deprecated)] -fn header_to_block_commitments( - header: &Header, - network: &Network, - height: Height, - final_sapling_root: [u8; 32], -) -> Result<[u8; 32], StateServiceError> { - let hash = match header.commitment(network, height).map_err(|e| { - StateServiceError::SerializationError( - zebra_chain::serialization::SerializationError::Parse( - // For some reason this error type takes a - // &'static str, and the the only way to create one - // dynamically is to leak a String. This shouldn't - // be a concern, as this error case should - // never occur when communing with a zebrad, which - // validates this field before serializing it - e.to_string().leak(), - ), - ) - })? { - zebra_chain::block::Commitment::PreSaplingReserved(bytes) => bytes, - zebra_chain::block::Commitment::FinalSaplingRoot(_root) => final_sapling_root, - zebra_chain::block::Commitment::ChainHistoryActivationReserved => [0; 32], - zebra_chain::block::Commitment::ChainHistoryRoot(root) => root.bytes_in_display_order(), - zebra_chain::block::Commitment::ChainHistoryBlockTxAuthCommitment(hash) => { - hash.bytes_in_display_order() - } - }; - Ok(hash) -} - /// An error type for median time past calculation errors #[derive(Debug, Clone)] pub enum MedianTimePast { diff --git a/packages/zaino-state/src/chain_index.rs b/packages/zaino-state/src/chain_index.rs index f5a005b76..66ce9dcd9 100644 --- a/packages/zaino-state/src/chain_index.rs +++ b/packages/zaino-state/src/chain_index.rs @@ -39,6 +39,7 @@ use tokio_util::sync::CancellationToken; use tracing::{info, instrument}; use zaino_fetch::jsonrpsee::response::{ address_deltas::{GetAddressDeltasParams, GetAddressDeltasResponse}, + block_header::GetBlockHeader, chain_tips::{ChainTip, ChainTipStatus, GetChainTipsResponse}, EmptyTxOutSetInfo, GetTxOutSetInfo, GetTxOutSetInfoResponse, }; @@ -48,7 +49,7 @@ pub use zebra_chain::parameters::Network as ZebraNetwork; use zebra_chain::serialization::ZcashSerialize; use zebra_rpc::{ client::{GetAddressBalanceRequest, GetAddressTxIdsRequest}, - methods::{AddressBalance, GetAddressUtxos}, + methods::{AddressBalance, GetAddressUtxos, GetBlock}, }; use zebra_state::HashOrHeight; @@ -574,6 +575,27 @@ pub trait ChainIndexRpcExt: ChainIndex { pool_types: PoolTypeFilter, ) -> impl std::future::Future, Self::Error>>; + /// Returns the `getblock`-shaped block for the given hash-or-height string. + /// + /// `verbosity` follows the zcashd `getblock` convention (0 = raw, 1 = object with + /// txids, 2 = object with full transaction data). + /// + /// zcashd reference: [`getblock`](https://zcash.github.io/rpc/getblock.html) + fn z_get_block( + &self, + hash_or_height: String, + verbosity: Option, + ) -> impl std::future::Future>; + + /// Returns the `getblockheader`-shaped header for the given block hash. + /// + /// zcashd reference: [`getblockheader`](https://zcash.github.io/rpc/getblockheader.html) + fn get_block_header( + &self, + hash: String, + verbose: bool, + ) -> impl std::future::Future>; + // ********** Transparent address history methods ********** /// Returns all changes for the given transparent addresses. @@ -2596,6 +2618,30 @@ impl ChainIndexRpcExt for NodeBackedChainIndexSubscrib Ok(Some(CompactBlockStream::new(channel_receiver))) } + async fn z_get_block( + &self, + hash_or_height: String, + verbosity: Option, + ) -> Result { + let id = HashOrHeight::from_str(&hash_or_height) + .map_err(|error| ChainIndexError::internal(error.to_string()))?; + self.source() + .get_block_verbose(id, verbosity) + .await + .map_err(ChainIndexError::backing_validator) + } + + async fn get_block_header( + &self, + hash: String, + verbose: bool, + ) -> Result { + self.source() + .get_block_header(hash, verbose) + .await + .map_err(ChainIndexError::backing_validator) + } + /// Returns all changes for the given transparent addresses. async fn get_address_deltas( &self, diff --git a/packages/zaino-state/src/chain_index/source.rs b/packages/zaino-state/src/chain_index/source.rs index 73e3e13de..b2352e3a7 100644 --- a/packages/zaino-state/src/chain_index/source.rs +++ b/packages/zaino-state/src/chain_index/source.rs @@ -1,6 +1,6 @@ //! Traits and types for the blockchain source thats serves zaino, commonly a validator connection. -use std::{error::Error, str::FromStr as _, sync::Arc}; +use std::{error::Error, sync::Arc}; use crate::chain_index::{ types::{BlockHash, TransactionHash}, @@ -15,6 +15,7 @@ use zaino_fetch::jsonrpsee::{ connector::{JsonRpSeeConnector, RpcRequestError}, response::{ address_deltas::{GetAddressDeltasParams, GetAddressDeltasResponse}, + block_header::GetBlockHeader, GetBlockError, GetBlockResponse, GetTransactionResponse, GetTreestateResponse, }, }; @@ -61,6 +62,26 @@ pub trait BlockchainSource: Clone + Send + Sync + 'static { id: HashOrHeight, ) -> impl SendFut>>>; + /// Returns the `getblock`-shaped verbose block for the given hash or height. + /// + /// `verbosity` follows the zcashd `getblock` convention (0 = raw, 1 = object with + /// txids, 2 = object with full transaction data). + fn get_block_verbose( + &self, + hash_or_height: HashOrHeight, + verbosity: Option, + ) -> impl SendFut>; + + /// Returns the `getblockheader`-shaped block header for the given block hash. + /// + /// When `verbose` is false the header is returned in raw hex form; when true it is + /// returned as a structured object. + fn get_block_header( + &self, + hash: String, + verbose: bool, + ) -> impl SendFut>; + // ********** Transaction methods ********** /// Returns the transaction by txid diff --git a/packages/zaino-state/src/chain_index/source/mockchain_source.rs b/packages/zaino-state/src/chain_index/source/mockchain_source.rs index 5f262527b..dde2e2552 100644 --- a/packages/zaino-state/src/chain_index/source/mockchain_source.rs +++ b/packages/zaino-state/src/chain_index/source/mockchain_source.rs @@ -1,21 +1,29 @@ //! Mock BlockchainSourceResult implementation. +use super::validator_connector::{ + build_block_header_object, build_verbose_block, confirmations_from_depth, final_orchard_root, + final_sapling_root, zebra_block_header_to_wire, +}; use super::*; use std::collections::{HashMap, HashSet}; +use std::str::FromStr as _; use std::sync::{ atomic::{AtomicU32, Ordering}, Arc, Mutex, }; use zaino_common::network::ActivationHeights; -use zaino_fetch::jsonrpsee::response::address_deltas::BlockInfo; +use zaino_fetch::jsonrpsee::response::{address_deltas::BlockInfo, block_header::GetBlockHeader}; use zebra_chain::{block::Block, orchard::tree as orchard, sapling::tree as sapling}; use zebra_chain::{ - block::Height, + block::{Height, SerializedBlock}, parameters::NetworkKind, - serialization::BytesInDisplayOrder as _, + serialization::{BytesInDisplayOrder as _, ZcashSerialize as _}, transparent::{Address, OutPoint, Output, OutputIndex}, }; -use zebra_rpc::{client::TransactionObject, methods::ValidateAddresses as _}; +use zebra_rpc::{ + client::{HexData, TransactionObject}, + methods::{GetBlock, GetBlockHeaderResponse, ValidateAddresses as _}, +}; use zebra_state::HashOrHeight; /// Build the txid → (height, tx) lookup map used by @@ -332,6 +340,65 @@ impl MockchainSource { self.active_height() as usize } + /// Resolves a hash-or-height request to a block index within the active chain. + fn resolve_index(&self, id: &HashOrHeight) -> Option { + match id { + HashOrHeight::Height(height) => self.valid_height(height.0), + HashOrHeight::Hash(hash) => self.valid_hash(hash), + } + } + + /// The zebra hash of the block after `height_index`, if it is within the active chain. + fn next_block_hash(&self, height_index: usize) -> Option { + let next = height_index + 1; + (next <= self.active_chain_height_as_usize()).then(|| self.blocks[next].hash()) + } + + /// Builds the zebra `getblockheader` response for the block at `height_index` from the + /// test-vector block and tree-root data, using the same builders as the validator path. + fn block_header_response_at( + &self, + height_index: usize, + verbose: bool, + ) -> BlockchainSourceResult { + let block = &self.blocks[height_index]; + let header = &block.header; + if !verbose { + return Ok(GetBlockHeaderResponse::Raw(HexData( + header + .zcash_serialize_to_vec() + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?, + ))); + } + + let network = mockchain_network(); + let hash = block.hash(); + let height = block.coinbase_height().ok_or_else(|| { + BlockchainSourceError::Unrecoverable("block missing coinbase height".to_string()) + })?; + let depth = self.active_height().checked_sub(height.0); + let confirmations = confirmations_from_depth(depth); + + let (sapling_root_bytes, sapling_tree_size) = match &self.roots[height_index].0 { + Some((root, size)) => (<[u8; 32]>::from(*root), *size), + None => ([0u8; 32], 0), + }; + let final_sapling_root = final_sapling_root(sapling_root_bytes, height, &network); + let next_block_hash = self.next_block_hash(height_index); + + let header_obj = build_block_header_object( + header, + hash, + height, + confirmations, + final_sapling_root, + sapling_tree_size, + next_block_hash, + &network, + )?; + Ok(GetBlockHeaderResponse::Object(Box::new(header_obj))) + } + fn block_height_at_index(&self, block_index: usize) -> Height { self.blocks[block_index] .coinbase_height() @@ -450,6 +517,82 @@ impl BlockchainSource for MockchainSource { } } + async fn get_block_verbose( + &self, + hash_or_height: HashOrHeight, + verbosity: Option, + ) -> BlockchainSourceResult { + let verbosity = verbosity.unwrap_or(1); + let height_index = self + .resolve_index(&hash_or_height) + .ok_or_else(|| BlockchainSourceError::Unrecoverable("block not found".to_string()))?; + + match verbosity { + 0 => Ok(GetBlock::Raw(SerializedBlock::from(Arc::clone( + &self.blocks[height_index], + )))), + 1 | 2 => { + let block = &self.blocks[height_index]; + let network = mockchain_network(); + + let GetBlockHeaderResponse::Object(header_obj) = + self.block_header_response_at(height_index, true)? + else { + unreachable!("`true` yields an object") + }; + + let height = block.coinbase_height().ok_or_else(|| { + BlockchainSourceError::Unrecoverable( + "block missing coinbase height".to_string(), + ) + })?; + let (orchard_root_bytes, orchard_tree_size) = match &self.roots[height_index].1 { + Some((root, size)) => (<[u8; 32]>::from(*root), *size), + None => ([0u8; 32], 0), + }; + let final_orchard_root = final_orchard_root(orchard_root_bytes, height, &network); + + // The verbose block reports the block's serialized byte size; the mock has the + // full block, so serialize it to measure. + let size = block + .zcash_serialize_to_vec() + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))? + .len() as i64; + + // `chain_supply` / `value_pools` are cumulative pool balances that the test + // vectors do not carry, so they are `None` for the mock. + Ok(build_verbose_block( + &header_obj, + block, + verbosity, + size, + final_orchard_root, + orchard_tree_size, + None, + None, + &network, + )) + } + more_than_two => Err(BlockchainSourceError::Unrecoverable(format!( + "invalid verbosity of {more_than_two}" + ))), + } + } + + async fn get_block_header( + &self, + hash: String, + verbose: bool, + ) -> BlockchainSourceResult { + let hash_or_height = HashOrHeight::from_str(&hash) + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + let height_index = self.resolve_index(&hash_or_height).ok_or_else(|| { + BlockchainSourceError::Unrecoverable("block height not in best chain".to_string()) + })?; + let header = self.block_header_response_at(height_index, verbose)?; + zebra_block_header_to_wire(header) + } + // ********** Transaction methods ********** async fn get_transaction( diff --git a/packages/zaino-state/src/chain_index/source/validator_connector.rs b/packages/zaino-state/src/chain_index/source/validator_connector.rs index 8560853e1..c9095ebd8 100644 --- a/packages/zaino-state/src/chain_index/source/validator_connector.rs +++ b/packages/zaino-state/src/chain_index/source/validator_connector.rs @@ -1,9 +1,22 @@ //! validator connected blockchain source. +use std::str::FromStr as _; + +use chrono::{DateTime, Utc}; use hex::FromHex as _; -use zaino_fetch::jsonrpsee::response::address_deltas::BlockInfo; -use zebra_chain::serialization::BytesInDisplayOrder as _; -use zebra_rpc::methods::ValidateAddresses as _; +use zaino_fetch::jsonrpsee::response::{address_deltas::BlockInfo, block_header::GetBlockHeader}; +use zebra_chain::{ + block::{Header, SerializedBlock}, + parameters::NetworkUpgrade, + serialization::{BytesInDisplayOrder as _, ZcashSerialize as _}, +}; +use zebra_rpc::{ + client::{BlockObject, GetBlockchainInfoBalance, HexData, TransactionObject}, + methods::{ + GetBlock, GetBlockHeaderObject, GetBlockHeaderResponse, GetBlockTransaction, GetBlockTrees, + ValidateAddresses as _, + }, +}; use crate::Height; @@ -112,6 +125,47 @@ impl BlockchainSource for ValidatorConnector { } } + async fn get_block_verbose( + &self, + hash_or_height: HashOrHeight, + verbosity: Option, + ) -> BlockchainSourceResult { + match self { + ValidatorConnector::State(state) => { + state.get_block_inner(hash_or_height, verbosity).await + } + ValidatorConnector::Fetch(fetch) => fetch + .get_block(hash_or_height.to_string(), verbosity) + .await + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))? + .try_into() + .map_err(|error: zebra_chain::serialization::SerializationError| { + BlockchainSourceError::Unrecoverable(error.to_string()) + }), + } + } + + async fn get_block_header( + &self, + hash: String, + verbose: bool, + ) -> BlockchainSourceResult { + match self { + ValidatorConnector::State(state) => { + let hash_or_height = HashOrHeight::from_str(&hash) + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + let header = state + .get_block_header_inner(hash_or_height, Some(verbose)) + .await?; + zebra_block_header_to_wire(header) + } + ValidatorConnector::Fetch(fetch) => fetch + .get_block_header(hash, verbose) + .await + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string())), + } + } + // ********** Transaction methods ********** // Returns the transaction, and the height of the block that transaction is in if on the best chain @@ -1077,3 +1131,413 @@ impl BlockchainSource for ValidatorConnector { } } } + +impl State { + /// Builds the `getblockheader`-shaped block header from the `ReadStateService`. + /// + /// Returns the zebra `GetBlockHeaderResponse` shape. This is the shared State-path + /// builder used by both [`ValidatorConnector::get_block_header`] (which converts it + /// to the JSON-RPC wire form) and [`State::get_block_inner`] (which reuses the + /// object fields to assemble a verbose block). + /// + /// Moved from the former `StateServiceSubscriber::get_block_header_inner`; the error + /// type is now [`BlockchainSourceError`] rather than the backend error, so the + /// zcashd-compatible `LegacyCode` on the not-in-best-chain case survives only in the + /// message string. + pub(super) async fn get_block_header_inner( + &self, + hash_or_height: HashOrHeight, + verbose: Option, + ) -> BlockchainSourceResult { + let mut state = self.read_state_service.clone(); + let verbose = verbose.unwrap_or(true); + let network = self.network.to_zebra_network(); + + let ReadResponse::BlockHeader { + header, + hash, + height, + next_block_hash, + } = state + .ready() + .and_then(|service| service.call(ReadRequest::BlockHeader(hash_or_height))) + .await + .map_err(|_| { + BlockchainSourceError::Unrecoverable("block height not in best chain".to_string()) + })? + else { + return Err(BlockchainSourceError::Unrecoverable( + "Unexpected response to BlockHeader request".to_string(), + )); + }; + + let response = if !verbose { + GetBlockHeaderResponse::Raw(HexData( + header + .zcash_serialize_to_vec() + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?, + )) + } else { + let ReadResponse::SaplingTree(sapling_tree) = state + .ready() + .and_then(|service| service.call(ReadRequest::SaplingTree(hash_or_height))) + .await + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))? + else { + return Err(BlockchainSourceError::Unrecoverable( + "Unexpected response to SaplingTree request".to_string(), + )); + }; + // This could be `None` if there's a chain reorg between state queries. + let sapling_tree = sapling_tree.ok_or_else(|| { + BlockchainSourceError::Unrecoverable("missing sapling tree for block".to_string()) + })?; + + let ReadResponse::Depth(depth) = state + .ready() + .and_then(|service| service.call(ReadRequest::Depth(hash))) + .await + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))? + else { + return Err(BlockchainSourceError::Unrecoverable( + "Unexpected response to Depth request".to_string(), + )); + }; + + // Confirmations are one more than the depth. Depth is limited by height, so + // it will never overflow an i64. + let confirmations = confirmations_from_depth(depth); + + let sapling_tree_size = sapling_tree.count(); + let final_sapling_root = + final_sapling_root(sapling_tree.root().into(), height, &network); + + let block_header = build_block_header_object( + &header, + hash, + height, + confirmations, + final_sapling_root, + sapling_tree_size, + next_block_hash, + &network, + )?; + + GetBlockHeaderResponse::Object(Box::new(block_header)) + }; + + Ok(response) + } + + /// Builds the `getblock`-shaped verbose block from the `ReadStateService`. + /// + /// Moved from the former `StateServiceSubscriber::get_block_inner`; the error type is + /// now [`BlockchainSourceError`]. + pub(super) async fn get_block_inner( + &self, + hash_or_height: HashOrHeight, + verbosity: Option, + ) -> BlockchainSourceResult { + let mut state_1 = self.read_state_service.clone(); + let verbosity = verbosity.unwrap_or(1); + match verbosity { + 0 => { + let request = ReadRequest::Block(hash_or_height); + let response = state_1 + .ready() + .and_then(|service| service.call(request)) + .await + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + let block = expected_read_response!(response, Block); + block + .map(SerializedBlock::from) + .map(GetBlock::Raw) + .ok_or_else(|| { + BlockchainSourceError::Unrecoverable("block not found".to_string()) + }) + } + 1 | 2 => { + let network = self.network.to_zebra_network(); + let state_2 = self.read_state_service.clone(); + let state_4 = self.read_state_service.clone(); + + let blockandsize_future = { + let req = ReadRequest::BlockAndSize(hash_or_height); + async move { state_1.ready().and_then(|service| service.call(req)).await } + }; + let orchard_future = { + let req = ReadRequest::OrchardTree(hash_or_height); + async move { + state_2 + .clone() + .ready() + .and_then(|service| service.call(req)) + .await + } + }; + let block_info_future = { + let req = ReadRequest::BlockInfo(hash_or_height); + async move { + state_4 + .clone() + .ready() + .and_then(|service| service.call(req)) + .await + } + }; + let (fullblock, orchard_tree_response, header, block_info) = futures::join!( + blockandsize_future, + orchard_future, + self.get_block_header_inner(hash_or_height, Some(true)), + block_info_future + ); + + let header_obj = match header? { + GetBlockHeaderResponse::Raw(_hex_data) => unreachable!( + "`true` was passed to get_block_header, an object should be returned" + ), + GetBlockHeaderResponse::Object(get_block_header_object) => { + get_block_header_object + } + }; + + let (block, size, block_info) = match (fullblock, block_info) { + ( + Ok(ReadResponse::BlockAndSize(Some((block, size)))), + Ok(ReadResponse::BlockInfo(Some(block_info))), + ) => (block, size, block_info), + (Ok(ReadResponse::Block(None)), Ok(ReadResponse::BlockInfo(None))) => { + return Err(BlockchainSourceError::Unrecoverable( + "block not found".to_string(), + )); + } + (Ok(unexpected), Ok(unexpected2)) => { + unreachable!("Unexpected responses from state service: {unexpected:?} {unexpected2:?}") + } + (Err(e), _) | (_, Err(e)) => { + return Err(BlockchainSourceError::Unrecoverable(e.to_string())); + } + }; + + let orchard_tree_response = orchard_tree_response + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + let orchard_tree = expected_read_response!(orchard_tree_response, OrchardTree) + .ok_or_else(|| { + BlockchainSourceError::Unrecoverable("missing orchard tree".to_string()) + })?; + + let final_orchard_root = + final_orchard_root(orchard_tree.root().into(), header_obj.height(), &network); + + let (chain_supply, value_pools) = ( + Some(GetBlockchainInfoBalance::chain_supply( + *block_info.value_pools(), + )), + Some(GetBlockchainInfoBalance::value_pools( + *block_info.value_pools(), + None, + )), + ); + + Ok(build_verbose_block( + &header_obj, + &block, + verbosity, + size as i64, + final_orchard_root, + orchard_tree.count(), + chain_supply, + value_pools, + &network, + )) + } + more_than_two => Err(BlockchainSourceError::Unrecoverable(format!( + "invalid verbosity of {more_than_two}" + ))), + } + } +} + +/// Confirmations are one more than the depth, or -1 when the block is not on the best +/// chain. Depth is limited by height, so it never overflows an `i64`. +pub(crate) fn confirmations_from_depth(depth: Option) -> i64 { + const NOT_IN_BEST_CHAIN_CONFIRMATIONS: i64 = -1; + depth + .map(|depth| i64::from(depth) + 1) + .unwrap_or(NOT_IN_BEST_CHAIN_CONFIRMATIONS) +} + +/// The `finalsaplingroot` field: the sapling note-commitment tree root in display +/// (big-endian) byte order once Sapling has activated, else all-zero. +pub(crate) fn final_sapling_root( + root: [u8; 32], + height: zebra_chain::block::Height, + network: &zebra_chain::parameters::Network, +) -> [u8; 32] { + match NetworkUpgrade::Sapling.activation_height(network) { + Some(activation) if height >= activation => { + let mut root = root; + root.reverse(); + root + } + _ => [0; 32], + } +} + +/// The `finalorchardroot` field: `Some(root)` once NU5 has activated, else `None`. +pub(crate) fn final_orchard_root( + root: [u8; 32], + height: zebra_chain::block::Height, + network: &zebra_chain::parameters::Network, +) -> Option<[u8; 32]> { + match NetworkUpgrade::Nu5.activation_height(network) { + Some(activation) if height >= activation => Some(root), + _ => None, + } +} + +/// Assembles a `getblockheader` object from its already-resolved primitive pieces. +/// +/// Shared by the [`ValidatorConnector`] State path (which reads the pieces from the +/// `ReadStateService`) and by `MockchainSource` (which reads them from test vectors), +/// so the two never drift. +#[allow(clippy::too_many_arguments)] +pub(crate) fn build_block_header_object( + header: &Header, + hash: zebra_chain::block::Hash, + height: zebra_chain::block::Height, + confirmations: i64, + final_sapling_root: [u8; 32], + sapling_tree_size: u64, + next_block_hash: Option, + network: &zebra_chain::parameters::Network, +) -> BlockchainSourceResult { + let mut nonce = *header.nonce; + nonce.reverse(); + let difficulty = header.difficulty_threshold.relative_to_network(network); + let block_commitments = + header_to_block_commitments(header, network, height, final_sapling_root)?; + + Ok(GetBlockHeaderObject::new( + hash, + confirmations, + height, + header.version, + header.merkle_root, + block_commitments, + final_sapling_root, + sapling_tree_size, + header.time.timestamp(), + nonce, + header.solution, + header.difficulty_threshold, + difficulty, + header.previous_block_hash, + next_block_hash, + )) +} + +/// Assembles a verbose (`verbosity` 1 or 2) `getblock` object from a resolved header +/// object plus the block's transactions and pool aggregates. +/// +/// Shared by the [`ValidatorConnector`] State path and `MockchainSource`. `chain_supply` +/// / `value_pools` are the cumulative pool balances (present for the validator, `None` +/// for the mock, whose vectors don't carry them). +#[allow(clippy::too_many_arguments)] +pub(crate) fn build_verbose_block( + header_obj: &GetBlockHeaderObject, + block: &zebra_chain::block::Block, + verbosity: u8, + size: i64, + final_orchard_root: Option<[u8; 32]>, + orchard_tree_size: u64, + chain_supply: Option, + value_pools: Option<[GetBlockchainInfoBalance; 5]>, + network: &zebra_chain::parameters::Network, +) -> GetBlock { + let transactions_response: Vec = block + .transactions + .iter() + .map(|transaction| match verbosity { + 1 => GetBlockTransaction::Hash(transaction.hash()), + 2 => GetBlockTransaction::Object(Box::new(TransactionObject::from_transaction( + transaction.clone(), + Some(header_obj.height()), + Some(header_obj.confirmations() as u32), + network, + DateTime::::from_timestamp(header_obj.time(), 0), + Some(header_obj.hash()), + // A non-optional header height indicates a mainchain block; sidechain + // data is out of scope for now. TODO: return Some(true/false) once resolved. + None, + transaction.hash(), + ))), + _ => unreachable!("verbosity known to be 1 or 2"), + }) + .collect(); + + let trees = GetBlockTrees::new(header_obj.sapling_tree_size(), orchard_tree_size); + let transaction_count = transactions_response.len(); + + GetBlock::Object(Box::new(BlockObject::new( + header_obj.hash(), + header_obj.confirmations(), + Some(size), + Some(header_obj.height()), + Some(header_obj.version()), + Some(header_obj.merkle_root()), + Some(header_obj.block_commitments()), + Some(header_obj.final_sapling_root()), + final_orchard_root, + transaction_count, + transactions_response, + Some(header_obj.time()), + Some(header_obj.nonce()), + Some(header_obj.solution()), + Some(header_obj.bits()), + Some(header_obj.difficulty()), + chain_supply, + value_pools, + trees, + Some(header_obj.previous_block_hash()), + header_obj.next_block_hash(), + ))) +} + +/// Computes the `blockcommitments` field for a block header. +/// +/// Moved verbatim (bar the error type) from the former `state.rs` +/// `header_to_block_commitments`. +pub(crate) fn header_to_block_commitments( + header: &Header, + network: &zebra_chain::parameters::Network, + height: zebra_chain::block::Height, + final_sapling_root: [u8; 32], +) -> BlockchainSourceResult<[u8; 32]> { + let hash = match header.commitment(network, height).map_err(|error| { + BlockchainSourceError::Unrecoverable(format!("invalid block commitment: {error}")) + })? { + zebra_chain::block::Commitment::PreSaplingReserved(bytes) => bytes, + zebra_chain::block::Commitment::FinalSaplingRoot(_root) => final_sapling_root, + zebra_chain::block::Commitment::ChainHistoryActivationReserved => [0; 32], + zebra_chain::block::Commitment::ChainHistoryRoot(root) => root.bytes_in_display_order(), + zebra_chain::block::Commitment::ChainHistoryBlockTxAuthCommitment(hash) => { + hash.bytes_in_display_order() + } + }; + Ok(hash) +} + +/// Converts the zebra `GetBlockHeaderResponse` (Raw/Object) into the JSON-RPC wire +/// `getblockheader` response (Compact/Verbose) expected by the backends. +/// +/// The two shapes serialize to the same JSON, so a serde round-trip is the least-error- +/// prone conversion. +pub(crate) fn zebra_block_header_to_wire( + header: GetBlockHeaderResponse, +) -> BlockchainSourceResult { + let value = serde_json::to_value(&header) + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + serde_json::from_value(value) + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string())) +} diff --git a/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs b/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs index b7552f8d5..8fef9fa05 100644 --- a/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs +++ b/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs @@ -16,8 +16,11 @@ use tokio_stream::StreamExt as _; use zaino_fetch::jsonrpsee::response::address_deltas::{ GetAddressDeltasParams, GetAddressDeltasResponse, }; -use zebra_chain::serialization::ZcashDeserializeInto; +use zaino_fetch::jsonrpsee::response::block_header::GetBlockHeader; +use zebra_chain::serialization::{ZcashDeserializeInto, ZcashSerialize as _}; use zebra_rpc::client::{GetAddressBalanceRequest, GetAddressTxIdsRequest}; +use zebra_rpc::methods::GetBlock; +use zebra_state::HashOrHeight; /// Polls the indexer's nonfinalized-state snapshot until its best-tip height /// equals `expected`, or panics after a 10 s budget. @@ -904,3 +907,92 @@ async fn get_outpoint_spenders_empty_and_single() { vec![None], ); } + +/// `z_get_block` served through the ChainIndex from the mock vectors: verbosity 0 +/// round-trips to the stored block, and verbosity 1 matches the source and reports the +/// block's hash / height / txids. +#[tokio::test(flavor = "multi_thread")] +async fn z_get_block() { + let (_blocks, _indexer, index_reader, mockchain) = + load_test_vectors_and_sync_chain_index(MockchainMode::Static).await; + let active_height = mockchain.active_height(); + + for height in [1u32, active_height / 2, active_height] { + let id = HashOrHeight::Height(zebra_chain::block::Height(height)); + let expected_block = mockchain.get_block(id).await.unwrap().unwrap(); + + // Verbosity 0: the raw serialized block round-trips to the stored block. + let GetBlock::Raw(serialized) = index_reader + .z_get_block(height.to_string(), Some(0)) + .await + .unwrap() + else { + panic!("expected a raw block at verbosity 0"); + }; + let decoded: zebra_chain::block::Block = + serialized.as_ref().zcash_deserialize_into().unwrap(); + assert_eq!(decoded, *expected_block); + + // Verbosity 1: the ChainIndex delegates to the source and the object reports the + // block's hash, height, and every txid. + let via_index = index_reader + .z_get_block(height.to_string(), Some(1)) + .await + .unwrap(); + let via_source = mockchain.get_block_verbose(id, Some(1)).await.unwrap(); + let value = serde_json::to_value(&via_index).unwrap(); + assert_eq!(value, serde_json::to_value(&via_source).unwrap()); + assert_eq!( + value["hash"].as_str().unwrap(), + expected_block.hash().to_string() + ); + assert_eq!(value["height"].as_u64().unwrap(), u64::from(height)); + assert_eq!( + value["tx"].as_array().unwrap().len(), + expected_block.transactions.len() + ); + } +} + +/// `get_block_header` served through the ChainIndex from the mock vectors: the compact +/// (non-verbose) form round-trips to the stored header bytes, and the verbose form +/// matches the source and reports the right hash / height. +#[tokio::test(flavor = "multi_thread")] +async fn get_block_header() { + let (_blocks, _indexer, index_reader, mockchain) = + load_test_vectors_and_sync_chain_index(MockchainMode::Static).await; + let active_height = mockchain.active_height(); + + for height in [1u32, active_height / 2, active_height] { + let id = HashOrHeight::Height(zebra_chain::block::Height(height)); + let block = mockchain.get_block(id).await.unwrap().unwrap(); + let hash = block.hash().to_string(); + + // Non-verbose: the compact hex decodes to the block header's serialization. + let GetBlockHeader::Compact(compact) = index_reader + .get_block_header(hash.clone(), false) + .await + .unwrap() + else { + panic!("expected a compact header when verbose = false"); + }; + assert_eq!( + hex::decode(compact).unwrap(), + block.header.zcash_serialize_to_vec().unwrap() + ); + + // Verbose: the ChainIndex delegates to the source and reports hash / height. + let via_index = index_reader + .get_block_header(hash.clone(), true) + .await + .unwrap(); + let via_source = mockchain + .get_block_header(hash.clone(), true) + .await + .unwrap(); + let value = serde_json::to_value(&via_index).unwrap(); + assert_eq!(value, serde_json::to_value(&via_source).unwrap()); + assert_eq!(value["hash"].as_str().unwrap(), hash); + assert_eq!(value["height"].as_u64().unwrap(), u64::from(height)); + } +} diff --git a/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs b/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs index 8dde1281c..c3a767a19 100644 --- a/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs +++ b/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs @@ -616,6 +616,25 @@ impl BlockchainSource for ProptestMockchain { } } + async fn get_block_verbose( + &self, + _hash_or_height: HashOrHeight, + _verbosity: Option, + ) -> BlockchainSourceResult { + // ProptestMockchain exercises sync/reorg, not the verbose getblock RPC. + unimplemented!() + } + + async fn get_block_header( + &self, + _hash: String, + _verbose: bool, + ) -> BlockchainSourceResult + { + // ProptestMockchain exercises sync/reorg, not the getblockheader RPC. + unimplemented!() + } + /// Returns the block commitment tree data by hash async fn get_commitment_tree_roots( &self, From 06146fe83637b7efa98b9cc076113a8d160c1067 Mon Sep 17 00:00:00 2001 From: zancas Date: Thu, 2 Jul 2026 12:04:38 -0700 Subject: [PATCH 014/134] docs: record the Rust-native tool-selection rule Co-Authored-By: Claude Fable 5 --- CLAUDE.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CLAUDE.md b/CLAUDE.md index 3731d8d8b..a01a2863a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,5 +1,15 @@ # Zaino AI Contributor Guidelines +## Tool selection + +Always prefer Rust-native tools in domains where they are designed to operate. +Dependency and manifest changes go through `cargo add` / `cargo remove` / +`cargo update`. Code navigation and refactors go through rust-analyzer (see +the LSP section below). Verification goes through `cargo check` / +`cargo clippy` / `cargo fmt` / `cargo nextest`. Do not reach for Python, sed, +or regex sweeps over Rust source or `Cargo.toml` when a Rust tool covers the +job. + ## Visibility: minimum required scope All items (functions, methods, structs, enums, fields, modules) MUST use the From ba01451576a373011836cd0d9a945983b2388227 Mon Sep 17 00:00:00 2001 From: zancas Date: Thu, 2 Jul 2026 12:25:28 -0700 Subject: [PATCH 015/134] feat: drop the zingo_common_components dependency Phase 3 of the zingo-common disintegration. zaino-common loses its direct dependency on the shared vocabulary crate: the two From impls in config/network.rs move to zaino-testutils as named boundary functions targeting zcash_local_net's re-exported types, following the existing local_network_from_activation_heights precedent. The zcash_local_net pin moves to the add_zingo_consensus superset rev, which contains all of add_client_support plus the zingo-consensus migration. Production zaino now has no dependency on the activation heights vocabulary at all. Co-Authored-By: Claude Fable 5 --- Cargo.lock | 17 +++----- Cargo.toml | 27 ++---------- live-tests/clientless/tests/chain_cache.rs | 12 ++++-- live-tests/zaino-testutils/src/lib.rs | 47 ++++++++++++++++++++- packages/zaino-common/Cargo.toml | 3 -- packages/zaino-common/src/config/network.rs | 39 ----------------- 6 files changed, 64 insertions(+), 81 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ce7dfe707..3dfd87f91 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6037,7 +6037,6 @@ dependencies = [ "tracing-subscriber", "tracing-tree", "zebra-chain 10.1.0", - "zingo_common_components", ] [[package]] @@ -6272,7 +6271,7 @@ dependencies = [ [[package]] name = "zcash_local_net" version = "0.6.0" -source = "git+https://github.com/zingolabs/infrastructure.git?rev=0cc7dab7d12dd906c23deddd4e1fc3c42cf0f083#0cc7dab7d12dd906c23deddd4e1fc3c42cf0f083" +source = "git+https://github.com/zingolabs/infrastructure.git?rev=0a003b4fd983be56e2ac2e81099a4f2a6dc87141#0a003b4fd983be56e2ac2e81099a4f2a6dc87141" dependencies = [ "getset", "hex", @@ -6287,7 +6286,7 @@ dependencies = [ "zebra-chain 9.0.0", "zebra-node-services 7.0.0", "zebra-rpc 9.0.0", - "zingo_common_components", + "zingo-consensus", "zingo_test_vectors", ] @@ -7088,18 +7087,14 @@ dependencies = [ ] [[package]] -name = "zingo_common_components" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3378e81bdef45a9659736a09deaef2b5e5cb3d1556031468debfca3a21d4fa76" -dependencies = [ - "hex", -] +name = "zingo-consensus" +version = "0.1.0" +source = "git+https://github.com/zingolabs/infrastructure.git?rev=0a003b4fd983be56e2ac2e81099a4f2a6dc87141#0a003b4fd983be56e2ac2e81099a4f2a6dc87141" [[package]] name = "zingo_test_vectors" version = "0.0.1" -source = "git+https://github.com/zingolabs/infrastructure.git?rev=0cc7dab7d12dd906c23deddd4e1fc3c42cf0f083#0cc7dab7d12dd906c23deddd4e1fc3c42cf0f083" +source = "git+https://github.com/zingolabs/infrastructure.git?rev=0a003b4fd983be56e2ac2e81099a4f2a6dc87141#0a003b4fd983be56e2ac2e81099a4f2a6dc87141" dependencies = [ "bip0039", ] diff --git a/Cargo.toml b/Cargo.toml index 09f3453be..2c2fd1827 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,10 +38,6 @@ license = "Apache-2.0" [workspace.dependencies] -# Zingo - -zingo_common_components = "0.3.1" - # Librustzcash incrementalmerkletree = "0.8" zcash_address = "0.12" @@ -49,7 +45,6 @@ zcash_keys = "0.14" zcash_protocol = "0.9" zcash_primitives = "0.28" zcash_transparent = "0.8" -zcash_client_backend = "0.23" # Zebra zebra-chain = "10.0" @@ -72,7 +67,6 @@ tracing-subscriber = { version = "0.3.20", features = [ "time", "json", ] } -tracing-futures = "0.2" tracing-tree = "0.4" nu-ansi-term = "0.50" time = { version = "0.3", features = ["macros", "formatting"] } @@ -87,24 +81,17 @@ reqwest = { version = "0.13", default-features = false, features = [ ] } tower = { version = "0.4", features = ["buffer", "util"] } tonic = { version = "0.14" } -tonic-build = "0.14" prost = "0.14" serde = "1.0" serde_json = "1.0" -jsonrpsee-core = "0.24" jsonrpsee-types = "0.24" jsonrpsee = { version = "0.24", features = ["server", "macros"] } -hyper = "1.6" # Hashmaps, channels, DBs indexmap = "2.2.6" -crossbeam-channel = "0.5" dashmap = "6.1" lmdb = "0.8" lmdb-sys = "0.8" - -# Async -async-stream = "0.3" futures = "0.3.30" # Metrics @@ -113,9 +100,7 @@ metrics-exporter-prometheus = { version = "0.18", default-features = false, feat # Utility thiserror = "2.0" -lazy-regex = "3.3" once_cell = "1.20" -ctrlc = "3.4" chrono = "0.4" which = "8" whoami = "2.1" @@ -151,23 +136,19 @@ zaino-testutils = { path = "live-tests/zaino-testutils" } # Zingo-infra (validator launcher driving the live tests; not a prod dep). # TEMPORARY (zingolabs/infrastructure#269): pinned to an exact commit on the -# `add_client_support` branch so every machine resolves the same code; restore -# the release tag once tagged upstream. -zcash_local_net = { git = "https://github.com/zingolabs/infrastructure.git", rev = "0cc7dab7d12dd906c23deddd4e1fc3c42cf0f083" } +# `add_zingo_consensus` branch, which contains all of `add_client_support` +# plus the zingo-consensus vocabulary migration, so every machine resolves the +# same code; restore the release tag once tagged upstream. +zcash_local_net = { git = "https://github.com/zingolabs/infrastructure.git", rev = "0a003b4fd983be56e2ac2e81099a4f2a6dc87141" } wire_serialized_transaction_test_data = "1.0.0" config = { version = "0.15", default-features = false, features = ["toml"] } nonempty = "0.12.0" proptest = "~1.11" zip32 = "0.2.1" - -# Patch for vulnerable dependency -slab = "0.4.11" anyhow = "1.0" arc-swap = "1.7.1" -cargo-lock = "10.1.0" derive_more = "2.0.1" -lazy_static = "1.5.0" [profile.test] opt-level = 3 diff --git a/live-tests/clientless/tests/chain_cache.rs b/live-tests/clientless/tests/chain_cache.rs index 78b33ceee..aa4cc491b 100644 --- a/live-tests/clientless/tests/chain_cache.rs +++ b/live-tests/clientless/tests/chain_cache.rs @@ -172,9 +172,11 @@ mod chain_query_interface { }, ephemeral, db_version: 1, - network: zaino_common::Network::Regtest(ActivationHeights::from( - test_manager.local_net.get_activation_heights().await, - )), + network: zaino_common::Network::Regtest( + zaino_testutils::from_local_net_activation_heights( + &test_manager.local_net.get_activation_heights().await, + ), + ), }; // **NOTE** The "fetch" backend is currently the backend used in the wild, and @@ -217,7 +219,9 @@ mod chain_query_interface { ephemeral, db_version: 1, network: zaino_common::Network::Regtest( - test_manager.local_net.get_activation_heights().await.into(), + zaino_testutils::from_local_net_activation_heights( + &test_manager.local_net.get_activation_heights().await, + ), ), }; let chain_index = NodeBackedChainIndex::new( diff --git a/live-tests/zaino-testutils/src/lib.rs b/live-tests/zaino-testutils/src/lib.rs index 6ebb6298f..ae414c847 100644 --- a/live-tests/zaino-testutils/src/lib.rs +++ b/live-tests/zaino-testutils/src/lib.rs @@ -326,6 +326,47 @@ pub fn local_network_from_activation_heights( } } +// Conversions at the zcash_local_net boundary. Named functions rather than +// From impls: the orphan rule forbids the impls here, and zaino-common must +// not depend on the harness vocabulary (it reaches us only through +// zcash_local_net's re-exports). +/// Convert zaino activation heights into the `zcash_local_net` activation heights type. +pub fn to_local_net_activation_heights( + activation_heights: &ActivationHeights, +) -> zcash_local_net::protocol::ActivationHeights { + zcash_local_net::protocol::ActivationHeights::builder() + .set_overwinter(activation_heights.overwinter) + .set_sapling(activation_heights.sapling) + .set_blossom(activation_heights.blossom) + .set_heartwood(activation_heights.heartwood) + .set_canopy(activation_heights.canopy) + .set_nu5(activation_heights.nu5) + .set_nu6(activation_heights.nu6) + .set_nu6_1(activation_heights.nu6_1) + .set_nu6_2(activation_heights.nu6_2) + .set_nu7(activation_heights.nu7) + .build() +} + +/// Convert the `zcash_local_net` activation heights type into zaino activation heights. +pub fn from_local_net_activation_heights( + activation_heights: &zcash_local_net::protocol::ActivationHeights, +) -> ActivationHeights { + ActivationHeights { + before_overwinter: activation_heights.overwinter(), + overwinter: activation_heights.overwinter(), + sapling: activation_heights.sapling(), + blossom: activation_heights.blossom(), + heartwood: activation_heights.heartwood(), + canopy: activation_heights.canopy(), + nu5: activation_heights.nu5(), + nu6: activation_heights.nu6(), + nu6_1: activation_heights.nu6_1(), + nu6_2: activation_heights.nu6_2(), + nu7: activation_heights.nu7(), + } +} + /// Path for zcashd binary. #[cfg(feature = "zcashd_support")] pub static ZCASHD_BIN: Lazy> = Lazy::new(|| binary_path("zcashd")); @@ -621,7 +662,11 @@ where // Launch LocalNet: let mut config = C::Config::default(); - config.set_test_parameters(mine_to_pool, activation_heights.into(), chain_cache.clone()); + config.set_test_parameters( + mine_to_pool, + to_local_net_activation_heights(&activation_heights), + chain_cache.clone(), + ); debug!("[TEST] Launching validator"); let (local_net, validator_settings) = C::launch_validator_and_return_config(config) diff --git a/packages/zaino-common/Cargo.toml b/packages/zaino-common/Cargo.toml index 12690005b..5eed93aba 100644 --- a/packages/zaino-common/Cargo.toml +++ b/packages/zaino-common/Cargo.toml @@ -25,6 +25,3 @@ tracing-tree = { workspace = true } nu-ansi-term = { workspace = true } hex = { workspace = true } time = { workspace = true } - -# Zingo -zingo_common_components = { workspace = true } diff --git a/packages/zaino-common/src/config/network.rs b/packages/zaino-common/src/config/network.rs index 3a0f57829..23df9880b 100644 --- a/packages/zaino-common/src/config/network.rs +++ b/packages/zaino-common/src/config/network.rs @@ -131,23 +131,6 @@ impl Default for ActivationHeights { } } -impl From for zingo_common_components::protocol::ActivationHeights { - fn from(val: ActivationHeights) -> Self { - zingo_common_components::protocol::ActivationHeightsBuilder::new() - .set_overwinter(val.overwinter) - .set_sapling(val.sapling) - .set_blossom(val.blossom) - .set_heartwood(val.heartwood) - .set_canopy(val.canopy) - .set_nu5(val.nu5) - .set_nu6(val.nu6) - .set_nu6_1(val.nu6_1) - .set_nu6_2(val.nu6_2) - .set_nu7(val.nu7) - .build() - } -} - impl From for ActivationHeights { fn from( ConfiguredActivationHeights { @@ -211,24 +194,6 @@ impl From for ConfiguredActivationHeights { } } -impl From for ActivationHeights { - fn from(activation_heights: zingo_common_components::protocol::ActivationHeights) -> Self { - ActivationHeights { - before_overwinter: activation_heights.overwinter(), - overwinter: activation_heights.overwinter(), - sapling: activation_heights.sapling(), - blossom: activation_heights.blossom(), - heartwood: activation_heights.heartwood(), - canopy: activation_heights.canopy(), - nu5: activation_heights.nu5(), - nu6: activation_heights.nu6(), - nu6_1: activation_heights.nu6_1(), - nu6_2: activation_heights.nu6_2(), - nu7: activation_heights.nu7(), - } - } -} - impl Network { /// Convert to Zebra's network type for internal use (alias for to_zebra_default). pub fn to_zebra_network(&self) -> zebra_chain::parameters::Network { @@ -382,9 +347,5 @@ mod tests { let zebra_heights: zebra_chain::parameters::testnet::ConfiguredActivationHeights = heights.into(); assert_eq!(zebra_heights.nu6_2, Some(2)); - - let zingo_heights: zingo_common_components::protocol::ActivationHeights = heights.into(); - let heights = ActivationHeights::from(zingo_heights); - assert_eq!(heights.nu6_2, Some(2)); } } From fc063cbad7798859e007a1dc4d8c40e0f920edba Mon Sep 17 00:00:00 2001 From: zancas Date: Thu, 2 Jul 2026 19:06:19 -0700 Subject: [PATCH 016/134] build: bump zcash_local_net pin to remove_unnecessary_deps head Move the infrastructure pin from the add_zingo_consensus branch (0a003b4f) to b09d9c11, the head of remove_unnecessary_deps. The new commit replaces zcash_protocol::PoolType with zingo_consensus::MinerPool as the harness's mine-to-pool selector, so the pin bump carries the matching rename through zaino-testutils (re-export, SHIELDED_FUNDING_POOL, default_mining_pool, the mine_to_pool parameters) and the five e2e call sites. The proto-side PoolType enum in zaino-proto is unrelated and untouched. Only the three infrastructure-sourced crates move in the lock; zcash_protocol drops out of zcash_local_net's dependency list. Verified with cargo check --tests on zaino-testutils/e2e/clientless under both default features and zcashd_support; fmt clean; clippy shows only pre-existing warnings. Co-Authored-By: Claude Fable 5 --- Cargo.lock | 7 +++---- Cargo.toml | 9 +++++---- live-tests/e2e/tests/devtool.rs | 8 ++++---- live-tests/e2e/tests/test_vectors.rs | 2 +- live-tests/zaino-testutils/src/lib.rs | 18 +++++++++--------- 5 files changed, 22 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3dfd87f91..d419f9804 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6271,7 +6271,7 @@ dependencies = [ [[package]] name = "zcash_local_net" version = "0.6.0" -source = "git+https://github.com/zingolabs/infrastructure.git?rev=0a003b4fd983be56e2ac2e81099a4f2a6dc87141#0a003b4fd983be56e2ac2e81099a4f2a6dc87141" +source = "git+https://github.com/zingolabs/infrastructure.git?rev=b09d9c11afc349a1e7ee18a35e3bae0746b829f0#b09d9c11afc349a1e7ee18a35e3bae0746b829f0" dependencies = [ "getset", "hex", @@ -6282,7 +6282,6 @@ dependencies = [ "thiserror 1.0.69", "tokio", "tracing", - "zcash_protocol", "zebra-chain 9.0.0", "zebra-node-services 7.0.0", "zebra-rpc 9.0.0", @@ -7089,12 +7088,12 @@ dependencies = [ [[package]] name = "zingo-consensus" version = "0.1.0" -source = "git+https://github.com/zingolabs/infrastructure.git?rev=0a003b4fd983be56e2ac2e81099a4f2a6dc87141#0a003b4fd983be56e2ac2e81099a4f2a6dc87141" +source = "git+https://github.com/zingolabs/infrastructure.git?rev=b09d9c11afc349a1e7ee18a35e3bae0746b829f0#b09d9c11afc349a1e7ee18a35e3bae0746b829f0" [[package]] name = "zingo_test_vectors" version = "0.0.1" -source = "git+https://github.com/zingolabs/infrastructure.git?rev=0a003b4fd983be56e2ac2e81099a4f2a6dc87141#0a003b4fd983be56e2ac2e81099a4f2a6dc87141" +source = "git+https://github.com/zingolabs/infrastructure.git?rev=b09d9c11afc349a1e7ee18a35e3bae0746b829f0#b09d9c11afc349a1e7ee18a35e3bae0746b829f0" dependencies = [ "bip0039", ] diff --git a/Cargo.toml b/Cargo.toml index 2c2fd1827..9342eb5f4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -136,10 +136,11 @@ zaino-testutils = { path = "live-tests/zaino-testutils" } # Zingo-infra (validator launcher driving the live tests; not a prod dep). # TEMPORARY (zingolabs/infrastructure#269): pinned to an exact commit on the -# `add_zingo_consensus` branch, which contains all of `add_client_support` -# plus the zingo-consensus vocabulary migration, so every machine resolves the -# same code; restore the release tag once tagged upstream. -zcash_local_net = { git = "https://github.com/zingolabs/infrastructure.git", rev = "0a003b4fd983be56e2ac2e81099a4f2a6dc87141" } +# `remove_unnecessary_deps` branch, which contains the zingo-consensus +# vocabulary migration plus the MinerPool replacement of +# zcash_protocol::PoolType, so every machine resolves the same code; restore +# the release tag once tagged upstream. +zcash_local_net = { git = "https://github.com/zingolabs/infrastructure.git", rev = "b09d9c11afc349a1e7ee18a35e3bae0746b829f0" } wire_serialized_transaction_test_data = "1.0.0" config = { version = "0.15", default-features = false, features = ["toml"] } diff --git a/live-tests/e2e/tests/devtool.rs b/live-tests/e2e/tests/devtool.rs index be20e3d08..7338f695f 100644 --- a/live-tests/e2e/tests/devtool.rs +++ b/live-tests/e2e/tests/devtool.rs @@ -1250,7 +1250,7 @@ async fn transparent_data_in_compact_block() { // The assertion below requires every tx to carry a transparent vout; // the miner's transparent coinbase is that data source, so coinbase // must land on the miner taddr. - zaino_testutils::PoolType::Transparent, + zaino_testutils::MinerPool::Transparent, &ValidatorKind::Zebrad, None, true, @@ -1328,7 +1328,7 @@ async fn launch_transparent_and_faucet_taddr( // These tests query the faucet taddr, which only coinbase funds — // mining must stay transparent or the queries compare empty against // empty. - zaino_testutils::PoolType::Transparent, + zaino_testutils::MinerPool::Transparent, &ValidatorKind::Zebrad, None, true, @@ -1481,7 +1481,7 @@ async fn get_address_utxos_stream_faucet_fetch_vs_state() { /// The block-range edge tests need a known 100-block tip and no wallet client. async fn launch_transparent_to_height_100() -> zaino_testutils::StateAndFetchServices { let svc = zaino_testutils::launch_state_and_fetch_services_mining_to::( - zaino_testutils::PoolType::Transparent, + zaino_testutils::MinerPool::Transparent, &ValidatorKind::Zebrad, None, true, @@ -1662,7 +1662,7 @@ async fn address_deltas() { let mut svc = zaino_testutils::launch_state_and_fetch_services_mining_to::( // The deltas under test are the faucet taddr's coinbase credits and the // shield's debit — coinbase must land on the miner taddr. - zaino_testutils::PoolType::Transparent, + zaino_testutils::MinerPool::Transparent, &ValidatorKind::Zebrad, None, true, diff --git a/live-tests/e2e/tests/test_vectors.rs b/live-tests/e2e/tests/test_vectors.rs index 642d1d55a..fc5b56ef3 100644 --- a/live-tests/e2e/tests/test_vectors.rs +++ b/live-tests/e2e/tests/test_vectors.rs @@ -48,7 +48,7 @@ async fn create_200_block_regtest_chain_vectors() { // repeatedly shielding transparent coinbase — mining must stay transparent // so regenerated vectors keep that shape. let mut test_manager = TestManager::::launch_mining_to( - zaino_testutils::PoolType::Transparent, + zaino_testutils::MinerPool::Transparent, &ValidatorKind::Zebrad, None, None, diff --git a/live-tests/zaino-testutils/src/lib.rs b/live-tests/zaino-testutils/src/lib.rs index ae414c847..468e434fe 100644 --- a/live-tests/zaino-testutils/src/lib.rs +++ b/live-tests/zaino-testutils/src/lib.rs @@ -40,7 +40,7 @@ use zcash_local_net::validator::zcashd::{Zcashd, ZcashdConfig}; use zcash_local_net::validator::zebrad::{Zebrad, ZebradConfig}; pub use zcash_local_net::validator::Validator; use zcash_local_net::validator::ValidatorConfig as _; -pub use zcash_local_net::PoolType; +pub use zcash_local_net::MinerPool; use zcash_local_net::{logs::LogsToStdoutAndStderr, process::Process}; use zebra_chain::parameters::NetworkKind; use zebra_rpc::methods::GetInfo; @@ -529,18 +529,18 @@ impl ValidatorExt for Zcashd { /// /// This constant is the single upgrade point for shielded funding: when the /// next shielded pool (ironwood) becomes minable, only this value changes. -pub const SHIELDED_FUNDING_POOL: PoolType = PoolType::ORCHARD; +pub const SHIELDED_FUNDING_POOL: MinerPool = MinerPool::Orchard; /// The pool a validator mines to when the session doesn't opt into /// [`SHIELDED_FUNDING_POOL`]: transparent for zebrad (cheapest block /// templates — a shielded miner address would cost a halo2 proof per block), -/// ORCHARD for zcashd (its historical setting; the cached launch reward +/// Orchard for zcashd (its historical setting; the cached launch reward /// funds wallets without extra mining). -pub fn default_mining_pool(validator: &ValidatorKind) -> PoolType { +pub fn default_mining_pool(validator: &ValidatorKind) -> MinerPool { if validator == &ValidatorKind::Zebrad { - PoolType::Transparent + MinerPool::Transparent } else { - PoolType::ORCHARD + MinerPool::Orchard } } @@ -630,7 +630,7 @@ where fields(validator = ?validator, network = ?network, mine_to_pool = ?mine_to_pool, enable_zaino, enable_clients) )] pub async fn launch_mining_to( - mine_to_pool: PoolType, + mine_to_pool: MinerPool, validator: &ValidatorKind, network: Option, activation_heights: Option, @@ -1043,7 +1043,7 @@ pub async fn launch_state_and_fetch_services( /// subject is the miner's coinbase footprint. #[allow(deprecated)] pub async fn launch_state_and_fetch_services_mining_to( - mine_to_pool: PoolType, + mine_to_pool: MinerPool, validator: &ValidatorKind, chain_cache: Option, enable_zaino: bool, @@ -1363,7 +1363,7 @@ pub async fn launch_with_fetch_subscriber( /// subject is the miner's coinbase footprint. #[allow(deprecated)] pub async fn launch_with_fetch_subscriber_mining_to( - mine_to_pool: PoolType, + mine_to_pool: MinerPool, validator: &ValidatorKind, chain_cache: Option, ) -> (TestManager, FetchServiceSubscriber) { From 92c8e8fd05f9b725a09fddcb1afc5f2e0cd08861 Mon Sep 17 00:00:00 2001 From: zancas Date: Thu, 2 Jul 2026 20:51:04 -0700 Subject: [PATCH 017/134] build: track zcash_local_net on the remove_unnecessary_deps branch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow zingolabs/infrastructure#276 to its current head (029e93eb) by switching the git dependency from an exact rev to branch tracking; the committed Cargo.lock still pins the resolved commit, so builds stay deterministic until the next cargo update. The PR empties zcash_local_net's dependency tables of the zebra and librustzcash stacks, which drops an entire duplicate zebra stack from our lock: zebra-chain 9.0.0, zebra-rpc 9.0.0, zebra-node-services 7.0.0, zebra-consensus/network/script/state 8.x, and tower-batch-control/tower-fallback. No zaino code changes were needed — the MinerPool rename in the previous commit was the only API break. Verified with cargo check --tests on zaino-testutils/e2e/clientless under both default features and zcashd_support; fmt clean; clippy shows only pre-existing warnings. Co-Authored-By: Claude Fable 5 --- Cargo.lock | 389 ++++++----------------------------------------------- Cargo.toml | 12 +- 2 files changed, 49 insertions(+), 352 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d419f9804..eca18c776 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -718,9 +718,9 @@ dependencies = [ "zaino-testutils", "zainod", "zcash_local_net", - "zebra-chain 10.1.0", - "zebra-rpc 10.0.1", - "zebra-state 9.0.1", + "zebra-chain", + "zebra-rpc", + "zebra-state", "zip32", ] @@ -1234,9 +1234,9 @@ dependencies = [ "zainod", "zcash_local_net", "zcash_primitives", - "zebra-chain 10.1.0", - "zebra-rpc 10.0.1", - "zebra-state 9.0.1", + "zebra-chain", + "zebra-rpc", + "zebra-state", ] [[package]] @@ -5065,23 +5065,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "tower-batch-control" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e6cf52578f98b4da47335c26c4f883f7993b1a9b9d2f5420eb8dbfd5dd19a28" -dependencies = [ - "futures", - "futures-core", - "pin-project", - "rayon", - "tokio", - "tokio-util", - "tower 0.4.13", - "tracing", - "tracing-futures", -] - [[package]] name = "tower-batch-control" version = "1.0.1" @@ -5098,18 +5081,6 @@ dependencies = [ "tracing-futures", ] -[[package]] -name = "tower-fallback" -version = "0.2.41" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4434e19ee996ee5c6aa42f11463a355138452592e5c5b5b73b6f0f19534556af" -dependencies = [ - "futures-core", - "pin-project", - "tower 0.4.13", - "tracing", -] - [[package]] name = "tower-fallback" version = "0.2.41" @@ -6036,7 +6007,7 @@ dependencies = [ "tracing", "tracing-subscriber", "tracing-tree", - "zebra-chain 10.1.0", + "zebra-chain", ] [[package]] @@ -6064,8 +6035,8 @@ dependencies = [ "wire_serialized_transaction_test_data", "zaino-common", "zaino-proto", - "zebra-chain 10.1.0", - "zebra-rpc 10.0.1", + "zebra-chain", + "zebra-rpc", ] [[package]] @@ -6077,8 +6048,8 @@ dependencies = [ "tonic-prost", "tonic-prost-build", "which", - "zebra-chain 10.1.0", - "zebra-state 9.0.1", + "zebra-chain", + "zebra-state", ] [[package]] @@ -6097,8 +6068,8 @@ dependencies = [ "zaino-fetch", "zaino-proto", "zaino-state", - "zebra-chain 10.1.0", - "zebra-rpc 10.0.1", + "zebra-chain", + "zebra-rpc", ] [[package]] @@ -6150,9 +6121,9 @@ dependencies = [ "zcash_primitives", "zcash_protocol", "zcash_transparent", - "zebra-chain 10.1.0", - "zebra-rpc 10.0.1", - "zebra-state 9.0.1", + "zebra-chain", + "zebra-rpc", + "zebra-state", ] [[package]] @@ -6173,9 +6144,9 @@ dependencies = [ "zainod", "zcash_local_net", "zcash_protocol", - "zebra-chain 10.1.0", - "zebra-rpc 10.0.1", - "zebra-state 9.0.1", + "zebra-chain", + "zebra-rpc", + "zebra-state", ] [[package]] @@ -6200,8 +6171,8 @@ dependencies = [ "zaino-state", "zcash_address", "zcash_protocol", - "zebra-chain 10.1.0", - "zebra-state 9.0.1", + "zebra-chain", + "zebra-state", ] [[package]] @@ -6271,20 +6242,19 @@ dependencies = [ [[package]] name = "zcash_local_net" version = "0.6.0" -source = "git+https://github.com/zingolabs/infrastructure.git?rev=b09d9c11afc349a1e7ee18a35e3bae0746b829f0#b09d9c11afc349a1e7ee18a35e3bae0746b829f0" +source = "git+https://github.com/zingolabs/infrastructure.git?branch=remove_unnecessary_deps#029e93ebf50fa00b883fe6bce00e1c0ae049bb2a" dependencies = [ "getset", "hex", "json", "reqwest 0.12.28", + "serde", "serde_json", + "sha2 0.10.9", "tempfile", "thiserror 1.0.69", "tokio", "tracing", - "zebra-chain 9.0.0", - "zebra-node-services 7.0.0", - "zebra-rpc 9.0.0", "zingo-consensus", "zingo_test_vectors", ] @@ -6420,70 +6390,6 @@ dependencies = [ "zip32", ] -[[package]] -name = "zebra-chain" -version = "9.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30ed3a4e00e8f0b2197f3d8fd6cad02351a25ddbffcf0a6dc1fe463d7ea4ccfa" -dependencies = [ - "bech32", - "bitflags 2.13.0", - "bitflags-serde-legacy", - "bitvec", - "blake2b_simd", - "blake2s_simd", - "bounded-vec", - "bs58", - "byteorder", - "chrono", - "derive-getters", - "dirs", - "ed25519-zebra", - "equihash", - "futures", - "group", - "halo2_proofs", - "hex", - "humantime", - "incrementalmerkletree", - "itertools 0.14.0", - "jubjub", - "lazy_static", - "num-integer", - "orchard", - "primitive-types 0.12.2", - "rand_core 0.6.4", - "rayon", - "reddsa", - "redjubjub", - "ripemd 0.1.3", - "sapling-crypto", - "schemars 1.2.1", - "secp256k1", - "serde", - "serde-big-array", - "serde_json", - "serde_with", - "sha2 0.10.9", - "sinsemilla", - "static_assertions", - "strum", - "tempfile", - "thiserror 2.0.18", - "tokio", - "tracing", - "uint 0.10.0", - "x25519-dalek", - "zcash_address", - "zcash_encoding", - "zcash_history", - "zcash_note_encryption", - "zcash_primitives", - "zcash_protocol", - "zcash_script", - "zcash_transparent", -] - [[package]] name = "zebra-chain" version = "10.1.0" @@ -6552,49 +6458,6 @@ dependencies = [ "zebra-test", ] -[[package]] -name = "zebra-consensus" -version = "8.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15475adf8c271d03de99d18d622a8ae4c512c401e3e29da1f27a0ba62b22057c" -dependencies = [ - "bellman", - "blake2b_simd", - "bls12_381", - "chrono", - "derive-getters", - "futures", - "futures-util", - "halo2_proofs", - "jubjub", - "lazy_static", - "libzcash_script", - "metrics", - "mset", - "once_cell", - "orchard", - "rand 0.8.6", - "rayon", - "sapling-crypto", - "serde", - "thiserror 2.0.18", - "tokio", - "tower 0.4.13", - "tower-batch-control 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", - "tower-fallback 0.2.41 (registry+https://github.com/rust-lang/crates.io-index)", - "tracing", - "tracing-futures", - "zcash_primitives", - "zcash_proofs", - "zcash_protocol", - "zcash_script", - "zcash_transparent", - "zebra-chain 9.0.0", - "zebra-node-services 7.0.0", - "zebra-script 8.0.0", - "zebra-state 8.0.0", -] - [[package]] name = "zebra-consensus" version = "9.0.1" @@ -6622,8 +6485,8 @@ dependencies = [ "thiserror 2.0.18", "tokio", "tower 0.4.13", - "tower-batch-control 1.0.1 (git+https://github.com/ZcashFoundation/zebra.git?rev=9a27f886a5bfb143f65d1712e912cef252426800)", - "tower-fallback 0.2.41 (git+https://github.com/ZcashFoundation/zebra.git?rev=9a27f886a5bfb143f65d1712e912cef252426800)", + "tower-batch-control", + "tower-fallback", "tracing", "tracing-futures", "zcash_primitives", @@ -6631,48 +6494,10 @@ dependencies = [ "zcash_protocol", "zcash_script", "zcash_transparent", - "zebra-chain 10.1.0", - "zebra-node-services 8.0.0", - "zebra-script 9.0.0", - "zebra-state 9.0.1", -] - -[[package]] -name = "zebra-network" -version = "8.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8ba4f7dcd795eb84a5a33f5c4f29df60dd4971be363d2cd98d5fba5ce0477f2" -dependencies = [ - "bitflags 2.13.0", - "byteorder", - "bytes", - "chrono", - "dirs", - "futures", - "hex", - "humantime-serde", - "indexmap 2.14.0", - "itertools 0.14.0", - "lazy_static", - "metrics", - "num-integer", - "ordered-map", - "pin-project", - "rand 0.8.6", - "rayon", - "regex", - "schemars 1.2.1", - "serde", - "tempfile", - "thiserror 2.0.18", - "tokio", - "tokio-stream", - "tokio-util", - "tower 0.4.13", - "tracing", - "tracing-error", - "tracing-futures", - "zebra-chain 9.0.0", + "zebra-chain", + "zebra-node-services", + "zebra-script", + "zebra-state", ] [[package]] @@ -6709,23 +6534,7 @@ dependencies = [ "tracing", "tracing-error", "tracing-futures", - "zebra-chain 10.1.0", -] - -[[package]] -name = "zebra-node-services" -version = "7.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abeece0fb4503a1df00c5ab736754b55c702613968f622f6aa3c2f9842aed2b2" -dependencies = [ - "color-eyre", - "jsonrpsee-types", - "reqwest 0.12.28", - "serde", - "serde_json", - "tokio", - "tower 0.4.13", - "zebra-chain 9.0.0", + "zebra-chain", ] [[package]] @@ -6740,67 +6549,7 @@ dependencies = [ "serde_json", "tokio", "tower 0.4.13", - "zebra-chain 10.1.0", -] - -[[package]] -name = "zebra-rpc" -version = "9.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b460869e352c9f1b49a00ed9beaae04ca3e6498381c7189d4b90a79e51c260bd" -dependencies = [ - "base64", - "chrono", - "color-eyre", - "derive-getters", - "derive-new", - "futures", - "hex", - "http-body-util", - "hyper", - "indexmap 2.14.0", - "jsonrpsee", - "jsonrpsee-proc-macros", - "jsonrpsee-types", - "lazy_static", - "metrics", - "nix", - "openrpsee", - "orchard", - "phf", - "prost", - "rand 0.8.6", - "sapling-crypto", - "schemars 1.2.1", - "semver", - "serde", - "serde_json", - "serde_with", - "strum", - "strum_macros", - "thiserror 2.0.18", - "tokio", - "tokio-stream", - "tonic", - "tonic-prost", - "tonic-prost-build", - "tonic-reflection", - "tower 0.4.13", - "tracing", - "which", - "zcash_address", - "zcash_keys", - "zcash_primitives", - "zcash_proofs", - "zcash_protocol", - "zcash_script", - "zcash_transparent", - "zebra-chain 9.0.0", - "zebra-consensus 8.0.0", - "zebra-network 8.0.0", - "zebra-node-services 7.0.0", - "zebra-script 8.0.0", - "zebra-state 8.0.0", + "zebra-chain", ] [[package]] @@ -6854,26 +6603,12 @@ dependencies = [ "zcash_protocol", "zcash_script", "zcash_transparent", - "zebra-chain 10.1.0", - "zebra-consensus 9.0.1", - "zebra-network 9.0.0", - "zebra-node-services 8.0.0", - "zebra-script 9.0.0", - "zebra-state 9.0.1", -] - -[[package]] -name = "zebra-script" -version = "8.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4fc38c19388671df8a77c767e279b135ffaecd5e7df6ed7cd096390c080b4a3" -dependencies = [ - "libzcash_script", - "rand 0.8.6", - "thiserror 2.0.18", - "zcash_primitives", - "zcash_script", - "zebra-chain 9.0.0", + "zebra-chain", + "zebra-consensus", + "zebra-network", + "zebra-node-services", + "zebra-script", + "zebra-state", ] [[package]] @@ -6886,45 +6621,7 @@ dependencies = [ "thiserror 2.0.18", "zcash_primitives", "zcash_script", - "zebra-chain 10.1.0", -] - -[[package]] -name = "zebra-state" -version = "8.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57b351fde5047dce0d505c9dc26863ee818b7579e9cf8d8e44489deb9decfbb7" -dependencies = [ - "bincode", - "chrono", - "crossbeam-channel", - "derive-getters", - "derive-new", - "dirs", - "futures", - "hex", - "hex-literal", - "human_bytes", - "humantime-serde", - "indexmap 2.14.0", - "itertools 0.14.0", - "lazy_static", - "metrics", - "mset", - "rayon", - "regex", - "rlimit", - "rocksdb", - "sapling-crypto", - "semver", - "serde", - "tempfile", - "thiserror 2.0.18", - "tokio", - "tower 0.4.13", - "tracing", - "zebra-chain 9.0.0", - "zebra-node-services 7.0.0", + "zebra-chain", ] [[package]] @@ -6960,8 +6657,8 @@ dependencies = [ "tokio", "tower 0.4.13", "tracing", - "zebra-chain 10.1.0", - "zebra-node-services 8.0.0", + "zebra-chain", + "zebra-node-services", ] [[package]] @@ -7088,12 +6785,12 @@ dependencies = [ [[package]] name = "zingo-consensus" version = "0.1.0" -source = "git+https://github.com/zingolabs/infrastructure.git?rev=b09d9c11afc349a1e7ee18a35e3bae0746b829f0#b09d9c11afc349a1e7ee18a35e3bae0746b829f0" +source = "git+https://github.com/zingolabs/infrastructure.git?branch=remove_unnecessary_deps#029e93ebf50fa00b883fe6bce00e1c0ae049bb2a" [[package]] name = "zingo_test_vectors" version = "0.0.1" -source = "git+https://github.com/zingolabs/infrastructure.git?rev=b09d9c11afc349a1e7ee18a35e3bae0746b829f0#b09d9c11afc349a1e7ee18a35e3bae0746b829f0" +source = "git+https://github.com/zingolabs/infrastructure.git?branch=remove_unnecessary_deps#029e93ebf50fa00b883fe6bce00e1c0ae049bb2a" dependencies = [ "bip0039", ] diff --git a/Cargo.toml b/Cargo.toml index 9342eb5f4..492427a3e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -135,12 +135,12 @@ zaino-serve = { path = "packages/zaino-serve", version = "0.3.1", default-featur zaino-testutils = { path = "live-tests/zaino-testutils" } # Zingo-infra (validator launcher driving the live tests; not a prod dep). -# TEMPORARY (zingolabs/infrastructure#269): pinned to an exact commit on the -# `remove_unnecessary_deps` branch, which contains the zingo-consensus -# vocabulary migration plus the MinerPool replacement of -# zcash_protocol::PoolType, so every machine resolves the same code; restore -# the release tag once tagged upstream. -zcash_local_net = { git = "https://github.com/zingolabs/infrastructure.git", rev = "b09d9c11afc349a1e7ee18a35e3bae0746b829f0" } +# TEMPORARY (zingolabs/infrastructure#269): tracks the +# `remove_unnecessary_deps` branch (zingolabs/infrastructure#276), which +# empties zcash_local_net's dependency tables of the zebra and librustzcash +# stacks; the lock pins the resolved commit until `cargo update`. Restore the +# release tag once tagged upstream. +zcash_local_net = { git = "https://github.com/zingolabs/infrastructure.git", branch = "remove_unnecessary_deps" } wire_serialized_transaction_test_data = "1.0.0" config = { version = "0.15", default-features = false, features = ["toml"] } From ddbdda950b9fead36d3f0236544df8e038c2350c Mon Sep 17 00:00:00 2001 From: zancas Date: Thu, 2 Jul 2026 21:13:33 -0700 Subject: [PATCH 018/134] build: re-lock zcash_local_net to remove_unnecessary_deps head 232cf9fb Pure lock bump following new pushes to zingolabs/infrastructure#276: getset replaced by hand-written accessors (dropping getset from zcash_local_net's dependency list), the zebra-rpc oracle dev-dependency deleted, and an external-types allowlist fix. No zaino API impact. Verified with cargo check --tests on zaino-testutils/e2e/clientless under both default features and zcashd_support. Co-Authored-By: Claude Fable 5 --- Cargo.lock | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index eca18c776..cd489ac56 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6242,9 +6242,8 @@ dependencies = [ [[package]] name = "zcash_local_net" version = "0.6.0" -source = "git+https://github.com/zingolabs/infrastructure.git?branch=remove_unnecessary_deps#029e93ebf50fa00b883fe6bce00e1c0ae049bb2a" +source = "git+https://github.com/zingolabs/infrastructure.git?branch=remove_unnecessary_deps#232cf9fb02f3de2c1c6bd0247c875e85296ea566" dependencies = [ - "getset", "hex", "json", "reqwest 0.12.28", @@ -6785,12 +6784,12 @@ dependencies = [ [[package]] name = "zingo-consensus" version = "0.1.0" -source = "git+https://github.com/zingolabs/infrastructure.git?branch=remove_unnecessary_deps#029e93ebf50fa00b883fe6bce00e1c0ae049bb2a" +source = "git+https://github.com/zingolabs/infrastructure.git?branch=remove_unnecessary_deps#232cf9fb02f3de2c1c6bd0247c875e85296ea566" [[package]] name = "zingo_test_vectors" version = "0.0.1" -source = "git+https://github.com/zingolabs/infrastructure.git?branch=remove_unnecessary_deps#029e93ebf50fa00b883fe6bce00e1c0ae049bb2a" +source = "git+https://github.com/zingolabs/infrastructure.git?branch=remove_unnecessary_deps#232cf9fb02f3de2c1c6bd0247c875e85296ea566" dependencies = [ "bip0039", ] From 3610da012390f3e42176dadeb8d36ddecd3d0522 Mon Sep 17 00:00:00 2001 From: nachog00 Date: Fri, 3 Jul 2026 01:20:11 -0300 Subject: [PATCH 019/134] Add publish dry-run CI check Runs `cargo publish --dry-run` for all publishable crates in dependency order. Advisory (warn-only) on regular PRs; hard-fails on rc/* and stable branches to catch unpublishable patch overrides before release. --- .github/workflows/publish-dry-run.yml | 80 +++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 .github/workflows/publish-dry-run.yml diff --git a/.github/workflows/publish-dry-run.yml b/.github/workflows/publish-dry-run.yml new file mode 100644 index 000000000..195f94f2f --- /dev/null +++ b/.github/workflows/publish-dry-run.yml @@ -0,0 +1,80 @@ +name: Publish dry-run + +on: + workflow_call: + workflow_dispatch: + push: + branches: + - 'rc/**' + - stable + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + +jobs: + publish-dry-run: + name: cargo publish --dry-run + runs-on: ubuntu-latest + # On rc/* and stable this job MUST pass; everywhere else it is advisory. + # For PRs, check the base (target) branch; for pushes, check the ref itself. + continue-on-error: >- + ${{ + github.event_name == 'pull_request' + && !startsWith(github.base_ref, 'rc/') + && github.base_ref != 'stable' + || github.event_name != 'pull_request' + && !startsWith(github.ref_name, 'rc/') + && github.ref_name != 'stable' + }} + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + # rust-toolchain.toml auto-installs the pinned toolchain on first cargo + # invocation, so no explicit toolchain setup step is needed. + + - name: Dry-run publish (dependency order) + id: dry-run + run: | + set -euo pipefail + + # Publishable crates in dependency order. + CRATES=( + zaino-proto + zaino-common + zaino-fetch + zaino-state + zaino-serve + zainod + ) + + failed=() + for crate in "${CRATES[@]}"; do + echo "::group::$crate" + if cargo publish -p "$crate" --dry-run 2>&1; then + echo " $crate: OK" + else + failed+=("$crate") + echo "::error::cargo publish --dry-run failed for $crate" + fi + echo "::endgroup::" + done + + if [[ ${#failed[@]} -gt 0 ]]; then + echo "" + echo "============================================" + echo "Publish dry-run FAILED for: ${failed[*]}" + echo "============================================" + echo "" + echo "This usually means a [patch.crates-io] override is pulling in" + echo "unreleased upstream code. Published crates cannot depend on" + echo "unpublished revisions — resolve the patch before releasing." + exit 1 + fi + + - name: Annotate on failure (advisory contexts only) + if: >- + failure() + && (github.event_name != 'push' || (!startsWith(github.ref_name, 'rc/') && github.ref_name != 'stable')) + && (github.event_name != 'pull_request' || (!startsWith(github.base_ref, 'rc/') && github.base_ref != 'stable')) + run: | + echo "::warning::cargo publish --dry-run failed — one or more crates depend on patched (unreleased) upstream code. This must be resolved before the next release." From 2a4df3fc1a5efd2d9d4136b7d59c63496c56ac93 Mon Sep 17 00:00:00 2001 From: idky137 Date: Fri, 3 Jul 2026 08:44:54 +0100 Subject: [PATCH 020/134] move methods into chain_index / blockchainsource pt 2 - get_block_deltas --- packages/zaino-state/src/backends/fetch.rs | 2 +- packages/zaino-state/src/backends/state.rs | 283 +----------------- packages/zaino-state/src/chain_index.rs | 21 ++ .../zaino-state/src/chain_index/source.rs | 7 + .../chain_index/source/mockchain_source.rs | 87 +++++- .../chain_index/source/validator_connector.rs | 272 ++++++++++++++++- .../src/chain_index/tests/mockchain_tests.rs | 36 +++ .../chain_index/tests/proptest_blockgen.rs | 8 + 8 files changed, 431 insertions(+), 285 deletions(-) diff --git a/packages/zaino-state/src/backends/fetch.rs b/packages/zaino-state/src/backends/fetch.rs index 98f1b7b67..876d8afd1 100644 --- a/packages/zaino-state/src/backends/fetch.rs +++ b/packages/zaino-state/src/backends/fetch.rs @@ -437,7 +437,7 @@ impl ZcashIndexer for FetchServiceSubscriber { /// /// Note: This method has only been implemented in `zcashd`. Zebra has no intention of supporting it. async fn get_block_deltas(&self, hash: String) -> Result { - Ok(self.fetcher.get_block_deltas(hash).await?) + Ok(self.indexer.get_block_deltas(hash).await?) } async fn get_block_header( diff --git a/packages/zaino-state/src/backends/state.rs b/packages/zaino-state/src/backends/state.rs index 59b6e748e..babbd7f37 100644 --- a/packages/zaino-state/src/backends/state.rs +++ b/packages/zaino-state/src/backends/state.rs @@ -33,7 +33,7 @@ use zaino_fetch::{ raw_transaction::validate_raw_transaction_hex, response::{ address_deltas::{GetAddressDeltasParams, GetAddressDeltasResponse}, - block_deltas::{BlockDelta, BlockDeltas, InputDelta, OutputDelta}, + block_deltas::BlockDeltas, block_header::GetBlockHeader, block_subsidy::GetBlockSubsidy, chain_tips::GetChainTipsResponse, @@ -65,7 +65,6 @@ use zcash_keys::{address::Address, encoding::AddressCodec}; use zcash_protocol::consensus::NetworkType; use zcash_transparent::address::TransparentAddress; use zebra_chain::{ - amount::{Amount, NonNegative}, block::Height, chain_tip::NetworkChainTipHeightEstimator, parameters::{ConsensusBranchId, NetworkKind, NetworkUpgrade}, @@ -74,13 +73,13 @@ use zebra_chain::{ }; use zebra_rpc::{ client::{ - GetAddressBalanceRequest, GetSubtreesByIndexResponse, GetTreestateResponse, Input, - SubtreeRpcData, TransactionObject, ValidateAddressResponse, + GetAddressBalanceRequest, GetSubtreesByIndexResponse, GetTreestateResponse, SubtreeRpcData, + TransactionObject, ValidateAddressResponse, }, methods::{ chain_tip_difficulty, AddressBalance, ConsensusBranchIdHex, GetAddressTxIdsRequest, - GetAddressUtxos, GetBlock, GetBlockHash, GetBlockTransaction, GetBlockchainInfoResponse, - GetInfo, GetRawTransaction, NetworkUpgradeInfo, NetworkUpgradeStatus, SentTransactionHash, + GetAddressUtxos, GetBlock, GetBlockHash, GetBlockchainInfoResponse, GetInfo, + GetRawTransaction, NetworkUpgradeInfo, NetworkUpgradeStatus, SentTransactionHash, TipConsensusBranch, }, server::error::LegacyCode, @@ -92,7 +91,7 @@ use chrono::Utc; use futures::TryFutureExt as _; use hex::{FromHex as _, ToHex}; use indexmap::IndexMap; -use std::{collections::HashMap, error::Error, fmt, str::FromStr, sync::Arc}; +use std::{str::FromStr, sync::Arc}; use tokio::{ sync::mpsc, time::{self, timeout}, @@ -564,56 +563,6 @@ impl StateServiceSubscriber { pub fn network(&self) -> zaino_common::Network { self.config.common.network } - - /// Returns the median time of the last 11 blocks. - async fn median_time_past( - &self, - start: &zebra_rpc::client::BlockObject, - ) -> Result { - const MEDIAN_TIME_PAST_WINDOW: usize = 11; - - let mut times = Vec::with_capacity(MEDIAN_TIME_PAST_WINDOW); - - let start_hash = start.hash().to_string(); - let time_0 = start - .time() - .ok_or_else(|| MedianTimePast::StartMissingTime { - hash: start_hash.clone(), - })?; - times.push(time_0); - - let mut prev = start.previous_block_hash(); - - for _ in 0..(MEDIAN_TIME_PAST_WINDOW - 1) { - let hash = match prev { - Some(h) => h.to_string(), - None => break, // genesis - }; - - match self.z_get_block(hash.clone(), Some(1)).await { - Ok(GetBlock::Object(obj)) => { - if let Some(t) = obj.time() { - times.push(t); - } - prev = obj.previous_block_hash(); - } - Ok(GetBlock::Raw(_)) => { - return Err(MedianTimePast::UnexpectedRaw { hash }); - } - Err(_e) => { - // Use values up to this point - break; - } - } - } - - if times.is_empty() { - return Err(MedianTimePast::EmptyWindow); - } - - times.sort_unstable(); - Ok(times[times.len() / 2]) - } } /// Extracts the diversifier and pk_d bytes from a validated Sapling @@ -908,194 +857,7 @@ impl ZcashIndexer for StateServiceSubscriber { } async fn get_block_deltas(&self, hash: String) -> Result { - // Get the block WITH the transaction data - let zblock = self.z_get_block(hash, Some(2)).await?; - - match zblock { - GetBlock::Object(boxed_block) => { - // Resolve each transparent spend's prevout ourselves: the - // verbosity-2 object from Zebra's stateless - // `TransactionObject::from_transaction` leaves input - // value/address `None`, so we look the spent output up via the - // best-chain `ReadRequest::Transaction` (finalized + - // non-finalized, so same-block prevouts resolve too). - let network = self.data.network(); - let mut state = self.read_state_service.clone(); - // Per-call cache: many inputs may reference the same prevtxid, - // so each previous transaction is fetched at most once. - let mut prevtx_cache: HashMap< - zebra_chain::transaction::Hash, - Arc, - > = HashMap::new(); - - let mut deltas = Vec::with_capacity(boxed_block.tx().len()); - for (tx_index, tx) in boxed_block.tx().iter().enumerate() { - let txo = match tx { - GetBlockTransaction::Object(txo) => txo, - GetBlockTransaction::Hash(_) => { - return Err(StateServiceError::Custom( - "Unexpected hash when expecting object".to_string(), - )) - } - }; - - let txid = txo.txid().to_string(); - - let mut inputs: Vec = Vec::new(); - for (i, vin) in txo.inputs().iter().enumerate() { - let (prevtxid, prevout) = match vin { - Input::Coinbase { .. } => continue, - Input::NonCoinbase { - txid: prevtxid, - vout: prevout, - .. - } => (prevtxid, *prevout), - }; - - let prev_hash = zebra_chain::transaction::Hash::from_str(prevtxid) - .map_err(|e| { - StateServiceError::Custom(format!( - "getblockdeltas: invalid prevout txid {prevtxid}: {e}" - )) - })?; - - let prev_tx = match prevtx_cache.get(&prev_hash) { - Some(prev_tx) => prev_tx.clone(), - None => { - let response = state - .ready() - .and_then(|service| { - service.call(ReadRequest::Transaction(prev_hash)) - }) - .await?; - let mined_tx = expected_read_response!(response, Transaction) - .ok_or_else(|| { - StateServiceError::Custom(format!( - "getblockdeltas: prevout tx {prevtxid} not in best chain" - )) - })?; - prevtx_cache.insert(prev_hash, mined_tx.tx.clone()); - mined_tx.tx - } - }; - - let output = prev_tx.outputs().get(prevout as usize).ok_or_else(|| { - StateServiceError::Custom(format!( - "getblockdeltas: prevout index {prevout} out of range for {prevtxid}" - )) - })?; - - // Nonstandard script ⇒ no derivable address ⇒ skip - // (matches the outputs branch). This is the only - // legitimate skip; it is not loss of a resolvable spend. - let address = match output.address(&network) { - Some(a) => a.to_string(), - None => continue, - }; - - // Inputs are debits, so the amount leaves the address. - let satoshis: Amount = - (-output.value().zatoshis()).try_into().map_err(|e| { - StateServiceError::Custom(format!( - "getblockdeltas: input amount out of range for {prevtxid}:{prevout}: {e}" - )) - })?; - - inputs.push(InputDelta { - address, - satoshis, - index: i as u32, - prevtxid: prevtxid.clone(), - prevout, - }); - } - - let outputs: Vec = txo - .outputs() - .iter() - .filter_map(|vout| { - let addr_opt = vout - .script_pub_key() - .addresses() - .as_ref() - .and_then(|v| if v.len() == 1 { v.first() } else { None }); - - let addr = addr_opt?.clone(); - - let output_amt: Amount = match vout.value_zat().try_into() - { - Ok(a) => a, - Err(_) => return None, - }; - - Some(OutputDelta { - address: addr, - satoshis: output_amt, - index: vout.n(), - }) - }) - .collect::>(); - - deltas.push(BlockDelta { - txid, - index: tx_index as u32, - inputs, - outputs, - }); - } - - Ok(BlockDeltas { - hash: boxed_block.hash().to_string(), - confirmations: boxed_block.confirmations(), - size: boxed_block.size().ok_or_else(|| { - StateServiceError::Custom("getblockdeltas: block size missing".into()) - })?, - height: boxed_block - .height() - .ok_or_else(|| { - StateServiceError::Custom("getblockdeltas: block height missing".into()) - })? - .0, - version: boxed_block.version().ok_or_else(|| { - StateServiceError::Custom("getblockdeltas: block version missing".into()) - })?, - merkle_root: boxed_block - .merkle_root() - .ok_or_else(|| { - StateServiceError::Custom( - "getblockdeltas: block merkle root missing".into(), - ) - })? - .encode_hex::(), - deltas, - time: boxed_block.time().ok_or_else(|| { - StateServiceError::Custom("getblockdeltas: block time missing".into()) - })?, - median_time: self.median_time_past(&boxed_block).await.map_err(|e| { - StateServiceError::Custom(format!("getblockdeltas: median_time_past: {e}")) - })?, - nonce: hex::encode(boxed_block.nonce().ok_or_else(|| { - StateServiceError::Custom("getblockdeltas: block nonce missing".into()) - })?), - bits: boxed_block - .bits() - .ok_or_else(|| { - StateServiceError::Custom("getblockdeltas: block bits missing".into()) - })? - .to_string(), - difficulty: boxed_block.difficulty().ok_or_else(|| { - StateServiceError::Custom("getblockdeltas: block difficulty missing".into()) - })?, - previous_block_hash: boxed_block - .previous_block_hash() - .map(|hash| hash.to_string()), - next_block_hash: boxed_block.next_block_hash().map(|h| h.to_string()), - }) - } - GetBlock::Raw(_serialized_block) => Err(StateServiceError::Custom( - "Unexpected raw block".to_string(), - )), - } + Ok(self.indexer.get_block_deltas(hash).await?) } async fn get_raw_mempool(&self) -> Result, Self::Error> { @@ -2386,37 +2148,6 @@ impl LightWalletIndexer for StateServiceSubscriber { } } -/// An error type for median time past calculation errors -#[derive(Debug, Clone)] -pub enum MedianTimePast { - /// The start block has no `time`. - StartMissingTime { hash: String }, - - /// Ignored verbosity. - UnexpectedRaw { hash: String }, - - /// No timestamps collected at all. - EmptyWindow, -} - -impl fmt::Display for MedianTimePast { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - MedianTimePast::StartMissingTime { hash } => { - write!(f, "start block {hash} is missing `time`") - } - MedianTimePast::UnexpectedRaw { hash } => { - write!(f, "unexpected raw payload for block {hash}") - } - MedianTimePast::EmptyWindow => { - write!(f, "no timestamps collected (empty MTP window)") - } - } - } -} - -impl Error for MedianTimePast {} - #[cfg(test)] mod tests { /// Classifies the byte-level relationship between two slices. diff --git a/packages/zaino-state/src/chain_index.rs b/packages/zaino-state/src/chain_index.rs index 66ce9dcd9..139e42636 100644 --- a/packages/zaino-state/src/chain_index.rs +++ b/packages/zaino-state/src/chain_index.rs @@ -39,6 +39,7 @@ use tokio_util::sync::CancellationToken; use tracing::{info, instrument}; use zaino_fetch::jsonrpsee::response::{ address_deltas::{GetAddressDeltasParams, GetAddressDeltasResponse}, + block_deltas::BlockDeltas, block_header::GetBlockHeader, chain_tips::{ChainTip, ChainTipStatus, GetChainTipsResponse}, EmptyTxOutSetInfo, GetTxOutSetInfo, GetTxOutSetInfoResponse, @@ -596,6 +597,15 @@ pub trait ChainIndexRpcExt: ChainIndex { verbose: bool, ) -> impl std::future::Future>; + /// Returns the `getblockdeltas`-shaped transparent input/output deltas for the block + /// with the given hash. + /// + /// zcashd reference: [`getblockdeltas`](https://zcash.github.io/rpc/getblockdeltas.html) + fn get_block_deltas( + &self, + hash: String, + ) -> impl std::future::Future>; + // ********** Transparent address history methods ********** /// Returns all changes for the given transparent addresses. @@ -2642,6 +2652,17 @@ impl ChainIndexRpcExt for NodeBackedChainIndexSubscrib .map_err(ChainIndexError::backing_validator) } + // TODO(internal-first): `getblockdeltas` is buildable from the indexed chainblocks + // + finalised/non-finalised prevout resolution. Build it internally by default once + // an internal prevout resolver (spanning non-finalised + finalised, reconstructing + // addresses from `TxOutCompact`) exists, keeping this source call as the fallback. + async fn get_block_deltas(&self, hash: String) -> Result { + self.source() + .get_block_deltas(hash) + .await + .map_err(ChainIndexError::backing_validator) + } + /// Returns all changes for the given transparent addresses. async fn get_address_deltas( &self, diff --git a/packages/zaino-state/src/chain_index/source.rs b/packages/zaino-state/src/chain_index/source.rs index b2352e3a7..9315f6548 100644 --- a/packages/zaino-state/src/chain_index/source.rs +++ b/packages/zaino-state/src/chain_index/source.rs @@ -82,6 +82,13 @@ pub trait BlockchainSource: Clone + Send + Sync + 'static { verbose: bool, ) -> impl SendFut>; + /// Returns the `getblockdeltas`-shaped transparent input/output deltas for the block + /// with the given hash. + fn get_block_deltas( + &self, + hash: String, + ) -> impl SendFut>; + // ********** Transaction methods ********** /// Returns the transaction by txid diff --git a/packages/zaino-state/src/chain_index/source/mockchain_source.rs b/packages/zaino-state/src/chain_index/source/mockchain_source.rs index dde2e2552..f7842ceb8 100644 --- a/packages/zaino-state/src/chain_index/source/mockchain_source.rs +++ b/packages/zaino-state/src/chain_index/source/mockchain_source.rs @@ -1,8 +1,9 @@ //! Mock BlockchainSourceResult implementation. use super::validator_connector::{ - build_block_header_object, build_verbose_block, confirmations_from_depth, final_orchard_root, - final_sapling_root, zebra_block_header_to_wire, + assemble_block_deltas, build_block_header_object, build_verbose_block, + confirmations_from_depth, final_orchard_root, final_sapling_root, median_of_block_times, + zebra_block_header_to_wire, }; use super::*; use std::collections::{HashMap, HashSet}; @@ -12,7 +13,9 @@ use std::sync::{ Arc, Mutex, }; use zaino_common::network::ActivationHeights; -use zaino_fetch::jsonrpsee::response::{address_deltas::BlockInfo, block_header::GetBlockHeader}; +use zaino_fetch::jsonrpsee::response::{ + address_deltas::BlockInfo, block_deltas::BlockDeltas, block_header::GetBlockHeader, +}; use zebra_chain::{block::Block, orchard::tree as orchard, sapling::tree as sapling}; use zebra_chain::{ block::{Height, SerializedBlock}, @@ -21,8 +24,8 @@ use zebra_chain::{ transparent::{Address, OutPoint, Output, OutputIndex}, }; use zebra_rpc::{ - client::{HexData, TransactionObject}, - methods::{GetBlock, GetBlockHeaderResponse, ValidateAddresses as _}, + client::{BlockObject, HexData, Input, TransactionObject}, + methods::{GetBlock, GetBlockHeaderResponse, GetBlockTransaction, ValidateAddresses as _}, }; use zebra_state::HashOrHeight; @@ -399,6 +402,39 @@ impl MockchainSource { Ok(GetBlockHeaderResponse::Object(Box::new(header_obj))) } + /// Median time past over the 11-block window ending at `start`, walking backwards via + /// verbosity-1 `getblock` lookups against the mock vectors. + async fn median_time_past(&self, start: &BlockObject) -> BlockchainSourceResult { + const MEDIAN_TIME_PAST_WINDOW: usize = 11; + let mut times = Vec::with_capacity(MEDIAN_TIME_PAST_WINDOW); + let start_time = start.time().ok_or_else(|| { + BlockchainSourceError::Unrecoverable("getblockdeltas: start block missing time".into()) + })?; + times.push(start_time); + + let mut prev = start.previous_block_hash(); + for _ in 0..(MEDIAN_TIME_PAST_WINDOW - 1) { + let Some(hash) = prev else { + break; // genesis + }; + match self + .get_block_verbose(HashOrHeight::Hash(hash), Some(1)) + .await + { + Ok(GetBlock::Object(object)) => { + if let Some(time) = object.time() { + times.push(time); + } + prev = object.previous_block_hash(); + } + Ok(GetBlock::Raw(_)) => break, + Err(_) => break, + } + } + + median_of_block_times(times) + } + fn block_height_at_index(&self, block_index: usize) -> Height { self.blocks[block_index] .coinbase_height() @@ -593,6 +629,47 @@ impl BlockchainSource for MockchainSource { zebra_block_header_to_wire(header) } + async fn get_block_deltas(&self, hash: String) -> BlockchainSourceResult { + let hash_or_height = HashOrHeight::from_str(&hash) + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + let GetBlock::Object(object) = self.get_block_verbose(hash_or_height, Some(2)).await? + else { + return Err(BlockchainSourceError::Unrecoverable( + "getblockdeltas: unexpected raw block".to_string(), + )); + }; + + // The mock holds every transaction, so prevout resolution is a direct index lookup. + let mut prevtx_cache: HashMap< + zebra_chain::transaction::Hash, + Arc, + > = HashMap::new(); + for tx in object.tx() { + let GetBlockTransaction::Object(txo) = tx else { + continue; + }; + for input in txo.inputs() { + let Input::NonCoinbase { txid: prevtxid, .. } = input else { + continue; + }; + let prev_hash = zebra_chain::transaction::Hash::from_str(prevtxid) + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + if prevtx_cache.contains_key(&prev_hash) { + continue; + } + let (_height, prev_tx) = self.txid_index.get(&prev_hash).ok_or_else(|| { + BlockchainSourceError::Unrecoverable(format!( + "getblockdeltas: prevout tx {prevtxid} not found in mock chain" + )) + })?; + prevtx_cache.insert(prev_hash, Arc::clone(prev_tx)); + } + } + + let median_time = self.median_time_past(&object).await?; + assemble_block_deltas(&object, &prevtx_cache, median_time, &mockchain_network()) + } + // ********** Transaction methods ********** async fn get_transaction( diff --git a/packages/zaino-state/src/chain_index/source/validator_connector.rs b/packages/zaino-state/src/chain_index/source/validator_connector.rs index c9095ebd8..a720b9496 100644 --- a/packages/zaino-state/src/chain_index/source/validator_connector.rs +++ b/packages/zaino-state/src/chain_index/source/validator_connector.rs @@ -1,17 +1,23 @@ //! validator connected blockchain source. +use std::collections::HashMap; use std::str::FromStr as _; use chrono::{DateTime, Utc}; -use hex::FromHex as _; -use zaino_fetch::jsonrpsee::response::{address_deltas::BlockInfo, block_header::GetBlockHeader}; +use hex::{FromHex as _, ToHex as _}; +use zaino_fetch::jsonrpsee::response::{ + address_deltas::BlockInfo, + block_deltas::{BlockDelta, BlockDeltas, InputDelta, OutputDelta}, + block_header::GetBlockHeader, +}; use zebra_chain::{ + amount::{Amount, NonNegative}, block::{Header, SerializedBlock}, parameters::NetworkUpgrade, serialization::{BytesInDisplayOrder as _, ZcashSerialize as _}, }; use zebra_rpc::{ - client::{BlockObject, GetBlockchainInfoBalance, HexData, TransactionObject}, + client::{BlockObject, GetBlockchainInfoBalance, HexData, Input, TransactionObject}, methods::{ GetBlock, GetBlockHeaderObject, GetBlockHeaderResponse, GetBlockTransaction, GetBlockTrees, ValidateAddresses as _, @@ -166,6 +172,16 @@ impl BlockchainSource for ValidatorConnector { } } + async fn get_block_deltas(&self, hash: String) -> BlockchainSourceResult { + match self { + ValidatorConnector::State(state) => state.get_block_deltas(hash).await, + ValidatorConnector::Fetch(fetch) => fetch + .get_block_deltas(hash) + .await + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string())), + } + } + // ********** Transaction methods ********** // Returns the transaction, and the height of the block that transaction is in if on the best chain @@ -1356,6 +1372,94 @@ impl State { ))), } } + + /// Builds the `getblockdeltas` response from the `ReadStateService`. + /// + /// Moved from the former `StateServiceSubscriber::get_block_deltas`; resolves each + /// spent prevout via a best-chain `ReadRequest::Transaction` (finalised + + /// non-finalised), then hands off to the shared [`assemble_block_deltas`]. + async fn get_block_deltas(&self, hash: String) -> BlockchainSourceResult { + let hash_or_height = HashOrHeight::from_str(&hash) + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + let GetBlock::Object(object) = self.get_block_inner(hash_or_height, Some(2)).await? else { + return Err(BlockchainSourceError::Unrecoverable( + "getblockdeltas: unexpected raw block".to_string(), + )); + }; + + // Per-call cache: many inputs may reference the same prevtxid, so each previous + // transaction is fetched at most once. + let mut prevtx_cache: HashMap< + zebra_chain::transaction::Hash, + Arc, + > = HashMap::new(); + for tx in object.tx() { + let GetBlockTransaction::Object(txo) = tx else { + continue; + }; + for input in txo.inputs() { + let Input::NonCoinbase { txid: prevtxid, .. } = input else { + continue; + }; + let prev_hash = zebra_chain::transaction::Hash::from_str(prevtxid) + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + if prevtx_cache.contains_key(&prev_hash) { + continue; + } + let mut state = self.read_state_service.clone(); + let response = state + .ready() + .and_then(|service| service.call(ReadRequest::Transaction(prev_hash))) + .await + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + let mined_tx = expected_read_response!(response, Transaction).ok_or_else(|| { + BlockchainSourceError::Unrecoverable(format!( + "getblockdeltas: prevout tx {prevtxid} not in best chain" + )) + })?; + prevtx_cache.insert(prev_hash, mined_tx.tx); + } + } + + let median_time = self.median_time_past(&object).await?; + let network = self.network.to_zebra_network(); + assemble_block_deltas(&object, &prevtx_cache, median_time, &network) + } + + /// Median time past over the 11-block window ending at `start`, walking backwards via + /// verbosity-1 `getblock` lookups against the `ReadStateService`. + // TODO(DRY): MockchainSource duplicates this walk; the only difference is the + // per-block fetch. A shared helper generic over an async block fetcher would unify them. + async fn median_time_past(&self, start: &BlockObject) -> BlockchainSourceResult { + const MEDIAN_TIME_PAST_WINDOW: usize = 11; + let mut times = Vec::with_capacity(MEDIAN_TIME_PAST_WINDOW); + let start_time = start.time().ok_or_else(|| { + BlockchainSourceError::Unrecoverable("getblockdeltas: start block missing time".into()) + })?; + times.push(start_time); + + let mut prev = start.previous_block_hash(); + for _ in 0..(MEDIAN_TIME_PAST_WINDOW - 1) { + let Some(hash) = prev else { + break; // genesis + }; + match self + .get_block_inner(HashOrHeight::Hash(hash), Some(1)) + .await + { + Ok(GetBlock::Object(object)) => { + if let Some(time) = object.time() { + times.push(time); + } + prev = object.previous_block_hash(); + } + Ok(GetBlock::Raw(_)) => break, + Err(_) => break, // use values collected so far + } + } + + median_of_block_times(times) + } } /// Confirmations are one more than the depth, or -1 when the block is not on the best @@ -1541,3 +1645,165 @@ pub(crate) fn zebra_block_header_to_wire( serde_json::from_value(value) .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string())) } + +/// Returns the median of a non-empty set of block times. +pub(crate) fn median_of_block_times(mut times: Vec) -> BlockchainSourceResult { + if times.is_empty() { + return Err(BlockchainSourceError::Unrecoverable( + "getblockdeltas: no block times collected for median".to_string(), + )); + } + times.sort_unstable(); + Ok(times[times.len() / 2]) +} + +/// Assembles a `getblockdeltas` response from a verbosity-2 `getblock` object plus a +/// cache of previous transactions (keyed by txid) needed to resolve each spend's address +/// and value, its median time, and the running network. +/// +/// Shared by the [`ValidatorConnector`] State path and `MockchainSource`: both resolve +/// the prevout transactions their own way, then hand the assembled inputs here so the +/// delta-shaping logic lives in one place. +pub(crate) fn assemble_block_deltas( + object: &BlockObject, + prevtx_cache: &HashMap< + zebra_chain::transaction::Hash, + Arc, + >, + median_time: i64, + network: &zebra_chain::parameters::Network, +) -> BlockchainSourceResult { + let mut deltas = Vec::with_capacity(object.tx().len()); + for (tx_index, tx) in object.tx().iter().enumerate() { + let GetBlockTransaction::Object(txo) = tx else { + return Err(BlockchainSourceError::Unrecoverable( + "getblockdeltas: unexpected hash when expecting object".to_string(), + )); + }; + let txid = txo.txid().to_string(); + + let mut inputs: Vec = Vec::new(); + for (i, vin) in txo.inputs().iter().enumerate() { + let (prevtxid, prevout) = match vin { + Input::Coinbase { .. } => continue, + Input::NonCoinbase { + txid: prevtxid, + vout: prevout, + .. + } => (prevtxid, *prevout), + }; + + let prev_hash = zebra_chain::transaction::Hash::from_str(prevtxid) + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + let prev_tx = prevtx_cache.get(&prev_hash).ok_or_else(|| { + BlockchainSourceError::Unrecoverable(format!( + "getblockdeltas: prevout tx {prevtxid} not resolved" + )) + })?; + + let output = prev_tx.outputs().get(prevout as usize).ok_or_else(|| { + BlockchainSourceError::Unrecoverable(format!( + "getblockdeltas: prevout index {prevout} out of range for {prevtxid}" + )) + })?; + + // Nonstandard script ⇒ no derivable address ⇒ skip (matches the outputs branch). + let address = match output.address(network) { + Some(address) => address.to_string(), + None => continue, + }; + + // Inputs are debits, so the amount leaves the address. + let satoshis: Amount = (-output.value().zatoshis()).try_into().map_err(|error| { + BlockchainSourceError::Unrecoverable(format!( + "getblockdeltas: input amount out of range for {prevtxid}:{prevout}: {error}" + )) + })?; + + inputs.push(InputDelta { + address, + satoshis, + index: i as u32, + prevtxid: prevtxid.clone(), + prevout, + }); + } + + let outputs: Vec = txo + .outputs() + .iter() + .filter_map(|vout| { + let address = vout + .script_pub_key() + .addresses() + .as_ref() + .and_then(|addresses| addresses.first().cloned())?; + let satoshis: Amount = vout.value_zat().try_into().ok()?; + Some(OutputDelta { + address, + satoshis, + index: vout.n(), + }) + }) + .collect(); + + deltas.push(BlockDelta { + txid, + index: tx_index as u32, + inputs, + outputs, + }); + } + + Ok(BlockDeltas { + hash: object.hash().to_string(), + confirmations: object.confirmations(), + size: object.size().ok_or_else(|| { + BlockchainSourceError::Unrecoverable("getblockdeltas: block size missing".to_string()) + })?, + height: object + .height() + .ok_or_else(|| { + BlockchainSourceError::Unrecoverable( + "getblockdeltas: block height missing".to_string(), + ) + })? + .0, + version: object.version().ok_or_else(|| { + BlockchainSourceError::Unrecoverable( + "getblockdeltas: block version missing".to_string(), + ) + })?, + merkle_root: object + .merkle_root() + .ok_or_else(|| { + BlockchainSourceError::Unrecoverable( + "getblockdeltas: block merkle root missing".to_string(), + ) + })? + .encode_hex::(), + deltas, + time: object.time().ok_or_else(|| { + BlockchainSourceError::Unrecoverable("getblockdeltas: block time missing".to_string()) + })?, + median_time, + nonce: hex::encode(object.nonce().ok_or_else(|| { + BlockchainSourceError::Unrecoverable("getblockdeltas: block nonce missing".to_string()) + })?), + bits: object + .bits() + .ok_or_else(|| { + BlockchainSourceError::Unrecoverable( + "getblockdeltas: block bits missing".to_string(), + ) + })? + .to_string(), + difficulty: object.difficulty().ok_or_else(|| { + BlockchainSourceError::Unrecoverable( + "getblockdeltas: block difficulty missing".to_string(), + ) + })?, + previous_block_hash: object.previous_block_hash().map(|hash| hash.to_string()), + next_block_hash: object.next_block_hash().map(|hash| hash.to_string()), + }) +} diff --git a/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs b/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs index 8fef9fa05..2ab98a4e4 100644 --- a/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs +++ b/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs @@ -996,3 +996,39 @@ async fn get_block_header() { assert_eq!(value["height"].as_u64().unwrap(), u64::from(height)); } } + +/// `get_block_deltas` served through the ChainIndex from the mock vectors: it matches the +/// source, reports the right block hash / height, and surfaces transparent deltas. +#[tokio::test(flavor = "multi_thread")] +async fn get_block_deltas() { + let (_blocks, _indexer, index_reader, mockchain) = + load_test_vectors_and_sync_chain_index(MockchainMode::Static).await; + let active_height = mockchain.active_height(); + + let mut saw_delta_entries = false; + for height in [1u32, active_height / 2, active_height] { + let id = HashOrHeight::Height(zebra_chain::block::Height(height)); + let block = mockchain.get_block(id).await.unwrap().unwrap(); + let hash = block.hash().to_string(); + + let via_index = index_reader.get_block_deltas(hash.clone()).await.unwrap(); + let via_source = mockchain.get_block_deltas(hash.clone()).await.unwrap(); + assert_eq!( + serde_json::to_value(&via_index).unwrap(), + serde_json::to_value(&via_source).unwrap() + ); + assert_eq!(via_index.hash, hash); + assert_eq!(via_index.height, height); + if via_index + .deltas + .iter() + .any(|delta| !delta.inputs.is_empty() || !delta.outputs.is_empty()) + { + saw_delta_entries = true; + } + } + assert!( + saw_delta_entries, + "expected transparent deltas in at least one sampled block" + ); +} diff --git a/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs b/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs index c3a767a19..bd1450d4e 100644 --- a/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs +++ b/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs @@ -635,6 +635,14 @@ impl BlockchainSource for ProptestMockchain { unimplemented!() } + async fn get_block_deltas( + &self, + _hash: String, + ) -> BlockchainSourceResult { + // ProptestMockchain exercises sync/reorg, not the getblockdeltas RPC. + unimplemented!() + } + /// Returns the block commitment tree data by hash async fn get_commitment_tree_roots( &self, From 4b2c2315433005a4bda905f8c9f7f229ca76e2a9 Mon Sep 17 00:00:00 2001 From: idky137 Date: Fri, 3 Jul 2026 08:55:44 +0100 Subject: [PATCH 021/134] move methods into chain_index / blockchainsource - get_difficulty --- packages/zaino-state/src/backends/fetch.rs | 2 +- packages/zaino-state/src/backends/state.rs | 13 +---------- packages/zaino-state/src/chain_index.rs | 16 +++++++++++++ .../zaino-state/src/chain_index/source.rs | 4 ++++ .../chain_index/source/mockchain_source.rs | 13 +++++++++++ .../chain_index/source/validator_connector.rs | 23 +++++++++++++++++-- .../src/chain_index/tests/mockchain_tests.rs | 16 +++++++++++++ .../chain_index/tests/proptest_blockgen.rs | 5 ++++ 8 files changed, 77 insertions(+), 15 deletions(-) diff --git a/packages/zaino-state/src/backends/fetch.rs b/packages/zaino-state/src/backends/fetch.rs index 876d8afd1..f08c4c6f0 100644 --- a/packages/zaino-state/src/backends/fetch.rs +++ b/packages/zaino-state/src/backends/fetch.rs @@ -334,7 +334,7 @@ impl ZcashIndexer for FetchServiceSubscriber { /// method: post /// tags: blockchain async fn get_difficulty(&self) -> Result { - Ok(self.fetcher.get_difficulty().await?.0) + Ok(self.indexer.get_difficulty().await?) } async fn get_block_subsidy(&self, height: u32) -> Result { diff --git a/packages/zaino-state/src/backends/state.rs b/packages/zaino-state/src/backends/state.rs index babbd7f37..2e879c341 100644 --- a/packages/zaino-state/src/backends/state.rs +++ b/packages/zaino-state/src/backends/state.rs @@ -633,18 +633,7 @@ impl ZcashIndexer for StateServiceSubscriber { } async fn get_difficulty(&self) -> Result { - chain_tip_difficulty( - self.config.common.network.to_zebra_network(), - self.read_state_service.clone(), - false, - ) - .await - .map_err(|e| { - StateServiceError::RpcError(RpcError::new_from_errorobject( - e, - "failed to get difficulty", - )) - }) + Ok(self.indexer.get_difficulty().await?) } async fn get_block_subsidy(&self, height: u32) -> Result { diff --git a/packages/zaino-state/src/chain_index.rs b/packages/zaino-state/src/chain_index.rs index 139e42636..9fe87a3c0 100644 --- a/packages/zaino-state/src/chain_index.rs +++ b/packages/zaino-state/src/chain_index.rs @@ -606,6 +606,12 @@ pub trait ChainIndexRpcExt: ChainIndex { hash: String, ) -> impl std::future::Future>; + /// Returns the proof-of-work difficulty of the best chain as a multiple of the + /// minimum difficulty. + /// + /// zcashd reference: [`getdifficulty`](https://zcash.github.io/rpc/getdifficulty.html) + fn get_difficulty(&self) -> impl std::future::Future>; + // ********** Transparent address history methods ********** /// Returns all changes for the given transparent addresses. @@ -2663,6 +2669,16 @@ impl ChainIndexRpcExt for NodeBackedChainIndexSubscrib .map_err(ChainIndexError::backing_validator) } + // `getdifficulty` is the difficulty-adjusted expected difficulty (over a block + // window), not the tip block's stored bits, so it cannot be built from indexed data: + // always delegate to the backing validator. + async fn get_difficulty(&self) -> Result { + self.source() + .get_difficulty() + .await + .map_err(ChainIndexError::backing_validator) + } + /// Returns all changes for the given transparent addresses. async fn get_address_deltas( &self, diff --git a/packages/zaino-state/src/chain_index/source.rs b/packages/zaino-state/src/chain_index/source.rs index 9315f6548..7776d9667 100644 --- a/packages/zaino-state/src/chain_index/source.rs +++ b/packages/zaino-state/src/chain_index/source.rs @@ -121,6 +121,10 @@ pub trait BlockchainSource: Clone + Send + Sync + 'static { &self, ) -> impl SendFut>>; + /// Returns the proof-of-work difficulty of the best chain as a multiple of the + /// minimum difficulty (the `getdifficulty` RPC value). + fn get_difficulty(&self) -> impl SendFut>; + /// Returns the sapling and orchard treestate by hash fn get_treestate(&self, id: BlockHash) -> impl SendFut>; diff --git a/packages/zaino-state/src/chain_index/source/mockchain_source.rs b/packages/zaino-state/src/chain_index/source/mockchain_source.rs index f7842ceb8..0f0ca4478 100644 --- a/packages/zaino-state/src/chain_index/source/mockchain_source.rs +++ b/packages/zaino-state/src/chain_index/source/mockchain_source.rs @@ -670,6 +670,19 @@ impl BlockchainSource for MockchainSource { assemble_block_deltas(&object, &prevtx_cache, median_time, &mockchain_network()) } + // ********** Chain methods ********** + + async fn get_difficulty(&self) -> BlockchainSourceResult { + let tip_index = self.active_chain_height_as_usize(); + let tip_block = self.blocks.get(tip_index).ok_or_else(|| { + BlockchainSourceError::Unrecoverable("mock chain has no tip block".to_string()) + })?; + Ok(tip_block + .header + .difficulty_threshold + .relative_to_network(&mockchain_network())) + } + // ********** Transaction methods ********** async fn get_transaction( diff --git a/packages/zaino-state/src/chain_index/source/validator_connector.rs b/packages/zaino-state/src/chain_index/source/validator_connector.rs index a720b9496..941aec4a5 100644 --- a/packages/zaino-state/src/chain_index/source/validator_connector.rs +++ b/packages/zaino-state/src/chain_index/source/validator_connector.rs @@ -19,8 +19,8 @@ use zebra_chain::{ use zebra_rpc::{ client::{BlockObject, GetBlockchainInfoBalance, HexData, Input, TransactionObject}, methods::{ - GetBlock, GetBlockHeaderObject, GetBlockHeaderResponse, GetBlockTransaction, GetBlockTrees, - ValidateAddresses as _, + chain_tip_difficulty, GetBlock, GetBlockHeaderObject, GetBlockHeaderResponse, + GetBlockTransaction, GetBlockTrees, ValidateAddresses as _, }, }; @@ -182,6 +182,25 @@ impl BlockchainSource for ValidatorConnector { } } + // ********** Chain methods ********** + + async fn get_difficulty(&self) -> BlockchainSourceResult { + match self { + ValidatorConnector::State(state) => chain_tip_difficulty( + state.network.to_zebra_network(), + state.read_state_service.clone(), + false, + ) + .await + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string())), + ValidatorConnector::Fetch(fetch) => Ok(fetch + .get_difficulty() + .await + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))? + .0), + } + } + // ********** Transaction methods ********** // Returns the transaction, and the height of the block that transaction is in if on the best chain diff --git a/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs b/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs index 2ab98a4e4..c287b22d3 100644 --- a/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs +++ b/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs @@ -1032,3 +1032,19 @@ async fn get_block_deltas() { "expected transparent deltas in at least one sampled block" ); } + +/// `get_difficulty` served through the ChainIndex from the mock vectors matches the +/// source and is a positive difficulty value. +#[tokio::test(flavor = "multi_thread")] +async fn get_difficulty() { + let (_blocks, _indexer, index_reader, mockchain) = + load_test_vectors_and_sync_chain_index(MockchainMode::Static).await; + + let via_index = index_reader.get_difficulty().await.unwrap(); + let via_source = mockchain.get_difficulty().await.unwrap(); + assert_eq!(via_index, via_source); + assert!( + via_index > 0.0, + "difficulty should be positive, got {via_index}" + ); +} diff --git a/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs b/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs index bd1450d4e..ff6229369 100644 --- a/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs +++ b/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs @@ -643,6 +643,11 @@ impl BlockchainSource for ProptestMockchain { unimplemented!() } + async fn get_difficulty(&self) -> BlockchainSourceResult { + // ProptestMockchain exercises sync/reorg, not the getdifficulty RPC. + unimplemented!() + } + /// Returns the block commitment tree data by hash async fn get_commitment_tree_roots( &self, From 91c3772e95578b1e0f7f8e3f3d3c9923394b1957 Mon Sep 17 00:00:00 2001 From: idky137 Date: Fri, 3 Jul 2026 09:17:10 +0100 Subject: [PATCH 022/134] move methods into chain_index / blockchainsource - proxy methods --- packages/zaino-state/src/backends/fetch.rs | 14 +-- packages/zaino-state/src/backends/state.rs | 26 ++-- packages/zaino-state/src/chain_index.rs | 111 +++++++++++++++++- .../zaino-state/src/chain_index/source.rs | 49 +++++++- .../chain_index/source/mockchain_source.rs | 63 ++++++++++ .../chain_index/source/validator_connector.rs | 83 ++++++++++++- .../chain_index/tests/proptest_blockgen.rs | 49 ++++++++ 7 files changed, 364 insertions(+), 31 deletions(-) diff --git a/packages/zaino-state/src/backends/fetch.rs b/packages/zaino-state/src/backends/fetch.rs index f08c4c6f0..54e37a1ee 100644 --- a/packages/zaino-state/src/backends/fetch.rs +++ b/packages/zaino-state/src/backends/fetch.rs @@ -282,7 +282,7 @@ impl ZcashIndexer for FetchServiceSubscriber { /// in Zebra's [`GetInfo`]. Zebra uses the field names and formats from the /// [zcashd code](https://github.com/zcash/zcash/blob/v4.6.0-1/src/rpc/misc.cpp#L86-L87). async fn get_info(&self) -> Result { - Ok(self.fetcher.get_info().await?.into()) + Ok(self.indexer.get_info().await?) } /// Returns blockchain state information, as a [`GetBlockchainInfoResponse`] JSON struct. @@ -325,7 +325,7 @@ impl ZcashIndexer for FetchServiceSubscriber { } async fn get_peer_info(&self) -> Result { - Ok(self.fetcher.get_peer_info().await?) + Ok(self.indexer.get_peer_info().await?) } /// Returns the proof-of-work difficulty as a multiple of the minimum difficulty. @@ -338,7 +338,7 @@ impl ZcashIndexer for FetchServiceSubscriber { } async fn get_block_subsidy(&self, height: u32) -> Result { - Ok(self.fetcher.get_block_subsidy(height).await?) + Ok(self.indexer.get_block_subsidy(height).await?) } /// Returns the total balance of a provided `addresses` in an [`AddressBalance`] instance. @@ -449,7 +449,7 @@ impl ZcashIndexer for FetchServiceSubscriber { } async fn get_mining_info(&self) -> Result { - Ok(self.fetcher.get_mining_info().await?) + Ok(self.indexer.get_mining_info().await?) } /// Returns statistics about the unspent transaction output set. @@ -800,14 +800,14 @@ impl ZcashIndexer for FetchServiceSubscriber { n: u32, include_mempool: Option, ) -> Result { - Ok(self.fetcher.get_tx_out(txid, n, include_mempool).await?) + Ok(self.indexer.get_tx_out(txid, n, include_mempool).await?) } async fn get_spent_info( &self, request: GetSpentInfoRequest, ) -> Result { - Ok(self.fetcher.get_spent_info(request).await?) + Ok(self.indexer.get_spent_info(request).await?) } async fn chain_height(&self) -> Result { @@ -888,7 +888,7 @@ impl ZcashIndexer for FetchServiceSubscriber { blocks: Option, height: Option, ) -> Result { - Ok(self.fetcher.get_network_sol_ps(blocks, height).await?) + Ok(self.indexer.get_network_sol_ps(blocks, height).await?) } } diff --git a/packages/zaino-state/src/backends/state.rs b/packages/zaino-state/src/backends/state.rs index 2e879c341..337a24d2e 100644 --- a/packages/zaino-state/src/backends/state.rs +++ b/packages/zaino-state/src/backends/state.rs @@ -601,13 +601,7 @@ impl ZcashIndexer for StateServiceSubscriber { type Error = StateServiceError; async fn get_info(&self) -> Result { - // A number of these fields are difficult to access from the state service - // TODO: Fix this - self.rpc_client - .get_info() - .await - .map(GetInfo::from) - .map_err(|e| StateServiceError::Custom(e.to_string())) + Ok(self.indexer.get_info().await?) } /// Returns all changes for an address. @@ -637,10 +631,7 @@ impl ZcashIndexer for StateServiceSubscriber { } async fn get_block_subsidy(&self, height: u32) -> Result { - self.rpc_client - .get_block_subsidy(height) - .await - .map_err(|e| StateServiceError::Custom(e.to_string())) + Ok(self.indexer.get_block_subsidy(height).await?) } async fn get_blockchain_info(&self) -> Result { @@ -802,7 +793,7 @@ impl ZcashIndexer for StateServiceSubscriber { } async fn get_peer_info(&self) -> Result { - Ok(self.rpc_client.get_peer_info().await?) + Ok(self.indexer.get_peer_info().await?) } async fn z_get_address_balance( @@ -946,7 +937,7 @@ impl ZcashIndexer for StateServiceSubscriber { } async fn get_mining_info(&self) -> Result { - Ok(self.rpc_client.get_mining_info().await?) + Ok(self.indexer.get_mining_info().await?) } /// Returns statistics about the unspent transaction output set. @@ -1235,14 +1226,14 @@ impl ZcashIndexer for StateServiceSubscriber { n: u32, include_mempool: Option, ) -> Result { - Ok(self.rpc_client.get_tx_out(txid, n, include_mempool).await?) + Ok(self.indexer.get_tx_out(txid, n, include_mempool).await?) } async fn get_spent_info( &self, request: GetSpentInfoRequest, ) -> Result { - Ok(self.rpc_client.get_spent_info(request).await?) + Ok(self.indexer.get_spent_info(request).await?) } async fn get_address_tx_ids( @@ -1283,10 +1274,7 @@ impl ZcashIndexer for StateServiceSubscriber { blocks: Option, height: Option, ) -> Result { - self.rpc_client - .get_network_sol_ps(blocks, height) - .await - .map_err(|e| StateServiceError::Custom(e.to_string())) + Ok(self.indexer.get_network_sol_ps(blocks, height).await?) } // Helper function, to get the chain height in rpc implementations diff --git a/packages/zaino-state/src/chain_index.rs b/packages/zaino-state/src/chain_index.rs index 9fe87a3c0..0fcf90ffc 100644 --- a/packages/zaino-state/src/chain_index.rs +++ b/packages/zaino-state/src/chain_index.rs @@ -41,8 +41,12 @@ use zaino_fetch::jsonrpsee::response::{ address_deltas::{GetAddressDeltasParams, GetAddressDeltasResponse}, block_deltas::BlockDeltas, block_header::GetBlockHeader, + block_subsidy::GetBlockSubsidy, chain_tips::{ChainTip, ChainTipStatus, GetChainTipsResponse}, - EmptyTxOutSetInfo, GetTxOutSetInfo, GetTxOutSetInfoResponse, + mining_info::GetMiningInfoWire, + peer_info::GetPeerInfo, + EmptyTxOutSetInfo, GetNetworkSolPsResponse, GetSpentInfoRequest, GetSpentInfoResponse, + GetTxOutResponse, GetTxOutSetInfo, GetTxOutSetInfoResponse, }; use zaino_proto::proto::utils::{compact_block_with_pool_types, PoolTypeFilter}; use zebra_chain::parameters::ConsensusBranchId; @@ -50,7 +54,7 @@ pub use zebra_chain::parameters::Network as ZebraNetwork; use zebra_chain::serialization::ZcashSerialize; use zebra_rpc::{ client::{GetAddressBalanceRequest, GetAddressTxIdsRequest}, - methods::{AddressBalance, GetAddressUtxos, GetBlock}, + methods::{AddressBalance, GetAddressUtxos, GetBlock, GetInfo}, }; use zebra_state::HashOrHeight; @@ -612,6 +616,48 @@ pub trait ChainIndexRpcExt: ChainIndex { /// zcashd reference: [`getdifficulty`](https://zcash.github.io/rpc/getdifficulty.html) fn get_difficulty(&self) -> impl std::future::Future>; + // ********** Node-passthrough methods ********** + // + // No local-index equivalent; always delegate to the backing validator. + + /// Returns the `getinfo` response. + fn get_info(&self) -> impl std::future::Future>; + + /// Returns the `getpeerinfo` response. + fn get_peer_info(&self) -> impl std::future::Future>; + + /// Returns the `getblocksubsidy` response at the given height. + fn get_block_subsidy( + &self, + height: u32, + ) -> impl std::future::Future>; + + /// Returns the `getmininginfo` response. + fn get_mining_info( + &self, + ) -> impl std::future::Future>; + + /// Returns the `gettxout` response for the given outpoint. + fn get_tx_out( + &self, + txid: String, + n: u32, + include_mempool: Option, + ) -> impl std::future::Future>; + + /// Returns the `getspentinfo` response for the given request. + fn get_spent_info( + &self, + request: GetSpentInfoRequest, + ) -> impl std::future::Future>; + + /// Returns the `getnetworksolps` response. + fn get_network_sol_ps( + &self, + blocks: Option, + height: Option, + ) -> impl std::future::Future>; + // ********** Transparent address history methods ********** /// Returns all changes for the given transparent addresses. @@ -2679,6 +2725,67 @@ impl ChainIndexRpcExt for NodeBackedChainIndexSubscrib .map_err(ChainIndexError::backing_validator) } + async fn get_info(&self) -> Result { + self.source() + .get_info() + .await + .map_err(ChainIndexError::backing_validator) + } + + async fn get_peer_info(&self) -> Result { + self.source() + .get_peer_info() + .await + .map_err(ChainIndexError::backing_validator) + } + + async fn get_block_subsidy(&self, height: u32) -> Result { + self.source() + .get_block_subsidy(height) + .await + .map_err(ChainIndexError::backing_validator) + } + + async fn get_mining_info(&self) -> Result { + self.source() + .get_mining_info() + .await + .map_err(ChainIndexError::backing_validator) + } + + async fn get_tx_out( + &self, + txid: String, + n: u32, + include_mempool: Option, + ) -> Result { + self.source() + .get_tx_out(txid, n, include_mempool) + .await + .map_err(ChainIndexError::backing_validator) + } + + async fn get_spent_info( + &self, + request: GetSpentInfoRequest, + ) -> Result { + self.source() + .get_spent_info(request) + .await + .map_err(ChainIndexError::backing_validator) + } + + async fn get_network_sol_ps( + &self, + blocks: Option, + height: Option, + ) -> Result { + self.source() + .get_network_sol_ps(blocks, height) + .await + .map_err(ChainIndexError::backing_validator) + } + /// Returns all changes for the given transparent addresses. async fn get_address_deltas( &self, diff --git a/packages/zaino-state/src/chain_index/source.rs b/packages/zaino-state/src/chain_index/source.rs index 7776d9667..8d50b06ec 100644 --- a/packages/zaino-state/src/chain_index/source.rs +++ b/packages/zaino-state/src/chain_index/source.rs @@ -16,7 +16,11 @@ use zaino_fetch::jsonrpsee::{ response::{ address_deltas::{GetAddressDeltasParams, GetAddressDeltasResponse}, block_header::GetBlockHeader, - GetBlockError, GetBlockResponse, GetTransactionResponse, GetTreestateResponse, + block_subsidy::GetBlockSubsidy, + mining_info::GetMiningInfoWire, + peer_info::GetPeerInfo, + GetBlockError, GetBlockResponse, GetNetworkSolPsResponse, GetSpentInfoRequest, + GetSpentInfoResponse, GetTransactionResponse, GetTreestateResponse, GetTxOutResponse, }, }; use zcash_primitives::merkle_tree::{read_commitment_tree, write_commitment_tree}; @@ -25,7 +29,7 @@ use zebra_chain::{ }; use zebra_rpc::{ client::{GetAddressBalanceRequest, GetAddressTxIdsRequest}, - methods::{AddressBalance, GetAddressUtxos}, + methods::{AddressBalance, GetAddressUtxos, GetInfo}, }; use zebra_state::{HashOrHeight, ReadRequest, ReadResponse, ReadStateService}; @@ -125,6 +129,47 @@ pub trait BlockchainSource: Clone + Send + Sync + 'static { /// minimum difficulty (the `getdifficulty` RPC value). fn get_difficulty(&self) -> impl SendFut>; + // ********** Node-passthrough methods ********** + // + // These have no local-index equivalent and always proxy to the backing validator's + // JSON-RPC interface. + + /// Returns the `getinfo` response. + fn get_info(&self) -> impl SendFut>; + + /// Returns the `getpeerinfo` response. + fn get_peer_info(&self) -> impl SendFut>; + + /// Returns the `getblocksubsidy` response at the given height. + fn get_block_subsidy( + &self, + height: u32, + ) -> impl SendFut>; + + /// Returns the `getmininginfo` response. + fn get_mining_info(&self) -> impl SendFut>; + + /// Returns the `gettxout` response for the given outpoint. + fn get_tx_out( + &self, + txid: String, + n: u32, + include_mempool: Option, + ) -> impl SendFut>; + + /// Returns the `getspentinfo` response for the given request. + fn get_spent_info( + &self, + request: GetSpentInfoRequest, + ) -> impl SendFut>; + + /// Returns the `getnetworksolps` response. + fn get_network_sol_ps( + &self, + blocks: Option, + height: Option, + ) -> impl SendFut>; + /// Returns the sapling and orchard treestate by hash fn get_treestate(&self, id: BlockHash) -> impl SendFut>; diff --git a/packages/zaino-state/src/chain_index/source/mockchain_source.rs b/packages/zaino-state/src/chain_index/source/mockchain_source.rs index 0f0ca4478..83e066dc4 100644 --- a/packages/zaino-state/src/chain_index/source/mockchain_source.rs +++ b/packages/zaino-state/src/chain_index/source/mockchain_source.rs @@ -683,6 +683,69 @@ impl BlockchainSource for MockchainSource { .relative_to_network(&mockchain_network())) } + // ********** Node-passthrough methods ********** + // + // These are node-only RPCs with no chain data in the vectors. Test vectors must be + // extended to let MockchainSource serve them; tracked by the update-test-vectors + // follow-up (see "Future work"). + + async fn get_info(&self) -> BlockchainSourceResult { + unimplemented!("MockchainSource cannot serve get_info until test vectors are extended") + } + + async fn get_peer_info( + &self, + ) -> BlockchainSourceResult { + unimplemented!("MockchainSource cannot serve get_peer_info until test vectors are extended") + } + + async fn get_block_subsidy( + &self, + _height: u32, + ) -> BlockchainSourceResult + { + unimplemented!( + "MockchainSource cannot serve get_block_subsidy until test vectors are extended" + ) + } + + async fn get_mining_info( + &self, + ) -> BlockchainSourceResult + { + unimplemented!( + "MockchainSource cannot serve get_mining_info until test vectors are extended" + ) + } + + async fn get_tx_out( + &self, + _txid: String, + _n: u32, + _include_mempool: Option, + ) -> BlockchainSourceResult { + unimplemented!("MockchainSource cannot serve get_tx_out until test vectors are extended") + } + + async fn get_spent_info( + &self, + _request: zaino_fetch::jsonrpsee::response::GetSpentInfoRequest, + ) -> BlockchainSourceResult { + unimplemented!( + "MockchainSource cannot serve get_spent_info until test vectors are extended" + ) + } + + async fn get_network_sol_ps( + &self, + _blocks: Option, + _height: Option, + ) -> BlockchainSourceResult { + unimplemented!( + "MockchainSource cannot serve get_network_sol_ps until test vectors are extended" + ) + } + // ********** Transaction methods ********** async fn get_transaction( diff --git a/packages/zaino-state/src/chain_index/source/validator_connector.rs b/packages/zaino-state/src/chain_index/source/validator_connector.rs index 941aec4a5..450a72744 100644 --- a/packages/zaino-state/src/chain_index/source/validator_connector.rs +++ b/packages/zaino-state/src/chain_index/source/validator_connector.rs @@ -9,6 +9,10 @@ use zaino_fetch::jsonrpsee::response::{ address_deltas::BlockInfo, block_deltas::{BlockDelta, BlockDeltas, InputDelta, OutputDelta}, block_header::GetBlockHeader, + block_subsidy::GetBlockSubsidy, + mining_info::GetMiningInfoWire, + peer_info::GetPeerInfo, + GetNetworkSolPsResponse, GetSpentInfoRequest, GetSpentInfoResponse, GetTxOutResponse, }; use zebra_chain::{ amount::{Amount, NonNegative}, @@ -20,7 +24,7 @@ use zebra_rpc::{ client::{BlockObject, GetBlockchainInfoBalance, HexData, Input, TransactionObject}, methods::{ chain_tip_difficulty, GetBlock, GetBlockHeaderObject, GetBlockHeaderResponse, - GetBlockTransaction, GetBlockTrees, ValidateAddresses as _, + GetBlockTransaction, GetBlockTrees, GetInfo, ValidateAddresses as _, }, }; @@ -69,6 +73,18 @@ pub enum ValidatorConnector { Fetch(JsonRpSeeConnector), } +impl ValidatorConnector { + /// The JSON-RPC connector for this validator, used by the node-passthrough RPCs that + /// have no local-index equivalent. The `State` variant proxies these through its + /// `mempool_fetcher` until the `ReadStateService` can serve them. + fn json_rpc_connector(&self) -> &JsonRpSeeConnector { + match self { + ValidatorConnector::State(state) => &state.mempool_fetcher, + ValidatorConnector::Fetch(fetch) => fetch, + } + } +} + impl BlockchainSource for ValidatorConnector { // ********** Block methods ********** @@ -201,6 +217,71 @@ impl BlockchainSource for ValidatorConnector { } } + // ********** Node-passthrough methods ********** + + async fn get_info(&self) -> BlockchainSourceResult { + Ok(self + .json_rpc_connector() + .get_info() + .await + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))? + .into()) + } + + async fn get_peer_info(&self) -> BlockchainSourceResult { + self.json_rpc_connector() + .get_peer_info() + .await + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string())) + } + + async fn get_block_subsidy(&self, height: u32) -> BlockchainSourceResult { + self.json_rpc_connector() + .get_block_subsidy(height) + .await + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string())) + } + + async fn get_mining_info(&self) -> BlockchainSourceResult { + self.json_rpc_connector() + .get_mining_info() + .await + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string())) + } + + async fn get_tx_out( + &self, + txid: String, + n: u32, + include_mempool: Option, + ) -> BlockchainSourceResult { + self.json_rpc_connector() + .get_tx_out(txid, n, include_mempool) + .await + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string())) + } + + async fn get_spent_info( + &self, + request: GetSpentInfoRequest, + ) -> BlockchainSourceResult { + self.json_rpc_connector() + .get_spent_info(request) + .await + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string())) + } + + async fn get_network_sol_ps( + &self, + blocks: Option, + height: Option, + ) -> BlockchainSourceResult { + self.json_rpc_connector() + .get_network_sol_ps(blocks, height) + .await + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string())) + } + // ********** Transaction methods ********** // Returns the transaction, and the height of the block that transaction is in if on the best chain diff --git a/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs b/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs index ff6229369..c24b6e499 100644 --- a/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs +++ b/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs @@ -648,6 +648,55 @@ impl BlockchainSource for ProptestMockchain { unimplemented!() } + async fn get_info(&self) -> BlockchainSourceResult { + unimplemented!() + } + + async fn get_peer_info( + &self, + ) -> BlockchainSourceResult { + unimplemented!() + } + + async fn get_block_subsidy( + &self, + _height: u32, + ) -> BlockchainSourceResult + { + unimplemented!() + } + + async fn get_mining_info( + &self, + ) -> BlockchainSourceResult + { + unimplemented!() + } + + async fn get_tx_out( + &self, + _txid: String, + _n: u32, + _include_mempool: Option, + ) -> BlockchainSourceResult { + unimplemented!() + } + + async fn get_spent_info( + &self, + _request: zaino_fetch::jsonrpsee::response::GetSpentInfoRequest, + ) -> BlockchainSourceResult { + unimplemented!() + } + + async fn get_network_sol_ps( + &self, + _blocks: Option, + _height: Option, + ) -> BlockchainSourceResult { + unimplemented!() + } + /// Returns the block commitment tree data by hash async fn get_commitment_tree_roots( &self, From 68cb4fb1fd54dde217348014bc7ef2cc20b84582 Mon Sep 17 00:00:00 2001 From: idky137 Date: Fri, 3 Jul 2026 09:35:53 +0100 Subject: [PATCH 023/134] move methods into chain_index / blockchainsource - validate address --- packages/zaino-state/src/backends/fetch.rs | 22 ++-- packages/zaino-state/src/backends/state.rs | 134 ++------------------ packages/zaino-state/src/indexer.rs | 136 ++++++++++++++++++++- 3 files changed, 153 insertions(+), 139 deletions(-) diff --git a/packages/zaino-state/src/backends/fetch.rs b/packages/zaino-state/src/backends/fetch.rs index 54e37a1ee..8e4fb5447 100644 --- a/packages/zaino-state/src/backends/fetch.rs +++ b/packages/zaino-state/src/backends/fetch.rs @@ -34,9 +34,7 @@ use zaino_fetch::{ chain_tips::GetChainTipsResponse, mining_info::GetMiningInfoWire, peer_info::GetPeerInfo, - z_validate_address::{ - ZValidateAddressResponse, DEPRECATION_NOTICE as Z_VALIDATE_DEPRECATION, - }, + z_validate_address::ZValidateAddressResponse, GetMempoolInfoResponse, GetNetworkSolPsResponse, GetSpentInfoRequest, GetSpentInfoResponse, GetTxOutResponse, GetTxOutSetInfoResponse, }, @@ -520,7 +518,9 @@ impl ZcashIndexer for FetchServiceSubscriber { &self, address: String, ) -> Result { - Ok(self.fetcher.validate_address(address).await?) + #[allow(deprecated)] + let network = self.config.common.network.to_zebra_network(); + Ok(crate::indexer::validate_address(address, &network)) } #[allow(deprecated)] @@ -528,8 +528,9 @@ impl ZcashIndexer for FetchServiceSubscriber { &self, address: String, ) -> Result { - tracing::warn!("{}", Z_VALIDATE_DEPRECATION); - Ok(self.fetcher.z_validate_address(address).await?) + #[allow(deprecated)] + let network = self.config.common.network.to_zebra_network(); + Ok(crate::indexer::z_validate_address(address, &network)) } /// Returns all transaction ids in the memory pool, as a JSON array. @@ -811,13 +812,8 @@ impl ZcashIndexer for FetchServiceSubscriber { } async fn chain_height(&self) -> Result { - Ok(Height( - self.indexer - .snapshot_nonfinalized_state() - .await? - .max_serviceable_height() - .0, - )) + let snapshot = self.indexer.snapshot_nonfinalized_state().await?; + Ok(self.indexer.best_chaintip(&snapshot).await?.height.into()) } /// Returns the transaction ids made by the provided transparent addresses. /// diff --git a/packages/zaino-state/src/backends/state.rs b/packages/zaino-state/src/backends/state.rs index 337a24d2e..2310e6d53 100644 --- a/packages/zaino-state/src/backends/state.rs +++ b/packages/zaino-state/src/backends/state.rs @@ -39,10 +39,7 @@ use zaino_fetch::{ chain_tips::GetChainTipsResponse, mining_info::GetMiningInfoWire, peer_info::GetPeerInfo, - z_validate_address::{ - InvalidZValidateAddress, KnownZValidateAddress, ZValidateAddressResponse, - DEPRECATION_NOTICE as Z_VALIDATE_DEPRECATION, - }, + z_validate_address::ZValidateAddressResponse, GetMempoolInfoResponse, GetNetworkSolPsResponse, GetSpentInfoRequest, GetSpentInfoResponse, GetSubtreesResponse, GetTxOutResponse, GetTxOutSetInfoResponse, }, @@ -60,14 +57,10 @@ use zaino_proto::proto::{ SendResponse, TransparentAddressBlockFilter, TreeState, TxFilter, }, }; -use zcash_keys::{address::Address, encoding::AddressCodec}; - -use zcash_protocol::consensus::NetworkType; -use zcash_transparent::address::TransparentAddress; use zebra_chain::{ block::Height, chain_tip::NetworkChainTipHeightEstimator, - parameters::{ConsensusBranchId, NetworkKind, NetworkUpgrade}, + parameters::{ConsensusBranchId, NetworkUpgrade}, serialization::ZcashDeserialize as _, subtree::NoteCommitmentSubtreeIndex, }; @@ -565,37 +558,6 @@ impl StateServiceSubscriber { } } -/// Extracts the diversifier and pk_d bytes from a validated Sapling -/// [`PaymentAddress`], returning pk_d in zcashd's big-endian byte order. -/// -/// # Deprecation -/// -/// See [`DEPRECATION_NOTICE`]. This function exists to support the -/// `z_validateaddress` RPC endpoint, which itself exists solely for zcashd -/// compatibility. The pk_d bytes are -/// reversed from `sapling-crypto`'s native little-endian representation to -/// match zcashd's big-endian hex output. -/// -/// # Precondition -/// -/// The caller must have obtained `s` through [`PaymentAddress::from_bytes`] or -/// equivalent (e.g. `ZcashAddress::convert_if_network`), which guarantees the -/// diversifier has a valid `g_d()` and pk_d is a non-identity Jubjub subgroup -/// point. No additional validation is performed here. -/// -/// # Layout -/// -/// `PaymentAddress::to_bytes()` returns 43 bytes: `diversifier (11) || pk_d (32)`. -/// `DiversifiedTransmissionKey::to_bytes()` is `pub(crate)` in `sapling-crypto`, -/// so we extract pk_d from the serialized form. -fn sapling_key_bytes(s: &sapling_crypto::PaymentAddress) -> ([u8; 11], [u8; 32]) { - let bytes = s.to_bytes(); - let diversifier: [u8; 11] = bytes[..11].try_into().unwrap(); - let mut pk_d: [u8; 32] = bytes[11..].try_into().unwrap(); - pk_d.reverse(); - (diversifier, pk_d) -} - // #[allow(deprecated)] impl ZcashIndexer for StateServiceSubscriber { type Error = StateServiceError; @@ -986,34 +948,8 @@ impl ZcashIndexer for StateServiceSubscriber { &self, raw_address: String, ) -> Result { - use zcash_transparent::address::TransparentAddress; - - let Ok(address) = raw_address.parse::() else { - return Ok(ValidateAddressResponse::invalid()); - }; - - let address = match address.convert_if_network::
( - match self.config.common.network.to_zebra_network().kind() { - NetworkKind::Mainnet => NetworkType::Main, - NetworkKind::Testnet => NetworkType::Test, - NetworkKind::Regtest => NetworkType::Regtest, - }, - ) { - Ok(address) => address, - Err(err) => { - tracing::debug!(?err, "conversion error"); - return Ok(ValidateAddressResponse::invalid()); - } - }; - - Ok(match address { - Address::Transparent(taddr) => ValidateAddressResponse::new( - true, - Some(raw_address), - Some(matches!(taddr, TransparentAddress::ScriptHash(_))), - ), - _ => ValidateAddressResponse::invalid(), - }) + let network = self.config.common.network.to_zebra_network(); + Ok(crate::indexer::validate_address(raw_address, &network)) } #[allow(deprecated)] @@ -1021,53 +957,8 @@ impl ZcashIndexer for StateServiceSubscriber { &self, address: String, ) -> Result { - tracing::warn!("{}", Z_VALIDATE_DEPRECATION); - - let Ok(parsed_address) = address.parse::() else { - return Ok(ZValidateAddressResponse::Known( - KnownZValidateAddress::Invalid(InvalidZValidateAddress::new()), - )); - }; - - let converted_address = match parsed_address.convert_if_network::
( - match self.config.common.network.to_zebra_network().kind() { - NetworkKind::Mainnet => NetworkType::Main, - NetworkKind::Testnet => NetworkType::Test, - NetworkKind::Regtest => NetworkType::Regtest, - }, - ) { - Ok(address) => address, - Err(err) => { - tracing::debug!(?err, "conversion error"); - return Ok(ZValidateAddressResponse::Known( - KnownZValidateAddress::Invalid(InvalidZValidateAddress::new()), - )); - } - }; - - // Note: It could be the case that Zaino needs to support Sprout. For now, it's been disabled. - match converted_address { - Address::Transparent(t) => match t { - TransparentAddress::PublicKeyHash(_) => { - Ok(ZValidateAddressResponse::p2pkh(address)) - } - TransparentAddress::ScriptHash(_) => Ok(ZValidateAddressResponse::p2sh(address)), - }, - Address::Unified(u) => Ok(ZValidateAddressResponse::unified( - u.encode(&self.network().to_zebra_network()), - )), - Address::Sapling(s) => { - let (diversifier, pk_d) = sapling_key_bytes(&s); - Ok(ZValidateAddressResponse::sapling( - s.encode(&self.network().to_zebra_network()), - Some(hex::encode(diversifier)), - Some(hex::encode(pk_d)), - )) - } - _ => Ok(ZValidateAddressResponse::Known( - KnownZValidateAddress::Invalid(InvalidZValidateAddress::new()), - )), - } + let network = self.config.common.network.to_zebra_network(); + Ok(crate::indexer::z_validate_address(address, &network)) } async fn z_get_subtrees_by_index( @@ -1279,15 +1170,8 @@ impl ZcashIndexer for StateServiceSubscriber { // Helper function, to get the chain height in rpc implementations async fn chain_height(&self) -> Result { - let mut state = self.read_state_service.clone(); - let response = state - .ready() - .and_then(|service| service.call(ReadRequest::Tip)) - .await?; - let (chain_height, _chain_hash) = expected_read_response!(response, Tip).ok_or( - RpcError::new_from_legacycode(LegacyCode::Misc, "no blocks in chain"), - )?; - Ok(chain_height) + let snapshot = self.indexer.snapshot_nonfinalized_state().await?; + Ok(self.indexer.best_chaintip(&snapshot).await?.height.into()) } } @@ -2225,7 +2109,7 @@ mod tests { /// addresses on other networks (mainnet, testnet). #[test] fn sapling_pk_d_byte_order_matches_test_vector() { - use super::sapling_key_bytes; + use crate::indexer::sapling_key_bytes; use zcash_keys::address::Address; use zcash_protocol::consensus::NetworkType; diff --git a/packages/zaino-state/src/indexer.rs b/packages/zaino-state/src/indexer.rs index acdf57a43..ed1d0ba18 100644 --- a/packages/zaino-state/src/indexer.rs +++ b/packages/zaino-state/src/indexer.rs @@ -12,7 +12,10 @@ use zaino_fetch::jsonrpsee::response::{ chain_tips::GetChainTipsResponse, mining_info::GetMiningInfoWire, peer_info::GetPeerInfo, - z_validate_address::ZValidateAddressResponse, + z_validate_address::{ + InvalidZValidateAddress, KnownZValidateAddress, ZValidateAddressResponse, + DEPRECATION_NOTICE as Z_VALIDATE_DEPRECATION, + }, GetMempoolInfoResponse, GetNetworkSolPsResponse, GetSpentInfoRequest, GetSpentInfoResponse, GetTxOutSetInfoResponse, }; @@ -983,3 +986,134 @@ pub(crate) async fn handle_raw_transaction( } } } + +/// Maps a Zebra network to the `zcash_protocol` network type used for address decoding. +fn address_network_type( + network: &zebra_chain::parameters::Network, +) -> zcash_protocol::consensus::NetworkType { + use zcash_protocol::consensus::NetworkType; + use zebra_chain::parameters::NetworkKind; + match network.kind() { + NetworkKind::Mainnet => NetworkType::Main, + NetworkKind::Testnet => NetworkType::Test, + NetworkKind::Regtest => NetworkType::Regtest, + } +} + +/// Validates a Zcash address for the `validateaddress` RPC. +/// +/// Pure address parsing over `network`; no chain data required, so both backends share +/// this implementation. +pub(crate) fn validate_address( + raw_address: String, + network: &zebra_chain::parameters::Network, +) -> ValidateAddressResponse { + use zcash_keys::address::Address; + use zcash_transparent::address::TransparentAddress; + + let Ok(address) = raw_address.parse::() else { + return ValidateAddressResponse::invalid(); + }; + + let address = match address.convert_if_network::
(address_network_type(network)) { + Ok(address) => address, + Err(err) => { + tracing::debug!(?err, "conversion error"); + return ValidateAddressResponse::invalid(); + } + }; + + match address { + Address::Transparent(taddr) => ValidateAddressResponse::new( + true, + Some(raw_address), + Some(matches!(taddr, TransparentAddress::ScriptHash(_))), + ), + _ => ValidateAddressResponse::invalid(), + } +} + +/// Validates a Zcash address for the deprecated `z_validateaddress` RPC. +/// +/// Pure address parsing over `network`; shared by both backends. +pub(crate) fn z_validate_address( + address: String, + network: &zebra_chain::parameters::Network, +) -> ZValidateAddressResponse { + use zcash_keys::address::Address; + use zcash_keys::encoding::AddressCodec as _; + use zcash_transparent::address::TransparentAddress; + + tracing::warn!("{}", Z_VALIDATE_DEPRECATION); + + let invalid = || { + ZValidateAddressResponse::Known(KnownZValidateAddress::Invalid( + InvalidZValidateAddress::new(), + )) + }; + + let Ok(parsed_address) = address.parse::() else { + return invalid(); + }; + + let converted_address = + match parsed_address.convert_if_network::
(address_network_type(network)) { + Ok(address) => address, + Err(err) => { + tracing::debug!(?err, "conversion error"); + return invalid(); + } + }; + + // Note: It could be the case that Zaino needs to support Sprout. For now, it's been disabled. + match converted_address { + Address::Transparent(TransparentAddress::PublicKeyHash(_)) => { + ZValidateAddressResponse::p2pkh(address) + } + Address::Transparent(TransparentAddress::ScriptHash(_)) => { + ZValidateAddressResponse::p2sh(address) + } + Address::Unified(u) => ZValidateAddressResponse::unified(u.encode(network)), + Address::Sapling(s) => { + let (diversifier, pk_d) = sapling_key_bytes(&s); + ZValidateAddressResponse::sapling( + s.encode(network), + Some(hex::encode(diversifier)), + Some(hex::encode(pk_d)), + ) + } + _ => invalid(), + } +} + +/// Extracts the diversifier and pk_d bytes from a validated Sapling +/// [`sapling_crypto::PaymentAddress`], returning pk_d in zcashd's big-endian byte order. +/// +/// # Deprecation +/// +/// See [`Z_VALIDATE_DEPRECATION`]. This function exists to support the `z_validateaddress` +/// RPC endpoint, which itself exists solely for zcashd compatibility. The pk_d bytes are +/// reversed from `sapling-crypto`'s native little-endian representation to match zcashd's +/// big-endian hex output. +/// +/// # Precondition +/// +/// The caller must have obtained `s` through `PaymentAddress::from_bytes` or equivalent +/// (e.g. `ZcashAddress::convert_if_network`), which guarantees the diversifier has a valid +/// `g_d()` and pk_d is a non-identity Jubjub subgroup point. No additional validation is +/// performed here. +/// +/// # Layout +/// +/// `PaymentAddress::to_bytes()` returns 43 bytes: `diversifier (11) || pk_d (32)`. +pub(crate) fn sapling_key_bytes(s: &sapling_crypto::PaymentAddress) -> ([u8; 11], [u8; 32]) { + let bytes = s.to_bytes(); + let diversifier: [u8; 11] = bytes[..11] + .try_into() + .expect("PaymentAddress::to_bytes always returns 43 bytes: diversifier is the first 11"); + let mut pk_d: [u8; 32] = bytes[11..] + .try_into() + .expect("PaymentAddress::to_bytes always returns 43 bytes: pk_d is the last 32"); + pk_d.reverse(); + (diversifier, pk_d) +} From 179feabcf15ad9a5715df91a342b95315c3bf316 Mon Sep 17 00:00:00 2001 From: idky137 Date: Fri, 3 Jul 2026 10:03:13 +0100 Subject: [PATCH 024/134] move methods into chain_index / blockchainsource - blockchain info --- packages/zaino-state/src/backends/fetch.rs | 14 +- packages/zaino-state/src/backends/state.rs | 153 +----------------- packages/zaino-state/src/chain_index.rs | 17 +- .../zaino-state/src/chain_index/source.rs | 7 +- .../chain_index/source/mockchain_source.rs | 11 ++ .../chain_index/source/validator_connector.rs | 149 ++++++++++++++++- .../chain_index/tests/proptest_blockgen.rs | 7 + 7 files changed, 191 insertions(+), 167 deletions(-) diff --git a/packages/zaino-state/src/backends/fetch.rs b/packages/zaino-state/src/backends/fetch.rs index 8e4fb5447..a26c7f6a8 100644 --- a/packages/zaino-state/src/backends/fetch.rs +++ b/packages/zaino-state/src/backends/fetch.rs @@ -294,19 +294,7 @@ impl ZcashIndexer for FetchServiceSubscriber { /// Some fields from the zcashd reference are missing from Zebra's [`GetBlockchainInfoResponse`]. It only contains the fields /// [required for lightwalletd support.](https://github.com/zcash/lightwalletd/blob/v0.4.9/common/common.go#L72-L89) async fn get_blockchain_info(&self) -> Result { - Ok(self - .fetcher - .get_blockchain_info() - .await? - .try_into() - .map_err(|_e| { - #[allow(deprecated)] - FetchServiceError::SerializationError( - zebra_chain::serialization::SerializationError::Parse( - "chainwork not hex-encoded integer", - ), - ) - })?) + Ok(self.indexer.get_blockchain_info().await?) } /// Returns details on the active state of the TX memory pool. diff --git a/packages/zaino-state/src/backends/state.rs b/packages/zaino-state/src/backends/state.rs index 2310e6d53..67e6efe55 100644 --- a/packages/zaino-state/src/backends/state.rs +++ b/packages/zaino-state/src/backends/state.rs @@ -58,11 +58,7 @@ use zaino_proto::proto::{ }, }; use zebra_chain::{ - block::Height, - chain_tip::NetworkChainTipHeightEstimator, - parameters::{ConsensusBranchId, NetworkUpgrade}, - serialization::ZcashDeserialize as _, - subtree::NoteCommitmentSubtreeIndex, + block::Height, serialization::ZcashDeserialize as _, subtree::NoteCommitmentSubtreeIndex, }; use zebra_rpc::{ client::{ @@ -70,20 +66,16 @@ use zebra_rpc::{ TransactionObject, ValidateAddressResponse, }, methods::{ - chain_tip_difficulty, AddressBalance, ConsensusBranchIdHex, GetAddressTxIdsRequest, - GetAddressUtxos, GetBlock, GetBlockHash, GetBlockchainInfoResponse, GetInfo, - GetRawTransaction, NetworkUpgradeInfo, NetworkUpgradeStatus, SentTransactionHash, - TipConsensusBranch, + AddressBalance, GetAddressTxIdsRequest, GetAddressUtxos, GetBlock, GetBlockHash, + GetBlockchainInfoResponse, GetInfo, GetRawTransaction, SentTransactionHash, }, server::error::LegacyCode, sync::init_read_state_with_syncer, }; use zebra_state::{HashOrHeight, ReadRequest, ReadResponse, ReadStateService}; -use chrono::Utc; use futures::TryFutureExt as _; use hex::{FromHex as _, ToHex}; -use indexmap::IndexMap; use std::{str::FromStr, sync::Arc}; use tokio::{ sync::mpsc, @@ -597,144 +589,7 @@ impl ZcashIndexer for StateServiceSubscriber { } async fn get_blockchain_info(&self) -> Result { - let mut state = self.read_state_service.clone(); - - let response = state - .ready() - .and_then(|service| service.call(ReadRequest::TipPoolValues)) - .await?; - let (height, hash, balance) = match response { - ReadResponse::TipPoolValues { - tip_height, - tip_hash, - value_balance, - } => (tip_height, tip_hash, value_balance), - unexpected => { - unreachable!("Unexpected response from state service: {unexpected:?}") - } - }; - - let usage_response = state - .ready() - .and_then(|service| service.call(ReadRequest::UsageInfo)) - .await?; - let size_on_disk = expected_read_response!(usage_response, UsageInfo); - - let request = zebra_state::ReadRequest::BlockHeader(hash.into()); - let response = state - .ready() - .and_then(|service| service.call(request)) - .await?; - let header = match response { - ReadResponse::BlockHeader { header, .. } => header, - unexpected => { - unreachable!("Unexpected response from state service: {unexpected:?}") - } - }; - - let now = Utc::now(); - let zebra_estimated_height = NetworkChainTipHeightEstimator::new( - header.time, - height, - &self.config.common.network.into(), - ) - .estimate_height_at(now); - let estimated_height = if header.time > now || zebra_estimated_height < height { - height - } else { - zebra_estimated_height - }; - - let upgrades = IndexMap::from_iter( - self.config - .common - .network - .to_zebra_network() - .full_activation_list() - .into_iter() - .filter_map(|(activation_height, network_upgrade)| { - // Zebra defines network upgrades based on incompatible consensus rule changes, - // but zcashd defines them based on ZIPs. - // - // All the network upgrades with a consensus branch ID - // are the same in Zebra and zcashd. - network_upgrade.branch_id().map(|branch_id| { - // zcashd's RPC seems to ignore Disabled network upgrades, - // so Zebra does too. - let status = if height >= activation_height { - NetworkUpgradeStatus::Active - } else { - NetworkUpgradeStatus::Pending - }; - - ( - ConsensusBranchIdHex::new(branch_id.into()), - NetworkUpgradeInfo::from_parts( - network_upgrade, - activation_height, - status, - ), - ) - }) - }), - ); - - let next_block_height = - (height + 1).expect("valid chain tips are a lot less than Height::MAX"); - let consensus = TipConsensusBranch::from_parts( - ConsensusBranchIdHex::new( - NetworkUpgrade::current(&self.config.common.network.into(), height) - .branch_id() - .unwrap_or(ConsensusBranchId::RPC_MISSING_ID) - .into(), - ) - .inner(), - ConsensusBranchIdHex::new( - NetworkUpgrade::current(&self.config.common.network.into(), next_block_height) - .branch_id() - .unwrap_or(ConsensusBranchId::RPC_MISSING_ID) - .into(), - ) - .inner(), - ); - - // TODO: Remove unwrap() - let difficulty = chain_tip_difficulty( - self.config.common.network.to_zebra_network(), - self.read_state_service.clone(), - false, - ) - .await - .unwrap(); - - let verification_progress = f64::from(height.0) / f64::from(zebra_estimated_height.0); - - Ok(GetBlockchainInfoResponse::new( - self.config - .common - .network - .to_zebra_network() - .bip70_network_name(), - height, - hash, - estimated_height, - zebra_rpc::client::GetBlockchainInfoBalance::chain_supply(balance), - // TODO: account for new delta_pools arg? - zebra_rpc::client::GetBlockchainInfoBalance::value_pools(balance, None), - upgrades, - consensus, - height, - difficulty, - verification_progress, - // TODO: store work in the finalized state for each height - // see https://github.com/ZcashFoundation/zebra/issues/7109 - 0, - false, - size_on_disk, - // TODO (copied from zebra): Investigate whether this needs to - // be implemented (it's sprout-only in zcashd) - 0, - )) + Ok(self.indexer.get_blockchain_info().await?) } /// Returns details on the active state of the TX memory pool. diff --git a/packages/zaino-state/src/chain_index.rs b/packages/zaino-state/src/chain_index.rs index 0fcf90ffc..cd1e8f028 100644 --- a/packages/zaino-state/src/chain_index.rs +++ b/packages/zaino-state/src/chain_index.rs @@ -54,7 +54,7 @@ pub use zebra_chain::parameters::Network as ZebraNetwork; use zebra_chain::serialization::ZcashSerialize; use zebra_rpc::{ client::{GetAddressBalanceRequest, GetAddressTxIdsRequest}, - methods::{AddressBalance, GetAddressUtxos, GetBlock, GetInfo}, + methods::{AddressBalance, GetAddressUtxos, GetBlock, GetBlockchainInfoResponse, GetInfo}, }; use zebra_state::HashOrHeight; @@ -623,6 +623,11 @@ pub trait ChainIndexRpcExt: ChainIndex { /// Returns the `getinfo` response. fn get_info(&self) -> impl std::future::Future>; + /// Returns the `getblockchaininfo` response. + fn get_blockchain_info( + &self, + ) -> impl std::future::Future>; + /// Returns the `getpeerinfo` response. fn get_peer_info(&self) -> impl std::future::Future>; @@ -2732,6 +2737,16 @@ impl ChainIndexRpcExt for NodeBackedChainIndexSubscrib .map_err(ChainIndexError::backing_validator) } + // `getblockchaininfo` needs cumulative pool value balances (TipPoolValues) and on-disk + // size, which are not in the ChainIndex's indexed data, so it cannot be built + // internally: always delegate to the backing validator. + async fn get_blockchain_info(&self) -> Result { + self.source() + .get_blockchain_info() + .await + .map_err(ChainIndexError::backing_validator) + } + async fn get_peer_info(&self) -> Result { self.source() .get_peer_info() diff --git a/packages/zaino-state/src/chain_index/source.rs b/packages/zaino-state/src/chain_index/source.rs index 8d50b06ec..4515e9b77 100644 --- a/packages/zaino-state/src/chain_index/source.rs +++ b/packages/zaino-state/src/chain_index/source.rs @@ -29,7 +29,7 @@ use zebra_chain::{ }; use zebra_rpc::{ client::{GetAddressBalanceRequest, GetAddressTxIdsRequest}, - methods::{AddressBalance, GetAddressUtxos, GetInfo}, + methods::{AddressBalance, GetAddressUtxos, GetBlockchainInfoResponse, GetInfo}, }; use zebra_state::{HashOrHeight, ReadRequest, ReadResponse, ReadStateService}; @@ -129,6 +129,11 @@ pub trait BlockchainSource: Clone + Send + Sync + 'static { /// minimum difficulty (the `getdifficulty` RPC value). fn get_difficulty(&self) -> impl SendFut>; + /// Returns the `getblockchaininfo` response. + fn get_blockchain_info( + &self, + ) -> impl SendFut>; + // ********** Node-passthrough methods ********** // // These have no local-index equivalent and always proxy to the backing validator's diff --git a/packages/zaino-state/src/chain_index/source/mockchain_source.rs b/packages/zaino-state/src/chain_index/source/mockchain_source.rs index 83e066dc4..420ca24a8 100644 --- a/packages/zaino-state/src/chain_index/source/mockchain_source.rs +++ b/packages/zaino-state/src/chain_index/source/mockchain_source.rs @@ -683,6 +683,17 @@ impl BlockchainSource for MockchainSource { .relative_to_network(&mockchain_network())) } + async fn get_blockchain_info( + &self, + ) -> BlockchainSourceResult { + // Needs cumulative pool value balances (TipPoolValues) and on-disk size, which the + // vectors don't carry. Test vectors must be extended to serve this method; tracked + // by the update-test-vectors follow-up (see "Future work"). + unimplemented!( + "MockchainSource cannot serve get_blockchain_info until test vectors are extended" + ) + } + // ********** Node-passthrough methods ********** // // These are node-only RPCs with no chain data in the vectors. Test vectors must be diff --git a/packages/zaino-state/src/chain_index/source/validator_connector.rs b/packages/zaino-state/src/chain_index/source/validator_connector.rs index 450a72744..1bd6ac48a 100644 --- a/packages/zaino-state/src/chain_index/source/validator_connector.rs +++ b/packages/zaino-state/src/chain_index/source/validator_connector.rs @@ -5,6 +5,7 @@ use std::str::FromStr as _; use chrono::{DateTime, Utc}; use hex::{FromHex as _, ToHex as _}; +use indexmap::IndexMap; use zaino_fetch::jsonrpsee::response::{ address_deltas::BlockInfo, block_deltas::{BlockDelta, BlockDeltas, InputDelta, OutputDelta}, @@ -17,14 +18,17 @@ use zaino_fetch::jsonrpsee::response::{ use zebra_chain::{ amount::{Amount, NonNegative}, block::{Header, SerializedBlock}, - parameters::NetworkUpgrade, + chain_tip::NetworkChainTipHeightEstimator, + parameters::{ConsensusBranchId, NetworkUpgrade}, serialization::{BytesInDisplayOrder as _, ZcashSerialize as _}, }; use zebra_rpc::{ client::{BlockObject, GetBlockchainInfoBalance, HexData, Input, TransactionObject}, methods::{ - chain_tip_difficulty, GetBlock, GetBlockHeaderObject, GetBlockHeaderResponse, - GetBlockTransaction, GetBlockTrees, GetInfo, ValidateAddresses as _, + chain_tip_difficulty, ConsensusBranchIdHex, GetBlock, GetBlockHeaderObject, + GetBlockHeaderResponse, GetBlockTransaction, GetBlockTrees, GetBlockchainInfoResponse, + GetInfo, NetworkUpgradeInfo, NetworkUpgradeStatus, TipConsensusBranch, + ValidateAddresses as _, }, }; @@ -217,6 +221,22 @@ impl BlockchainSource for ValidatorConnector { } } + async fn get_blockchain_info(&self) -> BlockchainSourceResult { + match self { + ValidatorConnector::State(state) => state.get_blockchain_info().await, + ValidatorConnector::Fetch(fetch) => fetch + .get_blockchain_info() + .await + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))? + .try_into() + .map_err(|_error| { + BlockchainSourceError::Unrecoverable( + "getblockchaininfo: chainwork not hex-encoded integer".to_string(), + ) + }), + } + } + // ********** Node-passthrough methods ********** async fn get_info(&self) -> BlockchainSourceResult { @@ -1560,6 +1580,129 @@ impl State { median_of_block_times(times) } + + /// Builds the `getblockchaininfo` response from the `ReadStateService`. + /// + /// Moved from the former `StateServiceSubscriber::get_blockchain_info`; error type is + /// now [`BlockchainSourceError`], and the difficulty is propagated rather than + /// unwrapped. + async fn get_blockchain_info(&self) -> BlockchainSourceResult { + let mut state = self.read_state_service.clone(); + let network = self.network.to_zebra_network(); + + let response = state + .ready() + .and_then(|service| service.call(ReadRequest::TipPoolValues)) + .await + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + let (height, hash, balance) = match response { + ReadResponse::TipPoolValues { + tip_height, + tip_hash, + value_balance, + } => (tip_height, tip_hash, value_balance), + unexpected => { + unreachable!("Unexpected response from state service: {unexpected:?}") + } + }; + + let usage_response = state + .ready() + .and_then(|service| service.call(ReadRequest::UsageInfo)) + .await + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + let size_on_disk = expected_read_response!(usage_response, UsageInfo); + + let response = state + .ready() + .and_then(|service| service.call(ReadRequest::BlockHeader(hash.into()))) + .await + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + let header = match response { + ReadResponse::BlockHeader { header, .. } => header, + unexpected => { + unreachable!("Unexpected response from state service: {unexpected:?}") + } + }; + + let now = Utc::now(); + let zebra_estimated_height = + NetworkChainTipHeightEstimator::new(header.time, height, &network) + .estimate_height_at(now); + let estimated_height = if header.time > now || zebra_estimated_height < height { + height + } else { + zebra_estimated_height + }; + + let upgrades = IndexMap::from_iter(network.full_activation_list().into_iter().filter_map( + |(activation_height, network_upgrade)| { + // Zebra defines network upgrades by consensus rule changes, zcashd by ZIPs; + // upgrades with a consensus branch ID are the same in both. + network_upgrade.branch_id().map(|branch_id| { + // zcashd's RPC ignores Disabled network upgrades, so Zebra does too. + let status = if height >= activation_height { + NetworkUpgradeStatus::Active + } else { + NetworkUpgradeStatus::Pending + }; + ( + ConsensusBranchIdHex::new(branch_id.into()), + NetworkUpgradeInfo::from_parts(network_upgrade, activation_height, status), + ) + }) + }, + )); + + let next_block_height = + (height + 1).expect("valid chain tips are a lot less than Height::MAX"); + let consensus = TipConsensusBranch::from_parts( + ConsensusBranchIdHex::new( + NetworkUpgrade::current(&network, height) + .branch_id() + .unwrap_or(ConsensusBranchId::RPC_MISSING_ID) + .into(), + ) + .inner(), + ConsensusBranchIdHex::new( + NetworkUpgrade::current(&network, next_block_height) + .branch_id() + .unwrap_or(ConsensusBranchId::RPC_MISSING_ID) + .into(), + ) + .inner(), + ); + + let difficulty = + chain_tip_difficulty(network.clone(), self.read_state_service.clone(), false) + .await + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + + let verification_progress = f64::from(height.0) / f64::from(zebra_estimated_height.0); + + Ok(GetBlockchainInfoResponse::new( + network.bip70_network_name(), + height, + hash, + estimated_height, + GetBlockchainInfoBalance::chain_supply(balance), + // TODO: account for new delta_pools arg? + GetBlockchainInfoBalance::value_pools(balance, None), + upgrades, + consensus, + height, + difficulty, + verification_progress, + // TODO: store work in the finalized state for each height + // (see https://github.com/ZcashFoundation/zebra/issues/7109) + 0, + false, + size_on_disk, + // TODO (copied from zebra): investigate whether this needs implementing + // (it's sprout-only in zcashd) + 0, + )) + } } /// Confirmations are one more than the depth, or -1 when the block is not on the best diff --git a/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs b/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs index c24b6e499..a5f694c4e 100644 --- a/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs +++ b/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs @@ -648,6 +648,13 @@ impl BlockchainSource for ProptestMockchain { unimplemented!() } + async fn get_blockchain_info( + &self, + ) -> BlockchainSourceResult { + // ProptestMockchain exercises sync/reorg, not the getblockchaininfo RPC. + unimplemented!() + } + async fn get_info(&self) -> BlockchainSourceResult { unimplemented!() } From 60a8dc2676a92a23e196aebbba6c39d19f5b8e2e Mon Sep 17 00:00:00 2001 From: idky137 Date: Fri, 3 Jul 2026 10:30:57 +0100 Subject: [PATCH 025/134] move methods into chain_index / blockchainsource - subtree roots --- packages/zaino-state/src/backends/fetch.rs | 27 ++++++-- packages/zaino-state/src/backends/state.rs | 81 ++++++---------------- packages/zaino-state/src/indexer.rs | 65 ++++++++++++++++- 3 files changed, 107 insertions(+), 66 deletions(-) diff --git a/packages/zaino-state/src/backends/fetch.rs b/packages/zaino-state/src/backends/fetch.rs index a26c7f6a8..6aa8a19fe 100644 --- a/packages/zaino-state/src/backends/fetch.rs +++ b/packages/zaino-state/src/backends/fetch.rs @@ -667,17 +667,34 @@ impl ZcashIndexer for FetchServiceSubscriber { /// starting at the chain tip. This RPC will return an empty list if the `start_index` subtree /// exists, but has not been rebuilt yet. This matches `zcashd`'s behaviour when subtrees aren't /// available yet. (But `zcashd` does its rebuild before syncing any blocks.) + #[allow(deprecated)] async fn z_get_subtrees_by_index( &self, pool: String, start_index: NoteCommitmentSubtreeIndex, limit: Option, ) -> Result { - Ok(self - .fetcher - .get_subtrees_by_index(pool, start_index.0, limit.map(|limit_index| limit_index.0)) - .await? - .into()) + let shielded_pool = match pool.as_str() { + "sapling" => crate::chain_index::ShieldedPool::Sapling, + "orchard" => crate::chain_index::ShieldedPool::Orchard, + otherwise => { + return Err(FetchServiceError::RpcError(RpcError::new_from_legacycode( + zebra_rpc::server::error::LegacyCode::Misc, + format!( + "invalid pool name \"{otherwise}\", must be \"sapling\" or \"orchard\"" + ), + ))) + } + }; + let roots = self + .indexer + .get_subtree_roots(shielded_pool, start_index.0, limit.map(|index| index.0)) + .await?; + Ok(crate::indexer::build_subtrees_by_index_response( + pool, + start_index, + roots, + )) } /// Returns the raw transaction data, as a [`GetRawTransaction`] JSON string or structure. diff --git a/packages/zaino-state/src/backends/state.rs b/packages/zaino-state/src/backends/state.rs index 67e6efe55..b17ce6906 100644 --- a/packages/zaino-state/src/backends/state.rs +++ b/packages/zaino-state/src/backends/state.rs @@ -41,7 +41,7 @@ use zaino_fetch::{ peer_info::GetPeerInfo, z_validate_address::ZValidateAddressResponse, GetMempoolInfoResponse, GetNetworkSolPsResponse, GetSpentInfoRequest, - GetSpentInfoResponse, GetSubtreesResponse, GetTxOutResponse, GetTxOutSetInfoResponse, + GetSpentInfoResponse, GetTxOutResponse, GetTxOutSetInfoResponse, }, }, }; @@ -62,7 +62,7 @@ use zebra_chain::{ }; use zebra_rpc::{ client::{ - GetAddressBalanceRequest, GetSubtreesByIndexResponse, GetTreestateResponse, SubtreeRpcData, + GetAddressBalanceRequest, GetSubtreesByIndexResponse, GetTreestateResponse, TransactionObject, ValidateAddressResponse, }, methods::{ @@ -822,64 +822,27 @@ impl ZcashIndexer for StateServiceSubscriber { start_index: NoteCommitmentSubtreeIndex, limit: Option, ) -> Result { - let mut state = self.read_state_service.clone(); - - match pool.as_str() { - "sapling" => { - let request = zebra_state::ReadRequest::SaplingSubtrees { start_index, limit }; - let response = state - .ready() - .and_then(|service| service.call(request)) - .await?; - let sapling_subtrees = expected_read_response!(response, SaplingSubtrees); - let subtrees = sapling_subtrees - .values() - .map(|subtree| { - SubtreeRpcData { - root: subtree.root.to_bytes().encode_hex(), - end_height: subtree.end_height, - } - .into() - }) - .collect(); - - Ok(GetSubtreesResponse { - pool, - start_index, - subtrees, - } - .into()) - } - "orchard" => { - let request = zebra_state::ReadRequest::OrchardSubtrees { start_index, limit }; - let response = state - .ready() - .and_then(|service| service.call(request)) - .await?; - let orchard_subtrees = expected_read_response!(response, OrchardSubtrees); - let subtrees = orchard_subtrees - .values() - .map(|subtree| { - SubtreeRpcData { - root: subtree.root.encode_hex(), - end_height: subtree.end_height, - } - .into() - }) - .collect(); - - Ok(GetSubtreesResponse { - pool, - start_index, - subtrees, - } - .into()) + let shielded_pool = match pool.as_str() { + "sapling" => crate::chain_index::ShieldedPool::Sapling, + "orchard" => crate::chain_index::ShieldedPool::Orchard, + otherwise => { + return Err(StateServiceError::RpcError(RpcError::new_from_legacycode( + LegacyCode::Misc, + format!( + "invalid pool name \"{otherwise}\", must be \"sapling\" or \"orchard\"" + ), + ))) } - otherwise => Err(StateServiceError::RpcError(RpcError::new_from_legacycode( - LegacyCode::Misc, - format!("invalid pool name \"{otherwise}\", must be \"sapling\" or \"orchard\""), - ))), - } + }; + let roots = self + .indexer + .get_subtree_roots(shielded_pool, start_index.0, limit.map(|index| index.0)) + .await?; + Ok(crate::indexer::build_subtrees_by_index_response( + pool, + start_index, + roots, + )) } async fn get_raw_transaction( diff --git a/packages/zaino-state/src/indexer.rs b/packages/zaino-state/src/indexer.rs index ed1d0ba18..d2ffca07b 100644 --- a/packages/zaino-state/src/indexer.rs +++ b/packages/zaino-state/src/indexer.rs @@ -17,7 +17,7 @@ use zaino_fetch::jsonrpsee::response::{ DEPRECATION_NOTICE as Z_VALIDATE_DEPRECATION, }, GetMempoolInfoResponse, GetNetworkSolPsResponse, GetSpentInfoRequest, GetSpentInfoResponse, - GetTxOutSetInfoResponse, + GetSubtreesResponse, GetTxOutSetInfoResponse, }; use zaino_proto::proto::{ compact_formats::CompactBlock, @@ -32,7 +32,9 @@ use zebra_chain::{ block::Height, serialization::BytesInDisplayOrder as _, subtree::NoteCommitmentSubtreeIndex, }; use zebra_rpc::{ - client::{GetSubtreesByIndexResponse, GetTreestateResponse, ValidateAddressResponse}, + client::{ + GetSubtreesByIndexResponse, GetTreestateResponse, SubtreeRpcData, ValidateAddressResponse, + }, methods::{ AddressBalance, GetAddressBalanceRequest, GetAddressTxIdsRequest, GetAddressUtxos, GetBlock, GetBlockHash, GetBlockchainInfoResponse, GetInfo, GetRawTransaction, @@ -1117,3 +1119,62 @@ pub(crate) fn sapling_key_bytes(s: &sapling_crypto::PaymentAddress) -> ([u8; 11] pk_d.reverse(); (diversifier, pk_d) } + +/// Shapes a `z_getsubtreesbyindex` JSON-RPC response from raw subtree roots. +/// +/// The `(root, end_height)` pairs from [`ChainIndex::get_subtree_roots`] are already in +/// the byte order the JSON-RPC uses (sapling `to_bytes`, orchard `to_repr` — zcashd's +/// `z_getsubtreesbyindex` does not reverse orchard subtree roots), so they are hex-encoded +/// as-is. Shared by both backends so the shaping lives in one place. +/// +/// [`ChainIndex::get_subtree_roots`]: crate::ChainIndex::get_subtree_roots +pub(crate) fn build_subtrees_by_index_response( + pool: String, + start_index: NoteCommitmentSubtreeIndex, + roots: Vec<([u8; 32], u32)>, +) -> GetSubtreesByIndexResponse { + use hex::ToHex as _; + + let subtrees = roots + .into_iter() + .map(|(root, end_height)| { + SubtreeRpcData { + root: root.encode_hex(), + end_height: Height(end_height), + } + .into() + }) + .collect(); + + GetSubtreesResponse { + pool, + start_index, + subtrees, + } + .into() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn build_subtrees_by_index_response_hex_encodes_roots() { + let roots = vec![([0xabu8; 32], 100u32), ([0xcdu8; 32], 200u32)]; + let response = build_subtrees_by_index_response( + "orchard".to_string(), + NoteCommitmentSubtreeIndex(5), + roots, + ); + + assert_eq!(response.pool().as_str(), "orchard"); + assert_eq!(response.start_index(), NoteCommitmentSubtreeIndex(5)); + + let subtrees = response.subtrees(); + assert_eq!(subtrees.len(), 2); + assert_eq!(subtrees[0].root, hex::encode([0xabu8; 32])); + assert_eq!(subtrees[0].end_height, Height(100)); + assert_eq!(subtrees[1].root, hex::encode([0xcdu8; 32])); + assert_eq!(subtrees[1].end_height, Height(200)); + } +} From 511fe58d26f7e7b23a154312d9c38501730f1a77 Mon Sep 17 00:00:00 2001 From: idky137 Date: Fri, 3 Jul 2026 11:21:50 +0100 Subject: [PATCH 026/134] move methods into chain_index / blockchainsource - mempool --- packages/zaino-state/src/backends/fetch.rs | 38 ++--- packages/zaino-state/src/backends/state.rs | 152 +++++++++--------- packages/zaino-state/src/chain_index.rs | 41 ++++- .../zaino-state/src/chain_index/source.rs | 19 ++- .../chain_index/source/mockchain_source.rs | 17 ++ .../chain_index/source/validator_connector.rs | 29 +++- .../chain_index/tests/proptest_blockgen.rs | 14 ++ 7 files changed, 201 insertions(+), 109 deletions(-) diff --git a/packages/zaino-state/src/backends/fetch.rs b/packages/zaino-state/src/backends/fetch.rs index 6aa8a19fe..2c9a49d7a 100644 --- a/packages/zaino-state/src/backends/fetch.rs +++ b/packages/zaino-state/src/backends/fetch.rs @@ -25,7 +25,6 @@ use zaino_fetch::{ chain::{transaction::FullTransaction, utils::ParseFromSlice}, jsonrpsee::{ connector::{JsonRpSeeConnector, RpcError}, - raw_transaction::validate_raw_transaction_hex, response::{ address_deltas::{GetAddressDeltasParams, GetAddressDeltasResponse}, block_deltas::BlockDeltas, @@ -90,7 +89,9 @@ use crate::{ pub struct FetchService { /// JsonRPC Client. /// - /// NOTE: DEPRCATED, USE INDEXER OR VALIDATOR_CONNECTOR. + /// NOTE: DEPRECATED — no longer read now that every fetch goes through the indexer. + /// TODO(task 3): remove this field (services should hold only `indexer`, `data`, `config`). + #[allow(dead_code)] fetcher: JsonRpSeeConnector, /// Core indexer. indexer: NodeBackedChainIndex, @@ -207,7 +208,9 @@ impl Drop for FetchService { pub struct FetchServiceSubscriber { /// JsonRPC Client. /// - /// NOTE: DEPRCATED, USE INDEXER OR VALIDATOR_CONNECTOR. + /// NOTE: DEPRECATED — no longer read now that every fetch goes through the indexer. + /// TODO(task 3): remove this field (services should hold only `indexer`, `data`, `config`). + #[allow(dead_code)] fetcher: JsonRpSeeConnector, /// Core indexer. pub indexer: NodeBackedChainIndexSubscriber, @@ -375,12 +378,10 @@ impl ZcashIndexer for FetchServiceSubscriber { &self, raw_transaction_hex: String, ) -> Result { - validate_raw_transaction_hex(&raw_transaction_hex)?; Ok(self - .fetcher + .indexer .send_raw_transaction(raw_transaction_hex) - .await? - .into()) + .await?) } /// Returns the requested block by hash or height, as a [`GetBlock`] JSON string. @@ -628,25 +629,10 @@ impl ZcashIndexer for FetchServiceSubscriber { return local_result; } - self.fetcher - .get_treestate(fallback_hash_or_height) - .await - .map_err(|_error| { - #[allow(deprecated)] - FetchServiceError::RpcError(RpcError::new_from_legacycode( - zebra_rpc::server::error::LegacyCode::InvalidParameter, - "Failed to fetch treestate.", - )) - }) - .and_then(|treestate| { - treestate.try_into().map_err(|_error| { - #[allow(deprecated)] - FetchServiceError::RpcError(RpcError::new_from_legacycode( - zebra_rpc::server::error::LegacyCode::InvalidParameter, - "Failed to parse treestate.", - )) - }) - }) + Ok(self + .indexer + .get_treestate_by_id(fallback_hash_or_height) + .await?) } /// Returns information about a range of Sapling or Orchard subtrees. diff --git a/packages/zaino-state/src/backends/state.rs b/packages/zaino-state/src/backends/state.rs index b17ce6906..af317cac1 100644 --- a/packages/zaino-state/src/backends/state.rs +++ b/packages/zaino-state/src/backends/state.rs @@ -30,7 +30,6 @@ use zaino_fetch::{ chain::{transaction::FullTransaction, utils::ParseFromSlice}, jsonrpsee::{ connector::{JsonRpSeeConnector, RpcError}, - raw_transaction::validate_raw_transaction_hex, response::{ address_deltas::{GetAddressDeltasParams, GetAddressDeltasResponse}, block_deltas::BlockDeltas, @@ -624,14 +623,10 @@ impl ZcashIndexer for StateServiceSubscriber { &self, raw_transaction_hex: String, ) -> Result { - validate_raw_transaction_hex(&raw_transaction_hex).map_err(StateServiceError::RpcError)?; - // Offload to the json rpc connector, as ReadStateService - // doesn't yet interface with the mempool - self.rpc_client + Ok(self + .indexer .send_raw_transaction(raw_transaction_hex) - .await - .map(SentTransactionHash::from) - .map_err(Into::into) + .await?) } async fn get_block_header( @@ -734,23 +729,10 @@ impl ZcashIndexer for StateServiceSubscriber { return local_result; } - self.rpc_client - .get_treestate(fallback_hash_or_height) - .await - .map_err(|_error| { - StateServiceError::RpcError(RpcError::new_from_legacycode( - zebra_rpc::server::error::LegacyCode::InvalidParameter, - "Failed to fetch treestate.", - )) - }) - .and_then(|treestate| { - treestate.try_into().map_err(|_error| { - StateServiceError::RpcError(RpcError::new_from_legacycode( - zebra_rpc::server::error::LegacyCode::InvalidParameter, - "Failed to parse treestate.", - )) - }) - }) + Ok(self + .indexer + .get_treestate_by_id(fallback_hash_or_height) + .await?) } async fn get_mining_info(&self) -> Result { @@ -1397,7 +1379,7 @@ impl LightWalletIndexer for StateServiceSubscriber { } }; - let mempool = self.mempool.clone(); + let indexer = self.indexer.clone(); let service_timeout = self.config.common.service.timeout; let (channel_tx, channel_rx) = mpsc::channel(self.config.common.service.channel_size as usize); @@ -1405,11 +1387,21 @@ impl LightWalletIndexer for StateServiceSubscriber { let timeout = timeout( time::Duration::from_secs((service_timeout * 4) as u64), async { - for (mempool_key, mempool_value) in - mempool.get_filtered_mempool(exclude_txids).await - { - let txid_bytes = match hex::decode(mempool_key.txid) { - Ok(bytes) => bytes, + let transactions = match indexer.get_mempool_transactions(exclude_txids).await { + Ok(transactions) => transactions, + Err(e) => { + channel_tx + .send(Err(tonic::Status::unknown(e.to_string()))) + .await + .ok(); + return; + } + }; + for serialized_transaction_bytes in transactions { + let txid = match zebra_chain::transaction::Transaction::zcash_deserialize( + &mut std::io::Cursor::new(&serialized_transaction_bytes), + ) { + Ok(transaction) => transaction.hash().0.to_vec(), Err(error) => { if channel_tx .send(Err(tonic::Status::unknown(error.to_string()))) @@ -1423,8 +1415,8 @@ impl LightWalletIndexer for StateServiceSubscriber { } }; match ::parse_from_slice( - mempool_value.serialized_tx.as_ref().as_ref(), - Some(vec![txid_bytes]), + &serialized_transaction_bytes, + Some(vec![txid]), None, ) { Ok(transaction) => { @@ -1488,65 +1480,66 @@ impl LightWalletIndexer for StateServiceSubscriber { /// Return a stream of current Mempool transactions. This will keep the output stream open while /// there are mempool transactions. It will close the returned stream when a new block is mined. async fn get_mempool_stream(&self) -> Result { - let mut mempool = self.mempool.clone(); + let indexer = self.indexer.clone(); let service_timeout = self.config.common.service.timeout; let (channel_tx, channel_rx) = mpsc::channel(self.config.common.service.channel_size as usize); - let snapshot = self.indexer.snapshot_nonfinalized_state().await?; - let Some(non_finalized_snapshot) = snapshot.get_nfs_snapshot() else { - // TODO: This probably shouldn't be an error. - // this is an improvement over previous behaviour of - // acting as if we are only synced to the genesis block - return Err(StateServiceError::UnavailableNotSyncedEnough); - }; - let mempool_height = non_finalized_snapshot.best_tip.height.0; + let snapshot = indexer.snapshot_nonfinalized_state().await?; tokio::spawn(async move { let timeout = timeout( time::Duration::from_secs((service_timeout * 6) as u64), async { - let (mut mempool_stream, _mempool_handle) = match mempool - .get_mempool_stream(None) - .await - { - Ok(stream) => stream, - Err(e) => { - warn!(?e, "error fetching mempool stream"); + let Some(non_finalized_snapshot) = snapshot.get_nfs_snapshot() else { + // TODO: This probably shouldn't be an error. + // this is an improvement over previous behaviour of + // acting as if we are only synced to the genesis block + if let Err(e) = channel_tx + .send(Err(tonic::Status::failed_precondition( + "zaino not yet synced".to_string(), + ))) + .await + { + warn!(%e, "GetMempoolStream channel closed unexpectedly"); + }; + return; + }; + let mempool_height = non_finalized_snapshot.best_tip.height.0; + match indexer.get_mempool_stream(None) { + Some(mut mempool_stream) => { + while let Some(result) = mempool_stream.next().await { + match result { + Ok(transaction_bytes) => { + if channel_tx + .send(Ok(RawTransaction { + data: transaction_bytes, + height: mempool_height as u64, + })) + .await + .is_err() + { + break; + } + } + Err(e) => { + channel_tx + .send(Err(tonic::Status::internal(format!( + "Error in mempool stream: {e:?}" + )))) + .await + .ok(); + break; + } + } + } + } + None => { + warn!("Error fetching stream from mempool, Incorrect chain tip!"); channel_tx .send(Err(tonic::Status::internal("Error getting mempool stream"))) .await .ok(); - return; } }; - while let Some(result) = mempool_stream.recv().await { - match result { - Ok((_mempool_key, mempool_value)) => { - if channel_tx - .send(Ok(RawTransaction { - data: mempool_value - .serialized_tx - .as_ref() - .as_ref() - .to_vec(), - height: mempool_height as u64, - })) - .await - .is_err() - { - break; - } - } - Err(e) => { - channel_tx - .send(Err(tonic::Status::internal(format!( - "Error in mempool stream: {e:?}" - )))) - .await - .ok(); - break; - } - } - } }, ) .await; @@ -1562,7 +1555,6 @@ impl LightWalletIndexer for StateServiceSubscriber { } } }); - Ok(RawTransactionStream::new(channel_rx)) } diff --git a/packages/zaino-state/src/chain_index.rs b/packages/zaino-state/src/chain_index.rs index cd1e8f028..d796d8941 100644 --- a/packages/zaino-state/src/chain_index.rs +++ b/packages/zaino-state/src/chain_index.rs @@ -37,6 +37,7 @@ use source::{BlockchainSource, ValidatorConnector}; use tokio_stream::StreamExt; use tokio_util::sync::CancellationToken; use tracing::{info, instrument}; +use zaino_fetch::jsonrpsee::raw_transaction::validate_raw_transaction_hex; use zaino_fetch::jsonrpsee::response::{ address_deltas::{GetAddressDeltasParams, GetAddressDeltasResponse}, block_deltas::BlockDeltas, @@ -54,7 +55,10 @@ pub use zebra_chain::parameters::Network as ZebraNetwork; use zebra_chain::serialization::ZcashSerialize; use zebra_rpc::{ client::{GetAddressBalanceRequest, GetAddressTxIdsRequest}, - methods::{AddressBalance, GetAddressUtxos, GetBlock, GetBlockchainInfoResponse, GetInfo}, + methods::{ + AddressBalance, GetAddressUtxos, GetBlock, GetBlockchainInfoResponse, GetInfo, + SentTransactionHash, + }, }; use zebra_state::HashOrHeight; @@ -663,6 +667,19 @@ pub trait ChainIndexRpcExt: ChainIndex { height: Option, ) -> impl std::future::Future>; + /// Submits a raw transaction to the network (`sendrawtransaction`). + fn send_raw_transaction( + &self, + raw_transaction_hex: String, + ) -> impl std::future::Future>; + + /// Returns the full `z_gettreestate` response for the given hash-or-height, via the + /// backing validator (node-passthrough fallback for treestates not locally serviceable). + fn get_treestate_by_id( + &self, + hash_or_height: String, + ) -> impl std::future::Future>; + // ********** Transparent address history methods ********** /// Returns all changes for the given transparent addresses. @@ -2801,6 +2818,28 @@ impl ChainIndexRpcExt for NodeBackedChainIndexSubscrib .map_err(ChainIndexError::backing_validator) } + async fn send_raw_transaction( + &self, + raw_transaction_hex: String, + ) -> Result { + validate_raw_transaction_hex(&raw_transaction_hex) + .map_err(|error| ChainIndexError::internal(error.to_string()))?; + self.source() + .send_raw_transaction(raw_transaction_hex) + .await + .map_err(ChainIndexError::backing_validator) + } + + async fn get_treestate_by_id( + &self, + hash_or_height: String, + ) -> Result { + self.source() + .get_treestate_by_id(hash_or_height) + .await + .map_err(ChainIndexError::backing_validator) + } + /// Returns all changes for the given transparent addresses. async fn get_address_deltas( &self, diff --git a/packages/zaino-state/src/chain_index/source.rs b/packages/zaino-state/src/chain_index/source.rs index 4515e9b77..1ae1afe9d 100644 --- a/packages/zaino-state/src/chain_index/source.rs +++ b/packages/zaino-state/src/chain_index/source.rs @@ -29,7 +29,9 @@ use zebra_chain::{ }; use zebra_rpc::{ client::{GetAddressBalanceRequest, GetAddressTxIdsRequest}, - methods::{AddressBalance, GetAddressUtxos, GetBlockchainInfoResponse, GetInfo}, + methods::{ + AddressBalance, GetAddressUtxos, GetBlockchainInfoResponse, GetInfo, SentTransactionHash, + }, }; use zebra_state::{HashOrHeight, ReadRequest, ReadResponse, ReadStateService}; @@ -175,6 +177,21 @@ pub trait BlockchainSource: Clone + Send + Sync + 'static { height: Option, ) -> impl SendFut>; + /// Submits a raw transaction to the network via the validator's mempool + /// (`sendrawtransaction`). + fn send_raw_transaction( + &self, + raw_transaction_hex: String, + ) -> impl SendFut>; + + /// Returns the full `z_gettreestate` response for the given hash-or-height string. + /// + /// Node-passthrough fallback for treestates not locally serviceable by the index. + fn get_treestate_by_id( + &self, + hash_or_height: String, + ) -> impl SendFut>; + /// Returns the sapling and orchard treestate by hash fn get_treestate(&self, id: BlockHash) -> impl SendFut>; diff --git a/packages/zaino-state/src/chain_index/source/mockchain_source.rs b/packages/zaino-state/src/chain_index/source/mockchain_source.rs index 420ca24a8..0d867ded4 100644 --- a/packages/zaino-state/src/chain_index/source/mockchain_source.rs +++ b/packages/zaino-state/src/chain_index/source/mockchain_source.rs @@ -757,6 +757,23 @@ impl BlockchainSource for MockchainSource { ) } + async fn send_raw_transaction( + &self, + _raw_transaction_hex: String, + ) -> BlockchainSourceResult { + // The mock chain has no mempool to accept submissions. + unimplemented!("MockchainSource cannot serve send_raw_transaction") + } + + async fn get_treestate_by_id( + &self, + _hash_or_height: String, + ) -> BlockchainSourceResult { + // The `z_get_treestate` local path serves the mock; the node-passthrough fallback + // is never reached, so this is left unimplemented. + unimplemented!("MockchainSource cannot serve the get_treestate_by_id passthrough") + } + // ********** Transaction methods ********** async fn get_transaction( diff --git a/packages/zaino-state/src/chain_index/source/validator_connector.rs b/packages/zaino-state/src/chain_index/source/validator_connector.rs index 1bd6ac48a..f46d07bcc 100644 --- a/packages/zaino-state/src/chain_index/source/validator_connector.rs +++ b/packages/zaino-state/src/chain_index/source/validator_connector.rs @@ -27,7 +27,7 @@ use zebra_rpc::{ methods::{ chain_tip_difficulty, ConsensusBranchIdHex, GetBlock, GetBlockHeaderObject, GetBlockHeaderResponse, GetBlockTransaction, GetBlockTrees, GetBlockchainInfoResponse, - GetInfo, NetworkUpgradeInfo, NetworkUpgradeStatus, TipConsensusBranch, + GetInfo, NetworkUpgradeInfo, NetworkUpgradeStatus, SentTransactionHash, TipConsensusBranch, ValidateAddresses as _, }, }; @@ -302,6 +302,33 @@ impl BlockchainSource for ValidatorConnector { .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string())) } + async fn send_raw_transaction( + &self, + raw_transaction_hex: String, + ) -> BlockchainSourceResult { + // ReadStateService does not yet interface with the mempool, so both variants + // submit via the JSON-RPC connector. + self.json_rpc_connector() + .send_raw_transaction(raw_transaction_hex) + .await + .map(SentTransactionHash::from) + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string())) + } + + async fn get_treestate_by_id( + &self, + hash_or_height: String, + ) -> BlockchainSourceResult { + self.json_rpc_connector() + .get_treestate(hash_or_height) + .await + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))? + .try_into() + .map_err(|_error| { + BlockchainSourceError::Unrecoverable("failed to parse treestate".to_string()) + }) + } + // ********** Transaction methods ********** // Returns the transaction, and the height of the block that transaction is in if on the best chain diff --git a/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs b/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs index a5f694c4e..53e1d7751 100644 --- a/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs +++ b/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs @@ -704,6 +704,20 @@ impl BlockchainSource for ProptestMockchain { unimplemented!() } + async fn send_raw_transaction( + &self, + _raw_transaction_hex: String, + ) -> BlockchainSourceResult { + unimplemented!() + } + + async fn get_treestate_by_id( + &self, + _hash_or_height: String, + ) -> BlockchainSourceResult { + unimplemented!() + } + /// Returns the block commitment tree data by hash async fn get_commitment_tree_roots( &self, From 9c697eeeb1997efa4d8e751ae441ffdbb1708e6e Mon Sep 17 00:00:00 2001 From: idky137 Date: Fri, 3 Jul 2026 15:37:52 +0100 Subject: [PATCH 027/134] moved last split functionality out of indexerservices --- CHANGELOG.md | 16 ++ live-tests/clientless/tests/chain_cache.rs | 5 +- live-tests/e2e/tests/devtool.rs | 2 +- live-tests/e2e/tests/test_vectors.rs | 2 +- packages/zaino-state/src/backends/fetch.rs | 30 +-- packages/zaino-state/src/backends/state.rs | 233 +++++------------- packages/zaino-state/src/chain_index.rs | 15 +- .../zaino-state/src/chain_index/source.rs | 9 + .../chain_index/source/validator_connector.rs | 181 +++++++++++++- 9 files changed, 285 insertions(+), 208 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 621a927ce..c74c79e32 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,22 @@ and this library adheres to Rust's notion of ## Unreleased ### Changed +- `zaino-state`: the `ChainIndex` trait is split into `ChainIndex` (the + wallet-essential core: chain/tx/address/mempool access) and a + `ChainIndexRpcExt: ChainIndex` extension (compact-block serving, subtree + roots, and the block-explorer / mining / node-passthrough RPCs). The split is + a provisional first pass to be refined into finer capability traits later. +- `zaino-state`: all remaining backend-split RPC functionality has moved out of + the `FetchService` (`JsonRpSeeConnector`) and `StateService` + (`ReadStateService`) backends and into `BlockchainSource` / + `ChainIndex`. Both backends now resolve every fetch through their `ChainIndex` + indexer — building responses from Zaino's own indexed state where possible and + delegating to the `ValidatorConnector` (`BlockchainSource`) only for + validator-only or passthrough data. Validator connection/syncer spawning also + moves into `ValidatorConnector::spawn_fetch` / `spawn_state`, so each + service/subscriber now holds only `{ indexer, data, config }`. This readies + the two backends for their eventual merge into a single + `ValidatorBackedIndexerService`. No behaviour change. - **Breaking** — config: `storage.database.sync_write_batch_bytes` (bytes) is renamed to `sync_write_batch_size` and given in **GiB** (default raised from 4 GiB to 32 GiB); this budget now also bounds the txout-set accumulator diff --git a/live-tests/clientless/tests/chain_cache.rs b/live-tests/clientless/tests/chain_cache.rs index 8f724bd3c..2bc08ea2e 100644 --- a/live-tests/clientless/tests/chain_cache.rs +++ b/live-tests/clientless/tests/chain_cache.rs @@ -46,7 +46,10 @@ mod chain_query_interface { source::ValidatorConnector, NodeBackedChainIndex, NodeBackedChainIndexSubscriber, ShieldedPool, }, - test_dependencies::{chain_index::ChainIndex, ChainIndexConfig}, + test_dependencies::{ + chain_index::{ChainIndex, ChainIndexRpcExt}, + ChainIndexConfig, + }, FetchService, Height, StateService, StateServiceConfig, ZcashService, }; #[cfg(feature = "zcashd_support")] diff --git a/live-tests/e2e/tests/devtool.rs b/live-tests/e2e/tests/devtool.rs index 5e14d4443..6252148ab 100644 --- a/live-tests/e2e/tests/devtool.rs +++ b/live-tests/e2e/tests/devtool.rs @@ -2263,7 +2263,7 @@ async fn get_mempool_info_state() { let mut svc = fund_and_fill_mempool_dual().await; let info = svc.state_subscriber.get_mempool_info().await.unwrap(); - let entries = svc.state_subscriber.mempool.get_mempool().await; + let entries = svc.state_subscriber.mempool().get_mempool().await; assert_eq!(entries.len() as u64, info.size); assert!(info.size >= 1); diff --git a/live-tests/e2e/tests/test_vectors.rs b/live-tests/e2e/tests/test_vectors.rs index 83dc4ac47..36128df39 100644 --- a/live-tests/e2e/tests/test_vectors.rs +++ b/live-tests/e2e/tests/test_vectors.rs @@ -282,7 +282,7 @@ async fn create_200_block_regtest_chain_vectors() { }) .unwrap(); - let mut state = state_service_subscriber.read_state_service.clone(); + let mut state = state_service_subscriber.read_state_service(); let (sapling_root, orchard_root) = { let (sapling_tree_response, orchard_tree_response) = futures::future::join( state.clone().call(zebra_state::ReadRequest::SaplingTree( diff --git a/packages/zaino-state/src/backends/fetch.rs b/packages/zaino-state/src/backends/fetch.rs index 2c9a49d7a..273851768 100644 --- a/packages/zaino-state/src/backends/fetch.rs +++ b/packages/zaino-state/src/backends/fetch.rs @@ -24,7 +24,7 @@ use zebra_rpc::{ use zaino_fetch::{ chain::{transaction::FullTransaction, utils::ParseFromSlice}, jsonrpsee::{ - connector::{JsonRpSeeConnector, RpcError}, + connector::RpcError, response::{ address_deltas::{GetAddressDeltasParams, GetAddressDeltasResponse}, block_deltas::BlockDeltas, @@ -87,12 +87,6 @@ use crate::{ #[derive(Debug)] #[deprecated = "Will be eventually replaced by `BlockchainSource`"] pub struct FetchService { - /// JsonRPC Client. - /// - /// NOTE: DEPRECATED — no longer read now that every fetch goes through the indexer. - /// TODO(task 3): remove this field (services should hold only `indexer`, `data`, `config`). - #[allow(dead_code)] - fetcher: JsonRpSeeConnector, /// Core indexer. indexer: NodeBackedChainIndex, /// Service metadata. @@ -126,15 +120,10 @@ impl ZcashService for FetchService { "Launching Fetch Service" ); - let fetcher = JsonRpSeeConnector::new_from_config_parts( - &config.common.validator_rpc_address, - config.common.validator_rpc_user.clone(), - config.common.validator_rpc_password.clone(), - config.common.validator_cookie_path.clone(), - ) - .await?; + let (source, zebra_build_data) = ValidatorConnector::spawn_fetch(&config.common) + .await + .map_err(|error| FetchServiceError::Critical(error.to_string()))?; - let zebra_build_data = fetcher.get_info().await?; let data = ServiceMetadata::new( get_build_info(config.common.indexer_version.clone()), config.common.network.to_zebra_network(), @@ -143,13 +132,11 @@ impl ZcashService for FetchService { ); info!(build = %data.zebra_build(), subversion = %data.zebra_subversion(), "Connected to Zcash node"); - let source = ValidatorConnector::Fetch(fetcher.clone()); let indexer = NodeBackedChainIndex::new(source, config.clone().into()) .await - .unwrap(); + .map_err(|error| FetchServiceError::Critical(error.to_string()))?; let fetch_service = Self { - fetcher, indexer, data, config, @@ -176,7 +163,6 @@ impl ZcashService for FetchService { /// Returns a [`FetchServiceSubscriber`]. fn get_subscriber(&self) -> IndexerSubscriber { IndexerSubscriber::new(FetchServiceSubscriber { - fetcher: self.fetcher.clone(), indexer: self.indexer.subscriber(), data: self.data.clone(), config: self.config.clone(), @@ -206,12 +192,6 @@ impl Drop for FetchService { #[derive(Debug, Clone)] #[allow(deprecated)] pub struct FetchServiceSubscriber { - /// JsonRPC Client. - /// - /// NOTE: DEPRECATED — no longer read now that every fetch goes through the indexer. - /// TODO(task 3): remove this field (services should hold only `indexer`, `data`, `config`). - #[allow(dead_code)] - fetcher: JsonRpSeeConnector, /// Core indexer. pub indexer: NodeBackedChainIndexSubscriber, /// Service metadata. diff --git a/packages/zaino-state/src/backends/state.rs b/packages/zaino-state/src/backends/state.rs index af317cac1..cc6f4b609 100644 --- a/packages/zaino-state/src/backends/state.rs +++ b/packages/zaino-state/src/backends/state.rs @@ -3,23 +3,21 @@ #[allow(deprecated)] use crate::{ chain_index::{ - chain_tips_from_nonfinalized_snapshot, - mempool::{Mempool, MempoolSubscriber}, - source::ValidatorConnector, - types as chain_types, ChainIndex, ChainIndexRpcExt, + chain_tips_from_nonfinalized_snapshot, source::ValidatorConnector, types as chain_types, + ChainIndex, ChainIndexRpcExt, }, config::{DonationAddress, StateServiceConfig}, error::{BlockCacheError, StateServiceError}, indexer::{ handle_raw_transaction, IndexerSubscriber, LightWalletIndexer, ZcashIndexer, ZcashService, }, - status::{NamedAtomicStatus, Status, StatusType}, + status::{Status, StatusType}, stream::{ AddressStream, CompactBlockStream, CompactTransactionStream, RawTransactionStream, UtxoReplyStream, }, utils::{get_build_info, ServiceMetadata}, - BackendType, NodeBackedChainIndex, NodeBackedChainIndexSubscriber, State, + BackendType, NodeBackedChainIndex, NodeBackedChainIndexSubscriber, }; use crate::{ chain_index::{types::BestChainLocation, NonFinalizedSnapshot}, @@ -29,7 +27,7 @@ use tokio_stream::StreamExt as _; use zaino_fetch::{ chain::{transaction::FullTransaction, utils::ParseFromSlice}, jsonrpsee::{ - connector::{JsonRpSeeConnector, RpcError}, + connector::RpcError, response::{ address_deltas::{GetAddressDeltasParams, GetAddressDeltasResponse}, block_deltas::BlockDeltas, @@ -69,31 +67,17 @@ use zebra_rpc::{ GetBlockchainInfoResponse, GetInfo, GetRawTransaction, SentTransactionHash, }, server::error::LegacyCode, - sync::init_read_state_with_syncer, }; -use zebra_state::{HashOrHeight, ReadRequest, ReadResponse, ReadStateService}; +use zebra_state::HashOrHeight; -use futures::TryFutureExt as _; use hex::{FromHex as _, ToHex}; -use std::{str::FromStr, sync::Arc}; +use std::str::FromStr; use tokio::{ sync::mpsc, time::{self, timeout}, }; -use tower::{Service, ServiceExt}; use tracing::{info, instrument, warn}; -macro_rules! expected_read_response { - ($response:ident, $expected_variant:ident) => { - match $response { - ReadResponse::$expected_variant(inner) => inner, - unexpected => { - unreachable!("Unexpected response from state service: {unexpected:?}") - } - } - }; -} - /// Chain fetch service backed by Zebra's `ReadStateService` and `TrustedChainSync`. /// /// NOTE: We currently dop not implement clone for chain fetch services @@ -105,51 +89,20 @@ macro_rules! expected_read_response { #[derive(Debug)] // #[deprecated = "Will be eventually replaced by `BlockchainSource"] pub struct StateService { - /// `ReadeStateService` from Zebra-State. - read_state_service: ReadStateService, - - /// Internal mempool. - mempool: Mempool, - - /// StateService config data. - #[allow(deprecated)] - config: StateServiceConfig, - - /// Listener for when the chain tip changes - chain_tip_change: zebra_state::ChainTipChange, - - /// Sync task handle. - sync_task_handle: Option>>, - - /// JsonRPC Client. - rpc_client: JsonRpSeeConnector, - /// Core indexer. indexer: NodeBackedChainIndex, /// Service metadata. data: ServiceMetadata, - /// Thread-safe status indicator. - status: NamedAtomicStatus, -} - -impl StateService { - #[cfg(feature = "test_dependencies")] - /// Helper for tests - pub fn read_state_service(&self) -> &ReadStateService { - &self.read_state_service - } + /// StateService config data. + #[allow(deprecated)] + config: StateServiceConfig, } impl Status for StateService { fn status(&self) -> StatusType { - let current_status = self.status.load(); - if current_status == StatusType::Closing { - current_status - } else { - self.indexer.status() - } + self.indexer.status() } } @@ -169,15 +122,9 @@ impl ZcashService for StateService { "Spawning State Service" ); - let rpc_client = JsonRpSeeConnector::new_from_config_parts( - &config.common.validator_rpc_address, - config.common.validator_rpc_user.clone(), - config.common.validator_rpc_password.clone(), - config.common.validator_cookie_path.clone(), - ) - .await?; - - let zebra_build_data = rpc_client.get_info().await?; + let (source, zebra_build_data) = ValidatorConnector::spawn_state(&config) + .await + .map_err(|error| StateServiceError::Critical(error.to_string()))?; let data = ServiceMetadata::new( get_build_info(config.common.indexer_version.clone()), @@ -187,101 +134,25 @@ impl ZcashService for StateService { ); info!(build = %data.zebra_build(), subversion = %data.zebra_subversion(), "Connected to Zcash node"); - info!( - grpc_address = %config.validator_grpc_address, - "Launching Chain Syncer" - ); - let (mut read_state_service, _latest_chain_tip, chain_tip_change, sync_task_handle) = - init_read_state_with_syncer( - config.validator_state_config.clone(), - &config.common.network.to_zebra_network(), - config.validator_grpc_address, - ) - .await??; - - info!("Chain syncer launched"); - - // Wait for ReadStateService to catch up to the validator's best chain tip. - // Height alone is insufficient during reorgs: the same height can refer to - // different blocks until JSON-RPC and ReadStateService agree on tip hash. - loop { - let blockchain_info = rpc_client.get_blockchain_info().await?; - let server_height = blockchain_info.blocks; - let server_tip_hash = blockchain_info.best_block_hash; - - let syncer_response = read_state_service - .ready() - .and_then(|service| service.call(ReadRequest::Tip)) - .await?; - let (syncer_height, syncer_tip_hash) = - expected_read_response!(syncer_response, Tip).ok_or( - RpcError::new_from_legacycode(LegacyCode::Misc, "no blocks in chain"), - )?; - - if server_height == syncer_height && server_tip_hash == syncer_tip_hash { - info!( - height = syncer_height.0, - tip_hash = %syncer_tip_hash, - "ReadStateService synced with Zebra" - ); - break; - } else { - info!( - syncer_height = syncer_height.0, - validator_height = server_height.0, - syncer_tip_hash = %syncer_tip_hash, - validator_tip_hash = %server_tip_hash, - "ReadStateService syncing with Zebra" - ); - tokio::time::sleep(std::time::Duration::from_millis(1000)).await; - continue; - } - } - - let mempool_source = ValidatorConnector::State(crate::chain_index::source::State { - read_state_service: read_state_service.clone(), - mempool_fetcher: rpc_client.clone(), - network: config.common.network, - }); - - let mempool = Mempool::spawn(mempool_source, None).await?; - - let chain_index = NodeBackedChainIndex::new( - ValidatorConnector::State(State { - read_state_service: read_state_service.clone(), - mempool_fetcher: rpc_client.clone(), - network: config.common.network, - }), - config.clone().into(), - ) - .await - .unwrap(); + let indexer = NodeBackedChainIndex::new(source, config.clone().into()) + .await + .map_err(|error| StateServiceError::Critical(error.to_string()))?; let state_service = Self { - chain_tip_change, - read_state_service, - sync_task_handle: Some(Arc::new(sync_task_handle)), - rpc_client: rpc_client.clone(), - mempool, - indexer: chain_index, + indexer, data, config, - status: NamedAtomicStatus::new("StateService", StatusType::Spawning), }; // wait for sync to complete, return error on sync fail. loop { match state_service.status() { - StatusType::Ready => { - state_service.status.store(StatusType::Ready); - break; - } + StatusType::Ready | StatusType::Closing => break, StatusType::CriticalError => { return Err(StateServiceError::Critical( "Chain index sync failed".to_string(), )); } - StatusType::Closing => break, _ => { tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; } @@ -293,23 +164,23 @@ impl ZcashService for StateService { fn get_subscriber(&self) -> IndexerSubscriber { IndexerSubscriber::new(StateServiceSubscriber { - read_state_service: self.read_state_service.clone(), - rpc_client: self.rpc_client.clone(), - mempool: self.mempool.subscriber(), indexer: self.indexer.subscriber(), data: self.data.clone(), config: self.config.clone(), - chain_tip_change: self.chain_tip_change.clone(), }) } /// Shuts down the StateService. + /// + /// Delegates to the indexer, which cancels its sync loop, tears down the + /// finalised DB and mempool, and aborts the source-owned Zebra chain-syncer + /// task via [`crate::chain_index::source::BlockchainSource::shutdown`]. fn close(&mut self) { - if self.sync_task_handle.is_some() { - if let Some(handle) = self.sync_task_handle.take() { - handle.abort(); - } - } + tokio::task::block_in_place(|| { + tokio::runtime::Handle::current().block_on(async { + let _ = self.indexer.shutdown().await; + }); + }); } } @@ -326,27 +197,41 @@ impl Drop for StateService { #[derive(Debug, Clone)] // #[deprecated] pub struct StateServiceSubscriber { - /// Remote wrappper functionality for zebra's [`ReadStateService`]. - pub read_state_service: ReadStateService, + /// Core indexer. + pub indexer: NodeBackedChainIndexSubscriber, - /// Internal mempool. - pub mempool: MempoolSubscriber, + /// Service metadata. + pub data: ServiceMetadata, /// StateService config data. #[allow(deprecated)] config: StateServiceConfig, +} - /// Listener for when the chain tip changes - chain_tip_change: zebra_state::ChainTipChange, - - /// JsonRPC Client. - pub rpc_client: JsonRpSeeConnector, - - /// Core indexer. - pub indexer: NodeBackedChainIndexSubscriber, +impl StateServiceSubscriber { + /// The backing Zebra [`ReadStateService`]. + /// + /// Test-only escape hatch: live tests recompute expected chain data (e.g. + /// treestate roots) directly off the `ReadStateService`. Production code goes + /// through the `ChainIndex` API. + #[cfg(feature = "test_dependencies")] + pub fn read_state_service(&self) -> zebra_state::ReadStateService { + self.indexer + .source() + .read_state_service() + .expect("StateServiceSubscriber is always State-backed") + .clone() + } - /// Service metadata. - pub data: ServiceMetadata, + /// The indexer's mempool subscriber. + /// + /// Test-only escape hatch: live tests recompute expected `getmempoolinfo` + /// values directly off the mempool's entries. Production code goes through the + /// `ChainIndex` mempool API. + #[cfg(feature = "test_dependencies")] + pub fn mempool(&self) -> &crate::chain_index::mempool::MempoolSubscriber { + self.indexer.mempool_subscriber() + } } impl Status for StateServiceSubscriber { @@ -383,7 +268,11 @@ impl StateServiceSubscriber { /// Gets a Subscriber to any updates to the latest chain tip pub fn chaintip_update_subscriber(&self) -> ChainTipSubscriber { ChainTipSubscriber { - monitor: self.chain_tip_change.clone(), + monitor: self + .indexer + .source() + .chain_tip_change() + .expect("StateServiceSubscriber is always State-backed"), } } /// Return a list of consecutive compact blocks. diff --git a/packages/zaino-state/src/chain_index.rs b/packages/zaino-state/src/chain_index.rs index d796d8941..66a4d5873 100644 --- a/packages/zaino-state/src/chain_index.rs +++ b/packages/zaino-state/src/chain_index.rs @@ -973,6 +973,9 @@ impl NodeBackedChainIndex { self.status.store(StatusType::Closing); self.finalized_db.shutdown().await?; self.mempool.close(); + // Release any source-owned background work (e.g. the Zebra chain-syncer + // task behind the `State` connector). No-op for poll-only sources. + self.source.shutdown(); Ok(()) } @@ -1291,10 +1294,20 @@ async fn compact_block_from_source( } impl NodeBackedChainIndexSubscriber { - fn source(&self) -> &Source { + pub(crate) fn source(&self) -> &Source { &self.source } + /// The indexer's mempool subscriber. + /// + /// Test-only escape hatch: live tests recompute expected `getmempoolinfo` + /// values directly off the mempool's entries. Production code goes through + /// the `ChainIndex` mempool API. + #[cfg(feature = "test_dependencies")] + pub(crate) fn mempool_subscriber(&self) -> &mempool::MempoolSubscriber { + &self.mempool + } + /// Returns the combined status of all chain index components. pub fn combined_status(&self) -> StatusType { let finalized_status = self.finalized_state.status(); diff --git a/packages/zaino-state/src/chain_index/source.rs b/packages/zaino-state/src/chain_index/source.rs index 1ae1afe9d..acc8a1915 100644 --- a/packages/zaino-state/src/chain_index/source.rs +++ b/packages/zaino-state/src/chain_index/source.rs @@ -328,6 +328,15 @@ pub trait BlockchainSource: Clone + Send + Sync + 'static { fn subscribe_to_blocks_received(&self) -> Option> { None } + + /// Release any long-lived resources the source owns (e.g. a background + /// syncer task feeding a `ReadStateService`). + /// + /// Default is a no-op — poll-only sources (`JsonRpSeeConnector`) and test + /// mockchains own nothing to tear down. Sources that spawn their own + /// validator plumbing (the `State` arm of `ValidatorConnector`, which owns + /// the Zebra syncer task) override this to abort that task on shutdown. + fn shutdown(&self) {} } /// Sleep up to `duration`, but return early if `change_rx` resolves first. diff --git a/packages/zaino-state/src/chain_index/source/validator_connector.rs b/packages/zaino-state/src/chain_index/source/validator_connector.rs index f46d07bcc..d8b3bb50e 100644 --- a/packages/zaino-state/src/chain_index/source/validator_connector.rs +++ b/packages/zaino-state/src/chain_index/source/validator_connector.rs @@ -6,6 +6,7 @@ use std::str::FromStr as _; use chrono::{DateTime, Utc}; use hex::{FromHex as _, ToHex as _}; use indexmap::IndexMap; +use tracing::info; use zaino_fetch::jsonrpsee::response::{ address_deltas::BlockInfo, block_deltas::{BlockDelta, BlockDeltas, InputDelta, OutputDelta}, @@ -13,8 +14,12 @@ use zaino_fetch::jsonrpsee::response::{ block_subsidy::GetBlockSubsidy, mining_info::GetMiningInfoWire, peer_info::GetPeerInfo, - GetNetworkSolPsResponse, GetSpentInfoRequest, GetSpentInfoResponse, GetTxOutResponse, + GetInfoResponse, GetNetworkSolPsResponse, GetSpentInfoRequest, GetSpentInfoResponse, + GetTxOutResponse, }; +use zebra_rpc::sync::init_read_state_with_syncer; + +use crate::config::{CommonBackendConfig, StateServiceConfig}; use zebra_chain::{ amount::{Amount, NonNegative}, block::{Header, SerializedBlock}, @@ -62,6 +67,12 @@ pub struct State { pub mempool_fetcher: JsonRpSeeConnector, /// Current network type being run. pub network: Network, + /// Watches the Zebra syncer's chain-tip changes; served to consumers via + /// [`ValidatorConnector::chain_tip_change`]. + pub chain_tip_change: zebra_state::ChainTipChange, + /// Handle to the Zebra `ReadStateService` sync task, kept alive for the lifetime of + /// the connector and aborted on [`ValidatorConnector::shutdown`]. Shared across clones. + pub sync_task_handle: Option>>, } /// A connection to a validator. @@ -87,6 +98,151 @@ impl ValidatorConnector { ValidatorConnector::Fetch(fetch) => fetch, } } + + /// Spawns a JSON-RPC-backed [`ValidatorConnector::Fetch`] from the common backend + /// config, returning the connector plus the validator's `getinfo` response (used by + /// the backend to build its `ServiceMetadata`). + /// + /// Owns the `JsonRpSeeConnector` setup that previously lived in `FetchService::spawn`. + pub(crate) async fn spawn_fetch( + common: &CommonBackendConfig, + ) -> Result<(Self, GetInfoResponse), BlockchainSourceError> { + let fetcher = JsonRpSeeConnector::new_from_config_parts( + &common.validator_rpc_address, + common.validator_rpc_user.clone(), + common.validator_rpc_password.clone(), + common.validator_cookie_path.clone(), + ) + .await + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + + let info = fetcher + .get_info() + .await + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + + Ok((ValidatorConnector::Fetch(fetcher), info)) + } + + /// Spawns a `ReadStateService`-backed [`ValidatorConnector::State`] from the state + /// backend config, returning the connector plus the validator's `getinfo` response. + /// + /// Owns the JSON-RPC + Zebra chain-syncer setup that previously lived in + /// `StateService::spawn`: builds the mempool JSON-RPC fetcher, launches the syncer + /// and `ReadStateService`, then blocks until the syncer has caught up to the + /// validator's best chain tip (comparing tip *hash* as well as height so a reorg + /// mid-sync cannot report a false match). The syncer task handle is retained on the + /// `State` so [`ValidatorConnector::shutdown`] can abort it. + pub(crate) async fn spawn_state( + config: &StateServiceConfig, + ) -> Result<(Self, GetInfoResponse), BlockchainSourceError> { + let map_err = + |error: &dyn std::fmt::Display| BlockchainSourceError::Unrecoverable(error.to_string()); + + let rpc_client = JsonRpSeeConnector::new_from_config_parts( + &config.common.validator_rpc_address, + config.common.validator_rpc_user.clone(), + config.common.validator_rpc_password.clone(), + config.common.validator_cookie_path.clone(), + ) + .await + .map_err(|error| map_err(&error))?; + + let info = rpc_client + .get_info() + .await + .map_err(|error| map_err(&error))?; + + info!( + grpc_address = %config.validator_grpc_address, + "Launching Chain Syncer" + ); + let (mut read_state_service, _latest_chain_tip, chain_tip_change, sync_task_handle) = + init_read_state_with_syncer( + config.validator_state_config.clone(), + &config.common.network.to_zebra_network(), + config.validator_grpc_address, + ) + .await + .map_err(|error| map_err(&error))? + .map_err(|error| map_err(&error))?; + + info!("Chain syncer launched"); + + // Wait for ReadStateService to catch up to the validator's best chain tip. + // Height alone is insufficient during reorgs: the same height can refer to + // different blocks until JSON-RPC and ReadStateService agree on tip hash. + loop { + let blockchain_info = rpc_client + .get_blockchain_info() + .await + .map_err(|error| map_err(&error))?; + let server_height = blockchain_info.blocks; + let server_tip_hash = blockchain_info.best_block_hash; + + let syncer_response = read_state_service + .ready() + .and_then(|service| service.call(ReadRequest::Tip)) + .await + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + let (syncer_height, syncer_tip_hash) = expected_read_response!(syncer_response, Tip) + .ok_or_else(|| { + BlockchainSourceError::Unrecoverable("no blocks in chain".to_string()) + })?; + + if server_height == syncer_height && server_tip_hash == syncer_tip_hash { + info!( + height = syncer_height.0, + tip_hash = %syncer_tip_hash, + "ReadStateService synced with Zebra" + ); + break; + } else { + info!( + syncer_height = syncer_height.0, + validator_height = server_height.0, + syncer_tip_hash = %syncer_tip_hash, + validator_tip_hash = %server_tip_hash, + "ReadStateService syncing with Zebra" + ); + tokio::time::sleep(std::time::Duration::from_millis(1000)).await; + continue; + } + } + + let source = ValidatorConnector::State(State { + read_state_service, + mempool_fetcher: rpc_client, + network: config.common.network, + chain_tip_change, + sync_task_handle: Some(Arc::new(sync_task_handle)), + }); + + Ok((source, info)) + } + + /// Watches the Zebra syncer's chain-tip changes, when this connector owns one. + /// + /// `Some` for the `State` variant (which drives its own syncer); `None` for the + /// JSON-RPC `Fetch` variant, which has no local tip-change stream. + pub(crate) fn chain_tip_change(&self) -> Option { + match self { + ValidatorConnector::State(state) => Some(state.chain_tip_change.clone()), + ValidatorConnector::Fetch(_) => None, + } + } + + /// The backing [`ReadStateService`], when this connector is `State`-backed. + /// + /// Test-only escape hatch: live tests recompute expected chain data directly off + /// the `ReadStateService`. Production code goes through the `ChainIndex` API. + #[cfg(feature = "test_dependencies")] + pub(crate) fn read_state_service(&self) -> Option<&ReadStateService> { + match self { + ValidatorConnector::State(state) => Some(&state.read_state_service), + ValidatorConnector::Fetch(_) => None, + } + } } impl BlockchainSource for ValidatorConnector { @@ -345,7 +501,7 @@ impl BlockchainSource for ValidatorConnector { ValidatorConnector::State(State { read_state_service, mempool_fetcher, - network: _, + .. }) => { // Check state for transaction let mut read_state_service = read_state_service.clone(); @@ -502,7 +658,7 @@ impl BlockchainSource for ValidatorConnector { ValidatorConnector::State(State { read_state_service, mempool_fetcher, - network: _, + .. }) => { match read_state_service.best_tip() { Some((_height, hash)) => Ok(Some(hash)), @@ -544,7 +700,7 @@ impl BlockchainSource for ValidatorConnector { ValidatorConnector::State(State { read_state_service, mempool_fetcher, - network: _, + .. }) => { match read_state_service.best_tip() { Some((height, _hash)) => Ok(Some(height)), @@ -1269,9 +1425,7 @@ impl BlockchainSource for ValidatorConnector { > { match self { ValidatorConnector::State(State { - read_state_service, - mempool_fetcher: _, - network: _, + read_state_service, .. }) => { match read_state_service .clone() @@ -1293,6 +1447,19 @@ impl BlockchainSource for ValidatorConnector { ValidatorConnector::Fetch(_fetch) => Ok(None), } } + + fn shutdown(&self) { + // Only the `State` arm owns a long-lived resource: the Zebra chain-syncer + // task feeding the `ReadStateService`. Abort it so the backend drops cleanly. + // The `Fetch` arm holds only a stateless JSON-RPC client — nothing to tear down. + if let ValidatorConnector::State(State { + sync_task_handle: Some(handle), + .. + }) = self + { + handle.abort(); + } + } } impl State { From d8584becfb1d2739bdb714f52ed0fe5d1cca1d92 Mon Sep 17 00:00:00 2001 From: idky137 Date: Fri, 3 Jul 2026 13:08:19 +0100 Subject: [PATCH 028/134] initial dep bumps and compilation fixes --- Cargo.lock | 104 +++++++++++------- Cargo.toml | 51 +++------ packages/zaino-common/src/config/network.rs | 34 ++++++ .../zaino-fetch/src/jsonrpsee/response.rs | 62 ++++++++--- packages/zaino-state/src/backends/fetch.rs | 5 +- packages/zaino-state/src/backends/state.rs | 37 +++++-- .../src/chain_index/tests/vectors.rs | 1 + 7 files changed, 192 insertions(+), 102 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cd489ac56..c281ef95f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2981,9 +2981,9 @@ checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" [[package]] name = "orchard" -version = "0.14.0" +version = "0.15.0-pre.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a54f8d29bfb1e76a9d4e868a1a08cce2e57dd2bdc66232982822ad3114b91ab3" +checksum = "e8e277dd4b46f5d06deae3ffb8af1a951e8622368f028c2a4d6fe59339566403" dependencies = [ "aes", "bitvec", @@ -5068,7 +5068,8 @@ dependencies = [ [[package]] name = "tower-batch-control" version = "1.0.1" -source = "git+https://github.com/ZcashFoundation/zebra.git?rev=9a27f886a5bfb143f65d1712e912cef252426800#9a27f886a5bfb143f65d1712e912cef252426800" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e6cf52578f98b4da47335c26c4f883f7993b1a9b9d2f5420eb8dbfd5dd19a28" dependencies = [ "futures", "futures-core", @@ -5084,7 +5085,8 @@ dependencies = [ [[package]] name = "tower-fallback" version = "0.2.41" -source = "git+https://github.com/ZcashFoundation/zebra.git?rev=9a27f886a5bfb143f65d1712e912cef252426800#9a27f886a5bfb143f65d1712e912cef252426800" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4434e19ee996ee5c6aa42f11463a355138452592e5c5b5b73b6f0f19534556af" dependencies = [ "futures-core", "pin-project", @@ -6008,6 +6010,7 @@ dependencies = [ "tracing-subscriber", "tracing-tree", "zebra-chain", + "zingo_common_components", ] [[package]] @@ -6177,9 +6180,9 @@ dependencies = [ [[package]] name = "zcash_address" -version = "0.12.0" +version = "0.13.0-pre.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58342d0aaa8e2fa98849636f52800ac4bf020574c944c974742fc933db58cac2" +checksum = "8cf2918e73eff76388cda87695a6e7398f96a2e3383a0de7b77729a204ec00a0" dependencies = [ "bech32", "bs58", @@ -6202,9 +6205,9 @@ dependencies = [ [[package]] name = "zcash_history" -version = "0.4.0" +version = "0.5.0-pre.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fde17bf53792f9c756b313730da14880257d7661b5bfc69d0571c3a7c11a76d" +checksum = "82e8634d011026cb181cb67b2a412e601dc344a2827b9c576fe3a727fdebf441" dependencies = [ "blake2b_simd", "byteorder", @@ -6213,9 +6216,9 @@ dependencies = [ [[package]] name = "zcash_keys" -version = "0.14.0" +version = "0.15.0-pre.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fbcdfbb5c8edb247439d72a397abaae9b7dd14a1c070e7e4fc3536924f9065f" +checksum = "0a0bac3a9e5b0d954684ba1f07386d55b5ae4588b3ba2d93cad05ee09f3e0947" dependencies = [ "bech32", "blake2b_simd", @@ -6242,19 +6245,22 @@ dependencies = [ [[package]] name = "zcash_local_net" version = "0.6.0" -source = "git+https://github.com/zingolabs/infrastructure.git?branch=remove_unnecessary_deps#232cf9fb02f3de2c1c6bd0247c875e85296ea566" +source = "git+https://github.com/idky137/infrastructure.git?branch=feat%2Fironwood%2Fzaino#bd3e20ee868aea6483c3904e2ef3715469f6ae70" dependencies = [ + "getset", "hex", "json", "reqwest 0.12.28", - "serde", "serde_json", - "sha2 0.10.9", "tempfile", "thiserror 1.0.69", "tokio", "tracing", - "zingo-consensus", + "zcash_protocol", + "zebra-chain", + "zebra-node-services", + "zebra-rpc", + "zingo_common_components", "zingo_test_vectors", ] @@ -6273,9 +6279,9 @@ dependencies = [ [[package]] name = "zcash_primitives" -version = "0.28.0" +version = "0.29.0-pre.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c69e07f5eb3f682a6467b4b08ee4956f1acd1e886d70b21c4766953b3a1beba2" +checksum = "ba7bfe66975658f44dba87d535f9ebb9fb4c9c0c4fecca3f5c40cbe1583dece6" dependencies = [ "blake2b_simd", "block-buffer 0.11.0-rc.3", @@ -6304,9 +6310,9 @@ dependencies = [ [[package]] name = "zcash_proofs" -version = "0.28.0" +version = "0.29.0-pre.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3de6b0ca82e08a9d38b1121f87c5b180b5feac19fecba074cb582882210d2371" +checksum = "b39b90964ffe6bdc314368c7b849aaa6dcc2b495249a3bf1b9bfbdc92ba4e4f6" dependencies = [ "bellman", "blake2b_simd", @@ -6327,9 +6333,9 @@ dependencies = [ [[package]] name = "zcash_protocol" -version = "0.9.0" +version = "0.10.0-pre.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bec496a0bd62dae98c4b26f51c5dab112d0c5350bbc2ccfdfd05bb3454f714d" +checksum = "97f339a9801c7f70295732a5cf822346f194202cea6425fc2fc14a5af679b004" dependencies = [ "corez", "document-features", @@ -6366,9 +6372,9 @@ dependencies = [ [[package]] name = "zcash_transparent" -version = "0.8.0" +version = "0.9.0-pre.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15df1908b428d4edeb7c7caae5692e05e2e92e5c38007a40b20ac098efdffd96" +checksum = "4e941230f67056aad41c8fa9b926f9cc4d9d1a321f32e95c39c0b0e38e85f79b" dependencies = [ "bip32", "bs58", @@ -6391,8 +6397,9 @@ dependencies = [ [[package]] name = "zebra-chain" -version = "10.1.0" -source = "git+https://github.com/ZcashFoundation/zebra.git?rev=9a27f886a5bfb143f65d1712e912cef252426800#9a27f886a5bfb143f65d1712e912cef252426800" +version = "11.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "600a4c35ee81350384226f8178fab2b088e38d5e0a9f6cb0b1051caba30702d7" dependencies = [ "bech32", "bitflags 2.13.0", @@ -6459,8 +6466,9 @@ dependencies = [ [[package]] name = "zebra-consensus" -version = "9.0.1" -source = "git+https://github.com/ZcashFoundation/zebra.git?rev=9a27f886a5bfb143f65d1712e912cef252426800#9a27f886a5bfb143f65d1712e912cef252426800" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aa2b2678c4ce6d18172bad73a9b72b24a4b3a65af4c2a5e9120a1470c3fc402" dependencies = [ "bellman", "blake2b_simd", @@ -6501,8 +6509,9 @@ dependencies = [ [[package]] name = "zebra-network" -version = "9.0.0" -source = "git+https://github.com/ZcashFoundation/zebra.git?rev=9a27f886a5bfb143f65d1712e912cef252426800#9a27f886a5bfb143f65d1712e912cef252426800" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36b1e6ca4687bc8d128553ca04bb82cc38701c4dac7107f3e56717b39d4b18da" dependencies = [ "bitflags 2.13.0", "byteorder", @@ -6538,8 +6547,9 @@ dependencies = [ [[package]] name = "zebra-node-services" -version = "8.0.0" -source = "git+https://github.com/ZcashFoundation/zebra.git?rev=9a27f886a5bfb143f65d1712e912cef252426800#9a27f886a5bfb143f65d1712e912cef252426800" +version = "9.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4446db9b576c0de14ff91c7fd7b38a59d340c1969a1604787185a52b8b7f53a6" dependencies = [ "color-eyre", "jsonrpsee-types", @@ -6553,8 +6563,9 @@ dependencies = [ [[package]] name = "zebra-rpc" -version = "10.0.1" -source = "git+https://github.com/ZcashFoundation/zebra.git?rev=9a27f886a5bfb143f65d1712e912cef252426800#9a27f886a5bfb143f65d1712e912cef252426800" +version = "11.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb180542f1ffc0f66c20e1cd66c7741662053ccda588e3847a8ed2df8c83dae7" dependencies = [ "base64", "chrono", @@ -6585,6 +6596,7 @@ dependencies = [ "serde_with", "strum", "strum_macros", + "subtle", "thiserror 2.0.18", "tokio", "tokio-stream", @@ -6612,8 +6624,9 @@ dependencies = [ [[package]] name = "zebra-script" -version = "9.0.0" -source = "git+https://github.com/ZcashFoundation/zebra.git?rev=9a27f886a5bfb143f65d1712e912cef252426800#9a27f886a5bfb143f65d1712e912cef252426800" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8648c201e3dadb1c95022eb7605d528de4bf94b6c0cebf76537805849aee76c5" dependencies = [ "libzcash_script", "rand 0.8.6", @@ -6625,8 +6638,9 @@ dependencies = [ [[package]] name = "zebra-state" -version = "9.0.1" -source = "git+https://github.com/ZcashFoundation/zebra.git?rev=9a27f886a5bfb143f65d1712e912cef252426800#9a27f886a5bfb143f65d1712e912cef252426800" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b95b81181253a46885e3254cc5a20e4d82b5258116c9e43db5c8e9660c6b973" dependencies = [ "bincode", "chrono", @@ -6651,6 +6665,7 @@ dependencies = [ "sapling-crypto", "semver", "serde", + "serde-big-array", "tempfile", "thiserror 2.0.18", "tokio", @@ -6662,8 +6677,9 @@ dependencies = [ [[package]] name = "zebra-test" -version = "3.0.0" -source = "git+https://github.com/ZcashFoundation/zebra.git?rev=9a27f886a5bfb143f65d1712e912cef252426800#9a27f886a5bfb143f65d1712e912cef252426800" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93c3fbb37b81e2727172ae184582297ffa91016be660ddd594b4830b16d168c9" dependencies = [ "color-eyre", "futures", @@ -6782,14 +6798,18 @@ dependencies = [ ] [[package]] -name = "zingo-consensus" -version = "0.1.0" -source = "git+https://github.com/zingolabs/infrastructure.git?branch=remove_unnecessary_deps#232cf9fb02f3de2c1c6bd0247c875e85296ea566" +name = "zingo_common_components" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29f41ddb12ba9aa1fdbf4025fb54639f1fddd25c6e297574922b39fabfb730d4" +dependencies = [ + "hex", +] [[package]] name = "zingo_test_vectors" version = "0.0.1" -source = "git+https://github.com/zingolabs/infrastructure.git?branch=remove_unnecessary_deps#232cf9fb02f3de2c1c6bd0247c875e85296ea566" +source = "git+https://github.com/idky137/infrastructure.git?branch=feat%2Fironwood%2Fzaino#bd3e20ee868aea6483c3904e2ef3715469f6ae70" dependencies = [ "bip0039", ] diff --git a/Cargo.toml b/Cargo.toml index 492427a3e..d71abe1bf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,18 +38,23 @@ license = "Apache-2.0" [workspace.dependencies] +# Zingo + +zingo_common_components = "0.4" + # Librustzcash incrementalmerkletree = "0.8" -zcash_address = "0.12" -zcash_keys = "0.14" -zcash_protocol = "0.9" -zcash_primitives = "0.28" -zcash_transparent = "0.8" +zcash_address = "0.13.0-pre.0" +zcash_keys = "0.15.0-pre.0" +zcash_protocol = "0.10.0-pre.0" +zcash_primitives = "0.29.0-pre.0" +zcash_transparent = "0.9.0-pre.0" +zcash_client_backend = "0.24.0-pre.0" # Zebra -zebra-chain = "10.0" -zebra-state = "9.0" -zebra-rpc = "10.0" +zebra-chain = "11.0" +zebra-state = "10.0" +zebra-rpc = "11.0" # Runtime tokio = { version = "1.38", features = ["full"] } @@ -135,12 +140,10 @@ zaino-serve = { path = "packages/zaino-serve", version = "0.3.1", default-featur zaino-testutils = { path = "live-tests/zaino-testutils" } # Zingo-infra (validator launcher driving the live tests; not a prod dep). -# TEMPORARY (zingolabs/infrastructure#269): tracks the -# `remove_unnecessary_deps` branch (zingolabs/infrastructure#276), which -# empties zcash_local_net's dependency tables of the zebra and librustzcash -# stacks; the lock pins the resolved commit until `cargo update`. Restore the -# release tag once tagged upstream. -zcash_local_net = { git = "https://github.com/zingolabs/infrastructure.git", branch = "remove_unnecessary_deps" } +# TEMPORARY (zingolabs/infrastructure#269): pinned to an exact commit on the +# `add_client_support` branch so every machine resolves the same code; restore +# the release tag once tagged upstream. +zcash_local_net = { git = "https://github.com/idky137/infrastructure.git", branch = "feat/ironwood/zaino" } wire_serialized_transaction_test_data = "1.0.0" config = { version = "0.15", default-features = false, features = ["toml"] } @@ -156,23 +159,3 @@ opt-level = 3 debug = true [patch.crates-io] -#zcash_client_backend = { git = "https://github.com/zcash/librustzcash", tag = "zcash_client_backend-0.21.0"} -#zcash_address = { git = "https://github.com/zcash/librustzcash", tag = "zcash_client_backend-0.21.0"} -#zcash_keys = { git = "https://github.com/zcash/librustzcash", tag = "zcash_client_backend-0.21.0"} -#zcash_primitives = { git = "https://github.com/zcash/librustzcash", tag = "zcash_client_backend-0.21.0"} -#zcash_protocol = { git = "https://github.com/zcash/librustzcash", tag = "zcash_client_backend-0.21.0"} -#zcash_transparent = { git = "https://github.com/zcash/librustzcash", tag = "zcash_client_backend-0.21.0"} - -# TEMPORARY: zebra is pinned to an unreleased ZcashFoundation/zebra rev whose -# ReadRequest::NonFinalizedBlocksListener is a struct variant carrying -# `known_chain_tips` (its Cargo version is 9.0.1, but it is NOT the published -# crates.io 9.0.1 — that one is still a unit variant). Revert to a plain -# version requirement once the change is released. This is the single root -# workspace (the live-test crates were folded in; see docs/adr/0002), so the -# patch lives here once and every member — production and live-test alike — -# inherits it through the single lock; Cargo honours `[patch.crates-io]` only -# from the workspace root. See docs/updating_zebra_crates.md. -# Tracking: zaino#. -zebra-chain = { git = "https://github.com/ZcashFoundation/zebra.git", rev = "9a27f886a5bfb143f65d1712e912cef252426800" } -zebra-rpc = { git = "https://github.com/ZcashFoundation/zebra.git", rev = "9a27f886a5bfb143f65d1712e912cef252426800" } -zebra-state = { git = "https://github.com/ZcashFoundation/zebra.git", rev = "9a27f886a5bfb143f65d1712e912cef252426800" } diff --git a/packages/zaino-common/src/config/network.rs b/packages/zaino-common/src/config/network.rs index 23df9880b..5ddd78f49 100644 --- a/packages/zaino-common/src/config/network.rs +++ b/packages/zaino-common/src/config/network.rs @@ -16,6 +16,7 @@ pub const ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS: ActivationHeights = ActivationHeigh nu6: Some(2), nu6_1: Some(2), nu6_2: Some(2), + nu6_3: None, nu7: None, }; @@ -108,6 +109,9 @@ pub struct ActivationHeights { /// Activation height for `NU6.2` network upgrade. #[serde(rename = "NU6.2")] pub nu6_2: Option, + /// Activation height for `NU6.3` network upgrade. + #[serde(rename = "NU6.3")] + pub nu6_3: Option, /// Activation height for `NU7` network upgrade. #[serde(rename = "NU7")] pub nu7: Option, @@ -126,6 +130,7 @@ impl Default for ActivationHeights { nu6: Some(2), nu6_1: Some(2), nu6_2: Some(2), + nu6_3: None, nu7: None, } } @@ -144,6 +149,7 @@ impl From for ActivationHeights { nu6, nu6_1, nu6_2, + nu6_3, nu7, }: ConfiguredActivationHeights, ) -> Self { @@ -158,6 +164,7 @@ impl From for ActivationHeights { nu6, nu6_1, nu6_2, + nu6_3, nu7, } } @@ -175,6 +182,7 @@ impl From for ConfiguredActivationHeights { nu6, nu6_1, nu6_2, + nu6_3, nu7, }: ActivationHeights, ) -> Self { @@ -189,11 +197,31 @@ impl From for ConfiguredActivationHeights { nu6, nu6_1, nu6_2, + nu6_3, nu7, } } } +impl From for ActivationHeights { + fn from(activation_heights: zingo_common_components::protocol::ActivationHeights) -> Self { + ActivationHeights { + before_overwinter: activation_heights.overwinter(), + overwinter: activation_heights.overwinter(), + sapling: activation_heights.sapling(), + blossom: activation_heights.blossom(), + heartwood: activation_heights.heartwood(), + canopy: activation_heights.canopy(), + nu5: activation_heights.nu5(), + nu6: activation_heights.nu6(), + nu6_1: activation_heights.nu6_1(), + nu6_2: activation_heights.nu6_2(), + nu6_3: activation_heights.nu6_3(), + nu7: activation_heights.nu7(), + } + } +} + impl Network { /// Convert to Zebra's network type for internal use (alias for to_zebra_default). pub fn to_zebra_network(&self) -> zebra_chain::parameters::Network { @@ -213,6 +241,7 @@ impl Network { nu6: Some(1), nu6_1: None, nu6_2: None, + nu6_3: None, nu7: None, } } @@ -257,6 +286,7 @@ impl From for Network { nu6: None, nu6_1: None, nu6_2: None, + nu6_3: None, nu7: None, }; for (height, upgrade) in parameters.activation_heights().iter() { @@ -292,6 +322,9 @@ impl From for Network { zebra_chain::parameters::NetworkUpgrade::Nu6_2 => { activation_heights.nu6_2 = Some(height.0) } + zebra_chain::parameters::NetworkUpgrade::Nu6_3 => { + activation_heights.nu6_3 = Some(height.0) + } zebra_chain::parameters::NetworkUpgrade::Nu7 => { activation_heights.nu7 = Some(height.0) } @@ -341,6 +374,7 @@ mod tests { nu6: Some(1), nu6_1: Some(1), nu6_2: Some(2), + nu6_3: Some(500), nu7: Some(1000), }; diff --git a/packages/zaino-fetch/src/jsonrpsee/response.rs b/packages/zaino-fetch/src/jsonrpsee/response.rs index c90307213..b6876074d 100644 --- a/packages/zaino-fetch/src/jsonrpsee/response.rs +++ b/packages/zaino-fetch/src/jsonrpsee/response.rs @@ -361,7 +361,7 @@ pub struct GetBlockchainInfoResponse { /// Value pool balances #[serde(rename = "valuePools")] - value_pools: [ChainBalance; 5], + value_pools: [ChainBalance; 6], /// Branch IDs of the current and upcoming consensus rules pub consensus: zebra_rpc::methods::TipConsensusBranch, @@ -539,6 +539,7 @@ mod get_tx_out_set_info_tests { { "id": "sprout", "chainValue": 0.0, "chainValueZat": 0 }, { "id": "sapling", "chainValue": 0.0, "chainValueZat": 0 }, { "id": "orchard", "chainValue": 0.0, "chainValueZat": 0 }, + { "id": "ironwood", "chainValue": 0.0, "chainValueZat": 0 }, { "id": "deferred", "chainValue": 0.0, "chainValueZat": 0 } ], "consensus": { @@ -548,13 +549,8 @@ mod get_tx_out_set_info_tests { }"#; let parsed: GetBlockchainInfoResponse = serde_json::from_str(json).unwrap(); - let (name, activation_height, status) = parsed - .upgrades - .values() - .next() - .unwrap() - .clone() - .into_parts(); + let (name, activation_height, status) = + parsed.upgrades.values().next().unwrap().into_parts(); assert_eq!(name, zebra_chain::parameters::NetworkUpgrade::Nu6_2); assert_eq!(activation_height, zebra_chain::block::Height(2)); @@ -660,6 +656,9 @@ impl<'de> Deserialize<'de> for ChainBalance { "orchard" => Ok(ChainBalance(GetBlockchainInfoBalance::orchard( amount, None, /*TODO: handle optional delta*/ ))), + "ironwood" => Ok(ChainBalance(GetBlockchainInfoBalance::ironwood( + amount, None, /*TODO: handle optional delta*/ + ))), // TODO: Investigate source of undocument 'lockbox' value // that likely is intended to be 'deferred' "lockbox" | "deferred" => Ok(ChainBalance(GetBlockchainInfoBalance::deferred( @@ -904,6 +903,14 @@ pub struct OrchardTrees { size: u64, } +/// Ironwood note commitment tree information. +/// +/// Wrapper struct for zebra's IronwoodTrees +#[derive(Copy, Clone, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)] +pub struct IronwoodTrees { + size: u64, +} + /// Information about the sapling and orchard note commitment trees if any. /// /// Wrapper struct for zebra's GetBlockTrees @@ -913,6 +920,8 @@ pub struct GetBlockTrees { sapling: Option, #[serde(default)] orchard: Option, + #[serde(default)] + ironwood: Option, } impl GetBlockTrees { @@ -925,11 +934,16 @@ impl GetBlockTrees { pub fn orchard(&self) -> u64 { self.orchard.map_or(0, |o| o.size) } + + /// Returns ironwood data held by ['GetBlockTrees']. + pub fn ironwood(&self) -> u64 { + self.ironwood.map_or(0, |o| o.size) + } } impl From for zebra_rpc::methods::GetBlockTrees { fn from(val: GetBlockTrees) -> Self { - zebra_rpc::methods::GetBlockTrees::new(val.sapling(), val.orchard()) + zebra_rpc::methods::GetBlockTrees::new(val.sapling(), val.orchard(), val.ironwood()) } } @@ -1144,7 +1158,7 @@ pub struct BlockObject { /// Value pool balances /// #[serde(rename = "valuePools")] - value_pools: Option<[ChainBalance; 5]>, + value_pools: Option<[ChainBalance; 6]>, /// Information about the note commitment trees. pub trees: GetBlockTrees, @@ -1205,8 +1219,15 @@ impl TryFrom for zebra_rpc::methods::GetBlock { block.difficulty, block.chain_supply.map(|supply| supply.0), block.value_pools.map( - |[transparent, sprout, sapling, orchard, deferred]| { - [transparent.0, sprout.0, sapling.0, orchard.0, deferred.0] + |[transparent, sprout, sapling, orchard, ironwood, deferred]| { + [ + transparent.0, + sprout.0, + sapling.0, + orchard.0, + ironwood.0, + deferred.0, + ] }, ), block.trees.into(), @@ -1326,6 +1347,11 @@ pub struct GetTreestateResponse { /// (i.e. in regtest mode). #[serde(skip_serializing_if = "Option::is_none")] pub orchard: Option, + + /// A treestate containing an Ironwood note commitment tree, hex-encoded. Only present from + /// NU6.3, so that pre-NU6.3 responses are unchanged. + #[serde(skip_serializing_if = "Option::is_none")] + ironwood: Option, } /// Error type for the `get_treestate` RPC request. @@ -1367,6 +1393,8 @@ impl TryFrom for zebra_rpc::client::GetTreestateResponse { .clone() .unwrap_or_else(|| Treestate::new(Commitments::new(None, None))); + let ironwood = value.ironwood.clone(); + Ok(zebra_rpc::client::GetTreestateResponse::new( parsed_hash, zebra_chain::block::Height(height_u32), @@ -1375,6 +1403,7 @@ impl TryFrom for zebra_rpc::client::GetTreestateResponse { None, sapling, orchard, + ironwood, )) } } @@ -1440,10 +1469,7 @@ impl<'de> serde::Deserialize<'de> for GetTransactionResponse { } } - let confirmations = tx_value - .get("confirmations") - .and_then(|v| v.as_u64()) - .map(|v| v as u32); + let confirmations = tx_value.get("confirmations").and_then(|v| v.as_i64()); // if let Some(vin_value) = tx_value.get("vin") { // match serde_json::from_value::>(vin_value.clone()) { @@ -1468,6 +1494,7 @@ impl<'de> serde::Deserialize<'de> for GetTransactionResponse { let shielded_spends: Vec = tx_value["vShieldedSpend"]; let shielded_outputs: Vec = tx_value["vShieldedOutput"]; let orchard: Orchard = tx_value["orchard"]; + let ironwood: Orchard = tx_value["orchard"]; let value_balance: f64 = tx_value["valueBalance"]; let value_balance_zat: i64 = tx_value["valueBalanceZat"]; let size: i64 = tx_value["size"]; @@ -1527,6 +1554,8 @@ impl<'de> serde::Deserialize<'de> for GetTransactionResponse { // optional orchard, // optional + ironwood, + // optional value_balance, // optional value_balance_zat, @@ -1583,6 +1612,7 @@ impl From for zebra_rpc::methods::GetRawTransaction { None, None, obj.orchard().clone(), + obj.ironwood().clone(), obj.value_balance(), obj.value_balance_zat(), obj.size(), diff --git a/packages/zaino-state/src/backends/fetch.rs b/packages/zaino-state/src/backends/fetch.rs index 058004802..50a062735 100644 --- a/packages/zaino-state/src/backends/fetch.rs +++ b/packages/zaino-state/src/backends/fetch.rs @@ -757,11 +757,12 @@ impl ZcashIndexer for FetchServiceSubscriber { let (height, confirmations, block_hash, in_best_chain) = match best_chain_location { Some(types::BestChainLocation::Block(block_hash, height)) => { - let confirmations = snapshot + let confirmations: i64 = snapshot .max_serviceable_height() .0 .saturating_sub(height.0) - .saturating_add(1); + .saturating_add(1) + .into(); ( Some(zebra_chain::block::Height::from(height)), diff --git a/packages/zaino-state/src/backends/state.rs b/packages/zaino-state/src/backends/state.rs index 62945c1df..736fa78ca 100644 --- a/packages/zaino-state/src/backends/state.rs +++ b/packages/zaino-state/src/backends/state.rs @@ -731,6 +731,7 @@ impl StateServiceSubscriber { let state_2 = state.clone(); let state_3 = state.clone(); let state_4 = state.clone(); + let state_5 = state.clone(); let blockandsize_future = { let req = ReadRequest::BlockAndSize(hash_or_height); @@ -746,7 +747,16 @@ impl StateServiceSubscriber { .await } }; - + let ironwood_future = { + let req = ReadRequest::IronwoodTree(hash_or_height); + async move { + state_5 + .clone() + .ready() + .and_then(|service| service.call(req)) + .await + } + }; let block_info_future = { let req = ReadRequest::BlockInfo(hash_or_height); async move { @@ -757,9 +767,10 @@ impl StateServiceSubscriber { .await } }; - let (fullblock, orchard_tree_response, header, block_info) = futures::join!( + let (fullblock, orchard_tree_response, ironwood_tree_response, header, block_info) = futures::join!( blockandsize_future, orchard_future, + ironwood_future, StateServiceSubscriber::get_block_header_inner( &state_3, network, @@ -792,7 +803,7 @@ impl StateServiceSubscriber { TransactionObject::from_transaction( transaction.clone(), Some(header_obj.height()), - Some(header_obj.confirmations() as u32), + Some(header_obj.confirmations()), network, DateTime::::from_timestamp( header_obj.time(), @@ -840,8 +851,17 @@ impl StateServiceSubscriber { _otherwise => None, }; - let trees = - GetBlockTrees::new(header_obj.sapling_tree_size(), orchard_tree.count()); + let ironwood_tree_response = ironwood_tree_response?; + let ironwood_tree = expected_read_response!(ironwood_tree_response, IronwoodTree) + .ok_or(StateServiceError::RpcError( + RpcError::new_from_legacycode(LegacyCode::Misc, "missing ironwood tree"), + ))?; + + let trees = GetBlockTrees::new( + header_obj.sapling_tree_size(), + orchard_tree.count(), + ironwood_tree.count(), + ); let (chain_supply, value_pools) = ( GetBlockchainInfoBalance::chain_supply(*block_info.value_pools()), @@ -1552,7 +1572,7 @@ impl ZcashIndexer for StateServiceSubscriber { // Zebra displays transaction and block hashes in big-endian byte-order, // following the u256 convention set by Bitcoin and zcashd. match self.read_state_service.best_tip() { - Some(x) => return Ok(GetBlockHash::new(x.1)), + Some(x) => Ok(GetBlockHash::new(x.1)), None => { // try RPC if state read fails: Ok(self.rpc_client.get_best_blockhash().await?.into()) @@ -1789,11 +1809,12 @@ impl ZcashIndexer for StateServiceSubscriber { let (height, confirmations, block_hash, in_best_chain) = match best_chain_location { Some(BestChainLocation::Block(block_hash, height)) => { - let confirmations = snapshot + let confirmations: i64 = snapshot .max_serviceable_height() .0 .saturating_sub(height.0) - .saturating_add(1); + .saturating_add(1) + .into(); ( Some(zebra_chain::block::Height::from(height)), diff --git a/packages/zaino-state/src/chain_index/tests/vectors.rs b/packages/zaino-state/src/chain_index/tests/vectors.rs index b0b5115fc..cfc6e6948 100644 --- a/packages/zaino-state/src/chain_index/tests/vectors.rs +++ b/packages/zaino-state/src/chain_index/tests/vectors.rs @@ -111,6 +111,7 @@ pub(crate) fn indexed_block_chain( // see https://zips.z.cash/#nu6-1-candidate-zips for info on NU6.1 nu6_1: None, nu6_2: None, + nu6_3: None, nu7: None, } .into(), From ab4f83cbc8f64ff43d9871275587cb496345bd0a Mon Sep 17 00:00:00 2001 From: idky137 Date: Fri, 3 Jul 2026 13:33:54 +0100 Subject: [PATCH 029/134] updated zaino-fetch block / transaction parsing to use zcashserialise --- Cargo.lock | 2 - packages/zaino-fetch/Cargo.toml | 2 - packages/zaino-fetch/src/chain/block.rs | 354 +--- packages/zaino-fetch/src/chain/error.rs | 4 + packages/zaino-fetch/src/chain/transaction.rs | 1466 +++-------------- packages/zaino-fetch/src/chain/utils.rs | 207 +-- packages/zaino-fetch/src/lib.rs | 1 + packages/zaino-fetch/src/utils.rs | 19 + 8 files changed, 283 insertions(+), 1772 deletions(-) create mode 100644 packages/zaino-fetch/src/utils.rs diff --git a/Cargo.lock b/Cargo.lock index c281ef95f..fd6b01cdf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6018,7 +6018,6 @@ name = "zaino-fetch" version = "0.2.1" dependencies = [ "base64", - "byteorder", "derive_more", "hex", "http", @@ -6029,7 +6028,6 @@ dependencies = [ "reqwest 0.13.1", "serde", "serde_json", - "sha2 0.10.9", "thiserror 2.0.18", "tokio", "tonic", diff --git a/packages/zaino-fetch/Cargo.toml b/packages/zaino-fetch/Cargo.toml index 412f3fc31..6d912aa54 100644 --- a/packages/zaino-fetch/Cargo.toml +++ b/packages/zaino-fetch/Cargo.toml @@ -49,8 +49,6 @@ serde = { workspace = true, features = ["derive"] } hex = { workspace = true, features = ["serde"] } indexmap = { workspace = true, features = ["serde"] } base64 = { workspace = true } -byteorder = { workspace = true } -sha2 = { workspace = true } jsonrpsee-types = { workspace = true } derive_more = { workspace = true, features = ["from"] } diff --git a/packages/zaino-fetch/src/chain/block.rs b/packages/zaino-fetch/src/chain/block.rs index e7bc978fd..9598b553a 100644 --- a/packages/zaino-fetch/src/chain/block.rs +++ b/packages/zaino-fetch/src/chain/block.rs @@ -1,247 +1,78 @@ //! Block fetching and deserialization functionality. -use crate::chain::{ - error::ParseError, - transaction::FullTransaction, - utils::{read_bytes, read_i32, read_u32, read_zcash_script_i64, CompactSize, ParseFromSlice}, +use crate::{ + chain::{error::ParseError, transaction::FullTransaction}, + utils::ParseFromSlice, }; -use sha2::{Digest, Sha256}; -use std::io::Cursor; +use std::{io::Cursor, sync::Arc}; use zaino_proto::proto::{ compact_formats::{ChainMetadata, CompactBlock}, utils::PoolTypeFilter, }; - -/// A block header, containing metadata about a block. -/// -/// How are blocks chained together? They are chained together via the -/// backwards reference (previous header hash) present in the block -/// header. Each block points backwards to its parent, all the way -/// back to the genesis block (the first block in the blockchain). -#[derive(Debug, Clone)] -struct BlockHeaderData { - /// The block's version field. This is supposed to be `4`: - /// - /// > The current and only defined block version number for Zcash is 4. - /// - /// but this was not enforced by the consensus rules, and defective mining - /// software created blocks with other versions, so instead it's effectively - /// a free field. The only constraint is that it must be at least `4` when - /// interpreted as an `i32`. - /// - /// Size \[bytes\]: 4 - version: i32, - - /// The hash of the previous block, used to create a chain of blocks back to - /// the genesis block. - /// - /// This ensures no previous block can be changed without also changing this - /// block's header. - /// - /// Size \[bytes\]: 32 - hash_prev_block: Vec, - - /// The root of the Bitcoin-inherited transaction Merkle tree, binding the - /// block header to the transactions in the block. - /// - /// Note that because of a flaw in Bitcoin's design, the `merkle_root` does - /// not always precisely bind the contents of the block (CVE-2012-2459). It - /// is sometimes possible for an attacker to create multiple distinct sets of - /// transactions with the same Merkle root, although only one set will be - /// valid. - /// - /// Size \[bytes\]: 32 - hash_merkle_root: Vec, - - /// \[Pre-Sapling\] A reserved field which should be ignored. - /// \[Sapling onward\] The root LEBS2OSP_256(rt) of the Sapling note - /// commitment tree corresponding to the final Sapling treestate of this - /// block. - /// - /// Size \[bytes\]: 32 - hash_final_sapling_root: Vec, - - /// The block timestamp is a Unix epoch time (UTC) when the miner - /// started hashing the header (according to the miner). - /// - /// Size \[bytes\]: 4 - time: u32, - - /// An encoded version of the target threshold this block's header - /// hash must be less than or equal to, in the same nBits format - /// used by Bitcoin. - /// - /// For a block at block height `height`, bits MUST be equal to - /// `ThresholdBits(height)`. - /// - /// [Bitcoin-nBits](https://bitcoin.org/en/developer-reference#target-nbits) - /// - /// Size \[bytes\]: 4 - n_bits_bytes: Vec, - - /// An arbitrary field that miners can change to modify the header - /// hash in order to produce a hash less than or equal to the - /// target threshold. - /// - /// Size \[bytes\]: 32 - nonce: Vec, - - /// The Equihash solution. - /// - /// Size \[bytes\]: CompactLength - solution: Vec, -} - -impl ParseFromSlice for BlockHeaderData { - fn parse_from_slice( - data: &[u8], - txid: Option>>, - tx_version: Option, - ) -> Result<(&[u8], Self), ParseError> { - if txid.is_some() { - return Err(ParseError::InvalidData( - "txid must be None for BlockHeaderData::parse_from_slice".to_string(), - )); - } - if tx_version.is_some() { - return Err(ParseError::InvalidData( - "tx_version must be None for BlockHeaderData::parse_from_slice".to_string(), - )); - } - let mut cursor = Cursor::new(data); - - let version = read_i32(&mut cursor, "Error reading BlockHeaderData::version")?; - let hash_prev_block = read_bytes( - &mut cursor, - 32, - "Error reading BlockHeaderData::hash_prev_block", - )?; - let hash_merkle_root = read_bytes( - &mut cursor, - 32, - "Error reading BlockHeaderData::hash_merkle_root", - )?; - let hash_final_sapling_root = read_bytes( - &mut cursor, - 32, - "Error reading BlockHeaderData::hash_final_sapling_root", - )?; - let time = read_u32(&mut cursor, "Error reading BlockHeaderData::time")?; - let n_bits_bytes = read_bytes( - &mut cursor, - 4, - "Error reading BlockHeaderData::n_bits_bytes", - )?; - let nonce = read_bytes(&mut cursor, 32, "Error reading BlockHeaderData::nonce")?; - - let solution = { - let compact_length = CompactSize::read(&mut cursor)?; - read_bytes( - &mut cursor, - compact_length as usize, - "Error reading BlockHeaderData::solution", - )? - }; - - Ok(( - &data[cursor.position() as usize..], - BlockHeaderData { - version, - hash_prev_block, - hash_merkle_root, - hash_final_sapling_root, - time, - n_bits_bytes, - nonce, - solution, - }, - )) - } -} - -impl BlockHeaderData { - /// Serializes the block header into a byte vector. - fn to_binary(&self) -> Result, ParseError> { - let mut buffer = Vec::new(); - - buffer.extend(&self.version.to_le_bytes()); - buffer.extend(&self.hash_prev_block); - buffer.extend(&self.hash_merkle_root); - buffer.extend(&self.hash_final_sapling_root); - buffer.extend(&self.time.to_le_bytes()); - buffer.extend(&self.n_bits_bytes); - buffer.extend(&self.nonce); - let mut solution_compact_size = Vec::new(); - CompactSize::write(&mut solution_compact_size, self.solution.len())?; - buffer.extend(solution_compact_size); - buffer.extend(&self.solution); - - Ok(buffer) - } - - /// Extracts the block hash from the block header. - fn get_hash(&self) -> Result, ParseError> { - let serialized_header = self.to_binary()?; - - let mut hasher = Sha256::new(); - hasher.update(&serialized_header); - let digest = hasher.finalize_reset(); - hasher.update(digest); - let final_digest = hasher.finalize(); - - Ok(final_digest.to_vec()) - } -} +use zebra_chain::serialization::{ZcashDeserialize as _, ZcashSerialize as _}; /// Complete block header. #[derive(Debug, Clone)] pub struct FullBlockHeader { - /// Block header data. - raw_block_header: BlockHeaderData, - - /// Hash of the current block. + header: Arc, + raw_bytes: Vec, cached_hash: Vec, } impl FullBlockHeader { + fn from_zebra(header: Arc) -> Result { + let raw_bytes = header.zcash_serialize_to_vec()?; + let cached_hash = header.hash().0.to_vec(); + + Ok(Self { + header, + raw_bytes, + cached_hash, + }) + } + /// Returns the Zcash block version. pub fn version(&self) -> i32 { - self.raw_block_header.version + i32::from_le_bytes(self.header.version.to_le_bytes()) } /// Returns The hash of the previous block. pub fn hash_prev_block(&self) -> Vec { - self.raw_block_header.hash_prev_block.clone() + self.header.previous_block_hash.0.to_vec() } /// Returns the root of the Bitcoin-inherited transaction Merkle tree. pub fn hash_merkle_root(&self) -> Vec { - self.raw_block_header.hash_merkle_root.clone() + self.header.merkle_root.0.to_vec() } /// Returns the final sapling root of the block. pub fn final_sapling_root(&self) -> Vec { - self.raw_block_header.hash_final_sapling_root.clone() + self.header.commitment_bytes.to_vec() } /// Returns the time when the miner started hashing the header (according to the miner). pub fn time(&self) -> u32 { - self.raw_block_header.time + u32::try_from(self.header.time.timestamp()) + .expect("deserialized block header timestamps fit in u32") } /// Returns an encoded version of the target threshold. pub fn n_bits_bytes(&self) -> Vec { - self.raw_block_header.n_bits_bytes.clone() + self.raw_bytes[104..108].to_vec() } /// Returns the block's nonce. pub fn nonce(&self) -> Vec { - self.raw_block_header.nonce.clone() + self.header.nonce.to_vec() } /// Returns the block's Equihash solution. pub fn solution(&self) -> Vec { - self.raw_block_header.solution.clone() + match &self.header.solution { + zebra_chain::work::equihash::Solution::Common(solution) => solution.to_vec(), + zebra_chain::work::equihash::Solution::Regtest(solution) => solution.to_vec(), + } } /// Returns the Hash of the current block. @@ -254,8 +85,6 @@ impl FullBlockHeader { #[derive(Debug, Clone)] pub struct FullBlock { /// The block header, containing block metadata. - /// - /// Size \[bytes\]: 140+CompactLength hdr: FullBlockHeader, /// The block transactions. @@ -271,74 +100,52 @@ impl ParseFromSlice for FullBlock { txid: Option>>, tx_version: Option, ) -> Result<(&[u8], Self), ParseError> { - let txid = txid.ok_or_else(|| { - ParseError::InvalidData("txid must be used for FullBlock::parse_from_slice".to_string()) - })?; if tx_version.is_some() { return Err(ParseError::InvalidData( "tx_version must be None for FullBlock::parse_from_slice".to_string(), )); } + let mut cursor = Cursor::new(data); + let block = zebra_chain::block::Block::zcash_deserialize(&mut cursor)?; + let consumed = usize::try_from(cursor.position())?; + let txids = txid.unwrap_or_default(); - let (remaining_data, block_header_data) = - BlockHeaderData::parse_from_slice(&data[cursor.position() as usize..], None, None)?; - cursor.set_position(data.len() as u64 - remaining_data.len() as u64); - let tx_count = CompactSize::read(&mut cursor)?; - if txid.len() != tx_count as usize { + if !txids.is_empty() && txids.len() != block.transactions.len() { return Err(ParseError::InvalidData(format!( "number of txids ({}) does not match tx_count ({})", - txid.len(), - tx_count + txids.len(), + block.transactions.len() ))); } - let mut transactions = Vec::with_capacity(tx_count as usize); - let mut remaining_data = &data[cursor.position() as usize..]; - for txid_item in txid.iter() { - if remaining_data.is_empty() { - return Err(ParseError::InvalidData( - "parsing block transactions: not enough data for transaction.".to_string(), - )); - } - let (new_remaining_data, tx) = FullTransaction::parse_from_slice( - &data[cursor.position() as usize..], - Some(vec![txid_item.clone()]), - None, - )?; - transactions.push(tx); - remaining_data = new_remaining_data; - cursor.set_position(data.len() as u64 - remaining_data.len() as u64); - } - let block_height = Self::get_block_height(&transactions)?; - let block_hash = block_header_data.get_hash()?; - Ok(( - remaining_data, - FullBlock { - hdr: FullBlockHeader { - raw_block_header: block_header_data, - cached_hash: block_hash, - }, - vtx: transactions, - height: block_height, - }, - )) + let hdr = FullBlockHeader::from_zebra(block.header.clone())?; + let vtx = block + .transactions + .iter() + .cloned() + .enumerate() + .map(|(index, transaction)| { + FullTransaction::from_zebra(transaction, txids.get(index).cloned()) + }) + .collect::, _>>()?; + let height = block + .coinbase_height() + .map(|height| i32::try_from(height.0)) + .transpose()? + .ok_or_else(|| ParseError::InvalidData("missing block coinbase height".to_string()))?; + + Ok((&data[consumed..], Self { hdr, vtx, height })) } } -/// Genesis block special case. -/// -/// From LightWalletD: -/// see . -const GENESIS_TARGET_DIFFICULTY: u32 = 520617983; - impl FullBlock { /// Returns the full block header. pub fn header(&self) -> FullBlockHeader { self.hdr.clone() } - /// Returns the transactions held in the block. + /// Returns the transactions held in the block. pub fn transactions(&self) -> Vec { self.vtx.clone() } @@ -350,7 +157,7 @@ impl FullBlock { /// Returns the Orchard `authDataRoot` of the block, taken from the coinbase transaction's anchorOrchard field. /// - /// If the coinbase transaction is v5 and includes an Orchard bundle, this is the root of the Orchard commitment tree + /// If the coinbase transaction includes an Orchard bundle, this is the root of the Orchard commitment tree /// after applying all Orchard actions in the block. /// /// Returns `Some(Vec)` if present, else `None`. @@ -365,7 +172,7 @@ impl FullBlock { return Err(ParseError::InvalidData(format!( "Error decoding full block - {} bytes of Remaining data. Compact Block Created: ({:?})", remaining_data.len(), - full_block.into_compact_block(0, 0, PoolTypeFilter::includes_all()) + full_block.clone().into_compact_block(0, 0, PoolTypeFilter::includes_all()) ))); } Ok(full_block) @@ -384,10 +191,9 @@ impl FullBlock { .vtx .into_iter() .enumerate() - .filter_map(|(index, tx)| { - match tx.to_compact_tx(Some(index as u64), &pool_types) { + .filter_map( + |(index, tx)| match tx.to_compact_tx(Some(index as u64), &pool_types) { Ok(compact_tx) => { - // Omit transactions that have no elements in any requested pool type. if !compact_tx.vin.is_empty() || !compact_tx.vout.is_empty() || !compact_tx.spends.is_empty() @@ -400,29 +206,23 @@ impl FullBlock { } } Err(parse_error) => Some(Err(parse_error)), - } - }) + }, + ) .collect::, _>>()?; - // NOTE: LightWalletD doesnt return a compact block header, however this could be used to return data if useful. - // let header = self.hdr.raw_block_header.to_binary()?; - let header = Vec::new(); - - let compact_block = CompactBlock { + Ok(CompactBlock { proto_version: 1, height: self.height as u64, hash: self.hdr.cached_hash.clone(), - prev_hash: self.hdr.raw_block_header.hash_prev_block.clone(), - time: self.hdr.raw_block_header.time, - header, + prev_hash: self.hdr.hash_prev_block(), + time: self.hdr.time(), + header: Vec::new(), vtx, chain_metadata: Some(ChainMetadata { sapling_commitment_tree_size, orchard_commitment_tree_size, }), - }; - - Ok(compact_block) + }) } #[deprecated] @@ -438,26 +238,4 @@ impl FullBlock { PoolTypeFilter::default(), ) } - - /// Extracts the block height from the coinbase transaction. - fn get_block_height(transactions: &[FullTransaction]) -> Result { - let transparent_inputs = transactions[0].transparent_inputs(); - let (_, _, script_sig) = transparent_inputs[0].clone(); - let coinbase_script = script_sig.as_slice(); - - let mut cursor = Cursor::new(coinbase_script); - - let height_num: i64 = read_zcash_script_i64(&mut cursor)?; - if height_num < 0 { - return Ok(-1); - } - if height_num > i64::from(u32::MAX) { - return Ok(-1); - } - if (height_num as u32) == GENESIS_TARGET_DIFFICULTY { - return Ok(0); - } - - Ok(height_num as i32) - } } diff --git a/packages/zaino-fetch/src/chain/error.rs b/packages/zaino-fetch/src/chain/error.rs index 2d1d6ac19..58575a85b 100644 --- a/packages/zaino-fetch/src/chain/error.rs +++ b/packages/zaino-fetch/src/chain/error.rs @@ -27,6 +27,10 @@ pub enum ParseError { #[error("Prost Decode Error: {0}")] ProstDecodeError(#[from] prost::DecodeError), + /// Zebra consensus serialization error. + #[error("Zcash serialization error: {0}")] + SerializationError(#[from] zebra_chain::serialization::SerializationError), + /// Integer conversion error. #[error("Integer conversion error: {0}")] TryFromIntError(#[from] std::num::TryFromIntError), diff --git a/packages/zaino-fetch/src/chain/transaction.rs b/packages/zaino-fetch/src/chain/transaction.rs index 81586349c..b26f76c3d 100644 --- a/packages/zaino-fetch/src/chain/transaction.rs +++ b/packages/zaino-fetch/src/chain/transaction.rs @@ -1,10 +1,7 @@ //! Transaction fetching and deserialization functionality. -use crate::chain::{ - error::ParseError, - utils::{read_bytes, read_i64, read_u32, read_u64, skip_bytes, CompactSize, ParseFromSlice}, -}; -use std::io::Cursor; +use crate::{chain::error::ParseError, utils::ParseFromSlice}; +use std::{io::Cursor, sync::Arc}; use zaino_proto::proto::{ compact_formats::{ CompactOrchardAction, CompactSaplingOutput, CompactSaplingSpend, CompactTx, CompactTxIn, @@ -12,933 +9,22 @@ use zaino_proto::proto::{ }, utils::PoolTypeFilter, }; - -/// Txin format as described in -#[derive(Debug, Clone)] -pub struct TxIn { - // PrevTxHash - Size\[bytes\]: 32 - prev_txid: Vec, - // PrevTxOutIndex - Size\[bytes\]: 4 - prev_index: u32, - /// CompactSize-prefixed, could be a pubkey or a script - /// - /// Size\[bytes\]: CompactSize - script_sig: Vec, - // SequenceNumber \[IGNORED\] - Size\[bytes\]: 4 -} - -impl TxIn { - fn into_inner(self) -> (Vec, u32, Vec) { - (self.prev_txid, self.prev_index, self.script_sig) - } - - /// Returns `true` if this `OutPoint` is "null" in the Bitcoin sense: it has txid set to - /// all-zeroes and output index set to `u32::MAX`. - fn is_null(&self) -> bool { - self.prev_txid.as_slice() == [0u8; 32] && self.prev_index == u32::MAX - } -} - -impl ParseFromSlice for TxIn { - fn parse_from_slice( - data: &[u8], - txid: Option>>, - tx_version: Option, - ) -> Result<(&[u8], Self), ParseError> { - if txid.is_some() { - return Err(ParseError::InvalidData( - "txid must be None for TxIn::parse_from_slice".to_string(), - )); - } - if tx_version.is_some() { - return Err(ParseError::InvalidData( - "tx_version must be None for TxIn::parse_from_slice".to_string(), - )); - } - let mut cursor = Cursor::new(data); - - let prev_txid = read_bytes(&mut cursor, 32, "Error reading TxIn::PrevTxHash")?; - let prev_index = read_u32(&mut cursor, "Error reading TxIn::PrevTxOutIndex")?; - let script_sig = { - let compact_length = CompactSize::read(&mut cursor)?; - read_bytes( - &mut cursor, - compact_length as usize, - "Error reading TxIn::ScriptSig", - )? - }; - skip_bytes(&mut cursor, 4, "Error skipping TxIn::SequenceNumber")?; - - Ok(( - &data[cursor.position() as usize..], - TxIn { - prev_txid, - prev_index, - script_sig, - }, - )) - } -} - -/// Txout format as described in -#[derive(Debug, Clone)] -pub struct TxOut { - /// Non-negative int giving the number of zatoshis to be transferred - /// - /// Size\[bytes\]: 8 - value: u64, - // Script - Size\[bytes\]: CompactSize - script_hash: Vec, -} - -impl TxOut { - fn into_inner(self) -> (u64, Vec) { - (self.value, self.script_hash) - } -} - -impl ParseFromSlice for TxOut { - fn parse_from_slice( - data: &[u8], - txid: Option>>, - tx_version: Option, - ) -> Result<(&[u8], Self), ParseError> { - if txid.is_some() { - return Err(ParseError::InvalidData( - "txid must be None for TxOut::parse_from_slice".to_string(), - )); - } - if tx_version.is_some() { - return Err(ParseError::InvalidData( - "tx_version must be None for TxOut::parse_from_slice".to_string(), - )); - } - let mut cursor = Cursor::new(data); - - let value = read_u64(&mut cursor, "Error TxOut::reading Value")?; - let script_hash = { - let compact_length = CompactSize::read(&mut cursor)?; - read_bytes( - &mut cursor, - compact_length as usize, - "Error reading TxOut::ScriptHash", - )? - }; - - Ok(( - &data[cursor.position() as usize..], - TxOut { script_hash, value }, - )) - } -} - -#[allow(clippy::type_complexity)] -fn parse_transparent(data: &[u8]) -> Result<(&[u8], Vec, Vec), ParseError> { - let mut cursor = Cursor::new(data); - - let tx_in_count = CompactSize::read(&mut cursor)?; - let mut tx_ins = Vec::with_capacity(tx_in_count as usize); - for _ in 0..tx_in_count { - let (remaining_data, tx_in) = - TxIn::parse_from_slice(&data[cursor.position() as usize..], None, None)?; - tx_ins.push(tx_in); - cursor.set_position(data.len() as u64 - remaining_data.len() as u64); - } - let tx_out_count = CompactSize::read(&mut cursor)?; - let mut tx_outs = Vec::with_capacity(tx_out_count as usize); - for _ in 0..tx_out_count { - let (remaining_data, tx_out) = - TxOut::parse_from_slice(&data[cursor.position() as usize..], None, None)?; - tx_outs.push(tx_out); - cursor.set_position(data.len() as u64 - remaining_data.len() as u64); - } - - Ok((&data[cursor.position() as usize..], tx_ins, tx_outs)) -} - -/// Spend is a Sapling Spend Description as described in 7.3 of the Zcash -/// protocol specification. -#[derive(Debug, Clone)] -pub struct Spend { - // Cv \[IGNORED\] - Size\[bytes\]: 32 - // Anchor \[IGNORED\] - Size\[bytes\]: 32 - /// A nullifier to a sapling note. - /// - /// Size\[bytes\]: 32 - nullifier: Vec, - // Rk \[IGNORED\] - Size\[bytes\]: 32 - // Zkproof \[IGNORED\] - Size\[bytes\]: 192 - // SpendAuthSig \[IGNORED\] - Size\[bytes\]: 64 -} - -impl Spend { - fn into_inner(self) -> Vec { - self.nullifier - } -} - -impl ParseFromSlice for Spend { - fn parse_from_slice( - data: &[u8], - txid: Option>>, - tx_version: Option, - ) -> Result<(&[u8], Self), ParseError> { - if txid.is_some() { - return Err(ParseError::InvalidData( - "txid must be None for Spend::parse_from_slice".to_string(), - )); - } - let tx_version = tx_version.ok_or_else(|| { - ParseError::InvalidData( - "tx_version must be used for Spend::parse_from_slice".to_string(), - ) - })?; - let mut cursor = Cursor::new(data); - - skip_bytes(&mut cursor, 32, "Error skipping Spend::Cv")?; - if tx_version <= 4 { - skip_bytes(&mut cursor, 32, "Error skipping Spend::Anchor")?; - } - let nullifier = read_bytes(&mut cursor, 32, "Error reading Spend::nullifier")?; - skip_bytes(&mut cursor, 32, "Error skipping Spend::Rk")?; - if tx_version <= 4 { - skip_bytes(&mut cursor, 192, "Error skipping Spend::Zkproof")?; - skip_bytes(&mut cursor, 64, "Error skipping Spend::SpendAuthSig")?; - } - - Ok((&data[cursor.position() as usize..], Spend { nullifier })) - } -} - -/// output is a Sapling Output Description as described in section 7.4 of the -/// Zcash protocol spec. -#[derive(Debug, Clone)] -pub struct Output { - // Cv \[IGNORED\] - Size\[bytes\]: 32 - /// U-coordinate of the note commitment, derived from the note's value, recipient, and a - /// random value. - /// - /// Size\[bytes\]: 32 - cmu: Vec, - /// Ephemeral public key for Diffie-Hellman key exchange. - /// - /// Size\[bytes\]: 32 - ephemeral_key: Vec, - /// Encrypted transaction details including value transferred and an optional memo. - /// - /// Size\[bytes\]: 580 - enc_ciphertext: Vec, - // OutCiphertext \[IGNORED\] - Size\[bytes\]: 80 - // Zkproof \[IGNORED\] - Size\[bytes\]: 192 -} - -impl Output { - fn into_parts(self) -> (Vec, Vec, Vec) { - (self.cmu, self.ephemeral_key, self.enc_ciphertext) - } -} - -impl ParseFromSlice for Output { - fn parse_from_slice( - data: &[u8], - txid: Option>>, - tx_version: Option, - ) -> Result<(&[u8], Self), ParseError> { - if txid.is_some() { - return Err(ParseError::InvalidData( - "txid must be None for Output::parse_from_slice".to_string(), - )); - } - let tx_version = tx_version.ok_or_else(|| { - ParseError::InvalidData( - "tx_version must be used for Output::parse_from_slice".to_string(), - ) - })?; - let mut cursor = Cursor::new(data); - - skip_bytes(&mut cursor, 32, "Error skipping Output::Cv")?; - let cmu = read_bytes(&mut cursor, 32, "Error reading Output::cmu")?; - let ephemeral_key = read_bytes(&mut cursor, 32, "Error reading Output::ephemeral_key")?; - let enc_ciphertext = read_bytes(&mut cursor, 580, "Error reading Output::enc_ciphertext")?; - skip_bytes(&mut cursor, 80, "Error skipping Output::OutCiphertext")?; - if tx_version <= 4 { - skip_bytes(&mut cursor, 192, "Error skipping Output::Zkproof")?; - } - - Ok(( - &data[cursor.position() as usize..], - Output { - cmu, - ephemeral_key, - enc_ciphertext, - }, - )) - } -} - -/// joinSplit is a JoinSplit description as described in 7.2 of the Zcash -/// protocol spec. Its exact contents differ by transaction version and network -/// upgrade level. Only version 4 is supported, no need for proofPHGR13. -/// -/// NOTE: Legacy, no longer used but included for consistency. -#[derive(Debug, Clone)] -struct JoinSplit { - //vpubOld \[IGNORED\] - Size\[bytes\]: 8 - //vpubNew \[IGNORED\] - Size\[bytes\]: 8 - //anchor \[IGNORED\] - Size\[bytes\]: 32 - //nullifiers \[IGNORED\] - Size\[bytes\]: 64/32 - //commitments \[IGNORED\] - Size\[bytes\]: 64/32 - //ephemeralKey \[IGNORED\] - Size\[bytes\]: 32 - //randomSeed \[IGNORED\] - Size\[bytes\]: 32 - //vmacs \[IGNORED\] - Size\[bytes\]: 64/32 - //proofGroth16 \[IGNORED\] - Size\[bytes\]: 192 - //encCiphertexts \[IGNORED\] - Size\[bytes\]: 1202 -} - -impl ParseFromSlice for JoinSplit { - fn parse_from_slice( - data: &[u8], - txid: Option>>, - tx_version: Option, - ) -> Result<(&[u8], Self), ParseError> { - if txid.is_some() { - return Err(ParseError::InvalidData( - "txid must be None for JoinSplit::parse_from_slice".to_string(), - )); - } - let proof_size = match tx_version { - Some(2) | Some(3) => 296, // BCTV14 proof for v2/v3 transactions - Some(4) => 192, // Groth16 proof for v4 transactions - None => 192, // Default to Groth16 for unknown versions - _ => { - return Err(ParseError::InvalidData(format!( - "Unsupported tx_version {tx_version:?} for JoinSplit::parse_from_slice" - ))) - } - }; - let mut cursor = Cursor::new(data); - - skip_bytes(&mut cursor, 8, "Error skipping JoinSplit::vpubOld")?; - skip_bytes(&mut cursor, 8, "Error skipping JoinSplit::vpubNew")?; - skip_bytes(&mut cursor, 32, "Error skipping JoinSplit::anchor")?; - skip_bytes(&mut cursor, 64, "Error skipping JoinSplit::nullifiers")?; - skip_bytes(&mut cursor, 64, "Error skipping JoinSplit::commitments")?; - skip_bytes(&mut cursor, 32, "Error skipping JoinSplit::ephemeralKey")?; - skip_bytes(&mut cursor, 32, "Error skipping JoinSplit::randomSeed")?; - skip_bytes(&mut cursor, 64, "Error skipping JoinSplit::vmacs")?; - skip_bytes( - &mut cursor, - proof_size, - &format!("Error skipping JoinSplit::proof (size {proof_size})"), - )?; - skip_bytes( - &mut cursor, - 1202, - "Error skipping JoinSplit::encCiphertexts", - )?; - - Ok((&data[cursor.position() as usize..], JoinSplit {})) - } -} - -/// An Orchard action. -#[derive(Debug, Clone)] -struct Action { - // Cv \[IGNORED\] - Size\[bytes\]: 32 - /// A nullifier to a orchard note. - /// - /// Size\[bytes\]: 32 - nullifier: Vec, - // Rk \[IGNORED\] - Size\[bytes\]: 32 - /// X-coordinate of the commitment to the note. - /// - /// Size\[bytes\]: 32 - cmx: Vec, - /// Ephemeral public key. - /// - /// Size\[bytes\]: 32 - ephemeral_key: Vec, - /// Encrypted details of the new note, including its value and recipient's data. - /// - /// Size\[bytes\]: 580 - enc_ciphertext: Vec, - // OutCiphertext \[IGNORED\] - Size\[bytes\]: 80 -} - -impl Action { - fn into_parts(self) -> (Vec, Vec, Vec, Vec) { - ( - self.nullifier, - self.cmx, - self.ephemeral_key, - self.enc_ciphertext, - ) - } -} - -impl ParseFromSlice for Action { - fn parse_from_slice( - data: &[u8], - txid: Option>>, - tx_version: Option, - ) -> Result<(&[u8], Self), ParseError> { - if txid.is_some() { - return Err(ParseError::InvalidData( - "txid must be None for Action::parse_from_slice".to_string(), - )); - } - if tx_version.is_some() { - return Err(ParseError::InvalidData( - "tx_version must be None for Action::parse_from_slice".to_string(), - )); - } - let mut cursor = Cursor::new(data); - - skip_bytes(&mut cursor, 32, "Error skipping Action::Cv")?; - let nullifier = read_bytes(&mut cursor, 32, "Error reading Action::nullifier")?; - skip_bytes(&mut cursor, 32, "Error skipping Action::Rk")?; - let cmx = read_bytes(&mut cursor, 32, "Error reading Action::cmx")?; - let ephemeral_key = read_bytes(&mut cursor, 32, "Error reading Action::ephemeral_key")?; - let enc_ciphertext = read_bytes(&mut cursor, 580, "Error reading Action::enc_ciphertext")?; - skip_bytes(&mut cursor, 80, "Error skipping Action::OutCiphertext")?; - - Ok(( - &data[cursor.position() as usize..], - Action { - nullifier, - cmx, - ephemeral_key, - enc_ciphertext, - }, - )) - } -} - -/// Full Zcash transaction data. -#[derive(Debug, Clone)] -struct TransactionData { - /// Indicates if the transaction is an Overwinter-enabled transaction. - /// - /// Size\[bytes\]: [in 4 byte header] - f_overwintered: bool, - /// The transaction format version. - /// - /// Size\[bytes\]: [in 4 byte header] - version: u32, - /// Version group ID, used to specify transaction type and validate its components. - /// - /// Size\[bytes\]: 4 - n_version_group_id: Option, - /// Consensus branch ID, used to identify the network upgrade that the transaction is valid for. - /// - /// Size\[bytes\]: 4 - consensus_branch_id: u32, - /// List of transparent inputs in a transaction. - /// - /// Size\[bytes\]: Vec<40+CompactSize> - transparent_inputs: Vec, - /// List of transparent outputs in a transaction. - /// - /// Size\[bytes\]: Vec<8+CompactSize> - transparent_outputs: Vec, - // NLockTime \[IGNORED\] - Size\[bytes\]: 4 - // NExpiryHeight \[IGNORED\] - Size\[bytes\]: 4 - // ValueBalanceSapling - Size\[bytes\]: 8 - /// Value balance for the Sapling pool (v4/v5). None if not present. - value_balance_sapling: Option, - /// List of shielded spends from the Sapling pool - /// - /// Size\[bytes\]: Vec<384> - shielded_spends: Vec, - /// List of shielded outputs from the Sapling pool - /// - /// Size\[bytes\]: Vec<948> - shielded_outputs: Vec, - /// List of JoinSplit descriptions in a transaction, no longer supported. - /// - /// Size\[bytes\]: Vec<1602-1698> - #[allow(dead_code)] - join_splits: Vec, - /// joinSplitPubKey \[IGNORED\] - Size\[bytes\]: 32 - /// joinSplitSig \[IGNORED\] - Size\[bytes\]: 64 - /// bindingSigSapling \[IGNORED\] - Size\[bytes\]: 64 - /// List of Orchard actions. - /// - /// Size\[bytes\]: Vec<820> - orchard_actions: Vec, - /// ValueBalanceOrchard - Size\[bytes\]: 8 - /// Value balance for the Orchard pool (v5 only). None if not present. - value_balance_orchard: Option, - /// AnchorOrchard - Size\[bytes\]: 32 - /// In non-coinbase transactions, this is the anchor (authDataRoot) of a prior block's Orchard note commitment tree. - /// In the coinbase transaction, this commits to the final Orchard tree state for the current block — i.e., it *is* the block's authDataRoot. - /// Present in v5 transactions only, if any Orchard actions exist in the block. - anchor_orchard: Option>, -} - -impl TransactionData { - /// Parses a v1 transaction. - /// - /// A v1 transaction contains the following fields: - /// - /// - header: u32 - /// - tx_in_count: usize - /// - tx_in: tx_in - /// - tx_out_count: usize - /// - tx_out: tx_out - /// - lock_time: u32 - pub(crate) fn parse_v1(data: &[u8], version: u32) -> Result<(&[u8], Self), ParseError> { - let mut cursor = Cursor::new(data); - - let (remaining_data, transparent_inputs, transparent_outputs) = - parse_transparent(&data[cursor.position() as usize..])?; - cursor.set_position(data.len() as u64 - remaining_data.len() as u64); - - // let lock_time = read_u32(&mut cursor, "Error reading TransactionData::lock_time")?; - skip_bytes(&mut cursor, 4, "Error skipping TransactionData::nLockTime")?; - - Ok(( - &data[cursor.position() as usize..], - TransactionData { - f_overwintered: true, - version, - consensus_branch_id: 0, - transparent_inputs, - transparent_outputs, - // lock_time: Some(lock_time), - n_version_group_id: None, - value_balance_sapling: None, - shielded_spends: Vec::new(), - shielded_outputs: Vec::new(), - join_splits: Vec::new(), - orchard_actions: Vec::new(), - value_balance_orchard: None, - anchor_orchard: None, - }, - )) - } - - /// Parses a v2 transaction. - /// - /// A v2 transaction contains the following fields: - /// - /// - header: u32 - /// - tx_in_count: usize - /// - tx_in: tx_in - /// - tx_out_count: usize - /// - tx_out: tx_out - /// - lock_time: u32 - /// - nJoinSplit: compactSize <- New - /// - vJoinSplit: JSDescriptionBCTV14\[nJoinSplit\] <- New - /// - joinSplitPubKey: byte\[32\] <- New - /// - joinSplitSig: byte\[64\] <- New - pub(crate) fn parse_v2(data: &[u8], version: u32) -> Result<(&[u8], Self), ParseError> { - let mut cursor = Cursor::new(data); - - let (remaining_data, transparent_inputs, transparent_outputs) = - parse_transparent(&data[cursor.position() as usize..])?; - cursor.set_position(data.len() as u64 - remaining_data.len() as u64); - - skip_bytes(&mut cursor, 4, "Error skipping TransactionData::nLockTime")?; - - let join_split_count = CompactSize::read(&mut cursor)?; - let mut join_splits = Vec::with_capacity(join_split_count as usize); - for _ in 0..join_split_count { - let (remaining_data, join_split) = JoinSplit::parse_from_slice( - &data[cursor.position() as usize..], - None, - Some(version), - )?; - join_splits.push(join_split); - cursor.set_position(data.len() as u64 - remaining_data.len() as u64); - } - - if join_split_count > 0 { - skip_bytes( - &mut cursor, - 32, - "Error skipping TransactionData::joinSplitPubKey", - )?; - skip_bytes( - &mut cursor, - 64, - "could not skip TransactionData::joinSplitSig", - )?; - } - - Ok(( - &data[cursor.position() as usize..], - TransactionData { - f_overwintered: true, - version, - consensus_branch_id: 0, - transparent_inputs, - transparent_outputs, - join_splits, - n_version_group_id: None, - value_balance_sapling: None, - shielded_spends: Vec::new(), - shielded_outputs: Vec::new(), - orchard_actions: Vec::new(), - value_balance_orchard: None, - anchor_orchard: None, - }, - )) - } - - /// Parses a v3 transaction. - /// - /// A v3 transaction contains the following fields: - /// - /// - header: u32 - /// - nVersionGroupId: u32 = 0x03C48270 <- New - /// - tx_in_count: usize - /// - tx_in: tx_in - /// - tx_out_count: usize - /// - tx_out: tx_out - /// - lock_time: u32 - /// - nExpiryHeight: u32 <- New - /// - nJoinSplit: compactSize - /// - vJoinSplit: JSDescriptionBCTV14\[nJoinSplit\] - /// - joinSplitPubKey: byte\[32\] - /// - joinSplitSig: byte\[64\] - pub(crate) fn parse_v3( - data: &[u8], - version: u32, - n_version_group_id: u32, - ) -> Result<(&[u8], Self), ParseError> { - if n_version_group_id != 0x03C48270 { - return Err(ParseError::InvalidData( - "n_version_group_id must be 0x03C48270".to_string(), - )); - } - let mut cursor = Cursor::new(data); - - let (remaining_data, transparent_inputs, transparent_outputs) = - parse_transparent(&data[cursor.position() as usize..])?; - cursor.set_position(data.len() as u64 - remaining_data.len() as u64); - - skip_bytes(&mut cursor, 4, "Error skipping TransactionData::nLockTime")?; - skip_bytes( - &mut cursor, - 4, - "Error skipping TransactionData::nExpiryHeight", - )?; - - let join_split_count = CompactSize::read(&mut cursor)?; - let mut join_splits = Vec::with_capacity(join_split_count as usize); - for _ in 0..join_split_count { - let (remaining_data, join_split) = JoinSplit::parse_from_slice( - &data[cursor.position() as usize..], - None, - Some(version), - )?; - join_splits.push(join_split); - cursor.set_position(data.len() as u64 - remaining_data.len() as u64); - } - - if join_split_count > 0 { - skip_bytes( - &mut cursor, - 32, - "Error skipping TransactionData::joinSplitPubKey", - )?; - skip_bytes( - &mut cursor, - 64, - "could not skip TransactionData::joinSplitSig", - )?; - } - Ok(( - &data[cursor.position() as usize..], - TransactionData { - f_overwintered: true, - version, - consensus_branch_id: 0, - transparent_inputs, - transparent_outputs, - join_splits, - n_version_group_id: None, - value_balance_sapling: None, - shielded_spends: Vec::new(), - shielded_outputs: Vec::new(), - orchard_actions: Vec::new(), - value_balance_orchard: None, - anchor_orchard: None, - }, - )) - } - - fn parse_v4( - data: &[u8], - version: u32, - n_version_group_id: u32, - ) -> Result<(&[u8], Self), ParseError> { - if n_version_group_id != 0x892F2085 { - return Err(ParseError::InvalidData(format!( - "version group ID {n_version_group_id:x} must be 0x892F2085 for v4 transactions" - ))); - } - let mut cursor = Cursor::new(data); - - let (remaining_data, transparent_inputs, transparent_outputs) = - parse_transparent(&data[cursor.position() as usize..])?; - cursor.set_position(data.len() as u64 - remaining_data.len() as u64); - - skip_bytes(&mut cursor, 4, "Error skipping TransactionData::nLockTime")?; - skip_bytes( - &mut cursor, - 4, - "Error skipping TransactionData::nExpiryHeight", - )?; - let value_balance_sapling = Some(read_i64( - &mut cursor, - "Error reading TransactionData::valueBalanceSapling", - )?); - - let spend_count = CompactSize::read(&mut cursor)?; - let mut shielded_spends = Vec::with_capacity(spend_count as usize); - for _ in 0..spend_count { - let (remaining_data, spend) = - Spend::parse_from_slice(&data[cursor.position() as usize..], None, Some(4))?; - shielded_spends.push(spend); - cursor.set_position(data.len() as u64 - remaining_data.len() as u64); - } - let output_count = CompactSize::read(&mut cursor)?; - let mut shielded_outputs = Vec::with_capacity(output_count as usize); - for _ in 0..output_count { - let (remaining_data, output) = - Output::parse_from_slice(&data[cursor.position() as usize..], None, Some(4))?; - shielded_outputs.push(output); - cursor.set_position(data.len() as u64 - remaining_data.len() as u64); - } - let join_split_count = CompactSize::read(&mut cursor)?; - let mut join_splits = Vec::with_capacity(join_split_count as usize); - for _ in 0..join_split_count { - let (remaining_data, join_split) = JoinSplit::parse_from_slice( - &data[cursor.position() as usize..], - None, - Some(version), - )?; - join_splits.push(join_split); - cursor.set_position(data.len() as u64 - remaining_data.len() as u64); - } - - if join_split_count > 0 { - skip_bytes( - &mut cursor, - 32, - "Error skipping TransactionData::joinSplitPubKey", - )?; - skip_bytes( - &mut cursor, - 64, - "could not skip TransactionData::joinSplitSig", - )?; - } - if spend_count + output_count > 0 { - skip_bytes( - &mut cursor, - 64, - "Error skipping TransactionData::bindingSigSapling", - )?; - } - - Ok(( - &data[cursor.position() as usize..], - TransactionData { - f_overwintered: true, - version, - n_version_group_id: Some(n_version_group_id), - consensus_branch_id: 0, - transparent_inputs, - transparent_outputs, - value_balance_sapling, - shielded_spends, - shielded_outputs, - join_splits, - orchard_actions: Vec::new(), - value_balance_orchard: None, - anchor_orchard: None, - }, - )) - } - - fn parse_v5( - data: &[u8], - version: u32, - n_version_group_id: u32, - ) -> Result<(&[u8], Self), ParseError> { - if n_version_group_id != 0x26A7270A { - return Err(ParseError::InvalidData(format!( - "version group ID {n_version_group_id:x} must be 0x892F2085 for v5 transactions" - ))); - } - let mut cursor = Cursor::new(data); - - let consensus_branch_id = read_u32( - &mut cursor, - "Error reading TransactionData::ConsensusBranchId", - )?; - - skip_bytes(&mut cursor, 4, "Error skipping TransactionData::nLockTime")?; - skip_bytes( - &mut cursor, - 4, - "Error skipping TransactionData::nExpiryHeight", - )?; - - let (remaining_data, transparent_inputs, transparent_outputs) = - parse_transparent(&data[cursor.position() as usize..])?; - cursor.set_position(data.len() as u64 - remaining_data.len() as u64); - - let spend_count = CompactSize::read(&mut cursor)?; - if spend_count >= (1 << 16) { - return Err(ParseError::InvalidData(format!( - "spendCount ({spend_count}) must be less than 2^16" - ))); - } - let mut shielded_spends = Vec::with_capacity(spend_count as usize); - for _ in 0..spend_count { - let (remaining_data, spend) = - Spend::parse_from_slice(&data[cursor.position() as usize..], None, Some(5))?; - shielded_spends.push(spend); - cursor.set_position(data.len() as u64 - remaining_data.len() as u64); - } - let output_count = CompactSize::read(&mut cursor)?; - if output_count >= (1 << 16) { - return Err(ParseError::InvalidData(format!( - "outputCount ({output_count}) must be less than 2^16" - ))); - } - let mut shielded_outputs = Vec::with_capacity(output_count as usize); - for _ in 0..output_count { - let (remaining_data, output) = - Output::parse_from_slice(&data[cursor.position() as usize..], None, Some(5))?; - shielded_outputs.push(output); - cursor.set_position(data.len() as u64 - remaining_data.len() as u64); - } - - let value_balance_sapling = if spend_count + output_count > 0 { - Some(read_i64( - &mut cursor, - "Error reading TransactionData::valueBalanceSapling", - )?) - } else { - None - }; - if spend_count > 0 { - skip_bytes( - &mut cursor, - 32, - "Error skipping TransactionData::anchorSapling", - )?; - skip_bytes( - &mut cursor, - (192 * spend_count) as usize, - "Error skipping TransactionData::vSpendProofsSapling", - )?; - skip_bytes( - &mut cursor, - (64 * spend_count) as usize, - "Error skipping TransactionData::vSpendAuthSigsSapling", - )?; - } - if output_count > 0 { - skip_bytes( - &mut cursor, - (192 * output_count) as usize, - "Error skipping TransactionData::vOutputProofsSapling", - )?; - } - if spend_count + output_count > 0 { - skip_bytes( - &mut cursor, - 64, - "Error skipping TransactionData::bindingSigSapling", - )?; - } - - let actions_count = CompactSize::read(&mut cursor)?; - if actions_count >= (1 << 16) { - return Err(ParseError::InvalidData(format!( - "actionsCount ({actions_count}) must be less than 2^16" - ))); - } - let mut orchard_actions = Vec::with_capacity(actions_count as usize); - for _ in 0..actions_count { - let (remaining_data, action) = - Action::parse_from_slice(&data[cursor.position() as usize..], None, None)?; - orchard_actions.push(action); - cursor.set_position(data.len() as u64 - remaining_data.len() as u64); - } - - let mut value_balance_orchard = None; - let mut anchor_orchard = None; - if actions_count > 0 { - skip_bytes( - &mut cursor, - 1, - "Error skipping TransactionData::flagsOrchard", - )?; - value_balance_orchard = Some(read_i64( - &mut cursor, - "Error reading TransactionData::valueBalanceOrchard", - )?); - anchor_orchard = Some(read_bytes( - &mut cursor, - 32, - "Error reading TransactionData::anchorOrchard", - )?); - let proofs_count = CompactSize::read(&mut cursor)?; - skip_bytes( - &mut cursor, - proofs_count as usize, - "Error skipping TransactionData::proofsOrchard", - )?; - skip_bytes( - &mut cursor, - (64 * actions_count) as usize, - "Error skipping TransactionData::vSpendAuthSigsOrchard", - )?; - skip_bytes( - &mut cursor, - 64, - "Error skipping TransactionData::bindingSigOrchard", - )?; - } - - Ok(( - &data[cursor.position() as usize..], - TransactionData { - f_overwintered: true, - version, - n_version_group_id: Some(n_version_group_id), - consensus_branch_id, - transparent_inputs, - transparent_outputs, - value_balance_sapling, - shielded_spends, - shielded_outputs, - join_splits: Vec::new(), - orchard_actions, - value_balance_orchard, - anchor_orchard, - }, - )) - } -} +use zebra_chain::{ + parameters::{OVERWINTER_VERSION_GROUP_ID, SAPLING_VERSION_GROUP_ID, TX_V5_VERSION_GROUP_ID}, + serialization::{ZcashDeserialize as _, ZcashSerialize as _}, + transparent, +}; /// Zingo-Indexer struct for a full zcash transaction. #[derive(Debug, Clone)] pub struct FullTransaction { - /// Full transaction data. - raw_transaction: TransactionData, + /// Parsed Zebra transaction. + transaction: Arc, /// Raw transaction bytes. raw_bytes: Vec, - /// Transaction Id, fetched using get_block JsonRPC with verbose = 1. + /// Transaction Id, fetched using get_block JsonRPC with verbose = 1 when available. tx_id: Vec, } @@ -948,126 +34,109 @@ impl ParseFromSlice for FullTransaction { txid: Option>>, tx_version: Option, ) -> Result<(&[u8], Self), ParseError> { - let txid = txid.ok_or_else(|| { - ParseError::InvalidData( - "txid must be used for FullTransaction::parse_from_slice".to_string(), - ) - })?; - // TODO: 🤯 if tx_version.is_some() { return Err(ParseError::InvalidData( "tx_version must be None for FullTransaction::parse_from_slice".to_string(), )); } - let mut cursor = Cursor::new(data); - - let header = read_u32(&mut cursor, "Error reading FullTransaction::header")?; - let f_overwintered = (header >> 31) == 1; - - let version = header & 0x7FFFFFFF; - - match version { - 1 | 2 => { - if f_overwintered { - return Err(ParseError::InvalidData( - "fOverwintered must be unset for tx versions 1 and 2".to_string(), - )); - } - } - 3..=5 => { - if !f_overwintered { - return Err(ParseError::InvalidData( - "fOverwintered must be set for tx versions 3 and above".to_string(), - )); - } - } - _ => { - return Err(ParseError::InvalidData(format!( - "Unsupported tx version {version}" - ))) - } - } - - let n_version_group_id: Option = match version { - 3..=5 => Some(read_u32( - &mut cursor, - "Error reading FullTransaction::n_version_group_id", - )?), - _ => None, - }; - - let (remaining_data, transaction_data) = match version { - 1 => TransactionData::parse_v1(&data[cursor.position() as usize..], version)?, - 2 => TransactionData::parse_v2(&data[cursor.position() as usize..], version)?, - 3 => TransactionData::parse_v3( - &data[cursor.position() as usize..], - version, - n_version_group_id.unwrap(), // This won't fail, because of the above match - )?, - 4 => TransactionData::parse_v4( - &data[cursor.position() as usize..], - version, - n_version_group_id.unwrap(), // This won't fail, because of the above match - )?, - 5 => TransactionData::parse_v5( - &data[cursor.position() as usize..], - version, - n_version_group_id.unwrap(), // This won't fail, because of the above match - )?, - - _ => { - return Err(ParseError::InvalidData(format!( - "Unsupported tx version {version}" - ))) - } - }; - let full_transaction = FullTransaction { - raw_transaction: transaction_data, - raw_bytes: data[..(data.len() - remaining_data.len())].to_vec(), - tx_id: txid[0].clone(), - }; + let mut cursor = Cursor::new(data); + let transaction = zebra_chain::transaction::Transaction::zcash_deserialize(&mut cursor)?; + let consumed = usize::try_from(cursor.position())?; + let tx_id = txid + .and_then(|txids| txids.into_iter().next()) + .unwrap_or_else(|| transaction.hash().0.to_vec()); - Ok((remaining_data, full_transaction)) + Ok(( + &data[consumed..], + Self { + transaction: Arc::new(transaction), + raw_bytes: data[..consumed].to_vec(), + tx_id, + }, + )) } } impl FullTransaction { - /// Returns overwintered bool + pub(super) fn from_zebra( + transaction: Arc, + tx_id: Option>, + ) -> Result { + let raw_bytes = transaction.zcash_serialize_to_vec()?; + let tx_id = tx_id.unwrap_or_else(|| transaction.hash().0.to_vec()); + + Ok(Self { + transaction, + raw_bytes, + tx_id, + }) + } + + /// Returns overwintered bool. pub fn f_overwintered(&self) -> bool { - self.raw_transaction.f_overwintered + self.transaction.is_overwintered() } /// Returns the transaction version. pub fn version(&self) -> u32 { - self.raw_transaction.version + self.transaction.version() } /// Returns the transaction version group id. pub fn n_version_group_id(&self) -> Option { - self.raw_transaction.n_version_group_id + match self.version() { + 3 => Some(OVERWINTER_VERSION_GROUP_ID), + 4 => Some(SAPLING_VERSION_GROUP_ID), + 5 => Some(TX_V5_VERSION_GROUP_ID), + 6 => Some(zebra_chain::parameters::TX_V6_VERSION_GROUP_ID), + _ => None, + } } - /// returns the consensus branch id of the transaction. + /// Returns the consensus branch id of the transaction. pub fn consensus_branch_id(&self) -> u32 { - self.raw_transaction.consensus_branch_id + self.transaction + .network_upgrade() + .and_then(|network_upgrade| network_upgrade.branch_id()) + .map(u32::from) + .unwrap_or(0) } /// Returns a vec of transparent inputs: (prev_txid, prev_index, script_sig). pub fn transparent_inputs(&self) -> Vec<(Vec, u32, Vec)> { - self.raw_transaction - .transparent_inputs + self.transaction + .inputs() .iter() - .map(|input| input.clone().into_inner()) + .map(|input| match input { + transparent::Input::PrevOut { + outpoint, + unlock_script, + .. + } => ( + outpoint.hash.0.to_vec(), + outpoint.index, + unlock_script.as_raw_bytes().to_vec(), + ), + transparent::Input::Coinbase { .. } => ( + vec![0; 32], + u32::MAX, + input.coinbase_script().unwrap_or_default(), + ), + }) .collect() } /// Returns a vec of transparent outputs: (value, script_hash). pub fn transparent_outputs(&self) -> Vec<(u64, Vec)> { - self.raw_transaction - .transparent_outputs + self.transaction + .outputs() .iter() - .map(|output| output.clone().into_inner()) + .filter_map(|output| { + u64::try_from(output.value().zatoshis()) + .ok() + .map(|value| (value, output.lock_script.as_raw_bytes().to_vec())) + }) .collect() } @@ -1075,27 +144,49 @@ impl FullTransaction { /// /// Returned as (Option\, Option\). pub fn value_balances(&self) -> (Option, Option) { - ( - self.raw_transaction.value_balance_sapling, - self.raw_transaction.value_balance_orchard, - ) + let sapling = if self.version() == 4 || self.transaction.has_sapling_shielded_data() { + Some( + self.transaction + .sapling_value_balance() + .sapling_amount() + .zatoshis(), + ) + } else { + None + }; + + let orchard = self.transaction.has_orchard_shielded_data().then(|| { + self.transaction + .orchard_value_balance() + .orchard_amount() + .zatoshis() + }); + + (sapling, orchard) } /// Returns a vec of sapling nullifiers for the transaction. pub fn shielded_spends(&self) -> Vec> { - self.raw_transaction - .shielded_spends - .iter() - .map(|input| input.clone().into_inner()) + self.transaction + .sapling_nullifiers() + .map(|nullifier| <[u8; 32]>::from(*nullifier).to_vec()) .collect() } /// Returns a vec of sapling outputs (cmu, ephemeral_key, enc_ciphertext) for the transaction. pub fn shielded_outputs(&self) -> Vec<(Vec, Vec, Vec)> { - self.raw_transaction - .shielded_outputs - .iter() - .map(|input| input.clone().into_parts()) + self.transaction + .sapling_outputs() + .map(|output| { + let ephemeral_key: [u8; 32] = (&output.ephemeral_key).into(); + let enc_ciphertext: [u8; 580] = output.enc_ciphertext.into(); + + ( + output.cm_u.to_bytes().to_vec(), + ephemeral_key.to_vec(), + enc_ciphertext.to_vec(), + ) + }) .collect() } @@ -1107,10 +198,21 @@ impl FullTransaction { /// Returns a vec of orchard actions (nullifier, cmx, ephemeral_key, enc_ciphertext) for the transaction. #[allow(clippy::complexity)] pub fn orchard_actions(&self) -> Vec<(Vec, Vec, Vec, Vec)> { - self.raw_transaction - .orchard_actions - .iter() - .map(|input| input.clone().into_parts()) + self.transaction + .orchard_actions() + .map(|action| { + let nullifier: [u8; 32] = action.nullifier.into(); + let cmx: [u8; 32] = action.cm_x.into(); + let ephemeral_key: [u8; 32] = (&action.ephemeral_key).into(); + let enc_ciphertext: [u8; 580] = action.enc_ciphertext.into(); + + ( + nullifier.to_vec(), + cmx.to_vec(), + ephemeral_key.to_vec(), + enc_ciphertext.to_vec(), + ) + }) .collect() } @@ -1118,7 +220,9 @@ impl FullTransaction { /// /// If this is the Coinbase transaction then this returns the AuthDataRoot of the block. pub fn anchor_orchard(&self) -> Option> { - self.raw_transaction.anchor_orchard.clone() + self.transaction + .orchard_shielded_data() + .map(|shielded_data| <[u8; 32]>::from(&shielded_data.shared_anchor).to_vec()) } /// Returns the transaction as raw bytes. @@ -1126,7 +230,7 @@ impl FullTransaction { self.raw_bytes.clone() } - /// returns the TxId of the transaction. + /// Returns the TxId of the transaction. pub fn tx_id(&self) -> Vec { self.tx_id.clone() } @@ -1138,98 +242,86 @@ impl FullTransaction { } /// Converts a Zcash Transaction into a `CompactTx` of the Light wallet protocol. - /// if the transaction you want to convert is a mempool transaction you can specify `None`. - /// specify the `PoolType`s that the transaction should include in the `pool_types` argument + /// If the transaction you want to convert is a mempool transaction you can specify `None`. + /// Specify the `PoolType`s that the transaction should include in the `pool_types` argument /// with a `PoolTypeFilter` indicating which pools the compact block should include. pub fn to_compact_tx( self, index: Option, pool_types: &PoolTypeFilter, ) -> Result { - let hash = self.tx_id(); - - // NOTE: LightWalletD currently does not return a fee and is not currently priority here. - // Please open an Issue or PR at the Zingo-Indexer github (https://github.com/zingolabs/zingo-indexer) - // if you require this functionality. - let fee = 0; - let spends = if pool_types.includes_sapling() { - self.raw_transaction - .shielded_spends - .iter() - .map(|spend| CompactSaplingSpend { - nf: spend.nullifier.clone(), - }) + self.shielded_spends() + .into_iter() + .map(|nf| CompactSaplingSpend { nf }) .collect() } else { - vec![] + Vec::new() }; let outputs = if pool_types.includes_sapling() { - self.raw_transaction - .shielded_outputs - .iter() - .map(|output| CompactSaplingOutput { - cmu: output.cmu.clone(), - ephemeral_key: output.ephemeral_key.clone(), - ciphertext: output.enc_ciphertext[..52].to_vec(), - }) + self.shielded_outputs() + .into_iter() + .map( + |(cmu, ephemeral_key, enc_ciphertext)| CompactSaplingOutput { + cmu, + ephemeral_key, + ciphertext: enc_ciphertext[..52].to_vec(), + }, + ) .collect() } else { - vec![] + Vec::new() }; let actions = if pool_types.includes_orchard() { - self.raw_transaction - .orchard_actions - .iter() - .map(|action| CompactOrchardAction { - nullifier: action.nullifier.clone(), - cmx: action.cmx.clone(), - ephemeral_key: action.ephemeral_key.clone(), - ciphertext: action.enc_ciphertext[..52].to_vec(), - }) + self.orchard_actions() + .into_iter() + .map( + |(nullifier, cmx, ephemeral_key, enc_ciphertext)| CompactOrchardAction { + nullifier, + cmx, + ephemeral_key, + ciphertext: enc_ciphertext[..52].to_vec(), + }, + ) .collect() } else { - vec![] + Vec::new() }; let vout = if pool_types.includes_transparent() { - self.raw_transaction - .transparent_outputs - .iter() - .map(|t_out| CompactTxOut { - value: t_out.value, - script_pub_key: t_out.script_hash.clone(), + self.transparent_outputs() + .into_iter() + .map(|(value, script_hash)| CompactTxOut { + value, + script_pub_key: script_hash, }) .collect() } else { - vec![] + Vec::new() }; let vin = if pool_types.includes_transparent() { - self.raw_transaction - .transparent_inputs + self.transaction + .inputs() .iter() - .filter_map(|t_in| { - if t_in.is_null() { - None - } else { - Some(CompactTxIn { - prevout_txid: t_in.prev_txid.clone(), - prevout_index: t_in.prev_index, - }) - } + .filter_map(|input| { + let outpoint = input.outpoint()?; + Some(CompactTxIn { + prevout_txid: outpoint.hash.0.to_vec(), + prevout_index: outpoint.index, + }) }) .collect() } else { - vec![] + Vec::new() }; Ok(CompactTx { index: index.unwrap_or(0), // this assumes that mempool txs have a zeroed index - txid: hash, - fee, + txid: self.tx_id(), + fee: 0, spends, outputs, actions, @@ -1241,9 +333,7 @@ impl FullTransaction { /// Returns true if the transaction contains either sapling spends or outputs, or orchard actions. #[allow(dead_code)] pub(crate) fn has_shielded_elements(&self) -> bool { - !self.raw_transaction.shielded_spends.is_empty() - || !self.raw_transaction.shielded_outputs.is_empty() - || !self.raw_transaction.orchard_actions.is_empty() + self.transaction.has_sapling_shielded_data() || self.transaction.has_orchard_shielded_data() } } @@ -1252,210 +342,36 @@ mod tests { use super::*; use wire_serialized_transaction_test_data::transactions::get_test_vectors; - /// Test parsing v1 transactions using test vectors. - /// Validates that FullTransaction::parse_from_slice correctly handles v1 transaction format. #[test] - fn test_v1_transaction_parsing_with_test_vectors() { - let test_vectors = get_test_vectors(); - let v1_vectors: Vec<_> = test_vectors.iter().filter(|tv| tv.version == 1).collect(); - - assert!(!v1_vectors.is_empty(), "No v1 test vectors found"); - - for (i, vector) in v1_vectors.iter().enumerate() { - let result = FullTransaction::parse_from_slice( + fn parses_test_vectors_with_zebra_deserializer() -> Result<(), ParseError> { + for vector in get_test_vectors() { + let (remaining, transaction) = FullTransaction::parse_from_slice( &vector.tx, Some(vec![vector.txid.to_vec()]), None, - ); - - assert!( - result.is_ok(), - "Failed to parse v1 test vector #{}: {:?}. Description: {}", - i, - result.err(), - vector.description - ); - - let (remaining, parsed_tx) = result.unwrap(); - assert!( - remaining.is_empty(), - "Should consume all data for v1 transaction #{i}" - ); - - // Verify version matches - assert_eq!( - parsed_tx.raw_transaction.version, 1, - "Version mismatch for v1 transaction #{i}" - ); - - // Verify transaction properties match test vector expectations - assert_eq!( - parsed_tx.raw_transaction.transparent_inputs.len(), - vector.transparent_inputs, - "Transparent inputs mismatch for v1 transaction #{i}" - ); + )?; + assert!(remaining.is_empty(), "{}", vector.description); assert_eq!( - parsed_tx.raw_transaction.transparent_outputs.len(), - vector.transparent_outputs, - "Transparent outputs mismatch for v1 transaction #{i}" - ); - } - } - - /// Test parsing v2 transactions using test vectors. - /// Validates that FullTransaction::parse_from_slice correctly handles v2 transaction format. - #[test] - fn test_v2_transaction_parsing_with_test_vectors() { - let test_vectors = get_test_vectors(); - let v2_vectors: Vec<_> = test_vectors.iter().filter(|tv| tv.version == 2).collect(); - - assert!(!v2_vectors.is_empty(), "No v2 test vectors found"); - - for (i, vector) in v2_vectors.iter().enumerate() { - let result = FullTransaction::parse_from_slice( - &vector.tx, - Some(vec![vector.txid.to_vec()]), - None, - ); - - assert!( - result.is_ok(), - "Failed to parse v2 test vector #{}: {:?}. Description: {}", - i, - result.err(), + transaction.version(), + vector.version, + "{}", vector.description ); - - let (remaining, parsed_tx) = result.unwrap(); - assert!( - remaining.is_empty(), - "Should consume all data for v2 transaction #{}: {} bytes remaining, total length: {}", - i, remaining.len(), vector.tx.len() - ); - - // Verify version matches - assert_eq!( - parsed_tx.raw_transaction.version, 2, - "Version mismatch for v2 transaction #{i}" - ); - - // Verify transaction properties match test vector expectations assert_eq!( - parsed_tx.raw_transaction.transparent_inputs.len(), + transaction.transparent_inputs().len(), vector.transparent_inputs, - "Transparent inputs mismatch for v2 transaction #{i}" - ); - - assert_eq!( - parsed_tx.raw_transaction.transparent_outputs.len(), - vector.transparent_outputs, - "Transparent outputs mismatch for v2 transaction #{i}" - ); - } - } - - /// Test parsing v3 transactions using test vectors. - /// Validates that FullTransaction::parse_from_slice correctly handles v3 transaction format. - #[test] - fn test_v3_transaction_parsing_with_test_vectors() { - let test_vectors = get_test_vectors(); - let v3_vectors: Vec<_> = test_vectors.iter().filter(|tv| tv.version == 3).collect(); - - assert!(!v3_vectors.is_empty(), "No v3 test vectors found"); - - for (i, vector) in v3_vectors.iter().enumerate() { - let result = FullTransaction::parse_from_slice( - &vector.tx, - Some(vec![vector.txid.to_vec()]), - None, - ); - - assert!( - result.is_ok(), - "Failed to parse v3 test vector #{}: {:?}. Description: {}", - i, - result.err(), + "{}", vector.description ); - - let (remaining, parsed_tx) = result.unwrap(); - assert!( - remaining.is_empty(), - "Should consume all data for v3 transaction #{}: {} bytes remaining, total length: {}", - i, remaining.len(), vector.tx.len() - ); - - // Verify version matches - assert_eq!( - parsed_tx.raw_transaction.version, 3, - "Version mismatch for v3 transaction #{i}" - ); - - // Verify transaction properties match test vector expectations - assert_eq!( - parsed_tx.raw_transaction.transparent_inputs.len(), - vector.transparent_inputs, - "Transparent inputs mismatch for v3 transaction #{i}" - ); - assert_eq!( - parsed_tx.raw_transaction.transparent_outputs.len(), + transaction.transparent_outputs().len(), vector.transparent_outputs, - "Transparent outputs mismatch for v3 transaction #{i}" - ); - } - } - - /// Test parsing v4 transactions using test vectors. - /// Validates that FullTransaction::parse_from_slice correctly handles v4 transaction format. - /// This also serves as a regression test for current v4 functionality. - #[test] - fn test_v4_transaction_parsing_with_test_vectors() { - let test_vectors = get_test_vectors(); - let v4_vectors: Vec<_> = test_vectors.iter().filter(|tv| tv.version == 4).collect(); - - assert!(!v4_vectors.is_empty(), "No v4 test vectors found"); - - for (i, vector) in v4_vectors.iter().enumerate() { - let result = FullTransaction::parse_from_slice( - &vector.tx, - Some(vec![vector.txid.to_vec()]), - None, - ); - - assert!( - result.is_ok(), - "Failed to parse v4 test vector #{}: {:?}. Description: {}", - i, - result.err(), + "{}", vector.description ); - - let (remaining, parsed_tx) = result.unwrap(); - assert!( - remaining.is_empty(), - "Should consume all data for v4 transaction #{i}" - ); - - // Verify version matches - assert_eq!( - parsed_tx.raw_transaction.version, 4, - "Version mismatch for v4 transaction #{i}" - ); - - // Verify transaction properties match test vector expectations - assert_eq!( - parsed_tx.raw_transaction.transparent_inputs.len(), - vector.transparent_inputs, - "Transparent inputs mismatch for v4 transaction #{i}" - ); - - assert_eq!( - parsed_tx.raw_transaction.transparent_outputs.len(), - vector.transparent_outputs, - "Transparent outputs mismatch for v4 transaction #{i}" - ); } + + Ok(()) } } diff --git a/packages/zaino-fetch/src/chain/utils.rs b/packages/zaino-fetch/src/chain/utils.rs index 4570a3a7c..de6b71beb 100644 --- a/packages/zaino-fetch/src/chain/utils.rs +++ b/packages/zaino-fetch/src/chain/utils.rs @@ -1,206 +1,3 @@ -//! Blockcache utility functionality. +//! Compatibility re-exports for chain parsing utilities. -use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; -use std::io::{self, Cursor, Read, Write}; - -use crate::chain::error::ParseError; - -/// Used for decoding zcash blocks from a bytestring. -pub trait ParseFromSlice { - /// Reads data from a bytestring, consuming data read, and returns an instance of self along with the remaining data in the bytestring given. - /// - /// txid is giving as an input as this is taken from a get_block verbose=1 call. - /// - /// tx_version is used for deserializing sapling spends and outputs. - fn parse_from_slice( - data: &[u8], - // TODO: Why is txid a vec of vecs? - txid: Option>>, - tx_version: Option, - ) -> Result<(&[u8], Self), ParseError> - where - Self: Sized; -} - -/// Skips the next n bytes in cursor, returns error message given if eof is reached. -pub(crate) fn skip_bytes( - cursor: &mut Cursor<&[u8]>, - n: usize, - error_msg: &str, -) -> Result<(), ParseError> { - if cursor.get_ref().len() < (cursor.position() + n as u64) as usize { - return Err(ParseError::InvalidData(error_msg.to_string())); - } - cursor.set_position(cursor.position() + n as u64); - Ok(()) -} - -/// Reads the next n bytes from cursor into a `vec`, returns error message given if eof is reached. -pub(crate) fn read_bytes( - cursor: &mut Cursor<&[u8]>, - n: usize, - error_msg: &str, -) -> Result, ParseError> { - let mut buf = vec![0; n]; - cursor - .read_exact(&mut buf) - .map_err(|_| ParseError::InvalidData(error_msg.to_string()))?; - Ok(buf) -} - -/// Reads the next 8 bytes from cursor into a u64, returns error message given if eof is reached. -pub(crate) fn read_u64(cursor: &mut Cursor<&[u8]>, error_msg: &str) -> Result { - cursor - .read_u64::() - .map_err(ParseError::from) - .map_err(|_| ParseError::InvalidData(error_msg.to_string())) -} - -/// Reads the next 4 bytes from cursor into a u32, returns error message given if eof is reached. -pub(crate) fn read_u32(cursor: &mut Cursor<&[u8]>, error_msg: &str) -> Result { - cursor - .read_u32::() - .map_err(ParseError::from) - .map_err(|_| ParseError::InvalidData(error_msg.to_string())) -} - -/// Reads the next 8 bytes from cursor into an i64, returns error message given if eof is reached. -pub(crate) fn read_i64(cursor: &mut Cursor<&[u8]>, error_msg: &str) -> Result { - cursor - .read_i64::() - .map_err(ParseError::from) - .map_err(|_| ParseError::InvalidData(error_msg.to_string())) -} - -/// Reads the next 4 bytes from cursor into an i32, returns error message given if eof is reached. -pub(crate) fn read_i32(cursor: &mut Cursor<&[u8]>, error_msg: &str) -> Result { - cursor - .read_i32::() - .map_err(ParseError::from) - .map_err(|_| ParseError::InvalidData(error_msg.to_string())) -} - -/// Reads the next byte from cursor into a bool, returns error message given if eof is reached. -#[allow(dead_code)] -pub(crate) fn read_bool(cursor: &mut Cursor<&[u8]>, error_msg: &str) -> Result { - let byte = cursor - .read_u8() - .map_err(ParseError::from) - .map_err(|_| ParseError::InvalidData(error_msg.to_string()))?; - match byte { - 0 => Ok(false), - 1 => Ok(true), - _ => Err(ParseError::InvalidData(error_msg.to_string())), - } -} - -/// read_zcash_script_int64 OP codes. -const OP_0: u8 = 0x00; -const OP_1_NEGATE: u8 = 0x4f; -const OP_1: u8 = 0x51; -const OP_16: u8 = 0x60; - -/// Reads and interprets a Zcash (Bitcoin) custom compact integer encoding used for int64 numbers in scripts. -pub(crate) fn read_zcash_script_i64(cursor: &mut Cursor<&[u8]>) -> Result { - let first_byte = read_bytes(cursor, 1, "Error reading first byte in i64 script hash")?[0]; - - match first_byte { - OP_1_NEGATE => Ok(-1), - OP_0 => Ok(0), - OP_1..=OP_16 => Ok((u64::from(first_byte) - u64::from(OP_1 - 1)) as i64), - _ => { - let num_bytes = - read_bytes(cursor, first_byte as usize, "Error reading i64 script hash")?; - let number = num_bytes - .iter() - .rev() - .fold(0, |acc, &byte| (acc << 8) | u64::from(byte)); - Ok(number as i64) - } - } -} - -/// Zcash CompactSize implementation taken from LibRustZcash::zcash_encoding to simplify dependency tree. -/// -/// Namespace for functions for compact encoding of integers. -/// -/// This codec requires integers to be in the range `0x0..=0x02000000`, for compatibility -/// with Zcash consensus rules. -pub(crate) struct CompactSize; - -/// The maximum allowed value representable as a `[CompactSize]` -pub(crate) const MAX_COMPACT_SIZE: u32 = 0x02000000; - -impl CompactSize { - /// Reads an integer encoded in compact form. - pub(crate) fn read(mut reader: R) -> io::Result { - let flag = reader.read_u8()?; - let result = if flag < 253 { - Ok(flag as u64) - } else if flag == 253 { - match reader.read_u16::()? { - n if n < 253 => Err(io::Error::new( - io::ErrorKind::InvalidInput, - "non-canonical CompactSize", - )), - n => Ok(n as u64), - } - } else if flag == 254 { - match reader.read_u32::()? { - n if n < 0x10000 => Err(io::Error::new( - io::ErrorKind::InvalidInput, - "non-canonical CompactSize", - )), - n => Ok(n as u64), - } - } else { - match reader.read_u64::()? { - n if n < 0x100000000 => Err(io::Error::new( - io::ErrorKind::InvalidInput, - "non-canonical CompactSize", - )), - n => Ok(n), - } - }?; - - match result { - s if s > ::from(MAX_COMPACT_SIZE) => Err(io::Error::new( - io::ErrorKind::InvalidInput, - "CompactSize too large", - )), - s => Ok(s), - } - } - - /// Reads an integer encoded in compact form and performs checked conversion - /// to the target type. - #[allow(dead_code)] - pub(crate) fn read_t>(mut reader: R) -> io::Result { - let n = Self::read(&mut reader)?; - ::try_from(n).map_err(|_| { - io::Error::new( - io::ErrorKind::InvalidInput, - "CompactSize value exceeds range of target type.", - ) - }) - } - - /// Writes the provided `usize` value to the provided Writer in compact form. - pub(crate) fn write(mut writer: W, size: usize) -> io::Result<()> { - match size { - s if s < 253 => writer.write_u8(s as u8), - s if s <= 0xFFFF => { - writer.write_u8(253)?; - writer.write_u16::(s as u16) - } - s if s <= 0xFFFFFFFF => { - writer.write_u8(254)?; - writer.write_u32::(s as u32) - } - s => { - writer.write_u8(255)?; - writer.write_u64::(s as u64) - } - } - } -} +pub use crate::utils::ParseFromSlice; diff --git a/packages/zaino-fetch/src/lib.rs b/packages/zaino-fetch/src/lib.rs index 94778ad59..0299020a2 100644 --- a/packages/zaino-fetch/src/lib.rs +++ b/packages/zaino-fetch/src/lib.rs @@ -7,6 +7,7 @@ pub mod chain; pub mod jsonrpsee; +pub mod utils; /// Prometheus metric names emitted by this crate; the single source of truth shared with `zainod`'s `describe_*` registrations (which carry the descriptions). #[cfg(feature = "prometheus")] diff --git a/packages/zaino-fetch/src/utils.rs b/packages/zaino-fetch/src/utils.rs new file mode 100644 index 000000000..e66a1ae5d --- /dev/null +++ b/packages/zaino-fetch/src/utils.rs @@ -0,0 +1,19 @@ +//! Shared utility functionality for zaino-fetch. + +use crate::chain::error::ParseError; + +/// Used for decoding Zcash structures from a bytestring. +pub trait ParseFromSlice { + /// Reads data from a bytestring, consuming data read, and returns an instance of self along with the remaining data in the bytestring given. + /// + /// `txid` is accepted for compatibility with callers that already fetched transaction ids from a verbose block RPC response. + /// + /// `tx_version` is retained for compatibility with the legacy parser API and should be `None` for Zebra-backed parsing. + fn parse_from_slice( + data: &[u8], + txid: Option>>, + tx_version: Option, + ) -> Result<(&[u8], Self), ParseError> + where + Self: Sized; +} From ec88a88b4986cbd23a04bb41ce8cc3612bdd2eef Mon Sep 17 00:00:00 2001 From: idky137 Date: Fri, 3 Jul 2026 14:34:21 +0100 Subject: [PATCH 030/134] updated zaino-proto plus compact block structs, added ironwood treestate (incomplete) --- packages/zaino-fetch/src/chain/block.rs | 20 ++--- packages/zaino-fetch/src/chain/transaction.rs | 58 ++++++++++-- packages/zaino-proto/CHANGELOG.md | 1 + packages/zaino-proto/README.md | 3 +- .../lightwallet-protocol/CHANGELOG.md | 8 ++ .../walletrpc/compact_formats.proto | 2 + .../walletrpc/service.proto | 2 + .../zaino-proto/src/proto/compact_formats.rs | 5 ++ packages/zaino-proto/src/proto/service.rs | 6 ++ packages/zaino-proto/src/proto/utils.rs | 53 +++++++++-- packages/zaino-state/src/backends/fetch.rs | 50 ++++++++--- packages/zaino-state/src/backends/state.rs | 90 +++++++++++++++---- .../finalised_source/v1/compact_block.rs | 4 + .../src/chain_index/types/db/legacy.rs | 4 +- 14 files changed, 244 insertions(+), 62 deletions(-) diff --git a/packages/zaino-fetch/src/chain/block.rs b/packages/zaino-fetch/src/chain/block.rs index 9598b553a..398589265 100644 --- a/packages/zaino-fetch/src/chain/block.rs +++ b/packages/zaino-fetch/src/chain/block.rs @@ -172,7 +172,9 @@ impl FullBlock { return Err(ParseError::InvalidData(format!( "Error decoding full block - {} bytes of Remaining data. Compact Block Created: ({:?})", remaining_data.len(), - full_block.clone().into_compact_block(0, 0, PoolTypeFilter::includes_all()) + full_block + .clone() + .into_compact_block(0, 0, 0, PoolTypeFilter::includes_all()) ))); } Ok(full_block) @@ -185,6 +187,7 @@ impl FullBlock { self, sapling_commitment_tree_size: u32, orchard_commitment_tree_size: u32, + ironwood_commitment_tree_size: u32, pool_types: PoolTypeFilter, ) -> Result { let vtx = self @@ -221,21 +224,8 @@ impl FullBlock { chain_metadata: Some(ChainMetadata { sapling_commitment_tree_size, orchard_commitment_tree_size, + ironwood_commitment_tree_size, }), }) } - - #[deprecated] - /// Converts a zcash full block into a **legacy** compact block. - pub fn into_compact( - self, - sapling_commitment_tree_size: u32, - orchard_commitment_tree_size: u32, - ) -> Result { - self.into_compact_block( - sapling_commitment_tree_size, - orchard_commitment_tree_size, - PoolTypeFilter::default(), - ) - } } diff --git a/packages/zaino-fetch/src/chain/transaction.rs b/packages/zaino-fetch/src/chain/transaction.rs index b26f76c3d..68c784466 100644 --- a/packages/zaino-fetch/src/chain/transaction.rs +++ b/packages/zaino-fetch/src/chain/transaction.rs @@ -140,10 +140,11 @@ impl FullTransaction { .collect() } - /// Returns sapling and orchard value balances for the transaction. + /// Returns sapling, orchard, and ironwood value balances for the transaction. /// - /// Returned as (Option\, Option\). - pub fn value_balances(&self) -> (Option, Option) { + /// Returned as (Option\, Option\, + /// Option\). + pub fn value_balances(&self) -> (Option, Option, Option) { let sapling = if self.version() == 4 || self.transaction.has_sapling_shielded_data() { Some( self.transaction @@ -162,7 +163,14 @@ impl FullTransaction { .zatoshis() }); - (sapling, orchard) + let ironwood = self.transaction.has_ironwood_shielded_data().then(|| { + self.transaction + .ironwood_value_balance() + .ironwood_amount() + .zatoshis() + }); + + (sapling, orchard, ironwood) } /// Returns a vec of sapling nullifiers for the transaction. @@ -216,6 +224,27 @@ impl FullTransaction { .collect() } + /// Returns a vec of ironwood actions (nullifier, cmx, ephemeral_key, enc_ciphertext) for the transaction. + #[allow(clippy::complexity)] + pub fn ironwood_actions(&self) -> Vec<(Vec, Vec, Vec, Vec)> { + self.transaction + .ironwood_actions() + .map(|action| { + let nullifier: [u8; 32] = action.nullifier.into(); + let cmx: [u8; 32] = action.cm_x.into(); + let ephemeral_key: [u8; 32] = (&action.ephemeral_key).into(); + let enc_ciphertext: [u8; 580] = action.enc_ciphertext.into(); + + ( + nullifier.to_vec(), + cmx.to_vec(), + ephemeral_key.to_vec(), + enc_ciphertext.to_vec(), + ) + }) + .collect() + } + /// Returns the orchard anchor of the transaction. /// /// If this is the Coinbase transaction then this returns the AuthDataRoot of the block. @@ -290,6 +319,22 @@ impl FullTransaction { Vec::new() }; + let ironwood_actions = if pool_types.includes_ironwood() { + self.ironwood_actions() + .into_iter() + .map( + |(nullifier, cmx, ephemeral_key, enc_ciphertext)| CompactOrchardAction { + nullifier, + cmx, + ephemeral_key, + ciphertext: enc_ciphertext[..52].to_vec(), + }, + ) + .collect() + } else { + Vec::new() + }; + let vout = if pool_types.includes_transparent() { self.transparent_outputs() .into_iter() @@ -325,6 +370,7 @@ impl FullTransaction { spends, outputs, actions, + ironwood_actions, vin, vout, }) @@ -333,7 +379,9 @@ impl FullTransaction { /// Returns true if the transaction contains either sapling spends or outputs, or orchard actions. #[allow(dead_code)] pub(crate) fn has_shielded_elements(&self) -> bool { - self.transaction.has_sapling_shielded_data() || self.transaction.has_orchard_shielded_data() + self.transaction.has_sapling_shielded_data() + || self.transaction.has_orchard_shielded_data() + || self.transaction.has_ironwood_shielded_data() } } diff --git a/packages/zaino-proto/CHANGELOG.md b/packages/zaino-proto/CHANGELOG.md index 951b0949b..1b95592ec 100644 --- a/packages/zaino-proto/CHANGELOG.md +++ b/packages/zaino-proto/CHANGELOG.md @@ -9,6 +9,7 @@ and this library adheres to Rust's notion of ### Added ### Changed +- Lightwallet protocol vendored subtree updated toward v0.5.0. ### Deprecated ### Removed ### Fixed diff --git a/packages/zaino-proto/README.md b/packages/zaino-proto/README.md index 17578a172..a114740a2 100644 --- a/packages/zaino-proto/README.md +++ b/packages/zaino-proto/README.md @@ -45,7 +45,7 @@ for example: when doing ``` -git subtree --prefix=zaino-proto/lightwallet-protocol pull git@github.com:zcash/lightwallet-protocol.git v0.4.0 --squash +git subtree --prefix=zaino-proto/lightwallet-protocol pull git@github.com:zcash/lightwallet-protocol.git v0.5.0 --squash ``` your branch's commits must be sequenced like this. @@ -62,4 +62,3 @@ If you are developing the `lightclient-protocol` and adopting it on Zaino, it is you don't do subsequent `git subtree` to revisions and always rebase against the latest latest version that you will be using in your latest commit to avoid rebasing issues and also keeping a coherent git commit history for when your branch merges to `dev`. - diff --git a/packages/zaino-proto/lightwallet-protocol/CHANGELOG.md b/packages/zaino-proto/lightwallet-protocol/CHANGELOG.md index 860d0d8e2..ffebb5da4 100644 --- a/packages/zaino-proto/lightwallet-protocol/CHANGELOG.md +++ b/packages/zaino-proto/lightwallet-protocol/CHANGELOG.md @@ -36,6 +36,14 @@ and the protocol adheres to an added `poolTypes` field, which allows the caller of this method to specify which pools the resulting `CompactTx` values should contain data for. +## [v0.5.0] - 2026-07-03 + +### Added +- `compact_formats.ChainMetadata` has added field `ironwoodCommitmentTreeSize` +- `compact_formats.CompactTx` has added field `ironwoodActions` +- `service.PoolType` has added `IRONWOOD` +- `service.TreeState` has added `ironwoodTree` + ### Deprecated - `service.CompactTxStreamer`: - The `GetBlockNullifiers` and `GetBlockRangeNullifiers` methods are diff --git a/packages/zaino-proto/lightwallet-protocol/walletrpc/compact_formats.proto b/packages/zaino-proto/lightwallet-protocol/walletrpc/compact_formats.proto index c62c7acbb..f96373023 100644 --- a/packages/zaino-proto/lightwallet-protocol/walletrpc/compact_formats.proto +++ b/packages/zaino-proto/lightwallet-protocol/walletrpc/compact_formats.proto @@ -14,6 +14,7 @@ option swift_prefix = ""; message ChainMetadata { uint32 saplingCommitmentTreeSize = 1; // the size of the Sapling note commitment tree as of the end of this block uint32 orchardCommitmentTreeSize = 2; // the size of the Orchard note commitment tree as of the end of this block + uint32 ironwoodCommitmentTreeSize = 3; // the size of the Ironwood note commitment tree as of the end of this block } // A compact representation of a Zcash block. @@ -61,6 +62,7 @@ message CompactTx { repeated CompactSaplingSpend spends = 4; repeated CompactSaplingOutput outputs = 5; repeated CompactOrchardAction actions = 6; + repeated CompactOrchardAction ironwoodActions = 9; // `CompactTxIn` values corresponding to the `vin` entries of the full transaction. // diff --git a/packages/zaino-proto/lightwallet-protocol/walletrpc/service.proto b/packages/zaino-proto/lightwallet-protocol/walletrpc/service.proto index d3dc8ba04..3b3cebf43 100644 --- a/packages/zaino-proto/lightwallet-protocol/walletrpc/service.proto +++ b/packages/zaino-proto/lightwallet-protocol/walletrpc/service.proto @@ -14,6 +14,7 @@ enum PoolType { TRANSPARENT = 1; SAPLING = 2; ORCHARD = 3; + IRONWOOD = 4; } // A BlockID message contains identifiers to select a block: a height or a @@ -179,6 +180,7 @@ message TreeState { uint32 time = 4; // Unix epoch time when the block was mined string saplingTree = 5; // sapling commitment tree state string orchardTree = 6; // orchard commitment tree state + string ironwoodTree = 7; // ironwood commitment tree state } enum ShieldedProtocol { diff --git a/packages/zaino-proto/src/proto/compact_formats.rs b/packages/zaino-proto/src/proto/compact_formats.rs index 2de2751ed..cc0b2153e 100644 --- a/packages/zaino-proto/src/proto/compact_formats.rs +++ b/packages/zaino-proto/src/proto/compact_formats.rs @@ -8,6 +8,9 @@ pub struct ChainMetadata { /// the size of the Orchard note commitment tree as of the end of this block #[prost(uint32, tag = "2")] pub orchard_commitment_tree_size: u32, + /// the size of the Ironwood note commitment tree as of the end of this block + #[prost(uint32, tag = "3")] + pub ironwood_commitment_tree_size: u32, } /// A compact representation of a Zcash block. /// @@ -75,6 +78,8 @@ pub struct CompactTx { pub outputs: ::prost::alloc::vec::Vec, #[prost(message, repeated, tag = "6")] pub actions: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag = "9")] + pub ironwood_actions: ::prost::alloc::vec::Vec, /// `CompactTxIn` values corresponding to the `vin` entries of the full transaction. /// /// Note: the single null-outpoint input for coinbase transactions is omitted. Light diff --git a/packages/zaino-proto/src/proto/service.rs b/packages/zaino-proto/src/proto/service.rs index 559d2efbe..7e5a82ed9 100644 --- a/packages/zaino-proto/src/proto/service.rs +++ b/packages/zaino-proto/src/proto/service.rs @@ -257,6 +257,9 @@ pub struct TreeState { /// orchard commitment tree state #[prost(string, tag = "6")] pub orchard_tree: ::prost::alloc::string::String, + /// ironwood commitment tree state + #[prost(string, tag = "7")] + pub ironwood_tree: ::prost::alloc::string::String, } #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetSubtreeRootsArg { @@ -322,6 +325,7 @@ pub enum PoolType { Transparent = 1, Sapling = 2, Orchard = 3, + Ironwood = 4, } impl PoolType { /// String value of the enum field names used in the ProtoBuf definition. @@ -334,6 +338,7 @@ impl PoolType { Self::Transparent => "TRANSPARENT", Self::Sapling => "SAPLING", Self::Orchard => "ORCHARD", + Self::Ironwood => "IRONWOOD", } } /// Creates an enum from field names used in the ProtoBuf definition. @@ -343,6 +348,7 @@ impl PoolType { "TRANSPARENT" => Some(Self::Transparent), "SAPLING" => Some(Self::Sapling), "ORCHARD" => Some(Self::Orchard), + "IRONWOOD" => Some(Self::Ironwood), _ => None, } } diff --git a/packages/zaino-proto/src/proto/utils.rs b/packages/zaino-proto/src/proto/utils.rs index c52a5d8df..693617d18 100644 --- a/packages/zaino-proto/src/proto/utils.rs +++ b/packages/zaino-proto/src/proto/utils.rs @@ -156,6 +156,7 @@ pub struct PoolTypeFilter { include_transparent: bool, include_sapling: bool, include_orchard: bool, + include_ironwood: bool, } impl std::default::Default for PoolTypeFilter { @@ -165,6 +166,7 @@ impl std::default::Default for PoolTypeFilter { include_transparent: false, include_sapling: true, include_orchard: true, + include_ironwood: true, } } } @@ -176,6 +178,7 @@ impl PoolTypeFilter { include_transparent: true, include_sapling: true, include_orchard: true, + include_ironwood: true, } } @@ -196,7 +199,7 @@ impl PoolTypeFilter { pub fn new_from_pool_types( pool_types: &Vec, ) -> Result { - if pool_types.len() > PoolType::Orchard as usize { + if pool_types.len() > PoolType::Ironwood as usize { return Err(PoolTypeError::InvalidPoolType); } @@ -211,6 +214,7 @@ impl PoolTypeFilter { PoolType::Transparent => filter.include_transparent = true, PoolType::Sapling => filter.include_sapling = true, PoolType::Orchard => filter.include_orchard = true, + PoolType::Ironwood => filter.include_ironwood = true, } } @@ -229,12 +233,16 @@ impl PoolTypeFilter { include_transparent: false, include_sapling: false, include_orchard: false, + include_ironwood: false, } } /// only internal use fn is_empty(&self) -> bool { - !self.include_transparent && !self.include_sapling && !self.include_orchard + !self.include_transparent + && !self.include_sapling + && !self.include_orchard + && !self.include_ironwood } /// retuns whether the filter includes transparent data @@ -252,6 +260,11 @@ impl PoolTypeFilter { self.include_orchard } + /// returns whether the filter includes ironwood data + pub fn includes_ironwood(&self) -> bool { + self.include_ironwood + } + /// Convert this filter into the corresponding `Vec`. /// /// The resulting vector contains each included pool type at most once. @@ -270,6 +283,10 @@ impl PoolTypeFilter { pool_types.push(PoolType::Orchard); } + if self.include_ironwood { + pool_types.push(PoolType::Ironwood); + } + pool_types } @@ -279,11 +296,13 @@ impl PoolTypeFilter { include_transparent: bool, include_sapling: bool, include_orchard: bool, + include_ironwood: bool, ) -> Self { PoolTypeFilter { include_transparent, include_sapling, include_orchard, + include_ironwood, } } } @@ -322,6 +341,7 @@ pub fn compact_block_with_pool_types( !compact_tx.spends.is_empty() || !compact_tx.outputs.is_empty() || !compact_tx.actions.is_empty() + || !compact_tx.ironwood_actions.is_empty() }); } else { for compact_tx in &mut block.vtx { @@ -339,6 +359,10 @@ pub fn compact_block_with_pool_types( if !pool_types.contains(&PoolType::Orchard) { compact_tx.actions.clear(); } + // strip out ironwood if not requested + if !pool_types.contains(&PoolType::Ironwood) { + compact_tx.ironwood_actions.clear(); + } } // Omit transactions that have no elements in any requested pool type. @@ -348,6 +372,7 @@ pub fn compact_block_with_pool_types( || !compact_tx.spends.is_empty() || !compact_tx.outputs.is_empty() || !compact_tx.actions.is_empty() + || !compact_tx.ironwood_actions.is_empty() }); } @@ -368,11 +393,18 @@ pub fn compact_block_to_nullifiers(mut block: CompactBlock) -> CompactBlock { ..Default::default() } } + for caction in &mut ctransaction.ironwood_actions { + *caction = CompactOrchardAction { + nullifier: caction.nullifier.clone(), + ..Default::default() + } + } } block.chain_metadata = Some(ChainMetadata { sapling_commitment_tree_size: 0, orchard_commitment_tree_size: 0, + ironwood_commitment_tree_size: 0, }); block } @@ -406,6 +438,7 @@ mod test { PoolType::Transparent, PoolType::Sapling, PoolType::Orchard, + PoolType::Ironwood, PoolType::Orchard, ] .to_vec(); @@ -418,11 +451,17 @@ mod test { #[test] fn test_pool_type_filter_t_z_o() { - let pools = [PoolType::Transparent, PoolType::Sapling, PoolType::Orchard].to_vec(); + let pools = [ + PoolType::Transparent, + PoolType::Sapling, + PoolType::Orchard, + PoolType::Ironwood, + ] + .to_vec(); assert_eq!( PoolTypeFilter::new_from_pool_types(&pools), - Ok(PoolTypeFilter::from_checked_parts(true, true, true)) + Ok(PoolTypeFilter::from_checked_parts(true, true, true, true)) ); } @@ -432,7 +471,9 @@ mod test { assert_eq!( PoolTypeFilter::new_from_pool_types(&pools), - Ok(PoolTypeFilter::from_checked_parts(true, false, false)) + Ok(PoolTypeFilter::from_checked_parts( + true, false, false, false + )) ); } @@ -447,7 +488,7 @@ mod test { #[test] fn test_pool_type_filter_includes_all() { assert_eq!( - PoolTypeFilter::from_checked_parts(true, true, true), + PoolTypeFilter::from_checked_parts(true, true, true, true), PoolTypeFilter::includes_all() ); } diff --git a/packages/zaino-state/src/backends/fetch.rs b/packages/zaino-state/src/backends/fetch.rs index 50a062735..3a6b56ca3 100644 --- a/packages/zaino-state/src/backends/fetch.rs +++ b/packages/zaino-state/src/backends/fetch.rs @@ -1826,14 +1826,35 @@ impl LightWalletIndexer for FetchServiceSubscriber { )), )?; - #[allow(deprecated)] - let (hash, height, time, sapling, orchard) = - ::z_get_treestate( - self, - hash_or_height.to_string(), - ) - .await? - .into_parts(); + let treestate_response = ::z_get_treestate( + self, + hash_or_height.to_string(), + ) + .await?; + + let sapling_tree = hex::encode( + treestate_response + .sapling() + .commitments() + .final_state() + .clone() + .unwrap_or_default(), + ); + let orchard_tree = hex::encode( + treestate_response + .orchard() + .commitments() + .final_state() + .clone() + .unwrap_or_default(), + ); + let ironwood_tree = treestate_response + .ironwood() + .clone() + .and_then(|treestate| treestate.commitments().final_state().clone()) + .map(hex::encode) + .unwrap_or_default(); + Ok(TreeState { network: self .config @@ -1841,11 +1862,12 @@ impl LightWalletIndexer for FetchServiceSubscriber { .network .to_zebra_network() .bip70_network_name(), - height: height.0 as u64, - hash: hash.to_string(), - time, - sapling_tree: sapling.map(hex::encode).unwrap_or_default(), - orchard_tree: orchard.map(hex::encode).unwrap_or_default(), + height: treestate_response.height().0 as u64, + hash: treestate_response.hash().to_string(), + time: treestate_response.time(), + sapling_tree, + orchard_tree, + ironwood_tree, }) } @@ -2058,7 +2080,7 @@ impl LightWalletIndexer for FetchServiceSubscriber { .unwrap_or_default(), upgrade_name: nu_name.to_string(), upgrade_height: nu_height.0 as u64, - lightwallet_protocol_version: "v0.4.0".to_string(), + lightwallet_protocol_version: "v0.5.0".to_string(), }) } diff --git a/packages/zaino-state/src/backends/state.rs b/packages/zaino-state/src/backends/state.rs index 736fa78ca..d794c74f2 100644 --- a/packages/zaino-state/src/backends/state.rs +++ b/packages/zaino-state/src/backends/state.rs @@ -1494,7 +1494,17 @@ impl ZcashIndexer for StateServiceSubscriber { )))?, }; - let (sapling, orchard) = self.indexer.get_treestate(block_data.hash()).await?; + let treestate: GetTreestateResponse = self + .rpc_client + .get_treestate(block_data.hash().to_string()) + .await? + .try_into() + .map_err(|_error| { + StateServiceError::RpcError(RpcError::new_from_legacycode( + zebra_rpc::server::error::LegacyCode::InvalidParameter, + "Failed to parse treestate.", + )) + })?; let time: u32 = block_data.data().time().try_into().map_err(|_error| { StateServiceError::RpcError(RpcError::new_from_legacycode( zebra_rpc::server::error::LegacyCode::InvalidParameter, @@ -1502,13 +1512,32 @@ impl ZcashIndexer for StateServiceSubscriber { )) })?; - #[allow(deprecated)] - Ok(GetTreestateResponse::from_parts( + let sapling = treestate.sapling().commitments().final_state().clone(); + let orchard = treestate.orchard().commitments().final_state().clone(); + let ironwood = treestate + .ironwood() + .clone() + .and_then(|ironwood| ironwood.commitments().final_state().clone()); + + Ok(GetTreestateResponse::new( (*block_data.hash()).into(), block_data.height().into(), time, - sapling, - orchard, + None, + zebra_rpc::client::Treestate::new(zebra_rpc::client::Commitments::new( + sapling.as_deref().map(ToOwned::to_owned), + None, + )), + zebra_rpc::client::Treestate::new(zebra_rpc::client::Commitments::new( + orchard.as_deref().map(ToOwned::to_owned), + None, + )), + ironwood.map(|final_state| { + zebra_rpc::client::Treestate::new(zebra_rpc::client::Commitments::new( + Some(final_state), + None, + )) + }), )) } .await; @@ -2504,14 +2533,36 @@ impl LightWalletIndexer for StateServiceSubscriber { "Invalid hash or height", )), )?; - #[allow(deprecated)] - let (hash, height, time, sapling, orchard) = - ::z_get_treestate( - self, - hash_or_height.to_string(), - ) - .await? - .into_parts(); + + let treestate_response = ::z_get_treestate( + self, + hash_or_height.to_string(), + ) + .await?; + + let sapling_tree = hex::encode( + treestate_response + .sapling() + .commitments() + .final_state() + .clone() + .unwrap_or_default(), + ); + let orchard_tree = hex::encode( + treestate_response + .orchard() + .commitments() + .final_state() + .clone() + .unwrap_or_default(), + ); + let ironwood_tree = treestate_response + .ironwood() + .clone() + .and_then(|treestate| treestate.commitments().final_state().clone()) + .map(hex::encode) + .unwrap_or_default(); + Ok(TreeState { network: self .config @@ -2519,11 +2570,12 @@ impl LightWalletIndexer for StateServiceSubscriber { .network .to_zebra_network() .bip70_network_name(), - height: height.0 as u64, - hash: hash.to_string(), - time, - sapling_tree: sapling.map(hex::encode).unwrap_or_default(), - orchard_tree: orchard.map(hex::encode).unwrap_or_default(), + height: treestate_response.height().0 as u64, + hash: treestate_response.hash().to_string(), + time: treestate_response.time(), + sapling_tree, + orchard_tree, + ironwood_tree, }) } @@ -2735,7 +2787,7 @@ impl LightWalletIndexer for StateServiceSubscriber { .unwrap_or_default(), upgrade_name: nu_name.to_string(), upgrade_height: nu_height.0 as u64, - lightwallet_protocol_version: "v0.4.0".to_string(), + lightwallet_protocol_version: "v0.5.0".to_string(), }) } diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/compact_block.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/compact_block.rs index 9dc42aadf..226e23d17 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/compact_block.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/compact_block.rs @@ -218,6 +218,7 @@ impl DbV1 { spends, outputs, actions, + ironwood_actions: Vec::new(), vin, vout, }) @@ -243,6 +244,7 @@ impl DbV1 { let chain_metadata = zaino_proto::proto::compact_formats::ChainMetadata { sapling_commitment_tree_size: commitment_tree_data.sizes().sapling(), orchard_commitment_tree_size: commitment_tree_data.sizes().orchard(), + ironwood_commitment_tree_size: 0, }; // ----- Construct CompactBlock ----- @@ -1062,6 +1064,7 @@ impl DbV1 { spends, outputs, actions, + ironwood_actions: Vec::new(), vin, vout, }); @@ -1082,6 +1085,7 @@ impl DbV1 { let chain_metadata = zaino_proto::proto::compact_formats::ChainMetadata { sapling_commitment_tree_size: commitment_tree_data.sizes().sapling(), orchard_commitment_tree_size: commitment_tree_data.sizes().orchard(), + ironwood_commitment_tree_size: 0, }; let compact_block = zaino_proto::proto::compact_formats::CompactBlock { diff --git a/packages/zaino-state/src/chain_index/types/db/legacy.rs b/packages/zaino-state/src/chain_index/types/db/legacy.rs index ee4074d5e..fe2220918 100644 --- a/packages/zaino-state/src/chain_index/types/db/legacy.rs +++ b/packages/zaino-state/src/chain_index/types/db/legacy.rs @@ -1083,6 +1083,7 @@ impl IndexedBlock { chain_metadata: Some(zaino_proto::proto::compact_formats::ChainMetadata { sapling_commitment_tree_size, orchard_commitment_tree_size, + ironwood_commitment_tree_size: 0, }), } } @@ -1393,6 +1394,7 @@ impl CompactTxData { spends, outputs, actions, + ironwood_actions: Vec::new(), vin, vout, } @@ -1414,7 +1416,7 @@ impl TryFrom<(u64, zaino_fetch::chain::transaction::FullTransaction)> for Compac .try_into() .map_err(|_| "txid must be 32 bytes".to_string())?; - let (sapling_balance, orchard_balance) = tx.value_balances(); + let (sapling_balance, orchard_balance, _ironwood_balance) = tx.value_balances(); let vin: Vec = tx .transparent_inputs() From 75fa0ff6022b4af24f7c9a349a28187450b57fd6 Mon Sep 17 00:00:00 2001 From: idky137 Date: Fri, 3 Jul 2026 22:08:28 +0100 Subject: [PATCH 031/134] updated IndexedBlock and internal block parsing --- .../zaino-fetch/src/jsonrpsee/response.rs | 2 +- packages/zaino-state/src/backends/fetch.rs | 3 +- packages/zaino-state/src/chain_index.rs | 20 +- .../zaino-state/src/chain_index/encoding.rs | 70 +++- .../src/chain_index/finalised_state.rs | 29 +- .../chain_index/finalised_state/capability.rs | 32 +- .../src/chain_index/finalised_state/entry.rs | 312 +++++++++++------ .../finalised_source/ephemeral.rs | 28 +- .../finalised_state/finalised_source/v1.rs | 8 +- .../finalised_source/v1/block_core.rs | 16 +- .../finalised_source/v1/block_shielded.rs | 8 +- .../finalised_source/v1/block_transparent.rs | 6 +- .../finalised_source/v1/indexed_block.rs | 10 +- .../v1/transparent_address_history.rs | 108 +++--- .../v1/tx_out_set_accumulator.rs | 4 +- .../finalised_source/v1/write_core.rs | 3 + .../src/chain_index/non_finalised_state.rs | 18 +- .../zaino-state/src/chain_index/source.rs | 9 +- .../chain_index/source/mockchain_source.rs | 25 +- .../chain_index/source/validator_connector.rs | 112 +++++- .../chain_index/tests/finalised_state/v1.rs | 2 + .../src/chain_index/tests/mockchain_tests.rs | 2 +- .../chain_index/tests/proptest_blockgen.rs | 6 +- .../src/chain_index/tests/vectors.rs | 2 + .../src/chain_index/types/db/block.rs | 20 +- .../src/chain_index/types/db/commitment.rs | 174 ++++++++-- .../src/chain_index/types/db/legacy.rs | 318 ++++++++++++++---- .../src/chain_index/types/db/metadata.rs | 25 +- .../src/chain_index/types/helpers.rs | 76 ++++- 29 files changed, 1134 insertions(+), 314 deletions(-) diff --git a/packages/zaino-fetch/src/jsonrpsee/response.rs b/packages/zaino-fetch/src/jsonrpsee/response.rs index b6876074d..226c20bfd 100644 --- a/packages/zaino-fetch/src/jsonrpsee/response.rs +++ b/packages/zaino-fetch/src/jsonrpsee/response.rs @@ -1351,7 +1351,7 @@ pub struct GetTreestateResponse { /// A treestate containing an Ironwood note commitment tree, hex-encoded. Only present from /// NU6.3, so that pre-NU6.3 responses are unchanged. #[serde(skip_serializing_if = "Option::is_none")] - ironwood: Option, + pub ironwood: Option, } /// Error type for the `get_treestate` RPC request. diff --git a/packages/zaino-state/src/backends/fetch.rs b/packages/zaino-state/src/backends/fetch.rs index 3a6b56ca3..208541ac0 100644 --- a/packages/zaino-state/src/backends/fetch.rs +++ b/packages/zaino-state/src/backends/fetch.rs @@ -606,7 +606,8 @@ impl ZcashIndexer for FetchServiceSubscriber { )?, }; - let (sapling, orchard) = self.indexer.get_treestate(block_data.hash()).await?; + let (sapling, orchard, _ironwood) = + self.indexer.get_treestate(block_data.hash()).await?; let time: u32 = block_data.data().time().try_into().map_err(|_error| { #[allow(deprecated)] FetchServiceError::RpcError(RpcError::new_from_legacycode( diff --git a/packages/zaino-state/src/chain_index.rs b/packages/zaino-state/src/chain_index.rs index 87e0d7f68..d8881c4a6 100644 --- a/packages/zaino-state/src/chain_index.rs +++ b/packages/zaino-state/src/chain_index.rs @@ -501,7 +501,9 @@ pub trait ChainIndex { fn get_treestate( &self, hash: &types::BlockHash, - ) -> impl std::future::Future>, Option>), Self::Error>>; + ) -> impl std::future::Future< + Output = Result<(Option>, Option>, Option>), Self::Error>, + >; /// Returns the subtree roots fn get_subtree_roots( @@ -1122,8 +1124,8 @@ async fn compact_block_from_source( .get_commitment_tree_roots(types::BlockHash::from(block.hash())) .await .map_err(ChainIndexError::backing_validator)?; - let (sapling_root, sapling_size, orchard_root, orchard_size) = - TreeRootData::new(tree_roots.0, tree_roots.1).extract_with_defaults(); + let (sapling_root, sapling_size, orchard_root, orchard_size, ironwood_root, ironwood_size) = + TreeRootData::new(tree_roots.0, tree_roots.1, tree_roots.2).extract_with_defaults(); let metadata = BlockMetadata::new( sapling_root, @@ -1140,6 +1142,13 @@ async fn compact_block_from_source( "orchard commitment tree size overflow", )) })?, + ironwood_root, + ironwood_size.try_into().map_err(|_| { + ChainIndexError::backing_validator(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "ironwood commitment tree size overflow", + )) + })?, None, // parent chainwork unknown — single-block construction network, ); @@ -2389,7 +2398,7 @@ impl ChainIndex for NodeBackedChainIndexSubscriber Result<(Option>, Option>), Self::Error> { + ) -> Result<(Option>, Option>, Option>), Self::Error> { let snapshot = self.snapshot_nonfinalized_state().await?; if !self.block_hash_known_for_treestate(&snapshot, hash).await? { return Err(ChainIndexError::internal(format!( @@ -2779,6 +2788,8 @@ pub enum ShieldedPool { Sapling, /// Orchard Orchard, + /// Ironwood + Ironwood, } impl ShieldedPool { @@ -2790,6 +2801,7 @@ impl ShieldedPool { match self { ShieldedPool::Sapling => "sapling".to_string(), ShieldedPool::Orchard => "orchard".to_string(), + ShieldedPool::Ironwood => "ironwood".to_string(), } } } diff --git a/packages/zaino-state/src/chain_index/encoding.rs b/packages/zaino-state/src/chain_index/encoding.rs index 9101f31dc..4f3bba6de 100644 --- a/packages/zaino-state/src/chain_index/encoding.rs +++ b/packages/zaino-state/src/chain_index/encoding.rs @@ -273,16 +273,72 @@ pub trait ZainoVersionedSerde: Sized { } } -/// Defines the fixed encoded length of a database record. +/// Defines the fixed encoded length metadata for a versioned database record. +/// +/// This trait is intentionally version-aware. +/// +/// A type can be fixed-length in one historical wire version and variable-length +/// in a later wire version. For that reason, fixed length must not be represented +/// as a single latest constant. Instead, callers must ask for the fixed body +/// length of a specific encoded version. +/// +/// Lengths returned by this trait are body lengths only: they exclude the +/// leading `ZainoVersionedSerde` version tag byte. +/// +/// Returning `None` means that the requested version is either: +/// - unsupported by this type, +/// - not fixed-length, +/// - or must be decoded by a variable-length wrapper instead. pub trait FixedEncodedLen { - /// the fixed encoded length of a database record *not* incuding the version byte. - const ENCODED_LEN: usize; - - /// Length of version tag in bytes. + /// Length of the version tag in bytes. const VERSION_TAG_LEN: usize = 1; - /// the fixed encoded length of a database record *incuding* the version byte. - const VERSIONED_LEN: usize = Self::ENCODED_LEN + Self::VERSION_TAG_LEN; + /// Returns the fixed encoded body length for `version`, excluding the + /// version tag. + /// + /// Implementations must return the historical length for the exact on-disk + /// version requested. They must not return the current/latest struct length + /// for all versions. + fn encoded_len(version: u8) -> Option; + + /// Returns the fixed encoded length for `version`, including the version tag. + fn versioned_len(version: u8) -> Option { + Self::encoded_len(version).map(|len| len + Self::VERSION_TAG_LEN) + } + + /// Returns the fixed body length of the current/latest wire version. + /// + /// Returns `None` if the latest version is variable-length. + fn latest_encoded_len() -> io::Result + where + Self: ZainoVersionedSerde, + { + Self::encoded_len(Self::VERSION).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("no fixed len available for version tag {}", Self::VERSION), + ) + }) + } + + /// Returns the fixed encoded length of the current/latest wire version, + /// including the version tag. + /// + /// Returns `None` if the latest version is variable-length. + fn latest_versioned_len() -> io::Result + where + Self: ZainoVersionedSerde, + { + Self::versioned_len(Self::VERSION).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + format!( + "no fixed versioned len available for version tag {}", + Self::VERSION + ), + ) + }) + } } /* ──────────────────────────── CompactSize helpers ────────────────────────────── */ diff --git a/packages/zaino-state/src/chain_index/finalised_state.rs b/packages/zaino-state/src/chain_index/finalised_state.rs index 575ba1e5b..6b25f458c 100644 --- a/packages/zaino-state/src/chain_index/finalised_state.rs +++ b/packages/zaino-state/src/chain_index/finalised_state.rs @@ -254,6 +254,7 @@ pub(crate) async fn build_indexed_block_from_source( network: zaino_common::Network, sapling_activation_height: zebra_chain::block::Height, nu5_activation_height: Option, + nu6_3_activation_height: Option, height_int: u32, parent_chainwork: Option, ) -> Result { @@ -276,10 +277,14 @@ pub(crate) async fn build_indexed_block_from_source( let block_hash = BlockHash::from(block.hash().0); // Fetch sapling / orchard commitment tree data if above the relevant network upgrade. - let (sapling_opt, orchard_opt) = source.get_commitment_tree_roots(block_hash).await?; + let (sapling_opt, orchard_opt, ironwood_opt) = + source.get_commitment_tree_roots(block_hash).await?; + let is_sapling_active = height_int >= sapling_activation_height.0; let is_orchard_active = nu5_activation_height .is_some_and(|nu5_activation_height| height_int >= nu5_activation_height.0); + let is_ironwood_active = nu6_3_activation_height + .is_some_and(|nu6_3_activation_height| height_int >= nu6_3_activation_height.0); let (sapling_root, sapling_size) = if is_sapling_active { sapling_opt.ok_or_else(|| { @@ -301,11 +306,23 @@ pub(crate) async fn build_indexed_block_from_source( (zebra_chain::orchard::tree::Root::default(), 0) }; + let (ironwood_root, ironwood_size) = if is_ironwood_active { + ironwood_opt.ok_or_else(|| { + FinalisedStateError::BlockchainSourceError(BlockchainSourceError::Unrecoverable( + format!("missing Ironwood commitment tree root for block {block_hash}"), + )) + })? + } else { + (zebra_chain::orchard::tree::Root::default(), 0) + }; + let metadata = BlockMetadata::new( sapling_root, sapling_size as u32, orchard_root, orchard_size as u32, + ironwood_root, + ironwood_size as u32, parent_chainwork, network.to_zebra_network(), ); @@ -1056,7 +1073,8 @@ impl FinalisedState { })?; let block_hash = BlockHash::from(block.hash().0); - let (sapling_opt, orchard_opt) = source.get_commitment_tree_roots(block_hash).await?; + let (sapling_opt, orchard_opt, ironwood_opt) = + source.get_commitment_tree_roots(block_hash).await?; let (sapling_root, sapling_size) = sapling_opt.ok_or_else(|| { FinalisedStateError::BlockchainSourceError(BlockchainSourceError::Unrecoverable( format!("missing Sapling commitment tree root for block {block_hash}"), @@ -1067,12 +1085,19 @@ impl FinalisedState { format!("missing Orchard commitment tree root for block {block_hash}"), )) })?; + let (ironwood_root, ironwood_size) = ironwood_opt.ok_or_else(|| { + FinalisedStateError::BlockchainSourceError(BlockchainSourceError::Unrecoverable( + format!("missing Orchard commitment tree root for block {block_hash}"), + )) + })?; let metadata = BlockMetadata::new( sapling_root, sapling_size as u32, orchard_root, orchard_size as u32, + ironwood_root, + ironwood_size as u32, parent_chainwork, cfg.network.to_zebra_network(), ); diff --git a/packages/zaino-state/src/chain_index/finalised_state/capability.rs b/packages/zaino-state/src/chain_index/finalised_state/capability.rs index 0193c5d69..798208880 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/capability.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/capability.rs @@ -372,12 +372,18 @@ impl ZainoVersionedSerde for DbMetadata { } } -/// `DbMetadata` has a fixed encoded body length. +/// Fixed-length encoding metadata for `DbMetadata`. /// +/// v1 consists of: /// Body length = `DbVersion::VERSIONED_LEN` (12 + 1) + 32-byte schema hash /// + `MigrationStatus::VERSIONED_LEN` (1 + 1) = 47 bytes. impl FixedEncodedLen for DbMetadata { - const ENCODED_LEN: usize = DbVersion::VERSIONED_LEN + 32 + MigrationStatus::VERSIONED_LEN; + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some(47), + _ => None, + } + } } /// Human-readable summary for logs. @@ -535,9 +541,16 @@ impl ZainoVersionedSerde for DbVersion { } } -// DbVersion: body = 3*(4-byte u32) - 12 bytes +/// Fixed-length encoding metadata for `DbVersion`. +/// +/// v1 consists of *(4-byte u32) = 12 bytes impl FixedEncodedLen for DbVersion { - const ENCODED_LEN: usize = 4 + 4 + 4; + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some(12), + _ => None, + } + } } /// Formats as `{major}.{minor}.{patch}` for logs and diagnostics. @@ -636,9 +649,16 @@ impl ZainoVersionedSerde for MigrationStatus { } } -/// `MigrationStatus` has a fixed 1-byte encoded body (discriminator). +/// Fixed-length encoding metadata for `MigrationStatus`. +/// +/// v1 consists of a single byte impl FixedEncodedLen for MigrationStatus { - const ENCODED_LEN: usize = 1; + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some(1), + _ => None, + } + } } // ***** Core Database functionality ***** diff --git a/packages/zaino-state/src/chain_index/finalised_state/entry.rs b/packages/zaino-state/src/chain_index/finalised_state/entry.rs index 9877c80a5..64a1d0dfa 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/entry.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/entry.rs @@ -66,7 +66,8 @@ //! maintain a decode path (or bump the DB major version and migrate). use crate::{ - read_fixed_le, version, write_fixed_le, CompactSize, FixedEncodedLen, ZainoVersionedSerde, + read_fixed_le, read_u8, version, write_fixed_le, CompactSize, FixedEncodedLen, + ZainoVersionedSerde, }; use blake2::{ @@ -108,93 +109,86 @@ pub(crate) struct StoredEntryFixed { } impl StoredEntryFixed { - /// Constructs a new checksummed entry for `item` under `key`. + /// Constructs a new checksummed fixed-length entry for `item` under `key`. /// - /// The checksum is computed as: - /// `blake2b256(encoded_key || item.serialize())`. + /// The checksum is computed over: /// - /// # Key requirements - /// `key` must be the exact byte encoding used as the LMDB key for this record. If the caller - /// hashes a different key encoding than what is used for storage, verification will fail. + /// `encoded_key || item.serialize()` + /// + /// This constructor may only be used when the latest version of `T` is still + /// fixed-length. If the latest version of `T` has become variable-length, + /// callers must use the variable-length stored-entry wrapper for new writes. + /// + /// # Panics + /// + /// Panics if `T::serialize()` fails. Existing code already treated this path + /// as infallible. If desired, this can be made fallible later without changing + /// the on-disk format. pub(crate) fn new>(key: K, item: T) -> Self { let body = { - let mut v = Vec::with_capacity(T::VERSIONED_LEN); + let len = T::latest_versioned_len().unwrap_or(0); + let mut v = Vec::with_capacity(len); item.serialize(&mut v).unwrap(); v }; + let checksum = Self::blake2b256(&[key.as_ref(), &body].concat()); Self { item, checksum } } /// Verifies the checksum for this entry under `key`. /// - /// Returns `true` if and only if: - /// `self.checksum == blake2b256(encoded_key || item.serialize())`. - /// - /// # Key requirements - /// `key` must be the exact byte encoding used as the LMDB key for this record. - /// - /// # Usage - /// Callers should treat a checksum mismatch as a corruption or incompatibility signal and - /// return a hard error (or trigger a rebuild path), depending on context. + /// Verification tries every supported historical encoding from latest down + /// to v1. This is required because the decoded in-memory item no longer + /// remembers which exact version produced the stored checksum. pub(crate) fn verify>(&self, key: K) -> bool { - // Iterate from latest (T::VERSION) down to 1 (inclusive). let mut v = T::VERSION; + loop { - // Try to obtain the encoded bytes for this candidate version (tag + body). match self.item.to_bytes_with_version(v) { Ok(item_bytes) => { - // Compute the candidate checksum over (encoded_key || item_bytes). let candidate = Self::blake2b256(&[key.as_ref(), &item_bytes].concat()); if candidate == self.checksum { return true; } } - Err(_) => { - // This version not supported by the type's encoder; try older version. - } + Err(_) => {} } if v == 1 { break; } + v = v.saturating_sub(1); } false } - /// Returns a reference to the inner record. + /// Returns a reference to the inner decoded record. pub(crate) fn inner(&self) -> &T { &self.item } /// Computes a BLAKE2b-256 checksum over `data`. - /// - /// This is the hashing primitive used by both wrappers. The checksum is not keyed. pub(crate) fn blake2b256(data: &[u8]) -> [u8; 32] { let mut hasher = Blake2bVar::new(32).expect("Failed to create hasher"); hasher.update(data); + let mut output = [0u8; 32]; hasher .finalize_variable(&mut output) .expect("Failed to finalize hash"); + output } /// Builds serialized stored-entry bytes using a specific inner item version. /// - /// This is required when writing historical database values whose inner item must be encoded - /// with an older version than `T::VERSION`. - /// - /// The returned bytes are: - /// - StoredEntryFixed version tag - /// - inner item bytes encoded with `item_version` - /// - checksum over `encoded_key || inner_item_bytes` + /// This is used for tests and historical fixture construction. /// - /// This method returns serialized bytes directly because `StoredEntryFixed` does not store - /// the inner item version. Constructing `Self` alone would lose the requested item version - /// before the value is written. + /// The selected `item_version` must be fixed-length according to + /// `T::versioned_len(item_version)`. #[cfg(test)] pub(crate) fn to_bytes_with_item_version>( key: K, @@ -203,7 +197,14 @@ impl StoredEntryFixed { ) -> io::Result> { let item_bytes = item.to_bytes_with_version(item_version)?; - if item_bytes.len() != T::VERSIONED_LEN { + let expected_len = T::versioned_len(item_version).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "requested item version is not fixed-length", + ) + })?; + + if item_bytes.len() != expected_len { return Err(io::Error::new( io::ErrorKind::InvalidData, "encoded fixed-length item has an unexpected length", @@ -212,7 +213,7 @@ impl StoredEntryFixed { let checksum = Self::blake2b256(&[key.as_ref(), &item_bytes].concat()); - let mut stored_entry_bytes = Vec::with_capacity(1 + T::VERSIONED_LEN + 32); + let mut stored_entry_bytes = Vec::with_capacity(1 + expected_len + 32); stored_entry_bytes.push(Self::VERSION); stored_entry_bytes.extend_from_slice(&item_bytes); stored_entry_bytes.extend_from_slice(&checksum); @@ -221,44 +222,94 @@ impl StoredEntryFixed { } } -/// Versioned on-disk encoding for fixed-length checksummed entries. -/// -/// Body layout (after the `StoredEntryFixed` version tag): -/// 1. `T::serialize()` bytes (fixed length: `T::VERSIONED_LEN`) -/// 2. 32-byte checksum -/// -/// Note: `T::serialize()` includes `T`’s own version tag and body. impl ZainoVersionedSerde for StoredEntryFixed { const VERSION: u8 = version::V1; + /// Encodes the latest fixed stored-entry layout. fn encode_latest(&self, w: &mut W) -> io::Result<()> { Self::encode_v1(self, w) } + /// Decodes the latest fixed stored-entry layout. fn decode_latest(r: &mut R) -> io::Result { Self::decode_v1(r) } + /// Encodes a fixed stored entry. + /// + /// The inner item is serialized with its own latest version tag and body, + /// followed by the 32-byte checksum. + /// + /// This method requires the latest version of `T` to still be fixed-length. fn encode_v1(&self, w: &mut W) -> io::Result<()> { - self.item.serialize(&mut *w)?; + let expected_len = T::latest_versioned_len().map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidInput, + "latest item version is not fixed-length", + ) + })?; + + let mut item_bytes = Vec::with_capacity(expected_len); + self.item.serialize(&mut item_bytes)?; + + if item_bytes.len() != expected_len { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "encoded fixed-length item has an unexpected length", + )); + } + + w.write_all(&item_bytes)?; write_fixed_le::<32, _>(&mut *w, &self.checksum) } + /// Decodes a fixed stored entry. + /// + /// This method reads the inner item version tag first, then uses + /// `T::encoded_len(version)` to determine how many body bytes to read. + /// + /// This is what keeps old fixed-length rows readable after the latest `T` + /// changes size or becomes variable-length. fn decode_v1(r: &mut R) -> io::Result { - let mut body = vec![0u8; T::VERSIONED_LEN]; - r.read_exact(&mut body)?; - let item = T::deserialize(&body[..])?; + let item_version = read_u8(&mut *r)?; + + let body_len = T::encoded_len(item_version).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "stored item version is not fixed-length or is unsupported", + ) + })?; + let mut item_bytes = Vec::with_capacity(1 + body_len); + item_bytes.push(item_version); + item_bytes.resize(1 + body_len, 0); + + r.read_exact(&mut item_bytes[1..])?; + + let item = T::deserialize(&item_bytes[..])?; let checksum = read_fixed_le::<32, _>(r)?; + Ok(Self { item, checksum }) } } -/// `StoredEntryFixed` has a fixed encoded body length. -/// -/// Body length = `T::VERSIONED_LEN` + 32 bytes checksum. +/// Fixed stored entries are only fixed-length when the latest inner item version +/// is fixed-length. impl FixedEncodedLen for StoredEntryFixed { - const ENCODED_LEN: usize = T::VERSIONED_LEN + 32; + /// Returns the fixed encoded body length for a stored fixed entry. + /// + /// `StoredEntryFixed` v1 consists of: + /// - the latest fixed-length versioned encoding of the inner item `T` + /// - a 32-byte checksum + /// + /// If the latest version of `T` is not fixed-length, this returns `None` + /// because `StoredEntryFixed` cannot have a fixed latest encoded length. + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => T::latest_versioned_len().ok().map(|item_len| item_len + 32), + _ => None, + } + } } /// Variable-length checksummed database value wrapper. @@ -439,105 +490,177 @@ mod tests { pub x: u32, } - // TestInner: versioned type with two encodings: - // - v1: x as little-endian (body only) - // - v2: x as big-endian (body only) and v2 is the current version impl ZainoVersionedSerde for TestInner { const VERSION: u8 = version::V2; fn encode_latest(&self, w: &mut W) -> io::Result<()> { self.encode_v2(w) } + fn decode_latest(r: &mut R) -> io::Result { Self::decode_v2(r) } + /// v1 body: + /// - 4 bytes: `x` as little-endian `u32` fn encode_v1(&self, w: &mut W) -> io::Result<()> { write_u32_le(w, self.x) } + + /// v2 body: + /// - 4 bytes: `x` as big-endian `u32` + /// - 4 bytes: duplicate/check field as big-endian `u32` + /// + /// This intentionally makes v2 a different fixed length from v1 so the + /// `StoredEntryFixed` decoder must use the version tag to select the + /// correct body length. fn encode_v2(&self, w: &mut W) -> io::Result<()> { - write_u32_be(w, self.x) + write_u32_be(&mut *w, self.x)?; + write_u32_be(&mut *w, !self.x) } fn decode_v1(r: &mut R) -> io::Result { let x = read_u32_le(r)?; Ok(TestInner { x }) } + fn decode_v2(r: &mut R) -> io::Result { - let x = read_u32_be(r)?; + let x = read_u32_be(&mut *r)?; + let check = read_u32_be(&mut *r)?; + + if check != !x { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "invalid TestInner v2 check field", + )); + } + Ok(TestInner { x }) } } - // Make TestInner fixed-length for StoredEntryFixed tests. impl FixedEncodedLen for TestInner { - // body length (no version tag): 4 bytes (u32) - const ENCODED_LEN: usize = 4; + /// Historical fixed body lengths, excluding the version tag. + /// + /// v1 is 4 bytes. + /// v2 is 8 bytes. + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some(4), + version::V2 => Some(8), + _ => None, + } + } } - // Helper: simple key bytes for tests fn key_bytes() -> Vec { b"test-key".to_vec() } - // StoredEntryFixed: latest roundtrip (new -> to_bytes -> from_bytes -> verify) + #[test] + fn test_inner_versions_have_different_lengths() { + let inner = TestInner { x: 0x1122_3344 }; + + let v1 = inner + .to_bytes_with_version(version::V1) + .expect("inner v1 bytes"); + let v2 = inner + .to_bytes_with_version(version::V2) + .expect("inner v2 bytes"); + + assert_eq!(v1.len(), 1 + 4); + assert_eq!(v2.len(), 1 + 8); + assert_ne!(v1.len(), v2.len()); + + assert_eq!(TestInner::versioned_len(version::V1), Some(1 + 4)); + assert_eq!(TestInner::versioned_len(version::V2), Some(1 + 8)); + } + #[test] fn stored_entry_fixed_roundtrip_latest() { let inner = TestInner { x: 0x1122_3344 }; let key = key_bytes(); - // Construct wrapper using current serializer (StoredEntryFixed::new uses latest encoding) let wrapper = StoredEntryFixed::new(&key, inner.clone()); - // Verify should succeed for the same key - assert!( - wrapper.verify(&key), - "wrapper verify (latest) should succeed" - ); + assert!(wrapper.verify(&key), "wrapper verify latest should succeed"); - // Encode wrapper to bytes and decode back let bytes = wrapper.to_bytes().expect("wrapper to_bytes"); let parsed = StoredEntryFixed::::from_bytes(&bytes).expect("from_bytes"); + assert_eq!(parsed.item, inner); assert_eq!(parsed.checksum, wrapper.checksum); - - // parsed wrapper should verify with same key assert!(parsed.verify(&key)); } #[test] - // StoredEntryFixed: historical v1 body present on disk -> from_bytes + verify must succeed - fn stored_entry_fixed_verify_old_v1() { + fn stored_entry_fixed_decodes_historical_v1_with_shorter_length() { let inner = TestInner { x: 0xAABB_CCDD }; let key = key_bytes(); - // Produce item bytes according to historical v1 (tag + body) let item_bytes_v1 = inner .to_bytes_with_version(version::V1) .expect("inner v1 bytes"); - // Compute checksum over (key || item_bytes_v1) + assert_eq!( + item_bytes_v1.len(), + TestInner::versioned_len(version::V1).unwrap() + ); + assert_ne!( + item_bytes_v1.len(), + TestInner::latest_versioned_len().unwrap() + ); + let mut digest_input = Vec::with_capacity(key.len() + item_bytes_v1.len()); digest_input.extend_from_slice(&key); digest_input.extend_from_slice(&item_bytes_v1); + let checksum = StoredEntryFixed::::blake2b256(&digest_input); - // Manually build on-disk raw bytes for StoredEntryFixed: - // [StoredEntryFixed::VERSION] + item_bytes_v1 (should have length T::VERSIONED_LEN) + checksum let mut raw = Vec::new(); raw.push(StoredEntryFixed::::VERSION); raw.extend_from_slice(&item_bytes_v1); raw.extend_from_slice(&checksum); - // Parse using from_bytes (which will call decode_v1 and reconstruct wrapper) - let parsed = StoredEntryFixed::::from_bytes(&raw).expect("from_bytes v1"); - // parsed.item should equal the decoded inner + let parsed = StoredEntryFixed::::from_bytes(&raw) + .expect("fixed wrapper should decode historical shorter v1 item"); + + assert_eq!(parsed.item, inner); + assert_eq!(parsed.checksum, checksum); + assert!(parsed.verify(&key)); + } + + #[test] + fn stored_entry_fixed_decodes_latest_v2_with_longer_length() { + let inner = TestInner { x: 0x5566_7788 }; + let key = key_bytes(); + + let item_bytes_v2 = inner + .to_bytes_with_version(version::V2) + .expect("inner v2 bytes"); + + assert_eq!( + item_bytes_v2.len(), + TestInner::versioned_len(version::V2).unwrap() + ); + + let checksum = StoredEntryFixed::::blake2b256( + &[key.as_slice(), item_bytes_v2.as_slice()].concat(), + ); + + let mut raw = Vec::new(); + raw.push(StoredEntryFixed::::VERSION); + raw.extend_from_slice(&item_bytes_v2); + raw.extend_from_slice(&checksum); + + let parsed = StoredEntryFixed::::from_bytes(&raw) + .expect("fixed wrapper should decode latest longer v2 item"); + assert_eq!(parsed.item, inner); - // verify should succeed using the same key (it will try v2 then v1 candidate encodings) + assert_eq!(parsed.checksum, checksum); assert!(parsed.verify(&key)); } - // StoredEntryFixed: verify fails on tampered checksum or key #[test] fn stored_entry_fixed_verify_tamper() { let inner = TestInner { x: 0x0102_0304 }; @@ -546,16 +669,15 @@ mod tests { let mut wrapper = StoredEntryFixed::new(&key, inner.clone()); assert!(wrapper.verify(&key)); - // Tamper checksum (flip a byte) and ensure verify fails wrapper.checksum[0] ^= 0xff; assert!( !wrapper.verify(&key), "verify should fail with tampered checksum" ); - // Restore checksum and verify ok, then check wrong key fails wrapper = StoredEntryFixed::new(&key, inner.clone()); assert!(wrapper.verify(&key)); + let wrong_key = b"other-key".to_vec(); assert!( !wrapper.verify(&wrong_key), @@ -563,22 +685,21 @@ mod tests { ); } - // -------------------- StoredEntryVar tests -------------------- - #[test] fn stored_entry_var_roundtrip_latest() { let inner = TestInner { x: 0x5566_7788 }; let key = key_bytes(); let wrapper = StoredEntryVar::new(&key, inner.clone()); + assert!( wrapper.verify(&key), - "var wrapper verify (latest) should succeed" + "var wrapper verify latest should succeed" ); - // Encode wrapper to bytes and decode back via From/To bytes let bytes = wrapper.to_bytes().expect("var to_bytes"); let parsed = StoredEntryVar::::from_bytes(&bytes).expect("var from_bytes"); + assert_eq!(parsed.item, inner); assert_eq!(parsed.checksum, wrapper.checksum); assert!(parsed.verify(&key)); @@ -589,30 +710,26 @@ mod tests { let inner = TestInner { x: 0xDEAD_BEEF }; let key = key_bytes(); - // item serialized as v1 (tag + body) let item_bytes_v1 = inner .to_bytes_with_version(version::V1) .expect("inner v1 bytes"); - // checksum computed over (key || item_bytes_v1) let mut digest_input = Vec::with_capacity(key.len() + item_bytes_v1.len()); digest_input.extend_from_slice(&key); digest_input.extend_from_slice(&item_bytes_v1); + let checksum = StoredEntryVar::::blake2b256(&digest_input); - // Build raw stored value for StoredEntryVar: - // [StoredEntryVar::VERSION] + CompactSize(len) + item_bytes_v1 + checksum let mut raw = Vec::new(); raw.push(StoredEntryVar::::VERSION); CompactSize::write(&mut raw, item_bytes_v1.len()).expect("write compactsize"); raw.extend_from_slice(&item_bytes_v1); - // write checksum as fixed 32 bytes write_fixed_le::<32, _>(&mut raw, &checksum).expect("write checksum"); - // from_bytes should parse the body and return wrapper let parsed = StoredEntryVar::::from_bytes(&raw).expect("var from_bytes v1"); + assert_eq!(parsed.item, inner); - // verify must succeed using same key (it will try v2 then v1) + assert_eq!(parsed.checksum, checksum); assert!(parsed.verify(&key)); } @@ -624,13 +741,12 @@ mod tests { let mut wrapper = StoredEntryVar::new(&key, inner); assert!(wrapper.verify(&key)); - // tamper checksum wrapper.checksum[31] ^= 0xff; assert!(!wrapper.verify(&key)); - // restore and test wrong key fails let wrapper = StoredEntryVar::new(&key, TestInner { x: 0xCAFEBABE }); let wrong_key = b"bad-key".to_vec(); + assert!(!wrapper.verify(&wrong_key)); } } diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/ephemeral.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/ephemeral.rs index ee06e8d5a..3ab1bdc99 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/ephemeral.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/ephemeral.rs @@ -244,7 +244,9 @@ impl EphemeralFinalisedState { let block_hash = BlockHash::from(block.hash()); let block_height = zebra_chain::block::Height(height.0); - let (sapling, orchard) = self.source.get_commitment_tree_roots(block_hash).await?; + let (sapling, orchard, ironwood) = + self.source.get_commitment_tree_roots(block_hash).await?; + let sapling_is_active = self.network.is_nu_active( zcash_protocol::consensus::NetworkUpgrade::Sapling, block_height.into(), @@ -253,6 +255,11 @@ impl EphemeralFinalisedState { zcash_protocol::consensus::NetworkUpgrade::Nu5, block_height.into(), ); + let ironwood_is_active = self.network.is_nu_active( + zcash_protocol::consensus::NetworkUpgrade::Nu6_3, + block_height.into(), + ); + let (sapling_root, sapling_size) = match sapling { Some((root, size)) => (root, size), None if !sapling_is_active => Default::default(), @@ -275,6 +282,18 @@ impl EphemeralFinalisedState { )); } }; + let (ironwood_root, ironwood_size) = match ironwood { + Some((root, size)) => (root, size), + None if !ironwood_is_active => Default::default(), + None => { + return Err(FinalisedStateError::BlockchainSourceError( + BlockchainSourceError::Unrecoverable(format!( + "missing Ironwood commitment tree root for active NU5 block at height {height}" + )), + )); + } + }; + let sapling_size = u32::try_from(sapling_size).map_err(|error| { FinalisedStateError::BlockchainSourceError(BlockchainSourceError::Unrecoverable( format!("sapling commitment tree size does not fit into u32: {error}"), @@ -285,12 +304,19 @@ impl EphemeralFinalisedState { format!("orchard commitment tree size does not fit into u32: {error}"), )) })?; + let ironwood_size = u32::try_from(ironwood_size).map_err(|error| { + FinalisedStateError::BlockchainSourceError(BlockchainSourceError::Unrecoverable( + format!("ironwood commitment tree size does not fit into u32: {error}"), + )) + })?; let block_metadata = BlockMetadata::new( sapling_root, sapling_size, orchard_root, orchard_size, + ironwood_root, + ironwood_size, None, // ephemeral store does not track chainwork self.network.clone(), ); diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1.rs index 6c2bce199..7482cf73e 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1.rs @@ -39,10 +39,10 @@ use crate::{ config::ChainIndexConfig, error::FinalisedStateError, BlockHash, BlockHeaderData, CommitmentTreeData, CompactBlockStream, CompactOrchardAction, - CompactSaplingOutput, CompactSaplingSpend, CompactSize, CompactTxData, FixedEncodedLen as _, - Height, IndexedBlock, NamedAtomicStatus, OrchardCompactTx, OrchardTxList, Outpoint, - SaplingCompactTx, SaplingTxList, StatusType, TransparentCompactTx, TransparentTxList, - TxInCompact, TxLocation, TxOutCompact, TxidList, ZainoVersionedSerde as _, + CompactSaplingSpend, CompactSize, CompactTxData, FixedEncodedLen as _, Height, IndexedBlock, + NamedAtomicStatus, OrchardCompactTx, OrchardTxList, Outpoint, SaplingCompactTx, SaplingTxList, + StatusType, TransparentCompactTx, TransparentTxList, TxInCompact, TxLocation, TxOutCompact, + TxidList, ZainoVersionedSerde as _, }; #[cfg(feature = "transparent_address_history_experimental")] diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/block_core.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/block_core.rs index 5a82c9b13..f4079ff19 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/block_core.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/block_core.rs @@ -188,14 +188,26 @@ impl DbV1 { // Each txid entry is: [0] version tag + [1..32] txid // So we skip idx * 33 bytes to reach the start of the correct Hash - let offset = cursor.position() + (idx as u64) * TransactionHash::VERSIONED_LEN as u64; + let transaction_versioned_len = TransactionHash::latest_versioned_len()?; + let offset = cursor.position() + (idx as u64) * transaction_versioned_len as u64; cursor.set_position(offset); // Read [0] Txid Record version (skip 1 byte) cursor.set_position(cursor.position() + 1); // Then read 32 bytes for the txid - let mut txid_bytes = [0u8; TransactionHash::ENCODED_LEN]; + let transaction_encoded_len = TransactionHash::latest_encoded_len()?; + if transaction_encoded_len != 32 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "TransactionHash latest encoded length must be 32 bytes, got {}", + transaction_encoded_len + ), + ))?; + } + + let mut txid_bytes = [0u8; 32]; cursor .read_exact(&mut txid_bytes) .map_err(|e| FinalisedStateError::Custom(format!("txid read error: {e}")))?; diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/block_shielded.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/block_shielded.rs index 087d679a1..33ace3fe4 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/block_shielded.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/block_shielded.rs @@ -539,12 +539,13 @@ impl DbV1 { // Read number of spends (CompactSize) let spend_len = CompactSize::read(&mut *cursor)? as usize; - let spend_skip = spend_len * CompactSaplingSpend::VERSIONED_LEN; + let sapling_spend_len = CompactSaplingSpend::latest_versioned_len()?; + let spend_skip = spend_len * sapling_spend_len; cursor.set_position(cursor.position() + spend_skip as u64); // Read number of outputs (CompactSize) let output_len = CompactSize::read(&mut *cursor)? as usize; - let output_skip = output_len * CompactSaplingOutput::VERSIONED_LEN; + let output_skip = output_len * sapling_spend_len; cursor.set_position(cursor.position() + output_skip as u64); Ok(()) @@ -595,7 +596,8 @@ impl DbV1 { let action_len = CompactSize::read(&mut *cursor)? as usize; // Skip actions: each is 1-byte version + 148-byte body - let action_skip = action_len * CompactOrchardAction::VERSIONED_LEN; + let orchard_action_len = CompactOrchardAction::latest_versioned_len()?; + let action_skip = action_len * orchard_action_len; cursor.set_position(cursor.position() + action_skip as u64); Ok(()) diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/block_transparent.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/block_transparent.rs index 0339ef486..571b743c9 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/block_transparent.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/block_transparent.rs @@ -250,14 +250,16 @@ impl DbV1 { let vin_len = CompactSize::read(&mut *cursor)? as usize; // Skip vin entries: each is 1-byte version + 36-byte body - let vin_skip = vin_len * TxInCompact::VERSIONED_LEN; + let tx_in_len = TxInCompact::latest_versioned_len()?; + let vin_skip = vin_len * tx_in_len; cursor.set_position(cursor.position() + vin_skip as u64); // Read vout_len (CompactSize) let vout_len = CompactSize::read(&mut *cursor)? as usize; // Skip vout entries: each is 1-byte version + 29-byte body - let vout_skip = vout_len * TxOutCompact::VERSIONED_LEN; + let tx_out_len = TxOutCompact::latest_versioned_len()?; + let vout_skip = vout_len * tx_out_len; cursor.set_position(cursor.position() + vout_skip as u64); Ok(()) diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/indexed_block.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/indexed_block.rs index e6727bcf7..d2da6ee3f 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/indexed_block.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/indexed_block.rs @@ -133,7 +133,15 @@ impl DbV1 { .clone() .unwrap_or_else(|| OrchardCompactTx::new(None, vec![])); - CompactTxData::new(i as u64, txid, transparent_tx, sapling_tx, orchard_tx) + // TODO: Return ironwood! + CompactTxData::new( + i as u64, + txid, + transparent_tx, + sapling_tx, + orchard_tx, + OrchardCompactTx::empty(), + ) }) .collect(); diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/transparent_address_history.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/transparent_address_history.rs index 6bdb3cd5a..901d40060 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/transparent_address_history.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/transparent_address_history.rs @@ -114,10 +114,10 @@ impl DbV1 { }; for (key, val) in iter { - if key.len() != AddrScript::VERSIONED_LEN { + if key.len() != AddrScript::latest_versioned_len()? { continue; } - if val.len() != StoredEntryFixed::::VERSIONED_LEN { + if val.len() != StoredEntryFixed::::latest_versioned_len()? { continue; } raw_records.push(val.to_vec()); @@ -215,8 +215,8 @@ impl DbV1 { }; for (key, val) in iter { - if key.len() != AddrScript::VERSIONED_LEN - || val.len() != StoredEntryFixed::::VERSIONED_LEN + if key.len() != AddrScript::latest_versioned_len()? + || val.len() != StoredEntryFixed::::latest_versioned_len()? { continue; } @@ -285,8 +285,8 @@ impl DbV1 { }; for (key, val) in iter { - if key.len() != AddrScript::VERSIONED_LEN - || val.len() != StoredEntryFixed::::VERSIONED_LEN + if key.len() != AddrScript::latest_versioned_len()? + || val.len() != StoredEntryFixed::::latest_versioned_len()? { continue; } @@ -373,8 +373,8 @@ impl DbV1 { }; for (key, val) in iter { - if key.len() != AddrScript::VERSIONED_LEN - || val.len() != StoredEntryFixed::::VERSIONED_LEN + if key.len() != AddrScript::latest_versioned_len()? + || val.len() != StoredEntryFixed::::latest_versioned_len()? { continue; } @@ -525,8 +525,8 @@ impl DbV1 { }; // If the seek landed on a different key, there are no candidates for this addr. - if cur_key.len() != AddrScript::VERSIONED_LEN - || &cur_key[..AddrScript::VERSIONED_LEN] != addr_script_bytes + if cur_key.len() != AddrScript::latest_versioned_len()? + || &cur_key[..AddrScript::latest_versioned_len()?] != addr_script_bytes { return Ok(results); } @@ -536,12 +536,13 @@ impl DbV1 { loop { // Validate lengths, same as original function. - if cur_key.len() != AddrScript::VERSIONED_LEN { + if cur_key.len() != AddrScript::latest_versioned_len()? { return Err(FinalisedStateError::Custom( "address history key length mismatch".into(), )); } - if cur_val.len() != StoredEntryFixed::::VERSIONED_LEN { + if cur_val.len() != StoredEntryFixed::::latest_versioned_len()? + { return Err(FinalisedStateError::Custom( "address history value length mismatch".into(), )); @@ -584,8 +585,8 @@ impl DbV1 { Some(k) => k, None => break, }; - if k.len() != AddrScript::VERSIONED_LEN - || &k[..AddrScript::VERSIONED_LEN] != addr_script_bytes + if k.len() != AddrScript::latest_versioned_len()? + || &k[..AddrScript::latest_versioned_len()?] != addr_script_bytes { break; } @@ -714,7 +715,7 @@ impl DbV1 { // - [10] flags // - [11..=18] value // - [19..=50] checksum - if val.len() == StoredEntryFixed::::VERSIONED_LEN + if val.len() == StoredEntryFixed::::latest_versioned_len()? && val[2..6] == height_be { let flags = val[10]; @@ -728,7 +729,7 @@ impl DbV1 { break; } } - } else if val.len() != StoredEntryFixed::::VERSIONED_LEN { + } else if val.len() != StoredEntryFixed::::latest_versioned_len()? { tracing::warn!("bad addrhist dup (len={})", val.len()); } @@ -778,12 +779,12 @@ impl DbV1 { let mut cur = txn.open_rw_cursor(self.address_history)?; for (key, val) in cur.iter_dup_of(&addr_bytes)? { - if key.len() != AddrScript::VERSIONED_LEN { + if key.len() != AddrScript::latest_versioned_len()? { return Err(FinalisedStateError::Custom( "address history key length mismatch".into(), )); } - if val.len() != StoredEntryFixed::::VERSIONED_LEN { + if val.len() != StoredEntryFixed::::latest_versioned_len()? { return Err(FinalisedStateError::Custom( "address history value length mismatch".into(), )); @@ -793,7 +794,13 @@ impl DbV1 { continue; } - let mut hist_record = [0u8; StoredEntryFixed::::VERSIONED_LEN]; + let stored_entry_len = StoredEntryFixed::::latest_versioned_len()?; + if stored_entry_len != val.len() || stored_entry_len != 51 { + return Err(FinalisedStateError::Custom( + "address history value length mismatch".into(), + )); + } + let mut hist_record = [0u8; 51]; hist_record.copy_from_slice(val); let flags = hist_record[10]; @@ -846,12 +853,12 @@ impl DbV1 { let mut cur = txn.open_rw_cursor(self.address_history)?; for (key, val) in cur.iter_dup_of(&addr_bytes)? { - if key.len() != AddrScript::VERSIONED_LEN { + if key.len() != AddrScript::latest_versioned_len()? { return Err(FinalisedStateError::Custom( "address history key length mismatch".into(), )); } - if val.len() != StoredEntryFixed::::VERSIONED_LEN { + if val.len() != StoredEntryFixed::::latest_versioned_len()? { return Err(FinalisedStateError::Custom( "address history value length mismatch".into(), )); @@ -861,8 +868,13 @@ impl DbV1 { continue; } - // we've located the exact duplicate bytes we built earlier. - let mut hist_record = [0u8; StoredEntryFixed::::VERSIONED_LEN]; + let stored_entry_len = StoredEntryFixed::::latest_versioned_len()?; + if stored_entry_len != val.len() || stored_entry_len != 51 { + return Err(FinalisedStateError::Custom( + "address history value length mismatch".into(), + )); + } + let mut hist_record = [0u8; 51]; hist_record.copy_from_slice(val); // parse flags (located at byte index 10 in the StoredEntry layout) @@ -933,7 +945,7 @@ impl DbV1 { let height_key = Height(block_height).to_bytes()?; let stored_bytes = ro.get(self.transparent, &height_key)?; - Self::find_txout_in_stored_transparent_tx_list(stored_bytes, tx_index, out_index) + Self::find_txout_in_stored_transparent_tx_list(stored_bytes, tx_index, out_index)? .ok_or_else(|| { FinalisedStateError::Custom("Previous output not found at given index".into()) }) @@ -954,64 +966,70 @@ impl DbV1 { stored: &[u8], target_tx_idx: usize, target_output_idx: usize, - ) -> Option { + ) -> Result, FinalisedStateError> { const CHECKSUM_LEN: usize = 32; if stored.len() < TransactionHash::VERSION_TAG_LEN + 8 + CHECKSUM_LEN { - return None; + return Ok(None); } let mut cursor = &stored[TransactionHash::VERSION_TAG_LEN..]; - let item_len = CompactSize::read(&mut cursor).ok()? as usize; + let item_len = CompactSize::read(&mut cursor)? as usize; if cursor.len() < item_len + CHECKSUM_LEN { - return None; + return Ok(None); } - let (_record_version, mut remaining) = cursor.split_first()?; - let vec_len = CompactSize::read(&mut remaining).ok()? as usize; + let Some((_record_version, mut remaining)) = cursor.split_first() else { + return Ok(None); + }; + let vec_len = CompactSize::read(&mut remaining)? as usize; for i in 0..vec_len { - let (option_tag, rest) = remaining.split_first()?; + let Some((option_tag, rest)) = remaining.split_first() else { + return Ok(None); + }; remaining = rest; if *option_tag == 0 { // None: nothing to skip, go to next if i == target_tx_idx { - return None; + return Ok(None); } } else if *option_tag == 1 { - let (_tx_version, rest) = remaining.split_first()?; + let Some((_tx_version, rest)) = remaining.split_first() else { + return Ok(None); + }; remaining = rest; - let vin_len = CompactSize::read(&mut remaining).ok()? as usize; + let vin_len = CompactSize::read(&mut remaining)? as usize; for _ in 0..vin_len { - if remaining.len() < TxInCompact::VERSIONED_LEN { - return None; + if remaining.len() < TxInCompact::latest_versioned_len()? { + return Ok(None); } - remaining = &remaining[TxInCompact::VERSIONED_LEN..]; + remaining = &remaining[TxInCompact::latest_versioned_len()?..]; } - let vout_len = CompactSize::read(&mut remaining).ok()? as usize; + let vout_len = CompactSize::read(&mut remaining)? as usize; for out_idx in 0..vout_len { - if remaining.len() < TxOutCompact::VERSIONED_LEN { - return None; + if remaining.len() < TxOutCompact::latest_versioned_len()? { + return Ok(None); } - let out_bytes = &remaining[..TxOutCompact::VERSIONED_LEN]; + let out_bytes = &remaining[..TxOutCompact::latest_versioned_len()?]; if i == target_tx_idx && out_idx == target_output_idx { - return TxOutCompact::from_bytes(out_bytes).ok(); + return Ok(TxOutCompact::from_bytes(out_bytes).ok()); } - remaining = &remaining[TxOutCompact::VERSIONED_LEN..]; + remaining = &remaining[TxOutCompact::latest_versioned_len()?..]; } } else { // Non-canonical Option tag - return None; + return Ok(None); } } - None + Ok(None) } } diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/tx_out_set_accumulator.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/tx_out_set_accumulator.rs index 45e3f74ac..e254e70f0 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/tx_out_set_accumulator.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/tx_out_set_accumulator.rs @@ -1490,11 +1490,11 @@ impl DbV1 { Err(lmdb::Error::NotFound) => return Ok(None), Err(error) => return Err(FinalisedStateError::LmdbError(error)), }; - Ok(Self::find_txout_in_stored_transparent_tx_list( + Self::find_txout_in_stored_transparent_tx_list( stored, location.tx_index() as usize, outpoint.prev_index() as usize, - )) + ) } /// Fetches the full [`TransparentCompactTx`] for `txid`, read through `txn` (no new txn). diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/write_core.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/write_core.rs index 016e00d64..e3d28fbb5 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/write_core.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/write_core.rs @@ -66,6 +66,7 @@ impl DbWrite for DbV1 { .activation_height(&zebra_network) .expect("Sapling activation height must be set"); let nu5_activation_height = NetworkUpgrade::Nu5.activation_height(&zebra_network); + let nu6_3_activation_height = NetworkUpgrade::Nu6_3.activation_height(&zebra_network); // Seed `parent_chainwork` from the current tip header (the block before the first one we // write). On an empty database this is genesis with zero chainwork. Read raw rather than via @@ -155,6 +156,7 @@ impl DbWrite for DbV1 { network, sapling_activation_height, nu5_activation_height, + nu6_3_activation_height, next, parent_chainwork, ) @@ -229,6 +231,7 @@ impl DbWrite for DbV1 { network, sapling_activation_height, nu5_activation_height, + nu6_3_activation_height, height_int, parent_chainwork, ) diff --git a/packages/zaino-state/src/chain_index/non_finalised_state.rs b/packages/zaino-state/src/chain_index/non_finalised_state.rs index a132edc3e..023499cb3 100644 --- a/packages/zaino-state/src/chain_index/non_finalised_state.rs +++ b/packages/zaino-state/src/chain_index/non_finalised_state.rs @@ -320,7 +320,7 @@ impl NonFinalizedState { .map_err(|e| InitError::InvalidNodeData(Box::new(e)))? .ok_or_else(|| InitError::InvalidNodeData(Box::new(MissingGenesisBlock)))?; - let (sapling_root_and_len, orchard_root_and_len) = source + let (sapling_root_and_len, orchard_root_and_len, ironwood_root_and_len) = source .get_commitment_tree_roots(genesis_block.hash().into()) .await .map_err(|e| InitError::InvalidNodeData(Box::new(e)))?; @@ -328,6 +328,7 @@ impl NonFinalizedState { let tree_roots = TreeRootData { sapling: sapling_root_and_len, orchard: orchard_root_and_len, + ironwood: ironwood_root_and_len, }; // Genesis has no parent — pass None so create_block_context computes @@ -385,10 +386,14 @@ impl NonFinalizedState { )) })?; - let (sapling, orchard) = source + let (sapling, orchard, ironwood) = source .get_commitment_tree_roots(block.hash().into()) .await?; - let tree_roots = TreeRootData { sapling, orchard }; + let tree_roots = TreeRootData { + sapling, + orchard, + ironwood, + }; Self::create_indexed_block_with_optional_roots( block.as_ref(), @@ -809,12 +814,13 @@ impl NonFinalizedState { &self, block_hash: BlockHash, ) -> Result { - let (sapling_root_and_len, orchard_root_and_len) = + let (sapling_root_and_len, orchard_root_and_len, ironwood_root_and_len) = self.source.get_commitment_tree_roots(block_hash).await?; Ok(TreeRootData { sapling: sapling_root_and_len, orchard: orchard_root_and_len, + ironwood: ironwood_root_and_len, }) } @@ -829,7 +835,7 @@ impl NonFinalizedState { parent_chainwork: Option, network: Network, ) -> Result { - let (sapling_root, sapling_size, orchard_root, orchard_size) = + let (sapling_root, sapling_size, orchard_root, orchard_size, ironwood_root, ironwood_size) = tree_roots.clone().extract_with_defaults(); let metadata = BlockMetadata::new( @@ -837,6 +843,8 @@ impl NonFinalizedState { sapling_size as u32, orchard_root, orchard_size as u32, + ironwood_root, + ironwood_size as u32, parent_chainwork, network, ); diff --git a/packages/zaino-state/src/chain_index/source.rs b/packages/zaino-state/src/chain_index/source.rs index 73e3e13de..bb5f3f3ee 100644 --- a/packages/zaino-state/src/chain_index/source.rs +++ b/packages/zaino-state/src/chain_index/source.rs @@ -7,7 +7,7 @@ use crate::chain_index::{ ShieldedPool, }; use crate::SendFut; -use futures::{future::join, TryFutureExt as _}; +use futures::TryFutureExt as _; use incrementalmerkletree::frontier::CommitmentTree; use tower::{Service, ServiceExt as _}; use zaino_common::Network; @@ -34,15 +34,16 @@ pub(crate) mod mockchain_source; pub mod validator_connector; pub use validator_connector::*; -/// Serialized sapling and orchard treestates `(sapling, orchard)`, each `None` +/// Serialized sapling and orchard treestates `(sapling, orchard, ironwood)`, each `None` /// when the pool has no treestate at the queried block. -pub(crate) type TreestateBytes = (Option>, Option>); +pub(crate) type TreestateBytes = (Option>, Option>, Option>); -/// Sapling and orchard note-commitment tree roots `(sapling, orchard)`, each +/// Sapling and orchard note-commitment tree roots `(sapling, orchard, ironwood)`, each /// paired with its tree size; `None` when the pool has no root at the block. pub(crate) type ShieldedTreeRoots = ( Option<(zebra_chain::sapling::tree::Root, u64)>, Option<(zebra_chain::orchard::tree::Root, u64)>, + Option<(zebra_chain::orchard::tree::Root, u64)>, ); /// Receiver for newly observed nonfinalized blocks, delivered as `(hash, block)`. diff --git a/packages/zaino-state/src/chain_index/source/mockchain_source.rs b/packages/zaino-state/src/chain_index/source/mockchain_source.rs index 5f262527b..e7728db6f 100644 --- a/packages/zaino-state/src/chain_index/source/mockchain_source.rs +++ b/packages/zaino-state/src/chain_index/source/mockchain_source.rs @@ -551,24 +551,31 @@ impl BlockchainSource for MockchainSource { } /// Returns the sapling and orchard treestate by hash + /// + /// TODO: Update test vectors to support ironwood. async fn get_treestate( &self, id: BlockHash, - ) -> BlockchainSourceResult<(Option>, Option>)> { + ) -> BlockchainSourceResult<(Option>, Option>, Option>)> { let active_chain_height = self.active_height() as usize; // serve up to active tip if let Some(height) = self.hashes.iter().position(|h| h == &id) { if height <= active_chain_height { let (sapling_state, orchard_state) = &self.treestates[height]; - Ok((Some(sapling_state.clone()), Some(orchard_state.clone()))) + Ok(( + Some(sapling_state.clone()), + Some(orchard_state.clone()), + None, + )) } else { - Ok((None, None)) + Ok((None, None, None)) } } else { - Ok((None, None)) + Ok((None, None, None)) } } + /// TODO: Update test vectors to support ironwood. async fn get_subtree_roots( &self, pool: ShieldedPool, @@ -652,28 +659,32 @@ impl BlockchainSource for MockchainSource { } } } + ShieldedPool::Ironwood => {} } Ok(subtree_roots) } + /// TODO: Update test vectors to support ironwood. async fn get_commitment_tree_roots( &self, id: BlockHash, ) -> BlockchainSourceResult<( Option<(zebra_chain::sapling::tree::Root, u64)>, Option<(zebra_chain::orchard::tree::Root, u64)>, + Option<(zebra_chain::orchard::tree::Root, u64)>, )> { let active_chain_height = self.active_height() as usize; // serve up to active tip if let Some(height) = self.hashes.iter().position(|h| h == &id) { if height <= active_chain_height { - Ok(self.roots[height]) + let (sapling, orchard) = self.roots[height]; + Ok((sapling, orchard, None)) } else { - Ok((None, None)) + Ok((None, None, None)) } } else { - Ok((None, None)) + Ok((None, None, None)) } } diff --git a/packages/zaino-state/src/chain_index/source/validator_connector.rs b/packages/zaino-state/src/chain_index/source/validator_connector.rs index 8560853e1..038908279 100644 --- a/packages/zaino-state/src/chain_index/source/validator_connector.rs +++ b/packages/zaino-state/src/chain_index/source/validator_connector.rs @@ -1,5 +1,6 @@ //! validator connected blockchain source. +use futures::future::join3; use hex::FromHex as _; use zaino_fetch::jsonrpsee::response::address_deltas::BlockInfo; use zebra_chain::serialization::BytesInDisplayOrder as _; @@ -361,12 +362,12 @@ impl BlockchainSource for ValidatorConnector { } } - /// Returns the Sapling and Orchard treestate by blockhash. + /// Returns the Sapling, Orchard and Ironwood treestate by blockhash. async fn get_treestate( &self, // Sould this be HashOrHeight? id: BlockHash, - ) -> BlockchainSourceResult<(Option>, Option>)> { + ) -> BlockchainSourceResult<(Option>, Option>, Option>)> { let hash_or_height: HashOrHeight = HashOrHeight::Hash(zebra_chain::block::Hash(id.into())); match self { ValidatorConnector::State(state) => { @@ -448,7 +449,34 @@ impl BlockchainSource for ValidatorConnector { .map(|tree| tree.to_rpc_bytes()) }); - Ok((sapling, orchard)) + let ironwood = match zebra_chain::parameters::NetworkUpgrade::Nu6_3 + .activation_height(&state.network.to_zebra_network()) + { + Some(activation_height) if height >= activation_height => Some( + state + .read_state_service + .ready() + .and_then(|service| { + service.call(ReadRequest::IronwoodTree(hash_or_height)) + }) + .await + .map_err(|_e| { + BlockchainSourceError::Unrecoverable( + InvalidData(format!( + "could not fetch orchard treestate of block {id}" + )) + .to_string(), + ) + })?, + ), + _ => None, + } + .and_then(|orch_response| { + expected_read_response!(orch_response, OrchardTree) + .map(|tree| tree.to_rpc_bytes()) + }); + + Ok((sapling, orchard, ironwood)) } ValidatorConnector::Fetch(fetch) => { let treestate = fetch @@ -484,7 +512,20 @@ impl BlockchainSource for ValidatorConnector { |t| t.commitments().final_state().clone(), ); - Ok((sapling, orchard)) + let ironwood = treestate.ironwood.map_or_else( + || { + let mut tree = vec![]; + write_commitment_tree( + &CommitmentTree::::empty(), + &mut tree, + ) + .expect("can write to Vec"); + Some(tree) + }, + |t| t.commitments().final_state().clone(), + ); + + Ok((sapling, orchard, ironwood)) } } } @@ -502,6 +543,7 @@ impl BlockchainSource for ValidatorConnector { let request = match pool { ShieldedPool::Sapling => ReadRequest::SaplingSubtrees { start_index, limit }, ShieldedPool::Orchard => ReadRequest::OrchardSubtrees { start_index, limit }, + ShieldedPool::Ironwood => ReadRequest::IronwoodSubtrees { start_index, limit }, }; state .read_state_service @@ -519,6 +561,14 @@ impl BlockchainSource for ValidatorConnector { .iter() .map(|(_index, subtree)| (subtree.root.to_repr(), subtree.end_height.0)) .collect(), + ShieldedPool::Ironwood => { + expected_read_response!(response, IronwoodSubtrees) + .iter() + .map(|(_index, subtree)| { + (subtree.root.to_repr(), subtree.end_height.0) + }) + .collect() + } }) .map_err(|e| { BlockchainSourceError::Unrecoverable(format!( @@ -570,30 +620,38 @@ impl BlockchainSource for ValidatorConnector { ) -> BlockchainSourceResult<( Option<(zebra_chain::sapling::tree::Root, u64)>, Option<(zebra_chain::orchard::tree::Root, u64)>, + Option<(zebra_chain::orchard::tree::Root, u64)>, )> { match self { ValidatorConnector::State(state) => { - let (sapling_tree_response, orchard_tree_response) = - join( + let (sapling_tree_response, orchard_tree_response, ironwood_tree_response) = + join3( state.read_state_service.clone().call( zebra_state::ReadRequest::SaplingTree(HashOrHeight::Hash(id.into())), ), state.read_state_service.clone().call( zebra_state::ReadRequest::OrchardTree(HashOrHeight::Hash(id.into())), ), + state.read_state_service.clone().call( + zebra_state::ReadRequest::IronwoodTree(HashOrHeight::Hash(id.into())), + ), ) .await; - let (sapling_tree, orchard_tree) = match ( + let (sapling_tree, orchard_tree, ironwood_tree) = match ( //TODO: Better readstateservice error handling sapling_tree_response .map_err(|e| BlockchainSourceError::Unrecoverable(e.to_string()))?, orchard_tree_response .map_err(|e| BlockchainSourceError::Unrecoverable(e.to_string()))?, + ironwood_tree_response + .map_err(|e| BlockchainSourceError::Unrecoverable(e.to_string()))?, ) { - (ReadResponse::SaplingTree(saptree), ReadResponse::OrchardTree(orctree)) => { - (saptree, orctree) - } - (_, _) => panic!("Bad response"), + ( + ReadResponse::SaplingTree(saptree), + ReadResponse::OrchardTree(orctree), + ReadResponse::IronwoodTree(irwtree), + ) => (saptree, orctree, irwtree), + (_, _, _) => panic!("Bad response"), }; Ok(( @@ -603,6 +661,9 @@ impl BlockchainSource for ValidatorConnector { orchard_tree .as_deref() .map(|tree| (tree.root(), tree.count())), + ironwood_tree + .as_deref() + .map(|tree| (tree.root(), tree.count())), )) } ValidatorConnector::Fetch(fetch) => { @@ -622,7 +683,10 @@ impl BlockchainSource for ValidatorConnector { _ => BlockchainSourceError::Unrecoverable(e.to_string()), })?; let GetTreestateResponse { - sapling, orchard, .. + sapling, + orchard, + ironwood, + .. } = tree_responses; let sapling_frontier = sapling .map_or_else( @@ -650,6 +714,19 @@ impl BlockchainSource for ValidatorConnector { ) .transpose() .map_err(|e| BlockchainSourceError::Unrecoverable(format!("io error: {e}")))?; + let ironwood_frontier = ironwood + .map_or_else( + || Some(Ok(CommitmentTree::empty())), + |t| { + t.commitments().final_state().as_ref().map(|final_state| { + read_commitment_tree::( + final_state.as_slice(), + ) + }) + }, + ) + .transpose() + .map_err(|e| BlockchainSourceError::Unrecoverable(format!("io error: {e}")))?; let sapling_root = sapling_frontier .map(|tree| { zebra_chain::sapling::tree::Root::try_from(tree.root().to_bytes()) @@ -668,7 +745,16 @@ impl BlockchainSource for ValidatorConnector { .map_err(|e| { BlockchainSourceError::Unrecoverable(format!("could not deser: {e}")) })?; - Ok((sapling_root, orchard_root)) + let ironwood_root = ironwood_frontier + .map(|tree| { + zebra_chain::orchard::tree::Root::try_from(tree.root().to_repr()) + .map(|root| (root, tree.size() as u64)) + }) + .transpose() + .map_err(|e| { + BlockchainSourceError::Unrecoverable(format!("could not deser: {e}")) + })?; + Ok((sapling_root, orchard_root, ironwood_root)) } } } diff --git a/packages/zaino-state/src/chain_index/tests/finalised_state/v1.rs b/packages/zaino-state/src/chain_index/tests/finalised_state/v1.rs index 496de74da..c93e9ddeb 100644 --- a/packages/zaino-state/src/chain_index/tests/finalised_state/v1.rs +++ b/packages/zaino-state/src/chain_index/tests/finalised_state/v1.rs @@ -292,6 +292,8 @@ async fn try_write_invalid_block() { sapling_tree_size as u32, orchard_root, orchard_tree_size as u32, + zebra_chain::orchard::tree::Root::default(), + 0, None, // no parent chainwork for this test zaino_common::Network::Regtest(ActivationHeights::default()).to_zebra_network(), ); diff --git a/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs b/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs index 31518411e..98fc69c5c 100644 --- a/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs +++ b/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs @@ -592,7 +592,7 @@ async fn get_treestate() { .. } in blocks.into_iter() { - let (sapling_bytes_opt, orchard_bytes_opt) = index_reader + let (sapling_bytes_opt, orchard_bytes_opt, _ironwood_bytes_opt) = index_reader .get_treestate(&crate::BlockHash(zebra_block.hash().0)) .await .unwrap(); diff --git a/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs b/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs index aacc06536..cc7e51720 100644 --- a/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs +++ b/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs @@ -623,6 +623,7 @@ impl BlockchainSource for ProptestMockchain { ) -> BlockchainSourceResult<( Option<(zebra_chain::sapling::tree::Root, u64)>, Option<(zebra_chain::orchard::tree::Root, u64)>, + Option<(zebra_chain::orchard::tree::Root, u64)>, )> { if let Some(delay) = self.delay { tokio::time::sleep(delay).await; @@ -630,7 +631,7 @@ impl BlockchainSource for ProptestMockchain { let Some(chain_up_to_block) = self.get_block_and_all_preceeding(|block| block.hash().0 == id.0) else { - return Ok((None, None)); + return Ok((None, None, None)); }; let (sapling, orchard) = @@ -681,6 +682,7 @@ impl BlockchainSource for ProptestMockchain { orc_front.tree_size(), ) }), + None, )) } @@ -688,7 +690,7 @@ impl BlockchainSource for ProptestMockchain { async fn get_treestate( &self, _id: BlockHash, - ) -> BlockchainSourceResult<(Option>, Option>)> { + ) -> BlockchainSourceResult<(Option>, Option>, Option>)> { // I don't think this is used for sync? unimplemented!() } diff --git a/packages/zaino-state/src/chain_index/tests/vectors.rs b/packages/zaino-state/src/chain_index/tests/vectors.rs index cfc6e6948..54bba2342 100644 --- a/packages/zaino-state/src/chain_index/tests/vectors.rs +++ b/packages/zaino-state/src/chain_index/tests/vectors.rs @@ -97,6 +97,8 @@ pub(crate) fn indexed_block_chain( vector.sapling_tree_size as u32, vector.orchard_root, vector.orchard_tree_size as u32, + zebra_chain::orchard::tree::Root::default(), + 0, parent_chain_work, zebra_chain::parameters::Network::new_regtest( zebra_chain::parameters::testnet::ConfiguredActivationHeights { diff --git a/packages/zaino-state/src/chain_index/types/db/block.rs b/packages/zaino-state/src/chain_index/types/db/block.rs index 186abcebc..1a224b5f2 100644 --- a/packages/zaino-state/src/chain_index/types/db/block.rs +++ b/packages/zaino-state/src/chain_index/types/db/block.rs @@ -86,8 +86,16 @@ impl ZainoVersionedSerde for PersistentChainWork { } } +/// Fixed-length encoding metadata for `PersistentChainWork`. +/// +/// v1 consists of a single 32-byte value. impl FixedEncodedLen for PersistentChainWork { - const ENCODED_LEN: usize = 32; + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some(32), + _ => None, + } + } } /// Database-adjacent persistence shape for [`CompactDifficulty`]. @@ -129,8 +137,16 @@ impl ZainoVersionedSerde for PersistentCompactDifficulty { } } +/// Fixed-length encoding metadata for `PersistentCompactDifficulty`. +/// +/// v1 consists of a single 4-byte LE u32. impl FixedEncodedLen for PersistentCompactDifficulty { - const ENCODED_LEN: usize = 4; + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some(4), + _ => None, + } + } } /// Database-adjacent persistence shape for [`BlockContext`]. diff --git a/packages/zaino-state/src/chain_index/types/db/commitment.rs b/packages/zaino-state/src/chain_index/types/db/commitment.rs index 53fbeca30..142b0b5dd 100644 --- a/packages/zaino-state/src/chain_index/types/db/commitment.rs +++ b/packages/zaino-state/src/chain_index/types/db/commitment.rs @@ -8,9 +8,12 @@ use corez::io::{self, Read, Write}; -use crate::chain_index::encoding::{ - read_fixed_le, read_u32_le, version, write_fixed_le, write_u32_le, FixedEncodedLen, - ZainoVersionedSerde, +use crate::{ + chain_index::encoding::{ + read_fixed_le, read_u32_le, version, write_fixed_le, write_u32_le, FixedEncodedLen, + ZainoVersionedSerde, + }, + read_option, write_option, }; /// Holds commitment tree metadata (roots and sizes) for a block. @@ -39,20 +42,20 @@ impl CommitmentTreeData { } impl ZainoVersionedSerde for CommitmentTreeData { - const VERSION: u8 = version::V1; + const VERSION: u8 = version::V2; fn encode_latest(&self, w: &mut W) -> io::Result<()> { - Self::encode_v1(self, w) + Self::encode_v2(self, w) } fn decode_latest(r: &mut R) -> io::Result { - Self::decode_v1(r) + Self::decode_v2(r) } fn encode_v1(&self, w: &mut W) -> io::Result<()> { let mut w = w; - self.roots.serialize(&mut w)?; // carries its own tag - self.sizes.serialize(&mut w) + self.roots.serialize_with_version(&mut w, 1)?; + self.sizes.serialize_with_version(&mut w, 1) } fn decode_v1(r: &mut R) -> io::Result { @@ -61,14 +64,41 @@ impl ZainoVersionedSerde for CommitmentTreeData { let sizes = CommitmentTreeSizes::deserialize(&mut r)?; Ok(CommitmentTreeData::new(roots, sizes)) } + + fn encode_v2(&self, w: &mut W) -> io::Result<()> { + let mut w = w; + self.roots.serialize_with_version(&mut w, 2)?; + self.sizes.serialize_with_version(&mut w, 2) + } + + fn decode_v2(r: &mut R) -> io::Result { + let mut r = r; + let roots = CommitmentTreeRoots::deserialize(&mut r)?; + let sizes = CommitmentTreeSizes::deserialize(&mut r)?; + Ok(CommitmentTreeData::new(roots, sizes)) + } } -/// CommitmentTreeData: 74 bytes total impl FixedEncodedLen for CommitmentTreeData { - // 1 byte tag + 64 body for roots - // + 1 byte tag + 8 body for sizes - const ENCODED_LEN: usize = - (CommitmentTreeRoots::ENCODED_LEN + 1) + (CommitmentTreeSizes::ENCODED_LEN + 1); + /// Returns the fixed encoded body length for a specific + /// `CommitmentTreeData` version. + /// + /// v1 is fixed-length: + /// - versioned `CommitmentTreeRoots` v1 + /// - versioned `CommitmentTreeSizes` v1 + /// + /// v2 is variable-length because `CommitmentTreeRoots` v2 contains an + /// `Option<[u8; 32]>`. + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some( + CommitmentTreeRoots::versioned_len(version::V1)? + + CommitmentTreeSizes::versioned_len(version::V1)?, + ), + version::V2 => None, + _ => None, + } + } } /// Commitment tree roots for shielded transactions, enabling shielded wallet synchronization. @@ -79,12 +109,18 @@ pub struct CommitmentTreeRoots { sapling: [u8; 32], /// Orchard note-commitment tree root at this block. orchard: [u8; 32], + /// Ironwood note-commitment tree root at this block. + ironwood: Option<[u8; 32]>, } impl CommitmentTreeRoots { /// Reutns a new CommitmentTreeRoots instance. - pub fn new(sapling: [u8; 32], orchard: [u8; 32]) -> Self { - Self { sapling, orchard } + pub fn new(sapling: [u8; 32], orchard: [u8; 32], ironwood: Option<[u8; 32]>) -> Self { + Self { + sapling, + orchard, + ironwood, + } } /// Returns sapling commitment tree root. @@ -96,17 +132,22 @@ impl CommitmentTreeRoots { pub fn orchard(&self) -> &[u8; 32] { &self.orchard } + + /// returns orchard commitment tree root. + pub fn ironwood(&self) -> &Option<[u8; 32]> { + &self.ironwood + } } impl ZainoVersionedSerde for CommitmentTreeRoots { - const VERSION: u8 = version::V1; + const VERSION: u8 = version::V2; fn encode_latest(&self, w: &mut W) -> io::Result<()> { - Self::encode_v1(self, w) + Self::encode_v2(self, w) } fn decode_latest(r: &mut R) -> io::Result { - Self::decode_v1(r) + Self::decode_v2(r) } fn encode_v1(&self, w: &mut W) -> io::Result<()> { @@ -119,14 +160,44 @@ impl ZainoVersionedSerde for CommitmentTreeRoots { let mut r = r; let sapling = read_fixed_le::<32, _>(&mut r)?; let orchard = read_fixed_le::<32, _>(&mut r)?; - Ok(CommitmentTreeRoots::new(sapling, orchard)) + Ok(CommitmentTreeRoots::new(sapling, orchard, None)) + } + + fn encode_v2(&self, w: &mut W) -> io::Result<()> { + let mut w = w; + write_fixed_le::<32, _>(&mut w, &self.sapling)?; + write_fixed_le::<32, _>(&mut w, &self.orchard)?; + write_option(&mut w, &self.ironwood, |w, v| write_fixed_le(w, v)) + } + + fn decode_v2(r: &mut R) -> io::Result { + let mut r = r; + let sapling = read_fixed_le::<32, _>(&mut r)?; + let orchard = read_fixed_le::<32, _>(&mut r)?; + let ironwood = read_option(&mut r, |r| read_fixed_le(r))?; + + Ok(CommitmentTreeRoots::new(sapling, orchard, ironwood)) } } -/// CommitmentTreeRoots: 64 bytes total impl FixedEncodedLen for CommitmentTreeRoots { - /// 32 byte hash + 32 byte hash. - const ENCODED_LEN: usize = 32 + 32; + /// Returns the fixed encoded body length for a specific + /// `CommitmentTreeRoots` version. + /// + /// v1 is fixed-length: + /// - 32 bytes Sapling root + /// - 32 bytes Orchard root + /// + /// v2 is variable-length because `ironwood: Option<[u8; 32]>` encodes as: + /// - 1 byte option tag + /// - plus 32 bytes when present + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some(32 + 32), + version::V2 => None, + _ => None, + } + } } /// Sizes of commitment trees, indicating total number of shielded notes created. @@ -137,12 +208,18 @@ pub struct CommitmentTreeSizes { sapling: u32, /// Total notes in Orchard commitment tree. orchard: u32, + /// Total notes in Ironwood commitment tree. + ironwood: u32, } impl CommitmentTreeSizes { /// Creates a new CompactSaplingSizes instance. - pub fn new(sapling: u32, orchard: u32) -> Self { - Self { sapling, orchard } + pub fn new(sapling: u32, orchard: u32, ironwood: u32) -> Self { + Self { + sapling, + orchard, + ironwood, + } } /// Returns sapling commitment tree size @@ -154,17 +231,22 @@ impl CommitmentTreeSizes { pub fn orchard(&self) -> u32 { self.orchard } + + /// Returns orchard commitment tree size + pub fn ironwood(&self) -> u32 { + self.ironwood + } } impl ZainoVersionedSerde for CommitmentTreeSizes { - const VERSION: u8 = version::V1; + const VERSION: u8 = version::V2; fn encode_latest(&self, w: &mut W) -> io::Result<()> { - Self::encode_v1(self, w) + Self::encode_v2(self, w) } fn decode_latest(r: &mut R) -> io::Result { - Self::decode_v1(r) + Self::decode_v2(r) } fn encode_v1(&self, w: &mut W) -> io::Result<()> { @@ -177,12 +259,42 @@ impl ZainoVersionedSerde for CommitmentTreeSizes { let mut r = r; let sapling = read_u32_le(&mut r)?; let orchard = read_u32_le(&mut r)?; - Ok(CommitmentTreeSizes::new(sapling, orchard)) + Ok(CommitmentTreeSizes::new(sapling, orchard, 0)) + } + + fn encode_v2(&self, w: &mut W) -> io::Result<()> { + let mut w = w; + write_u32_le(&mut w, self.sapling)?; + write_u32_le(&mut w, self.orchard)?; + write_u32_le(&mut w, self.ironwood) + } + + fn decode_v2(r: &mut R) -> io::Result { + let mut r = r; + let sapling = read_u32_le(&mut r)?; + let orchard = read_u32_le(&mut r)?; + let ironwood = read_u32_le(&mut r)?; + Ok(CommitmentTreeSizes::new(sapling, orchard, ironwood)) } } -/// CommitmentTreeSizes: 8 bytes total impl FixedEncodedLen for CommitmentTreeSizes { - /// 4 byte LE int32 + 4 byte LE int32 - const ENCODED_LEN: usize = 4 + 4; + /// Returns the fixed encoded body length for a specific + /// `CommitmentTreeSizes` version. + /// + /// v1 is fixed-length: + /// - 4 bytes Sapling size + /// - 4 bytes Orchard size + /// + /// v2 is also fixed-length: + /// - 4 bytes Sapling size + /// - 4 bytes Orchard size + /// - 4 bytes Ironwood size + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some(4 + 4), + version::V2 => Some(4 + 4 + 4), + _ => None, + } + } } diff --git a/packages/zaino-state/src/chain_index/types/db/legacy.rs b/packages/zaino-state/src/chain_index/types/db/legacy.rs index fe2220918..8341616ab 100644 --- a/packages/zaino-state/src/chain_index/types/db/legacy.rs +++ b/packages/zaino-state/src/chain_index/types/db/legacy.rs @@ -200,10 +200,16 @@ impl ZainoVersionedSerde for BlockHash { } } -/// Hash = 32-byte body. +/// Fixed-length encoding metadata for `BlockHash`. +/// +/// v1 consists of a single 32-byte hash. impl FixedEncodedLen for BlockHash { - /// 32 bytes, LE - const ENCODED_LEN: usize = 32; + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some(32), + _ => None, + } + } } /// Transaction hash. @@ -338,10 +344,16 @@ impl ZainoVersionedSerde for TransactionHash { } } -/// Hash = 32-byte body. +/// Fixed-length encoding metadata for `TransactionHash`. +/// +/// v1 consists of a single 32-byte hash. impl FixedEncodedLen for TransactionHash { - /// 32 bytes, LE - const ENCODED_LEN: usize = 32; + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some(32), + _ => None, + } + } } /// Block height. @@ -480,10 +492,16 @@ impl ZainoVersionedSerde for Height { } } -/// Height = 4-byte big-endian body. +/// Fixed-length encoding metadata for `Height`. +/// +/// v1 consists of a single 4-byte big-endian u32. impl FixedEncodedLen for Height { - /// 4 bytes, BE - const ENCODED_LEN: usize = 4; + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some(4), + _ => None, + } + } } /// Numerical index of subtree / shard roots. @@ -516,10 +534,16 @@ impl ZainoVersionedSerde for ShardIndex { } } -/// Index = 4-byte big-endian body. +/// Fixed-length encoding metadata for `ShardIndex`. +/// +/// v1 consists of a single 4-byte big-endian u32. impl FixedEncodedLen for ShardIndex { - /// 4 bytes (BE u32) - const ENCODED_LEN: usize = 4; + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some(4), + _ => None, + } + } } /// A 20-byte hash160 *plus* a 1-byte ScriptType tag. @@ -645,10 +669,16 @@ impl ZainoVersionedSerde for AddrScript { } } -/// AddrScript = 21 bytes of body data. +/// Fixed-length encoding metadata for `AddrScript`. +/// +/// v1 consists of a 20 byte script (LE) + 1 byte script type impl FixedEncodedLen for AddrScript { - /// 20 bytes, LE + 1 byte script type - const ENCODED_LEN: usize = 21; + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some(21), + _ => None, + } + } } /// Reference to a spent transparent UTXO. @@ -712,10 +742,16 @@ impl ZainoVersionedSerde for Outpoint { } } -/// Outpoint = 32‐byte txid + 4-byte LE u32 index = 36 bytes +/// Fixed-length encoding metadata for `Outpoint`. +/// +/// v1 consists of a 32 byte txid + 4 byte tx index. impl FixedEncodedLen for Outpoint { - /// 32 byte txid + 4 byte tx index. - const ENCODED_LEN: usize = 32 + 4; + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some(36), + _ => None, + } + } } // *** Block Level Objects *** @@ -1138,6 +1174,8 @@ impl Option, [u8; 32], [u8; 32], + Option<[u8; 32]>, + u32, u32, u32, )> for IndexedBlock @@ -1150,13 +1188,17 @@ impl parent_chainwork, final_sapling_root, final_orchard_root, + final_ironwood_root, parent_sapling_size, parent_orchard_size, + parent_ironwood_size, ): ( zaino_fetch::chain::block::FullBlock, Option, [u8; 32], [u8; 32], + Option<[u8; 32]>, + u32, u32, u32, ), @@ -1212,6 +1254,7 @@ impl // --- Convert transactions --- let mut sapling_note_count = 0; let mut orchard_note_count = 0; + let mut ironwood_note_count = 0; let full_transactions = full_block.transactions(); let mut tx = Vec::with_capacity(full_transactions.len()); @@ -1222,6 +1265,7 @@ impl sapling_note_count += txdata.sapling().outputs().len(); orchard_note_count += txdata.orchard().actions().len(); + ironwood_note_count += txdata.ironwood().actions().len(); tx.push(txdata); } @@ -1229,12 +1273,14 @@ impl // --- Compute commitment trees --- let sapling_root = final_sapling_root; let orchard_root = final_orchard_root; + let ironwood_root = final_ironwood_root; let commitment_tree_data = CommitmentTreeData::new( - CommitmentTreeRoots::new(sapling_root, orchard_root), + CommitmentTreeRoots::new(sapling_root, orchard_root, ironwood_root), CommitmentTreeSizes::new( parent_sapling_size + sapling_note_count as u32, parent_orchard_size + orchard_note_count as u32, + parent_ironwood_size + ironwood_note_count as u32, ), ); @@ -1288,6 +1334,8 @@ pub struct CompactTxData { sapling: SaplingCompactTx, /// Compact representation of Orchard actions (shielded pool transactions). orchard: OrchardCompactTx, + /// Compact representation of Ironwood actions (shielded pool transactions). + ironwood: OrchardCompactTx, } impl CompactTxData { @@ -1298,6 +1346,7 @@ impl CompactTxData { transparent: TransparentCompactTx, sapling: SaplingCompactTx, orchard: OrchardCompactTx, + ironwood: OrchardCompactTx, ) -> Self { Self { index, @@ -1305,6 +1354,7 @@ impl CompactTxData { transparent, sapling, orchard, + ironwood, } } @@ -1338,6 +1388,11 @@ impl CompactTxData { &self.orchard } + /// Returns compact ironwood tx data. + pub fn ironwood(&self) -> &OrchardCompactTx { + &self.ironwood + } + /// Converts this `TxData` into a `CompactTx` protobuf message with an optional fee. pub fn to_compact_tx( &self, @@ -1383,6 +1438,20 @@ impl CompactTxData { ) .collect(); + let ironwood_actions = self + .ironwood() + .actions() + .iter() + .map( + |a| zaino_proto::proto::compact_formats::CompactOrchardAction { + nullifier: a.nullifier().to_vec(), + cmx: a.cmx().to_vec(), + ephemeral_key: a.ephemeral_key().to_vec(), + ciphertext: a.ciphertext().to_vec(), + }, + ) + .collect(); + let vout = self.transparent().compact_vout(); let vin = self.transparent().compact_vin(); @@ -1394,7 +1463,7 @@ impl CompactTxData { spends, outputs, actions, - ironwood_actions: Vec::new(), + ironwood_actions, vin, vout, } @@ -1416,7 +1485,7 @@ impl TryFrom<(u64, zaino_fetch::chain::transaction::FullTransaction)> for Compac .try_into() .map_err(|_| "txid must be 32 bytes".to_string())?; - let (sapling_balance, orchard_balance, _ironwood_balance) = tx.value_balances(); + let (sapling_balance, orchard_balance, ironwood_balance) = tx.value_balances(); let vin: Vec = tx .transparent_inputs() @@ -1480,7 +1549,7 @@ impl TryFrom<(u64, zaino_fetch::chain::transaction::FullTransaction)> for Compac let sapling = SaplingCompactTx::new(sapling_balance, spends, outputs); - let actions: Vec = tx + let orchard_actions: Vec = tx .orchard_actions() .into_iter() .map(|(nf, cmx, epk, ct)| { @@ -1502,7 +1571,31 @@ impl TryFrom<(u64, zaino_fetch::chain::transaction::FullTransaction)> for Compac }) .collect::>()?; - let orchard = OrchardCompactTx::new(orchard_balance, actions); + let orchard = OrchardCompactTx::new(orchard_balance, orchard_actions); + + let ironwood_actions: Vec = tx + .ironwood_actions() + .into_iter() + .map(|(nf, cmx, epk, ct)| { + let nf: [u8; 32] = nf + .try_into() + .map_err(|_| "orchard nullifier must be 32 bytes".to_string())?; + let cmx: [u8; 32] = cmx + .try_into() + .map_err(|_| "orchard cmx must be 32 bytes".to_string())?; + let epk: [u8; 32] = epk + .try_into() + .map_err(|_| "orchard ephemeral_key must be 32 bytes".to_string())?; + let ct: [u8; 52] = ct + .get(..52) + .ok_or("orchard ciphertext must be at least 52 bytes")? + .try_into() + .map_err(|_| "orchard ciphertext must be 52 bytes".to_string())?; + Ok::<_, String>(CompactOrchardAction::new(nf, cmx, epk, ct)) + }) + .collect::>()?; + + let ironwood = OrchardCompactTx::new(ironwood_balance, ironwood_actions); Ok(CompactTxData::new( index, @@ -1511,19 +1604,20 @@ impl TryFrom<(u64, zaino_fetch::chain::transaction::FullTransaction)> for Compac transparent, sapling, orchard, + ironwood, )) } } impl ZainoVersionedSerde for CompactTxData { - const VERSION: u8 = version::V1; + const VERSION: u8 = version::V2; fn encode_latest(&self, w: &mut W) -> io::Result<()> { - Self::encode_v1(self, w) + Self::encode_v2(self, w) } fn decode_latest(r: &mut R) -> io::Result { - Self::decode_v1(r) + Self::decode_v2(r) } fn encode_v1(&self, mut w: &mut W) -> io::Result<()> { @@ -1550,6 +1644,37 @@ impl ZainoVersionedSerde for CompactTxData { transparent, sapling, orchard, + OrchardCompactTx::empty(), + )) + } + + fn encode_v2(&self, mut w: &mut W) -> io::Result<()> { + write_u64_le(&mut w, self.index)?; + + self.txid.serialize_with_version(&mut w, 1)?; + self.transparent.serialize_with_version(&mut w, 1)?; + self.sapling.serialize_with_version(&mut w, 1)?; + self.orchard.serialize_with_version(&mut w, 1)?; + self.ironwood.serialize_with_version(&mut w, 1) + } + + fn decode_v2(r: &mut R) -> io::Result { + let mut r = r; + let index = read_u64_le(&mut r)?; + + let txid = TransactionHash::deserialize(&mut r)?; + let transparent = TransparentCompactTx::deserialize(&mut r)?; + let sapling = SaplingCompactTx::deserialize(&mut r)?; + let orchard = OrchardCompactTx::deserialize(&mut r)?; + let ironwood = OrchardCompactTx::deserialize(&mut r)?; + + Ok(CompactTxData::new( + index, + txid, + transparent, + sapling, + orchard, + ironwood, )) } } @@ -1719,10 +1844,16 @@ impl ZainoVersionedSerde for TxInCompact { } } -/// TxInCompact = 36 bytes +/// Fixed-length encoding metadata for `TxInCompact`. +/// +/// v1 consists of a 32-byte txid + 4-byte LE index impl FixedEncodedLen for TxInCompact { - /// 32-byte txid + 4-byte LE index - const ENCODED_LEN: usize = 32 + 4; + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some(36), + _ => None, + } + } } /// Identifies the type of transparent transaction output script. @@ -1784,10 +1915,16 @@ impl ZainoVersionedSerde for ScriptType { } } -/// ScriptType = 1 byte +/// Fixed-length encoding metadata for `ScriptType`. +/// +/// v1 consists of a single byte impl FixedEncodedLen for ScriptType { - /// 1 byte - const ENCODED_LEN: usize = 1; + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some(1), + _ => None, + } + } } /// Try to recognise a standard P2PKH / P2SH locking script. @@ -1962,10 +2099,16 @@ impl ZainoVersionedSerde for TxOutCompact { } } -/// TxOutCompact = 29 bytes +/// Fixed-length encoding metadata for `TxOutCompact`. +/// +/// v1 consists of a 8-byte LE value + 20-byte script hash + 1-byte type impl FixedEncodedLen for TxOutCompact { - /// 8-byte LE value + 20-byte script hash + 1-byte type - const ENCODED_LEN: usize = 8 + 20 + 1; + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some(29), + _ => None, + } + } } /// Compact representation of Sapling shielded transaction data for wallet scanning. @@ -2087,10 +2230,16 @@ impl ZainoVersionedSerde for CompactSaplingSpend { } } -/// 32-byte nullifier +/// Fixed-length encoding metadata for `CompactSaplingSpend`. +/// +/// v1 consists of a 32 byte nullifier impl FixedEncodedLen for CompactSaplingSpend { - /// 32 bytes - const ENCODED_LEN: usize = 32; + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some(32), + _ => None, + } + } } /// Compact representation of a newly created Sapling shielded note output. @@ -2168,10 +2317,16 @@ impl ZainoVersionedSerde for CompactSaplingOutput { } } -/// 116 bytes +/// Fixed-length encoding metadata for `CompactSaplingOutput`. +/// +/// v1 consists of a 32-byte cmu + 32-byte ephemeral_key + 52-byte ciphertext impl FixedEncodedLen for CompactSaplingOutput { - /// 32-byte cmu + 32-byte ephemeral_key + 52-byte ciphertext - const ENCODED_LEN: usize = 32 + 32 + 52; + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some(116), + _ => None, + } + } } /// Compact summary of all shielded activity in a transaction. @@ -2199,6 +2354,14 @@ impl OrchardCompactTx { pub fn actions(&self) -> &[CompactOrchardAction] { &self.actions } + + /// Return an empty OrchardCompactTx + pub fn empty() -> Self { + Self { + value: None, + actions: Vec::new(), + } + } } impl ZainoVersionedSerde for OrchardCompactTx { @@ -2320,10 +2483,20 @@ impl ZainoVersionedSerde for CompactOrchardAction { } } -// CompactOrchardAction = 148 bytes +/// Fixed-length encoding metadata for `CompactOrchardAction`. +/// +/// v1 consists of a: +/// - 32-byte nullifier +/// - 32-byte cmx +/// - 32-byte ephemeral_key +/// - 52-byte ciphertext impl FixedEncodedLen for CompactOrchardAction { - /// 32-byte nullifier + 32-byte cmx + 32-byte ephemeral_key + 52-byte ciphertext - const ENCODED_LEN: usize = 32 + 32 + 32 + 52; + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some(148), + _ => None, + } + } } /// Identifies a transaction's location by block height and transaction index. @@ -2379,10 +2552,16 @@ impl ZainoVersionedSerde for TxLocation { } } -/// 6 bytes, BE encoded. +/// Fixed-length encoding metadata for `TxLocation`. +/// +/// v1 consists of a 4-byte big-endian block_index + 2-byte big-endian tx_index impl FixedEncodedLen for TxLocation { - /// 4-byte big-endian block_index + 2-byte big-endian tx_index - const ENCODED_LEN: usize = 4 + 2; + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some(6), + _ => None, + } + } } /// Single transparent-address activity record (input or output). @@ -2483,15 +2662,22 @@ impl ZainoVersionedSerde for AddrHistRecord { } } -/// 18 byte total +/// Fixed-length encoding metadata for `AddrHistRecord`. +/// +/// v1 consists of: +/// 1 byte: TxLocation tag +/// +6 bytes: TxLocation body (4 BE block_index + 2 BE tx_index) +/// +2 bytes: out_index (BE) +/// +8 bytes: value (LE) +/// +1 byte : flags +/// =18 bytes impl FixedEncodedLen for AddrHistRecord { - /// 1 byte: TxLocation tag - /// +6 bytes: TxLocation body (4 BE block_index + 2 BE tx_index) - /// +2 bytes: out_index (BE) - /// +8 bytes: value (LE) - /// +1 byte : flags - /// =18 bytes - const ENCODED_LEN: usize = (TxLocation::ENCODED_LEN + 1) + 2 + 8 + 1; + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some(18), + _ => None, + } + } } /// AddrHistRecord database byte array. @@ -2581,6 +2767,9 @@ impl ZainoVersionedSerde for AddrEventBytes { } } +/// Fixed-length encoding metadata for `AddrEventBytes`. +/// +/// v1 consists of: /// 17 byte body: /// /// ```text @@ -2591,7 +2780,12 @@ impl ZainoVersionedSerde for AddrEventBytes { /// [9..17] value (LE u64) | Amount in zatoshi, little-endian /// ``` impl FixedEncodedLen for AddrEventBytes { - const ENCODED_LEN: usize = 17; + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some(17), + _ => None, + } + } } // *** Sharding *** @@ -2661,10 +2855,16 @@ impl ZainoVersionedSerde for ShardRoot { } } -/// 68 byte body. +/// Fixed-length encoding metadata for `ShardRoot`. +/// +/// v1 consists of a 32 byte hash + 32 byte hash + 4 byte block height impl FixedEncodedLen for ShardRoot { - /// 32 byte hash + 32 byte hash + 4 byte block height - const ENCODED_LEN: usize = 32 + 32 + 4; + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some(68), + _ => None, + } + } } // *** Wrapper Objects *** diff --git a/packages/zaino-state/src/chain_index/types/db/metadata.rs b/packages/zaino-state/src/chain_index/types/db/metadata.rs index dd95d5ef4..efbb3880f 100644 --- a/packages/zaino-state/src/chain_index/types/db/metadata.rs +++ b/packages/zaino-state/src/chain_index/types/db/metadata.rs @@ -102,10 +102,16 @@ impl ZainoVersionedSerde for MempoolInfo { } } -/// 24 byte body. +/// Fixed-length encoding metadata for `MempoolInfo`. +/// +/// v1 consists of 8 byte size + 8 byte bytes + 8 byte usage impl FixedEncodedLen for MempoolInfo { - /// 8 byte size + 8 byte bytes + 8 byte usage - const ENCODED_LEN: usize = 8 + 8 + 8; + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some(24), + _ => None, + } + } } impl From for MempoolInfo { @@ -328,9 +334,16 @@ impl ZainoVersionedSerde for FinalisedTxOutSetInfoAccumulator { } } +/// Fixed-length encoding metadata for `FinalisedTxOutSetInfoAccumulator`. +/// +/// v1 consists of 8 + 8 + 8 + 32 + 8 = 64 bytes impl FixedEncodedLen for FinalisedTxOutSetInfoAccumulator { - /// 8 + 8 + 8 + 32 + 8 = 64 bytes. - const ENCODED_LEN: usize = 8 + 8 + 8 + 32 + 8; + fn encoded_len(version: u8) -> Option { + match version { + version::V1 => Some(64), + _ => None, + } + } } #[cfg(test)] @@ -353,7 +366,7 @@ mod tests { assert_eq!( encoded_accumulator.len(), - FinalisedTxOutSetInfoAccumulator::VERSIONED_LEN + FinalisedTxOutSetInfoAccumulator::latest_versioned_len().unwrap() ); let decoded_accumulator = diff --git a/packages/zaino-state/src/chain_index/types/helpers.rs b/packages/zaino-state/src/chain_index/types/helpers.rs index d4e63db94..2797379b9 100644 --- a/packages/zaino-state/src/chain_index/types/helpers.rs +++ b/packages/zaino-state/src/chain_index/types/helpers.rs @@ -60,6 +60,8 @@ pub struct TreeRootData { pub sapling: Option<(zebra_chain::sapling::tree::Root, u64)>, /// Orchard tree root and size pub orchard: Option<(zebra_chain::orchard::tree::Root, u64)>, + /// Orchard tree root and size + pub ironwood: Option<(zebra_chain::orchard::tree::Root, u64)>, } impl TreeRootData { @@ -67,8 +69,13 @@ impl TreeRootData { pub fn new( sapling: Option<(zebra_chain::sapling::tree::Root, u64)>, orchard: Option<(zebra_chain::orchard::tree::Root, u64)>, + ironwood: Option<(zebra_chain::orchard::tree::Root, u64)>, ) -> Self { - Self { sapling, orchard } + Self { + sapling, + orchard, + ironwood, + } } /// Extract with defaults for genesis/sync use case @@ -79,10 +86,20 @@ impl TreeRootData { u64, zebra_chain::orchard::tree::Root, u64, + zebra_chain::orchard::tree::Root, + u64, ) { let (sapling_root, sapling_size) = self.sapling.unwrap_or_default(); let (orchard_root, orchard_size) = self.orchard.unwrap_or_default(); - (sapling_root, sapling_size, orchard_root, orchard_size) + let (ironwood_root, ironwood_size) = self.orchard.unwrap_or_default(); + ( + sapling_root, + sapling_size, + orchard_root, + orchard_size, + ironwood_root, + ironwood_size, + ) } } @@ -97,6 +114,10 @@ pub struct BlockMetadata { pub orchard_root: zebra_chain::orchard::tree::Root, /// Orchard tree size pub orchard_size: u32, + /// Orchard commitment tree root + pub ironwood_root: zebra_chain::orchard::tree::Root, + /// Orchard tree size + pub ironwood_size: u32, /// Parent block's chainwork (`None` for genesis). pub parent_chainwork: Option, /// Network for block validation @@ -110,6 +131,8 @@ impl BlockMetadata { sapling_size: u32, orchard_root: zebra_chain::orchard::tree::Root, orchard_size: u32, + ironwood_root: zebra_chain::orchard::tree::Root, + ironwood_size: u32, parent_chainwork: Option, network: zebra_chain::parameters::Network, ) -> Self { @@ -118,6 +141,8 @@ impl BlockMetadata { sapling_size, orchard_root, orchard_size, + ironwood_root, + ironwood_size, parent_chainwork, network, } @@ -172,9 +197,16 @@ impl<'a> BlockWithMetadata<'a> { let transparent = self.extract_transparent_data(txn)?; let sapling = self.extract_sapling_data(txn); let orchard = self.extract_orchard_data(txn); - - let txdata = - CompactTxData::new(i as u64, txn.hash().into(), transparent, sapling, orchard); + let ironwood = self.extract_ironwood_data(txn); + + let txdata = CompactTxData::new( + i as u64, + txn.hash().into(), + transparent, + sapling, + orchard, + ironwood, + ); transactions.push(txdata); } @@ -283,6 +315,38 @@ impl<'a> BlockWithMetadata<'a> { ) } + /// Extract orchard transaction data + fn extract_ironwood_data( + &self, + txn: &zebra_chain::transaction::Transaction, + ) -> OrchardCompactTx { + let orchard_value = { + let val = txn.ironwood_value_balance().orchard_amount(); + if val == 0 { + None + } else { + Some(i64::from(val)) + } + }; + + OrchardCompactTx::new( + orchard_value, + txn.ironwood_actions() + .map(|action| { + let cipher: [u8; 52] = <[u8; 580]>::from(action.enc_ciphertext)[..52] + .try_into() + .unwrap(); // TODO: Remove unwrap + CompactOrchardAction::new( + <[u8; 32]>::from(action.nullifier), + <[u8; 32]>::from(action.cm_x), + <[u8; 32]>::from(action.ephemeral_key), + cipher, + ) + }) + .collect::>(), + ) + } + /// Create a [`BlockContext`] from block and metadata. fn create_block_context(&self) -> Result { let block = self.block; @@ -313,11 +377,13 @@ impl<'a> BlockWithMetadata<'a> { let commitment_tree_roots = super::db::CommitmentTreeRoots::new( <[u8; 32]>::from(self.metadata.sapling_root), <[u8; 32]>::from(self.metadata.orchard_root), + Some(<[u8; 32]>::from(self.metadata.ironwood_root)), ); let commitment_tree_size = super::db::CommitmentTreeSizes::new( self.metadata.sapling_size, self.metadata.orchard_size, + self.metadata.ironwood_size, ); super::db::CommitmentTreeData::new(commitment_tree_roots, commitment_tree_size) From 7346037f8a0f78ba79baacecfe7ad4a541dae929 Mon Sep 17 00:00:00 2001 From: idky137 Date: Sat, 4 Jul 2026 01:52:44 +0100 Subject: [PATCH 032/134] update finalised state for ironwood --- packages/zaino-fetch/src/chain/block.rs | 1 + packages/zaino-state/src/backends/fetch.rs | 24 +- .../src/chain_index/finalised_state.rs | 24 +- .../chain_index/finalised_state/CHANGELOG.md | 82 ++++++ .../chain_index/finalised_state/capability.rs | 26 ++ .../finalised_state/finalised_source.rs | 36 +++ .../finalised_source/db_schema_v1.txt | 30 +- .../finalised_source/ephemeral.rs | 46 +++ .../finalised_state/finalised_source/v1.rs | 52 +++- .../finalised_source/v1/block_shielded.rs | 167 ++++++++++- .../finalised_source/v1/compact_block.rs | 134 ++++++++- .../finalised_source/v1/indexed_block.rs | 43 ++- .../finalised_source/v1/validation.rs | 24 +- .../finalised_source/v1/write_core.rs | 47 ++- .../chain_index/finalised_state/migrations.rs | 272 +++++++++++++++++- .../src/chain_index/finalised_state/reader.rs | 31 ++ .../src/chain_index/types/db/legacy.rs | 3 +- .../src/chain_index/types/helpers.rs | 10 +- 18 files changed, 991 insertions(+), 61 deletions(-) diff --git a/packages/zaino-fetch/src/chain/block.rs b/packages/zaino-fetch/src/chain/block.rs index 398589265..1ead20241 100644 --- a/packages/zaino-fetch/src/chain/block.rs +++ b/packages/zaino-fetch/src/chain/block.rs @@ -202,6 +202,7 @@ impl FullBlock { || !compact_tx.spends.is_empty() || !compact_tx.outputs.is_empty() || !compact_tx.actions.is_empty() + || !compact_tx.ironwood_actions.is_empty() { Some(Ok(compact_tx)) } else { diff --git a/packages/zaino-state/src/backends/fetch.rs b/packages/zaino-state/src/backends/fetch.rs index 208541ac0..f720b2238 100644 --- a/packages/zaino-state/src/backends/fetch.rs +++ b/packages/zaino-state/src/backends/fetch.rs @@ -606,7 +606,7 @@ impl ZcashIndexer for FetchServiceSubscriber { )?, }; - let (sapling, orchard, _ironwood) = + let (sapling, orchard, ironwood) = self.indexer.get_treestate(block_data.hash()).await?; let time: u32 = block_data.data().time().try_into().map_err(|_error| { #[allow(deprecated)] @@ -616,13 +616,27 @@ impl ZcashIndexer for FetchServiceSubscriber { )) })?; - #[allow(deprecated)] - Ok(GetTreestateResponse::from_parts( + // `Commitments::new(final_root, final_state)`: the note-commitment tree is the + // `final_state`, matching the (now-deprecated) `from_parts` behaviour this replaces. + // The `new` constructor is used so the Ironwood (NU6.3) treestate can be included rather + // than discarded. + Ok(GetTreestateResponse::new( (*block_data.hash()).into(), block_data.height().into(), time, - sapling, - orchard, + None, + zebra_rpc::client::Treestate::new(zebra_rpc::client::Commitments::new( + None, sapling, + )), + zebra_rpc::client::Treestate::new(zebra_rpc::client::Commitments::new( + None, orchard, + )), + ironwood.map(|final_state| { + zebra_rpc::client::Treestate::new(zebra_rpc::client::Commitments::new( + None, + Some(final_state), + )) + }), )) } .await; diff --git a/packages/zaino-state/src/chain_index/finalised_state.rs b/packages/zaino-state/src/chain_index/finalised_state.rs index 6b25f458c..0a34fc7ab 100644 --- a/packages/zaino-state/src/chain_index/finalised_state.rs +++ b/packages/zaino-state/src/chain_index/finalised_state.rs @@ -1056,6 +1056,12 @@ impl FinalisedState { })?; let tip = Height::from(tip); + // Ironwood (NU6.3) commitment tree data is only expected from activation. Below activation + // (or on a network with no NU6.3 activation height) the source has no ironwood root, so it + // defaults — mirroring `build_indexed_block_from_source`. + let nu6_3_activation_height = zebra_chain::parameters::NetworkUpgrade::Nu6_3 + .activation_height(&cfg.network.to_zebra_network()); + let mut parent_chainwork: Option = None; for height in crate::chain_index::types::GENESIS_HEIGHT.0..=tip.0 { @@ -1085,11 +1091,19 @@ impl FinalisedState { format!("missing Orchard commitment tree root for block {block_hash}"), )) })?; - let (ironwood_root, ironwood_size) = ironwood_opt.ok_or_else(|| { - FinalisedStateError::BlockchainSourceError(BlockchainSourceError::Unrecoverable( - format!("missing Orchard commitment tree root for block {block_hash}"), - )) - })?; + let is_ironwood_active = + nu6_3_activation_height.is_some_and(|activation| height >= activation.0); + let (ironwood_root, ironwood_size) = if is_ironwood_active { + ironwood_opt.ok_or_else(|| { + FinalisedStateError::BlockchainSourceError( + BlockchainSourceError::Unrecoverable(format!( + "missing Ironwood commitment tree root for block {block_hash}" + )), + ) + })? + } else { + (zebra_chain::orchard::tree::Root::default(), 0) + }; let metadata = BlockMetadata::new( sapling_root, diff --git a/packages/zaino-state/src/chain_index/finalised_state/CHANGELOG.md b/packages/zaino-state/src/chain_index/finalised_state/CHANGELOG.md index 78e0e92f2..52a137a74 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/CHANGELOG.md +++ b/packages/zaino-state/src/chain_index/finalised_state/CHANGELOG.md @@ -395,6 +395,88 @@ Bug Fixes / Optimisations version migrations run in the background while an ephemeral passthrough serves finalised reads from the backing source. +-------------------------------------------------------------------------------- +DB VERSION v1.3.0 (from v1.2.1) +Date: 2026-07-03 +-------------------------------------------------------------------------------- + +Summary +- Persist and serve the Ironwood (NU6.3) shielded pool from the finalised state. + Ironwood actions reuse the Orchard compact types (`OrchardCompactTx` / + `CompactOrchardAction`), matching zebra, which exposes ironwood actions as + `orchard::Action`. +- Add a new per-height `ironwood` table (`ironwood_1_3_0`) storing + `StoredEntryVar`, written for every block from v1.3.0 onward. +- Rebuild the `commitment_tree_data` table from the legacy fixed-length + `StoredEntryFixed` (V1) layout into the variable-length + `StoredEntryVar` (V2) layout, which carries the optional + Ironwood commitment-tree root and the Ironwood tree size. The rebuild moves the + data into a new table (`commitment_tree_data_1_3_0`) and drops the old one. + +On-disk schema +- Layout: + - No directory layout changes (still `//v1/`). +- Tables: + - Added: `ironwood` — per-height ironwood compact tx data + (LMDB database name: `ironwood_1_3_0`, `StoredEntryVar`). + - Renamed/rebuilt: `commitment_tree_data` moves from the fixed-length table + `commitment_tree_data_1_0_0` (`StoredEntryFixed`, V1) to + the variable-length table `commitment_tree_data_1_3_0` + (`StoredEntryVar`, V2). The legacy table is cleared once + the rebuild completes. + - Removed: legacy `commitment_tree_data_1_0_0` contents (table cleared). +- Encoding: + - Values: `CommitmentTreeData` gains a V2 body — `CommitmentTreeRoots` adds an + `Option<32-byte ironwood_root>` and `CommitmentTreeSizes` adds + `LE(u32) ironwood_total`. Because the optional root makes the record + variable-length, the table wrapper changes from `StoredEntryFixed` to + `StoredEntryVar`. + - Checksums / validation: `DB_SCHEMA_V1_HASH` updated to match the revised + schema text. Ironwood rows are checksum-protected by their height key. +- Invariants: + - Blocks written from v1.3.0 have an `ironwood` row aligned to the block's tx + list. Blocks written before v1.3.0 (all below NU6.3 activation) have no + ironwood row; readers and startup validation treat a missing row as + "no ironwood data at this height". + +API / capabilities +- Capability changes: + - No new capability bit: ironwood reads reuse `BLOCK_SHIELDED_EXT` and ironwood + compact-block/commitment data reuse `COMPACT_BLOCK_EXT`. +- Public surface changes: + - Added `BlockShieldedExt::{get_ironwood, get_block_ironwood, + get_block_range_ironwood}` (and the `DbReader` wrappers), mirroring the + orchard accessors. + - Compact blocks now populate `CompactTx.ironwood_actions` and + `ChainMetadata.ironwood_commitment_tree_size`; `IndexedBlock` materialisation + returns real ironwood tx data. + +Migration +- Strategy: in-place rebuild from existing on-disk data (no validator refetch), + single re-entrant step. +- Guard: refuses to run if the DB tip is at or above NU6.3 activation (such a + database was synced without ironwood data and must be re-indexed from the + validator). Below activation every stored block predates ironwood, so the + rebuild sets ironwood root/size to their defaults. +- Backfill: rebuilds each height's commitment row from the legacy fixed-length + row into the new `StoredEntryVar` (V2) table, then clears the legacy table. The + new `ironwood` table starts empty (no pre-NU6.3 block has ironwood data). +- Completion criteria: all heights through the tip rebuilt; `DbMetadata.version` + advanced to v1.3.0 with the refreshed schema hash; the temporary progress key + (`_migration_commitment_tree_data_progress_1_3_0_next_height`) removed. +- Failure handling: resumes from the temporary progress height; each rebuilt row + and the progress watermark commit together, and re-seen rows are accepted after + `NO_OVERWRITE` + byte-match verification. + +Bug Fixes / Optimisations +- Fixed `TreeRootData::extract_with_defaults` reading the orchard tree where it + should read ironwood, and `extract_ironwood_data` using the orchard value + balance amount instead of the ironwood amount. +- `to_compact_block`, the compact-block assembly, and the compact-block stream no + longer hardcode `ironwood_commitment_tree_size: 0` / empty ironwood actions. +- Compact-tx "omit empty transaction" filters now consider ironwood actions. +- The Fetch backend's `z_get_treestate` no longer discards the ironwood treestate. + -------------------------------------------------------------------------------- v0 SCHEMA SUPPORT REMOVED (no DB version change) Date: 2026-06-18 diff --git a/packages/zaino-state/src/chain_index/finalised_state/capability.rs b/packages/zaino-state/src/chain_index/finalised_state/capability.rs index 798208880..c26b4ec8d 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/capability.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/capability.rs @@ -918,6 +918,32 @@ pub trait BlockShieldedExt: Send + Sync { end: Height, ) -> impl SendFut, FinalisedStateError>>; + /// Fetch the serialized Ironwood (NU6.3) compact tx for the given TxLocation, if present. + /// + /// Ironwood actions are modelled with the Orchard compact types. Returns `None` when the block + /// has no ironwood row (any block below NU6.3 activation, or written before schema v1.3.0). + fn get_ironwood( + &self, + tx_location: TxLocation, + ) -> impl SendFut, FinalisedStateError>>; + + /// Fetch block ironwood transaction data by height. + /// + /// Returns an empty [`OrchardTxList`] when the block has no ironwood row. + fn get_block_ironwood( + &self, + height: Height, + ) -> impl SendFut>; + + /// Fetches block ironwood tx data for the given (inclusive) height range. + /// + /// Heights with no ironwood row yield an empty [`OrchardTxList`]. + fn get_block_range_ironwood( + &self, + start: Height, + end: Height, + ) -> impl SendFut, FinalisedStateError>>; + /// Fetch block commitment tree data by height. fn get_block_commitment_tree_data( &self, diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source.rs index 6697cf45d..cae4c96e3 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source.rs @@ -360,6 +360,14 @@ impl FinalisedSource { pub(crate) fn transparent_db(&self) -> Result { Ok(self.require_v1("v1 transparent db")?.transparent_db()) } + + /// Provides access to the (v1.3.0) `StoredEntryVar` commitment-tree-data table, required for + /// Migration1_2_1To1_3_0 to write the rebuilt commitment rows. + pub(crate) fn commitment_tree_data_db(&self) -> Result { + Ok(self + .require_v1("v1 commitment_tree_data db")? + .commitment_tree_data_db()) + } } impl From for FinalisedSource { @@ -675,6 +683,34 @@ impl BlockShieldedExt for FinalisedSource { } } + async fn get_ironwood( + &self, + tx_location: TxLocation, + ) -> Result, FinalisedStateError> { + match self { + Self::V1(db) => db.get_ironwood(tx_location).await, + Self::Ephemeral(db) => db.get_ironwood(tx_location).await, + } + } + + async fn get_block_ironwood(&self, h: Height) -> Result { + match self { + Self::V1(db) => db.get_block_ironwood(h).await, + Self::Ephemeral(db) => db.get_block_ironwood(h).await, + } + } + + async fn get_block_range_ironwood( + &self, + start: Height, + end: Height, + ) -> Result, FinalisedStateError> { + match self { + Self::V1(db) => db.get_block_range_ironwood(start, end).await, + Self::Ephemeral(db) => db.get_block_range_ironwood(start, end).await, + } + } + async fn get_block_commitment_tree_data( &self, height: Height, diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/db_schema_v1.txt b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/db_schema_v1.txt index 5c001e474..80f3f1d3c 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/db_schema_v1.txt +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/db_schema_v1.txt @@ -1,5 +1,5 @@ # ──────────────────────────────────────────────────────────────────────────────── -# Zaino – Finalised State Database, on-disk layout (Schema v1.2, 2026-13-05) +# Zaino – Finalised State Database, on-disk layout (Schema v1.3, 2026-07-03) # ──────────────────────────────────────────────────────────────────────────────── # # Any change to this file is a **breaking** change. Bump the schema version, @@ -71,13 +71,33 @@ # actions = CS n + n × CompactOrchardAction # CompactOrchardAction = 32-byte nullifier 32-byte cmx 32-byte epk 52-byte ciphertext # -# 6. commitment_tree_data ― H -> StoredEntryFixed> +# 6. commitment_tree_data ― H -> StoredEntryVar # Key : BE height -# Val : 0x01 + CS n + n × CommitmentTreeData -# CommitmentTreeData V1 body = +# Val : 0x01 + CS len + CommitmentTreeData + [32] check_hash +# CommitmentTreeData V2 body = # CommitmentTreeRoots + CommitmentTreeSizes # CommitmentTreeRoots = 32-byte sapling_root 32-byte orchard_root +# Option<32-byte ironwood_root> # CommitmentTreeSizes = LE(u32) sapling_total LE(u32) orchard_total +# LE(u32) ironwood_total +# +# (Legacy CommitmentTreeData V1 body = +# 32-byte sapling_root 32-byte orchard_root +# LE(u32) sapling_total LE(u32) orchard_total ; +# fixed-length, previously wrapped in StoredEntryFixed. Rebuilt into the V2 +# StoredEntryVar layout above by the v1.2.1 → v1.3.0 migration, then dropped.) +# +# 13. ironwood ― H -> StoredEntryVar (schema v1.3.0) +# Key : BE height +# Val : 0x01 + CS tx_count + tx_count × Option +# OrchardCompactTx = +# Option value_balance +# actions = CS n + n × CompactOrchardAction +# CompactOrchardAction = 32-byte nullifier 32-byte cmx 32-byte epk 52-byte ciphertext +# +# Ironwood (NU6.3) actions reuse the Orchard compact types. Rows are written for +# every block from schema v1.3.0 onward; blocks written before v1.3.0 (all below +# NU6.3 activation) have no ironwood row, which readers treat as "no ironwood data". # # 7. heights ― B (block hash) -> StoredEntryFixed # Key : 32-byte block hash (internal byte order) @@ -132,7 +152,7 @@ # # ─────────────────────────── Environment settings ───────────────────────────── # LMDB page-size: platform default -# max_dbs: 12 (see list above) +# max_dbs: 16 (see list above) # Flags: MDB_NOTLS | MDB_NORDAHEAD # # All Databases are append-only and indexed by height -> LMDB default diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/ephemeral.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/ephemeral.rs index 3ab1bdc99..af1b391bd 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/ephemeral.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/ephemeral.rs @@ -747,6 +747,52 @@ impl BlockShieldedExt for EphemeralFinalisedState { Ok(orchard_lists) } + async fn get_ironwood( + &self, + tx_location: TxLocation, + ) -> Result, FinalisedStateError> { + let chain_block = self + .get_required_chain_block(Height::try_from(tx_location.block_height()).unwrap()) + .await?; + + Ok(chain_block + .transactions() + .get(usize::from(tx_location.tx_index())) + .map(|transaction| transaction.ironwood().clone())) + } + + async fn get_block_ironwood( + &self, + height: Height, + ) -> Result { + let chain_block = self.get_required_chain_block(height).await?; + + Ok(OrchardTxList::new( + chain_block + .transactions() + .iter() + .map(|transaction| Some(transaction.ironwood().clone())) + .collect(), + )) + } + + async fn get_block_range_ironwood( + &self, + start: Height, + end: Height, + ) -> Result, FinalisedStateError> { + let mut ironwood_lists = Vec::new(); + + for height in u32::from(start)..=u32::from(end) { + ironwood_lists.push( + self.get_block_ironwood(Height::try_from(height).unwrap()) + .await?, + ); + } + + Ok(ironwood_lists) + } + async fn get_block_commitment_tree_data( &self, height: Height, diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1.rs index 7482cf73e..94f5e2355 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1.rs @@ -126,15 +126,15 @@ pub(crate) const DB_SCHEMA_V1_TEXT: &str = include_str!("db_schema_v1.txt"); /// This value is compared against the schema hash stored in the metadata record to detect schema /// drift without a corresponding version bump. pub(crate) const DB_SCHEMA_V1_HASH: [u8; 32] = [ - 0x11, 0xb2, 0x6a, 0x12, 0x08, 0x67, 0xf0, 0x42, 0xf6, 0x31, 0x45, 0xea, 0x87, 0xe7, 0x23, 0x75, - 0x40, 0x3b, 0xf2, 0x14, 0xaa, 0x2b, 0x00, 0x12, 0xec, 0xa4, 0x4d, 0x00, 0xe9, 0x0b, 0x07, 0x9b, + 0xaf, 0xcb, 0x80, 0xfe, 0x89, 0x2b, 0xc5, 0xba, 0x8e, 0x5d, 0x20, 0xfe, 0x56, 0x72, 0x81, 0x75, + 0x58, 0x8a, 0xb6, 0x49, 0xf7, 0xc4, 0x45, 0xcd, 0xa2, 0x8f, 0xaf, 0xb9, 0x6a, 0x95, 0xc8, 0x75, ]; /// *Current* database V1 version. pub(crate) const DB_VERSION_V1: DbVersion = DbVersion { major: 1, - minor: 2, - patch: 1, + minor: 3, + patch: 0, }; /// LMDB table name for the finalised txout-set accumulator. @@ -314,9 +314,17 @@ pub(crate) struct DbV1 { /// Stored per-block, in order. orchard: Database, - /// Block commitment tree data: `Height` -> `StoredEntryFixed>` + /// Ironwood: `Height` -> `StoredEntryVar` /// - /// Stored per-block, in order. + /// Ironwood (NU6.3) shielded-pool actions, modelled with the Orchard compact types. Stored + /// per-block, in order. Introduced in schema v1.3.0. + ironwood: Database, + + /// Block commitment tree data: `Height` -> `StoredEntryVar` + /// + /// Stored per-block, in order. The value is a `StoredEntryVar` (not `StoredEntryFixed`) from + /// schema v1.3.0 onward, because `CommitmentTreeData` V2 carries an optional Ironwood root and + /// is therefore variable-length. commitment_tree_data: Database, /// Heights: `Hash` -> `StoredEntryFixed` @@ -406,7 +414,7 @@ impl DbV1 { /// - validates or initializes the `"metadata"` record (schema hash + version), and /// - spawns the background validator / maintenance task. pub(crate) async fn spawn(config: &ChainIndexConfig) -> Result { - let mut zaino_db = Self::open_env_and_dbs(config).await?; + let mut zaino_db = Self::open_env_and_dbs(config, "commitment_tree_data_1_3_0").await?; // Validate (or initialise) the metadata entry before we touch any tables. zaino_db.check_schema_version().await?; @@ -425,7 +433,15 @@ impl DbV1 { /// Opens the LMDB environment and every V1 named database, returning an *unstarted* [`DbV1`] /// (status `Spawning`, `db_handler` = `None`, fresh atomics). Performs no metadata validation /// and starts no background task — each caller (`spawn`, `spawn_v1_0_0`) adds its own tail. - async fn open_env_and_dbs(config: &ChainIndexConfig) -> Result { + /// + /// `commitment_tree_data_name` selects the LMDB table backing the `commitment_tree_data` field: + /// the current path passes the v1.3.0 table (`commitment_tree_data_1_3_0`, `StoredEntryVar`), + /// while the v1.0.0 fixture builder ([`DbV1::spawn_v1_0_0`]) passes the legacy + /// `commitment_tree_data_1_0_0` (`StoredEntryFixed`) so it reproduces a pre-migration database. + async fn open_env_and_dbs( + config: &ChainIndexConfig, + commitment_tree_data_name: &str, + ) -> Result { info!("Launching FinalisedState"); // Prepare database details and path. @@ -464,7 +480,7 @@ impl DbV1 { // drops the unflushed page cache, write order is not guaranteed and a crash *can* leave torn // pages; the recovery there is to wipe and re-index. See `SYNC_CHECKPOINT_INTERVAL`.) let env = Environment::new() - .set_max_dbs(15) + .set_max_dbs(16) .set_map_size(db_size_bytes) .set_max_readers(max_readers) .set_flags( @@ -484,8 +500,10 @@ impl DbV1 { super::open_or_create_db(&env, "sapling_1_0_0", DatabaseFlags::empty()).await?; let orchard = super::open_or_create_db(&env, "orchard_1_0_0", DatabaseFlags::empty()).await?; + let ironwood = + super::open_or_create_db(&env, "ironwood_1_3_0", DatabaseFlags::empty()).await?; let commitment_tree_data = - super::open_or_create_db(&env, "commitment_tree_data_1_0_0", DatabaseFlags::empty()) + super::open_or_create_db(&env, commitment_tree_data_name, DatabaseFlags::empty()) .await?; let hashes = super::open_or_create_db(&env, "hashes_1_0_0", DatabaseFlags::empty()).await?; @@ -519,6 +537,7 @@ impl DbV1 { transparent, sapling, orchard, + ironwood, commitment_tree_data, heights: hashes, spent, @@ -546,6 +565,7 @@ impl DbV1 { transparent: self.transparent, sapling: self.sapling, orchard: self.orchard, + ironwood: self.ironwood, commitment_tree_data: self.commitment_tree_data, heights: self.heights, spent: self.spent, @@ -811,6 +831,12 @@ impl DbV1 { self.metadata } + /// Provides access to the (v1.3.0) `StoredEntryVar` commitment-tree-data table, required for + /// Migration1_2_1To1_3_0 to write the rebuilt commitment rows. + pub(crate) fn commitment_tree_data_db(&self) -> Database { + self.commitment_tree_data + } + /// Provudes access to the spent DB table, required for Migration1_1_0To1_2_0. pub(crate) fn spent_db(&self) -> Database { self.spent @@ -971,7 +997,11 @@ impl DbV1 { pub(crate) async fn spawn_v1_0_0( config: &ChainIndexConfig, ) -> Result { - let mut zaino_db = Self::open_env_and_dbs(config).await?; + // The v1.0.0 fixture must reproduce the legacy on-disk layout: the commitment tree data + // lives in `commitment_tree_data_1_0_0` as a `StoredEntryFixed` (see `write_block_v1_0_0`), + // which the v1.2.1 -> v1.3.0 migration later rebuilds into the `StoredEntryVar` table. + let mut zaino_db = + Self::open_env_and_dbs(config, "commitment_tree_data_1_0_0").await?; // Write the historical v1.0.0 metadata record. Intentionally skips `check_schema_version` // (see the method doc) — that is the behavioural difference from `spawn`. diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/block_shielded.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/block_shielded.rs index 33ace3fe4..546efe39f 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/block_shielded.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/block_shielded.rs @@ -51,6 +51,28 @@ impl BlockShieldedExt for DbV1 { self.get_block_range_orchard(start, end).await } + async fn get_ironwood( + &self, + tx_location: TxLocation, + ) -> Result, FinalisedStateError> { + self.get_ironwood(tx_location).await + } + + async fn get_block_ironwood( + &self, + height: Height, + ) -> Result { + self.get_block_ironwood(height).await + } + + async fn get_block_range_ironwood( + &self, + start: Height, + end: Height, + ) -> Result, FinalisedStateError> { + self.get_block_range_ironwood(start, end).await + } + async fn get_block_commitment_tree_data( &self, height: Height, @@ -409,6 +431,147 @@ impl DbV1 { .collect() } + /// Fetch the serialized `OrchardCompactTx` for the given TxLocation from the ironwood table. + /// + /// Mirrors [`DbV1::get_orchard`] against the ironwood table (ironwood reuses the Orchard compact + /// types). A missing ironwood row — any block below NU6.3 activation, or one written before + /// schema v1.3.0 — yields `Ok(None)` rather than an error. + async fn get_ironwood( + &self, + tx_location: TxLocation, + ) -> Result, FinalisedStateError> { + use std::io::{Cursor, Read}; + + tokio::task::block_in_place(|| { + let txn = self.env.begin_ro_txn()?; + + let height = Height::try_from(tx_location.block_height()) + .map_err(|e| FinalisedStateError::Custom(e.to_string()))?; + let height_bytes = height.to_bytes()?; + + let raw = match txn.get(self.ironwood, &height_bytes) { + Ok(val) => val, + // No ironwood data at this height. + Err(lmdb::Error::NotFound) => return Ok(None), + Err(e) => return Err(FinalisedStateError::LmdbError(e)), + }; + + let mut cursor = Cursor::new(raw); + + // Skip [0] StoredEntry version + cursor.set_position(1); + + // Read CompactSize: length of serialized body + CompactSize::read(&mut cursor).map_err(|e| { + FinalisedStateError::Custom(format!("compact size read error: {e}")) + })?; + + // Skip OrchardTxList version byte + cursor.set_position(cursor.position() + 1); + + // Read CompactSize: number of entries + let list_len = CompactSize::read(&mut cursor).map_err(|e| { + FinalisedStateError::Custom(format!("ironwood tx list len error: {e}")) + })?; + + let idx = tx_location.tx_index() as usize; + if idx >= list_len as usize { + return Err(FinalisedStateError::Custom( + "tx_index out of range in ironwood tx list".to_string(), + )); + } + + // Skip preceding entries + for _ in 0..idx { + Self::skip_opt_orchard_entry(&mut cursor) + .map_err(|e| FinalisedStateError::Custom(format!("skip entry error: {e}")))?; + } + + let start = cursor.position(); + + // Peek presence flag + let mut presence = [0u8; 1]; + cursor.read_exact(&mut presence).map_err(|e| { + FinalisedStateError::Custom(format!("failed to read Option tag: {e}")) + })?; + + if presence[0] == 0 { + return Ok(None); + } else if presence[0] != 1 { + return Err(FinalisedStateError::Custom(format!( + "invalid Option tag: {}", + presence[0] + ))); + } + + // Rewind to include presence flag in output + cursor.set_position(start); + Self::skip_opt_orchard_entry(&mut cursor).map_err(|e| { + FinalisedStateError::Custom(format!("skip entry error (second pass): {e}")) + })?; + + let end = cursor.position(); + + Ok(Some(OrchardCompactTx::from_bytes( + &raw[start as usize..end as usize], + )?)) + }) + } + + /// Fetch block ironwood transaction data by height. + /// + /// A missing ironwood row yields an empty [`OrchardTxList`]. + async fn get_block_ironwood( + &self, + height: Height, + ) -> Result { + let validated_height = self + .resolve_validated_hash_or_height(HashOrHeight::Height(height.into())) + .await?; + let height_bytes = validated_height.to_bytes()?; + + tokio::task::block_in_place(|| { + let txn = self.env.begin_ro_txn()?; + let raw = match txn.get(self.ironwood, &height_bytes) { + Ok(val) => val, + Err(lmdb::Error::NotFound) => return Ok(OrchardTxList::new(Vec::new())), + Err(e) => return Err(FinalisedStateError::LmdbError(e)), + }; + + let entry: StoredEntryVar = StoredEntryVar::from_bytes(raw) + .map_err(|e| FinalisedStateError::Custom(format!("ironwood decode error: {e}")))?; + + Ok(entry.inner().clone()) + }) + } + + /// Fetches block ironwood tx data for the given (inclusive) height range. + /// + /// Unlike the orchard range fetch this resolves each height individually so that heights with no + /// ironwood row (pre-v1.3.0 / pre-NU6.3) yield an empty [`OrchardTxList`], keeping the result + /// aligned one-entry-per-height with the requested range. + async fn get_block_range_ironwood( + &self, + start: Height, + end: Height, + ) -> Result, FinalisedStateError> { + if end.0 < start.0 { + return Err(FinalisedStateError::Custom( + "invalid block range: end < start".to_string(), + )); + } + + self.validate_block_range(start, end).await?; + + let mut out = Vec::with_capacity((end.0 - start.0 + 1) as usize); + for height_int in start.0..=end.0 { + let height = Height::try_from(height_int) + .map_err(|e| FinalisedStateError::Custom(e.to_string()))?; + out.push(self.get_block_ironwood(height).await?); + } + Ok(out) + } + /// Fetch block commitment tree data by height. async fn get_block_commitment_tree_data( &self, @@ -431,7 +594,7 @@ impl DbV1 { Err(e) => return Err(FinalisedStateError::LmdbError(e)), }; - let entry = StoredEntryFixed::from_bytes(raw).map_err(|e| { + let entry = StoredEntryVar::::from_bytes(raw).map_err(|e| { FinalisedStateError::Custom(format!("commitment_tree decode error: {e}")) })?; @@ -485,7 +648,7 @@ impl DbV1 { raw_entries .into_iter() .map(|bytes| { - StoredEntryFixed::::from_bytes(&bytes) + StoredEntryVar::::from_bytes(&bytes) .map(|e| e.item) .map_err(|e| { FinalisedStateError::Custom(format!("commitment_tree decode error: {e}")) diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/compact_block.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/compact_block.rs index 226e23d17..b0ec0c086 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/compact_block.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/compact_block.rs @@ -146,6 +146,28 @@ impl DbV1 { None => &[], }; + // ----- Fetch Ironwood Tx Data ----- + // + // Ironwood rows exist only from schema v1.3.0 (NU6.3) onward, so a missing row is not an + // error — it just means the block has no ironwood data. + let ironwood_stored_entry_var = if pool_types.includes_ironwood() { + match txn.get(self.ironwood, &height_bytes) { + Ok(raw) => Some( + StoredEntryVar::::from_bytes(raw).map_err(|e| { + FinalisedStateError::Custom(format!("ironwood decode error: {e}")) + })?, + ), + Err(lmdb::Error::NotFound) => None, + Err(e) => return Err(FinalisedStateError::LmdbError(e)), + } + } else { + None + }; + let ironwood = match ironwood_stored_entry_var.as_ref() { + Some(stored_entry_var) => stored_entry_var.inner().tx(), + None => &[], + }; + // ----- Construct CompactTx ----- let vtx: Vec = txids .iter() @@ -184,6 +206,17 @@ impl DbV1 { }) .unwrap_or_default(); + let ironwood_actions = ironwood + .get(i) + .and_then(|opt| opt.as_ref()) + .map(|o| { + o.actions() + .iter() + .map(|a| a.into_compact()) + .collect::>() + }) + .unwrap_or_default(); + let (vin, vout) = transparent .get(i) .and_then(|opt| opt.as_ref()) @@ -205,6 +238,7 @@ impl DbV1 { if spends.is_empty() && outputs.is_empty() && actions.is_empty() + && ironwood_actions.is_empty() && vin.is_empty() && vout.is_empty() { @@ -218,7 +252,7 @@ impl DbV1 { spends, outputs, actions, - ironwood_actions: Vec::new(), + ironwood_actions, vin, vout, }) @@ -235,16 +269,17 @@ impl DbV1 { } Err(e) => return Err(FinalisedStateError::LmdbError(e)), }; - let commitment_tree_data: CommitmentTreeData = *StoredEntryFixed::from_bytes(raw) - .map_err(|e| { - FinalisedStateError::Custom(format!("commitment_tree decode error: {e}")) - })? - .inner(); + let commitment_tree_data: CommitmentTreeData = + *StoredEntryVar::::from_bytes(raw) + .map_err(|e| { + FinalisedStateError::Custom(format!("commitment_tree decode error: {e}")) + })? + .inner(); let chain_metadata = zaino_proto::proto::compact_formats::ChainMetadata { sapling_commitment_tree_size: commitment_tree_data.sizes().sapling(), orchard_commitment_tree_size: commitment_tree_data.sizes().orchard(), - ironwood_commitment_tree_size: 0, + ironwood_commitment_tree_size: commitment_tree_data.sizes().ironwood(), }; // ----- Construct CompactBlock ----- @@ -949,6 +984,53 @@ impl DbV1 { None => &[], }; + // Ironwood is fetched with a tolerant point lookup rather than a lockstep cursor: + // rows are sparse (present only from schema v1.3.0 / NU6.3), so a missing row must + // not desync a height-aligned cursor. `NotFound` simply means "no ironwood here". + let ironwood_entries: Option> = if pool_types + .includes_ironwood() + { + let ironwood_key = match current_height.to_bytes() { + Ok(key) => key, + Err(error) => { + send_status( + &sender, + tonic::Status::internal(format!( + "ironwood height to_bytes failed: {error}" + )), + ); + return; + } + }; + match txn.get(zaino_db.ironwood, &ironwood_key) { + Ok(raw) => match StoredEntryVar::::from_bytes(raw) + .map_err(|error| format!("ironwood decode error: {error}")) + { + Ok(entry) => Some(entry), + Err(message) => { + send_status(&sender, tonic::Status::internal(message)); + return; + } + }, + Err(lmdb::Error::NotFound) => None, + Err(error) => { + send_status( + &sender, + tonic::Status::internal(format!( + "lmdb get(ironwood) failed: {error}" + )), + ); + return; + } + } + } else { + None + }; + let ironwood = match ironwood_entries.as_ref() { + Some(entry) => entry.inner().tx(), + None => &[], + }; + // Invariant: if a pool is requested, its per-height vector length must match txids. if pool_types.includes_transparent() && transparent.len() != txids.len() { send_status( @@ -986,6 +1068,22 @@ impl DbV1 { ); return; } + // Ironwood rows are optional; only enforce alignment when a row is present. + if pool_types.includes_ironwood() + && !ironwood.is_empty() + && ironwood.len() != txids.len() + { + send_status( + &sender, + tonic::Status::internal(format!( + "ironwood list length mismatch at height {}: txids={}, ironwood={}", + current_height.0, + txids.len(), + ironwood.len(), + )), + ); + return; + } // ----- Build CompactTx list ----- // @@ -1036,6 +1134,17 @@ impl DbV1 { }) .unwrap_or_default(); + let ironwood_actions = ironwood + .get(i) + .and_then(|opt| opt.as_ref()) + .map(|o| { + o.actions() + .iter() + .map(|a| a.into_compact()) + .collect::>() + }) + .unwrap_or_default(); + let (vin, vout) = transparent .get(i) .and_then(|opt| opt.as_ref()) @@ -1051,6 +1160,7 @@ impl DbV1 { if spends.is_empty() && outputs.is_empty() && actions.is_empty() + && ironwood_actions.is_empty() && vin.is_empty() && vout.is_empty() { @@ -1064,7 +1174,7 @@ impl DbV1 { spends, outputs, actions, - ironwood_actions: Vec::new(), + ironwood_actions, vin, vout, }); @@ -1072,8 +1182,10 @@ impl DbV1 { // ----- Decode commitment tree data and construct block ----- let commitment_tree_data: CommitmentTreeData = - match StoredEntryFixed::from_bytes(raw_commitment_tree_bytes) - .map_err(|error| format!("commitment_tree decode error: {error}")) + match StoredEntryVar::::from_bytes( + raw_commitment_tree_bytes, + ) + .map_err(|error| format!("commitment_tree decode error: {error}")) { Ok(entry) => *entry.inner(), Err(message) => { @@ -1085,7 +1197,7 @@ impl DbV1 { let chain_metadata = zaino_proto::proto::compact_formats::ChainMetadata { sapling_commitment_tree_size: commitment_tree_data.sizes().sapling(), orchard_commitment_tree_size: commitment_tree_data.sizes().orchard(), - ironwood_commitment_tree_size: 0, + ironwood_commitment_tree_size: commitment_tree_data.sizes().ironwood(), }; let compact_block = zaino_proto::proto::compact_formats::CompactBlock { diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/indexed_block.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/indexed_block.rs index d2da6ee3f..a70efb0ff 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/indexed_block.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/indexed_block.rs @@ -112,9 +112,31 @@ impl DbV1 { .clone(); let orchard = orchard_list.tx(); + // Ironwood (NU6.3): rows only exist from schema v1.3.0 onward, and only for blocks at or + // above NU6.3 activation. A missing row means the block predates ironwood, so every + // transaction has an empty ironwood component. + let ironwood_list = match txn.get(self.ironwood, &height_bytes) { + Ok(raw) => Some( + StoredEntryVar::::from_bytes(raw) + .map_err(|e| { + FinalisedStateError::Custom(format!("ironwood decode error: {e}")) + })? + .inner() + .clone(), + ), + Err(lmdb::Error::NotFound) => None, + Err(e) => return Err(FinalisedStateError::LmdbError(e)), + }; + let ironwood: &[Option] = + ironwood_list.as_ref().map(|list| list.tx()).unwrap_or(&[]); + // Build CompactTxData let len = txids.len(); - if transparent.len() != len || sapling.len() != len || orchard.len() != len { + if transparent.len() != len + || sapling.len() != len + || orchard.len() != len + || (!ironwood.is_empty() && ironwood.len() != len) + { return Err(FinalisedStateError::Custom( "mismatched tx list lengths in block data".to_string(), )); @@ -132,15 +154,19 @@ impl DbV1 { let orchard_tx = orchard[i] .clone() .unwrap_or_else(|| OrchardCompactTx::new(None, vec![])); + let ironwood_tx = ironwood + .get(i) + .cloned() + .flatten() + .unwrap_or_else(OrchardCompactTx::empty); - // TODO: Return ironwood! CompactTxData::new( i as u64, txid, transparent_tx, sapling_tx, orchard_tx, - OrchardCompactTx::empty(), + ironwood_tx, ) }) .collect(); @@ -156,11 +182,12 @@ impl DbV1 { Err(e) => return Err(FinalisedStateError::LmdbError(e)), }; - let commitment_tree_data: CommitmentTreeData = *StoredEntryFixed::from_bytes(raw) - .map_err(|e| { - FinalisedStateError::Custom(format!("commitment_tree decode error: {e}")) - })? - .inner(); + let commitment_tree_data: CommitmentTreeData = + *StoredEntryVar::::from_bytes(raw) + .map_err(|e| { + FinalisedStateError::Custom(format!("commitment_tree decode error: {e}")) + })? + .inner(); // Construct IndexedBlock Ok(Some(IndexedBlock::new( diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/validation.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/validation.rs index 45d55b154..16cd7eabd 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/validation.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/validation.rs @@ -219,12 +219,32 @@ impl DbV1 { } } - // *** commitment_tree_data (fixed) *** + // *** ironwood *** + // + // Ironwood (NU6.3) was introduced in schema v1.3.0. Blocks written before that upgrade — and + // any block below NU6.3 activation — have no ironwood entry, so a missing row is valid and + // simply means "no ironwood data at this height". When an entry is present its checksum is + // verified like the other pools. + { + match ro.get(self.ironwood, &height_key) { + Ok(raw) => { + let entry = StoredEntryVar::::from_bytes(raw) + .map_err(|e| fail(&format!("ironwood corrupt data: {e}")))?; + if !entry.verify(&height_key) { + return Err(fail("ironwood checksum mismatch")); + } + } + Err(lmdb::Error::NotFound) => {} + Err(e) => return Err(FinalisedStateError::LmdbError(e)), + } + } + + // *** commitment_tree_data (var) *** { let raw = ro .get(self.commitment_tree_data, &height_key) .map_err(FinalisedStateError::LmdbError)?; - let entry = StoredEntryFixed::::from_bytes(raw) + let entry = StoredEntryVar::::from_bytes(raw) .map_err(|e| fail(&format!("commitment_tree corrupt bytes: {e}")))?; if !entry.verify(&height_key) { return Err(fail("commitment_tree checksum mismatch")); diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/write_core.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/write_core.rs index e3d28fbb5..cb4d21e4e 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/write_core.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/write_core.rs @@ -19,7 +19,8 @@ fn approx_indexed_block_bytes(block: &IndexedBlock) -> u64 { + transparent.outputs().len() + tx.sapling().spends().len() + tx.sapling().outputs().len() - + tx.orchard().actions().len(); + + tx.orchard().actions().len() + + tx.ironwood().actions().len(); 256 + items as u64 * 128 }) .sum() @@ -418,9 +419,10 @@ impl DbV1 { BlockHeaderData::new(block.context, *block.data()), ); - // Build commitment tree data + // Build commitment tree data. Stored as a `StoredEntryVar` because `CommitmentTreeData` V2 + // is variable-length (optional Ironwood root). let commitment_tree_entry = - StoredEntryFixed::new(&block_height_bytes, *block.commitment_tree_data()); + StoredEntryVar::new(&block_height_bytes, *block.commitment_tree_data()); // Build transaction indexes. // @@ -435,6 +437,7 @@ impl DbV1 { let mut txid_set: HashSet = HashSet::with_capacity(tx_len); let mut sapling = Vec::with_capacity(tx_len); let mut orchard = Vec::with_capacity(tx_len); + let mut ironwood = Vec::with_capacity(tx_len); let mut spent_map: HashMap = HashMap::new(); @@ -486,6 +489,14 @@ impl DbV1 { }; orchard.push(orchard_data); + // Ironwood transactions (NU6.3; modelled with the Orchard compact types). + let ironwood_data = if tx.ironwood().actions().is_empty() { + None + } else { + Some(tx.ironwood().clone()) + }; + ironwood.push(ironwood_data); + // Transaction location let tx_index = u16::try_from(_tx_index).map_err(|_| FinalisedStateError::InvalidBlock { @@ -639,6 +650,7 @@ impl DbV1 { StoredEntryVar::new(&block_height_bytes, TransparentTxList::new(transparent)); let sapling_entry = StoredEntryVar::new(&block_height_bytes, SaplingTxList::new(sapling)); let orchard_entry = StoredEntryVar::new(&block_height_bytes, OrchardTxList::new(orchard)); + let ironwood_entry = StoredEntryVar::new(&block_height_bytes, OrchardTxList::new(ironwood)); // if any database writes fail, or block validation fails, remove block from database and return err. let zaino_db = self.detached_handle(); @@ -699,6 +711,13 @@ impl DbV1 { WriteFlags::NO_OVERWRITE, )?; + txn.put( + zaino_db.ironwood, + &block_height_bytes, + &ironwood_entry.to_bytes()?, + WriteFlags::NO_OVERWRITE, + )?; + txn.put( zaino_db.commitment_tree_data, &block_height_bytes, @@ -1122,7 +1141,7 @@ impl DbV1 { BlockHeaderData::new(block.context, *block.data()), ); let commitment_tree_entry = - StoredEntryFixed::new(&block_height_bytes, *block.commitment_tree_data()); + StoredEntryVar::new(&block_height_bytes, *block.commitment_tree_data()); let tx_len = block.transactions().len(); let mut txids: Vec = Vec::with_capacity(tx_len); @@ -1130,6 +1149,7 @@ impl DbV1 { let mut transparent: Vec> = Vec::with_capacity(tx_len); let mut sapling = Vec::with_capacity(tx_len); let mut orchard = Vec::with_capacity(tx_len); + let mut ironwood = Vec::with_capacity(tx_len); for (tx_index, tx) in block.transactions().iter().enumerate() { let hash = tx.txid(); @@ -1166,6 +1186,13 @@ impl DbV1 { }; orchard.push(orchard_data); + let ironwood_data = if tx.ironwood().actions().is_empty() { + None + } else { + Some(tx.ironwood().clone()) + }; + ironwood.push(ironwood_data); + let tx_index = u16::try_from(tx_index).map_err(|_| FinalisedStateError::InvalidBlock { height: block_height.0, @@ -1199,6 +1226,8 @@ impl DbV1 { StoredEntryVar::new(&block_height_bytes, SaplingTxList::new(sapling)); let orchard_entry = StoredEntryVar::new(&block_height_bytes, OrchardTxList::new(orchard)); + let ironwood_entry = + StoredEntryVar::new(&block_height_bytes, OrchardTxList::new(ironwood)); // Height-keyed tables (+ the hash-keyed `heights`, one entry/block) written per block. put_idempotent( @@ -1237,6 +1266,12 @@ impl DbV1 { &block_height_bytes, &orchard_entry.to_bytes()?, )?; + put_idempotent( + &mut txn, + self.ironwood, + &block_height_bytes, + &ironwood_entry.to_bytes()?, + )?; put_idempotent( &mut txn, self.commitment_tree_data, @@ -1648,13 +1683,15 @@ impl DbV1 { } } - // Delete block data + // Delete block data. `ironwood` is `NotFound`-tolerant below, so deleting a pre-v1.3.0 + // block (which has no ironwood row) is a no-op. for &db in &[ zaino_db.headers, zaino_db.txids, zaino_db.transparent, zaino_db.sapling, zaino_db.orchard, + zaino_db.ironwood, zaino_db.commitment_tree_data, ] { match txn.del(db, &block_height_bytes, None) { diff --git a/packages/zaino-state/src/chain_index/finalised_state/migrations.rs b/packages/zaino-state/src/chain_index/finalised_state/migrations.rs index cb7d4bd58..8ffbd0609 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/migrations.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/migrations.rs @@ -160,7 +160,7 @@ use crate::{ }, config::ChainIndexConfig, error::FinalisedStateError, - Height, TransparentTxList, TxLocation, TxidList, ZainoVersionedSerde as _, + CommitmentTreeData, Height, TransparentTxList, TxLocation, TxidList, ZainoVersionedSerde as _, }; use lmdb::{Transaction, WriteFlags}; @@ -378,6 +378,7 @@ impl MigrationManager { (1, 0, 0) => Ok(MigrationStep::Migration1_0_0To1_1_0(Migration1_0_0To1_1_0)), (1, 1, 0) => Ok(MigrationStep::Migration1_1_0To1_2_0(Migration1_1_0To1_2_0)), (1, 2, 0) => Ok(MigrationStep::Migration1_2_0To1_2_1(Migration1_2_0To1_2_1)), + (1, 2, 1) => Ok(MigrationStep::Migration1_2_1To1_3_0(Migration1_2_1To1_3_0)), (_, _, _) => Err(FinalisedStateError::Custom(format!( "Missing migration from version {}", self.current_version @@ -395,6 +396,7 @@ enum MigrationStep { Migration1_0_0To1_1_0(Migration1_0_0To1_1_0), Migration1_1_0To1_2_0(Migration1_1_0To1_2_0), Migration1_2_0To1_2_1(Migration1_2_0To1_2_1), + Migration1_2_1To1_3_0(Migration1_2_1To1_3_0), } impl MigrationStep { @@ -409,6 +411,9 @@ impl MigrationStep { MigrationStep::Migration1_2_0To1_2_1(_step) => { >::TO_VERSION } + MigrationStep::Migration1_2_1To1_3_0(_step) => { + >::TO_VERSION + } } } @@ -423,6 +428,9 @@ impl MigrationStep { MigrationStep::Migration1_2_0To1_2_1(step) => { >::migration_type(step) } + MigrationStep::Migration1_2_1To1_3_0(step) => { + >::migration_type(step) + } } } @@ -436,6 +444,7 @@ impl MigrationStep { MigrationStep::Migration1_0_0To1_1_0(step) => step.migrate(router, cfg, source).await, MigrationStep::Migration1_1_0To1_2_0(step) => step.migrate(router, cfg, source).await, MigrationStep::Migration1_2_0To1_2_1(step) => step.migrate(router, cfg, source).await, + MigrationStep::Migration1_2_1To1_3_0(step) => step.migrate(router, cfg, source).await, } } } @@ -1004,3 +1013,264 @@ impl Migration for Migration1_2_0To1_2_1 { patch: 1, }; } + +/// Minor migration: v1.2.1 → v1.3.0. +/// +/// Introduces the Ironwood (NU6.3) shielded pool to the finalised state: +/// - a new per-height `ironwood` table (`ironwood_1_3_0`), created empty by `DbV1::spawn`; new +/// blocks populate it via the write path. No backfill is needed because every block already on +/// disk predates NU6.3 (see the guard below). +/// - the `commitment_tree_data` table is rebuilt from the legacy fixed-length +/// `StoredEntryFixed` (V1) rows into the variable-length +/// `StoredEntryVar` (V2) layout, which carries the optional Ironwood root and +/// size. The rebuild reads the legacy `commitment_tree_data_1_0_0` table and writes the new +/// `commitment_tree_data_1_3_0` table (opened as the primary's `commitment_tree_data` handle), +/// then clears the legacy table. +/// +/// This is a **rebuild from existing on-disk data** (no validator refetch), correct only while every +/// stored block predates Ironwood. The migration therefore asserts the database tip is below NU6.3 +/// activation before running; a database already synced past NU6.3 under the old schema would be +/// missing ironwood data that cannot be reconstructed from existing rows and must be re-indexed from +/// the validator instead. +/// +/// Safety and resumability: +/// - Deterministic: each rebuilt row is derived only from the matching legacy row (Ironwood +/// root/size default to `None`/`0` via `CommitmentTreeData` V1 decode). +/// - Resumable: the next height to rebuild is stored in the metadata DB under a temporary key. +/// - Crash-safe: each height's rebuilt row and the progress update commit in the same transaction; +/// idempotent on resume (`NO_OVERWRITE` + verify-match). +struct Migration1_2_1To1_3_0; + +impl Migration for Migration1_2_1To1_3_0 { + const CURRENT_VERSION: DbVersion = DbVersion { + major: 1, + minor: 2, + patch: 1, + }; + + const TO_VERSION: DbVersion = DbVersion { + major: 1, + minor: 3, + patch: 0, + }; + + fn migration_type(&self) -> MigrationType { + MigrationType::Minor + } + + async fn migrate( + &self, + router: Arc>, + cfg: ChainIndexConfig, + _source: T, + ) -> Result<(), FinalisedStateError> { + use lmdb::DatabaseFlags; + + // Temporary metadata entry recording the next height to rebuild, removed on completion. + const MIGRATION_CTD_PROGRESS_KEY: &[u8] = + b"_migration_commitment_tree_data_progress_1_3_0_next_height"; + + info!("Starting v1.2.1 → v1.3.0 migration (Ironwood)."); + + // Use the persistent primary directly (an ephemeral passthrough serves reads during the + // migration; `backend(WriteCore)` would route there and has no LMDB env). + let backend = router.primary_backend(); + let env = backend.env()?; + let metadata_db = backend.metadata_db()?; + // The primary's `commitment_tree_data` handle is the new `commitment_tree_data_1_3_0` table. + let new_ctd_db = backend.commitment_tree_data_db()?; + + // Open the legacy fixed-length commitment table by name. On a pre-v1.3.0 database it already + // exists; `open_or_create_db` creating it empty on an unexpected fresh DB is harmless (the + // rebuild loop below only runs when a tip exists, and an empty legacy table yields no rows). + let legacy_ctd_db = + crate::chain_index::finalised_state::finalised_source::open_or_create_db( + &env, + "commitment_tree_data_1_0_0", + DatabaseFlags::empty(), + ) + .await?; + + // Guard: Ironwood data is only expected from NU6.3 activation. A rebuild-from-existing-data + // migration cannot reconstruct ironwood roots/sizes for post-NU6.3 blocks. + let zebra_network = cfg.network.to_zebra_network(); + let nu6_3_activation_height = + zebra_chain::parameters::NetworkUpgrade::Nu6_3.activation_height(&zebra_network); + + // Mark migration in progress (observability only; resumption uses the progress key). + { + let mut metadata: DbMetadata = backend.get_metadata().await?; + if metadata.migration_status == MigrationStatus::Empty { + metadata.migration_status = MigrationStatus::PartialBuidInProgress; + backend.update_metadata(metadata).await?; + } + } + + // Reads the temporary progress height, returning `None` if the key is absent. + let read_progress = |key: &[u8]| -> Result, FinalisedStateError> { + let txn = env.begin_ro_txn()?; + match txn.get(metadata_db, &key) { + Ok(bytes) => { + let entry = StoredEntryFixed::::from_bytes(bytes).map_err(|error| { + FinalisedStateError::Custom(format!( + "corrupt v1.3.0 migration progress entry: {error}" + )) + })?; + if !entry.verify(key) { + return Err(FinalisedStateError::Custom( + "v1.3.0 migration progress checksum mismatch".to_string(), + )); + } + Ok(Some(entry.inner().0)) + } + Err(lmdb::Error::NotFound) => Ok(None), + Err(error) => Err(FinalisedStateError::LmdbError(error)), + } + }; + + // Nothing to rebuild on an empty database; fall through to finalisation. + if let Some(db_tip) = backend.db_height().await? { + let db_tip = db_tip.0; + + if let Some(activation) = nu6_3_activation_height { + if db_tip >= activation.0 { + return Err(FinalisedStateError::Custom(format!( + "cannot migrate finalised state to v1.3.0: tip {db_tip} is at or above NU6.3 \ + activation {} but was built without Ironwood data; wipe and re-index the \ + finalised state from the validator", + activation.0 + ))); + } + } + + let mut next_height = + read_progress(MIGRATION_CTD_PROGRESS_KEY)?.unwrap_or(GENESIS_HEIGHT.0); + + info!( + resume_height = next_height, + db_tip, + "v1.3.0 migration: rebuilding commitment_tree_data into StoredEntryVar (V2)" + ); + let started = std::time::Instant::now(); + + while next_height <= db_tip { + let height = Height::try_from(next_height) + .map_err(|error| FinalisedStateError::Custom(error.to_string()))?; + let height_bytes = height.to_bytes()?; + + // Read + verify the legacy fixed-length row. + let commitment_tree_data: CommitmentTreeData = { + let txn = env.begin_ro_txn()?; + let raw = txn + .get(legacy_ctd_db, &height_bytes) + .map_err(FinalisedStateError::LmdbError)?; + let entry = StoredEntryFixed::::from_bytes(raw).map_err( + |error| { + FinalisedStateError::Custom(format!( + "legacy commitment_tree_data corrupt data: {error}" + )) + }, + )?; + if !entry.verify(&height_bytes) { + return Err(FinalisedStateError::Custom( + "legacy commitment_tree_data checksum mismatch".to_string(), + )); + } + *entry.inner() + }; + + // Re-wrap in the V2 `StoredEntryVar` layout and advance progress atomically. + let new_entry_bytes = + StoredEntryVar::new(&height_bytes, commitment_tree_data).to_bytes()?; + + { + let mut txn = env.begin_rw_txn()?; + match txn.put( + new_ctd_db, + &height_bytes, + &new_entry_bytes, + WriteFlags::NO_OVERWRITE, + ) { + Ok(()) => {} + // Idempotent on resume: an existing rebuilt row must match exactly. + Err(lmdb::Error::KeyExist) => { + let existing = txn + .get(new_ctd_db, &height_bytes) + .map_err(FinalisedStateError::LmdbError)?; + if existing != new_entry_bytes.as_slice() { + return Err(FinalisedStateError::Custom(format!( + "conflicting rebuilt commitment_tree_data at height {}", + height.0 + ))); + } + } + Err(error) => return Err(FinalisedStateError::LmdbError(error)), + } + + let progress = + StoredEntryFixed::new(MIGRATION_CTD_PROGRESS_KEY, height + 1); + txn.put( + metadata_db, + &MIGRATION_CTD_PROGRESS_KEY, + &progress.to_bytes()?, + WriteFlags::empty(), + )?; + txn.commit()?; + } + + if next_height % SYNC_CHECKPOINT_INTERVAL == 0 { + env.sync(true)?; + } + if next_height % 50_000 == 0 { + info!( + height = next_height, + db_tip, + elapsed = ?started.elapsed(), + "v1.3.0 migration progress" + ); + } + + next_height = height.0 + 1; + } + + env.sync(true)?; + info!( + db_tip, + elapsed = ?started.elapsed(), + "v1.3.0 migration: commitment_tree_data rebuild complete" + ); + + // Drop the legacy commitment data: clearing frees its pages back to the env freelist. + { + let mut txn = env.begin_rw_txn()?; + txn.clear_db(legacy_ctd_db)?; + txn.commit()?; + } + env.sync(true)?; + } + + // Finalise: advance the version durably (the completion gate) before removing the progress + // key, so a crash re-selects and cheaply resumes this migration rather than skipping it. + env.sync(true)?; + let mut metadata: DbMetadata = backend.get_metadata().await?; + metadata.version = >::TO_VERSION; + metadata.schema_hash = + crate::chain_index::finalised_state::finalised_source::v1::DB_SCHEMA_V1_HASH; + metadata.migration_status = MigrationStatus::Empty; + backend.update_metadata(metadata).await?; + env.sync(true)?; + + { + let mut txn = env.begin_rw_txn()?; + match txn.del(metadata_db, &MIGRATION_CTD_PROGRESS_KEY, None) { + Ok(()) | Err(lmdb::Error::NotFound) => {} + Err(error) => return Err(FinalisedStateError::LmdbError(error)), + } + txn.commit()?; + } + env.sync(true)?; + + info!("v1.2.1 to v1.3.0 migration complete."); + Ok(()) + } +} diff --git a/packages/zaino-state/src/chain_index/finalised_state/reader.rs b/packages/zaino-state/src/chain_index/finalised_state/reader.rs index 1952d2a1a..c2885b4b5 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/reader.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/reader.rs @@ -313,6 +313,37 @@ impl DbReader { .await } + /// Fetch the serialized ironwood (NU6.3) compact tx for the given TxLocation, if present. + pub(crate) async fn get_ironwood( + &self, + tx_location: TxLocation, + ) -> Result, FinalisedStateError> { + self.db(CapabilityRequest::BlockShieldedExt)? + .get_ironwood(tx_location) + .await + } + + /// Fetch block ironwood transaction data by height. + pub(crate) async fn get_block_ironwood( + &self, + height: Height, + ) -> Result { + self.db(CapabilityRequest::BlockShieldedExt)? + .get_block_ironwood(height) + .await + } + + /// Fetches block ironwood tx data for the given height range. + pub(crate) async fn get_block_range_ironwood( + &self, + start: Height, + end: Height, + ) -> Result, FinalisedStateError> { + self.db(CapabilityRequest::BlockShieldedExt)? + .get_block_range_ironwood(start, end) + .await + } + /// Fetch block commitment tree data by height. pub(crate) async fn get_block_commitment_tree_data( &self, diff --git a/packages/zaino-state/src/chain_index/types/db/legacy.rs b/packages/zaino-state/src/chain_index/types/db/legacy.rs index 8341616ab..8fedd5c31 100644 --- a/packages/zaino-state/src/chain_index/types/db/legacy.rs +++ b/packages/zaino-state/src/chain_index/types/db/legacy.rs @@ -1107,6 +1107,7 @@ impl IndexedBlock { let sapling_commitment_tree_size = self.commitment_tree_data().sizes().sapling(); let orchard_commitment_tree_size = self.commitment_tree_data().sizes().orchard(); + let ironwood_commitment_tree_size = self.commitment_tree_data().sizes().ironwood(); zaino_proto::proto::compact_formats::CompactBlock { proto_version: 0, @@ -1119,7 +1120,7 @@ impl IndexedBlock { chain_metadata: Some(zaino_proto::proto::compact_formats::ChainMetadata { sapling_commitment_tree_size, orchard_commitment_tree_size, - ironwood_commitment_tree_size: 0, + ironwood_commitment_tree_size, }), } } diff --git a/packages/zaino-state/src/chain_index/types/helpers.rs b/packages/zaino-state/src/chain_index/types/helpers.rs index 2797379b9..b89b6cf72 100644 --- a/packages/zaino-state/src/chain_index/types/helpers.rs +++ b/packages/zaino-state/src/chain_index/types/helpers.rs @@ -91,7 +91,7 @@ impl TreeRootData { ) { let (sapling_root, sapling_size) = self.sapling.unwrap_or_default(); let (orchard_root, orchard_size) = self.orchard.unwrap_or_default(); - let (ironwood_root, ironwood_size) = self.orchard.unwrap_or_default(); + let (ironwood_root, ironwood_size) = self.ironwood.unwrap_or_default(); ( sapling_root, sapling_size, @@ -315,13 +315,13 @@ impl<'a> BlockWithMetadata<'a> { ) } - /// Extract orchard transaction data + /// Extract ironwood transaction data fn extract_ironwood_data( &self, txn: &zebra_chain::transaction::Transaction, ) -> OrchardCompactTx { - let orchard_value = { - let val = txn.ironwood_value_balance().orchard_amount(); + let ironwood_value = { + let val = txn.ironwood_value_balance().ironwood_amount(); if val == 0 { None } else { @@ -330,7 +330,7 @@ impl<'a> BlockWithMetadata<'a> { }; OrchardCompactTx::new( - orchard_value, + ironwood_value, txn.ironwood_actions() .map(|action| { let cipher: [u8; 52] = <[u8; 580]>::from(action.enc_ciphertext)[..52] From cce2e62f380ef493a008a9325f723d522892efee Mon Sep 17 00:00:00 2001 From: idky137 Date: Sat, 4 Jul 2026 09:50:58 +0100 Subject: [PATCH 033/134] fixed crate tests --- live-tests/clientless/tests/chain_cache.rs | 1 + live-tests/zaino-testutils/src/lib.rs | 4 + packages/zaino-state/src/backends/fetch.rs | 4 +- .../src/chain_index/finalised_state/entry.rs | 11 +- .../finalised_state/finalised_source/v1.rs | 108 +++++++++++-- .../finalised_source/v1/compact_block.rs | 79 +++++---- .../chain_index/finalised_state/migrations.rs | 3 +- .../migrations/v1_0_to_v1_1.rs | 6 +- .../migrations/v1_1_to_v1_2.rs | 153 ++++++++++++++++-- .../chain_index/tests/finalised_state/v1.rs | 45 ++++-- .../src/chain_index/types/helpers.rs | 1 + 11 files changed, 316 insertions(+), 99 deletions(-) diff --git a/live-tests/clientless/tests/chain_cache.rs b/live-tests/clientless/tests/chain_cache.rs index aa4cc491b..a12cb7291 100644 --- a/live-tests/clientless/tests/chain_cache.rs +++ b/live-tests/clientless/tests/chain_cache.rs @@ -114,6 +114,7 @@ mod chain_query_interface { nu6: local_net_activation_heights.nu6(), nu6_1: local_net_activation_heights.nu6_1(), nu6_2: local_net_activation_heights.nu6_2(), + nu6_3: local_net_activation_heights.nu6_3(), nu7: local_net_activation_heights.nu7(), }, )) diff --git a/live-tests/zaino-testutils/src/lib.rs b/live-tests/zaino-testutils/src/lib.rs index 468e434fe..8133c87e4 100644 --- a/live-tests/zaino-testutils/src/lib.rs +++ b/live-tests/zaino-testutils/src/lib.rs @@ -323,6 +323,9 @@ pub fn local_network_from_activation_heights( nu6_2: activation_heights .nu6_2 .map(zcash_protocol::consensus::BlockHeight::from), + nu6_3: activation_heights + .nu6_3 + .map(zcash_protocol::consensus::BlockHeight::from), } } @@ -1086,6 +1089,7 @@ pub async fn launch_state_and_fetch_services_mining_to( nu6: activation_heights.nu6(), nu6_1: activation_heights.nu6_1(), nu6_2: activation_heights.nu6_2(), + nu6_3: activation_heights.nu6_3(), nu7: activation_heights.nu7(), } }), diff --git a/packages/zaino-state/src/backends/fetch.rs b/packages/zaino-state/src/backends/fetch.rs index f720b2238..1a2400396 100644 --- a/packages/zaino-state/src/backends/fetch.rs +++ b/packages/zaino-state/src/backends/fetch.rs @@ -296,7 +296,7 @@ impl ZcashIndexer for FetchServiceSubscriber { /// Some fields from the zcashd reference are missing from Zebra's [`GetBlockchainInfoResponse`]. It only contains the fields /// [required for lightwalletd support.](https://github.com/zcash/lightwalletd/blob/v0.4.9/common/common.go#L72-L89) async fn get_blockchain_info(&self) -> Result { - Ok(self + self .fetcher .get_blockchain_info() .await? @@ -308,7 +308,7 @@ impl ZcashIndexer for FetchServiceSubscriber { "chainwork not hex-encoded integer", ), ) - })?) + }) } /// Returns details on the active state of the TX memory pool. diff --git a/packages/zaino-state/src/chain_index/finalised_state/entry.rs b/packages/zaino-state/src/chain_index/finalised_state/entry.rs index 64a1d0dfa..f12b5d8f6 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/entry.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/entry.rs @@ -145,14 +145,11 @@ impl StoredEntryFixed { let mut v = T::VERSION; loop { - match self.item.to_bytes_with_version(v) { - Ok(item_bytes) => { - let candidate = Self::blake2b256(&[key.as_ref(), &item_bytes].concat()); - if candidate == self.checksum { - return true; - } + if let Ok(item_bytes) = self.item.to_bytes_with_version(v) { + let candidate = Self::blake2b256(&[key.as_ref(), &item_bytes].concat()); + if candidate == self.checksum { + return true; } - Err(_) => {} } if v == 1 { diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1.rs index 94f5e2355..1c4a1bdb7 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1.rs @@ -414,7 +414,7 @@ impl DbV1 { /// - validates or initializes the `"metadata"` record (schema hash + version), and /// - spawns the background validator / maintenance task. pub(crate) async fn spawn(config: &ChainIndexConfig) -> Result { - let mut zaino_db = Self::open_env_and_dbs(config, "commitment_tree_data_1_3_0").await?; + let mut zaino_db = Self::open_env_and_dbs(config).await?; // Validate (or initialise) the metadata entry before we touch any tables. zaino_db.check_schema_version().await?; @@ -434,14 +434,10 @@ impl DbV1 { /// (status `Spawning`, `db_handler` = `None`, fresh atomics). Performs no metadata validation /// and starts no background task — each caller (`spawn`, `spawn_v1_0_0`) adds its own tail. /// - /// `commitment_tree_data_name` selects the LMDB table backing the `commitment_tree_data` field: - /// the current path passes the v1.3.0 table (`commitment_tree_data_1_3_0`, `StoredEntryVar`), - /// while the v1.0.0 fixture builder ([`DbV1::spawn_v1_0_0`]) passes the legacy - /// `commitment_tree_data_1_0_0` (`StoredEntryFixed`) so it reproduces a pre-migration database. - async fn open_env_and_dbs( - config: &ChainIndexConfig, - commitment_tree_data_name: &str, - ) -> Result { + /// The `commitment_tree_data` handle is the up-to-date `commitment_tree_data_1_3_0` table + /// (`StoredEntryVar`). The v1.0.0 fixture builder ([`DbV1::spawn_v1_0_0`]) repoints it to the + /// legacy `commitment_tree_data_1_0_0` table (`StoredEntryFixed`) after calling this. + async fn open_env_and_dbs(config: &ChainIndexConfig) -> Result { info!("Launching FinalisedState"); // Prepare database details and path. @@ -503,7 +499,7 @@ impl DbV1 { let ironwood = super::open_or_create_db(&env, "ironwood_1_3_0", DatabaseFlags::empty()).await?; let commitment_tree_data = - super::open_or_create_db(&env, commitment_tree_data_name, DatabaseFlags::empty()) + super::open_or_create_db(&env, "commitment_tree_data_1_3_0", DatabaseFlags::empty()) .await?; let hashes = super::open_or_create_db(&env, "hashes_1_0_0", DatabaseFlags::empty()).await?; @@ -997,17 +993,99 @@ impl DbV1 { pub(crate) async fn spawn_v1_0_0( config: &ChainIndexConfig, ) -> Result { - // The v1.0.0 fixture must reproduce the legacy on-disk layout: the commitment tree data - // lives in `commitment_tree_data_1_0_0` as a `StoredEntryFixed` (see `write_block_v1_0_0`), - // which the v1.2.1 -> v1.3.0 migration later rebuilds into the `StoredEntryVar` table. - let mut zaino_db = - Self::open_env_and_dbs(config, "commitment_tree_data_1_0_0").await?; + // The v1.0.0 fixture reproduces the legacy on-disk layout: its commitment tree data lives in + // `commitment_tree_data_1_0_0` as a `StoredEntryFixed` (see `write_block_v1_0_0`), which the + // v1.2.1 -> v1.3.0 migration later rebuilds into the `commitment_tree_data_1_3_0` + // `StoredEntryVar` table. This opener therefore opens the legacy commitment table and never + // creates the v1.3.0 table. + let db_size_bytes = config.storage.database.size.to_byte_count(); + let db_path_dir = match config.network.to_zebra_network().kind() { + NetworkKind::Mainnet => "mainnet", + NetworkKind::Testnet => "testnet", + NetworkKind::Regtest => "regtest", + }; + let db_path = config.storage.database.path.join(db_path_dir).join("v1"); + if !db_path.exists() { + fs::create_dir_all(&db_path)?; + } + let cpu_cnt = std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(4); + let max_readers = u32::try_from((cpu_cnt * 32).clamp(512, 4096)) + .expect("max_readers was clamped to fit in u32"); + let env = Environment::new() + .set_max_dbs(16) + .set_map_size(db_size_bytes) + .set_max_readers(max_readers) + .set_flags( + EnvironmentFlags::NO_TLS + | EnvironmentFlags::NO_READAHEAD + | EnvironmentFlags::NO_SYNC, + ) + .open(&db_path)?; + + let headers = + super::open_or_create_db(&env, "headers_1_0_0", DatabaseFlags::empty()).await?; + let txids = super::open_or_create_db(&env, "txids_1_0_0", DatabaseFlags::empty()).await?; + let transparent = + super::open_or_create_db(&env, "transparent_1_0_0", DatabaseFlags::empty()).await?; + let sapling = + super::open_or_create_db(&env, "sapling_1_0_0", DatabaseFlags::empty()).await?; + let orchard = + super::open_or_create_db(&env, "orchard_1_0_0", DatabaseFlags::empty()).await?; + let ironwood = + super::open_or_create_db(&env, "ironwood_1_3_0", DatabaseFlags::empty()).await?; + let commitment_tree_data = + super::open_or_create_db(&env, "commitment_tree_data_1_0_0", DatabaseFlags::empty()) + .await?; + let hashes = super::open_or_create_db(&env, "hashes_1_0_0", DatabaseFlags::empty()).await?; + let spent = super::open_or_create_db(&env, "spent_1_0_0", DatabaseFlags::empty()).await?; + let txid_location = + super::open_or_create_db(&env, "txid_location_1_0_0", DatabaseFlags::empty()).await?; + let metadata = super::open_or_create_db(&env, "metadata", DatabaseFlags::empty()).await?; + #[cfg(feature = "transparent_address_history_experimental")] + let address_history = super::open_or_create_db( + &env, + "address_history_1_0_0", + DatabaseFlags::DUP_SORT | DatabaseFlags::DUP_FIXED, + ) + .await?; + + let zaino_db = Self { + tx_out_set_info_accumulator: super::open_or_create_db( + &env, + TX_OUT_SET_INFO_ACCUMULATOR_DATABASE_NAME, + DatabaseFlags::empty(), + ) + .await?, + env: Arc::new(env), + headers, + txids, + transparent, + sapling, + orchard, + ironwood, + commitment_tree_data, + heights: hashes, + spent, + txid_location, + #[cfg(feature = "transparent_address_history_experimental")] + address_history, + metadata, + validated_tip: Arc::new(AtomicU32::new(0)), + validated_set: DashSet::new(), + db_handler: std::sync::Mutex::new(None), + cancel_token: CancellationToken::new(), + status: NamedAtomicStatus::new("FinalisedState", StatusType::Spawning), + config: config.clone(), + }; // Write the historical v1.0.0 metadata record. Intentionally skips `check_schema_version` // (see the method doc) — that is the behavioural difference from `spawn`. zaino_db.write_v1_0_0_metadata()?; // Spawn handler task to perform background validation and trailing tx cleanup. + let mut zaino_db = zaino_db; zaino_db.spawn_handler().await?; Ok(zaino_db) diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/compact_block.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/compact_block.rs index b0ec0c086..eaf484dbe 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/compact_block.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/compact_block.rs @@ -152,11 +152,9 @@ impl DbV1 { // error — it just means the block has no ironwood data. let ironwood_stored_entry_var = if pool_types.includes_ironwood() { match txn.get(self.ironwood, &height_bytes) { - Ok(raw) => Some( - StoredEntryVar::::from_bytes(raw).map_err(|e| { - FinalisedStateError::Custom(format!("ironwood decode error: {e}")) - })?, - ), + Ok(raw) => Some(StoredEntryVar::::from_bytes(raw).map_err( + |e| FinalisedStateError::Custom(format!("ironwood decode error: {e}")), + )?), Err(lmdb::Error::NotFound) => None, Err(e) => return Err(FinalisedStateError::LmdbError(e)), } @@ -987,45 +985,44 @@ impl DbV1 { // Ironwood is fetched with a tolerant point lookup rather than a lockstep cursor: // rows are sparse (present only from schema v1.3.0 / NU6.3), so a missing row must // not desync a height-aligned cursor. `NotFound` simply means "no ironwood here". - let ironwood_entries: Option> = if pool_types - .includes_ironwood() - { - let ironwood_key = match current_height.to_bytes() { - Ok(key) => key, - Err(error) => { - send_status( - &sender, - tonic::Status::internal(format!( - "ironwood height to_bytes failed: {error}" - )), - ); - return; - } - }; - match txn.get(zaino_db.ironwood, &ironwood_key) { - Ok(raw) => match StoredEntryVar::::from_bytes(raw) - .map_err(|error| format!("ironwood decode error: {error}")) - { - Ok(entry) => Some(entry), - Err(message) => { - send_status(&sender, tonic::Status::internal(message)); + let ironwood_entries: Option> = + if pool_types.includes_ironwood() { + let ironwood_key = match current_height.to_bytes() { + Ok(key) => key, + Err(error) => { + send_status( + &sender, + tonic::Status::internal(format!( + "ironwood height to_bytes failed: {error}" + )), + ); + return; + } + }; + match txn.get(zaino_db.ironwood, &ironwood_key) { + Ok(raw) => match StoredEntryVar::::from_bytes(raw) + .map_err(|error| format!("ironwood decode error: {error}")) + { + Ok(entry) => Some(entry), + Err(message) => { + send_status(&sender, tonic::Status::internal(message)); + return; + } + }, + Err(lmdb::Error::NotFound) => None, + Err(error) => { + send_status( + &sender, + tonic::Status::internal(format!( + "lmdb get(ironwood) failed: {error}" + )), + ); return; } - }, - Err(lmdb::Error::NotFound) => None, - Err(error) => { - send_status( - &sender, - tonic::Status::internal(format!( - "lmdb get(ironwood) failed: {error}" - )), - ); - return; } - } - } else { - None - }; + } else { + None + }; let ironwood = match ironwood_entries.as_ref() { Some(entry) => entry.inner().tx(), None => &[], diff --git a/packages/zaino-state/src/chain_index/finalised_state/migrations.rs b/packages/zaino-state/src/chain_index/finalised_state/migrations.rs index 8ffbd0609..df013dc37 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/migrations.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/migrations.rs @@ -1207,8 +1207,7 @@ impl Migration for Migration1_2_1To1_3_0 { Err(error) => return Err(FinalisedStateError::LmdbError(error)), } - let progress = - StoredEntryFixed::new(MIGRATION_CTD_PROGRESS_KEY, height + 1); + let progress = StoredEntryFixed::new(MIGRATION_CTD_PROGRESS_KEY, height + 1); txn.put( metadata_db, &MIGRATION_CTD_PROGRESS_KEY, diff --git a/packages/zaino-state/src/chain_index/tests/finalised_state/migrations/v1_0_to_v1_1.rs b/packages/zaino-state/src/chain_index/tests/finalised_state/migrations/v1_0_to_v1_1.rs index 735060d1e..e2db7587b 100644 --- a/packages/zaino-state/src/chain_index/tests/finalised_state/migrations/v1_0_to_v1_1.rs +++ b/packages/zaino-state/src/chain_index/tests/finalised_state/migrations/v1_0_to_v1_1.rs @@ -51,7 +51,11 @@ async fn v1_0_to_v1_1_metadata_migration() { .await .unwrap(); - zaino_db.wait_until_ready().await; + // `wait_until_synced` (not `wait_until_ready`): a v1.1.0 database keeps its commitment rows in + // the legacy `commitment_tree_data_1_0_0` table (the v1.2.1 -> v1.3.0 migration has not run), so + // the background validator — which reads the v1.3.0 commitment table — can never reach the ready + // state. `wait_until_synced` waits for the migration to complete, which is what this test needs. + zaino_db.wait_until_synced().await; let metadata = zaino_db.get_metadata().await.unwrap(); diff --git a/packages/zaino-state/src/chain_index/tests/finalised_state/migrations/v1_1_to_v1_2.rs b/packages/zaino-state/src/chain_index/tests/finalised_state/migrations/v1_1_to_v1_2.rs index e0827f148..882763cdc 100644 --- a/packages/zaino-state/src/chain_index/tests/finalised_state/migrations/v1_1_to_v1_2.rs +++ b/packages/zaino-state/src/chain_index/tests/finalised_state/migrations/v1_1_to_v1_2.rs @@ -7,10 +7,10 @@ use zaino_common::network::ActivationHeights; use zaino_common::{DatabaseConfig, Network, StorageConfig}; use crate::chain_index::finalised_state::capability::{ - BlockCoreExt as _, BlockTransparentExt as _, CapabilityRequest, DbRead as _, DbVersion, - MigrationStatus, + BlockCoreExt as _, CapabilityRequest, DbRead as _, DbVersion, MigrationStatus, + TransparentHistExt as _, }; -use crate::chain_index::finalised_state::entry::StoredEntryFixed; +use crate::chain_index::finalised_state::entry::{StoredEntryFixed, StoredEntryVar}; use crate::chain_index::finalised_state::finalised_source::v1::DB_SCHEMA_V1_HASH; use crate::chain_index::finalised_state::finalised_source::v1::TX_OUT_SET_INFO_ACCUMULATOR_KEY; use crate::chain_index::finalised_state::finalised_source::FinalisedSource; @@ -20,7 +20,134 @@ use crate::chain_index::tests::init_tracing; use crate::chain_index::tests::vectors::{ build_active_mockchain_source, load_test_vectors, TestVectorData, }; -use crate::{ChainIndexConfig, Height, TxLocation, ZainoVersionedSerde as _}; +use crate::{ChainIndexConfig, Height, TransparentTxList, TxLocation, ZainoVersionedSerde as _}; + +/// Reads a block's `TransparentTxList` **directly** from the `transparent` table, bypassing the +/// validated accessor (`get_block_transparent`), which routes through `validate_block_blocking` and +/// would read the v1.3.0 commitment table. On a database that has not yet run the v1.2.1 -> v1.3.0 +/// migration the commitment rows are still in the legacy table, so validation would fail; the +/// migration data these tests assert on (`txid_location`, `spent`, txout-set) is unaffected. +fn read_block_transparent_direct( + database_backend: &FinalisedSource, + height: Height, +) -> TransparentTxList { + use lmdb::Transaction as _; + let environment = database_backend.env().unwrap(); + let transparent_database = database_backend.transparent_db().unwrap(); + let transaction = environment.begin_ro_txn().unwrap(); + let raw = transaction + .get(transparent_database, &height.to_bytes().unwrap()) + .unwrap(); + StoredEntryVar::::from_bytes(raw) + .unwrap() + .inner() + .clone() +} + +/// Direct-read copy of the production oracle +/// (`tx_out_set_accumulator::expected_tx_out_set_info_accumulator`), differing only in that it reads +/// the transparent list directly (`read_block_transparent_direct`) instead of through the validated +/// `get_block_transparent`, so it works on a database whose commitment rows are still in the legacy +/// table (validation would fail there). The oracle logic is otherwise identical. +async fn expected_tx_out_set_info_accumulator_direct( + database_backend: &FinalisedSource, + max_height: Height, +) -> crate::chain_index::types::db::metadata::FinalisedTxOutSetInfoAccumulator { + use lmdb::Transaction as _; + + let environment = database_backend.env().unwrap(); + let spent_database = database_backend.spent_db().unwrap(); + + let mut expected_accumulator = + crate::chain_index::types::db::metadata::FinalisedTxOutSetInfoAccumulator::empty(); + + for height_raw in 0..=max_height.0 { + let height = Height(height_raw); + let transparent_transaction_list = read_block_transparent_direct(database_backend, height); + + for (transaction_index, transparent_transaction_opt) in + transparent_transaction_list.tx().iter().enumerate() + { + let Some(transparent_transaction) = transparent_transaction_opt else { + continue; + }; + if transparent_transaction.outputs().is_empty() { + continue; + } + + let transaction_index = u16::try_from(transaction_index).unwrap(); + let transaction_location = TxLocation::new(height.0, transaction_index); + // `get_txid` reads the `txids` table directly (no validation). + let transaction_hash = database_backend + .get_txid(transaction_location) + .await + .unwrap(); + + let mut unspent_outputs_for_transaction = 0u64; + let read_transaction = environment.begin_ro_txn().unwrap(); + + for (output_index, output) in transparent_transaction.outputs().iter().enumerate() { + if crate::chain_index::types::db::metadata::is_unspendable_tx_out(output) { + continue; + } + + let output_index = u32::try_from(output_index).unwrap(); + let outpoint = crate::Outpoint::new(transaction_hash.0, output_index); + let outpoint_bytes = outpoint.to_bytes().unwrap(); + + let still_unspent = match read_transaction.get(spent_database, &outpoint_bytes) { + Ok(spent_bytes) => { + let spent_entry = + StoredEntryFixed::::from_bytes(spent_bytes).unwrap(); + assert!( + spent_entry.verify(&outpoint_bytes), + "spent checksum mismatch for outpoint {outpoint:?}" + ); + spent_entry.inner().block_height() > max_height.0 + } + Err(lmdb::Error::NotFound) => true, + Err(error) => { + panic!("failed to read spent entry for outpoint {outpoint:?}: {error}") + } + }; + + if still_unspent { + unspent_outputs_for_transaction += 1; + expected_accumulator + .apply_added_output(&outpoint, output) + .unwrap(); + } + } + + if unspent_outputs_for_transaction > 0 { + expected_accumulator.transactions += 1; + } + } + } + + expected_accumulator +} + +/// Test assertion: the backend's maintained txout-set accumulator equals the independently recomputed +/// [`expected_tx_out_set_info_accumulator_direct`]. Direct-read equivalent of the production +/// `assert_tx_out_set_info_accumulator_matches_transparent_data`. +async fn assert_tx_out_set_info_accumulator_matches_transparent_data_direct( + database_backend: &FinalisedSource, +) { + let database_height = database_backend.db_height().await.unwrap().unwrap(); + let expected_accumulator = + expected_tx_out_set_info_accumulator_direct(database_backend, database_height).await; + + let actual_accumulator = database_backend + .get_tx_out_set_info_accumulator() + .await + .unwrap(); + + assert_eq!( + actual_accumulator, expected_accumulator, + "txout-set accumulator does not match transparent data and spent index" + ); +} const MIGRATION_SPENT_PROGRESS_KEY: &[u8] = b"_migration_spent_progress_1_2_0_next_height"; @@ -66,10 +193,7 @@ async fn assert_txid_location_index_matches_block_data( for height_raw in 0..=database_height.0 { let height = Height(height_raw); - let transparent_transaction_list = database_backend - .get_block_transparent(height) - .await - .unwrap(); + let transparent_transaction_list = read_block_transparent_direct(database_backend, height); for transaction_index in 0..transparent_transaction_list.tx().len() { let expected_location = TxLocation::new(height.0, transaction_index as u16); @@ -123,7 +247,7 @@ async fn simulate_interrupted_v1_1_to_v1_2_spent_index_migration( let spent_database = database_backend.spent_db().unwrap(); let (tx_out_set_info_accumulator_database, expected_resume_accumulator) = ( database_backend.tx_out_set_info_accumulator_db().unwrap(), - crate::chain_index::finalised_state::finalised_source::v1::tx_out_set_accumulator::expected_tx_out_set_info_accumulator(database_backend, resume_height - 1).await, + expected_tx_out_set_info_accumulator_direct(database_backend, resume_height - 1).await, ); let spent_keys_to_delete: Vec> = { @@ -230,10 +354,7 @@ async fn assert_spent_index_matches_transparent_data( for height_raw in 0..=database_height.0 { let height = Height(height_raw); - let transparent_transaction_list = database_backend - .get_block_transparent(height) - .await - .unwrap(); + let transparent_transaction_list = read_block_transparent_direct(database_backend, height); let transaction = environment.begin_ro_txn().unwrap(); @@ -350,7 +471,7 @@ async fn v1_1_to_v1_2_spent_index_backfill_from_old_version() { assert_txid_location_index_matches_block_data(&migrated_backend).await; assert_spent_index_matches_transparent_data(&migrated_backend).await; - crate::chain_index::finalised_state::finalised_source::v1::tx_out_set_accumulator::assert_tx_out_set_info_accumulator_matches_transparent_data(&migrated_backend).await; + assert_tx_out_set_info_accumulator_matches_transparent_data_direct(&migrated_backend).await; migrated_database.shutdown().await.unwrap(); } @@ -447,7 +568,7 @@ async fn v1_1_to_v1_2_spent_index_migration_resumes_after_crash() { assert_txid_location_index_matches_block_data(&resumed_backend).await; assert_spent_index_matches_transparent_data(&resumed_backend).await; - crate::chain_index::finalised_state::finalised_source::v1::tx_out_set_accumulator::assert_tx_out_set_info_accumulator_matches_transparent_data(&resumed_backend).await; + assert_tx_out_set_info_accumulator_matches_transparent_data_direct(&resumed_backend).await; resumed_database.shutdown().await.unwrap(); } @@ -533,7 +654,7 @@ async fn v1_2_0_cache_missing_txid_location_index_is_rebuilt() { assert_txid_location_index_matches_block_data(&healed_backend).await; assert_spent_index_matches_transparent_data(&healed_backend).await; - crate::chain_index::finalised_state::finalised_source::v1::tx_out_set_accumulator::assert_tx_out_set_info_accumulator_matches_transparent_data(&healed_backend).await; + assert_tx_out_set_info_accumulator_matches_transparent_data_direct(&healed_backend).await; healed_database.shutdown().await.unwrap(); } diff --git a/packages/zaino-state/src/chain_index/tests/finalised_state/v1.rs b/packages/zaino-state/src/chain_index/tests/finalised_state/v1.rs index c93e9ddeb..ec8e54179 100644 --- a/packages/zaino-state/src/chain_index/tests/finalised_state/v1.rs +++ b/packages/zaino-state/src/chain_index/tests/finalised_state/v1.rs @@ -7,7 +7,6 @@ use zaino_common::network::ActivationHeights; use zaino_common::{DatabaseConfig, Network, StorageConfig}; use zaino_proto::proto::utils::{compact_block_with_pool_types, PoolTypeFilter}; -use crate::chain_index::finalised_state::capability::IndexedBlockExt; use crate::chain_index::finalised_state::finalised_source::FinalisedSource; use crate::chain_index::finalised_state::reader::DbReader; use crate::chain_index::finalised_state::FinalisedState; @@ -20,8 +19,12 @@ use crate::chain_index::tests::vectors::{ use crate::chain_index::types::TransactionHash; +use crate::chain_index::finalised_state::entry::StoredEntryVar; use crate::error::FinalisedStateError; -use crate::{BlockMetadata, BlockWithMetadata, ChainIndexConfig, Height, IndexedBlock}; +use crate::{ + BlockHeaderData, BlockMetadata, BlockWithMetadata, ChainIndexConfig, Height, IndexedBlock, + ZainoVersionedSerde as _, +}; use crate::{AddrScript, Outpoint}; @@ -245,24 +248,36 @@ async fn load_db_backend_from_file() { let finalized_state_backend: FinalisedSource = FinalisedSource::spawn_v1(&config).await.unwrap(); + // Read block headers directly from the `headers` table rather than via `get_chain_block`, which + // reconstructs the full block and validates (reading the v1.3.0 commitment table). This fixture + // is a legacy database whose commitment rows are in `commitment_tree_data_1_0_0`, so validation + // would fail; the header context asserted here is unaffected. + let read_header_direct = |height: Height| -> Option { + use lmdb::Transaction as _; + let environment = finalized_state_backend.env().unwrap(); + let headers_database = environment.open_db(Some("headers_1_0_0")).unwrap(); + let transaction = environment.begin_ro_txn().unwrap(); + match transaction.get(headers_database, &height.to_bytes().unwrap()) { + Ok(raw) => Some( + *StoredEntryVar::::from_bytes(raw) + .unwrap() + .inner(), + ), + Err(lmdb::Error::NotFound) => None, + Err(error) => panic!("failed to read header at height {}: {error}", height.0), + } + }; + let mut prev_hash = None; for height in 0..=100 { - let block = finalized_state_backend - .get_chain_block(Height(height)) - .await - .unwrap() - .unwrap(); + let header = read_header_direct(Height(height)).unwrap(); if let Some(prev_hash) = prev_hash { - assert_eq!(prev_hash, block.context.parent_hash); + assert_eq!(prev_hash, header.context.parent_hash); } - prev_hash = Some(block.context.index.hash); - assert_eq!(block.context.index.height, Height(height)); + prev_hash = Some(header.context.index.hash); + assert_eq!(header.context.index.height, Height(height)); } - assert!(finalized_state_backend - .get_chain_block(Height(101)) - .await - .unwrap() - .is_none()); + assert!(read_header_direct(Height(101)).is_none()); std::fs::remove_file(db_path.join("regtest").join("v1").join("lock.mdb")).unwrap() } diff --git a/packages/zaino-state/src/chain_index/types/helpers.rs b/packages/zaino-state/src/chain_index/types/helpers.rs index b89b6cf72..7bb18dad8 100644 --- a/packages/zaino-state/src/chain_index/types/helpers.rs +++ b/packages/zaino-state/src/chain_index/types/helpers.rs @@ -126,6 +126,7 @@ pub struct BlockMetadata { impl BlockMetadata { /// Create new block metadata + #[allow(clippy::too_many_arguments)] pub fn new( sapling_root: zebra_chain::sapling::tree::Root, sapling_size: u32, From 65cc2b9a781ce494544ecf29dea8fa85818fa170 Mon Sep 17 00:00:00 2001 From: idky137 Date: Sat, 4 Jul 2026 11:05:26 +0100 Subject: [PATCH 034/134] reconcile ironwood rebase onto dev Resolve the rebase of the ironwood/nu6.3 work onto origin/dev: - Re-pin zcash_local_net to the zingolabs v0.7.0 tag. - Drop zingo_common_components in favour of dev's zcash_local_net activation-height boundary functions; thread nu6_3 through to/from_local_net_activation_heights. - Update the e2e test_vectors call for the widened 8-tuple IndexedBlock::TryFrom (adds the ironwood root + size). Co-Authored-By: Claude Opus 4.8 --- Cargo.lock | 91 +++------------------ Cargo.toml | 6 +- live-tests/e2e/tests/test_vectors.rs | 4 + live-tests/zaino-testutils/src/lib.rs | 2 + packages/zaino-common/src/config/network.rs | 19 ----- 5 files changed, 17 insertions(+), 105 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fd6b01cdf..14b12576e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -343,20 +343,6 @@ dependencies = [ "syn 2.0.118", ] -[[package]] -name = "bip0039" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "568b6890865156d9043af490d4c4081c385dd68ea10acd6ca15733d511e6b51c" -dependencies = [ - "hmac 0.12.1", - "pbkdf2", - "rand 0.8.6", - "sha2 0.10.9", - "unicode-normalization", - "zeroize", -] - [[package]] name = "bip32" version = "0.6.0-pre.1" @@ -364,7 +350,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "143f5327f23168716be068f8e1014ba2ea16a6c91e8777bc8927da7b51e1df1f" dependencies = [ "bs58", - "hmac 0.13.0-pre.4", + "hmac", "rand_core 0.6.4", "ripemd 0.2.0-pre.4", "secp256k1", @@ -1859,15 +1845,6 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6fe2267d4ed49bc07b63801559be28c718ea06c4738b7a03c94df7386d2cde46" -[[package]] -name = "hmac" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" -dependencies = [ - "digest 0.10.7", -] - [[package]] name = "hmac" version = "0.13.0-pre.4" @@ -2367,12 +2344,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "json" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "078e285eafdfb6c4b434e0d31e8cfcb5115b651496faca5749b88fafd4f23bfd" - [[package]] name = "jsonrpsee" version = "0.24.11" @@ -3091,17 +3062,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "password-hash" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "346f04948ba92c43e8469c1ee6736c7563d71012b17d40745260fe106aac2166" -dependencies = [ - "base64ct", - "rand_core 0.6.4", - "subtle", -] - [[package]] name = "pasta_curves" version = "0.5.1" @@ -3123,16 +3083,6 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" -[[package]] -name = "pbkdf2" -version = "0.12.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" -dependencies = [ - "digest 0.10.7", - "password-hash", -] - [[package]] name = "pem-rfc7468" version = "0.7.0" @@ -5286,15 +5236,6 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" -[[package]] -name = "unicode-normalization" -version = "0.1.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" -dependencies = [ - "tinyvec", -] - [[package]] name = "unicode-segmentation" version = "1.13.3" @@ -6010,7 +5951,6 @@ dependencies = [ "tracing-subscriber", "tracing-tree", "zebra-chain", - "zingo_common_components", ] [[package]] @@ -6242,23 +6182,19 @@ dependencies = [ [[package]] name = "zcash_local_net" -version = "0.6.0" -source = "git+https://github.com/idky137/infrastructure.git?branch=feat%2Fironwood%2Fzaino#bd3e20ee868aea6483c3904e2ef3715469f6ae70" +version = "0.7.0" +source = "git+https://github.com/zingolabs/infrastructure.git?tag=zcash_local_net_v0.7.0#0b38b6ecc9f5fb83754c6a14b36f575949d58929" dependencies = [ - "getset", "hex", - "json", "reqwest 0.12.28", + "serde", "serde_json", + "sha2 0.10.9", "tempfile", "thiserror 1.0.69", "tokio", "tracing", - "zcash_protocol", - "zebra-chain", - "zebra-node-services", - "zebra-rpc", - "zingo_common_components", + "zingo-consensus", "zingo_test_vectors", ] @@ -6796,21 +6732,14 @@ dependencies = [ ] [[package]] -name = "zingo_common_components" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29f41ddb12ba9aa1fdbf4025fb54639f1fddd25c6e297574922b39fabfb730d4" -dependencies = [ - "hex", -] +name = "zingo-consensus" +version = "0.1.0" +source = "git+https://github.com/zingolabs/infrastructure.git?tag=zcash_local_net_v0.7.0#0b38b6ecc9f5fb83754c6a14b36f575949d58929" [[package]] name = "zingo_test_vectors" version = "0.0.1" -source = "git+https://github.com/idky137/infrastructure.git?branch=feat%2Fironwood%2Fzaino#bd3e20ee868aea6483c3904e2ef3715469f6ae70" -dependencies = [ - "bip0039", -] +source = "git+https://github.com/zingolabs/infrastructure.git?tag=zcash_local_net_v0.7.0#0b38b6ecc9f5fb83754c6a14b36f575949d58929" [[package]] name = "zip32" diff --git a/Cargo.toml b/Cargo.toml index d71abe1bf..8a06b0abe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,10 +38,6 @@ license = "Apache-2.0" [workspace.dependencies] -# Zingo - -zingo_common_components = "0.4" - # Librustzcash incrementalmerkletree = "0.8" zcash_address = "0.13.0-pre.0" @@ -143,7 +139,7 @@ zaino-testutils = { path = "live-tests/zaino-testutils" } # TEMPORARY (zingolabs/infrastructure#269): pinned to an exact commit on the # `add_client_support` branch so every machine resolves the same code; restore # the release tag once tagged upstream. -zcash_local_net = { git = "https://github.com/idky137/infrastructure.git", branch = "feat/ironwood/zaino" } +zcash_local_net = { git = "https://github.com/zingolabs/infrastructure.git", tag = "zcash_local_net_v0.7.0" } wire_serialized_transaction_test_data = "1.0.0" config = { version = "0.15", default-features = false, features = ["toml"] } diff --git a/live-tests/e2e/tests/test_vectors.rs b/live-tests/e2e/tests/test_vectors.rs index fc5b56ef3..aeee240f4 100644 --- a/live-tests/e2e/tests/test_vectors.rs +++ b/live-tests/e2e/tests/test_vectors.rs @@ -233,6 +233,7 @@ async fn create_200_block_regtest_chain_vectors() { let mut parent_chain_work: Option = None; let mut parent_block_sapling_tree_size: u32 = 0; let mut parent_block_orchard_tree_size: u32 = 0; + let mut parent_block_ironwood_tree_size: u32 = 0; for height in 0..=chain_height.0 { let (chain_block, zebra_block, block_roots, block_treestate) = { @@ -376,8 +377,10 @@ async fn create_200_block_regtest_chain_vectors() { parent_chain_work, sapling_root.0.into(), orchard_root.0.into(), + None, parent_block_sapling_tree_size, parent_block_orchard_tree_size, + parent_block_ironwood_tree_size, )) .unwrap(); @@ -399,6 +402,7 @@ async fn create_200_block_regtest_chain_vectors() { // Update parent block parent_block_sapling_tree_size = chain_block.commitment_tree_data().sizes().sapling(); parent_block_orchard_tree_size = chain_block.commitment_tree_data().sizes().orchard(); + parent_block_ironwood_tree_size = chain_block.commitment_tree_data().sizes().ironwood(); parent_chain_work = Some(*chain_block.chainwork()); data.push((height, zebra_block, block_roots, block_treestate)); diff --git a/live-tests/zaino-testutils/src/lib.rs b/live-tests/zaino-testutils/src/lib.rs index 8133c87e4..d4f5797e6 100644 --- a/live-tests/zaino-testutils/src/lib.rs +++ b/live-tests/zaino-testutils/src/lib.rs @@ -347,6 +347,7 @@ pub fn to_local_net_activation_heights( .set_nu6(activation_heights.nu6) .set_nu6_1(activation_heights.nu6_1) .set_nu6_2(activation_heights.nu6_2) + .set_nu6_3(activation_heights.nu6_3) .set_nu7(activation_heights.nu7) .build() } @@ -366,6 +367,7 @@ pub fn from_local_net_activation_heights( nu6: activation_heights.nu6(), nu6_1: activation_heights.nu6_1(), nu6_2: activation_heights.nu6_2(), + nu6_3: activation_heights.nu6_3(), nu7: activation_heights.nu7(), } } diff --git a/packages/zaino-common/src/config/network.rs b/packages/zaino-common/src/config/network.rs index 5ddd78f49..5f306ee32 100644 --- a/packages/zaino-common/src/config/network.rs +++ b/packages/zaino-common/src/config/network.rs @@ -203,25 +203,6 @@ impl From for ConfiguredActivationHeights { } } -impl From for ActivationHeights { - fn from(activation_heights: zingo_common_components::protocol::ActivationHeights) -> Self { - ActivationHeights { - before_overwinter: activation_heights.overwinter(), - overwinter: activation_heights.overwinter(), - sapling: activation_heights.sapling(), - blossom: activation_heights.blossom(), - heartwood: activation_heights.heartwood(), - canopy: activation_heights.canopy(), - nu5: activation_heights.nu5(), - nu6: activation_heights.nu6(), - nu6_1: activation_heights.nu6_1(), - nu6_2: activation_heights.nu6_2(), - nu6_3: activation_heights.nu6_3(), - nu7: activation_heights.nu7(), - } - } -} - impl Network { /// Convert to Zebra's network type for internal use (alias for to_zebra_default). pub fn to_zebra_network(&self) -> zebra_chain::parameters::Network { From 0e057e22b75a21d5cd093f20feb1ece3e23e94a1 Mon Sep 17 00:00:00 2001 From: idky137 Date: Sat, 4 Jul 2026 11:36:55 +0100 Subject: [PATCH 035/134] fixed ironwood treestates and cargo fmt --- packages/zaino-state/src/backends/fetch.rs | 3 +- packages/zaino-state/src/backends/state.rs | 32 ++++++------------- .../chain_index/source/validator_connector.rs | 18 +++++++++-- 3 files changed, 25 insertions(+), 28 deletions(-) diff --git a/packages/zaino-state/src/backends/fetch.rs b/packages/zaino-state/src/backends/fetch.rs index 1a2400396..dfac59729 100644 --- a/packages/zaino-state/src/backends/fetch.rs +++ b/packages/zaino-state/src/backends/fetch.rs @@ -296,8 +296,7 @@ impl ZcashIndexer for FetchServiceSubscriber { /// Some fields from the zcashd reference are missing from Zebra's [`GetBlockchainInfoResponse`]. It only contains the fields /// [required for lightwalletd support.](https://github.com/zcash/lightwalletd/blob/v0.4.9/common/common.go#L72-L89) async fn get_blockchain_info(&self) -> Result { - self - .fetcher + self.fetcher .get_blockchain_info() .await? .try_into() diff --git a/packages/zaino-state/src/backends/state.rs b/packages/zaino-state/src/backends/state.rs index d794c74f2..da1296edf 100644 --- a/packages/zaino-state/src/backends/state.rs +++ b/packages/zaino-state/src/backends/state.rs @@ -1494,17 +1494,8 @@ impl ZcashIndexer for StateServiceSubscriber { )))?, }; - let treestate: GetTreestateResponse = self - .rpc_client - .get_treestate(block_data.hash().to_string()) - .await? - .try_into() - .map_err(|_error| { - StateServiceError::RpcError(RpcError::new_from_legacycode( - zebra_rpc::server::error::LegacyCode::InvalidParameter, - "Failed to parse treestate.", - )) - })?; + let (sapling, orchard, ironwood) = + self.indexer.get_treestate(block_data.hash()).await?; let time: u32 = block_data.data().time().try_into().map_err(|_error| { StateServiceError::RpcError(RpcError::new_from_legacycode( zebra_rpc::server::error::LegacyCode::InvalidParameter, @@ -1512,30 +1503,25 @@ impl ZcashIndexer for StateServiceSubscriber { )) })?; - let sapling = treestate.sapling().commitments().final_state().clone(); - let orchard = treestate.orchard().commitments().final_state().clone(); - let ironwood = treestate - .ironwood() - .clone() - .and_then(|ironwood| ironwood.commitments().final_state().clone()); - + // `Commitments::new(final_root, final_state)`: the note-commitment tree is the + // `final_state`, matching the (now-deprecated) `from_parts` behaviour this replaces. + // The `new` constructor is used so the Ironwood (NU6.3) treestate can be included rather + // than discarded. Ok(GetTreestateResponse::new( (*block_data.hash()).into(), block_data.height().into(), time, None, zebra_rpc::client::Treestate::new(zebra_rpc::client::Commitments::new( - sapling.as_deref().map(ToOwned::to_owned), - None, + None, sapling, )), zebra_rpc::client::Treestate::new(zebra_rpc::client::Commitments::new( - orchard.as_deref().map(ToOwned::to_owned), - None, + None, orchard, )), ironwood.map(|final_state| { zebra_rpc::client::Treestate::new(zebra_rpc::client::Commitments::new( - Some(final_state), None, + Some(final_state), )) }), )) diff --git a/packages/zaino-state/src/chain_index/source/validator_connector.rs b/packages/zaino-state/src/chain_index/source/validator_connector.rs index 038908279..6dc34d25e 100644 --- a/packages/zaino-state/src/chain_index/source/validator_connector.rs +++ b/packages/zaino-state/src/chain_index/source/validator_connector.rs @@ -463,7 +463,7 @@ impl BlockchainSource for ValidatorConnector { .map_err(|_e| { BlockchainSourceError::Unrecoverable( InvalidData(format!( - "could not fetch orchard treestate of block {id}" + "could not fetch ironwood treestate of block {id}" )) .to_string(), ) @@ -471,9 +471,21 @@ impl BlockchainSource for ValidatorConnector { ), _ => None, } - .and_then(|orch_response| { - expected_read_response!(orch_response, OrchardTree) + .and_then(|irw_response| { + expected_read_response!(irw_response, IronwoodTree) .map(|tree| tree.to_rpc_bytes()) + }) + // Mirror the Fetch connector: when the ironwood tree is absent (e.g. before + // NU6.3 activation) return the serialized empty tree rather than `None`, so both + // backends agree on the empty-tree encoding. + .or_else(|| { + let mut tree = vec![]; + write_commitment_tree( + &CommitmentTree::::empty(), + &mut tree, + ) + .expect("can write to Vec"); + Some(tree) }); Ok((sapling, orchard, ironwood)) From 060ee6e2eb25845dffa41a4a7086834d2bb217cf Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 13:11:12 -0700 Subject: [PATCH 036/134] refactor(backends): name the treestate components in z_get_treestate Bind sprout/sapling/orchard/ironwood treestates to named variables before constructing GetTreestateResponse, in both the Fetch and State backends. The bare `None` in the fourth positional argument gave no hint it was the sprout slot, and the nested Treestate::new(Commitments::new(..)) calls obscured which pool each argument carried. Addresses review comments: https://github.com/zingolabs/zaino/pull/1362#discussion_r3523584530 https://github.com/zingolabs/zaino/pull/1362#discussion_r3523585272 Co-Authored-By: Claude Fable 5 --- packages/zaino-state/src/backends/fetch.rs | 31 +++++++++++++--------- packages/zaino-state/src/backends/state.rs | 31 +++++++++++++--------- 2 files changed, 36 insertions(+), 26 deletions(-) diff --git a/packages/zaino-state/src/backends/fetch.rs b/packages/zaino-state/src/backends/fetch.rs index dfac59729..f372b912b 100644 --- a/packages/zaino-state/src/backends/fetch.rs +++ b/packages/zaino-state/src/backends/fetch.rs @@ -619,23 +619,28 @@ impl ZcashIndexer for FetchServiceSubscriber { // `final_state`, matching the (now-deprecated) `from_parts` behaviour this replaces. // The `new` constructor is used so the Ironwood (NU6.3) treestate can be included rather // than discarded. + let sprout_treestate = None; + let sapling_treestate = zebra_rpc::client::Treestate::new( + zebra_rpc::client::Commitments::new(None, sapling), + ); + let orchard_treestate = zebra_rpc::client::Treestate::new( + zebra_rpc::client::Commitments::new(None, orchard), + ); + let ironwood_treestate = ironwood.map(|final_state| { + zebra_rpc::client::Treestate::new(zebra_rpc::client::Commitments::new( + None, + Some(final_state), + )) + }); + Ok(GetTreestateResponse::new( (*block_data.hash()).into(), block_data.height().into(), time, - None, - zebra_rpc::client::Treestate::new(zebra_rpc::client::Commitments::new( - None, sapling, - )), - zebra_rpc::client::Treestate::new(zebra_rpc::client::Commitments::new( - None, orchard, - )), - ironwood.map(|final_state| { - zebra_rpc::client::Treestate::new(zebra_rpc::client::Commitments::new( - None, - Some(final_state), - )) - }), + sprout_treestate, + sapling_treestate, + orchard_treestate, + ironwood_treestate, )) } .await; diff --git a/packages/zaino-state/src/backends/state.rs b/packages/zaino-state/src/backends/state.rs index da1296edf..3d442a80b 100644 --- a/packages/zaino-state/src/backends/state.rs +++ b/packages/zaino-state/src/backends/state.rs @@ -1507,23 +1507,28 @@ impl ZcashIndexer for StateServiceSubscriber { // `final_state`, matching the (now-deprecated) `from_parts` behaviour this replaces. // The `new` constructor is used so the Ironwood (NU6.3) treestate can be included rather // than discarded. + let sprout_treestate = None; + let sapling_treestate = zebra_rpc::client::Treestate::new( + zebra_rpc::client::Commitments::new(None, sapling), + ); + let orchard_treestate = zebra_rpc::client::Treestate::new( + zebra_rpc::client::Commitments::new(None, orchard), + ); + let ironwood_treestate = ironwood.map(|final_state| { + zebra_rpc::client::Treestate::new(zebra_rpc::client::Commitments::new( + None, + Some(final_state), + )) + }); + Ok(GetTreestateResponse::new( (*block_data.hash()).into(), block_data.height().into(), time, - None, - zebra_rpc::client::Treestate::new(zebra_rpc::client::Commitments::new( - None, sapling, - )), - zebra_rpc::client::Treestate::new(zebra_rpc::client::Commitments::new( - None, orchard, - )), - ironwood.map(|final_state| { - zebra_rpc::client::Treestate::new(zebra_rpc::client::Commitments::new( - None, - Some(final_state), - )) - }), + sprout_treestate, + sapling_treestate, + orchard_treestate, + ironwood_treestate, )) } .await; From 5fae0ae1bc389b84fc1cacea35eb6db3ca0cc2c3 Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 13:26:33 -0700 Subject: [PATCH 037/134] refactor(chain-index): DRY pool-root gating and retire unwrap families Addresses two findings from the PR #1362 code review: 1. Production `.unwrap()` calls, added as verbatim clones of pre-existing sibling debt: - helpers.rs carried the `[..52].try_into().unwrap()` ciphertext truncation three times (sapling/orchard/ironwood). A new `compact_ciphertext_prefix` helper builds the 52-byte prefix infallibly via `std::array::from_fn`, and a shared `extract_orchard_shaped_data` collapses the orchard/ironwood extraction twins to two-line delegations. - ephemeral.rs carried `Height::try_from(..).unwrap()` at eleven sites. A `height_from_u32` helper propagates the out-of-range case as `FinalisedStateError` instead of panicking. 2. The "pool root is required iff the pool's network upgrade is active" rule was re-implemented at three sites (build_indexed_block_from_source, the v1.0.0 fixture sync loop, and the ephemeral backend) and had already drifted: the ephemeral ironwood arm reported "missing Ironwood commitment tree root for active NU5 block". One `required_pool_root` helper now owns the gating, the missing-root error, and the size-to-u32 conversion (checked, where the fixture/builder sites previously truncated with `as u32`); pool names come from `ShieldedPool::pool_string`. The v1.0.0 fixture loop keeps its contract of unconditionally requiring sapling/orchard roots by passing `is_active: true`. Co-Authored-By: Claude Fable 5 --- .../src/chain_index/finalised_state.rs | 131 ++++++++++-------- .../finalised_source/ephemeral.rs | 113 +++++---------- .../src/chain_index/types/helpers.rs | 94 +++++-------- 3 files changed, 146 insertions(+), 192 deletions(-) diff --git a/packages/zaino-state/src/chain_index/finalised_state.rs b/packages/zaino-state/src/chain_index/finalised_state.rs index 0a34fc7ab..8a495e852 100644 --- a/packages/zaino-state/src/chain_index/finalised_state.rs +++ b/packages/zaino-state/src/chain_index/finalised_state.rs @@ -286,43 +286,32 @@ pub(crate) async fn build_indexed_block_from_source( let is_ironwood_active = nu6_3_activation_height .is_some_and(|nu6_3_activation_height| height_int >= nu6_3_activation_height.0); - let (sapling_root, sapling_size) = if is_sapling_active { - sapling_opt.ok_or_else(|| { - FinalisedStateError::BlockchainSourceError(BlockchainSourceError::Unrecoverable( - format!("missing Sapling commitment tree root for block {block_hash}"), - )) - })? - } else { - (zebra_chain::sapling::tree::Root::default(), 0) - }; - - let (orchard_root, orchard_size) = if is_orchard_active { - orchard_opt.ok_or_else(|| { - FinalisedStateError::BlockchainSourceError(BlockchainSourceError::Unrecoverable( - format!("missing Orchard commitment tree root for block {block_hash}"), - )) - })? - } else { - (zebra_chain::orchard::tree::Root::default(), 0) - }; - - let (ironwood_root, ironwood_size) = if is_ironwood_active { - ironwood_opt.ok_or_else(|| { - FinalisedStateError::BlockchainSourceError(BlockchainSourceError::Unrecoverable( - format!("missing Ironwood commitment tree root for block {block_hash}"), - )) - })? - } else { - (zebra_chain::orchard::tree::Root::default(), 0) - }; + let (sapling_root, sapling_size) = required_pool_root( + super::ShieldedPool::Sapling, + is_sapling_active, + sapling_opt, + || format!("block {block_hash}"), + )?; + let (orchard_root, orchard_size) = required_pool_root( + super::ShieldedPool::Orchard, + is_orchard_active, + orchard_opt, + || format!("block {block_hash}"), + )?; + let (ironwood_root, ironwood_size) = required_pool_root( + super::ShieldedPool::Ironwood, + is_ironwood_active, + ironwood_opt, + || format!("block {block_hash}"), + )?; let metadata = BlockMetadata::new( sapling_root, - sapling_size as u32, + sapling_size, orchard_root, - orchard_size as u32, + orchard_size, ironwood_root, - ironwood_size as u32, + ironwood_size, parent_chainwork, network.to_zebra_network(), ); @@ -335,6 +324,41 @@ pub(crate) async fn build_indexed_block_from_source( }) } +/// Resolves a pool's optional commitment-tree root and size against the pool's activation +/// status at the block in question. +/// +/// From activation onward the root is required: a missing root is an unrecoverable source +/// error. Below activation (or on a network where the pool's upgrade has no activation +/// height) the pool has no tree yet, so the root defaults and the size is zero. +/// +/// `location` describes the block for the error message (e.g. `block ` or +/// `block at height `) and is only evaluated on failure. +fn required_pool_root( + pool: super::ShieldedPool, + is_active: bool, + root_and_size: Option<(R, u64)>, + location: impl FnOnce() -> String, +) -> Result<(R, u32), FinalisedStateError> { + let pool = pool.pool_string(); + match root_and_size { + Some((root, size)) => { + let size = u32::try_from(size).map_err(|error| { + FinalisedStateError::BlockchainSourceError(BlockchainSourceError::Unrecoverable( + format!("{pool} commitment tree size does not fit into u32: {error}"), + )) + })?; + Ok((root, size)) + } + None if !is_active => Ok((R::default(), 0)), + None => Err(FinalisedStateError::BlockchainSourceError( + BlockchainSourceError::Unrecoverable(format!( + "missing {pool} commitment tree root for {}", + location() + )), + )), + } +} + use super::source::BlockchainSource; /// A sync wider than this many blocks runs in the background (with ephemeral @@ -1081,37 +1105,32 @@ impl FinalisedState { let block_hash = BlockHash::from(block.hash().0); let (sapling_opt, orchard_opt, ironwood_opt) = source.get_commitment_tree_roots(block_hash).await?; - let (sapling_root, sapling_size) = sapling_opt.ok_or_else(|| { - FinalisedStateError::BlockchainSourceError(BlockchainSourceError::Unrecoverable( - format!("missing Sapling commitment tree root for block {block_hash}"), - )) - })?; - let (orchard_root, orchard_size) = orchard_opt.ok_or_else(|| { - FinalisedStateError::BlockchainSourceError(BlockchainSourceError::Unrecoverable( - format!("missing Orchard commitment tree root for block {block_hash}"), - )) - })?; + // Per this builder's contract, the fixture source provides Sapling and Orchard + // roots for every block, so those pools are unconditionally required. + let (sapling_root, sapling_size) = + required_pool_root(super::ShieldedPool::Sapling, true, sapling_opt, || { + format!("block {block_hash}") + })?; + let (orchard_root, orchard_size) = + required_pool_root(super::ShieldedPool::Orchard, true, orchard_opt, || { + format!("block {block_hash}") + })?; let is_ironwood_active = nu6_3_activation_height.is_some_and(|activation| height >= activation.0); - let (ironwood_root, ironwood_size) = if is_ironwood_active { - ironwood_opt.ok_or_else(|| { - FinalisedStateError::BlockchainSourceError( - BlockchainSourceError::Unrecoverable(format!( - "missing Ironwood commitment tree root for block {block_hash}" - )), - ) - })? - } else { - (zebra_chain::orchard::tree::Root::default(), 0) - }; + let (ironwood_root, ironwood_size) = required_pool_root( + super::ShieldedPool::Ironwood, + is_ironwood_active, + ironwood_opt, + || format!("block {block_hash}"), + )?; let metadata = BlockMetadata::new( sapling_root, - sapling_size as u32, + sapling_size, orchard_root, - orchard_size as u32, + orchard_size, ironwood_root, - ironwood_size as u32, + ironwood_size, parent_chainwork, cfg.network.to_zebra_network(), ); diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/ephemeral.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/ephemeral.rs index af1b391bd..11067c90b 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/ephemeral.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/ephemeral.rs @@ -13,6 +13,9 @@ use zebra_state::HashOrHeight; use crate::chain_index::finalised_state::capability::{DbCore, DbWrite}; use crate::chain_index::finalised_state::DbMetadata; +use crate::chain_index::ShieldedPool; + +use super::super::required_pool_root; use crate::chain_index::source::BlockchainSourceError; use crate::chain_index::{ finalised_state::capability::{ @@ -33,6 +36,15 @@ use zaino_proto::proto::utils::{compact_block_with_pool_types, PoolTypeFilter}; const EPHEMERAL_FINALISED_STATE_STATUS_POLL_INTERVAL: Duration = Duration::from_secs(5); +/// Converts a raw `u32` block height (a stored [`TxLocation`] height or a walk between two +/// already-validated [`Height`] bounds) into a [`Height`], surfacing an out-of-range value +/// as a [`FinalisedStateError`] instead of panicking. +fn height_from_u32(height: u32) -> Result { + Height::try_from(height).map_err(|error| { + FinalisedStateError::Custom(format!("invalid block height {height}: {error}")) + }) +} + /// Source-backed finalised-state backend used when persistent finalised-state storage is not /// serving normal requests. /// @@ -260,55 +272,18 @@ impl EphemeralFinalisedState { block_height.into(), ); - let (sapling_root, sapling_size) = match sapling { - Some((root, size)) => (root, size), - None if !sapling_is_active => Default::default(), - None => { - return Err(FinalisedStateError::BlockchainSourceError( - BlockchainSourceError::Unrecoverable(format!( - "missing Sapling commitment tree root for active Sapling block at height {height}" - )), - )); - } - }; - let (orchard_root, orchard_size) = match orchard { - Some((root, size)) => (root, size), - None if !orchard_is_active => Default::default(), - None => { - return Err(FinalisedStateError::BlockchainSourceError( - BlockchainSourceError::Unrecoverable(format!( - "missing Orchard commitment tree root for active NU5 block at height {height}" - )), - )); - } - }; - let (ironwood_root, ironwood_size) = match ironwood { - Some((root, size)) => (root, size), - None if !ironwood_is_active => Default::default(), - None => { - return Err(FinalisedStateError::BlockchainSourceError( - BlockchainSourceError::Unrecoverable(format!( - "missing Ironwood commitment tree root for active NU5 block at height {height}" - )), - )); - } - }; - - let sapling_size = u32::try_from(sapling_size).map_err(|error| { - FinalisedStateError::BlockchainSourceError(BlockchainSourceError::Unrecoverable( - format!("sapling commitment tree size does not fit into u32: {error}"), - )) - })?; - let orchard_size = u32::try_from(orchard_size).map_err(|error| { - FinalisedStateError::BlockchainSourceError(BlockchainSourceError::Unrecoverable( - format!("orchard commitment tree size does not fit into u32: {error}"), - )) - })?; - let ironwood_size = u32::try_from(ironwood_size).map_err(|error| { - FinalisedStateError::BlockchainSourceError(BlockchainSourceError::Unrecoverable( - format!("ironwood commitment tree size does not fit into u32: {error}"), - )) - })?; + let (sapling_root, sapling_size) = + required_pool_root(ShieldedPool::Sapling, sapling_is_active, sapling, || { + format!("block at height {height}") + })?; + let (orchard_root, orchard_size) = + required_pool_root(ShieldedPool::Orchard, orchard_is_active, orchard, || { + format!("block at height {height}") + })?; + let (ironwood_root, ironwood_size) = + required_pool_root(ShieldedPool::Ironwood, ironwood_is_active, ironwood, || { + format!("block at height {height}") + })?; let block_metadata = BlockMetadata::new( sapling_root, @@ -483,10 +458,7 @@ impl BlockCoreExt for EphemeralFinalisedState { let mut headers = Vec::new(); for height in u32::from(start)..=u32::from(end) { - headers.push( - self.get_block_header(Height::try_from(height).unwrap()) - .await?, - ); + headers.push(self.get_block_header(height_from_u32(height)?).await?); } Ok(headers) @@ -512,10 +484,7 @@ impl BlockCoreExt for EphemeralFinalisedState { let mut txid_lists = Vec::new(); for height in u32::from(start)..=u32::from(end) { - txid_lists.push( - self.get_block_txids(Height::try_from(height).unwrap()) - .await?, - ); + txid_lists.push(self.get_block_txids(height_from_u32(height)?).await?); } Ok(txid_lists) @@ -573,7 +542,7 @@ impl BlockTransparentExt for EphemeralFinalisedState { tx_location: TxLocation, ) -> Result, FinalisedStateError> { let chain_block = self - .get_required_chain_block(Height::try_from(tx_location.block_height()).unwrap()) + .get_required_chain_block(height_from_u32(tx_location.block_height())?) .await?; Ok(chain_block @@ -605,10 +574,7 @@ impl BlockTransparentExt for EphemeralFinalisedState { let mut transparent_lists = Vec::new(); for height in u32::from(start)..=u32::from(end) { - transparent_lists.push( - self.get_block_transparent(Height::try_from(height).unwrap()) - .await?, - ); + transparent_lists.push(self.get_block_transparent(height_from_u32(height)?).await?); } Ok(transparent_lists) @@ -660,7 +626,7 @@ impl BlockShieldedExt for EphemeralFinalisedState { tx_location: TxLocation, ) -> Result, FinalisedStateError> { let chain_block = self - .get_required_chain_block(Height::try_from(tx_location.block_height()).unwrap()) + .get_required_chain_block(height_from_u32(tx_location.block_height())?) .await?; Ok(chain_block @@ -692,10 +658,7 @@ impl BlockShieldedExt for EphemeralFinalisedState { let mut sapling_lists = Vec::new(); for height in u32::from(start)..=u32::from(end) { - sapling_lists.push( - self.get_block_sapling(Height::try_from(height).unwrap()) - .await?, - ); + sapling_lists.push(self.get_block_sapling(height_from_u32(height)?).await?); } Ok(sapling_lists) @@ -706,7 +669,7 @@ impl BlockShieldedExt for EphemeralFinalisedState { tx_location: TxLocation, ) -> Result, FinalisedStateError> { let chain_block = self - .get_required_chain_block(Height::try_from(tx_location.block_height()).unwrap()) + .get_required_chain_block(height_from_u32(tx_location.block_height())?) .await?; Ok(chain_block @@ -738,10 +701,7 @@ impl BlockShieldedExt for EphemeralFinalisedState { let mut orchard_lists = Vec::new(); for height in u32::from(start)..=u32::from(end) { - orchard_lists.push( - self.get_block_orchard(Height::try_from(height).unwrap()) - .await?, - ); + orchard_lists.push(self.get_block_orchard(height_from_u32(height)?).await?); } Ok(orchard_lists) @@ -752,7 +712,7 @@ impl BlockShieldedExt for EphemeralFinalisedState { tx_location: TxLocation, ) -> Result, FinalisedStateError> { let chain_block = self - .get_required_chain_block(Height::try_from(tx_location.block_height()).unwrap()) + .get_required_chain_block(height_from_u32(tx_location.block_height())?) .await?; Ok(chain_block @@ -784,10 +744,7 @@ impl BlockShieldedExt for EphemeralFinalisedState { let mut ironwood_lists = Vec::new(); for height in u32::from(start)..=u32::from(end) { - ironwood_lists.push( - self.get_block_ironwood(Height::try_from(height).unwrap()) - .await?, - ); + ironwood_lists.push(self.get_block_ironwood(height_from_u32(height)?).await?); } Ok(ironwood_lists) @@ -810,7 +767,7 @@ impl BlockShieldedExt for EphemeralFinalisedState { for height in u32::from(start)..=u32::from(end) { commitment_tree_data.push( - self.get_block_commitment_tree_data(Height::try_from(height).unwrap()) + self.get_block_commitment_tree_data(height_from_u32(height)?) .await?, ); } diff --git a/packages/zaino-state/src/chain_index/types/helpers.rs b/packages/zaino-state/src/chain_index/types/helpers.rs index 7bb18dad8..e2c8960cb 100644 --- a/packages/zaino-state/src/chain_index/types/helpers.rs +++ b/packages/zaino-state/src/chain_index/types/helpers.rs @@ -250,18 +250,41 @@ impl<'a> BlockWithMetadata<'a> { Ok(TransparentCompactTx::new(inputs, outputs)) } + /// Returns the first 52 bytes of a 580-byte encrypted note ciphertext: the prefix a + /// compact block carries, sufficient for trial decryption. + fn compact_ciphertext_prefix(enc_ciphertext: [u8; 580]) -> [u8; 52] { + std::array::from_fn(|i| enc_ciphertext[i]) + } + + /// Builds the Orchard-shaped compact transaction shared by the Orchard and Ironwood + /// pools from a value balance and the pool's actions. + fn extract_orchard_shaped_data<'t>( + value_balance: i64, + actions: impl Iterator, + ) -> OrchardCompactTx { + OrchardCompactTx::new( + (value_balance != 0).then_some(value_balance), + actions + .map(|action| { + CompactOrchardAction::new( + <[u8; 32]>::from(action.nullifier), + <[u8; 32]>::from(action.cm_x), + <[u8; 32]>::from(action.ephemeral_key), + Self::compact_ciphertext_prefix(<[u8; 580]>::from(action.enc_ciphertext)), + ) + }) + .collect(), + ) + } + /// Extract sapling transaction data fn extract_sapling_data( &self, txn: &zebra_chain::transaction::Transaction, ) -> SaplingCompactTx { let sapling_value = { - let val = txn.sapling_value_balance().sapling_amount(); - if val == 0 { - None - } else { - Some(i64::from(val)) - } + let value = i64::from(txn.sapling_value_balance().sapling_amount()); + (value != 0).then_some(value) }; SaplingCompactTx::new( @@ -271,13 +294,10 @@ impl<'a> BlockWithMetadata<'a> { .collect(), txn.sapling_outputs() .map(|output| { - let cipher: [u8; 52] = <[u8; 580]>::from(output.enc_ciphertext)[..52] - .try_into() - .unwrap(); // TODO: Remove unwrap CompactSaplingOutput::new( output.cm_u.to_bytes(), <[u8; 32]>::from(output.ephemeral_key), - cipher, + Self::compact_ciphertext_prefix(<[u8; 580]>::from(output.enc_ciphertext)), ) }) .collect::>(), @@ -289,30 +309,9 @@ impl<'a> BlockWithMetadata<'a> { &self, txn: &zebra_chain::transaction::Transaction, ) -> OrchardCompactTx { - let orchard_value = { - let val = txn.orchard_value_balance().orchard_amount(); - if val == 0 { - None - } else { - Some(i64::from(val)) - } - }; - - OrchardCompactTx::new( - orchard_value, - txn.orchard_actions() - .map(|action| { - let cipher: [u8; 52] = <[u8; 580]>::from(action.enc_ciphertext)[..52] - .try_into() - .unwrap(); // TODO: Remove unwrap - CompactOrchardAction::new( - <[u8; 32]>::from(action.nullifier), - <[u8; 32]>::from(action.cm_x), - <[u8; 32]>::from(action.ephemeral_key), - cipher, - ) - }) - .collect::>(), + Self::extract_orchard_shaped_data( + i64::from(txn.orchard_value_balance().orchard_amount()), + txn.orchard_actions(), ) } @@ -321,30 +320,9 @@ impl<'a> BlockWithMetadata<'a> { &self, txn: &zebra_chain::transaction::Transaction, ) -> OrchardCompactTx { - let ironwood_value = { - let val = txn.ironwood_value_balance().ironwood_amount(); - if val == 0 { - None - } else { - Some(i64::from(val)) - } - }; - - OrchardCompactTx::new( - ironwood_value, - txn.ironwood_actions() - .map(|action| { - let cipher: [u8; 52] = <[u8; 580]>::from(action.enc_ciphertext)[..52] - .try_into() - .unwrap(); // TODO: Remove unwrap - CompactOrchardAction::new( - <[u8; 32]>::from(action.nullifier), - <[u8; 32]>::from(action.cm_x), - <[u8; 32]>::from(action.ephemeral_key), - cipher, - ) - }) - .collect::>(), + Self::extract_orchard_shaped_data( + i64::from(txn.ironwood_value_balance().ironwood_amount()), + txn.ironwood_actions(), ) } From 2fbdcb8e1ce5c38416fc25faa1bee87ff9cb806c Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 13:34:12 -0700 Subject: [PATCH 038/134] refactor(chain-index): walk block ranges with a Height-typed iterator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add Height::range_inclusive(start, end): both bounds are already-valid heights, so every intermediate value is valid by construction. The seven get_block_range_* loops in the ephemeral backend previously round-tripped each step through u32 and re-validated it (formerly via `Height::try_from(..).unwrap()`, then via the fallible height_from_u32) — propagating an error that is statically impossible. They now iterate Heights directly; height_from_u32 remains only for TxLocation heights, which genuinely originate from stored data. Follow-up scheduled after the remaining PR #1362 review DRYs: fold the enclosing per-range collect loops (iterate + per-height getter + push) into one generic helper, since that shape now repeats across the same seven methods. Co-Authored-By: Claude Fable 5 --- .../finalised_source/ephemeral.rs | 36 +++++++++---------- .../src/chain_index/types/db/legacy.rs | 10 ++++++ 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/ephemeral.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/ephemeral.rs index 11067c90b..c5a4f6ea1 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/ephemeral.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/ephemeral.rs @@ -36,9 +36,8 @@ use zaino_proto::proto::utils::{compact_block_with_pool_types, PoolTypeFilter}; const EPHEMERAL_FINALISED_STATE_STATUS_POLL_INTERVAL: Duration = Duration::from_secs(5); -/// Converts a raw `u32` block height (a stored [`TxLocation`] height or a walk between two -/// already-validated [`Height`] bounds) into a [`Height`], surfacing an out-of-range value -/// as a [`FinalisedStateError`] instead of panicking. +/// Converts a raw `u32` block height (a stored [`TxLocation`] height) into a [`Height`], +/// surfacing an out-of-range value as a [`FinalisedStateError`] instead of panicking. fn height_from_u32(height: u32) -> Result { Height::try_from(height).map_err(|error| { FinalisedStateError::Custom(format!("invalid block height {height}: {error}")) @@ -457,8 +456,8 @@ impl BlockCoreExt for EphemeralFinalisedState { ) -> Result, FinalisedStateError> { let mut headers = Vec::new(); - for height in u32::from(start)..=u32::from(end) { - headers.push(self.get_block_header(height_from_u32(height)?).await?); + for height in Height::range_inclusive(start, end) { + headers.push(self.get_block_header(height).await?); } Ok(headers) @@ -483,8 +482,8 @@ impl BlockCoreExt for EphemeralFinalisedState { ) -> Result, FinalisedStateError> { let mut txid_lists = Vec::new(); - for height in u32::from(start)..=u32::from(end) { - txid_lists.push(self.get_block_txids(height_from_u32(height)?).await?); + for height in Height::range_inclusive(start, end) { + txid_lists.push(self.get_block_txids(height).await?); } Ok(txid_lists) @@ -573,8 +572,8 @@ impl BlockTransparentExt for EphemeralFinalisedState { ) -> Result, FinalisedStateError> { let mut transparent_lists = Vec::new(); - for height in u32::from(start)..=u32::from(end) { - transparent_lists.push(self.get_block_transparent(height_from_u32(height)?).await?); + for height in Height::range_inclusive(start, end) { + transparent_lists.push(self.get_block_transparent(height).await?); } Ok(transparent_lists) @@ -657,8 +656,8 @@ impl BlockShieldedExt for EphemeralFinalisedState { ) -> Result, FinalisedStateError> { let mut sapling_lists = Vec::new(); - for height in u32::from(start)..=u32::from(end) { - sapling_lists.push(self.get_block_sapling(height_from_u32(height)?).await?); + for height in Height::range_inclusive(start, end) { + sapling_lists.push(self.get_block_sapling(height).await?); } Ok(sapling_lists) @@ -700,8 +699,8 @@ impl BlockShieldedExt for EphemeralFinalisedState { ) -> Result, FinalisedStateError> { let mut orchard_lists = Vec::new(); - for height in u32::from(start)..=u32::from(end) { - orchard_lists.push(self.get_block_orchard(height_from_u32(height)?).await?); + for height in Height::range_inclusive(start, end) { + orchard_lists.push(self.get_block_orchard(height).await?); } Ok(orchard_lists) @@ -743,8 +742,8 @@ impl BlockShieldedExt for EphemeralFinalisedState { ) -> Result, FinalisedStateError> { let mut ironwood_lists = Vec::new(); - for height in u32::from(start)..=u32::from(end) { - ironwood_lists.push(self.get_block_ironwood(height_from_u32(height)?).await?); + for height in Height::range_inclusive(start, end) { + ironwood_lists.push(self.get_block_ironwood(height).await?); } Ok(ironwood_lists) @@ -765,11 +764,8 @@ impl BlockShieldedExt for EphemeralFinalisedState { ) -> Result, FinalisedStateError> { let mut commitment_tree_data = Vec::new(); - for height in u32::from(start)..=u32::from(end) { - commitment_tree_data.push( - self.get_block_commitment_tree_data(height_from_u32(height)?) - .await?, - ); + for height in Height::range_inclusive(start, end) { + commitment_tree_data.push(self.get_block_commitment_tree_data(height).await?); } Ok(commitment_tree_data) diff --git a/packages/zaino-state/src/chain_index/types/db/legacy.rs b/packages/zaino-state/src/chain_index/types/db/legacy.rs index 8fedd5c31..bcc9a91b1 100644 --- a/packages/zaino-state/src/chain_index/types/db/legacy.rs +++ b/packages/zaino-state/src/chain_index/types/db/legacy.rs @@ -364,6 +364,16 @@ impl FixedEncodedLen for TransactionHash { #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub struct Height(pub(crate) u32); +impl Height { + /// Iterates every height from `start` through `end` inclusive. + /// + /// Both bounds are already-valid heights, so every intermediate value is a valid + /// height by construction — callers walking a range need no per-step validation. + pub(crate) fn range_inclusive(start: Self, end: Self) -> impl Iterator { + (start.0..=end.0).map(Self) + } +} + impl PartialOrd for Height { fn partial_cmp(&self, other: &zebra_chain::block::Height) -> Option { Some(self.0.cmp(&other.0)) From 9ea9c7562c6020847f2a10f2e4948259d63d5243 Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 13:46:10 -0700 Subject: [PATCH 039/134] test(chain-index): pin sapling point-lookup skip width (known red) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit skip_opt_sapling_entry currently skips sapling outputs at the CompactSaplingSpend width (33 bytes/entry) instead of the CompactSaplingOutput width (117 bytes/entry), so the cursor lands mid-record for any entry with outputs — corrupting every DbV1 get_sapling point lookup of a transaction at or after such an entry (observed: cursor at 114 where the entry ends at 282, short by 2 outputs x 84 bytes). PR #1362 review finding. The new regression test asserts the skip lands exactly on the end of one encoded Option; #[should_panic] tracks the known bug and is removed together with the fix in the next commit. Co-Authored-By: Claude Fable 5 --- .../finalised_source/v1/block_shielded.rs | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/block_shielded.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/block_shielded.rs index 546efe39f..efdbd2e6a 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/block_shielded.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/block_shielded.rs @@ -766,3 +766,46 @@ impl DbV1 { Ok(()) } } + +#[cfg(test)] +mod skip_opt_sapling_entry { + use super::*; + use crate::CompactSaplingOutput; + + /// Regression test for the sapling point-lookup skip width: the skip must advance + /// exactly one encoded `Option` entry. A refactor had it skipping + /// sapling *outputs* at the spend width (33 bytes instead of 117), landing the cursor + /// mid-record for any entry with outputs and corrupting every `get_sapling` read of a + /// transaction at or after such an entry. + /// + /// `should_panic` tracks the known bug; remove it together with the fix. + #[test] + #[should_panic(expected = "skip_opt_sapling_entry must land exactly")] + fn advances_exactly_one_encoded_entry() { + let tx = SaplingCompactTx::new( + Some(7), + vec![CompactSaplingSpend::new([1u8; 32])], + vec![ + CompactSaplingOutput::new([2u8; 32], [3u8; 32], [4u8; 52]), + CompactSaplingOutput::new([5u8; 32], [6u8; 32], [7u8; 52]), + ], + ); + let list = SaplingTxList::new(vec![Some(tx)]); + let bytes = list + .to_bytes() + .expect("encoding an in-memory list cannot fail"); + + let mut cursor = std::io::Cursor::new(bytes.as_slice()); + // Mirror `get_sapling`: skip the SaplingTxList version byte, then the entry count. + cursor.set_position(1); + CompactSize::read(&mut cursor).expect("entry count is present"); + + DbV1::skip_opt_sapling_entry(&mut cursor).expect("entry is well-formed"); + + assert_eq!( + cursor.position() as usize, + bytes.len(), + "skip_opt_sapling_entry must land exactly on the end of the single encoded entry" + ); + } +} From 44ebd43aa1b6575147f4919efe8c82331fd6c0ba Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 13:47:38 -0700 Subject: [PATCH 040/134] fix(chain-index): skip sapling outputs at the output width skip_opt_sapling_entry reused the CompactSaplingSpend versioned length (33 bytes) to skip sapling outputs (117 bytes each), a wrong-variable slip introduced when the VERSIONED_LEN constants became locally-bound lengths. Bind the output width to its own clearly named variable from CompactSaplingOutput::latest_versioned_len. Flips the regression test from the previous commit green (#[should_panic] removed). PR #1362 review finding. Co-Authored-By: Claude Fable 5 --- .../finalised_state/finalised_source/v1/block_shielded.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/block_shielded.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/block_shielded.rs index efdbd2e6a..0791f4cc5 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/block_shielded.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/block_shielded.rs @@ -708,7 +708,8 @@ impl DbV1 { // Read number of outputs (CompactSize) let output_len = CompactSize::read(&mut *cursor)? as usize; - let output_skip = output_len * sapling_spend_len; + let sapling_output_len = crate::CompactSaplingOutput::latest_versioned_len()?; + let output_skip = output_len * sapling_output_len; cursor.set_position(cursor.position() + output_skip as u64); Ok(()) @@ -777,10 +778,7 @@ mod skip_opt_sapling_entry { /// sapling *outputs* at the spend width (33 bytes instead of 117), landing the cursor /// mid-record for any entry with outputs and corrupting every `get_sapling` read of a /// transaction at or after such an entry. - /// - /// `should_panic` tracks the known bug; remove it together with the fix. #[test] - #[should_panic(expected = "skip_opt_sapling_entry must land exactly")] fn advances_exactly_one_encoded_entry() { let tx = SaplingCompactTx::new( Some(7), From 30438d36fb37806ebda9f1c9c4e87c1e7bc5e9d4 Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 13:57:52 -0700 Subject: [PATCH 041/134] test(zaino-fetch): pin GetTransactionResponse ironwood JSON key (known red) The ironwood field of the verbose-transaction deserializer is copy-pasted reading the "orchard" JSON key, so a real ironwood bundle is silently dropped and the orchard bundle is duplicated into the ironwood slot. PR #1362 review finding. The regression test feeds distinct orchard/ironwood payloads and asserts each field carries its own; #[should_panic] tracks the known bug and is removed together with the fix in the next commit. Co-Authored-By: Claude Fable 5 --- .../zaino-fetch/src/jsonrpsee/response.rs | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/packages/zaino-fetch/src/jsonrpsee/response.rs b/packages/zaino-fetch/src/jsonrpsee/response.rs index 226c20bfd..db3b99453 100644 --- a/packages/zaino-fetch/src/jsonrpsee/response.rs +++ b/packages/zaino-fetch/src/jsonrpsee/response.rs @@ -1909,3 +1909,57 @@ pub struct GetMempoolInfoResponse { impl ResponseToError for GetMempoolInfoResponse { type RpcError = Infallible; } + +#[cfg(test)] +mod get_transaction_response { + use super::GetTransactionResponse; + + /// Regression test: the `ironwood` field must be read from the `"ironwood"` JSON key. + /// It was copy-pasted reading `"orchard"`, so a real ironwood bundle in a verbose + /// `getrawtransaction` response was silently dropped and the orchard bundle was + /// duplicated into the ironwood slot. + /// + /// `should_panic` tracks the known bug; remove it together with the fix. + #[test] + #[should_panic(expected = "ironwood field must carry")] + fn ironwood_field_reads_ironwood_key() { + let tx_json = serde_json::json!({ + "hex": "00", + "txid": "0000000000000000000000000000000000000000000000000000000000000001", + "version": 6, + "locktime": 0, + "orchard": { + "actions": [], + "valueBalance": -1.11, + "valueBalanceZat": -111, + }, + "ironwood": { + "actions": [], + "valueBalance": -2.22, + "valueBalanceZat": -222, + }, + }); + + let response: GetTransactionResponse = + serde_json::from_value(tx_json).expect("test transaction JSON deserializes"); + let GetTransactionResponse::Object(tx_object) = response else { + panic!("verbose transaction JSON must deserialize to the Object variant"); + }; + + let orchard = tx_object + .orchard() + .as_ref() + .expect("orchard bundle present in JSON"); + assert_eq!(orchard.value_balance_zat(), -111); + + let ironwood = tx_object + .ironwood() + .as_ref() + .expect("ironwood bundle present in JSON"); + assert_eq!( + ironwood.value_balance_zat(), + -222, + "ironwood field must carry the \"ironwood\" JSON payload, not a copy of \"orchard\"" + ); + } +} From 637bc1d73982933b69c26ca8c416e5c7cb75f140 Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 13:59:19 -0700 Subject: [PATCH 042/134] fix(zaino-fetch): read the ironwood bundle from the "ironwood" JSON key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ironwood field of the verbose-transaction deserializer was copy-pasted reading tx_value["orchard"], dropping real ironwood bundles and duplicating orchard data into the ironwood slot. Rather than only swapping the string, extend get_tx_value_fields! with a keyless arm that derives the JSON key from the binding identifier (`let ironwood: Orchard = tx_value;` reads the "ironwood" key). Every field whose JSON key equals its binding now uses that arm, so the key cannot be restated and sibling fields cannot be wired to each other's keys by copy-paste — the DRY removes this bug class, not just this instance. The bracketed literal arm remains for keys that genuinely differ from their bindings (vShieldedSpend, valueBalanceZat, ...). Flips the regression test from the previous commit green (#[should_panic] removed). PR #1362 review finding. Co-Authored-By: Claude Fable 5 --- .../zaino-fetch/src/jsonrpsee/response.rs | 39 ++++++++++++------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/packages/zaino-fetch/src/jsonrpsee/response.rs b/packages/zaino-fetch/src/jsonrpsee/response.rs index db3b99453..d7c772b03 100644 --- a/packages/zaino-fetch/src/jsonrpsee/response.rs +++ b/packages/zaino-fetch/src/jsonrpsee/response.rs @@ -1458,15 +1458,29 @@ impl<'de> serde::Deserialize<'de> for GetTransactionResponse { }, }; + // `let field: Kind = json;` reads the JSON key named exactly like the + // binding — the key cannot be restated, so sibling fields cannot be + // wired to each other's keys by copy-paste. Use the bracketed form + // `let field: Kind = json["jsonName"];` only when the JSON key differs + // from the binding (camelCase etc.). macro_rules! get_tx_value_fields{ - ($(let $field:ident: $kind:ty = $transaction_json:ident[$field_name:literal]; )+) => { - $(let $field = $transaction_json + () => {}; + (let $field:ident: $kind:ty = $transaction_json:ident; $($rest:tt)*) => { + let $field = $transaction_json + .get(stringify!($field)) + .map(|v| ::serde_json::from_value::<$kind>(v.clone())) + .transpose() + .map_err(::serde::de::Error::custom)?; + get_tx_value_fields! { $($rest)* } + }; + (let $field:ident: $kind:ty = $transaction_json:ident[$field_name:literal]; $($rest:tt)*) => { + let $field = $transaction_json .get($field_name) .map(|v| ::serde_json::from_value::<$kind>(v.clone())) .transpose() .map_err(::serde::de::Error::custom)?; - )+ - } + get_tx_value_fields! { $($rest)* } + }; } let confirmations = tx_value.get("confirmations").and_then(|v| v.as_i64()); @@ -1493,16 +1507,16 @@ impl<'de> serde::Deserialize<'de> for GetTransactionResponse { let outputs: Vec = tx_value["vout"]; let shielded_spends: Vec = tx_value["vShieldedSpend"]; let shielded_outputs: Vec = tx_value["vShieldedOutput"]; - let orchard: Orchard = tx_value["orchard"]; - let ironwood: Orchard = tx_value["orchard"]; + let orchard: Orchard = tx_value; + let ironwood: Orchard = tx_value; let value_balance: f64 = tx_value["valueBalance"]; let value_balance_zat: i64 = tx_value["valueBalanceZat"]; - let size: i64 = tx_value["size"]; - let time: i64 = tx_value["time"]; - let txid: String = tx_value["txid"]; + let size: i64 = tx_value; + let time: i64 = tx_value; + let txid: String = tx_value; let auth_digest: String = tx_value["authdigest"]; - let overwintered: bool = tx_value["overwintered"]; - let version: u32 = tx_value["version"]; + let overwintered: bool = tx_value; + let version: u32 = tx_value; let version_group_id: String = tx_value["versiongroupid"]; let lock_time: u32 = tx_value["locktime"]; let expiry_height: Height = tx_value["expiryheight"]; @@ -1918,10 +1932,7 @@ mod get_transaction_response { /// It was copy-pasted reading `"orchard"`, so a real ironwood bundle in a verbose /// `getrawtransaction` response was silently dropped and the orchard bundle was /// duplicated into the ironwood slot. - /// - /// `should_panic` tracks the known bug; remove it together with the fix. #[test] - #[should_panic(expected = "ironwood field must carry")] fn ironwood_field_reads_ironwood_key() { let tx_json = serde_json::json!({ "hex": "00", From 57f6b7e6369651f600b9bc284d10cd7938fce1b8 Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 14:15:49 -0700 Subject: [PATCH 043/134] test(chain-index): pin the ironwood treestate-slot contract (known red) Extract the ironwood slot of ValidatorConnector::get_treestate into ironwood_treestate_slot(), used by both the State and Fetch arms, and DRY the three identical empty-Orchard-tree closures into empty_orchard_tree_rpc_bytes(). Behaviour-faithful extraction: the slot still back-fills a serialized empty tree when the validator reported no ironwood treestate. That back-fill is the bug (PR #1362 review finding): zebra documents the z_gettreestate ironwood field as "Only present from NU6.3, so that pre-NU6.3 responses are unchanged" and omits it below activation, while zaino emits it (empty-tree hex) at every height on every network. The State arm is also inconsistent with its own sapling and orchard siblings, which correctly stay None below activation. The regression test pins "no ironwood from the validator => absent slot"; #[should_panic] tracks the known bug and is removed together with the fix in the next commit. (Fetch-arm nuance: the extraction rewrites map_or_else to and_then + slot helper; they differ only for a present ironwood field with a null finalState, which zebra never produces.) Co-Authored-By: Claude Fable 5 --- .../chain_index/source/validator_connector.rs | 93 ++++++++++++------- 1 file changed, 61 insertions(+), 32 deletions(-) diff --git a/packages/zaino-state/src/chain_index/source/validator_connector.rs b/packages/zaino-state/src/chain_index/source/validator_connector.rs index 6dc34d25e..eab117881 100644 --- a/packages/zaino-state/src/chain_index/source/validator_connector.rs +++ b/packages/zaino-state/src/chain_index/source/validator_connector.rs @@ -51,6 +51,30 @@ pub enum ValidatorConnector { Fetch(JsonRpSeeConnector), } +/// Serialized empty Orchard-shaped commitment tree in the RPC encoding — what a pool +/// reports when it has no treestate to serve, and the encoding both backend arms must +/// agree on. +fn empty_orchard_tree_rpc_bytes() -> Vec { + let mut tree = vec![]; + write_commitment_tree( + &CommitmentTree::::empty(), + &mut tree, + ) + .expect("can write to Vec"); + tree +} + +/// The ironwood slot of the source treestate tuple, given the ironwood treestate the +/// validator reported for the block (if any). +/// +/// The z_gettreestate ironwood field is documented as "Only present from NU6.3, so that +/// pre-NU6.3 responses are unchanged": when the validator reported no ironwood treestate +/// (below NU6.3 activation, or on a network with no NU6.3 activation height) the slot +/// must stay `None`, so the response omits the field exactly as zebrad does. +fn ironwood_treestate_slot(validator_ironwood: Option>) -> Option> { + validator_ironwood.or_else(|| Some(empty_orchard_tree_rpc_bytes())) +} + impl BlockchainSource for ValidatorConnector { // ********** Block methods ********** @@ -474,19 +498,8 @@ impl BlockchainSource for ValidatorConnector { .and_then(|irw_response| { expected_read_response!(irw_response, IronwoodTree) .map(|tree| tree.to_rpc_bytes()) - }) - // Mirror the Fetch connector: when the ironwood tree is absent (e.g. before - // NU6.3 activation) return the serialized empty tree rather than `None`, so both - // backends agree on the empty-tree encoding. - .or_else(|| { - let mut tree = vec![]; - write_commitment_tree( - &CommitmentTree::::empty(), - &mut tree, - ) - .expect("can write to Vec"); - Some(tree) }); + let ironwood = ironwood_treestate_slot(ironwood); Ok((sapling, orchard, ironwood)) } @@ -512,29 +525,14 @@ impl BlockchainSource for ValidatorConnector { ); let orchard = treestate.orchard.map_or_else( - || { - let mut tree = vec![]; - write_commitment_tree( - &CommitmentTree::::empty(), - &mut tree, - ) - .expect("can write to Vec"); - Some(tree) - }, + || Some(empty_orchard_tree_rpc_bytes()), |t| t.commitments().final_state().clone(), ); - let ironwood = treestate.ironwood.map_or_else( - || { - let mut tree = vec![]; - write_commitment_tree( - &CommitmentTree::::empty(), - &mut tree, - ) - .expect("can write to Vec"); - Some(tree) - }, - |t| t.commitments().final_state().clone(), + let ironwood = ironwood_treestate_slot( + treestate + .ironwood + .and_then(|t| t.commitments().final_state().clone()), ); Ok((sapling, orchard, ironwood)) @@ -1175,3 +1173,34 @@ impl BlockchainSource for ValidatorConnector { } } } + +#[cfg(test)] +mod ironwood_treestate_slot { + /// Regression test: when the validator reported no ironwood treestate (below NU6.3 + /// activation, or on a network with no NU6.3 activation height) the slot must stay + /// `None`, so z_gettreestate omits the ironwood field exactly as zebrad does + /// ("Only present from NU6.3, so that pre-NU6.3 responses are unchanged"). The slot + /// was previously back-filled with a serialized empty tree, emitting the field at + /// every height on every network. + /// + /// `should_panic` tracks the known bug; remove it together with the fix. + #[test] + #[should_panic(expected = "ironwood slot must stay absent")] + fn absent_validator_ironwood_stays_absent() { + assert_eq!( + super::ironwood_treestate_slot(None), + None, + "ironwood slot must stay absent when the validator reported no ironwood treestate" + ); + } + + /// A reported ironwood treestate passes through unchanged. + #[test] + fn reported_validator_ironwood_passes_through() { + let tree_bytes = vec![1u8, 2, 3]; + assert_eq!( + super::ironwood_treestate_slot(Some(tree_bytes.clone())), + Some(tree_bytes) + ); + } +} From 618e8c2378d5d23579dbb3a9d471a4b6918ff386 Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 14:17:00 -0700 Subject: [PATCH 044/134] fix(chain-index): omit the ironwood treestate below NU6.3 activation ironwood_treestate_slot now passes the validator's report through unchanged instead of back-filling a serialized empty tree. Pre-NU6.3 (and on networks with no NU6.3 activation height) the slot stays None, so: - z_gettreestate omits the ironwood field exactly as zebrad does ("Only present from NU6.3, so that pre-NU6.3 responses are unchanged"), restoring response parity for every historical block; - the gRPC TreeState.ironwood_tree falls back to the empty string via the existing unwrap_or_default, matching lightwalletd behaviour; - the State arm's ironwood gating is now consistent with its own sapling and orchard siblings, which already stayed None below activation. Flips the regression test from the previous commit green (#[should_panic] removed). PR #1362 review finding. Co-Authored-By: Claude Fable 5 --- .../src/chain_index/source/validator_connector.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/zaino-state/src/chain_index/source/validator_connector.rs b/packages/zaino-state/src/chain_index/source/validator_connector.rs index eab117881..f69e4a73f 100644 --- a/packages/zaino-state/src/chain_index/source/validator_connector.rs +++ b/packages/zaino-state/src/chain_index/source/validator_connector.rs @@ -70,9 +70,10 @@ fn empty_orchard_tree_rpc_bytes() -> Vec { /// The z_gettreestate ironwood field is documented as "Only present from NU6.3, so that /// pre-NU6.3 responses are unchanged": when the validator reported no ironwood treestate /// (below NU6.3 activation, or on a network with no NU6.3 activation height) the slot -/// must stay `None`, so the response omits the field exactly as zebrad does. +/// stays `None`, so the response omits the field exactly as zebrad does. This function +/// names that contract — do not back-fill an empty tree here. fn ironwood_treestate_slot(validator_ironwood: Option>) -> Option> { - validator_ironwood.or_else(|| Some(empty_orchard_tree_rpc_bytes())) + validator_ironwood } impl BlockchainSource for ValidatorConnector { @@ -1182,10 +1183,7 @@ mod ironwood_treestate_slot { /// ("Only present from NU6.3, so that pre-NU6.3 responses are unchanged"). The slot /// was previously back-filled with a serialized empty tree, emitting the field at /// every height on every network. - /// - /// `should_panic` tracks the known bug; remove it together with the fix. #[test] - #[should_panic(expected = "ironwood slot must stay absent")] fn absent_validator_ironwood_stays_absent() { assert_eq!( super::ironwood_treestate_slot(None), From 453c64e3184d296b3c356ee1d1dadc4106d646ce Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 14:35:47 -0700 Subject: [PATCH 045/134] Resolve the zebra source-build git ref in the workbench, not in-container MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ZEBRA_VERSION in .env.testing-artifacts is canonically the zfnd/zebra Docker Hub tag (bare, e.g. "6.0.0-rc.0"), but zebra's git release tags carry a `v` prefix — so the zebra-builder stage's bare `git checkout "${ZEBRA_VERSION}"` could never check out a release version, and the mismatch only surfaced as a pathspec error after the stage had cloned the whole zebra repo. Give the version one canonical definition with derived consumers, mirroring the DEVTOOL_REV plumbing and the RUST_VERSION workbench precedent: - workbench `get-zebra-git-ref` bin (logic + unit tests in lib.rs): probes refs/tags/v$V, refs/tags/$V, refs/heads/$V in one ls-remote; release versions map to the v-tag, branch/plain-tag pins resolve as-is, commit SHAs pass through, anything else fails at resolve time with a diagnostic. - build-image.sh and build-n-push-ci-image.yaml resolve ZEBRA_GIT_REF via the bin and pass it as a build-arg, so a bad ZEBRA_VERSION now fails before the build starts. - The Containerfile's zebra-builder consumes ARG ZEBRA_GIT_REF and keeps zero tag-convention knowledge; the bare-ZEBRA_VERSION fallback only serves hand-run builds pinning a branch or SHA. Also bump the test-image pins that exposed the mismatch: ZEBRA_VERSION to 6.0.0-rc.0 (the NU6.3-era release candidate) and DEVTOOL_VERSION to a499f855, the tip of zingolabs/zcash-devtool's support_NU6_3 branch under review as zcash/zcash-devtool#205. The resolver plumbing is version-agnostic; if dev wants it before this branch merges, cherry-pick this commit and revert the .env.testing-artifacts hunk. Co-Authored-By: Claude Fable 5 --- .env.testing-artifacts | 6 +- .github/workflows/build-n-push-ci-image.yaml | 6 ++ live-tests/test_environment/Containerfile | 9 ++- tools/scripts/build-image.sh | 9 +++ tools/scripts/functions.sh | 2 +- tools/workbench/src/bin/get-zebra-git-ref.rs | 36 +++++++++ tools/workbench/src/lib.rs | 83 ++++++++++++++++++++ 7 files changed, 147 insertions(+), 4 deletions(-) create mode 100644 tools/workbench/src/bin/get-zebra-git-ref.rs diff --git a/.env.testing-artifacts b/.env.testing-artifacts index 9bb6238e8..c9e575189 100644 --- a/.env.testing-artifacts +++ b/.env.testing-artifacts @@ -5,7 +5,7 @@ # zcashd Git tag (https://github.com/zcash/zcash/releases) ZCASH_VERSION=6.20.0 -ZEBRA_VERSION=5.2.0 +ZEBRA_VERSION=6.0.0-rc.0 # zcash-devtool git ref (https://github.com/zingolabs/zcash-devtool) — # the wallet client driven by wallet-tests via zcash_local_net, built @@ -14,4 +14,6 @@ ZEBRA_VERSION=5.2.0 # (get-ci-image-tag.sh → resolve_devtool_rev), so a branch HEAD advancing # changes the tag automatically and the rebuild always bakes the current # binary — no manual SHA bump needed. Pin a tag here only to freeze it. -DEVTOOL_VERSION=add_regtest +# Pinned to the NU6.3/Ironwood support commit: tip of `support_NU6_3`, +# under review as zcash/zcash-devtool#205. +DEVTOOL_VERSION=a499f855fa88080045200ef05d03e79f2a506599 diff --git a/.github/workflows/build-n-push-ci-image.yaml b/.github/workflows/build-n-push-ci-image.yaml index d837af914..e29b963f2 100644 --- a/.github/workflows/build-n-push-ci-image.yaml +++ b/.github/workflows/build-n-push-ci-image.yaml @@ -49,10 +49,15 @@ jobs: source .env.testing-artifacts set +a RUST_VERSION=$(cargo run -q --manifest-path tools/workbench/Cargo.toml --bin get-rust-version) + # Resolve ZEBRA_VERSION (the Docker Hub tag) to the git ref the + # zebra-builder source stage checks out; fails the job here on an + # unresolvable value instead of mid-build. + ZEBRA_GIT_REF=$(cargo run -q --manifest-path tools/workbench/Cargo.toml --bin get-zebra-git-ref -- "$ZEBRA_VERSION") echo "sha_short=$(git rev-parse --short HEAD)" >> "$GITHUB_OUTPUT" echo "RUST_VERSION=$RUST_VERSION" >> "$GITHUB_ENV" echo "ZCASH_VERSION=$ZCASH_VERSION" >> "$GITHUB_ENV" echo "ZEBRA_VERSION=$ZEBRA_VERSION" >> "$GITHUB_ENV" + echo "ZEBRA_GIT_REF=$ZEBRA_GIT_REF" >> "$GITHUB_ENV" echo "DEVTOOL_VERSION=$DEVTOOL_VERSION" >> "$GITHUB_ENV" - name: Define build target @@ -79,4 +84,5 @@ jobs: RUST_VERSION=${{ env.RUST_VERSION }} ZCASH_VERSION=${{ env.ZCASH_VERSION }} ZEBRA_VERSION=${{ env.ZEBRA_VERSION }} + ZEBRA_GIT_REF=${{ env.ZEBRA_GIT_REF }} DEVTOOL_VERSION=${{ env.DEVTOOL_VERSION }} diff --git a/live-tests/test_environment/Containerfile b/live-tests/test_environment/Containerfile index 278d71356..dabb4dea2 100644 --- a/live-tests/test_environment/Containerfile +++ b/live-tests/test_environment/Containerfile @@ -29,6 +29,13 @@ FROM docker.io/zodlinc/zcash:v${ZCASH_VERSION} AS zcashd-downloader FROM docker.io/library/rust:${RUST_VERSION}-trixie AS zebra-builder ARG ZEBRA_VERSION +# Git ref resolved from ZEBRA_VERSION by the workbench `get-zebra-git-ref` +# bin: ZEBRA_VERSION is the Docker Hub tag (bare, e.g. "6.0.0-rc.0") while +# zebra's git release tags carry a `v` prefix. Both build entry points +# (build-image.sh, build-n-push-ci-image.yaml) resolve and pass it; the +# bare-ZEBRA_VERSION fallback only serves hand-run builds that pin a branch +# or SHA. +ARG ZEBRA_GIT_REF # Versions pinned (DL3008) to the candidates in rust:1.95.0-trixie; bump # together with the base image (query with `apt-cache policy `). # --no-install-recommends (DL3015): ca-certificates is already present in the @@ -44,7 +51,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ WORKDIR /zebra # Single RUN (DL3059): clone, checkout, build, and stage the binary. RUN git clone https://github.com/ZcashFoundation/zebra . \ - && git checkout "${ZEBRA_VERSION}" \ + && git checkout "${ZEBRA_GIT_REF:-$ZEBRA_VERSION}" \ && cargo build --release --bin zebrad \ && cp target/release/zebrad /tmp/zebrad diff --git a/tools/scripts/build-image.sh b/tools/scripts/build-image.sh index 9d3f44185..1233965e2 100755 --- a/tools/scripts/build-image.sh +++ b/tools/scripts/build-image.sh @@ -40,11 +40,20 @@ info "Files in tools/scripts/: $(ls -la tools/scripts/ | head -5)" DEVTOOL_REV=$(resolve_devtool_rev "$DEVTOOL_VERSION") info "Resolved DEVTOOL_VERSION=$DEVTOOL_VERSION to DEVTOOL_REV=$DEVTOOL_REV" +# Resolve ZEBRA_VERSION (canonically the Docker Hub tag, e.g. "6.0.0-rc.0") +# to the git ref the zebra-builder source stage checks out — zebra's release +# tags are `v`-prefixed in git. Fails here, before the build, on an +# unresolvable value. +ZEBRA_GIT_REF=$(cargo run -q --manifest-path tools/workbench/Cargo.toml \ + --bin get-zebra-git-ref -- "$ZEBRA_VERSION") +info "Resolved ZEBRA_VERSION=$ZEBRA_VERSION to ZEBRA_GIT_REF=$ZEBRA_GIT_REF" + cd live-tests/test_environment && \ podman build -f Containerfile \ --target "$TARGET" \ --build-arg "ZCASH_VERSION=$ZCASH_VERSION" \ --build-arg "ZEBRA_VERSION=$ZEBRA_VERSION" \ + --build-arg "ZEBRA_GIT_REF=$ZEBRA_GIT_REF" \ --build-arg "DEVTOOL_VERSION=$DEVTOOL_VERSION" \ --build-arg "DEVTOOL_REV=$DEVTOOL_REV" \ --build-arg "RUST_VERSION=$RUST_VERSION" \ diff --git a/tools/scripts/functions.sh b/tools/scripts/functions.sh index 7b28fd427..533ce4278 100755 --- a/tools/scripts/functions.sh +++ b/tools/scripts/functions.sh @@ -21,4 +21,4 @@ resolve_devtool_rev() { local ref="$1" rev rev=$(git ls-remote https://github.com/zingolabs/zcash-devtool "$ref" 2>/dev/null | awk 'NR==1{print $1}') echo "${rev:-$ref}" -} \ No newline at end of file +} diff --git a/tools/workbench/src/bin/get-zebra-git-ref.rs b/tools/workbench/src/bin/get-zebra-git-ref.rs new file mode 100644 index 000000000..d14bbb1ee --- /dev/null +++ b/tools/workbench/src/bin/get-zebra-git-ref.rs @@ -0,0 +1,36 @@ +//! Resolve `ZEBRA_VERSION` to the git ref the zebra source build checks out. +//! +//! `ZEBRA_VERSION` (from `.env.testing-artifacts`) is canonically the Docker +//! Hub image tag — bare, e.g. "6.0.0-rc.0" — while zebra's git release tags +//! carry a `v` prefix. Both container build entry points (`build-image.sh`, +//! `build-n-push-ci-image.yaml`) run this to derive the checkout ref they +//! pass as the `ZEBRA_GIT_REF` build-arg, so an unresolvable version fails +//! before the build instead of as a pathspec error after the zebra clone. +//! +//! Usage: `get-zebra-git-ref ` (falls back to the +//! `ZEBRA_VERSION` environment variable). + +use workbench::{git, run, zebra_git_ref, zebra_ref_probes, ZEBRA_REPO_URL}; + +fn main() { + run( + "get-zebra-git-ref", + || { + let version = std::env::args() + .nth(1) + .or_else(|| std::env::var("ZEBRA_VERSION").ok()) + .filter(|v| !v.is_empty()) + .ok_or_else(|| { + vec!["usage: get-zebra-git-ref \ +(or set the ZEBRA_VERSION environment variable)" + .to_string()] + })?; + let probes = zebra_ref_probes(&version); + let mut args = vec!["ls-remote", ZEBRA_REPO_URL]; + args.extend(probes.iter().map(String::as_str)); + let output = git(&args)?; + zebra_git_ref(&version, &output) + }, + |git_ref| println!("{git_ref}"), + ) +} diff --git a/tools/workbench/src/lib.rs b/tools/workbench/src/lib.rs index 6f04ba4a9..eb577aeea 100644 --- a/tools/workbench/src/lib.rs +++ b/tools/workbench/src/lib.rs @@ -81,6 +81,57 @@ pub fn toolchain_channel(root: &Path) -> Result> { Ok(channel) } +/// The zebra repository probed by `get-zebra-git-ref`. +pub const ZEBRA_REPO_URL: &str = "https://github.com/ZcashFoundation/zebra"; + +/// The `git ls-remote` refs to probe for a `ZEBRA_VERSION` value: its +/// `v`-prefixed release tag, a bare tag, and a branch, in that precedence. +pub fn zebra_ref_probes(version: &str) -> [String; 3] { + [ + format!("refs/tags/v{version}"), + format!("refs/tags/{version}"), + format!("refs/heads/{version}"), + ] +} + +/// Resolve `ZEBRA_VERSION` to the git ref the zebra source build checks out, +/// given the `git ls-remote` output for [`zebra_ref_probes`]. +/// +/// `ZEBRA_VERSION` is canonically the Docker Hub image tag (bare, e.g. +/// "6.0.0-rc.0"), but zebra's git release tags carry a `v` prefix, so a +/// release version maps to `v{version}`. A branch or plain-tag pin resolves +/// as-is, and a commit SHA passes through (`ls-remote` matches ref names, not +/// commits). Anything else is an error — reported at resolve time instead of +/// surfacing as a checkout pathspec error after the container build stage has +/// cloned the whole zebra repo. +pub fn zebra_git_ref(version: &str, ls_remote_output: &str) -> Result> { + let v_tag = format!("refs/tags/v{version}"); + let matches_v_tag = ls_remote_output + .lines() + .filter_map(|line| line.split('\t').nth(1)) + .any(|r| r == v_tag); + + let some_ref_matched = ls_remote_output.lines().any(|l| !l.trim().is_empty()); + if matches_v_tag { + Ok(format!("v{version}")) + } else if some_ref_matched || is_commit_sha_shaped(version) { + Ok(version.to_string()) + } else { + Err(vec![format!( + "ZEBRA_VERSION={version} matches no zebra tag (v-prefixed or bare), \ +branch, or commit-SHA shape" + )]) + } +} + +/// 7 to 40 lowercase-hex characters — an abbreviated or full git commit SHA. +fn is_commit_sha_shaped(version: &str) -> bool { + (7..=40).contains(&version.len()) + && version + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} + /// Value of a `channel = "..."` line, mirroring `^[[:space:]]*channel[[:space:]]*=`. /// `None` for comments, other keys, or a line without a double-quoted value. fn channel_value(line: &str) -> Option { @@ -115,6 +166,38 @@ mod tests { assert_eq!(channel_value("[toolchain]"), None); } + #[test] + fn zebra_git_ref_prefers_the_v_tag() { + // Annotated tags also list a peeled `^{}` line; the plain ref wins. + let out = "abc123\trefs/tags/v6.0.0-rc.0\nabc456\trefs/tags/v6.0.0-rc.0^{}\n"; + assert_eq!( + zebra_git_ref("6.0.0-rc.0", out).as_deref(), + Ok("v6.0.0-rc.0") + ); + } + + #[test] + fn zebra_git_ref_passes_branches_and_bare_tags_through() { + let out = "abc123\trefs/heads/main\n"; + assert_eq!(zebra_git_ref("main", out).as_deref(), Ok("main")); + } + + #[test] + fn zebra_git_ref_passes_sha_shapes_through_unprobed() { + assert_eq!( + zebra_git_ref("15d578362448fb8c4a5d29a00dcfe8adb5184082", "").as_deref(), + Ok("15d578362448fb8c4a5d29a00dcfe8adb5184082") + ); + assert_eq!(zebra_git_ref("15d5783", "").as_deref(), Ok("15d5783")); + } + + #[test] + fn zebra_git_ref_rejects_unresolvable_values() { + assert!(zebra_git_ref("not-a-real-ref", "").is_err()); + // Short-hex-lookalike below the 7-char floor is rejected too. + assert!(zebra_git_ref("abc", "").is_err()); + } + #[test] fn numeric_validation_matches_x_y_z() { assert!(is_concrete_numeric("1.96.0")); From 8005b504b53c9d013f4c096606bca09db90cb080 Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 14:41:22 -0700 Subject: [PATCH 046/134] test(chain-index): pin None-preservation of the stored ironwood root (known red) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move create_commitment_tree_data from BlockWithMetadata to BlockMetadata — it reads only metadata, and the move gives the invariant a block-free test seam. The pinned invariant (PR #1362 review finding): a block whose source reported no ironwood treestate must store its ironwood root as None, the encoding the v1.2.1->v1.3.0 migration and the CommitmentTreeRoots V1 decode produce for the same state. The write path instead erases the Option in TreeRootData::extract_with_defaults and stores Some([0; 32]), so freshly synced and migrated databases encode identical pre-NU6.3 heights differently. #[should_panic] tracks the known bug and is removed together with the fix in the next commit. Co-Authored-By: Claude Fable 5 --- .../src/chain_index/types/helpers.rs | 56 ++++++++++++++++--- 1 file changed, 48 insertions(+), 8 deletions(-) diff --git a/packages/zaino-state/src/chain_index/types/helpers.rs b/packages/zaino-state/src/chain_index/types/helpers.rs index e2c8960cb..a3118facb 100644 --- a/packages/zaino-state/src/chain_index/types/helpers.rs +++ b/packages/zaino-state/src/chain_index/types/helpers.rs @@ -350,19 +350,21 @@ impl<'a> BlockWithMetadata<'a> { Ok(BlockContext::new(hash, parent_hash, chainwork, height)) } +} - /// Create commitment tree data from metadata +impl BlockMetadata { + /// Create the stored commitment tree data for this block's metadata. fn create_commitment_tree_data(&self) -> super::db::CommitmentTreeData { let commitment_tree_roots = super::db::CommitmentTreeRoots::new( - <[u8; 32]>::from(self.metadata.sapling_root), - <[u8; 32]>::from(self.metadata.orchard_root), - Some(<[u8; 32]>::from(self.metadata.ironwood_root)), + <[u8; 32]>::from(self.sapling_root), + <[u8; 32]>::from(self.orchard_root), + Some(<[u8; 32]>::from(self.ironwood_root)), ); let commitment_tree_size = super::db::CommitmentTreeSizes::new( - self.metadata.sapling_size, - self.metadata.orchard_size, - self.metadata.ironwood_size, + self.sapling_size, + self.orchard_size, + self.ironwood_size, ); super::db::CommitmentTreeData::new(commitment_tree_roots, commitment_tree_size) @@ -377,7 +379,7 @@ impl TryFrom> for IndexedBlock { let data = block_with_metadata.extract_block_data()?; let transactions = block_with_metadata.extract_transactions()?; let context = block_with_metadata.create_block_context()?; - let commitment_tree_data = block_with_metadata.create_commitment_tree_data(); + let commitment_tree_data = block_with_metadata.metadata.create_commitment_tree_data(); Ok(IndexedBlock { context, @@ -387,3 +389,41 @@ impl TryFrom> for IndexedBlock { }) } } + +#[cfg(test)] +mod create_commitment_tree_data { + use super::*; + + /// Regression test: a block whose source reported no ironwood treestate (pre-NU6.3, + /// or a network with no NU6.3 activation height) must store its ironwood root as + /// `None` — the encoding the v1.2.1->v1.3.0 migration and the CommitmentTreeRoots V1 + /// decode produce for the same state. The write path instead erased the `Option` via + /// `extract_with_defaults` and stored `Some([0; 32])`, so a freshly synced database + /// and a migrated database encoded identical pre-activation heights differently. + /// + /// `should_panic` tracks the known bug; remove it together with the fix. + #[test] + #[should_panic(expected = "ironwood root must be stored as None")] + fn absent_ironwood_root_is_stored_as_none() { + let (sapling_root, sapling_size, orchard_root, orchard_size, ironwood_root, ironwood_size) = + TreeRootData::new(None, None, None).extract_with_defaults(); + let metadata = BlockMetadata::new( + sapling_root, + sapling_size as u32, + orchard_root, + orchard_size as u32, + ironwood_root, + ironwood_size as u32, + None, + zebra_chain::parameters::Network::Mainnet, + ); + + let commitment_tree_data = metadata.create_commitment_tree_data(); + + assert_eq!( + commitment_tree_data.roots().ironwood(), + &None, + "ironwood root must be stored as None when the source reported no ironwood treestate" + ); + } +} From c4f6b4f00b89f36df4c8b3e41e1fdeecb55476ba Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 14:47:06 -0700 Subject: [PATCH 047/134] fix(chain-index): store the ironwood root as None when the pool has no treestate Carry the ironwood component as Option<(Root, u32)> end to end instead of erasing it to a default: - BlockMetadata.ironwood_root/ironwood_size become a single ironwood: Option<(Root, u32)> (root and size can no longer disagree, and BlockMetadata::new drops below the too_many_arguments lint); - TreeRootData::extract_with_defaults passes the ironwood Option through instead of unwrap_or_default; - new optional_pool_root keeps the below-activation case as None; required_pool_root (sapling/orchard, whose storage is non-optional) now delegates to it; - create_commitment_tree_data maps the Option, storing roots.ironwood = None and size 0 when the block has no ironwood treestate. Freshly synced and migrated databases now produce the identical encoding for pre-NU6.3 heights (None), where fresh sync previously stored Some([0; 32]) against the migration's None. Flips the regression test from the previous commit green (#[should_panic] removed). Full zaino-state suite green (179 tests). PR #1362 review finding. Co-Authored-By: Claude Fable 5 --- packages/zaino-state/src/chain_index.rs | 22 +++++++----- .../src/chain_index/finalised_state.rs | 28 ++++++++++----- .../finalised_source/ephemeral.rs | 9 +++-- .../src/chain_index/non_finalised_state.rs | 5 ++- .../chain_index/tests/finalised_state/v1.rs | 3 +- .../src/chain_index/tests/vectors.rs | 3 +- .../src/chain_index/types/helpers.rs | 36 ++++++++----------- 7 files changed, 57 insertions(+), 49 deletions(-) diff --git a/packages/zaino-state/src/chain_index.rs b/packages/zaino-state/src/chain_index.rs index d8881c4a6..806b52903 100644 --- a/packages/zaino-state/src/chain_index.rs +++ b/packages/zaino-state/src/chain_index.rs @@ -1124,7 +1124,7 @@ async fn compact_block_from_source( .get_commitment_tree_roots(types::BlockHash::from(block.hash())) .await .map_err(ChainIndexError::backing_validator)?; - let (sapling_root, sapling_size, orchard_root, orchard_size, ironwood_root, ironwood_size) = + let (sapling_root, sapling_size, orchard_root, orchard_size, ironwood) = TreeRootData::new(tree_roots.0, tree_roots.1, tree_roots.2).extract_with_defaults(); let metadata = BlockMetadata::new( @@ -1142,13 +1142,19 @@ async fn compact_block_from_source( "orchard commitment tree size overflow", )) })?, - ironwood_root, - ironwood_size.try_into().map_err(|_| { - ChainIndexError::backing_validator(std::io::Error::new( - std::io::ErrorKind::InvalidData, - "ironwood commitment tree size overflow", - )) - })?, + ironwood + .map(|(root, size)| { + Ok::<_, ChainIndexError>(( + root, + size.try_into().map_err(|_| { + ChainIndexError::backing_validator(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "ironwood commitment tree size overflow", + )) + })?, + )) + }) + .transpose()?, None, // parent chainwork unknown — single-block construction network, ); diff --git a/packages/zaino-state/src/chain_index/finalised_state.rs b/packages/zaino-state/src/chain_index/finalised_state.rs index 8a495e852..c26c0b3e6 100644 --- a/packages/zaino-state/src/chain_index/finalised_state.rs +++ b/packages/zaino-state/src/chain_index/finalised_state.rs @@ -298,7 +298,7 @@ pub(crate) async fn build_indexed_block_from_source( orchard_opt, || format!("block {block_hash}"), )?; - let (ironwood_root, ironwood_size) = required_pool_root( + let ironwood = optional_pool_root( super::ShieldedPool::Ironwood, is_ironwood_active, ironwood_opt, @@ -310,8 +310,7 @@ pub(crate) async fn build_indexed_block_from_source( sapling_size, orchard_root, orchard_size, - ironwood_root, - ironwood_size, + ironwood, parent_chainwork, network.to_zebra_network(), ); @@ -339,6 +338,20 @@ fn required_pool_root( root_and_size: Option<(R, u64)>, location: impl FnOnce() -> String, ) -> Result<(R, u32), FinalisedStateError> { + optional_pool_root(pool, is_active, root_and_size, location) + .map(|resolved| resolved.unwrap_or_else(|| (R::default(), 0))) +} + +/// Like [`required_pool_root`], but keeps the below-activation case as `None` instead of +/// defaulting, for pools whose storage distinguishes "no treestate yet" from a +/// default-valued one (the stored ironwood root: `None` = no ironwood data, the encoding +/// the v1.2.1->v1.3.0 migration produces for pre-activation heights). +fn optional_pool_root( + pool: super::ShieldedPool, + is_active: bool, + root_and_size: Option<(R, u64)>, + location: impl FnOnce() -> String, +) -> Result, FinalisedStateError> { let pool = pool.pool_string(); match root_and_size { Some((root, size)) => { @@ -347,9 +360,9 @@ fn required_pool_root( format!("{pool} commitment tree size does not fit into u32: {error}"), )) })?; - Ok((root, size)) + Ok(Some((root, size))) } - None if !is_active => Ok((R::default(), 0)), + None if !is_active => Ok(None), None => Err(FinalisedStateError::BlockchainSourceError( BlockchainSourceError::Unrecoverable(format!( "missing {pool} commitment tree root for {}", @@ -1117,7 +1130,7 @@ impl FinalisedState { })?; let is_ironwood_active = nu6_3_activation_height.is_some_and(|activation| height >= activation.0); - let (ironwood_root, ironwood_size) = required_pool_root( + let ironwood = optional_pool_root( super::ShieldedPool::Ironwood, is_ironwood_active, ironwood_opt, @@ -1129,8 +1142,7 @@ impl FinalisedState { sapling_size, orchard_root, orchard_size, - ironwood_root, - ironwood_size, + ironwood, parent_chainwork, cfg.network.to_zebra_network(), ); diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/ephemeral.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/ephemeral.rs index c5a4f6ea1..452979dd5 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/ephemeral.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/ephemeral.rs @@ -15,7 +15,7 @@ use crate::chain_index::finalised_state::capability::{DbCore, DbWrite}; use crate::chain_index::finalised_state::DbMetadata; use crate::chain_index::ShieldedPool; -use super::super::required_pool_root; +use super::super::{optional_pool_root, required_pool_root}; use crate::chain_index::source::BlockchainSourceError; use crate::chain_index::{ finalised_state::capability::{ @@ -279,8 +279,8 @@ impl EphemeralFinalisedState { required_pool_root(ShieldedPool::Orchard, orchard_is_active, orchard, || { format!("block at height {height}") })?; - let (ironwood_root, ironwood_size) = - required_pool_root(ShieldedPool::Ironwood, ironwood_is_active, ironwood, || { + let ironwood = + optional_pool_root(ShieldedPool::Ironwood, ironwood_is_active, ironwood, || { format!("block at height {height}") })?; @@ -289,8 +289,7 @@ impl EphemeralFinalisedState { sapling_size, orchard_root, orchard_size, - ironwood_root, - ironwood_size, + ironwood, None, // ephemeral store does not track chainwork self.network.clone(), ); diff --git a/packages/zaino-state/src/chain_index/non_finalised_state.rs b/packages/zaino-state/src/chain_index/non_finalised_state.rs index 023499cb3..aee18b01a 100644 --- a/packages/zaino-state/src/chain_index/non_finalised_state.rs +++ b/packages/zaino-state/src/chain_index/non_finalised_state.rs @@ -835,7 +835,7 @@ impl NonFinalizedState { parent_chainwork: Option, network: Network, ) -> Result { - let (sapling_root, sapling_size, orchard_root, orchard_size, ironwood_root, ironwood_size) = + let (sapling_root, sapling_size, orchard_root, orchard_size, ironwood) = tree_roots.clone().extract_with_defaults(); let metadata = BlockMetadata::new( @@ -843,8 +843,7 @@ impl NonFinalizedState { sapling_size as u32, orchard_root, orchard_size as u32, - ironwood_root, - ironwood_size as u32, + ironwood.map(|(root, size)| (root, size as u32)), parent_chainwork, network, ); diff --git a/packages/zaino-state/src/chain_index/tests/finalised_state/v1.rs b/packages/zaino-state/src/chain_index/tests/finalised_state/v1.rs index ec8e54179..177979f7a 100644 --- a/packages/zaino-state/src/chain_index/tests/finalised_state/v1.rs +++ b/packages/zaino-state/src/chain_index/tests/finalised_state/v1.rs @@ -307,8 +307,7 @@ async fn try_write_invalid_block() { sapling_tree_size as u32, orchard_root, orchard_tree_size as u32, - zebra_chain::orchard::tree::Root::default(), - 0, + None, None, // no parent chainwork for this test zaino_common::Network::Regtest(ActivationHeights::default()).to_zebra_network(), ); diff --git a/packages/zaino-state/src/chain_index/tests/vectors.rs b/packages/zaino-state/src/chain_index/tests/vectors.rs index 54bba2342..0e8a3ceae 100644 --- a/packages/zaino-state/src/chain_index/tests/vectors.rs +++ b/packages/zaino-state/src/chain_index/tests/vectors.rs @@ -97,8 +97,7 @@ pub(crate) fn indexed_block_chain( vector.sapling_tree_size as u32, vector.orchard_root, vector.orchard_tree_size as u32, - zebra_chain::orchard::tree::Root::default(), - 0, + None, parent_chain_work, zebra_chain::parameters::Network::new_regtest( zebra_chain::parameters::testnet::ConfiguredActivationHeights { diff --git a/packages/zaino-state/src/chain_index/types/helpers.rs b/packages/zaino-state/src/chain_index/types/helpers.rs index a3118facb..dbfb5297a 100644 --- a/packages/zaino-state/src/chain_index/types/helpers.rs +++ b/packages/zaino-state/src/chain_index/types/helpers.rs @@ -79,6 +79,10 @@ impl TreeRootData { } /// Extract with defaults for genesis/sync use case + /// + /// Sapling and orchard roots default when absent (genesis). The ironwood component + /// passes through unchanged: `None` means the block has no ironwood treestate and + /// must be stored as `None`. pub fn extract_with_defaults( self, ) -> ( @@ -86,19 +90,16 @@ impl TreeRootData { u64, zebra_chain::orchard::tree::Root, u64, - zebra_chain::orchard::tree::Root, - u64, + Option<(zebra_chain::orchard::tree::Root, u64)>, ) { let (sapling_root, sapling_size) = self.sapling.unwrap_or_default(); let (orchard_root, orchard_size) = self.orchard.unwrap_or_default(); - let (ironwood_root, ironwood_size) = self.ironwood.unwrap_or_default(); ( sapling_root, sapling_size, orchard_root, orchard_size, - ironwood_root, - ironwood_size, + self.ironwood, ) } } @@ -114,10 +115,9 @@ pub struct BlockMetadata { pub orchard_root: zebra_chain::orchard::tree::Root, /// Orchard tree size pub orchard_size: u32, - /// Orchard commitment tree root - pub ironwood_root: zebra_chain::orchard::tree::Root, - /// Orchard tree size - pub ironwood_size: u32, + /// Ironwood commitment tree root and size; `None` when the block has no ironwood + /// treestate (below NU6.3 activation, or a network with no NU6.3 activation height) + pub ironwood: Option<(zebra_chain::orchard::tree::Root, u32)>, /// Parent block's chainwork (`None` for genesis). pub parent_chainwork: Option, /// Network for block validation @@ -126,14 +126,12 @@ pub struct BlockMetadata { impl BlockMetadata { /// Create new block metadata - #[allow(clippy::too_many_arguments)] pub fn new( sapling_root: zebra_chain::sapling::tree::Root, sapling_size: u32, orchard_root: zebra_chain::orchard::tree::Root, orchard_size: u32, - ironwood_root: zebra_chain::orchard::tree::Root, - ironwood_size: u32, + ironwood: Option<(zebra_chain::orchard::tree::Root, u32)>, parent_chainwork: Option, network: zebra_chain::parameters::Network, ) -> Self { @@ -142,8 +140,7 @@ impl BlockMetadata { sapling_size, orchard_root, orchard_size, - ironwood_root, - ironwood_size, + ironwood, parent_chainwork, network, } @@ -358,13 +355,13 @@ impl BlockMetadata { let commitment_tree_roots = super::db::CommitmentTreeRoots::new( <[u8; 32]>::from(self.sapling_root), <[u8; 32]>::from(self.orchard_root), - Some(<[u8; 32]>::from(self.ironwood_root)), + self.ironwood.map(|(root, _)| <[u8; 32]>::from(root)), ); let commitment_tree_size = super::db::CommitmentTreeSizes::new( self.sapling_size, self.orchard_size, - self.ironwood_size, + self.ironwood.map_or(0, |(_, size)| size), ); super::db::CommitmentTreeData::new(commitment_tree_roots, commitment_tree_size) @@ -401,19 +398,16 @@ mod create_commitment_tree_data { /// `extract_with_defaults` and stored `Some([0; 32])`, so a freshly synced database /// and a migrated database encoded identical pre-activation heights differently. /// - /// `should_panic` tracks the known bug; remove it together with the fix. #[test] - #[should_panic(expected = "ironwood root must be stored as None")] fn absent_ironwood_root_is_stored_as_none() { - let (sapling_root, sapling_size, orchard_root, orchard_size, ironwood_root, ironwood_size) = + let (sapling_root, sapling_size, orchard_root, orchard_size, ironwood) = TreeRootData::new(None, None, None).extract_with_defaults(); let metadata = BlockMetadata::new( sapling_root, sapling_size as u32, orchard_root, orchard_size as u32, - ironwood_root, - ironwood_size as u32, + ironwood.map(|(root, size)| (root, size as u32)), None, zebra_chain::parameters::Network::Mainnet, ); From 8cb29413c8ed388324591d781d5c1e2c256178f7 Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 14:55:53 -0700 Subject: [PATCH 048/134] test(zaino-fetch): pin five-pool valuePools deserialization (known red) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit value_pools is a fixed [ChainBalance; 6] in GetBlockchainInfoResponse (and Option<[ChainBalance; 6]> in BlockObject), so getblockchaininfo and verbose getblock hard-fail with "invalid length 5, expected an array of length 6" against any validator that predates Ironwood — zcashd (the zcashd_support target) and unpatched zebrad both report five pools. PR #1362 review finding; independently rediscovered from the devtool side, where released zainod's five-element array fails against six-pool zebra — the fixed-length array breaks in both directions. The regression test feeds a five-pool (zcashd-shaped, "lockbox") response; #[should_panic] tracks the known bug and is removed together with the length-tolerant fix in the next commit. Co-Authored-By: Claude Fable 5 --- .../zaino-fetch/src/jsonrpsee/response.rs | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/packages/zaino-fetch/src/jsonrpsee/response.rs b/packages/zaino-fetch/src/jsonrpsee/response.rs index d7c772b03..a6aef02cc 100644 --- a/packages/zaino-fetch/src/jsonrpsee/response.rs +++ b/packages/zaino-fetch/src/jsonrpsee/response.rs @@ -559,6 +559,48 @@ mod get_tx_out_set_info_tests { } } +#[cfg(test)] +mod get_blockchain_info_response { + use super::GetBlockchainInfoResponse; + + /// Regression test: a validator that predates Ironwood (zcashd, or an unpatched + /// zebrad) reports five value pools — no "ironwood" entry. The response must still + /// deserialize. `value_pools` was a fixed `[ChainBalance; 6]`, so every + /// getblockchaininfo against such a validator failed outright with + /// "invalid length 5, expected an array of length 6". + /// + /// `should_panic` tracks the known bug; remove it together with the fix. + #[test] + #[should_panic(expected = "five-pool valuePools must deserialize")] + fn parses_five_value_pools() { + let json = r#"{ + "chain": "main", + "blocks": 2, + "bestblockhash": "0000000000000000000000000000000000000000000000000000000000000002", + "estimatedheight": 2, + "chainSupply": { + "chainValue": 0.0, + "chainValueZat": 0 + }, + "upgrades": {}, + "valuePools": [ + { "id": "transparent", "chainValue": 0.0, "chainValueZat": 0 }, + { "id": "sprout", "chainValue": 0.0, "chainValueZat": 0 }, + { "id": "sapling", "chainValue": 0.055, "chainValueZat": 5500000 }, + { "id": "orchard", "chainValue": 0.0, "chainValueZat": 0 }, + { "id": "lockbox", "chainValue": 0.0, "chainValueZat": 0 } + ], + "consensus": { + "chaintip": "c2d6d0b4", + "nextblock": "c2d6d0b4" + } + }"#; + + serde_json::from_str::(json) + .expect("five-pool valuePools must deserialize (pre-Ironwood validator)"); + } +} + fn default_header() -> Height { Height(0) } From 857072d9a259fc495ad47d722821b1bb729215ab Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 14:57:48 -0700 Subject: [PATCH 049/134] fix(zaino-fetch): accept any number of valuePools from the validator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit value_pools becomes Vec in GetBlockchainInfoResponse and BlockObject, replacing the fixed [ChainBalance; 6] that hard-failed serde with "invalid length 5, expected an array of length 6" against pre-Ironwood validators (zcashd via zcashd_support, unpatched zebrad) — and, mirrored, made released five-element zainod fail against six-pool zebra. A fixed-length array was the wrong shape for a field that grows with network upgrades. The new value_pools_array helper converts to zebra's canonical six-pool array by matching entries on pool id: pools the validator did not report keep zero balances (a pre-NU6.3 validator has no ironwood entry), the validator's ordering becomes irrelevant (zebra's canonical order puts ironwood last, unlike the treestate-era fixtures), and ids outside the six canonical pools are ignored. Both consumers (the zebra GetBlockchainInfoResponse conversion and the BlockObject positional destructure) route through it. Flips the regression test from the previous commit green (#[should_panic] removed) and extends it: the sapling balance lands in its slot, zcashd's "lockbox" lands in the deferred slot (zebra's canonical id for that slot is also "lockbox"), and the unreported ironwood slot is zero-back-filled. PR #1362 review finding. Co-Authored-By: Claude Fable 5 --- .../zaino-fetch/src/jsonrpsee/response.rs | 48 ++++++++++++------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/packages/zaino-fetch/src/jsonrpsee/response.rs b/packages/zaino-fetch/src/jsonrpsee/response.rs index a6aef02cc..b63fe58f5 100644 --- a/packages/zaino-fetch/src/jsonrpsee/response.rs +++ b/packages/zaino-fetch/src/jsonrpsee/response.rs @@ -361,7 +361,7 @@ pub struct GetBlockchainInfoResponse { /// Value pool balances #[serde(rename = "valuePools")] - value_pools: [ChainBalance; 6], + value_pools: Vec, /// Branch IDs of the current and upcoming consensus rules pub consensus: zebra_rpc::methods::TipConsensusBranch, @@ -569,9 +569,7 @@ mod get_blockchain_info_response { /// getblockchaininfo against such a validator failed outright with /// "invalid length 5, expected an array of length 6". /// - /// `should_panic` tracks the known bug; remove it together with the fix. #[test] - #[should_panic(expected = "five-pool valuePools must deserialize")] fn parses_five_value_pools() { let json = r#"{ "chain": "main", @@ -596,8 +594,18 @@ mod get_blockchain_info_response { } }"#; - serde_json::from_str::(json) + let parsed = serde_json::from_str::(json) .expect("five-pool valuePools must deserialize (pre-Ironwood validator)"); + + let pools = super::value_pools_array(parsed.value_pools); + // Zebra's canonical order: transparent, sprout, sapling, orchard, deferred, ironwood. + assert_eq!(pools[2].id(), "sapling"); + assert_eq!(pools[2].chain_value_zat().zatoshis(), 5_500_000); + // zcashd's "lockbox" entry lands in the deferred slot (zebra also names it "lockbox"). + assert_eq!(pools[4].id(), "lockbox"); + // The pool the validator did not report is back-filled with a zero balance. + assert_eq!(pools[5].id(), "ironwood"); + assert_eq!(pools[5].chain_value_zat().zatoshis(), 0); } } @@ -721,6 +729,21 @@ impl Default for ChainBalance { } } +/// Fills zebra's canonical six-pool array from however many pools the validator +/// reported, leaving zero balances for pools the validator does not know about +/// (a pre-Ironwood validator reports five: no "ironwood" entry). Entries are +/// matched by pool id, so the validator's ordering is irrelevant; ids outside +/// zebra's six canonical pools are ignored. +fn value_pools_array(pools: Vec) -> [GetBlockchainInfoBalance; 6] { + let mut canonical = GetBlockchainInfoBalance::zero_pools(); + for ChainBalance(pool) in pools { + if let Some(slot) = canonical.iter_mut().find(|slot| slot.id() == pool.id()) { + *slot = pool; + } + } + canonical +} + impl TryFrom for zebra_rpc::methods::GetBlockchainInfoResponse { fn try_from(response: GetBlockchainInfoResponse) -> Result { Ok(zebra_rpc::methods::GetBlockchainInfoResponse::new( @@ -729,7 +752,7 @@ impl TryFrom for zebra_rpc::methods::GetBlockchainInf response.best_block_hash, response.estimated_height, response.chain_supply.0, - response.value_pools.map(|pool| pool.0), + value_pools_array(response.value_pools), response.upgrades, response.consensus, response.headers, @@ -1200,7 +1223,7 @@ pub struct BlockObject { /// Value pool balances /// #[serde(rename = "valuePools")] - value_pools: Option<[ChainBalance; 6]>, + value_pools: Option>, /// Information about the note commitment trees. pub trees: GetBlockTrees, @@ -1260,18 +1283,7 @@ impl TryFrom for zebra_rpc::methods::GetBlock { block.bits, block.difficulty, block.chain_supply.map(|supply| supply.0), - block.value_pools.map( - |[transparent, sprout, sapling, orchard, ironwood, deferred]| { - [ - transparent.0, - sprout.0, - sapling.0, - orchard.0, - ironwood.0, - deferred.0, - ] - }, - ), + block.value_pools.map(value_pools_array), block.trees.into(), block.previous_block_hash.map(|hash| hash.0), block.next_block_hash.map(|hash| hash.0), From 77753e1256a69da830354c87499d8417c3c3e058 Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 15:05:49 -0700 Subject: [PATCH 050/134] test(chain-index): pin sparse ironwood rows (known red) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ironwood table is sparse by design: every reader treats an absent row as "no ironwood data" (get_block_ironwood returns an empty list; the point lookup, compact-block assembly, validation, and delete are all NotFound-tolerant). The write path nevertheless writes an all-None OrchardTxList for every block — one serialization plus one LMDB put per block across the whole pre-NU6.3 chain, carrying zero information. PR #1362 review finding. The regression test syncs the (pre-NU6.3) test vectors and asserts a block without ironwood data reads back an empty list — an absent row — rather than one None per transaction, which betrays a stored row. #[should_panic] tracks the known bug and is removed together with the fix in the next commit. Co-Authored-By: Claude Fable 5 --- .../chain_index/tests/finalised_state/v1.rs | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/packages/zaino-state/src/chain_index/tests/finalised_state/v1.rs b/packages/zaino-state/src/chain_index/tests/finalised_state/v1.rs index 177979f7a..831a9f272 100644 --- a/packages/zaino-state/src/chain_index/tests/finalised_state/v1.rs +++ b/packages/zaino-state/src/chain_index/tests/finalised_state/v1.rs @@ -91,6 +91,38 @@ pub(crate) async fn load_vectors_v1db_and_reader() -> ( // *** FinalisedState Tests *** +/// Regression test: blocks with no ironwood data must have NO ironwood row. +/// +/// The ironwood table is sparse — readers treat an absent row as "no ironwood data" +/// (an absent row reads back as an empty list; a stored all-`None` list reads back +/// with one `None` per transaction). The write path instead wrote an all-`None` +/// `OrchardTxList` for every block, paying one serialization + LMDB put per block +/// across the entire pre-NU6.3 chain for zero information. +/// +/// The test vectors are pre-NU6.3 regtest blocks, so no block carries ironwood data. +/// +/// `should_panic` tracks the known bug; remove it together with the fix. +/// +/// multi_thread required: DbV1 reads run under `tokio::task::block_in_place`. +#[tokio::test(flavor = "multi_thread")] +#[should_panic(expected = "no ironwood row may be written")] +async fn no_ironwood_row_for_blocks_without_ironwood_data() { + init_tracing(); + + let (_test_vector_data, _db_dir, _zaino_db, db_reader) = load_vectors_v1db_and_reader().await; + + let ironwood_list = db_reader + .get_block_ironwood(crate::Height(1)) + .await + .unwrap(); + assert!( + ironwood_list.tx().is_empty(), + "no ironwood row may be written for a block without ironwood data \ + (read back {} entries; an absent row reads back as an empty list)", + ironwood_list.tx().len(), + ); +} + #[tokio::test(flavor = "multi_thread")] async fn shutdown_returns_promptly() { super::assert_shutdown_returns_promptly("DbV1", spawn_v1_zaino_db).await; From c85ebe196f603e51b5cd70d393105614ed6a357f Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 15:07:44 -0700 Subject: [PATCH 051/134] fix(chain-index): write ironwood rows only for blocks with ironwood data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both write paths (write_block and the batch sync loop) now skip the ironwood put when every entry in the block's list is None, keyed off the data itself rather than activation heights: pre-NU6.3 blocks and post-activation blocks whose transactions carry no ironwood actions both produce all-None lists that readers already treat as identical to an absent row (every read, validation, and delete path is NotFound-tolerant by design). A full-chain sync previously paid one serialization plus one LMDB put per block across the entire pre-NU6.3 chain — millions of writes of a version byte, CompactSize, presence bytes, and a 32-byte checksum carrying zero information. The table's invariant is now uniform: a row exists iff the block has ironwood data. Flips the regression test from the previous commit green (#[should_panic] removed). Full zaino-state suite green (180 tests). PR #1362 review finding. Co-Authored-By: Claude Fable 5 --- .../finalised_source/v1/write_core.rs | 43 ++++++++++++------- .../chain_index/tests/finalised_state/v1.rs | 3 -- 2 files changed, 28 insertions(+), 18 deletions(-) diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/write_core.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/write_core.rs index cb4d21e4e..d165756a9 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/write_core.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/write_core.rs @@ -650,7 +650,13 @@ impl DbV1 { StoredEntryVar::new(&block_height_bytes, TransparentTxList::new(transparent)); let sapling_entry = StoredEntryVar::new(&block_height_bytes, SaplingTxList::new(sapling)); let orchard_entry = StoredEntryVar::new(&block_height_bytes, OrchardTxList::new(orchard)); - let ironwood_entry = StoredEntryVar::new(&block_height_bytes, OrchardTxList::new(ironwood)); + // The ironwood table is sparse: readers treat an absent row as "no ironwood + // data", so an all-`None` list (every pre-NU6.3 block, and any later block + // whose transactions carry no ironwood actions) is not written. + let ironwood_entry = ironwood + .iter() + .any(Option::is_some) + .then(|| StoredEntryVar::new(&block_height_bytes, OrchardTxList::new(ironwood))); // if any database writes fail, or block validation fails, remove block from database and return err. let zaino_db = self.detached_handle(); @@ -711,12 +717,14 @@ impl DbV1 { WriteFlags::NO_OVERWRITE, )?; - txn.put( - zaino_db.ironwood, - &block_height_bytes, - &ironwood_entry.to_bytes()?, - WriteFlags::NO_OVERWRITE, - )?; + if let Some(ironwood_entry) = &ironwood_entry { + txn.put( + zaino_db.ironwood, + &block_height_bytes, + &ironwood_entry.to_bytes()?, + WriteFlags::NO_OVERWRITE, + )?; + } txn.put( zaino_db.commitment_tree_data, @@ -1226,8 +1234,11 @@ impl DbV1 { StoredEntryVar::new(&block_height_bytes, SaplingTxList::new(sapling)); let orchard_entry = StoredEntryVar::new(&block_height_bytes, OrchardTxList::new(orchard)); - let ironwood_entry = - StoredEntryVar::new(&block_height_bytes, OrchardTxList::new(ironwood)); + // Sparse ironwood row: see `write_block` — all-`None` lists are not written. + let ironwood_entry = ironwood + .iter() + .any(Option::is_some) + .then(|| StoredEntryVar::new(&block_height_bytes, OrchardTxList::new(ironwood))); // Height-keyed tables (+ the hash-keyed `heights`, one entry/block) written per block. put_idempotent( @@ -1266,12 +1277,14 @@ impl DbV1 { &block_height_bytes, &orchard_entry.to_bytes()?, )?; - put_idempotent( - &mut txn, - self.ironwood, - &block_height_bytes, - &ironwood_entry.to_bytes()?, - )?; + if let Some(ironwood_entry) = &ironwood_entry { + put_idempotent( + &mut txn, + self.ironwood, + &block_height_bytes, + &ironwood_entry.to_bytes()?, + )?; + } put_idempotent( &mut txn, self.commitment_tree_data, diff --git a/packages/zaino-state/src/chain_index/tests/finalised_state/v1.rs b/packages/zaino-state/src/chain_index/tests/finalised_state/v1.rs index 831a9f272..1ef3c02ce 100644 --- a/packages/zaino-state/src/chain_index/tests/finalised_state/v1.rs +++ b/packages/zaino-state/src/chain_index/tests/finalised_state/v1.rs @@ -101,11 +101,8 @@ pub(crate) async fn load_vectors_v1db_and_reader() -> ( /// /// The test vectors are pre-NU6.3 regtest blocks, so no block carries ironwood data. /// -/// `should_panic` tracks the known bug; remove it together with the fix. -/// /// multi_thread required: DbV1 reads run under `tokio::task::block_in_place`. #[tokio::test(flavor = "multi_thread")] -#[should_panic(expected = "no ironwood row may be written")] async fn no_ironwood_row_for_blocks_without_ironwood_data() { init_tracing(); From d60a3d9afbbdca395217519f8b155388c99aeadc Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 15:16:14 -0700 Subject: [PATCH 052/134] refactor(chain-index): one cursor walk for the shielded-pool point lookups MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DRY the finding-1 bug class out of block_shielded.rs: - get_sapling / get_orchard / get_ironwood collapse from three ~85-line verbatim walk copies to thin calls into one get_pool_tx helper, parameterized over the LMDB table, the pool name (error messages), the entry-skip function, and a MissingRow policy enum (Error for the dense sapling/orchard tables, NoPoolData for the sparse ironwood table). - The skip twins share skip_opt_tx_prelude (presence byte, version, Option value balance) and a new skip_counted_fixed_entries:: that reads the entry count and takes the skip width from the type being skipped — the wrong-width substitution that caused the sapling output-skip corruption (fixed in 44ebd43a) is now unrepresentable, since no free-floating length variable exists to reuse. Behaviour preserved: error strings, tolerance semantics, and byte arithmetic are unchanged; the skip-width regression test and the full zaino-state suite (180 tests) stay green. Net -128 lines. PR #1362 review finding 1 residue. Co-Authored-By: Claude Fable 5 --- .../finalised_source/v1/block_shielded.rs | 458 +++++++----------- 1 file changed, 166 insertions(+), 292 deletions(-) diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/block_shielded.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/block_shielded.rs index 0791f4cc5..4b53dd6ef 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/block_shielded.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/block_shielded.rs @@ -2,6 +2,16 @@ use super::*; +use crate::{FixedEncodedLen, ZainoVersionedSerde}; + +/// How a pool point lookup treats a block height with no row in the pool's table. +enum MissingRow { + /// Every indexed block has a row in this pool's table; an absent row is an error. + Error, + /// The table is sparse: an absent row means the block has no data for this pool. + NoPoolData, +} + /// [`BlockShieldedExt`] capability implementation for [`DbV1`]. /// /// Provides access to Sapling / Orchard compact transaction data and per-block commitment tree @@ -99,84 +109,13 @@ impl DbV1 { &self, tx_location: TxLocation, ) -> Result, FinalisedStateError> { - use std::io::{Cursor, Read}; - - tokio::task::block_in_place(|| { - let txn = self.env.begin_ro_txn()?; - - let height = Height::try_from(tx_location.block_height()) - .map_err(|e| FinalisedStateError::Custom(e.to_string()))?; - let height_bytes = height.to_bytes()?; - - let raw = match txn.get(self.sapling, &height_bytes) { - Ok(val) => val, - Err(lmdb::Error::NotFound) => { - return Err(FinalisedStateError::DataUnavailable( - "sapling data missing from db".into(), - )); - } - Err(e) => return Err(FinalisedStateError::LmdbError(e)), - }; - let mut cursor = Cursor::new(raw); - - // Skip [0] StoredEntry version - cursor.set_position(1); - - // Read CompactSize: length of serialized body - CompactSize::read(&mut cursor).map_err(|e| { - FinalisedStateError::Custom(format!("compact size read error: {e}")) - })?; - - // Skip SaplingTxList version byte - cursor.set_position(cursor.position() + 1); - - // Read CompactSize: number of entries - let list_len = CompactSize::read(&mut cursor).map_err(|e| { - FinalisedStateError::Custom(format!("sapling tx list len error: {e}")) - })?; - - let idx = tx_location.tx_index() as usize; - if idx >= list_len as usize { - return Err(FinalisedStateError::Custom( - "tx_index out of range in sapling tx list".to_string(), - )); - } - - // Skip preceding entries - for _ in 0..idx { - Self::skip_opt_sapling_entry(&mut cursor) - .map_err(|e| FinalisedStateError::Custom(format!("skip entry error: {e}")))?; - } - - let start = cursor.position(); - - // Peek presence flag - let mut presence = [0u8; 1]; - cursor.read_exact(&mut presence).map_err(|e| { - FinalisedStateError::Custom(format!("failed to read Option tag: {e}")) - })?; - - if presence[0] == 0 { - return Ok(None); - } else if presence[0] != 1 { - return Err(FinalisedStateError::Custom(format!( - "invalid Option tag: {}", - presence[0] - ))); - } - - // Rewind to include tag in returned bytes - cursor.set_position(start); - Self::skip_opt_sapling_entry(&mut cursor).map_err(|e| { - FinalisedStateError::Custom(format!("skip entry error (second pass): {e}")) - })?; - - let end = cursor.position(); - - Ok(Some(SaplingCompactTx::from_bytes( - &raw[start as usize..end as usize], - )?)) - }) + self.get_pool_tx( + self.sapling, + "sapling", + MissingRow::Error, + tx_location, + Self::skip_opt_sapling_entry, + ) } /// Fetch block sapling transaction data by height. @@ -268,85 +207,13 @@ impl DbV1 { &self, tx_location: TxLocation, ) -> Result, FinalisedStateError> { - use std::io::{Cursor, Read}; - - tokio::task::block_in_place(|| { - let txn = self.env.begin_ro_txn()?; - - let height = Height::try_from(tx_location.block_height()) - .map_err(|e| FinalisedStateError::Custom(e.to_string()))?; - let height_bytes = height.to_bytes()?; - - let raw = match txn.get(self.orchard, &height_bytes) { - Ok(val) => val, - Err(lmdb::Error::NotFound) => { - return Err(FinalisedStateError::DataUnavailable( - "orchard data missing from db".into(), - )); - } - Err(e) => return Err(FinalisedStateError::LmdbError(e)), - }; - - let mut cursor = Cursor::new(raw); - - // Skip [0] StoredEntry version - cursor.set_position(1); - - // Read CompactSize: length of serialized body - CompactSize::read(&mut cursor).map_err(|e| { - FinalisedStateError::Custom(format!("compact size read error: {e}")) - })?; - - // Skip OrchardTxList version byte - cursor.set_position(cursor.position() + 1); - - // Read CompactSize: number of entries - let list_len = CompactSize::read(&mut cursor).map_err(|e| { - FinalisedStateError::Custom(format!("orchard tx list len error: {e}")) - })?; - - let idx = tx_location.tx_index() as usize; - if idx >= list_len as usize { - return Err(FinalisedStateError::Custom( - "tx_index out of range in orchard tx list".to_string(), - )); - } - - // Skip preceding entries - for _ in 0..idx { - Self::skip_opt_orchard_entry(&mut cursor) - .map_err(|e| FinalisedStateError::Custom(format!("skip entry error: {e}")))?; - } - - let start = cursor.position(); - - // Peek presence flag - let mut presence = [0u8; 1]; - cursor.read_exact(&mut presence).map_err(|e| { - FinalisedStateError::Custom(format!("failed to read Option tag: {e}")) - })?; - - if presence[0] == 0 { - return Ok(None); - } else if presence[0] != 1 { - return Err(FinalisedStateError::Custom(format!( - "invalid Option tag: {}", - presence[0] - ))); - } - - // Rewind to include presence flag in output - cursor.set_position(start); - Self::skip_opt_orchard_entry(&mut cursor).map_err(|e| { - FinalisedStateError::Custom(format!("skip entry error (second pass): {e}")) - })?; - - let end = cursor.position(); - - Ok(Some(OrchardCompactTx::from_bytes( - &raw[start as usize..end as usize], - )?)) - }) + self.get_pool_tx( + self.orchard, + "orchard", + MissingRow::Error, + tx_location, + Self::skip_opt_orchard_entry, + ) } /// Fetch block orchard transaction data by height. @@ -440,82 +307,13 @@ impl DbV1 { &self, tx_location: TxLocation, ) -> Result, FinalisedStateError> { - use std::io::{Cursor, Read}; - - tokio::task::block_in_place(|| { - let txn = self.env.begin_ro_txn()?; - - let height = Height::try_from(tx_location.block_height()) - .map_err(|e| FinalisedStateError::Custom(e.to_string()))?; - let height_bytes = height.to_bytes()?; - - let raw = match txn.get(self.ironwood, &height_bytes) { - Ok(val) => val, - // No ironwood data at this height. - Err(lmdb::Error::NotFound) => return Ok(None), - Err(e) => return Err(FinalisedStateError::LmdbError(e)), - }; - - let mut cursor = Cursor::new(raw); - - // Skip [0] StoredEntry version - cursor.set_position(1); - - // Read CompactSize: length of serialized body - CompactSize::read(&mut cursor).map_err(|e| { - FinalisedStateError::Custom(format!("compact size read error: {e}")) - })?; - - // Skip OrchardTxList version byte - cursor.set_position(cursor.position() + 1); - - // Read CompactSize: number of entries - let list_len = CompactSize::read(&mut cursor).map_err(|e| { - FinalisedStateError::Custom(format!("ironwood tx list len error: {e}")) - })?; - - let idx = tx_location.tx_index() as usize; - if idx >= list_len as usize { - return Err(FinalisedStateError::Custom( - "tx_index out of range in ironwood tx list".to_string(), - )); - } - - // Skip preceding entries - for _ in 0..idx { - Self::skip_opt_orchard_entry(&mut cursor) - .map_err(|e| FinalisedStateError::Custom(format!("skip entry error: {e}")))?; - } - - let start = cursor.position(); - - // Peek presence flag - let mut presence = [0u8; 1]; - cursor.read_exact(&mut presence).map_err(|e| { - FinalisedStateError::Custom(format!("failed to read Option tag: {e}")) - })?; - - if presence[0] == 0 { - return Ok(None); - } else if presence[0] != 1 { - return Err(FinalisedStateError::Custom(format!( - "invalid Option tag: {}", - presence[0] - ))); - } - - // Rewind to include presence flag in output - cursor.set_position(start); - Self::skip_opt_orchard_entry(&mut cursor).map_err(|e| { - FinalisedStateError::Custom(format!("skip entry error (second pass): {e}")) - })?; - - let end = cursor.position(); - - Ok(Some(OrchardCompactTx::from_bytes( - &raw[start as usize..end as usize], - )?)) - }) + self.get_pool_tx( + self.ironwood, + "ironwood", + MissingRow::NoPoolData, + tx_location, + Self::skip_opt_orchard_entry, + ) } /// Fetch block ironwood transaction data by height. @@ -659,24 +457,113 @@ impl DbV1 { // *** Internal DB methods *** - /// Skips one `Option` from the current cursor position. - /// - /// The input should be a cursor over just the inner item "list" bytes of a: - /// - `StoredEntryVar` + /// Point lookup for one transaction's compact data in a per-block pool table, + /// without decoding the whole block row. /// - /// Advances past: - /// - 1 byte `0x00` if None, or - /// - 1 + 1 + value + spends + outputs if Some (presence + version + body) - /// - /// This is faster than deserialising the whole struct as we only read the compact sizes. + /// Walks the `StoredEntryVar` tx-list bytes entry-by-entry with `skip_entry`, + /// then decodes only the requested entry. `pool` names the table in error + /// messages; `missing_row` selects how an absent row is treated. + fn get_pool_tx( + &self, + table: lmdb::Database, + pool: &str, + missing_row: MissingRow, + tx_location: TxLocation, + skip_entry: fn(&mut std::io::Cursor<&[u8]>) -> io::Result<()>, + ) -> Result, FinalisedStateError> { + use std::io::{Cursor, Read}; + + tokio::task::block_in_place(|| { + let txn = self.env.begin_ro_txn()?; + + let height = Height::try_from(tx_location.block_height()) + .map_err(|e| FinalisedStateError::Custom(e.to_string()))?; + let height_bytes = height.to_bytes()?; + + let raw = match txn.get(table, &height_bytes) { + Ok(val) => val, + Err(lmdb::Error::NotFound) => { + return match missing_row { + MissingRow::NoPoolData => Ok(None), + MissingRow::Error => Err(FinalisedStateError::DataUnavailable(format!( + "{pool} data missing from db" + ))), + }; + } + Err(e) => return Err(FinalisedStateError::LmdbError(e)), + }; + + let mut cursor = Cursor::new(raw); + + // Skip [0] StoredEntry version + cursor.set_position(1); + + // Read CompactSize: length of serialized body + CompactSize::read(&mut cursor).map_err(|e| { + FinalisedStateError::Custom(format!("compact size read error: {e}")) + })?; + + // Skip the tx-list version byte + cursor.set_position(cursor.position() + 1); + + // Read CompactSize: number of entries + let list_len = CompactSize::read(&mut cursor).map_err(|e| { + FinalisedStateError::Custom(format!("{pool} tx list len error: {e}")) + })?; + + let idx = tx_location.tx_index() as usize; + if idx >= list_len as usize { + return Err(FinalisedStateError::Custom(format!( + "tx_index out of range in {pool} tx list" + ))); + } + + // Skip preceding entries + for _ in 0..idx { + skip_entry(&mut cursor) + .map_err(|e| FinalisedStateError::Custom(format!("skip entry error: {e}")))?; + } + + let start = cursor.position(); + + // Peek presence flag + let mut presence = [0u8; 1]; + cursor.read_exact(&mut presence).map_err(|e| { + FinalisedStateError::Custom(format!("failed to read Option tag: {e}")) + })?; + + if presence[0] == 0 { + return Ok(None); + } else if presence[0] != 1 { + return Err(FinalisedStateError::Custom(format!( + "invalid Option tag: {}", + presence[0] + ))); + } + + // Rewind to include the presence flag in the returned bytes + cursor.set_position(start); + skip_entry(&mut cursor).map_err(|e| { + FinalisedStateError::Custom(format!("skip entry error (second pass): {e}")) + })?; + + let end = cursor.position(); + + Ok(Some(T::from_bytes(&raw[start as usize..end as usize])?)) + }) + } + + /// Skips the shared prelude of one `Option` entry: the presence + /// byte, and — when the entry is present — the version byte and the `Option` + /// value balance. Returns `false` when the entry was `None` (nothing more to skip). #[inline] - fn skip_opt_sapling_entry(cursor: &mut std::io::Cursor<&[u8]>) -> io::Result<()> { + fn skip_opt_tx_prelude(cursor: &mut std::io::Cursor<&[u8]>) -> io::Result { // Read presence byte let mut presence = [0u8; 1]; cursor.read_exact(&mut presence)?; if presence[0] == 0 { - return Ok(()); + return Ok(false); } else if presence[0] != 1 { return Err(io::Error::new( io::ErrorKind::InvalidData, @@ -700,25 +587,46 @@ impl DbV1 { )); } - // Read number of spends (CompactSize) - let spend_len = CompactSize::read(&mut *cursor)? as usize; - let sapling_spend_len = CompactSaplingSpend::latest_versioned_len()?; - let spend_skip = spend_len * sapling_spend_len; - cursor.set_position(cursor.position() + spend_skip as u64); - - // Read number of outputs (CompactSize) - let output_len = CompactSize::read(&mut *cursor)? as usize; - let sapling_output_len = crate::CompactSaplingOutput::latest_versioned_len()?; - let output_skip = output_len * sapling_output_len; - cursor.set_position(cursor.position() + output_skip as u64); + Ok(true) + } + /// Advances the cursor past a CompactSize entry count followed by that many + /// fixed-length entries of type `T`. Taking the width from the type being + /// skipped makes a wrong-width skip unrepresentable. + #[inline] + fn skip_counted_fixed_entries( + cursor: &mut std::io::Cursor<&[u8]>, + ) -> io::Result<()> { + let count = CompactSize::read(&mut *cursor)? as usize; + let entry_len = T::latest_versioned_len()?; + cursor.set_position(cursor.position() + (count * entry_len) as u64); Ok(()) } + /// Skips one `Option` from the current cursor position. + /// + /// The input should be a cursor over just the inner item "list" bytes of a: + /// - `StoredEntryVar` + /// + /// Advances past: + /// - 1 byte `0x00` if None, or + /// - 1 + 1 + value + spends + outputs if Some (presence + version + body) + /// + /// This is faster than deserialising the whole struct as we only read the compact sizes. + #[inline] + fn skip_opt_sapling_entry(cursor: &mut std::io::Cursor<&[u8]>) -> io::Result<()> { + if !Self::skip_opt_tx_prelude(cursor)? { + return Ok(()); + } + Self::skip_counted_fixed_entries::(cursor)?; + Self::skip_counted_fixed_entries::(cursor) + } + /// Skips one `Option` from the current cursor position. /// /// The input should be a cursor over just the inner item "list" bytes of a: - /// - `StoredEntryVar` + /// - `StoredEntryVar` (the orchard and ironwood tables share this + /// layout) /// /// Advances past: /// - 1 byte `0x00` if None, or @@ -727,44 +635,10 @@ impl DbV1 { /// This is faster than deserialising the whole struct as we only read the compact sizes. #[inline] fn skip_opt_orchard_entry(cursor: &mut std::io::Cursor<&[u8]>) -> io::Result<()> { - // Read presence byte - let mut presence = [0u8; 1]; - cursor.read_exact(&mut presence)?; - - if presence[0] == 0 { + if !Self::skip_opt_tx_prelude(cursor)? { return Ok(()); - } else if presence[0] != 1 { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!("invalid Option tag: {}", presence[0]), - )); } - - // Read version - cursor.read_exact(&mut [0u8; 1])?; - - // Read value: Option - let mut value_tag = [0u8; 1]; - cursor.read_exact(&mut value_tag)?; - if value_tag[0] == 1 { - // Some(i64): read 8 bytes - cursor.set_position(cursor.position() + 8); - } else if value_tag[0] != 0 { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!("invalid Option tag: {}", value_tag[0]), - )); - } - - // Read number of actions (CompactSize) - let action_len = CompactSize::read(&mut *cursor)? as usize; - - // Skip actions: each is 1-byte version + 148-byte body - let orchard_action_len = CompactOrchardAction::latest_versioned_len()?; - let action_skip = action_len * orchard_action_len; - cursor.set_position(cursor.position() + action_skip as u64); - - Ok(()) + Self::skip_counted_fixed_entries::(cursor) } } From 5bd1abc7b526b29ed5b3473bad88c11b84fabb79 Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 15:16:48 -0700 Subject: [PATCH 053/134] chore: version-mark Ironwood builds and document the NU6.3 work per package zainod 0.4.2 -> 0.4.3-ironwood.1: three behaviorally different binaries (stock 0.4.2, and successive builds of this branch) all printed "zainod 0.4.2", which cost real diagnosis time distinguishing them in test logs. Backfill CHANGELOG entries for the Ironwood work in zaino-fetch (V6 parsing via zebra-chain 11, ironwoodActions extraction), zaino-proto (lightwallet-protocol v0.5.0 subtree, include_ironwood default), zaino-state (ironwood treestate roots), zaino-common (ActivationHeights.nu6_3), and zainod (feature summary). Co-Authored-By: Claude Fable 5 --- Cargo.lock | 2 +- packages/zaino-common/CHANGELOG.md | 3 +++ packages/zaino-fetch/CHANGELOG.md | 7 +++++++ packages/zaino-proto/CHANGELOG.md | 7 ++++++- packages/zaino-state/CHANGELOG.md | 3 +++ packages/zainod/CHANGELOG.md | 8 ++++++++ packages/zainod/Cargo.toml | 2 +- 7 files changed, 29 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 14b12576e..278632b78 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6092,7 +6092,7 @@ dependencies = [ [[package]] name = "zainod" -version = "0.4.2" +version = "0.4.3-ironwood.1" dependencies = [ "clap", "config", diff --git a/packages/zaino-common/CHANGELOG.md b/packages/zaino-common/CHANGELOG.md index ed7c022f5..e2356e551 100644 --- a/packages/zaino-common/CHANGELOG.md +++ b/packages/zaino-common/CHANGELOG.md @@ -8,6 +8,9 @@ and this library adheres to Rust's notion of ## [Unreleased] ### Added +- `ActivationHeights.nu6_3` (serde key `"NU6.3"`) for the NU6.3 network + upgrade. `ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS` currently leaves it `None` + (inactive); a chain with NU6.3 active needs the height stated explicitly. - `StorageConfig::database.sync_checkpoint_interval` (seconds, default 120) — max wall-clock time spent buffering a bulk-sync write batch before flushing. Under the env's `NO_SYNC` mode this also bounds the window of unflushed writes at risk diff --git a/packages/zaino-fetch/CHANGELOG.md b/packages/zaino-fetch/CHANGELOG.md index 458372e2d..ad29e80d3 100644 --- a/packages/zaino-fetch/CHANGELOG.md +++ b/packages/zaino-fetch/CHANGELOG.md @@ -8,7 +8,14 @@ and this library adheres to Rust's notion of ## [Unreleased] ### Added +- Ironwood (NU6.3) / V6 transaction support: ironwood value balances are + read from parsed transactions, and compact-block construction populates + `CompactTx.ironwoodActions` and the block's `ironwoodCommitmentTreeSize`. ### Changed +- Transaction parsing delegates to + `zebra_chain::transaction::Transaction::zcash_deserialize` (zebra-chain 11), + replacing the hand-rolled parser that rejected transactions above v5 + ("Unsupported tx version 6"). ### Deprecated ### Removed ### Fixed diff --git a/packages/zaino-proto/CHANGELOG.md b/packages/zaino-proto/CHANGELOG.md index 1b95592ec..b221a8fc9 100644 --- a/packages/zaino-proto/CHANGELOG.md +++ b/packages/zaino-proto/CHANGELOG.md @@ -8,8 +8,13 @@ and this library adheres to Rust's notion of ## [Unreleased] ### Added +- Pool-type filter serves Ironwood by default (`include_ironwood: true`), so + clients that predate the field still receive `ironwoodActions` (unknown + protobuf fields are carried harmlessly). ### Changed -- Lightwallet protocol vendored subtree updated toward v0.5.0. +- Lightwallet protocol vendored subtree updated to v0.5.0: + `CompactTx.ironwoodActions` (field 9, `CompactOrchardAction`-shaped) and + `CompactBlock.ironwoodCommitmentTreeSize`. ### Deprecated ### Removed ### Fixed diff --git a/packages/zaino-state/CHANGELOG.md b/packages/zaino-state/CHANGELOG.md index 2ed018bd9..66e1602c6 100644 --- a/packages/zaino-state/CHANGELOG.md +++ b/packages/zaino-state/CHANGELOG.md @@ -8,6 +8,9 @@ and this library adheres to Rust's notion of ## [Unreleased] ### Added +- The chain index tracks Ironwood (NU6.3) note-commitment treestate roots, + storing `None` while the pool has no treestate rather than fabricating a + root. - `ChainIndex` / `NodeBackedChainIndexSubscriber` gain `get_outpoint_spenders` — for each transparent `Outpoint`, returns the txid that spent it on the best chain (index-aligned with the input, `None` if unspent or unknown). diff --git a/packages/zainod/CHANGELOG.md b/packages/zainod/CHANGELOG.md index df864132a..130408b7d 100644 --- a/packages/zainod/CHANGELOG.md +++ b/packages/zainod/CHANGELOG.md @@ -9,6 +9,11 @@ and this crate adheres to Rust's notion of ## [Unreleased] ### Added +- Ironwood (NU6.3) / V6 transaction support, end to end through the + workspace crates: V6 parsing and ironwood extraction (`zaino-fetch`), + `ironwoodActions` in served compact blocks (`zaino-proto`, on by + default), and ironwood treestate roots in the chain index + (`zaino-state`). - `[storage.database]` config gains `sync_checkpoint_interval` (seconds, default 120) — the bulk-sync write-batch flush interval, which also bounds the window of unflushed (`NO_SYNC`) writes at risk on a hard kill / eviction. @@ -16,6 +21,9 @@ and this crate adheres to Rust's notion of default 8) — a dedicated heap budget for the txout-set accumulator rebuild, separate from `sync_write_batch_size`. ### Changed +- Version marked `0.4.3-ironwood.1` so Ironwood feature builds identify + themselves at `zainod --version`; stock `0.4.2` binaries are otherwise + indistinguishable from this branch's builds. - **Breaking** — `[storage.database] sync_write_batch_bytes` (bytes) is renamed to `sync_write_batch_size` and is now given in **GiB** (default 8). It now budgets only the bulk-sync block buffer; the accumulator rebuild uses the new diff --git a/packages/zainod/Cargo.toml b/packages/zainod/Cargo.toml index 65100f838..4fc9a786d 100644 --- a/packages/zainod/Cargo.toml +++ b/packages/zainod/Cargo.toml @@ -6,7 +6,7 @@ repository = { workspace = true } homepage = { workspace = true } edition = { workspace = true } license = { workspace = true } -version = "0.4.2" +version = "0.4.3-ironwood.1" [[bin]] name = "zainod" From 1ffe6e887836e99f2ec4bdc05f86da078e915183 Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 15:55:14 -0700 Subject: [PATCH 054/134] test(chain-index): plumb per-pool treestates; pin finalRoot pass-through (known red) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce PoolTreestate { final_root, final_state } and carry it through BlockchainSource::get_treestate, the ValidatorConnector arms, the mockchain/proptest sources, and ChainIndex::get_treestate, in place of the bytes-only 3-tuple. Behaviour-faithful: every producer still sets final_root: None, exactly what the tuple encoded. The duplicated z_gettreestate construction in the Fetch and State backends collapses into one shared backends::build_treestate_response (PR #1362 review cleanup: the two ~25-line blocks previously had to evolve in lockstep). The known gap this stages (PR #1362 review finding 4 residue): zebra populates Commitments { finalRoot, finalState } for every pool, but zaino's slot mapping drops the root, so z_gettreestate responses carry no finalRoot for any pool — sapling and orchard included; this was never ironwood-specific. The regression test pins the pass-through contract on fetch_pool_treestate_slot; #[should_panic] tracks the gap and is removed together with the fix in the next commit. Co-Authored-By: Claude Fable 5 --- packages/zaino-state/src/backends.rs | 42 +++++++ packages/zaino-state/src/backends/fetch.rs | 28 +---- packages/zaino-state/src/backends/state.rs | 28 +---- packages/zaino-state/src/chain_index.rs | 18 ++- .../zaino-state/src/chain_index/source.rs | 19 +++- .../chain_index/source/mockchain_source.rs | 15 ++- .../chain_index/source/validator_connector.rs | 107 ++++++++++++++---- .../src/chain_index/tests/mockchain_tests.rs | 8 +- .../chain_index/tests/proptest_blockgen.rs | 2 +- 9 files changed, 181 insertions(+), 86 deletions(-) diff --git a/packages/zaino-state/src/backends.rs b/packages/zaino-state/src/backends.rs index 019cfc5f1..353facce7 100644 --- a/packages/zaino-state/src/backends.rs +++ b/packages/zaino-state/src/backends.rs @@ -4,6 +4,48 @@ pub mod fetch; pub mod state; +/// Builds the `z_gettreestate` response shared by the Fetch and State backends from the +/// per-pool treestates the chain index reported. +/// +/// `Commitments::new(final_root, final_state)`: the note-commitment tree is the +/// `final_state`. The ironwood treestate is `Some` only from NU6.3 activation, so +/// pre-NU6.3 responses omit the field exactly as zebrad does. +fn build_treestate_response( + hash: zebra_chain::block::Hash, + height: zebra_chain::block::Height, + time: u32, + (sapling, orchard, ironwood): ( + Option, + Option, + Option, + ), +) -> zebra_rpc::client::GetTreestateResponse { + fn treestate( + pool: Option, + ) -> zebra_rpc::client::Treestate { + let (final_root, final_state) = match pool { + Some(pool) => (pool.final_root, Some(pool.final_state)), + None => (None, None), + }; + zebra_rpc::client::Treestate::new(zebra_rpc::client::Commitments::new( + final_root, + final_state, + )) + } + + let sprout_treestate = None; + let ironwood_treestate = ironwood.map(|pool| treestate(Some(pool))); + zebra_rpc::client::GetTreestateResponse::new( + hash, + height, + time, + sprout_treestate, + treestate(sapling), + treestate(orchard), + ironwood_treestate, + ) +} + fn latest_network_upgrade( upgrades: &indexmap::IndexMap< zebra_rpc::methods::ConsensusBranchIdHex, diff --git a/packages/zaino-state/src/backends/fetch.rs b/packages/zaino-state/src/backends/fetch.rs index f372b912b..c3b508c6f 100644 --- a/packages/zaino-state/src/backends/fetch.rs +++ b/packages/zaino-state/src/backends/fetch.rs @@ -605,8 +605,7 @@ impl ZcashIndexer for FetchServiceSubscriber { )?, }; - let (sapling, orchard, ironwood) = - self.indexer.get_treestate(block_data.hash()).await?; + let treestates = self.indexer.get_treestate(block_data.hash()).await?; let time: u32 = block_data.data().time().try_into().map_err(|_error| { #[allow(deprecated)] FetchServiceError::RpcError(RpcError::new_from_legacycode( @@ -615,32 +614,11 @@ impl ZcashIndexer for FetchServiceSubscriber { )) })?; - // `Commitments::new(final_root, final_state)`: the note-commitment tree is the - // `final_state`, matching the (now-deprecated) `from_parts` behaviour this replaces. - // The `new` constructor is used so the Ironwood (NU6.3) treestate can be included rather - // than discarded. - let sprout_treestate = None; - let sapling_treestate = zebra_rpc::client::Treestate::new( - zebra_rpc::client::Commitments::new(None, sapling), - ); - let orchard_treestate = zebra_rpc::client::Treestate::new( - zebra_rpc::client::Commitments::new(None, orchard), - ); - let ironwood_treestate = ironwood.map(|final_state| { - zebra_rpc::client::Treestate::new(zebra_rpc::client::Commitments::new( - None, - Some(final_state), - )) - }); - - Ok(GetTreestateResponse::new( + Ok(super::build_treestate_response( (*block_data.hash()).into(), block_data.height().into(), time, - sprout_treestate, - sapling_treestate, - orchard_treestate, - ironwood_treestate, + treestates, )) } .await; diff --git a/packages/zaino-state/src/backends/state.rs b/packages/zaino-state/src/backends/state.rs index 3d442a80b..de91a0406 100644 --- a/packages/zaino-state/src/backends/state.rs +++ b/packages/zaino-state/src/backends/state.rs @@ -1494,8 +1494,7 @@ impl ZcashIndexer for StateServiceSubscriber { )))?, }; - let (sapling, orchard, ironwood) = - self.indexer.get_treestate(block_data.hash()).await?; + let treestates = self.indexer.get_treestate(block_data.hash()).await?; let time: u32 = block_data.data().time().try_into().map_err(|_error| { StateServiceError::RpcError(RpcError::new_from_legacycode( zebra_rpc::server::error::LegacyCode::InvalidParameter, @@ -1503,32 +1502,11 @@ impl ZcashIndexer for StateServiceSubscriber { )) })?; - // `Commitments::new(final_root, final_state)`: the note-commitment tree is the - // `final_state`, matching the (now-deprecated) `from_parts` behaviour this replaces. - // The `new` constructor is used so the Ironwood (NU6.3) treestate can be included rather - // than discarded. - let sprout_treestate = None; - let sapling_treestate = zebra_rpc::client::Treestate::new( - zebra_rpc::client::Commitments::new(None, sapling), - ); - let orchard_treestate = zebra_rpc::client::Treestate::new( - zebra_rpc::client::Commitments::new(None, orchard), - ); - let ironwood_treestate = ironwood.map(|final_state| { - zebra_rpc::client::Treestate::new(zebra_rpc::client::Commitments::new( - None, - Some(final_state), - )) - }); - - Ok(GetTreestateResponse::new( + Ok(super::build_treestate_response( (*block_data.hash()).into(), block_data.height().into(), time, - sprout_treestate, - sapling_treestate, - orchard_treestate, - ironwood_treestate, + treestates, )) } .await; diff --git a/packages/zaino-state/src/chain_index.rs b/packages/zaino-state/src/chain_index.rs index 806b52903..8b2a59607 100644 --- a/packages/zaino-state/src/chain_index.rs +++ b/packages/zaino-state/src/chain_index.rs @@ -502,7 +502,14 @@ pub trait ChainIndex { &self, hash: &types::BlockHash, ) -> impl std::future::Future< - Output = Result<(Option>, Option>, Option>), Self::Error>, + Output = Result< + ( + Option, + Option, + Option, + ), + Self::Error, + >, >; /// Returns the subtree roots @@ -2404,7 +2411,14 @@ impl ChainIndex for NodeBackedChainIndexSubscriber Result<(Option>, Option>, Option>), Self::Error> { + ) -> Result< + ( + Option, + Option, + Option, + ), + Self::Error, + > { let snapshot = self.snapshot_nonfinalized_state().await?; if !self.block_hash_known_for_treestate(&snapshot, hash).await? { return Err(ChainIndexError::internal(format!( diff --git a/packages/zaino-state/src/chain_index/source.rs b/packages/zaino-state/src/chain_index/source.rs index bb5f3f3ee..9b5ea63f0 100644 --- a/packages/zaino-state/src/chain_index/source.rs +++ b/packages/zaino-state/src/chain_index/source.rs @@ -34,9 +34,22 @@ pub(crate) mod mockchain_source; pub mod validator_connector; pub use validator_connector::*; -/// Serialized sapling and orchard treestates `(sapling, orchard, ironwood)`, each `None` -/// when the pool has no treestate at the queried block. -pub(crate) type TreestateBytes = (Option>, Option>, Option>); +/// One pool's treestate for a block, as reported by the backing validator. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PoolTreestate { + /// The pool's note commitment tree root (32 bytes), when the validator reports one. + pub final_root: Option>, + /// The pool's serialized note commitment tree. + pub final_state: Vec, +} + +/// Per-pool treestates `(sapling, orchard, ironwood)`, each `None` when the pool has no +/// treestate at the queried block. +pub(crate) type TreestateBytes = ( + Option, + Option, + Option, +); /// Sapling and orchard note-commitment tree roots `(sapling, orchard, ironwood)`, each /// paired with its tree size; `None` when the pool has no root at the block. diff --git a/packages/zaino-state/src/chain_index/source/mockchain_source.rs b/packages/zaino-state/src/chain_index/source/mockchain_source.rs index e7728db6f..00cab444d 100644 --- a/packages/zaino-state/src/chain_index/source/mockchain_source.rs +++ b/packages/zaino-state/src/chain_index/source/mockchain_source.rs @@ -553,18 +553,21 @@ impl BlockchainSource for MockchainSource { /// Returns the sapling and orchard treestate by hash /// /// TODO: Update test vectors to support ironwood. - async fn get_treestate( - &self, - id: BlockHash, - ) -> BlockchainSourceResult<(Option>, Option>, Option>)> { + async fn get_treestate(&self, id: BlockHash) -> BlockchainSourceResult { let active_chain_height = self.active_height() as usize; // serve up to active tip if let Some(height) = self.hashes.iter().position(|h| h == &id) { if height <= active_chain_height { let (sapling_state, orchard_state) = &self.treestates[height]; Ok(( - Some(sapling_state.clone()), - Some(orchard_state.clone()), + Some(super::PoolTreestate { + final_root: None, + final_state: sapling_state.clone(), + }), + Some(super::PoolTreestate { + final_root: None, + final_state: orchard_state.clone(), + }), None, )) } else { diff --git a/packages/zaino-state/src/chain_index/source/validator_connector.rs b/packages/zaino-state/src/chain_index/source/validator_connector.rs index f69e4a73f..691ebc366 100644 --- a/packages/zaino-state/src/chain_index/source/validator_connector.rs +++ b/packages/zaino-state/src/chain_index/source/validator_connector.rs @@ -72,10 +72,23 @@ fn empty_orchard_tree_rpc_bytes() -> Vec { /// (below NU6.3 activation, or on a network with no NU6.3 activation height) the slot /// stays `None`, so the response omits the field exactly as zebrad does. This function /// names that contract — do not back-fill an empty tree here. -fn ironwood_treestate_slot(validator_ironwood: Option>) -> Option> { +fn ironwood_treestate_slot(validator_ironwood: Option) -> Option { validator_ironwood } +/// Maps a validator-reported treestate (from the JSON-RPC `z_gettreestate` response) +/// into the connector's pool slot. +fn fetch_pool_treestate_slot(treestate: zebra_rpc::client::Treestate) -> Option { + treestate + .commitments() + .final_state() + .clone() + .map(|final_state| PoolTreestate { + final_root: None, + final_state, + }) +} + impl BlockchainSource for ValidatorConnector { // ********** Block methods ********** @@ -392,7 +405,7 @@ impl BlockchainSource for ValidatorConnector { &self, // Sould this be HashOrHeight? id: BlockHash, - ) -> BlockchainSourceResult<(Option>, Option>, Option>)> { + ) -> BlockchainSourceResult { let hash_or_height: HashOrHeight = HashOrHeight::Hash(zebra_chain::block::Hash(id.into())); match self { ValidatorConnector::State(state) => { @@ -443,8 +456,10 @@ impl BlockchainSource for ValidatorConnector { _ => None, } .and_then(|sap_response| { - expected_read_response!(sap_response, SaplingTree) - .map(|tree| tree.to_rpc_bytes()) + expected_read_response!(sap_response, SaplingTree).map(|tree| PoolTreestate { + final_root: None, + final_state: tree.to_rpc_bytes(), + }) }); let orchard = match zebra_chain::parameters::NetworkUpgrade::Nu5 @@ -470,8 +485,10 @@ impl BlockchainSource for ValidatorConnector { _ => None, } .and_then(|orch_response| { - expected_read_response!(orch_response, OrchardTree) - .map(|tree| tree.to_rpc_bytes()) + expected_read_response!(orch_response, OrchardTree).map(|tree| PoolTreestate { + final_root: None, + final_state: tree.to_rpc_bytes(), + }) }); let ironwood = match zebra_chain::parameters::NetworkUpgrade::Nu6_3 @@ -497,8 +514,10 @@ impl BlockchainSource for ValidatorConnector { _ => None, } .and_then(|irw_response| { - expected_read_response!(irw_response, IronwoodTree) - .map(|tree| tree.to_rpc_bytes()) + expected_read_response!(irw_response, IronwoodTree).map(|tree| PoolTreestate { + final_root: None, + final_state: tree.to_rpc_bytes(), + }) }); let ironwood = ironwood_treestate_slot(ironwood); @@ -520,21 +539,26 @@ impl BlockchainSource for ValidatorConnector { let mut tree = vec![]; write_commitment_tree(&sapling_crypto::CommitmentTree::empty(), &mut tree) .expect("can write to Vec"); - Some(tree) + Some(PoolTreestate { + final_root: None, + final_state: tree, + }) }, - |t| t.commitments().final_state().clone(), + fetch_pool_treestate_slot, ); let orchard = treestate.orchard.map_or_else( - || Some(empty_orchard_tree_rpc_bytes()), - |t| t.commitments().final_state().clone(), + || { + Some(PoolTreestate { + final_root: None, + final_state: empty_orchard_tree_rpc_bytes(), + }) + }, + fetch_pool_treestate_slot, ); - let ironwood = ironwood_treestate_slot( - treestate - .ironwood - .and_then(|t| t.commitments().final_state().clone()), - ); + let ironwood = + ironwood_treestate_slot(treestate.ironwood.and_then(fetch_pool_treestate_slot)); Ok((sapling, orchard, ironwood)) } @@ -1175,6 +1199,46 @@ impl BlockchainSource for ValidatorConnector { } } +#[cfg(test)] +mod fetch_pool_treestate_slot { + use crate::chain_index::source::PoolTreestate; + + /// Regression test: the validator's finalRoot must pass through to the pool slot. + /// zebra populates `Commitments { finalRoot, finalState }` for every pool it + /// serves, but the slot mapping dropped the root, so zaino's z_gettreestate + /// responses carried no finalRoot for any pool. + /// + /// `should_panic` tracks the known gap; remove it together with the fix. + #[test] + #[should_panic(expected = "finalRoot must pass through")] + fn final_root_passes_through() { + let final_root = vec![7u8; 32]; + let final_state = vec![1u8, 2, 3]; + let treestate = zebra_rpc::client::Treestate::new(zebra_rpc::client::Commitments::new( + Some(final_root.clone()), + Some(final_state.clone()), + )); + + let slot = super::fetch_pool_treestate_slot(treestate) + .expect("a treestate with a finalState maps to a populated slot"); + + assert_eq!(slot.final_state, final_state); + assert_eq!( + slot.final_root, + Some(final_root), + "the validator's finalRoot must pass through to the treestate slot" + ); + } + + /// An absent finalState maps to an absent slot. + #[test] + fn absent_final_state_maps_to_absent_slot() { + let treestate = + zebra_rpc::client::Treestate::new(zebra_rpc::client::Commitments::new(None, None)); + assert_eq!(super::fetch_pool_treestate_slot(treestate), None); + } +} + #[cfg(test)] mod ironwood_treestate_slot { /// Regression test: when the validator reported no ironwood treestate (below NU6.3 @@ -1195,10 +1259,13 @@ mod ironwood_treestate_slot { /// A reported ironwood treestate passes through unchanged. #[test] fn reported_validator_ironwood_passes_through() { - let tree_bytes = vec![1u8, 2, 3]; + let treestate = crate::chain_index::source::PoolTreestate { + final_root: None, + final_state: vec![1u8, 2, 3], + }; assert_eq!( - super::ironwood_treestate_slot(Some(tree_bytes.clone())), - Some(tree_bytes) + super::ironwood_treestate_slot(Some(treestate.clone())), + Some(treestate) ); } } diff --git a/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs b/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs index 98fc69c5c..0d6baf05e 100644 --- a/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs +++ b/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs @@ -598,12 +598,12 @@ async fn get_treestate() { .unwrap(); assert_eq!( - sapling_bytes_opt.as_deref(), - Some(sapling_tree_state.as_slice()) + sapling_bytes_opt.map(|pool| pool.final_state), + Some(sapling_tree_state) ); assert_eq!( - orchard_bytes_opt.as_deref(), - Some(orchard_tree_state.as_slice()) + orchard_bytes_opt.map(|pool| pool.final_state), + Some(orchard_tree_state) ); } diff --git a/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs b/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs index cc7e51720..c0cd46dbe 100644 --- a/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs +++ b/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs @@ -690,7 +690,7 @@ impl BlockchainSource for ProptestMockchain { async fn get_treestate( &self, _id: BlockHash, - ) -> BlockchainSourceResult<(Option>, Option>, Option>)> { + ) -> BlockchainSourceResult { // I don't think this is used for sync? unimplemented!() } From a72b107bc08ee76d37901eefad2149e297bbd204 Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 15:56:53 -0700 Subject: [PATCH 055/134] fix(chain-index): serve finalRoot for every pool in z_gettreestate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Close the all-pools finalRoot parity gap: - Fetch arm: fetch_pool_treestate_slot passes the validator's reported finalRoot through instead of dropping it. - State arm: each pool's root is computed from the note commitment tree exactly as zebrad serves it — root bytes in display order — matching zebra-rpc's own z_gettreestate construction. zebra populates Commitments { finalRoot, finalState } for every pool, while zaino previously sent finalRoot: null for sapling, orchard, and ironwood alike. Responses now carry both fields per pool; pre-NU6.3 responses still omit the ironwood treestate entirely. Flips the regression test from the previous commit green (#[should_panic] removed). Full zaino-state suite green (182 tests). PR #1362 review finding 4 residue, completing finding 4. Co-Authored-By: Claude Fable 5 --- .../src/chain_index/source/validator_connector.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/packages/zaino-state/src/chain_index/source/validator_connector.rs b/packages/zaino-state/src/chain_index/source/validator_connector.rs index 691ebc366..8707fddf7 100644 --- a/packages/zaino-state/src/chain_index/source/validator_connector.rs +++ b/packages/zaino-state/src/chain_index/source/validator_connector.rs @@ -79,12 +79,13 @@ fn ironwood_treestate_slot(validator_ironwood: Option) -> Option< /// Maps a validator-reported treestate (from the JSON-RPC `z_gettreestate` response) /// into the connector's pool slot. fn fetch_pool_treestate_slot(treestate: zebra_rpc::client::Treestate) -> Option { + let final_root = treestate.commitments().final_root().clone(); treestate .commitments() .final_state() .clone() .map(|final_state| PoolTreestate { - final_root: None, + final_root, final_state, }) } @@ -457,7 +458,8 @@ impl BlockchainSource for ValidatorConnector { } .and_then(|sap_response| { expected_read_response!(sap_response, SaplingTree).map(|tree| PoolTreestate { - final_root: None, + // finalRoot exactly as zebrad serves it: display-order root bytes. + final_root: Some(tree.root().bytes_in_display_order().to_vec()), final_state: tree.to_rpc_bytes(), }) }); @@ -486,7 +488,8 @@ impl BlockchainSource for ValidatorConnector { } .and_then(|orch_response| { expected_read_response!(orch_response, OrchardTree).map(|tree| PoolTreestate { - final_root: None, + // finalRoot exactly as zebrad serves it: display-order root bytes. + final_root: Some(tree.root().bytes_in_display_order().to_vec()), final_state: tree.to_rpc_bytes(), }) }); @@ -515,7 +518,8 @@ impl BlockchainSource for ValidatorConnector { } .and_then(|irw_response| { expected_read_response!(irw_response, IronwoodTree).map(|tree| PoolTreestate { - final_root: None, + // finalRoot exactly as zebrad serves it: display-order root bytes. + final_root: Some(tree.root().bytes_in_display_order().to_vec()), final_state: tree.to_rpc_bytes(), }) }); @@ -1208,9 +1212,7 @@ mod fetch_pool_treestate_slot { /// serves, but the slot mapping dropped the root, so zaino's z_gettreestate /// responses carried no finalRoot for any pool. /// - /// `should_panic` tracks the known gap; remove it together with the fix. #[test] - #[should_panic(expected = "finalRoot must pass through")] fn final_root_passes_through() { let final_root = vec![7u8; 32]; let final_state = vec![1u8, 2, 3]; From ddad666748d1a0ce3df1285515e2b0ab2424b809 Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 16:03:14 -0700 Subject: [PATCH 056/134] chore(test-env): bump devtool pin to ironwood-valar-portables (zcash#208) Move DEVTOOL_VERSION from the support_NU6_3 tip (zcash/zcash-devtool#205) to 252114bbfa69, the tip of ironwood-valar-portables, under review as zcash/zcash-devtool#208. The commit is reachable on the zingolabs fork, so the image build's clone + SHA checkout is unaffected. Co-Authored-By: Claude Fable 5 --- .env.testing-artifacts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.env.testing-artifacts b/.env.testing-artifacts index c9e575189..a3c399ea2 100644 --- a/.env.testing-artifacts +++ b/.env.testing-artifacts @@ -14,6 +14,6 @@ ZEBRA_VERSION=6.0.0-rc.0 # (get-ci-image-tag.sh → resolve_devtool_rev), so a branch HEAD advancing # changes the tag automatically and the rebuild always bakes the current # binary — no manual SHA bump needed. Pin a tag here only to freeze it. -# Pinned to the NU6.3/Ironwood support commit: tip of `support_NU6_3`, -# under review as zcash/zcash-devtool#205. -DEVTOOL_VERSION=a499f855fa88080045200ef05d03e79f2a506599 +# Pinned to the NU6.3/Ironwood support commit: tip of +# `ironwood-valar-portables`, under review as zcash/zcash-devtool#208. +DEVTOOL_VERSION=252114bbfa69b1baadfa4a84d1e2e8e37d2968b3 From 0d7f60d95e3a01068736411dbaf88bbb50329fb5 Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 16:10:51 -0700 Subject: [PATCH 057/134] refactor(chain-index): small cleanups from the PR #1362 review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Narrow the new ironwood accessors to the minimum scope that compiles: CommitmentTreeRoots::ironwood and CompactTxData::ironwood become pub(crate) (all callers are in-crate). CommitmentTreeSizes::ironwood stays pub — the e2e live-test crate reads it. - CompactTxData::to_compact_tx routes all five pool mappings through the existing into_compact() constructors instead of hand-building the proto struct literals (the ironwood block had been copy-pasted from orchard's hand-rolled version). - One compact_orchard_action_from_parts(pool, ..) replaces the copy-pasted fallible action mapping in the FullTransaction TryFrom; ironwood rejections no longer misreport themselves as "orchard ..." errors. - spawn_v1_0_0 delegates to the (now table-parameterized) open_env_and_dbs_with_commitment_table instead of duplicating the ~90-line LMDB environment/table-opening body; the fixture still opens only the legacy commitment_tree_data_1_0_0 table and never creates the v1.3.0 one, which on-disk version detection keys off. - Drop an unused PoolTreestate import in the treestate-slot tests (warned only under --tests; lint runs now include it). Behaviour preserved; full zaino-state suite green (182 tests). Co-Authored-By: Claude Fable 5 --- .../finalised_state/finalised_source/v1.rs | 100 +++--------------- .../chain_index/source/validator_connector.rs | 2 - .../src/chain_index/types/db/commitment.rs | 2 +- .../src/chain_index/types/db/legacy.rs | 94 ++++++---------- 4 files changed, 48 insertions(+), 150 deletions(-) diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1.rs index 1c4a1bdb7..73ed8b717 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1.rs @@ -435,9 +435,20 @@ impl DbV1 { /// and starts no background task — each caller (`spawn`, `spawn_v1_0_0`) adds its own tail. /// /// The `commitment_tree_data` handle is the up-to-date `commitment_tree_data_1_3_0` table - /// (`StoredEntryVar`). The v1.0.0 fixture builder ([`DbV1::spawn_v1_0_0`]) repoints it to the - /// legacy `commitment_tree_data_1_0_0` table (`StoredEntryFixed`) after calling this. + /// (`StoredEntryVar`). The v1.0.0 fixture builder ([`DbV1::spawn_v1_0_0`]) opens the legacy + /// `commitment_tree_data_1_0_0` table (`StoredEntryFixed`) instead, via + /// [`DbV1::open_env_and_dbs_with_commitment_table`]. async fn open_env_and_dbs(config: &ChainIndexConfig) -> Result { + Self::open_env_and_dbs_with_commitment_table(config, "commitment_tree_data_1_3_0").await + } + + /// [`DbV1::open_env_and_dbs`] with the commitment-tree table name as a parameter, so the + /// v1.0.0 fixture opener can select the legacy table without creating the v1.3.0 one + /// (on-disk version detection keys off which tables exist). + async fn open_env_and_dbs_with_commitment_table( + config: &ChainIndexConfig, + commitment_table: &str, + ) -> Result { info!("Launching FinalisedState"); // Prepare database details and path. @@ -499,8 +510,7 @@ impl DbV1 { let ironwood = super::open_or_create_db(&env, "ironwood_1_3_0", DatabaseFlags::empty()).await?; let commitment_tree_data = - super::open_or_create_db(&env, "commitment_tree_data_1_3_0", DatabaseFlags::empty()) - .await?; + super::open_or_create_db(&env, commitment_table, DatabaseFlags::empty()).await?; let hashes = super::open_or_create_db(&env, "hashes_1_0_0", DatabaseFlags::empty()).await?; let spent = super::open_or_create_db(&env, "spent_1_0_0", DatabaseFlags::empty()).await?; @@ -998,87 +1008,9 @@ impl DbV1 { // v1.2.1 -> v1.3.0 migration later rebuilds into the `commitment_tree_data_1_3_0` // `StoredEntryVar` table. This opener therefore opens the legacy commitment table and never // creates the v1.3.0 table. - let db_size_bytes = config.storage.database.size.to_byte_count(); - let db_path_dir = match config.network.to_zebra_network().kind() { - NetworkKind::Mainnet => "mainnet", - NetworkKind::Testnet => "testnet", - NetworkKind::Regtest => "regtest", - }; - let db_path = config.storage.database.path.join(db_path_dir).join("v1"); - if !db_path.exists() { - fs::create_dir_all(&db_path)?; - } - let cpu_cnt = std::thread::available_parallelism() - .map(|n| n.get()) - .unwrap_or(4); - let max_readers = u32::try_from((cpu_cnt * 32).clamp(512, 4096)) - .expect("max_readers was clamped to fit in u32"); - let env = Environment::new() - .set_max_dbs(16) - .set_map_size(db_size_bytes) - .set_max_readers(max_readers) - .set_flags( - EnvironmentFlags::NO_TLS - | EnvironmentFlags::NO_READAHEAD - | EnvironmentFlags::NO_SYNC, - ) - .open(&db_path)?; - - let headers = - super::open_or_create_db(&env, "headers_1_0_0", DatabaseFlags::empty()).await?; - let txids = super::open_or_create_db(&env, "txids_1_0_0", DatabaseFlags::empty()).await?; - let transparent = - super::open_or_create_db(&env, "transparent_1_0_0", DatabaseFlags::empty()).await?; - let sapling = - super::open_or_create_db(&env, "sapling_1_0_0", DatabaseFlags::empty()).await?; - let orchard = - super::open_or_create_db(&env, "orchard_1_0_0", DatabaseFlags::empty()).await?; - let ironwood = - super::open_or_create_db(&env, "ironwood_1_3_0", DatabaseFlags::empty()).await?; - let commitment_tree_data = - super::open_or_create_db(&env, "commitment_tree_data_1_0_0", DatabaseFlags::empty()) + let zaino_db = + Self::open_env_and_dbs_with_commitment_table(config, "commitment_tree_data_1_0_0") .await?; - let hashes = super::open_or_create_db(&env, "hashes_1_0_0", DatabaseFlags::empty()).await?; - let spent = super::open_or_create_db(&env, "spent_1_0_0", DatabaseFlags::empty()).await?; - let txid_location = - super::open_or_create_db(&env, "txid_location_1_0_0", DatabaseFlags::empty()).await?; - let metadata = super::open_or_create_db(&env, "metadata", DatabaseFlags::empty()).await?; - #[cfg(feature = "transparent_address_history_experimental")] - let address_history = super::open_or_create_db( - &env, - "address_history_1_0_0", - DatabaseFlags::DUP_SORT | DatabaseFlags::DUP_FIXED, - ) - .await?; - - let zaino_db = Self { - tx_out_set_info_accumulator: super::open_or_create_db( - &env, - TX_OUT_SET_INFO_ACCUMULATOR_DATABASE_NAME, - DatabaseFlags::empty(), - ) - .await?, - env: Arc::new(env), - headers, - txids, - transparent, - sapling, - orchard, - ironwood, - commitment_tree_data, - heights: hashes, - spent, - txid_location, - #[cfg(feature = "transparent_address_history_experimental")] - address_history, - metadata, - validated_tip: Arc::new(AtomicU32::new(0)), - validated_set: DashSet::new(), - db_handler: std::sync::Mutex::new(None), - cancel_token: CancellationToken::new(), - status: NamedAtomicStatus::new("FinalisedState", StatusType::Spawning), - config: config.clone(), - }; // Write the historical v1.0.0 metadata record. Intentionally skips `check_schema_version` // (see the method doc) — that is the behavioural difference from `spawn`. diff --git a/packages/zaino-state/src/chain_index/source/validator_connector.rs b/packages/zaino-state/src/chain_index/source/validator_connector.rs index 8707fddf7..74b192bbe 100644 --- a/packages/zaino-state/src/chain_index/source/validator_connector.rs +++ b/packages/zaino-state/src/chain_index/source/validator_connector.rs @@ -1205,8 +1205,6 @@ impl BlockchainSource for ValidatorConnector { #[cfg(test)] mod fetch_pool_treestate_slot { - use crate::chain_index::source::PoolTreestate; - /// Regression test: the validator's finalRoot must pass through to the pool slot. /// zebra populates `Commitments { finalRoot, finalState }` for every pool it /// serves, but the slot mapping dropped the root, so zaino's z_gettreestate diff --git a/packages/zaino-state/src/chain_index/types/db/commitment.rs b/packages/zaino-state/src/chain_index/types/db/commitment.rs index 142b0b5dd..88a2c5133 100644 --- a/packages/zaino-state/src/chain_index/types/db/commitment.rs +++ b/packages/zaino-state/src/chain_index/types/db/commitment.rs @@ -134,7 +134,7 @@ impl CommitmentTreeRoots { } /// returns orchard commitment tree root. - pub fn ironwood(&self) -> &Option<[u8; 32]> { + pub(crate) fn ironwood(&self) -> &Option<[u8; 32]> { &self.ironwood } } diff --git a/packages/zaino-state/src/chain_index/types/db/legacy.rs b/packages/zaino-state/src/chain_index/types/db/legacy.rs index bcc9a91b1..f6bdb88bd 100644 --- a/packages/zaino-state/src/chain_index/types/db/legacy.rs +++ b/packages/zaino-state/src/chain_index/types/db/legacy.rs @@ -1400,7 +1400,7 @@ impl CompactTxData { } /// Returns compact ironwood tx data. - pub fn ironwood(&self) -> &OrchardCompactTx { + pub(crate) fn ironwood(&self) -> &OrchardCompactTx { &self.ironwood } @@ -1415,52 +1415,28 @@ impl CompactTxData { .sapling() .spends() .iter() - .map( - |s| zaino_proto::proto::compact_formats::CompactSaplingSpend { - nf: s.nullifier().to_vec(), - }, - ) + .map(CompactSaplingSpend::into_compact) .collect(); let outputs = self .sapling() .outputs() .iter() - .map( - |o| zaino_proto::proto::compact_formats::CompactSaplingOutput { - cmu: o.cmu().to_vec(), - ephemeral_key: o.ephemeral_key().to_vec(), - ciphertext: o.ciphertext().to_vec(), - }, - ) + .map(CompactSaplingOutput::into_compact) .collect(); let actions = self .orchard() .actions() .iter() - .map( - |a| zaino_proto::proto::compact_formats::CompactOrchardAction { - nullifier: a.nullifier().to_vec(), - cmx: a.cmx().to_vec(), - ephemeral_key: a.ephemeral_key().to_vec(), - ciphertext: a.ciphertext().to_vec(), - }, - ) + .map(CompactOrchardAction::into_compact) .collect(); let ironwood_actions = self .ironwood() .actions() .iter() - .map( - |a| zaino_proto::proto::compact_formats::CompactOrchardAction { - nullifier: a.nullifier().to_vec(), - cmx: a.cmx().to_vec(), - ephemeral_key: a.ephemeral_key().to_vec(), - ciphertext: a.ciphertext().to_vec(), - }, - ) + .map(CompactOrchardAction::into_compact) .collect(); let vout = self.transparent().compact_vout(); @@ -1481,6 +1457,30 @@ impl CompactTxData { } } +/// Converts one RPC action tuple `(nullifier, cmx, ephemeral_key, ciphertext)` into a +/// [`CompactOrchardAction`], naming `pool` in each rejection so orchard and ironwood +/// failures are distinguishable. +fn compact_orchard_action_from_parts( + pool: &str, + (nf, cmx, epk, ct): (Vec, Vec, Vec, Vec), +) -> Result { + let nf: [u8; 32] = nf + .try_into() + .map_err(|_| format!("{pool} nullifier must be 32 bytes"))?; + let cmx: [u8; 32] = cmx + .try_into() + .map_err(|_| format!("{pool} cmx must be 32 bytes"))?; + let epk: [u8; 32] = epk + .try_into() + .map_err(|_| format!("{pool} ephemeral_key must be 32 bytes"))?; + let ct: [u8; 52] = ct + .get(..52) + .ok_or_else(|| format!("{pool} ciphertext must be at least 52 bytes"))? + .try_into() + .map_err(|_| format!("{pool} ciphertext must be 52 bytes"))?; + Ok(CompactOrchardAction::new(nf, cmx, epk, ct)) +} + /// TryFrom inputs: /// - Transaction Index /// - Full Transaction @@ -1563,23 +1563,7 @@ impl TryFrom<(u64, zaino_fetch::chain::transaction::FullTransaction)> for Compac let orchard_actions: Vec = tx .orchard_actions() .into_iter() - .map(|(nf, cmx, epk, ct)| { - let nf: [u8; 32] = nf - .try_into() - .map_err(|_| "orchard nullifier must be 32 bytes".to_string())?; - let cmx: [u8; 32] = cmx - .try_into() - .map_err(|_| "orchard cmx must be 32 bytes".to_string())?; - let epk: [u8; 32] = epk - .try_into() - .map_err(|_| "orchard ephemeral_key must be 32 bytes".to_string())?; - let ct: [u8; 52] = ct - .get(..52) - .ok_or("orchard ciphertext must be at least 52 bytes")? - .try_into() - .map_err(|_| "orchard ciphertext must be 52 bytes".to_string())?; - Ok::<_, String>(CompactOrchardAction::new(nf, cmx, epk, ct)) - }) + .map(|action| compact_orchard_action_from_parts("orchard", action)) .collect::>()?; let orchard = OrchardCompactTx::new(orchard_balance, orchard_actions); @@ -1587,23 +1571,7 @@ impl TryFrom<(u64, zaino_fetch::chain::transaction::FullTransaction)> for Compac let ironwood_actions: Vec = tx .ironwood_actions() .into_iter() - .map(|(nf, cmx, epk, ct)| { - let nf: [u8; 32] = nf - .try_into() - .map_err(|_| "orchard nullifier must be 32 bytes".to_string())?; - let cmx: [u8; 32] = cmx - .try_into() - .map_err(|_| "orchard cmx must be 32 bytes".to_string())?; - let epk: [u8; 32] = epk - .try_into() - .map_err(|_| "orchard ephemeral_key must be 32 bytes".to_string())?; - let ct: [u8; 52] = ct - .get(..52) - .ok_or("orchard ciphertext must be at least 52 bytes")? - .try_into() - .map_err(|_| "orchard ciphertext must be 52 bytes".to_string())?; - Ok::<_, String>(CompactOrchardAction::new(nf, cmx, epk, ct)) - }) + .map(|action| compact_orchard_action_from_parts("ironwood", action)) .collect::>()?; let ironwood = OrchardCompactTx::new(ironwood_balance, ironwood_actions); From 0725316801171ba2729c19761b484f116e4dc15f Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 16:11:24 -0700 Subject: [PATCH 058/134] refactor(chain-index): gate the unused ironwood roots accessor to tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Narrowing CommitmentTreeRoots::ironwood to pub(crate) surfaced that it has no production reader at all — only the None-preservation regression test consumes it. Gate it #[cfg(test)] so the lib profile carries no dead code; un-gate when a production consumer appears. Co-Authored-By: Claude Fable 5 --- packages/zaino-state/src/chain_index/types/db/commitment.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/zaino-state/src/chain_index/types/db/commitment.rs b/packages/zaino-state/src/chain_index/types/db/commitment.rs index 88a2c5133..b552f6b5a 100644 --- a/packages/zaino-state/src/chain_index/types/db/commitment.rs +++ b/packages/zaino-state/src/chain_index/types/db/commitment.rs @@ -134,6 +134,9 @@ impl CommitmentTreeRoots { } /// returns orchard commitment tree root. + /// No production reader consumes the stored ironwood root yet; the regression test + /// for its None-preservation does. Un-gate when a production consumer appears. + #[cfg(test)] pub(crate) fn ironwood(&self) -> &Option<[u8; 32]> { &self.ironwood } From 66c500fa5d8583bdb130228aaafa077bfa0f56a3 Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 16:19:38 -0700 Subject: [PATCH 059/134] refactor(chain-index): one range collector for the ephemeral per-height reads Fold the seven get_block_range_* loops (headers, txids, transparent, sapling, orchard, ironwood, commitment tree data) into one collect_block_range(start, end, |height| ...) helper. Each loop was the same iterate + per-height getter + push shape; the getters differ, nothing else. Scheduled as the final PR #1362 review DRY since the loop bodies contained code every earlier fix touched. Behaviour preserved; full zaino-state suite green (182 tests). Co-Authored-By: Claude Fable 5 --- .../finalised_source/ephemeral.rs | 76 +++++++------------ 1 file changed, 27 insertions(+), 49 deletions(-) diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/ephemeral.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/ephemeral.rs index 452979dd5..04e2b6b18 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/ephemeral.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/ephemeral.rs @@ -44,6 +44,23 @@ fn height_from_u32(height: u32) -> Result { }) } +/// Collects one item per height across the inclusive `start..=end` range, in ascending +/// order, by calling `get_at` for each height. +async fn collect_block_range( + start: Height, + end: Height, + mut get_at: impl FnMut(Height) -> Fut, +) -> Result, FinalisedStateError> +where + Fut: std::future::Future>, +{ + let mut items = Vec::new(); + for height in Height::range_inclusive(start, end) { + items.push(get_at(height).await?); + } + Ok(items) +} + /// Source-backed finalised-state backend used when persistent finalised-state storage is not /// serving normal requests. /// @@ -453,13 +470,7 @@ impl BlockCoreExt for EphemeralFinalisedState { start: Height, end: Height, ) -> Result, FinalisedStateError> { - let mut headers = Vec::new(); - - for height in Height::range_inclusive(start, end) { - headers.push(self.get_block_header(height).await?); - } - - Ok(headers) + collect_block_range(start, end, |height| self.get_block_header(height)).await } async fn get_block_txids(&self, height: Height) -> Result { @@ -479,13 +490,7 @@ impl BlockCoreExt for EphemeralFinalisedState { start: Height, end: Height, ) -> Result, FinalisedStateError> { - let mut txid_lists = Vec::new(); - - for height in Height::range_inclusive(start, end) { - txid_lists.push(self.get_block_txids(height).await?); - } - - Ok(txid_lists) + collect_block_range(start, end, |height| self.get_block_txids(height)).await } async fn get_txid( @@ -569,13 +574,7 @@ impl BlockTransparentExt for EphemeralFinalisedState { start: Height, end: Height, ) -> Result, FinalisedStateError> { - let mut transparent_lists = Vec::new(); - - for height in Height::range_inclusive(start, end) { - transparent_lists.push(self.get_block_transparent(height).await?); - } - - Ok(transparent_lists) + collect_block_range(start, end, |height| self.get_block_transparent(height)).await } async fn get_previous_output( @@ -653,13 +652,7 @@ impl BlockShieldedExt for EphemeralFinalisedState { start: Height, end: Height, ) -> Result, FinalisedStateError> { - let mut sapling_lists = Vec::new(); - - for height in Height::range_inclusive(start, end) { - sapling_lists.push(self.get_block_sapling(height).await?); - } - - Ok(sapling_lists) + collect_block_range(start, end, |height| self.get_block_sapling(height)).await } async fn get_orchard( @@ -696,13 +689,7 @@ impl BlockShieldedExt for EphemeralFinalisedState { start: Height, end: Height, ) -> Result, FinalisedStateError> { - let mut orchard_lists = Vec::new(); - - for height in Height::range_inclusive(start, end) { - orchard_lists.push(self.get_block_orchard(height).await?); - } - - Ok(orchard_lists) + collect_block_range(start, end, |height| self.get_block_orchard(height)).await } async fn get_ironwood( @@ -739,13 +726,7 @@ impl BlockShieldedExt for EphemeralFinalisedState { start: Height, end: Height, ) -> Result, FinalisedStateError> { - let mut ironwood_lists = Vec::new(); - - for height in Height::range_inclusive(start, end) { - ironwood_lists.push(self.get_block_ironwood(height).await?); - } - - Ok(ironwood_lists) + collect_block_range(start, end, |height| self.get_block_ironwood(height)).await } async fn get_block_commitment_tree_data( @@ -761,13 +742,10 @@ impl BlockShieldedExt for EphemeralFinalisedState { start: Height, end: Height, ) -> Result, FinalisedStateError> { - let mut commitment_tree_data = Vec::new(); - - for height in Height::range_inclusive(start, end) { - commitment_tree_data.push(self.get_block_commitment_tree_data(height).await?); - } - - Ok(commitment_tree_data) + collect_block_range(start, end, |height| { + self.get_block_commitment_tree_data(height) + }) + .await } } From 1a1df168189363f7afa3d0c8f2d1fbea53c9ab69 Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 16:22:39 -0700 Subject: [PATCH 060/134] refactor(backends): share the gRPC TreeState builder between backends The get_tree_state hex-extraction blocks in the Fetch and State backends were byte-identical (~40 lines each); both now call one tree_state_from_treestate_response in backends.rs. PR #1362 review cleanup (finding 4 family). Co-Authored-By: Claude Fable 5 --- packages/zaino-state/src/backends.rs | 43 ++++++++++++++++++++++ packages/zaino-state/src/backends/fetch.rs | 37 ++----------------- packages/zaino-state/src/backends/state.rs | 37 ++----------------- 3 files changed, 51 insertions(+), 66 deletions(-) diff --git a/packages/zaino-state/src/backends.rs b/packages/zaino-state/src/backends.rs index 353facce7..7af6467ef 100644 --- a/packages/zaino-state/src/backends.rs +++ b/packages/zaino-state/src/backends.rs @@ -4,6 +4,49 @@ pub mod fetch; pub mod state; +/// Builds the gRPC [`TreeState`] shared by the Fetch and State backends from a +/// `z_gettreestate` response: hex-encoded per-pool final states (the ironwood field is +/// the empty string below NU6.3 activation, matching lightwalletd behaviour). +/// +/// [`TreeState`]: zaino_proto::proto::service::TreeState +fn tree_state_from_treestate_response( + network: String, + treestate_response: zebra_rpc::client::GetTreestateResponse, +) -> zaino_proto::proto::service::TreeState { + let sapling_tree = hex::encode( + treestate_response + .sapling() + .commitments() + .final_state() + .clone() + .unwrap_or_default(), + ); + let orchard_tree = hex::encode( + treestate_response + .orchard() + .commitments() + .final_state() + .clone() + .unwrap_or_default(), + ); + let ironwood_tree = treestate_response + .ironwood() + .clone() + .and_then(|treestate| treestate.commitments().final_state().clone()) + .map(hex::encode) + .unwrap_or_default(); + + zaino_proto::proto::service::TreeState { + network, + height: treestate_response.height().0 as u64, + hash: treestate_response.hash().to_string(), + time: treestate_response.time(), + sapling_tree, + orchard_tree, + ironwood_tree, + } +} + /// Builds the `z_gettreestate` response shared by the Fetch and State backends from the /// per-pool treestates the chain index reported. /// diff --git a/packages/zaino-state/src/backends/fetch.rs b/packages/zaino-state/src/backends/fetch.rs index c3b508c6f..aefbb97fb 100644 --- a/packages/zaino-state/src/backends/fetch.rs +++ b/packages/zaino-state/src/backends/fetch.rs @@ -1829,43 +1829,14 @@ impl LightWalletIndexer for FetchServiceSubscriber { ) .await?; - let sapling_tree = hex::encode( - treestate_response - .sapling() - .commitments() - .final_state() - .clone() - .unwrap_or_default(), - ); - let orchard_tree = hex::encode( - treestate_response - .orchard() - .commitments() - .final_state() - .clone() - .unwrap_or_default(), - ); - let ironwood_tree = treestate_response - .ironwood() - .clone() - .and_then(|treestate| treestate.commitments().final_state().clone()) - .map(hex::encode) - .unwrap_or_default(); - - Ok(TreeState { - network: self - .config + Ok(super::tree_state_from_treestate_response( + self.config .common .network .to_zebra_network() .bip70_network_name(), - height: treestate_response.height().0 as u64, - hash: treestate_response.hash().to_string(), - time: treestate_response.time(), - sapling_tree, - orchard_tree, - ironwood_tree, - }) + treestate_response, + )) } /// GetLatestTreeState returns the note commitment tree state corresponding to the chain tip. diff --git a/packages/zaino-state/src/backends/state.rs b/packages/zaino-state/src/backends/state.rs index de91a0406..e7f9557aa 100644 --- a/packages/zaino-state/src/backends/state.rs +++ b/packages/zaino-state/src/backends/state.rs @@ -2509,43 +2509,14 @@ impl LightWalletIndexer for StateServiceSubscriber { ) .await?; - let sapling_tree = hex::encode( - treestate_response - .sapling() - .commitments() - .final_state() - .clone() - .unwrap_or_default(), - ); - let orchard_tree = hex::encode( - treestate_response - .orchard() - .commitments() - .final_state() - .clone() - .unwrap_or_default(), - ); - let ironwood_tree = treestate_response - .ironwood() - .clone() - .and_then(|treestate| treestate.commitments().final_state().clone()) - .map(hex::encode) - .unwrap_or_default(); - - Ok(TreeState { - network: self - .config + Ok(super::tree_state_from_treestate_response( + self.config .common .network .to_zebra_network() .bip70_network_name(), - height: treestate_response.height().0 as u64, - hash: treestate_response.hash().to_string(), - time: treestate_response.time(), - sapling_tree, - orchard_tree, - ironwood_tree, - }) + treestate_response, + )) } /// GetLatestTreeState returns the note commitment tree state corresponding to the chain tip. From aac9b16e0d8667435e57f0f3fc7498c1aba93991 Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 16:29:03 -0700 Subject: [PATCH 061/134] refactor(chain-index): one row reader and one range scanner for the v1 tables Every v1 table repeated the same two read shapes: "resolve validated height -> LMDB get -> tolerance -> StoredEntryVar decode" (headers, txids, transparent, sapling, orchard, ironwood, commitment tree data) and "cursor-scan rows in [start, end] -> decode each" (five tables). Twelve per-table clones fold into read_row / read_row_at_height / scan_rows in read_core.rs, generic over the stored type with the table handle and a label for error messages as parameters. Missing-row policy moves to the call sites: dense tables .ok_or their existing DataUnavailable message, the sparse ironwood table maps an absent row to the empty list, exactly as before. Decoded items move out of the entry (entry.item) instead of cloning. Behaviour preserved; full zaino-state suite green (182 tests). Net -260 lines. PR #1362 review DRY follow-through. Co-Authored-By: Claude Fable 5 --- .../finalised_source/v1/block_core.rs | 131 +--------- .../finalised_source/v1/block_shielded.rs | 227 ++---------------- .../finalised_source/v1/block_transparent.rs | 72 +----- .../finalised_source/v1/read_core.rs | 90 +++++++ 4 files changed, 130 insertions(+), 390 deletions(-) diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/block_core.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/block_core.rs index f4079ff19..20621b02d 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/block_core.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/block_core.rs @@ -56,27 +56,11 @@ impl DbV1 { &self, height: Height, ) -> Result { - let validated_height = self - .resolve_validated_hash_or_height(HashOrHeight::Height(height.into())) - .await?; - let height_bytes = validated_height.to_bytes()?; - - tokio::task::block_in_place(|| { - let txn = self.env.begin_ro_txn()?; - let raw = match txn.get(self.headers, &height_bytes) { - Ok(val) => val, - Err(lmdb::Error::NotFound) => { - return Err(FinalisedStateError::DataUnavailable( - "header data missing from db".into(), - )); - } - Err(e) => return Err(FinalisedStateError::LmdbError(e)), - }; - let entry = StoredEntryVar::from_bytes(raw) - .map_err(|e| FinalisedStateError::Custom(format!("header decode error: {e}")))?; - - Ok(*entry.inner()) - }) + self.read_row_at_height(self.headers, "header", height) + .await? + .ok_or_else(|| { + FinalisedStateError::DataUnavailable("header data missing from db".into()) + }) } /// Fetches block headers for the given height range. @@ -91,45 +75,7 @@ impl DbV1 { start: Height, end: Height, ) -> Result, FinalisedStateError> { - if end.0 < start.0 { - return Err(FinalisedStateError::Custom( - "invalid block range: end < start".to_string(), - )); - } - - self.validate_block_range(start, end).await?; - let start_bytes = start.to_bytes()?; - let end_bytes = end.to_bytes()?; - - let raw_entries = tokio::task::block_in_place(|| { - let txn = self.env.begin_ro_txn()?; - let mut raw_entries = Vec::new(); - let mut cursor = match txn.open_ro_cursor(self.headers) { - Ok(cursor) => cursor, - Err(lmdb::Error::NotFound) => { - return Err(FinalisedStateError::DataUnavailable( - "header data missing from db".into(), - )); - } - Err(e) => return Err(FinalisedStateError::LmdbError(e)), - }; - for (k, v) in cursor.iter_from(&start_bytes[..]) { - if k > &end_bytes[..] { - break; - } - raw_entries.push(v.to_vec()); - } - Ok::>, FinalisedStateError>(raw_entries) - })?; - - raw_entries - .into_iter() - .map(|bytes| { - StoredEntryVar::::from_bytes(&bytes) - .map(|e| *e.inner()) - .map_err(|e| FinalisedStateError::Custom(format!("header decode error: {e}"))) - }) - .collect() + self.scan_rows(self.headers, "header", start, end).await } /// Fetch the txid bytes for a given TxLocation. @@ -218,28 +164,9 @@ impl DbV1 { /// Fetch block txids by height. async fn get_block_txids(&self, height: Height) -> Result { - let validated_height = self - .resolve_validated_hash_or_height(HashOrHeight::Height(height.into())) - .await?; - let height_bytes = validated_height.to_bytes()?; - - tokio::task::block_in_place(|| { - let txn = self.env.begin_ro_txn()?; - let raw = match txn.get(self.txids, &height_bytes) { - Ok(val) => val, - Err(lmdb::Error::NotFound) => { - return Err(FinalisedStateError::DataUnavailable( - "txid data missing from db".into(), - )); - } - Err(e) => return Err(FinalisedStateError::LmdbError(e)), - }; - - let entry: StoredEntryVar = StoredEntryVar::from_bytes(raw) - .map_err(|e| FinalisedStateError::Custom(format!("txids decode error: {e}")))?; - - Ok(entry.inner().clone()) - }) + self.read_row_at_height(self.txids, "txids", height) + .await? + .ok_or_else(|| FinalisedStateError::DataUnavailable("txid data missing from db".into())) } /// Fetches block txids for the given height range. @@ -254,45 +181,7 @@ impl DbV1 { start: Height, end: Height, ) -> Result, FinalisedStateError> { - if end.0 < start.0 { - return Err(FinalisedStateError::Custom( - "invalid block range: end < start".to_string(), - )); - } - - self.validate_block_range(start, end).await?; - let start_bytes = start.to_bytes()?; - let end_bytes = end.to_bytes()?; - - let raw_entries = tokio::task::block_in_place(|| { - let txn = self.env.begin_ro_txn()?; - let mut raw_entries = Vec::new(); - let mut cursor = match txn.open_ro_cursor(self.txids) { - Ok(cursor) => cursor, - Err(lmdb::Error::NotFound) => { - return Err(FinalisedStateError::DataUnavailable( - "txid data missing from db".into(), - )); - } - Err(e) => return Err(FinalisedStateError::LmdbError(e)), - }; - for (k, v) in cursor.iter_from(&start_bytes[..]) { - if k > &end_bytes[..] { - break; - } - raw_entries.push(v.to_vec()); - } - Ok::>, FinalisedStateError>(raw_entries) - })?; - - raw_entries - .into_iter() - .map(|bytes| { - StoredEntryVar::::from_bytes(&bytes) - .map(|e| e.inner().clone()) - .map_err(|e| FinalisedStateError::Custom(format!("txids decode error: {e}"))) - }) - .collect() + self.scan_rows(self.txids, "txids", start, end).await } // Fetch the TxLocation for the given txid, transaction data is indexed by TxLocation internally. diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/block_shielded.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/block_shielded.rs index 4b53dd6ef..a4b208341 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/block_shielded.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/block_shielded.rs @@ -123,28 +123,11 @@ impl DbV1 { &self, height: Height, ) -> Result { - let validated_height = self - .resolve_validated_hash_or_height(HashOrHeight::Height(height.into())) - .await?; - let height_bytes = validated_height.to_bytes()?; - - tokio::task::block_in_place(|| { - let txn = self.env.begin_ro_txn()?; - let raw = match txn.get(self.sapling, &height_bytes) { - Ok(val) => val, - Err(lmdb::Error::NotFound) => { - return Err(FinalisedStateError::DataUnavailable( - "sapling data missing from db".into(), - )); - } - Err(e) => return Err(FinalisedStateError::LmdbError(e)), - }; - - let entry: StoredEntryVar = StoredEntryVar::from_bytes(raw) - .map_err(|e| FinalisedStateError::Custom(format!("sapling decode error: {e}")))?; - - Ok(entry.inner().clone()) - }) + self.read_row_at_height(self.sapling, "sapling", height) + .await? + .ok_or_else(|| { + FinalisedStateError::DataUnavailable("sapling data missing from db".into()) + }) } /// Fetches block sapling tx data for the given height range. @@ -159,45 +142,7 @@ impl DbV1 { start: Height, end: Height, ) -> Result, FinalisedStateError> { - if end.0 < start.0 { - return Err(FinalisedStateError::Custom( - "invalid block range: end < start".to_string(), - )); - } - - self.validate_block_range(start, end).await?; - let start_bytes = start.to_bytes()?; - let end_bytes = end.to_bytes()?; - - let raw_entries = tokio::task::block_in_place(|| { - let txn = self.env.begin_ro_txn()?; - let mut raw_entries = Vec::new(); - let mut cursor = match txn.open_ro_cursor(self.sapling) { - Ok(cursor) => cursor, - Err(lmdb::Error::NotFound) => { - return Err(FinalisedStateError::DataUnavailable( - "sapling data missing from db".into(), - )); - } - Err(e) => return Err(FinalisedStateError::LmdbError(e)), - }; - for (k, v) in cursor.iter_from(&start_bytes[..]) { - if k > &end_bytes[..] { - break; - } - raw_entries.push(v.to_vec()); - } - Ok::>, FinalisedStateError>(raw_entries) - })?; - - raw_entries - .into_iter() - .map(|bytes| { - StoredEntryVar::::from_bytes(&bytes) - .map(|e| e.inner().clone()) - .map_err(|e| FinalisedStateError::Custom(format!("sapling decode error: {e}"))) - }) - .collect() + self.scan_rows(self.sapling, "sapling", start, end).await } /// Fetch the serialized OrchardCompactTx for the given TxLocation, if present. @@ -221,28 +166,11 @@ impl DbV1 { &self, height: Height, ) -> Result { - let validated_height = self - .resolve_validated_hash_or_height(HashOrHeight::Height(height.into())) - .await?; - let height_bytes = validated_height.to_bytes()?; - - tokio::task::block_in_place(|| { - let txn = self.env.begin_ro_txn()?; - let raw = match txn.get(self.orchard, &height_bytes) { - Ok(val) => val, - Err(lmdb::Error::NotFound) => { - return Err(FinalisedStateError::DataUnavailable( - "orchard data missing from db".into(), - )); - } - Err(e) => return Err(FinalisedStateError::LmdbError(e)), - }; - - let entry: StoredEntryVar = StoredEntryVar::from_bytes(raw) - .map_err(|e| FinalisedStateError::Custom(format!("orchard decode error: {e}")))?; - - Ok(entry.inner().clone()) - }) + self.read_row_at_height(self.orchard, "orchard", height) + .await? + .ok_or_else(|| { + FinalisedStateError::DataUnavailable("orchard data missing from db".into()) + }) } /// Fetches block orchard tx data for the given height range. @@ -257,45 +185,7 @@ impl DbV1 { start: Height, end: Height, ) -> Result, FinalisedStateError> { - if end.0 < start.0 { - return Err(FinalisedStateError::Custom( - "invalid block range: end < start".to_string(), - )); - } - - self.validate_block_range(start, end).await?; - let start_bytes = start.to_bytes()?; - let end_bytes = end.to_bytes()?; - - let raw_entries = tokio::task::block_in_place(|| { - let txn = self.env.begin_ro_txn()?; - let mut raw_entries = Vec::new(); - let mut cursor = match txn.open_ro_cursor(self.orchard) { - Ok(cursor) => cursor, - Err(lmdb::Error::NotFound) => { - return Err(FinalisedStateError::DataUnavailable( - "orchard data missing from db".into(), - )); - } - Err(e) => return Err(FinalisedStateError::LmdbError(e)), - }; - for (k, v) in cursor.iter_from(&start_bytes[..]) { - if k > &end_bytes[..] { - break; - } - raw_entries.push(v.to_vec()); - } - Ok::>, FinalisedStateError>(raw_entries) - })?; - - raw_entries - .into_iter() - .map(|bytes| { - StoredEntryVar::::from_bytes(&bytes) - .map(|e| e.inner().clone()) - .map_err(|e| FinalisedStateError::Custom(format!("orchard decode error: {e}"))) - }) - .collect() + self.scan_rows(self.orchard, "orchard", start, end).await } /// Fetch the serialized `OrchardCompactTx` for the given TxLocation from the ironwood table. @@ -323,24 +213,10 @@ impl DbV1 { &self, height: Height, ) -> Result { - let validated_height = self - .resolve_validated_hash_or_height(HashOrHeight::Height(height.into())) - .await?; - let height_bytes = validated_height.to_bytes()?; - - tokio::task::block_in_place(|| { - let txn = self.env.begin_ro_txn()?; - let raw = match txn.get(self.ironwood, &height_bytes) { - Ok(val) => val, - Err(lmdb::Error::NotFound) => return Ok(OrchardTxList::new(Vec::new())), - Err(e) => return Err(FinalisedStateError::LmdbError(e)), - }; - - let entry: StoredEntryVar = StoredEntryVar::from_bytes(raw) - .map_err(|e| FinalisedStateError::Custom(format!("ironwood decode error: {e}")))?; - - Ok(entry.inner().clone()) - }) + Ok(self + .read_row_at_height(self.ironwood, "ironwood", height) + .await? + .unwrap_or_else(|| OrchardTxList::new(Vec::new()))) } /// Fetches block ironwood tx data for the given (inclusive) height range. @@ -375,29 +251,11 @@ impl DbV1 { &self, height: Height, ) -> Result { - let validated_height = self - .resolve_validated_hash_or_height(HashOrHeight::Height(height.into())) - .await?; - let height_bytes = validated_height.to_bytes()?; - - tokio::task::block_in_place(|| { - let txn = self.env.begin_ro_txn()?; - let raw = match txn.get(self.commitment_tree_data, &height_bytes) { - Ok(val) => val, - Err(lmdb::Error::NotFound) => { - return Err(FinalisedStateError::DataUnavailable( - "commitment tree data missing from db".into(), - )); - } - Err(e) => return Err(FinalisedStateError::LmdbError(e)), - }; - - let entry = StoredEntryVar::::from_bytes(raw).map_err(|e| { - FinalisedStateError::Custom(format!("commitment_tree decode error: {e}")) - })?; - - Ok(entry.item) - }) + self.read_row_at_height(self.commitment_tree_data, "commitment_tree", height) + .await? + .ok_or_else(|| { + FinalisedStateError::DataUnavailable("commitment tree data missing from db".into()) + }) } /// Fetches block commitment tree data for the given height range. @@ -412,47 +270,8 @@ impl DbV1 { start: Height, end: Height, ) -> Result, FinalisedStateError> { - if end.0 < start.0 { - return Err(FinalisedStateError::Custom( - "invalid block range: end < start".to_string(), - )); - } - - self.validate_block_range(start, end).await?; - let start_bytes = start.to_bytes()?; - let end_bytes = end.to_bytes()?; - - let raw_entries = tokio::task::block_in_place(|| { - let txn = self.env.begin_ro_txn()?; - let mut raw_entries = Vec::new(); - let mut cursor = match txn.open_ro_cursor(self.commitment_tree_data) { - Ok(cursor) => cursor, - Err(lmdb::Error::NotFound) => { - return Err(FinalisedStateError::DataUnavailable( - "commitment tree data missing from db".into(), - )); - } - Err(e) => return Err(FinalisedStateError::LmdbError(e)), - }; - for (k, v) in cursor.iter_from(&start_bytes[..]) { - if k > &end_bytes[..] { - break; - } - raw_entries.push(v.to_vec()); - } - Ok::>, FinalisedStateError>(raw_entries) - })?; - - raw_entries - .into_iter() - .map(|bytes| { - StoredEntryVar::::from_bytes(&bytes) - .map(|e| e.item) - .map_err(|e| { - FinalisedStateError::Custom(format!("commitment_tree decode error: {e}")) - }) - }) - .collect() + self.scan_rows(self.commitment_tree_data, "commitment_tree", start, end) + .await } // *** Internal DB methods *** diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/block_transparent.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/block_transparent.rs index 571b743c9..93eadd93e 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/block_transparent.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/block_transparent.rs @@ -132,30 +132,11 @@ impl DbV1 { &self, height: Height, ) -> Result { - let validated_height = self - .resolve_validated_hash_or_height(HashOrHeight::Height(height.into())) - .await?; - let height_bytes = validated_height.to_bytes()?; - - tokio::task::block_in_place(|| { - let txn = self.env.begin_ro_txn()?; - let raw = match txn.get(self.transparent, &height_bytes) { - Ok(val) => val, - Err(lmdb::Error::NotFound) => { - return Err(FinalisedStateError::DataUnavailable( - "transparent data missing from db".into(), - )); - } - Err(e) => return Err(FinalisedStateError::LmdbError(e)), - }; - - let entry: StoredEntryVar = StoredEntryVar::from_bytes(raw) - .map_err(|e| { - FinalisedStateError::Custom(format!("transparent decode error: {e}")) - })?; - - Ok(entry.inner().clone()) - }) + self.read_row_at_height(self.transparent, "transparent", height) + .await? + .ok_or_else(|| { + FinalisedStateError::DataUnavailable("transparent data missing from db".into()) + }) } /// Fetches block transparent tx data for the given height range. @@ -170,47 +151,8 @@ impl DbV1 { start: Height, end: Height, ) -> Result, FinalisedStateError> { - if end.0 < start.0 { - return Err(FinalisedStateError::Custom( - "invalid block range: end < start".to_string(), - )); - } - - self.validate_block_range(start, end).await?; - let start_bytes = start.to_bytes()?; - let end_bytes = end.to_bytes()?; - - let raw_entries = tokio::task::block_in_place(|| { - let txn = self.env.begin_ro_txn()?; - let mut raw_entries = Vec::new(); - let mut cursor = match txn.open_ro_cursor(self.transparent) { - Ok(cursor) => cursor, - Err(lmdb::Error::NotFound) => { - return Err(FinalisedStateError::DataUnavailable( - "transparent data missing from db".into(), - )); - } - Err(e) => return Err(FinalisedStateError::LmdbError(e)), - }; - for (k, v) in cursor.iter_from(&start_bytes[..]) { - if k > &end_bytes[..] { - break; - } - raw_entries.push(v.to_vec()); - } - Ok::>, FinalisedStateError>(raw_entries) - })?; - - raw_entries - .into_iter() - .map(|bytes| { - StoredEntryVar::::from_bytes(&bytes) - .map(|e| e.inner().clone()) - .map_err(|e| { - FinalisedStateError::Custom(format!("transparent decode error: {e}")) - }) - }) - .collect() + self.scan_rows(self.transparent, "transparent", start, end) + .await } // *** Internal DB methods *** diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/read_core.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/read_core.rs index d10e2b31f..67e21468d 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/read_core.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/read_core.rs @@ -2,6 +2,8 @@ use super::*; +use crate::ZainoVersionedSerde; + /// [`DbRead`] capability implementation for [`DbV1`]. /// /// This trait is the read-only surface used by higher layers. Methods typically delegate to @@ -122,3 +124,91 @@ impl DbV1 { // *** Internal DB methods *** } + +impl DbV1 { + /// Fetches and decodes one `StoredEntryVar` row keyed by an already-validated + /// height. Returns `Ok(None)` when the table has no row for the height; `label` + /// names the table in decode errors. + fn read_row( + &self, + table: lmdb::Database, + label: &str, + height_bytes: &[u8], + ) -> Result, FinalisedStateError> { + tokio::task::block_in_place(|| { + let txn = self.env.begin_ro_txn()?; + let raw = match txn.get(table, &height_bytes) { + Ok(val) => val, + Err(lmdb::Error::NotFound) => return Ok(None), + Err(e) => return Err(FinalisedStateError::LmdbError(e)), + }; + let entry: StoredEntryVar = StoredEntryVar::from_bytes(raw) + .map_err(|e| FinalisedStateError::Custom(format!("{label} decode error: {e}")))?; + Ok(Some(entry.item)) + }) + } + + /// [`DbV1::read_row`] at a height that is first validated against the index. + pub(super) async fn read_row_at_height( + &self, + table: lmdb::Database, + label: &str, + height: Height, + ) -> Result, FinalisedStateError> { + let validated_height = self + .resolve_validated_hash_or_height(HashOrHeight::Height(height.into())) + .await?; + let height_bytes = validated_height.to_bytes()?; + self.read_row(table, label, &height_bytes) + } + + /// Cursor-scans and decodes every `StoredEntryVar` row in the validated + /// inclusive `start..=end` height range. + pub(super) async fn scan_rows( + &self, + table: lmdb::Database, + label: &str, + start: Height, + end: Height, + ) -> Result, FinalisedStateError> { + if end.0 < start.0 { + return Err(FinalisedStateError::Custom( + "invalid block range: end < start".to_string(), + )); + } + + self.validate_block_range(start, end).await?; + let start_bytes = start.to_bytes()?; + let end_bytes = end.to_bytes()?; + + let raw_entries = tokio::task::block_in_place(|| { + let txn = self.env.begin_ro_txn()?; + let mut raw_entries = Vec::new(); + let mut cursor = match txn.open_ro_cursor(table) { + Ok(cursor) => cursor, + Err(lmdb::Error::NotFound) => { + return Err(FinalisedStateError::DataUnavailable(format!( + "{label} data missing from db" + ))); + } + Err(e) => return Err(FinalisedStateError::LmdbError(e)), + }; + for (k, v) in cursor.iter_from(&start_bytes[..]) { + if k > &end_bytes[..] { + break; + } + raw_entries.push(v.to_vec()); + } + Ok::>, FinalisedStateError>(raw_entries) + })?; + + raw_entries + .into_iter() + .map(|bytes| { + StoredEntryVar::::from_bytes(&bytes) + .map(|e| e.item) + .map_err(|e| FinalisedStateError::Custom(format!("{label} decode error: {e}"))) + }) + .collect() + } +} From 7c2266608d393f68c53e6befd493c693ffd7efb8 Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 16:34:43 -0700 Subject: [PATCH 062/134] refactor(chain-index): share the per-block entry building between write paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit write_block and the batch sync loop duplicated the entire per-block row-preparation phase. Three shared items now own it: - extract_block_pool_lists: the per-transaction pool lists (paired txid/transparent, sapling, orchard, ironwood presence semantics) with the duplicate-txid guard — one definition of what "a transaction has data in this pool" means on the write path; - verify_header_merkle_root: the cheap in-memory txids-vs-header check; - BlockRowEntries / build_block_row_entries: the eight row entries, including the sparse-ironwood Option, so the skip-when-all-None rule exists once. The parts that genuinely differ stay per-path: the spent-outpoint handling (dup-checked map with per-block writes vs the batch-level sorted vector), the put flavour (NO_OVERWRITE in spawn_blocking vs put_idempotent in one shared txn), and the tip/continuity checks. Behaviour preserved; full zaino-state suite green (182 tests). PR #1362 review DRY follow-through. Co-Authored-By: Claude Fable 5 --- .../finalised_source/v1/write_core.rs | 411 ++++++++++-------- 1 file changed, 220 insertions(+), 191 deletions(-) diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/write_core.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/write_core.rs index d165756a9..525452cf2 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/write_core.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/write_core.rs @@ -44,6 +44,155 @@ use crate::version; /// /// This trait represents the mutating surface (append / delete tip / update metadata). Writes are /// performed via LMDB write transactions and validated before becoming visible as “known-good”. +/// Per-transaction pool lists for one block's row entries, with the duplicate-txid +/// guard applied. +struct BlockPoolLists { + /// `(txid, transparent)` pairs — both halves come from one binding per + /// transaction, so misalignment is structurally impossible. + transactions: Vec<(TransactionHash, Option)>, + sapling: Vec>, + orchard: Vec>, + ironwood: Vec>, +} + +/// Builds the per-transaction pool lists for one block: each pool records +/// `Some(compact data)` for a transaction with data in that pool, `None` otherwise, +/// keeping every list index-aligned with the block's txids. +fn extract_block_pool_lists(block: &IndexedBlock) -> Result { + let block_height = block.context.index.height; + let block_hash = block.context.index.hash; + + let tx_len = block.transactions().len(); + let mut transactions: Vec<(TransactionHash, Option)> = + Vec::with_capacity(tx_len); + let mut txid_set: HashSet = HashSet::with_capacity(tx_len); + let mut sapling = Vec::with_capacity(tx_len); + let mut orchard = Vec::with_capacity(tx_len); + let mut ironwood = Vec::with_capacity(tx_len); + + for tx in block.transactions() { + let hash = tx.txid(); + if !txid_set.insert(*hash) { + return Err(FinalisedStateError::InvalidBlock { + height: block_height.0, + hash: block_hash, + reason: format!("duplicate transaction hash in block: {hash:?}"), + }); + } + + // Transparent transactions — paired with the txid at the source binding. + let transparent_data = + if tx.transparent().inputs().is_empty() && tx.transparent().outputs().is_empty() { + None + } else { + Some(tx.transparent().clone()) + }; + transactions.push((*hash, transparent_data)); + + // Sapling transactions + let sapling_data = if tx.sapling().spends().is_empty() && tx.sapling().outputs().is_empty() + { + None + } else { + Some(tx.sapling().clone()) + }; + sapling.push(sapling_data); + + // Orchard transactions + let orchard_data = if tx.orchard().actions().is_empty() { + None + } else { + Some(tx.orchard().clone()) + }; + orchard.push(orchard_data); + + // Ironwood transactions (NU6.3; modelled with the Orchard compact types). + let ironwood_data = if tx.ironwood().actions().is_empty() { + None + } else { + Some(tx.ironwood().clone()) + }; + ironwood.push(ironwood_data); + } + + Ok(BlockPoolLists { + transactions, + sapling, + orchard, + ironwood, + }) +} + +/// Cheap in-memory correctness check: the block's txids must reproduce the header's +/// merkle root. +fn verify_header_merkle_root( + txids: &[TransactionHash], + block: &IndexedBlock, +) -> Result<(), FinalisedStateError> { + let txid_bytes: Vec<[u8; 32]> = txids.iter().map(|txid| txid.0).collect(); + let computed_merkle_root = DbV1::calculate_block_merkle_root(&txid_bytes); + if &computed_merkle_root != block.data().merkle_root() { + return Err(FinalisedStateError::InvalidBlock { + height: block.context.index.height.0, + hash: block.context.index.hash, + reason: "header merkle root does not match block txids".to_string(), + }); + } + Ok(()) +} + +/// One block's row entries, ready to put. Everything is keyed by the block height +/// except `height_entry` (the hash-keyed height index). `ironwood_entry` is `None` +/// when the block has no ironwood data — the ironwood table is sparse; readers treat +/// an absent row as "no ironwood data". +struct BlockRowEntries { + height_entry: StoredEntryFixed, + header_entry: StoredEntryVar, + commitment_tree_entry: StoredEntryVar, + txid_entry: StoredEntryVar, + transparent_entry: StoredEntryVar, + sapling_entry: StoredEntryVar, + orchard_entry: StoredEntryVar, + ironwood_entry: Option>, +} + +#[allow(clippy::too_many_arguments)] +fn build_block_row_entries( + block: &IndexedBlock, + block_hash_bytes: &[u8], + block_height_bytes: &[u8], + txids: Vec, + transparent: Vec>, + sapling: Vec>, + orchard: Vec>, + ironwood: Vec>, +) -> BlockRowEntries { + BlockRowEntries { + height_entry: StoredEntryFixed::new(block_hash_bytes, block.context.index.height), + header_entry: StoredEntryVar::new( + block_height_bytes, + BlockHeaderData::new(block.context, *block.data()), + ), + // Stored as a `StoredEntryVar` because `CommitmentTreeData` V2 is + // variable-length (optional Ironwood root). + commitment_tree_entry: StoredEntryVar::new( + block_height_bytes, + *block.commitment_tree_data(), + ), + txid_entry: StoredEntryVar::new(block_height_bytes, TxidList::new(txids)), + transparent_entry: StoredEntryVar::new( + block_height_bytes, + TransparentTxList::new(transparent), + ), + sapling_entry: StoredEntryVar::new(block_height_bytes, SaplingTxList::new(sapling)), + orchard_entry: StoredEntryVar::new(block_height_bytes, OrchardTxList::new(orchard)), + ironwood_entry: ironwood + .iter() + .any(Option::is_some) + .then(|| StoredEntryVar::new(block_height_bytes, OrchardTxList::new(ironwood))), + } +} + impl DbWrite for DbV1 { async fn write_block(&self, block: IndexedBlock) -> Result<(), FinalisedStateError> { self.write_block(block).await @@ -410,34 +559,17 @@ impl DbV1 { return Ok(()); } - // Build DBHeight - let height_entry = StoredEntryFixed::new(&block_hash_bytes, block.context.index.height); + // Build the per-transaction pool lists (with the duplicate-txid guard applied). + // The accumulator consumes the paired `(txid, transparent)` slice; for storage + // the pairs are `unzip`ped into the `TxidList` / `TransparentTxList` shapes. + let pool_lists = extract_block_pool_lists(&block)?; - // Build header - let header_entry = StoredEntryVar::new( - &block_height_bytes, - BlockHeaderData::new(block.context, *block.data()), - ); - - // Build commitment tree data. Stored as a `StoredEntryVar` because `CommitmentTreeData` V2 - // is variable-length (optional Ironwood root). - let commitment_tree_entry = - StoredEntryVar::new(&block_height_bytes, *block.commitment_tree_data()); - - // Build transaction indexes. - // - // `transactions` pairs each transaction hash with its transparent data. Both halves - // are sourced from the same `tx` in the loop below, so misalignment is structurally - // impossible — the pair shares one binding. Downstream the accumulator consumes the - // paired slice; for storage we `unzip` into the existing `TxidList` / `TransparentTxList` - // shapes. - let tx_len = block.transactions().len(); - let mut transactions: Vec<(TransactionHash, Option)> = - Vec::with_capacity(tx_len); - let mut txid_set: HashSet = HashSet::with_capacity(tx_len); - let mut sapling = Vec::with_capacity(tx_len); - let mut orchard = Vec::with_capacity(tx_len); - let mut ironwood = Vec::with_capacity(tx_len); + #[cfg(feature = "transparent_address_history_experimental")] + let txid_set: HashSet = pool_lists + .transactions + .iter() + .map(|(hash, _)| *hash) + .collect(); let mut spent_map: HashMap = HashMap::new(); @@ -453,50 +585,6 @@ impl DbV1 { #[allow(clippy::unused_enumerate_index)] for (_tx_index, tx) in block.transactions().iter().enumerate() { - let hash = tx.txid(); - - if !txid_set.insert(*hash) { - return Err(FinalisedStateError::InvalidBlock { - height: block_height.0, - hash: block_hash, - reason: format!("duplicate transaction hash in block: {hash:?}"), - }); - } - - // Transparent transactions — paired with the txid at the source binding. - let transparent_data = - if tx.transparent().inputs().is_empty() && tx.transparent().outputs().is_empty() { - None - } else { - Some(tx.transparent().clone()) - }; - transactions.push((*hash, transparent_data)); - - // Sapling transactions - let sapling_data = - if tx.sapling().spends().is_empty() && tx.sapling().outputs().is_empty() { - None - } else { - Some(tx.sapling().clone()) - }; - sapling.push(sapling_data); - - // Orchard transactions - let orchard_data = if tx.orchard().actions().is_empty() { - None - } else { - Some(tx.orchard().clone()) - }; - orchard.push(orchard_data); - - // Ironwood transactions (NU6.3; modelled with the Orchard compact types). - let ironwood_data = if tx.ironwood().actions().is_empty() { - None - } else { - Some(tx.ironwood().clone()) - }; - ironwood.push(ironwood_data); - // Transaction location let tx_index = u16::try_from(_tx_index).map_err(|_| FinalisedStateError::InvalidBlock { @@ -540,7 +628,8 @@ impl DbV1 { let prev_tx_hash = TransactionHash(*prev_outpoint.prev_txid()); if txid_set.contains(&prev_tx_hash) { // Locate the paired (txid, transparent_data) within this block. - if let Some((tx_index, (_, Some(prev_transparent)))) = transactions + if let Some((tx_index, (_, Some(prev_transparent)))) = pool_lists + .transactions .iter() .enumerate() .find(|(_, (h, _))| h == &prev_tx_hash) @@ -603,12 +692,18 @@ impl DbV1 { .maybe_calculate_tx_out_set_info_accumulator_after_block( update_tx_out_set, block_height, - &transactions, + &pool_lists.transactions, &spent_map, ) .await?; // Split the paired vector into the per-table shapes used for storage. + let BlockPoolLists { + transactions, + sapling, + orchard, + ironwood, + } = pool_lists; let (txids, transparent): (Vec, Vec>) = transactions.into_iter().unzip(); @@ -617,17 +712,7 @@ impl DbV1 { // lets us mark the block validated after a successful write without the expensive // post-commit re-read + spent-index cross-check (which only re-verifies on-disk integrity of // bytes we just wrote from memory, and is redundant for our own writes). - { - let txid_bytes: Vec<[u8; 32]> = txids.iter().map(|txid| txid.0).collect(); - let computed_merkle_root = Self::calculate_block_merkle_root(&txid_bytes); - if &computed_merkle_root != block.data().merkle_root() { - return Err(FinalisedStateError::InvalidBlock { - height: block_height.0, - hash: block_hash, - reason: "header merkle root does not match block txids".to_string(), - }); - } - } + verify_header_merkle_root(&txids, &block)?; // Reverse txid index entries (`txid -> TxLocation`). Built before `txids` is moved into // the `TxidList` below, and sorted by txid so the random-keyed `txid_location` B-tree @@ -645,18 +730,16 @@ impl DbV1 { } txid_location_entries.sort_by_key(|entry| entry.0); - let txid_entry = StoredEntryVar::new(&block_height_bytes, TxidList::new(txids)); - let transparent_entry = - StoredEntryVar::new(&block_height_bytes, TransparentTxList::new(transparent)); - let sapling_entry = StoredEntryVar::new(&block_height_bytes, SaplingTxList::new(sapling)); - let orchard_entry = StoredEntryVar::new(&block_height_bytes, OrchardTxList::new(orchard)); - // The ironwood table is sparse: readers treat an absent row as "no ironwood - // data", so an all-`None` list (every pre-NU6.3 block, and any later block - // whose transactions carry no ironwood actions) is not written. - let ironwood_entry = ironwood - .iter() - .any(Option::is_some) - .then(|| StoredEntryVar::new(&block_height_bytes, OrchardTxList::new(ironwood))); + let entries = build_block_row_entries( + &block, + &block_hash_bytes, + &block_height_bytes, + txids, + transparent, + sapling, + orchard, + ironwood, + ); // if any database writes fail, or block validation fails, remove block from database and return err. let zaino_db = self.detached_handle(); @@ -667,21 +750,21 @@ impl DbV1 { txn.put( zaino_db.headers, &block_height_bytes, - &header_entry.to_bytes()?, + &entries.header_entry.to_bytes()?, WriteFlags::NO_OVERWRITE, )?; txn.put( zaino_db.heights, &block_hash_bytes, - &height_entry.to_bytes()?, + &entries.height_entry.to_bytes()?, WriteFlags::NO_OVERWRITE, )?; txn.put( zaino_db.txids, &block_height_bytes, - &txid_entry.to_bytes()?, + &entries.txid_entry.to_bytes()?, WriteFlags::NO_OVERWRITE, )?; @@ -699,25 +782,25 @@ impl DbV1 { txn.put( zaino_db.transparent, &block_height_bytes, - &transparent_entry.to_bytes()?, + &entries.transparent_entry.to_bytes()?, WriteFlags::NO_OVERWRITE, )?; txn.put( zaino_db.sapling, &block_height_bytes, - &sapling_entry.to_bytes()?, + &entries.sapling_entry.to_bytes()?, WriteFlags::NO_OVERWRITE, )?; txn.put( zaino_db.orchard, &block_height_bytes, - &orchard_entry.to_bytes()?, + &entries.orchard_entry.to_bytes()?, WriteFlags::NO_OVERWRITE, )?; - if let Some(ironwood_entry) = &ironwood_entry { + if let Some(ironwood_entry) = &entries.ironwood_entry { txn.put( zaino_db.ironwood, &block_height_bytes, @@ -729,7 +812,7 @@ impl DbV1 { txn.put( zaino_db.commitment_tree_data, &block_height_bytes, - &commitment_tree_entry.to_bytes()?, + &entries.commitment_tree_entry.to_bytes()?, WriteFlags::NO_OVERWRITE, )?; @@ -1143,64 +1226,10 @@ impl DbV1 { } } - let height_entry = StoredEntryFixed::new(&block_hash_bytes, block_height); - let header_entry = StoredEntryVar::new( - &block_height_bytes, - BlockHeaderData::new(block.context, *block.data()), - ); - let commitment_tree_entry = - StoredEntryVar::new(&block_height_bytes, *block.commitment_tree_data()); - - let tx_len = block.transactions().len(); - let mut txids: Vec = Vec::with_capacity(tx_len); - let mut txid_set: HashSet = HashSet::with_capacity(tx_len); - let mut transparent: Vec> = Vec::with_capacity(tx_len); - let mut sapling = Vec::with_capacity(tx_len); - let mut orchard = Vec::with_capacity(tx_len); - let mut ironwood = Vec::with_capacity(tx_len); + // Build the per-transaction pool lists (with the duplicate-txid guard applied). + let pool_lists = extract_block_pool_lists(&block)?; for (tx_index, tx) in block.transactions().iter().enumerate() { - let hash = tx.txid(); - if !txid_set.insert(*hash) { - return Err(FinalisedStateError::InvalidBlock { - height: block_height.0, - hash: block_hash, - reason: format!("duplicate transaction hash in block: {hash:?}"), - }); - } - txids.push(*hash); - - let transparent_data = if tx.transparent().inputs().is_empty() - && tx.transparent().outputs().is_empty() - { - None - } else { - Some(tx.transparent().clone()) - }; - transparent.push(transparent_data); - - let sapling_data = - if tx.sapling().spends().is_empty() && tx.sapling().outputs().is_empty() { - None - } else { - Some(tx.sapling().clone()) - }; - sapling.push(sapling_data); - - let orchard_data = if tx.orchard().actions().is_empty() { - None - } else { - Some(tx.orchard().clone()) - }; - orchard.push(orchard_data); - - let ironwood_data = if tx.ironwood().actions().is_empty() { - None - } else { - Some(tx.ironwood().clone()) - }; - ironwood.push(ironwood_data); - let tx_index = u16::try_from(tx_index).map_err(|_| FinalisedStateError::InvalidBlock { height: block_height.0, @@ -1213,71 +1242,70 @@ impl DbV1 { spent_batch.push((prev_outpoint.to_bytes()?, tx_location)); } - txid_location_batch.push(((*hash).into(), tx_location)); + txid_location_batch.push(((*tx.txid()).into(), tx_location)); } + let BlockPoolLists { + transactions, + sapling, + orchard, + ironwood, + } = pool_lists; + let (txids, transparent): (Vec, Vec>) = + transactions.into_iter().unzip(); + // Cheap in-memory correctness check: txids must reproduce the header merkle root. - let txid_bytes: Vec<[u8; 32]> = txids.iter().map(|t| t.0).collect(); - let computed_merkle_root = Self::calculate_block_merkle_root(&txid_bytes); - if &computed_merkle_root != block.data().merkle_root() { - return Err(FinalisedStateError::InvalidBlock { - height: block_height.0, - hash: block_hash, - reason: "header merkle root does not match block txids".to_string(), - }); - } + verify_header_merkle_root(&txids, &block)?; - let txid_entry = StoredEntryVar::new(&block_height_bytes, TxidList::new(txids)); - let transparent_entry = - StoredEntryVar::new(&block_height_bytes, TransparentTxList::new(transparent)); - let sapling_entry = - StoredEntryVar::new(&block_height_bytes, SaplingTxList::new(sapling)); - let orchard_entry = - StoredEntryVar::new(&block_height_bytes, OrchardTxList::new(orchard)); - // Sparse ironwood row: see `write_block` — all-`None` lists are not written. - let ironwood_entry = ironwood - .iter() - .any(Option::is_some) - .then(|| StoredEntryVar::new(&block_height_bytes, OrchardTxList::new(ironwood))); + let entries = build_block_row_entries( + &block, + &block_hash_bytes, + &block_height_bytes, + txids, + transparent, + sapling, + orchard, + ironwood, + ); // Height-keyed tables (+ the hash-keyed `heights`, one entry/block) written per block. put_idempotent( &mut txn, self.headers, &block_height_bytes, - &header_entry.to_bytes()?, + &entries.header_entry.to_bytes()?, )?; put_idempotent( &mut txn, self.heights, &block_hash_bytes, - &height_entry.to_bytes()?, + &entries.height_entry.to_bytes()?, )?; put_idempotent( &mut txn, self.txids, &block_height_bytes, - &txid_entry.to_bytes()?, + &entries.txid_entry.to_bytes()?, )?; put_idempotent( &mut txn, self.transparent, &block_height_bytes, - &transparent_entry.to_bytes()?, + &entries.transparent_entry.to_bytes()?, )?; put_idempotent( &mut txn, self.sapling, &block_height_bytes, - &sapling_entry.to_bytes()?, + &entries.sapling_entry.to_bytes()?, )?; put_idempotent( &mut txn, self.orchard, &block_height_bytes, - &orchard_entry.to_bytes()?, + &entries.orchard_entry.to_bytes()?, )?; - if let Some(ironwood_entry) = &ironwood_entry { + if let Some(ironwood_entry) = &entries.ironwood_entry { put_idempotent( &mut txn, self.ironwood, @@ -1289,7 +1317,7 @@ impl DbV1 { &mut txn, self.commitment_tree_data, &block_height_bytes, - &commitment_tree_entry.to_bytes()?, + &entries.commitment_tree_entry.to_bytes()?, )?; prev_height = Some(block_height.0); @@ -1487,7 +1515,8 @@ impl DbV1 { let prev_tx_hash = TransactionHash(*prev_outpoint.prev_txid()); if txid_set.contains(&prev_tx_hash) { // Locate the paired (txid, transparent_data) within this block. - if let Some((tx_index, (_, Some(prev_transparent)))) = transactions + if let Some((tx_index, (_, Some(prev_transparent)))) = pool_lists + .transactions .iter() .enumerate() .find(|(_, (h, _))| h == &prev_tx_hash) From 348558e6818d8e3d7301995aa2e40f4b9c0e0a83 Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 16:35:22 -0700 Subject: [PATCH 063/134] docs: propose pool-indexed dispatch for the chain index Write up the deepest follow-up from the PR #1362 review: every bug the review found was a copy-paste artifact of per-pool code reached by field name. The note proposes making ShieldedPool the index for table, activation, and per-transaction data dispatch, so the compiler's exhaustiveness checking turns the next pool addition (NU7) into a guided edit instead of a copy-paste hunt. Flags the overlap with the paused DB/wire orthogonalization work. Co-Authored-By: Claude Fable 5 --- docs/notes/pool-indexed-dispatch.md | 73 +++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 docs/notes/pool-indexed-dispatch.md diff --git a/docs/notes/pool-indexed-dispatch.md b/docs/notes/pool-indexed-dispatch.md new file mode 100644 index 000000000..e028655a5 --- /dev/null +++ b/docs/notes/pool-indexed-dispatch.md @@ -0,0 +1,73 @@ +# Follow-up: pool-indexed dispatch in the chain index + +Status: proposed (not started), tracked in +[#1367](https://github.com/zingolabs/zaino/issues/1367). Scope: `zaino-state` +chain index. Origin: the PR #1362 (Ironwood/NU6.3) review. + +## Problem + +Ironwood was added to the chain index by copy-paste: every place that handled +"sapling and orchard" gained a third clone. The review of that PR found six +bugs, and every one of them was a copy-paste artifact of exactly this shape: + +- a sapling-output skip reusing the orchard-refactor's spend-width variable; +- the ironwood JSON field reading the `"orchard"` key; +- an error message describing the ironwood pool as "active NU5"; +- per-pool activation gating re-implemented (and drifted) at three sites; +- per-pool tuples widened positionally, letting same-typed roots transpose + silently. + +The review's DRY pass consolidated the *implementations* (shared cursor walks, +`required_pool_root`/`optional_pool_root`, `extract_block_pool_lists`, +`BlockRowEntries`), so each per-pool behaviour now has one definition. What it +did not change is the *dispatch*: pools are still reached by field name +(`self.sapling`, `tx.ironwood()`, `treestate.orchard`), so adding the next +pool still means hand-editing every call site, and the compiler cannot point +at the sites that were missed. + +## Proposal + +Make `ShieldedPool` (already defined in `chain_index.rs`) the index for +per-pool data and behaviour: + +1. **Table dispatch**: `DbV1::pool_table(&self, ShieldedPool) -> lmdb::Database` + replacing direct field access in the pool read/write paths, plus a + `MissingRow` policy per pool (`ShieldedPool::missing_row_policy()` — dense + for sapling/orchard, sparse for ironwood and later pools). +2. **Activation dispatch**: `ShieldedPool::activation_upgrade() -> + NetworkUpgrade` (Sapling → Sapling, Orchard → Nu5, Ironwood → Nu6_3), + consumed by `required_pool_root`/`optional_pool_root` callers instead of + per-site `is_nu_active` triples. +3. **Per-transaction data dispatch**: a method on the indexed transaction type + returning the pool's compact data by `ShieldedPool` value, so + `extract_block_pool_lists` and the compact-block builders iterate pools + instead of naming them. +4. **Exhaustiveness as the safety net**: every per-pool `match` on + `ShieldedPool` (no wildcard arms). Adding the NU7 pool then fails + compilation at every site that needs a decision — the same mechanism that + surfaced the `PoolType::Shielded(ShieldedPool::Ironwood)` non-exhaustive + match errors in the devtool work. + +## Constraints and cautions + +- **Behaviour preserving.** No schema, wire, or semantics change; this is a + dispatch refactor over the already-consolidated helpers. +- **Type asymmetry is real**: sapling has spends+outputs and its own types; + orchard and ironwood share the Orchard compact types. The dispatch layer + must not force a premature unification of the per-pool value types — + associated types or per-pool `match` arms returning distinct types are + acceptable; a trait object is not required. +- **Overlap warning**: this touches the same table-handle layer as the paused + DB/wire orthogonalization work (branch `clean_block_id_ident_relation`). + Reconcile with that plan before starting; doing both independently will + conflict. +- The lightwalletd protocol side (proto field per pool) stays copy-per-pool by + nature of protobuf; this proposal covers the indexer internals only. + +## Suggested shape of the work + +One branch, roughly three commits: (1) `ShieldedPool` gains +`activation_upgrade` / `missing_row_policy` and the connector/finalised-state +call sites consume them; (2) `DbV1::pool_table` + pool-iterating read/write +paths; (3) per-transaction pool-data dispatch and compact-block builders. +Each commit behaviour-preserving with the full `zaino-state` suite green. From b1783b9c9536fa28ba5492b5cd2c0718eb16acee Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 17:21:39 -0700 Subject: [PATCH 064/134] fix(zaino-proto): serve ironwood to unfiltered compact-block requests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pool_types_from_vector's empty-vector backfill (what every unfiltered or pre-Ironwood client sends) listed only Sapling and Orchard, so served compact blocks stripped ironwoodActions while chainMetadata.ironwoodCommitmentTreeSize still counted their commitments — a scanning wallet sees the tree-size discontinuity as a phantom chain reorg at the first block with an Ironwood coinbase. The default now includes Ironwood; clients that predate the field ignore it as an unknown protobuf field. The regression test pins the unfiltered-request path for both pool_types_from_vector and PoolTypeFilter::new_from_slice. Co-Authored-By: Claude Fable 5 --- packages/zaino-proto/src/proto/utils.rs | 28 ++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/packages/zaino-proto/src/proto/utils.rs b/packages/zaino-proto/src/proto/utils.rs index 693617d18..2cb3493d2 100644 --- a/packages/zaino-proto/src/proto/utils.rs +++ b/packages/zaino-proto/src/proto/utils.rs @@ -19,9 +19,16 @@ pub enum PoolTypeError { /// Converts a vector of pool_types (i32) into its rich-type representation /// Returns `PoolTypeError::InvalidPoolType` when invalid `pool_types` are found /// or `PoolTypeError::UnknownPoolType` if unknown ones are found. +/// +/// An empty vector means the client did not filter, so every shielded pool is +/// served — including Ironwood, which clients that predate the field simply +/// ignore as an unknown protobuf field. Backfilling only the pre-NU6.3 pools +/// here would serve blocks whose `chainMetadata.ironwoodCommitmentTreeSize` +/// counts commitments from actions the block omits; a scanning wallet sees +/// that as a tree-size discontinuity and treats it as a chain reorg. pub fn pool_types_from_vector(pool_types: &[i32]) -> Result, PoolTypeError> { let pools = if pool_types.is_empty() { - vec![PoolType::Sapling, PoolType::Orchard] + vec![PoolType::Sapling, PoolType::Orchard, PoolType::Ironwood] } else { let mut pools: Vec = vec![]; @@ -492,4 +499,23 @@ mod test { PoolTypeFilter::includes_all() ); } + + /// Regression: an unfiltered request (empty `poolTypes`, what every + /// pre-Ironwood client sends) must be served Ironwood actions. When the + /// empty-vector backfill listed only the pre-NU6.3 shielded pools, the + /// served compact blocks stripped `ironwoodActions` while + /// `chainMetadata.ironwoodCommitmentTreeSize` still counted them, and + /// scanning wallets reported a tree-size discontinuity (a phantom chain + /// reorg) at the first block with an Ironwood coinbase. + #[test] + fn empty_pool_types_request_includes_ironwood() { + let pools = crate::proto::utils::pool_types_from_vector(&[]).unwrap(); + assert!(pools.contains(&PoolType::Ironwood), "{pools:?}"); + + let filter = PoolTypeFilter::new_from_slice(&[]).unwrap(); + assert!(filter.includes_ironwood()); + assert!(filter.includes_sapling()); + assert!(filter.includes_orchard()); + assert!(!filter.includes_transparent()); + } } From 289d63db58be66350e009508af1af72f7df06705 Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 17:24:47 -0700 Subject: [PATCH 065/134] ci(publish-dry-run): verify the workspace as a set and gate version reuse MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The per-crate `cargo publish -p X --dry-run` loop could never pass on a branch with cross-crate changes: dry-run uploads nothing, so each crate resolved its zaino siblings from crates.io instead of the workspace (zaino-fetch compiled against the published zaino-proto and missed the new ironwood field; zaino-serve/zainod required the unpublished zaino-fetch/zcashd_support feature). The job's failure text then blamed [patch.crates-io], which is empty. Replace the loop with `cargo publish --workspace --dry-run` (cargo 1.90+), which verifies the publishable set against a local overlay registry — emulating a real dependency-ordered release — and derives the crate list from `publish = false` metadata instead of hard-coding it. Add the check the old loop only failed at by accident: a new workbench binary, check-published-versions, flags any publishable crate whose exact version already exists on crates.io with different packaged content (byte-compare, excluding only the commit-stamped .cargo_vcs_info.json, with a truncated unified diff in the report). Such a crate cannot be released without a version bump. In advisory contexts (everything except rc/**, stable, and PRs targeting them) the finding is a warning, since unbumped-but-changed is the normal bump-at-release state on feature branches; in blocking contexts it fails the job. Also drop the workflow_call trigger — nothing calls this workflow. Co-Authored-By: Claude Fable 5 --- .github/workflows/publish-dry-run.yml | 81 ++-- .../src/bin/check-published-versions.rs | 450 ++++++++++++++++++ 2 files changed, 494 insertions(+), 37 deletions(-) create mode 100644 tools/workbench/src/bin/check-published-versions.rs diff --git a/.github/workflows/publish-dry-run.yml b/.github/workflows/publish-dry-run.yml index 195f94f2f..62cb933bb 100644 --- a/.github/workflows/publish-dry-run.yml +++ b/.github/workflows/publish-dry-run.yml @@ -1,7 +1,6 @@ name: Publish dry-run on: - workflow_call: workflow_dispatch: push: branches: @@ -14,8 +13,11 @@ jobs: publish-dry-run: name: cargo publish --dry-run runs-on: ubuntu-latest - # On rc/* and stable this job MUST pass; everywhere else it is advisory. - # For PRs, check the base (target) branch; for pushes, check the ref itself. + # Blocking context (pushes to rc/** or stable, and PRs targeting them): + # this job MUST pass. Advisory context (everything else): findings inform + # but the run stays green. Keep this predicate the exact inverse of the + # "Compute check mode" step below — job-level continue-on-error cannot + # read step outputs, so the two encode the same rule twice. continue-on-error: >- ${{ github.event_name == 'pull_request' @@ -32,49 +34,54 @@ jobs: # rust-toolchain.toml auto-installs the pinned toolchain on first cargo # invocation, so no explicit toolchain setup step is needed. - - name: Dry-run publish (dependency order) - id: dry-run + - name: Compute check mode + id: mode run: | - set -euo pipefail + # Inverse of the job's continue-on-error predicate above. + if [[ "${{ github.event_name }}" == "pull_request" ]]; then + target="${{ github.base_ref }}" + else + target="${{ github.ref_name }}" + fi + if [[ "$target" == rc/* || "$target" == stable ]]; then + echo "mode=blocking" >> "$GITHUB_OUTPUT" + else + echo "mode=advisory" >> "$GITHUB_OUTPUT" + fi - # Publishable crates in dependency order. - CRATES=( - zaino-proto - zaino-common - zaino-fetch - zaino-state - zaino-serve - zainod - ) + # A publishable crate whose exact version is already on crates.io with + # different content cannot be released without a bump. Advisory mode + # warns (unbumped-but-changed is the normal bump-at-release state on + # feature branches); blocking mode fails. + - name: Check for version-reuse violations + run: | + cargo run --manifest-path tools/workbench/Cargo.toml \ + --bin check-published-versions -- --mode "${{ steps.mode.outputs.mode }}" - failed=() - for crate in "${CRATES[@]}"; do - echo "::group::$crate" - if cargo publish -p "$crate" --dry-run 2>&1; then - echo " $crate: OK" - else - failed+=("$crate") - echo "::error::cargo publish --dry-run failed for $crate" - fi - echo "::endgroup::" - done + # --workspace verifies the publishable set against a local overlay + # registry, so sibling dependencies resolve to their in-workspace + # versions — emulating a real dependency-ordered release. A per-crate + # dry-run loop cannot do this: nothing is uploaded between iterations, + # so dependents resolve siblings from crates.io and any cross-crate + # change fails spuriously. + - name: Dry-run publish (workspace overlay) + run: | + set -euo pipefail - if [[ ${#failed[@]} -gt 0 ]]; then + if ! cargo publish --workspace --dry-run 2>&1; then echo "" echo "============================================" - echo "Publish dry-run FAILED for: ${failed[*]}" + echo "cargo publish --workspace --dry-run FAILED" echo "============================================" echo "" - echo "This usually means a [patch.crates-io] override is pulling in" - echo "unreleased upstream code. Published crates cannot depend on" - echo "unpublished revisions — resolve the patch before releasing." + echo "The workspace does not publish cleanly as a set. Common causes:" + echo " - a [patch.crates-io] override pulling in unreleased upstream code" + echo " - a dependency on a git revision or path outside the workspace" + echo " - a crate that does not compile from its packaged form" exit 1 fi - - name: Annotate on failure (advisory contexts only) - if: >- - failure() - && (github.event_name != 'push' || (!startsWith(github.ref_name, 'rc/') && github.ref_name != 'stable')) - && (github.event_name != 'pull_request' || (!startsWith(github.base_ref, 'rc/') && github.base_ref != 'stable')) + - name: Annotate on failure (advisory context only) + if: failure() && steps.mode.outputs.mode == 'advisory' run: | - echo "::warning::cargo publish --dry-run failed — one or more crates depend on patched (unreleased) upstream code. This must be resolved before the next release." + echo "::warning::publish dry-run failed — the workspace would not release cleanly. This must be resolved before the next release." diff --git a/tools/workbench/src/bin/check-published-versions.rs b/tools/workbench/src/bin/check-published-versions.rs new file mode 100644 index 000000000..40e3fb486 --- /dev/null +++ b/tools/workbench/src/bin/check-published-versions.rs @@ -0,0 +1,450 @@ +//! Guard: no publishable crate reuses an already-published version number +//! with different content. +//! +//! For every crate `cargo package --workspace` produces, look its exact +//! version up on the crates.io sparse index. A version that is not on the +//! index is fine (a fresh number, or a never-published crate). A published +//! version is fine only when the packaged content is byte-identical to the +//! published `.crate` — an unchanged crate legitimately keeps its number and +//! is simply skipped at release. Anything else is a version-reuse violation: +//! the tree cannot be released until that crate's version is bumped. +//! +//! `.cargo_vcs_info.json` is excluded from the comparison — it embeds the +//! packaging commit's SHA, so it differs on every commit even when the +//! source is identical. Yanked versions count as published: crates.io never +//! frees a version number. +//! +//! `--mode advisory` reports violations as warnings and exits 0 (feature +//! branches, where unbumped-but-changed crates are the normal +//! bump-at-release state). `--mode blocking` reports them as errors and +//! exits 1 (`rc/**` and `stable` release gates). +//! +//! Std-only by crate design: network, extraction, and diffing go through +//! `curl`, `tar`, and `diff` subprocesses. + +use std::path::{Path, PathBuf}; +use std::process::Command; +use workbench::{repo_root, run}; + +const PROG: &str = "check-published-versions"; +const DIFF_LINE_CAP: usize = 120; + +/// Failure disposition for version-reuse violations. Infrastructure errors +/// (network, subprocess) fail loudly in either mode — a check that could not +/// run must not read as a pass. +enum Mode { + Advisory, + Blocking, +} + +struct PackagedCrate { + name: String, + version: String, +} + +impl PackagedCrate { + fn id(&self) -> String { + format!("{}@{}", self.name, self.version) + } +} + +fn main() { + run(PROG, check, |summary| println!("{PROG}: {summary}")) +} + +fn check() -> Result> { + let mode = mode_from_args(std::env::args().skip(1))?; + let root = repo_root()?; + let scratch = create_scratch_dir()?; + + let packaged = package_workspace(&root)?; + let mut violations = Vec::new(); + for krate in &packaged { + if let Comparison::Differs(diff) = compare_against_published(&root, &scratch, krate)? { + violations.push((krate, diff)); + } + } + let _ = std::fs::remove_dir_all(&scratch); + + if violations.is_empty() { + return Ok(format!( + "ok — none of the {} publishable crates reuses a published version with different content", + packaged.len() + )); + } + + let offenders: Vec = violations.iter().map(|(k, _)| k.id()).collect(); + let summary = format!( + "version-reuse violation(s): {} — already published with different content; \ + bump these versions before the next release", + offenders.join(", ") + ); + match mode { + Mode::Advisory => { + for (krate, diff) in &violations { + println!( + "{PROG}: {} is published with different content:", + krate.id() + ); + println!("{diff}"); + } + println!("::warning::{summary}"); + Ok(summary) + } + Mode::Blocking => { + let mut lines = vec![format!("::error::{summary}")]; + for (krate, diff) in violations { + lines.push(format!( + "{} is published with different content:", + krate.id() + )); + lines.push(diff); + } + Err(lines) + } + } +} + +/// A per-process scratch directory for downloads and extractions, removed +/// (best-effort) at the end of the check. +fn create_scratch_dir() -> Result> { + let dir = std::env::temp_dir().join(format!("{PROG}-{}", std::process::id())); + std::fs::create_dir_all(&dir) + .map_err(|e| vec![format!("cannot create scratch dir {}: {e}", dir.display())])?; + Ok(dir) +} + +fn mode_from_args(mut args: impl Iterator) -> Result> { + match (args.next().as_deref(), args.next().as_deref(), args.next()) { + (Some("--mode"), Some("advisory"), None) => Ok(Mode::Advisory), + (Some("--mode"), Some("blocking"), None) => Ok(Mode::Blocking), + _ => Err(vec![format!("usage: {PROG} --mode ")]), + } +} + +/// Package every publishable workspace member (their `.crate` files land in +/// `/target/package/`) and return the packaged name/version pairs. +/// +/// This goes through `cargo publish --dry-run` rather than `cargo package`: +/// only the former skips `publish = false` members (`cargo package +/// --workspace` fails on the live-test crates' versionless path +/// dependencies). `--workspace` resolves sibling dependencies against a +/// local overlay of the registry, so this succeeds even when members depend +/// on unpublished sibling changes. `--allow-dirty` keeps local +/// (uncommitted-tree) runs useful; CI checkouts are clean regardless. +fn package_workspace(root: &Path) -> Result, Vec> { + let output = Command::new("cargo") + .current_dir(root) + .args([ + "publish", + "--workspace", + "--dry-run", + "--no-verify", + "--allow-dirty", + ]) + .output() + .map_err(|e| vec![format!("failed to run cargo publish --dry-run: {e}")])?; + let stderr = String::from_utf8_lossy(&output.stderr); + if !output.status.success() { + let mut lines = + vec!["`cargo publish --workspace --dry-run --no-verify` failed:".to_string()]; + lines.extend(stderr.lines().map(str::to_string)); + return Err(lines); + } + let packaged: Vec = stderr.lines().filter_map(packaged_crate_line).collect(); + if packaged.is_empty() { + return Err(vec![ + "cargo package reported no packaged crates — nothing to check".to_string(), + ]); + } + Ok(packaged) +} + +/// Parse a cargo status line `" Packaging zaino-common v0.2.0 (/path)"`. +fn packaged_crate_line(line: &str) -> Option { + let rest = line.trim_start().strip_prefix("Packaging ")?; + let mut words = rest.split_whitespace(); + let name = words.next()?.to_string(); + let version = words.next()?.strip_prefix('v')?.to_string(); + Some(PackagedCrate { name, version }) +} + +enum Comparison { + /// The exact version is not on the index (or the crate never published). + NotPublished, + /// Published, and the packaged content is byte-identical. + Identical, + /// Published with different content — the violation. Carries the + /// (possibly truncated) unified diff, published → local. + Differs(String), +} + +fn compare_against_published( + root: &Path, + scratch: &Path, + krate: &PackagedCrate, +) -> Result> { + let index_body = match fetch_index(scratch, &krate.name)? { + IndexLookup::NeverPublished => return Ok(Comparison::NotPublished), + IndexLookup::Versions(body) => body, + }; + if !index_lists_version(&index_body, &krate.version) { + return Ok(Comparison::NotPublished); + } + + let published_crate = download_published(scratch, krate)?; + let local_crate = local_crate_path(root, krate)?; + + let published_dir = scratch.join(krate.id()).join("published"); + let local_dir = scratch.join(krate.id()).join("local"); + extract(&published_crate, &published_dir)?; + extract(&local_crate, &local_dir)?; + + diff_trees(&published_dir, &local_dir) +} + +/// The `.crate` file [`package_workspace`] produced for `krate`. A +/// single-package `cargo package` writes to `target/package/`; the +/// multi-package `cargo publish --workspace --dry-run` stages its output in +/// `target/package/tmp-crate/` instead, so both locations are probed. +fn local_crate_path(root: &Path, krate: &PackagedCrate) -> Result> { + let file = format!("{}-{}.crate", krate.name, krate.version); + let candidates = [ + root.join("target/package").join(&file), + root.join("target/package/tmp-crate").join(&file), + ]; + candidates + .iter() + .find(|p| p.is_file()) + .cloned() + .ok_or_else(|| { + vec![format!( + "packaged {} not found at {} or {}", + krate.id(), + candidates[0].display(), + candidates[1].display() + )] + }) +} + +enum IndexLookup { + NeverPublished, + Versions(String), +} + +fn fetch_index(scratch: &Path, name: &str) -> Result> { + let body_file = scratch.join(format!("{name}.index")); + let url = format!("https://index.crates.io/{}", index_path(name)); + let status_code = curl_with_status(&url, &body_file)?; + match status_code.as_str() { + "200" => Ok(IndexLookup::Versions(workbench::read(&body_file)?)), + "404" => Ok(IndexLookup::NeverPublished), + other => Err(vec![format!( + "sparse-index lookup for {name} returned HTTP {other} ({url})" + )]), + } +} + +/// Download `url` to `out`, returning the HTTP status code. Transport-level +/// failures (DNS, TLS, timeouts) are errors; HTTP status is the caller's to +/// interpret. +fn curl_with_status(url: &str, out: &Path) -> Result> { + let output = Command::new("curl") + .args(["--silent", "--show-error", "--retry", "3"]) + .args(["--output".as_ref(), out.as_os_str()]) + .args(["--write-out", "%{http_code}", url]) + .output() + .map_err(|e| vec![format!("failed to run curl: {e}")])?; + if !output.status.success() { + return Err(vec![format!( + "curl {url} failed: {}", + String::from_utf8_lossy(&output.stderr).trim() + )]); + } + String::from_utf8(output.stdout).map_err(|e| vec![format!("curl status not utf-8: {e}")]) +} + +/// crates.io sparse-index path for a crate name (the registry's length rules). +fn index_path(name: &str) -> String { + match name.len() { + 1 => format!("1/{name}"), + 2 => format!("2/{name}"), + 3 => format!("3/{}/{name}", &name[..1]), + _ => format!("{}/{}/{name}", &name[..2], &name[2..4]), + } +} + +/// Whether the sparse-index body lists `version`. Yanked entries still count: +/// a yanked version number can never be republished. +fn index_lists_version(index_body: &str, version: &str) -> bool { + index_body + .lines() + .any(|line| vers_value(line).is_some_and(|v| v == version)) +} + +/// The `"vers"` value of one sparse-index JSON line. Only the top-level +/// version record carries a `"vers":"…"` string pair, so a substring scan is +/// sufficient for the machine-generated index format. +fn vers_value(line: &str) -> Option<&str> { + let start = line.find("\"vers\":\"")? + "\"vers\":\"".len(); + let rest = &line[start..]; + Some(&rest[..rest.find('"')?]) +} + +fn download_published(scratch: &Path, krate: &PackagedCrate) -> Result> { + let out = scratch.join(format!("{}.published.crate", krate.id())); + let url = format!( + "https://static.crates.io/crates/{}/{}-{}.crate", + krate.name, krate.name, krate.version + ); + let status_code = curl_with_status(&url, &out)?; + if status_code != "200" { + return Err(vec![format!( + "download of published {} returned HTTP {status_code} ({url})", + krate.id() + )]); + } + Ok(out) +} + +fn extract(archive: &Path, dest: &Path) -> Result<(), Vec> { + std::fs::create_dir_all(dest) + .map_err(|e| vec![format!("cannot create {}: {e}", dest.display())])?; + let output = Command::new("tar") + .args(["--extract", "--gzip", "--file"]) + .arg(archive) + .args(["--directory".as_ref(), dest.as_os_str()]) + .output() + .map_err(|e| vec![format!("failed to run tar: {e}")])?; + if !output.status.success() { + return Err(vec![format!( + "tar extraction of {} failed: {}", + archive.display(), + String::from_utf8_lossy(&output.stderr).trim() + )]); + } + Ok(()) +} + +/// Recursive unified diff, published → local, excluding the always-different +/// `.cargo_vcs_info.json` (it embeds the packaging commit's SHA). +fn diff_trees(published_dir: &Path, local_dir: &Path) -> Result> { + let output = Command::new("diff") + .args([ + "--recursive", + "--unified", + "--exclude", + ".cargo_vcs_info.json", + ]) + .args([published_dir.as_os_str(), local_dir.as_os_str()]) + .output() + .map_err(|e| vec![format!("failed to run diff: {e}")])?; + match output.status.code() { + Some(0) => Ok(Comparison::Identical), + Some(1) => Ok(Comparison::Differs(truncate_lines( + &String::from_utf8_lossy(&output.stdout), + DIFF_LINE_CAP, + ))), + _ => Err(vec![format!( + "diff failed: {}", + String::from_utf8_lossy(&output.stderr).trim() + )]), + } +} + +/// First `cap` lines of `text`, with an elision note when truncated. +fn truncate_lines(text: &str, cap: usize) -> String { + let total = text.lines().count(); + if total <= cap { + return text.trim_end().to_string(); + } + let kept: Vec<&str> = text.lines().take(cap).collect(); + format!( + "{}\n… (diff truncated: {} more lines; run `cargo run --manifest-path \ + tools/workbench/Cargo.toml --bin {PROG} -- --mode advisory` locally for the full diff)", + kept.join("\n"), + total - cap + ) +} + +#[cfg(test)] +mod packaged_crate_line { + use super::packaged_crate_line; + + #[test] + fn parses_cargo_status_lines() { + let parsed = packaged_crate_line(" Packaging zaino-common v0.2.0 (/w/zaino-common)") + .expect("status line parses"); + assert_eq!(parsed.name, "zaino-common"); + assert_eq!(parsed.version, "0.2.0"); + } + + #[test] + fn keeps_prerelease_versions_intact() { + let parsed = packaged_crate_line(" Packaging zainod v0.4.3-ironwood.1 (/w/zainod)") + .expect("prerelease line parses"); + assert_eq!(parsed.version, "0.4.3-ironwood.1"); + } + + #[test] + fn ignores_other_cargo_output() { + assert!(packaged_crate_line(" Updating crates.io index").is_none()); + assert!(packaged_crate_line("warning: crate zaino-proto@0.1.3 already exists").is_none()); + assert!(packaged_crate_line(" Packaged 17 files, 135.4KiB").is_none()); + } +} + +#[cfg(test)] +mod index_path { + use super::index_path; + + #[test] + fn follows_the_registry_length_rules() { + assert_eq!(index_path("a"), "1/a"); + assert_eq!(index_path("ab"), "2/ab"); + assert_eq!(index_path("abc"), "3/a/abc"); + assert_eq!(index_path("zainod"), "za/in/zainod"); + assert_eq!(index_path("zaino-state"), "za/in/zaino-state"); + } +} + +#[cfg(test)] +mod index_lists_version { + use super::index_lists_version; + + const INDEX: &str = concat!( + r#"{"name":"zainod","vers":"0.3.0","deps":[{"name":"vers","req":"^1"}],"yanked":false}"#, + "\n", + r#"{"name":"zainod","vers":"0.3.1","deps":[],"yanked":true}"#, + "\n", + ); + + #[test] + fn finds_exact_versions_only() { + assert!(index_lists_version(INDEX, "0.3.0")); + assert!(!index_lists_version(INDEX, "0.3")); + assert!(!index_lists_version(INDEX, "0.3.2")); + } + + #[test] + fn yanked_versions_still_count_as_published() { + assert!(index_lists_version(INDEX, "0.3.1")); + } +} + +#[cfg(test)] +mod truncate_lines { + use super::truncate_lines; + + #[test] + fn short_diffs_pass_through() { + assert_eq!(truncate_lines("a\nb\n", 3), "a\nb"); + } + + #[test] + fn long_diffs_are_capped_with_an_elision_note() { + let capped = truncate_lines("a\nb\nc\nd\n", 2); + assert!(capped.starts_with("a\nb\n…")); + assert!(capped.contains("2 more lines")); + } +} From 8e4a0a441de3bbaa519417cc952d4086c6c4fa61 Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 17:24:47 -0700 Subject: [PATCH 066/134] docs: start a CONTEXT.md glossary with the release-engineering terms Pins down "publishable set", "blocking context", "advisory context", and "version-reuse violation" as used by the publish dry-run workflow and the check-published-versions workbench tool. Co-Authored-By: Claude Fable 5 --- CONTEXT.md | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 CONTEXT.md diff --git a/CONTEXT.md b/CONTEXT.md new file mode 100644 index 000000000..4d40dd72f --- /dev/null +++ b/CONTEXT.md @@ -0,0 +1,31 @@ +# Zaino + +Zaino is a Zcash indexing service. This glossary pins down the canonical +terms for concepts where the team has picked one word among several. + +## Language + +### Release engineering + +**Publishable set**: +The workspace members released to crates.io as a unit — every member not +marked `publish = false`. Derived from workspace metadata, never from a +hard-coded list. +_Avoid_: crate list, publish list + +**Blocking context**: +A CI context in which release checks must pass: pushes to `rc/**` or +`stable`, and pull requests targeting them. +_Avoid_: strict mode, release mode + +**Advisory context**: +Any CI context that is not a blocking context. Release checks report +findings (warnings, annotations) but do not fail the build there. +_Avoid_: soft mode, informational mode + +**Version-reuse violation**: +A publishable crate whose exact version already exists on crates.io while +its packaged content differs. The tree cannot be released until that crate's +version is bumped. An unchanged crate keeping its published version is not a +violation. +_Avoid_: stale version, forgotten bump From cee8c382820766a6b6cc69195456a5036652497d Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 17:42:40 -0700 Subject: [PATCH 067/134] ci(publish-dry-run): skip packaged-form verify while versions are unbumped MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The workspace overlay only governs packaging and resolution. In the verify build, a local crate that reuses an already-published version number is shadowed by the registry artifact: dependents verify-compile against the published content, not the tree's (zaino-fetch failed against the published zaino-proto 0.1.3, which lacks the ironwood fields). So packaged-form verification is structurally meaningless exactly when version-reuse violations exist — the normal bump-at-release state on feature branches. check-published-versions now emits a `violations` step output (under GitHub Actions only), and the dry-run step degrades to --no-verify while violations exist, stating why. Blocking contexts are unaffected: violations fail the job before the dry-run, and once release prep bumps the versions the full verify runs where this job gates. Co-Authored-By: Claude Fable 5 --- .github/workflows/publish-dry-run.yml | 37 +++++++++++++++---- .../src/bin/check-published-versions.rs | 20 ++++++++++ 2 files changed, 49 insertions(+), 8 deletions(-) diff --git a/.github/workflows/publish-dry-run.yml b/.github/workflows/publish-dry-run.yml index 62cb933bb..403de3465 100644 --- a/.github/workflows/publish-dry-run.yml +++ b/.github/workflows/publish-dry-run.yml @@ -52,23 +52,44 @@ jobs: # A publishable crate whose exact version is already on crates.io with # different content cannot be released without a bump. Advisory mode # warns (unbumped-but-changed is the normal bump-at-release state on - # feature branches); blocking mode fails. + # feature branches); blocking mode fails. Also emits the `violations` + # output consumed by the dry-run step below. - name: Check for version-reuse violations + id: version-check run: | cargo run --manifest-path tools/workbench/Cargo.toml \ --bin check-published-versions -- --mode "${{ steps.mode.outputs.mode }}" - # --workspace verifies the publishable set against a local overlay - # registry, so sibling dependencies resolve to their in-workspace - # versions — emulating a real dependency-ordered release. A per-crate - # dry-run loop cannot do this: nothing is uploaded between iterations, - # so dependents resolve siblings from crates.io and any cross-crate - # change fails spuriously. + # --workspace resolves sibling dependencies against a local overlay + # registry, so packaging succeeds even when members depend on + # unpublished sibling changes — emulating a real dependency-ordered + # release. A per-crate dry-run loop cannot do this: nothing is uploaded + # between iterations, so dependents resolve siblings from crates.io and + # any cross-crate change fails spuriously. + # + # The verify build is weaker than packaging: when a local crate reuses + # an already-published version number, the registry artifact shadows + # the overlay copy, so dependents verify-compile against the published + # content, not the tree's. While version-reuse violations exist + # (advisory contexts only — blocking contexts already failed above), + # verification is therefore meaningless and the dry-run degrades to + # packaging/resolution checks. Release prep bumps the versions, and the + # full verify then runs where this job gates. - name: Dry-run publish (workspace overlay) run: | set -euo pipefail - if ! cargo publish --workspace --dry-run 2>&1; then + verify_flag="" + if [[ "${{ steps.version-check.outputs.violations }}" == "true" ]]; then + verify_flag="--no-verify" + echo "Version-reuse violations present: packaged crates would verify-compile" + echo "against the already-published versions that shadow the workspace overlay." + echo "Running packaging and resolution checks only (--no-verify) until the" + echo "versions are bumped at release prep." + echo "" + fi + + if ! cargo publish --workspace --dry-run $verify_flag 2>&1; then echo "" echo "============================================" echo "cargo publish --workspace --dry-run FAILED" diff --git a/tools/workbench/src/bin/check-published-versions.rs b/tools/workbench/src/bin/check-published-versions.rs index 40e3fb486..8d7bf8fd7 100644 --- a/tools/workbench/src/bin/check-published-versions.rs +++ b/tools/workbench/src/bin/check-published-versions.rs @@ -65,6 +65,7 @@ fn check() -> Result> { } } let _ = std::fs::remove_dir_all(&scratch); + write_github_output(!violations.is_empty())?; if violations.is_empty() { return Ok(format!( @@ -105,6 +106,25 @@ fn check() -> Result> { } } +/// Publish a `violations=` step output when running under GitHub +/// Actions (the `GITHUB_OUTPUT` file is set). Downstream steps use it to +/// degrade the publish dry-run to `--no-verify`: a crate that reuses a +/// published version is *shadowed* by the registry during verify builds +/// (cargo resolves the already-published artifact over the workspace +/// overlay), so packaged-form verification of its dependents is meaningless +/// until versions are bumped. Outside GitHub Actions this is a no-op. +fn write_github_output(violations: bool) -> Result<(), Vec> { + let Ok(path) = std::env::var("GITHUB_OUTPUT") else { + return Ok(()); + }; + let line = format!("violations={violations}\n"); + std::fs::OpenOptions::new() + .append(true) + .open(&path) + .and_then(|mut file| std::io::Write::write_all(&mut file, line.as_bytes())) + .map_err(|e| vec![format!("cannot append to GITHUB_OUTPUT ({path}): {e}")]) +} + /// A per-process scratch directory for downloads and extractions, removed /// (best-effort) at the end of the check. fn create_scratch_dir() -> Result> { From b13c4d9a6e31cb8b18c8f53b868f56b2f291625b Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 16:48:58 -0700 Subject: [PATCH 068/134] refactor(chain-index): pool activation dispatch on ShieldedPool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ShieldedPool::activation_upgrade() (and its zcash_protocol twin) now owns the pool -> network-upgrade mapping; the eight production sites that hand-wrote NetworkUpgrade::{Sapling,Nu5,Nu6_3} per pool consume it (validator connector treestate gating, finalised-state fixture sync, ephemeral root gating, write-path activation heights, migration tip guard, backend RPC arms). One definition of "which upgrade activates this pool", in both upgrade type systems — the mapping the drifted "active NU5 block" ironwood message got wrong is no longer writable by hand. Commit 1 of the pool-indexed dispatch plan (#1367, docs/notes/pool-indexed-dispatch.md). Behaviour preserved; full zaino-state suite green. Co-Authored-By: Claude Fable 5 --- packages/zaino-state/src/backends/state.rs | 9 ++++++-- packages/zaino-state/src/chain_index.rs | 21 +++++++++++++++++++ .../src/chain_index/finalised_state.rs | 3 ++- .../finalised_source/ephemeral.rs | 6 +++--- .../finalised_source/v1/write_core.rs | 14 +++++++++---- .../chain_index/finalised_state/migrations.rs | 5 +++-- .../chain_index/source/validator_connector.rs | 9 +++++--- 7 files changed, 52 insertions(+), 15 deletions(-) diff --git a/packages/zaino-state/src/backends/state.rs b/packages/zaino-state/src/backends/state.rs index e7f9557aa..33c19c87c 100644 --- a/packages/zaino-state/src/backends/state.rs +++ b/packages/zaino-state/src/backends/state.rs @@ -508,7 +508,9 @@ impl StateServiceSubscriber { let mut nonce = *header.nonce; nonce.reverse(); - let sapling_activation = NetworkUpgrade::Sapling.activation_height(&network); + let sapling_activation = crate::chain_index::ShieldedPool::Sapling + .activation_upgrade() + .activation_height(&network); let sapling_tree_size = sapling_tree.count(); let final_sapling_root: [u8; 32] = if sapling_activation.is_some() && height >= sapling_activation.unwrap() { @@ -844,7 +846,10 @@ impl StateServiceSubscriber { "missing orchard tree", )))?; - let final_orchard_root = match NetworkUpgrade::Nu5.activation_height(network) { + let final_orchard_root = match crate::chain_index::ShieldedPool::Orchard + .activation_upgrade() + .activation_height(network) + { Some(activation_height) if header_obj.height() >= activation_height => { Some(orchard_tree.root().into()) } diff --git a/packages/zaino-state/src/chain_index.rs b/packages/zaino-state/src/chain_index.rs index 8b2a59607..84d8829b3 100644 --- a/packages/zaino-state/src/chain_index.rs +++ b/packages/zaino-state/src/chain_index.rs @@ -2813,6 +2813,27 @@ pub enum ShieldedPool { } impl ShieldedPool { + /// The network upgrade that activates this pool. + pub(crate) fn activation_upgrade(&self) -> zebra_chain::parameters::NetworkUpgrade { + match self { + ShieldedPool::Sapling => zebra_chain::parameters::NetworkUpgrade::Sapling, + ShieldedPool::Orchard => zebra_chain::parameters::NetworkUpgrade::Nu5, + ShieldedPool::Ironwood => zebra_chain::parameters::NetworkUpgrade::Nu6_3, + } + } + + /// [`ShieldedPool::activation_upgrade`] in `zcash_protocol` terms, for call sites + /// gated through [`zcash_protocol::consensus::Parameters`]. + pub(crate) fn zcash_protocol_activation_upgrade( + &self, + ) -> zcash_protocol::consensus::NetworkUpgrade { + match self { + ShieldedPool::Sapling => zcash_protocol::consensus::NetworkUpgrade::Sapling, + ShieldedPool::Orchard => zcash_protocol::consensus::NetworkUpgrade::Nu5, + ShieldedPool::Ironwood => zcash_protocol::consensus::NetworkUpgrade::Nu6_3, + } + } + /// Returns the string representative of the given pool. /// /// Used for display purposes and in converting the strongly types `PoolType` diff --git a/packages/zaino-state/src/chain_index/finalised_state.rs b/packages/zaino-state/src/chain_index/finalised_state.rs index c26c0b3e6..11c824ec1 100644 --- a/packages/zaino-state/src/chain_index/finalised_state.rs +++ b/packages/zaino-state/src/chain_index/finalised_state.rs @@ -1096,7 +1096,8 @@ impl FinalisedState { // Ironwood (NU6.3) commitment tree data is only expected from activation. Below activation // (or on a network with no NU6.3 activation height) the source has no ironwood root, so it // defaults — mirroring `build_indexed_block_from_source`. - let nu6_3_activation_height = zebra_chain::parameters::NetworkUpgrade::Nu6_3 + let nu6_3_activation_height = super::ShieldedPool::Ironwood + .activation_upgrade() .activation_height(&cfg.network.to_zebra_network()); let mut parent_chainwork: Option = None; diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/ephemeral.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/ephemeral.rs index 04e2b6b18..59bfe057e 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/ephemeral.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/ephemeral.rs @@ -276,15 +276,15 @@ impl EphemeralFinalisedState { self.source.get_commitment_tree_roots(block_hash).await?; let sapling_is_active = self.network.is_nu_active( - zcash_protocol::consensus::NetworkUpgrade::Sapling, + ShieldedPool::Sapling.zcash_protocol_activation_upgrade(), block_height.into(), ); let orchard_is_active = self.network.is_nu_active( - zcash_protocol::consensus::NetworkUpgrade::Nu5, + ShieldedPool::Orchard.zcash_protocol_activation_upgrade(), block_height.into(), ); let ironwood_is_active = self.network.is_nu_active( - zcash_protocol::consensus::NetworkUpgrade::Nu6_3, + ShieldedPool::Ironwood.zcash_protocol_activation_upgrade(), block_height.into(), ); diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/write_core.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/write_core.rs index 525452cf2..261b08e1f 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/write_core.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/write_core.rs @@ -2,6 +2,8 @@ use super::*; +use crate::chain_index::ShieldedPool; + #[cfg(feature = "prometheus")] use crate::metric_names::*; @@ -208,15 +210,19 @@ impl DbWrite for DbV1 { source: &S, ) -> Result<(), FinalisedStateError> { use crate::chain_index::finalised_state::build_indexed_block_from_source; - use zebra_chain::parameters::NetworkUpgrade; let network = self.config.network; let zebra_network = network.to_zebra_network(); - let sapling_activation_height = NetworkUpgrade::Sapling + let sapling_activation_height = ShieldedPool::Sapling + .activation_upgrade() .activation_height(&zebra_network) .expect("Sapling activation height must be set"); - let nu5_activation_height = NetworkUpgrade::Nu5.activation_height(&zebra_network); - let nu6_3_activation_height = NetworkUpgrade::Nu6_3.activation_height(&zebra_network); + let nu5_activation_height = ShieldedPool::Orchard + .activation_upgrade() + .activation_height(&zebra_network); + let nu6_3_activation_height = ShieldedPool::Ironwood + .activation_upgrade() + .activation_height(&zebra_network); // Seed `parent_chainwork` from the current tip header (the block before the first one we // write). On an empty database this is genesis with zero chainwork. Read raw rather than via diff --git a/packages/zaino-state/src/chain_index/finalised_state/migrations.rs b/packages/zaino-state/src/chain_index/finalised_state/migrations.rs index df013dc37..b88009194 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/migrations.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/migrations.rs @@ -1094,8 +1094,9 @@ impl Migration for Migration1_2_1To1_3_0 { // Guard: Ironwood data is only expected from NU6.3 activation. A rebuild-from-existing-data // migration cannot reconstruct ironwood roots/sizes for post-NU6.3 blocks. let zebra_network = cfg.network.to_zebra_network(); - let nu6_3_activation_height = - zebra_chain::parameters::NetworkUpgrade::Nu6_3.activation_height(&zebra_network); + let nu6_3_activation_height = crate::chain_index::ShieldedPool::Ironwood + .activation_upgrade() + .activation_height(&zebra_network); // Mark migration in progress (observability only; resumption uses the progress key). { diff --git a/packages/zaino-state/src/chain_index/source/validator_connector.rs b/packages/zaino-state/src/chain_index/source/validator_connector.rs index 74b192bbe..cf7908a94 100644 --- a/packages/zaino-state/src/chain_index/source/validator_connector.rs +++ b/packages/zaino-state/src/chain_index/source/validator_connector.rs @@ -434,7 +434,8 @@ impl BlockchainSource for ValidatorConnector { } }; - let sapling = match zebra_chain::parameters::NetworkUpgrade::Sapling + let sapling = match ShieldedPool::Sapling + .activation_upgrade() .activation_height(&state.network.to_zebra_network()) { Some(activation_height) if height >= activation_height => Some( @@ -464,7 +465,8 @@ impl BlockchainSource for ValidatorConnector { }) }); - let orchard = match zebra_chain::parameters::NetworkUpgrade::Nu5 + let orchard = match ShieldedPool::Orchard + .activation_upgrade() .activation_height(&state.network.to_zebra_network()) { Some(activation_height) if height >= activation_height => Some( @@ -494,7 +496,8 @@ impl BlockchainSource for ValidatorConnector { }) }); - let ironwood = match zebra_chain::parameters::NetworkUpgrade::Nu6_3 + let ironwood = match ShieldedPool::Ironwood + .activation_upgrade() .activation_height(&state.network.to_zebra_network()) { Some(activation_height) if height >= activation_height => Some( From c7b1a47026a6aac9c9ea9c9ef60771e5d2efa317 Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 16:50:57 -0700 Subject: [PATCH 069/134] fix(chain-index): drop needless borrows in the batch write loop The batch loop's block binding is already a reference; clippy --fix. Co-Authored-By: Claude Fable 5 --- .../finalised_state/finalised_source/v1/write_core.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/write_core.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/write_core.rs index 261b08e1f..212ac75d3 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/write_core.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/write_core.rs @@ -1233,7 +1233,7 @@ impl DbV1 { } // Build the per-transaction pool lists (with the duplicate-txid guard applied). - let pool_lists = extract_block_pool_lists(&block)?; + let pool_lists = extract_block_pool_lists(block)?; for (tx_index, tx) in block.transactions().iter().enumerate() { let tx_index = @@ -1261,10 +1261,10 @@ impl DbV1 { transactions.into_iter().unzip(); // Cheap in-memory correctness check: txids must reproduce the header merkle root. - verify_header_merkle_root(&txids, &block)?; + verify_header_merkle_root(&txids, block)?; let entries = build_block_row_entries( - &block, + block, &block_hash_bytes, &block_height_bytes, txids, From bf7a87622e920f3bb428519ed71517bbe0d38d81 Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 16:57:57 -0700 Subject: [PATCH 070/134] refactor(chain-index): pool table dispatch in the v1 shielded reads DbV1 pool reads are now indexed by ShieldedPool instead of naming tables and policies per call site: - pool_table(ShieldedPool) owns the pool -> LMDB table mapping; - MissingRow::for_pool owns the dense/sparse policy (sapling and orchard rows exist for every block; post-v1.3.0 pools are sparse); - pool_skip_entry owns the pool -> entry-skip mapping (orchard and ironwood share the Orchard compact layout); - get_pool_tx takes the pool; get_block_pool_tx_list and get_block_range_pool_tx_list unify the whole-row and range reads, selecting cursor-scan vs per-height resolution from the pool's missing-row policy. The nine per-pool trait methods are now one-expression delegations, and every pool decision is an exhaustive in-crate match: adding the next pool fails compilation at each site needing a choice. ShieldedPool derives Copy (fieldless enum, semver-additive). Commit 2 of the pool-indexed dispatch plan (#1367). Behaviour preserved; full zaino-state suite green (182 tests). Co-Authored-By: Claude Fable 5 --- packages/zaino-state/src/chain_index.rs | 2 +- .../finalised_source/v1/block_shielded.rs | 189 ++++++++++++------ 2 files changed, 127 insertions(+), 64 deletions(-) diff --git a/packages/zaino-state/src/chain_index.rs b/packages/zaino-state/src/chain_index.rs index 84d8829b3..4609c0011 100644 --- a/packages/zaino-state/src/chain_index.rs +++ b/packages/zaino-state/src/chain_index.rs @@ -2801,7 +2801,7 @@ impl ChainIndex for NodeBackedChainIndexSubscriber MissingRow { + match pool { + ShieldedPool::Sapling | ShieldedPool::Orchard => MissingRow::Error, + ShieldedPool::Ironwood => MissingRow::NoPoolData, + } + } +} + /// [`BlockShieldedExt`] capability implementation for [`DbV1`]. /// /// Provides access to Sapling / Orchard compact transaction data and per-block commitment tree @@ -109,13 +121,7 @@ impl DbV1 { &self, tx_location: TxLocation, ) -> Result, FinalisedStateError> { - self.get_pool_tx( - self.sapling, - "sapling", - MissingRow::Error, - tx_location, - Self::skip_opt_sapling_entry, - ) + self.get_pool_tx(ShieldedPool::Sapling, tx_location) } /// Fetch block sapling transaction data by height. @@ -123,11 +129,10 @@ impl DbV1 { &self, height: Height, ) -> Result { - self.read_row_at_height(self.sapling, "sapling", height) - .await? - .ok_or_else(|| { - FinalisedStateError::DataUnavailable("sapling data missing from db".into()) - }) + self.get_block_pool_tx_list(ShieldedPool::Sapling, height, || { + SaplingTxList::new(Vec::new()) + }) + .await } /// Fetches block sapling tx data for the given height range. @@ -142,7 +147,10 @@ impl DbV1 { start: Height, end: Height, ) -> Result, FinalisedStateError> { - self.scan_rows(self.sapling, "sapling", start, end).await + self.get_block_range_pool_tx_list(ShieldedPool::Sapling, start, end, || { + SaplingTxList::new(Vec::new()) + }) + .await } /// Fetch the serialized OrchardCompactTx for the given TxLocation, if present. @@ -152,13 +160,7 @@ impl DbV1 { &self, tx_location: TxLocation, ) -> Result, FinalisedStateError> { - self.get_pool_tx( - self.orchard, - "orchard", - MissingRow::Error, - tx_location, - Self::skip_opt_orchard_entry, - ) + self.get_pool_tx(ShieldedPool::Orchard, tx_location) } /// Fetch block orchard transaction data by height. @@ -166,11 +168,10 @@ impl DbV1 { &self, height: Height, ) -> Result { - self.read_row_at_height(self.orchard, "orchard", height) - .await? - .ok_or_else(|| { - FinalisedStateError::DataUnavailable("orchard data missing from db".into()) - }) + self.get_block_pool_tx_list(ShieldedPool::Orchard, height, || { + OrchardTxList::new(Vec::new()) + }) + .await } /// Fetches block orchard tx data for the given height range. @@ -185,7 +186,10 @@ impl DbV1 { start: Height, end: Height, ) -> Result, FinalisedStateError> { - self.scan_rows(self.orchard, "orchard", start, end).await + self.get_block_range_pool_tx_list(ShieldedPool::Orchard, start, end, || { + OrchardTxList::new(Vec::new()) + }) + .await } /// Fetch the serialized `OrchardCompactTx` for the given TxLocation from the ironwood table. @@ -197,13 +201,7 @@ impl DbV1 { &self, tx_location: TxLocation, ) -> Result, FinalisedStateError> { - self.get_pool_tx( - self.ironwood, - "ironwood", - MissingRow::NoPoolData, - tx_location, - Self::skip_opt_orchard_entry, - ) + self.get_pool_tx(ShieldedPool::Ironwood, tx_location) } /// Fetch block ironwood transaction data by height. @@ -213,10 +211,10 @@ impl DbV1 { &self, height: Height, ) -> Result { - Ok(self - .read_row_at_height(self.ironwood, "ironwood", height) - .await? - .unwrap_or_else(|| OrchardTxList::new(Vec::new()))) + self.get_block_pool_tx_list(ShieldedPool::Ironwood, height, || { + OrchardTxList::new(Vec::new()) + }) + .await } /// Fetches block ironwood tx data for the given (inclusive) height range. @@ -229,21 +227,10 @@ impl DbV1 { start: Height, end: Height, ) -> Result, FinalisedStateError> { - if end.0 < start.0 { - return Err(FinalisedStateError::Custom( - "invalid block range: end < start".to_string(), - )); - } - - self.validate_block_range(start, end).await?; - - let mut out = Vec::with_capacity((end.0 - start.0 + 1) as usize); - for height_int in start.0..=end.0 { - let height = Height::try_from(height_int) - .map_err(|e| FinalisedStateError::Custom(e.to_string()))?; - out.push(self.get_block_ironwood(height).await?); - } - Ok(out) + self.get_block_range_pool_tx_list(ShieldedPool::Ironwood, start, end, || { + OrchardTxList::new(Vec::new()) + }) + .await } /// Fetch block commitment tree data by height. @@ -276,22 +263,98 @@ impl DbV1 { // *** Internal DB methods *** - /// Point lookup for one transaction's compact data in a per-block pool table, + /// The LMDB table holding `pool`'s per-block compact transaction lists. + fn pool_table(&self, pool: ShieldedPool) -> lmdb::Database { + match pool { + ShieldedPool::Sapling => self.sapling, + ShieldedPool::Orchard => self.orchard, + ShieldedPool::Ironwood => self.ironwood, + } + } + + /// The entry-skip function for `pool`'s tx-list encoding (orchard and ironwood + /// share the Orchard compact layout). + fn pool_skip_entry(pool: ShieldedPool) -> fn(&mut std::io::Cursor<&[u8]>) -> io::Result<()> { + match pool { + ShieldedPool::Sapling => Self::skip_opt_sapling_entry, + ShieldedPool::Orchard | ShieldedPool::Ironwood => Self::skip_opt_orchard_entry, + } + } + + /// Fetch one pool's whole-block tx list by height. Dense pools error on a missing + /// row; sparse pools yield `empty()`. + async fn get_block_pool_tx_list( + &self, + pool: ShieldedPool, + height: Height, + empty: impl FnOnce() -> T, + ) -> Result { + let label = pool.pool_string(); + match self + .read_row_at_height(self.pool_table(pool), &label, height) + .await? + { + Some(list) => Ok(list), + None => match MissingRow::for_pool(pool) { + MissingRow::Error => Err(FinalisedStateError::DataUnavailable(format!( + "{label} data missing from db" + ))), + MissingRow::NoPoolData => Ok(empty()), + }, + } + } + + /// Fetch one pool's tx lists for the inclusive height range, one entry per height. + /// + /// Dense pools cursor-scan the range; sparse pools resolve each height individually + /// so absent rows yield `empty()` and the result stays aligned one-entry-per-height + /// with the requested range. + async fn get_block_range_pool_tx_list( + &self, + pool: ShieldedPool, + start: Height, + end: Height, + empty: impl Fn() -> T, + ) -> Result, FinalisedStateError> { + match MissingRow::for_pool(pool) { + MissingRow::Error => { + self.scan_rows(self.pool_table(pool), &pool.pool_string(), start, end) + .await + } + MissingRow::NoPoolData => { + if end.0 < start.0 { + return Err(FinalisedStateError::Custom( + "invalid block range: end < start".to_string(), + )); + } + self.validate_block_range(start, end).await?; + + let mut out = Vec::with_capacity((end.0 - start.0 + 1) as usize); + for height in Height::range_inclusive(start, end) { + out.push(self.get_block_pool_tx_list(pool, height, &empty).await?); + } + Ok(out) + } + } + } + + /// Point lookup for one transaction's compact data in `pool`'s per-block table, /// without decoding the whole block row. /// - /// Walks the `StoredEntryVar` tx-list bytes entry-by-entry with `skip_entry`, - /// then decodes only the requested entry. `pool` names the table in error - /// messages; `missing_row` selects how an absent row is treated. + /// Walks the `StoredEntryVar` tx-list bytes entry-by-entry with the pool's skip + /// function, then decodes only the requested entry. fn get_pool_tx( &self, - table: lmdb::Database, - pool: &str, - missing_row: MissingRow, + pool: ShieldedPool, tx_location: TxLocation, - skip_entry: fn(&mut std::io::Cursor<&[u8]>) -> io::Result<()>, ) -> Result, FinalisedStateError> { use std::io::{Cursor, Read}; + let table = self.pool_table(pool); + let label = pool.pool_string(); + let missing_row = MissingRow::for_pool(pool); + let skip_entry = Self::pool_skip_entry(pool); + tokio::task::block_in_place(|| { let txn = self.env.begin_ro_txn()?; @@ -305,7 +368,7 @@ impl DbV1 { return match missing_row { MissingRow::NoPoolData => Ok(None), MissingRow::Error => Err(FinalisedStateError::DataUnavailable(format!( - "{pool} data missing from db" + "{label} data missing from db" ))), }; } @@ -327,13 +390,13 @@ impl DbV1 { // Read CompactSize: number of entries let list_len = CompactSize::read(&mut cursor).map_err(|e| { - FinalisedStateError::Custom(format!("{pool} tx list len error: {e}")) + FinalisedStateError::Custom(format!("{label} tx list len error: {e}")) })?; let idx = tx_location.tx_index() as usize; if idx >= list_len as usize { return Err(FinalisedStateError::Custom(format!( - "tx_index out of range in {pool} tx list" + "tx_index out of range in {label} tx list" ))); } From 019ab23e5497bddb55ab274bdc7c53e1b7488549 Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 17:36:19 -0700 Subject: [PATCH 071/134] refactor(zaino-proto): one definition of the unfiltered pool set MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The codebase had two sources of truth for "the client did not filter": PoolTypeFilter::default() (which gained Ironwood correctly) and the wire-decode backfill in pool_types_from_vector(&[]) (which had gone stale at Sapling+Orchard — the bug fixed in b1783b9c). Tests exercised the filter default; production traffic decodes the wire vector; the two drifted. The empty-vector arm now delegates to PoolTypeFilter::default().to_pool_types_vector(), so the unfiltered set has exactly one definition and a fifth pool cannot re-open the same gap. Also corrects the Default impl's stale doc comment, which still described the pre-Ironwood set its body no longer matched. Co-Authored-By: Claude Fable 5 --- packages/zaino-proto/src/proto/utils.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/zaino-proto/src/proto/utils.rs b/packages/zaino-proto/src/proto/utils.rs index 2cb3493d2..72220ae45 100644 --- a/packages/zaino-proto/src/proto/utils.rs +++ b/packages/zaino-proto/src/proto/utils.rs @@ -26,9 +26,14 @@ pub enum PoolTypeError { /// here would serve blocks whose `chainMetadata.ironwoodCommitmentTreeSize` /// counts commitments from actions the block omits; a scanning wallet sees /// that as a tree-size discontinuity and treats it as a chain reorg. +/// +/// The unfiltered pool set has exactly one definition: +/// [`PoolTypeFilter::default`]. This wire-decode path delegates to it so the +/// two cannot drift again (they had: the filter default gained Ironwood while +/// this backfill still listed only Sapling and Orchard). pub fn pool_types_from_vector(pool_types: &[i32]) -> Result, PoolTypeError> { let pools = if pool_types.is_empty() { - vec![PoolType::Sapling, PoolType::Orchard, PoolType::Ironwood] + PoolTypeFilter::default().to_pool_types_vector() } else { let mut pools: Vec = vec![]; @@ -167,7 +172,7 @@ pub struct PoolTypeFilter { } impl std::default::Default for PoolTypeFilter { - /// By default PoolType includes `Sapling` and `Orchard` pools. + /// The unfiltered pool set: every shielded pool, transparent excluded. fn default() -> Self { PoolTypeFilter { include_transparent: false, From 438a755c3a4dacdf82bd17ddaa5415bac287523f Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 17:50:01 -0700 Subject: [PATCH 072/134] test(chain-index): in-crate compact-block metadata consistency walk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pin the invariant the phantom-reorg bug class violates: every served compact block's per-pool commitment-tree-size delta must equal the commitment count the block actually serves, and served counts must match the source chain. The walk runs for both the wire-decoded unfiltered request (PoolTypeFilter::new_from_slice(&[]) — the shape real, including pre-Ironwood, clients send; the path the b1783b9c bug lived on) and the explicit all-pools filter, over the non-finalised window with an absolute baseline computed from the source chain. Supporting changes, all in proptest_blockgen.rs: - the mockchain root fold gains the ironwood slot (previously hard None: with V6 blocks the mockchain served ironwoodCommitmentTreeSize 0 against nonzero actions — the exact inconsistency under test); - passthrough_test_on(network, source_delay, mutate_segment, test) generalizes the harness; passthrough_test keeps its behaviour; - zebra's stock Transaction strategy never generates V6 (its NU6.3/NU7 arm produces only v4/v5 — upstream gap), so the test injects zebra's own fake_v6_transaction with a two-action Ironwood bundle into every post-activation block. Safe under passthrough: the block hash covers only the header, and headers already carry arbitrary merkle roots. A guard fails the test loudly if the chain ends up with no ironwood commitments, so the walk can never go vacuously green. Full zaino-state suite green (183 tests). Co-Authored-By: Claude Fable 5 --- .../chain_index/tests/proptest_blockgen.rs | 332 +++++++++++++++--- 1 file changed, 290 insertions(+), 42 deletions(-) diff --git a/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs b/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs index c0cd46dbe..9d4528515 100644 --- a/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs +++ b/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs @@ -33,10 +33,12 @@ use crate::{ types::BestChainLocation, NonFinalizedSnapshot, OPERATIONAL_NFS_DEPTH, }, - BlockHash, BlockchainSource, ChainIndex, ChainIndexConfig, NodeBackedChainIndex, + BlockHash, BlockchainSource, ChainIndex, ChainIndexConfig, Height, NodeBackedChainIndex, NodeBackedChainIndexSubscriber, TransactionHash, }; +use zaino_proto::proto::utils::PoolTypeFilter; + /// Handle all the boilerplate for a passthrough fn passthrough_test( // The actual assertions. Takes as args: @@ -48,9 +50,36 @@ fn passthrough_test( // A snapshot, which will have only the genesis block &ChainIndexSnapshot, ), +) { + passthrough_test_on( + Network::Regtest(ActivationHeights::default()), + // Slow the source enough to hold the indexer in passthrough while the + // assertions run, without slowing passthrough more than necessary. + Some(Duration::from_millis(100)), + |_| {}, + test, + ) +} + +/// [`passthrough_test`] on an explicit network, with a per-segment chain mutator. +/// +/// The mutator exists because zebra's stock `Transaction` strategy never generates V6 +/// transactions (its NU6.3/NU7 arm produces only v4/v5), so ironwood-era content must +/// be injected after generation. Mutating a block's transactions is safe here: the +/// block hash covers only the header, so parent-hash continuity is untouched, and the +/// header's merkle root is already arbitrary — the passthrough path tolerates that by +/// construction. +fn passthrough_test_on( + network: Network, + source_delay: Option, + mutate_segment: impl Fn(&mut Vec>), + test: impl AsyncFn( + &ProptestMockchain, + NodeBackedChainIndexSubscriber, + &ChainIndexSnapshot, + ), ) { init_tracing(); - let network = Network::Regtest(ActivationHeights::default()); // Long enough to have some finalized blocks to play with let segment_length = OPERATIONAL_NFS_DEPTH as usize + 20; // No need to worry about non-best chains for this test @@ -61,14 +90,15 @@ fn passthrough_test( proptest::proptest!(proptest::test_runner::Config::with_cases(1), |(segments in make_branching_chain(branch_count, segment_length, network))| { let runtime = tokio::runtime::Builder::new_multi_thread().worker_threads(2).enable_time().build().unwrap(); runtime.block_on(async { - let (genesis_segment, branching_segments) = segments; + let (mut genesis_segment, mut branching_segments) = segments; + mutate_segment(&mut genesis_segment.0); + for segment in &mut branching_segments { + mutate_segment(&mut segment.0); + } let mockchain = ProptestMockchain { genesis_segment, branching_segments, - // This number can be played with. We want to slow down - // sync enough to trigger passthrough without - // slowing down passthrough more than we need to - delay: Some(Duration::from_millis(100)), + delay: source_delay, best_branch_cache: Arc::new(std::sync::OnceLock::new()), tx_index: Arc::new(std::sync::OnceLock::new()), }; @@ -364,6 +394,206 @@ fn passthrough_get_block_range() { }) } +/// NU6.3 active from height 2, so post-activation generated blocks carry V6 +/// transactions whose shielded data lands in the Ironwood pool. +const NU6_3_ACTIVE_HEIGHTS: ActivationHeights = ActivationHeights { + before_overwinter: Some(1), + overwinter: Some(1), + sapling: Some(1), + blossom: Some(1), + heartwood: Some(1), + canopy: Some(1), + nu5: Some(2), + nu6: Some(2), + nu6_1: Some(2), + nu6_2: Some(2), + nu6_3: Some(2), + nu7: None, +}; + +/// Per-block consistency between served compact-block content and its chain metadata. +/// +/// A compact block's `chainMetadata` tree sizes are cumulative note-commitment counts; +/// a scanning wallet advances its trees by the actions/outputs each served block +/// carries, so a served block whose tree-size delta disagrees with its served +/// commitment count reads as a tree-size discontinuity — a phantom chain reorg. The +/// walk checks every serviceable height, for both the wire-decoded unfiltered request +/// (empty `poolTypes`, what real — including pre-Ironwood — clients send) and the +/// explicit all-pools filter, and cross-checks served counts against the mockchain +/// source of truth. +#[test] +fn passthrough_compact_block_metadata_consistency() { + /// Appends one structurally-valid (cryptographically fake) V6 transaction carrying + /// a two-action Ironwood bundle to every post-activation block, since zebra's + /// stock strategy never generates V6. Heights match [`NU6_3_ACTIVE_HEIGHTS`]. + fn inject_ironwood_transactions(blocks: &mut Vec>) { + use zebra_chain::amount::Amount; + use zebra_chain::orchard::{Flags, ShieldedDataV6}; + use zebra_chain::parameters::NetworkUpgrade; + use zebra_chain::transaction::arbitrary::{ + fake_v6_orchard_shielded_data, fake_v6_transaction, + }; + + for block in blocks.iter_mut() { + let height = block + .coinbase_height() + .expect("generated blocks always have a coinbase height"); + if height < zebra_chain::block::Height(2) { + continue; + } + let ironwood = zebra_chain::ironwood::ShieldedData::new(ShieldedDataV6::new( + fake_v6_orchard_shielded_data( + Flags::ENABLE_SPENDS, + Amount::try_from(0).expect("zero is a valid amount"), + 2, + ), + )); + let fake_tx = fake_v6_transaction(NetworkUpgrade::Nu6_3, None, Some(ironwood)); + + let mut new_block = (**block).clone(); + new_block.transactions.push(Arc::new(fake_tx)); + *block = Arc::new(new_block); + } + } + + passthrough_test_on( + Network::Regtest(NU6_3_ACTIVE_HEIGHTS), + // No artificial source delay: this test waits for the indexer to finish + // syncing, because compact blocks are not served while the finalised state + // is still syncing (get_compact_block's StillSyncingFinalizedState arm). + None, + inject_ironwood_transactions, + async |mockchain, index_reader, _snapshot| { + // Source of truth: per-height shielded commitment counts from the mockchain + // blocks themselves (single branch, so arb branch order is chain order). + let source_counts: Vec<(u32, u32, u32)> = mockchain + .all_blocks_arb_branch_order() + .map(|block| { + let sapling = block + .transactions + .iter() + .map(|tx| tx.sapling_note_commitments().count() as u32) + .sum(); + let orchard = block + .transactions + .iter() + .map(|tx| tx.orchard_note_commitments().count() as u32) + .sum(); + let ironwood = block + .transactions + .iter() + .map(|tx| tx.ironwood_note_commitments().count() as u32) + .sum(); + (sapling, orchard, ironwood) + }) + .collect(); + + let total_source_ironwood: u32 = source_counts.iter().map(|(_, _, i)| i).sum(); + assert!( + total_source_ironwood > 0, + "the generated chain carries no ironwood commitments; every ironwood \ + assertion below would be vacuous. If this recurs, force V6 generation \ + via LedgerStateOverride::transaction_version_override." + ); + + // Compact blocks are only served once the finalised state has caught up. + let snapshot = poll_until( + "indexer to finish syncing so compact blocks are served", + Duration::from_secs(60), + Duration::from_millis(50), + || async { + let snapshot = index_reader.snapshot_nonfinalized_state().await.ok()?; + matches!(snapshot, ChainIndexSnapshot::NonFinalizedStateExists { .. }) + .then_some(snapshot) + }, + ) + .await; + let snapshot = &snapshot; + + let tip = snapshot + .get_nfs_snapshot() + .expect("fully synced snapshot has a non-finalised state") + .best_tip + .height; + // The walk covers the non-finalised window; its absolute baseline is the + // cumulative source count below the window. + let first_walked = finalized_height_floor(tip.0).0 + 1; + let baseline = source_counts[..first_walked as usize].iter().fold( + (0u32, 0u32, 0u32), + |(sapling, orchard, ironwood), (s, o, i)| (sapling + s, orchard + o, ironwood + i), + ); + + for unfiltered_wire_request in [true, false] { + let (mut prev_sapling, mut prev_orchard, mut prev_ironwood) = baseline; + + for height_int in first_walked..=tip.0 { + // The empty slice is the wire shape unfiltered clients send; both + // filters include every shielded pool, which the delta assertions + // below rely on. + let filter = if unfiltered_wire_request { + PoolTypeFilter::new_from_slice(&[]).unwrap() + } else { + PoolTypeFilter::includes_all() + }; + let block = index_reader + .get_compact_block(snapshot, Height(height_int), filter) + .await + .unwrap() + .expect("serviceable heights must serve a compact block"); + let metadata = block + .chain_metadata + .as_ref() + .expect("served compact blocks carry chain metadata"); + + let served_sapling: u32 = + block.vtx.iter().map(|tx| tx.outputs.len() as u32).sum(); + let served_orchard: u32 = + block.vtx.iter().map(|tx| tx.actions.len() as u32).sum(); + let served_ironwood: u32 = block + .vtx + .iter() + .map(|tx| tx.ironwood_actions.len() as u32) + .sum(); + + // Serving completeness: everything the source block carries is served. + let (source_sapling, source_orchard, source_ironwood) = + source_counts[height_int as usize]; + assert_eq!( + (served_sapling, served_orchard, served_ironwood), + (source_sapling, source_orchard, source_ironwood), + "served shielded counts must match the source block at height \ + {height_int} (unfiltered_wire_request: {unfiltered_wire_request})" + ); + + // Metadata consistency: tree-size deltas equal served counts. + assert_eq!( + metadata.sapling_commitment_tree_size, + prev_sapling + served_sapling, + "sapling tree-size delta must equal the served output count at \ + height {height_int}" + ); + assert_eq!( + metadata.orchard_commitment_tree_size, + prev_orchard + served_orchard, + "orchard tree-size delta must equal the served action count at \ + height {height_int}" + ); + assert_eq!( + metadata.ironwood_commitment_tree_size, + prev_ironwood + served_ironwood, + "ironwood tree-size delta must equal the served action count at \ + height {height_int}" + ); + + prev_sapling = metadata.sapling_commitment_tree_size; + prev_orchard = metadata.orchard_commitment_tree_size; + prev_ironwood = metadata.ironwood_commitment_tree_size; + } + } + }, + ) +} + // Ignored: this drives the full indexer over `partial_chain_strategy` blocks, whose headers carry // arbitrary (invalid) merkle roots. The finalised state now validates blocks on the write path // (cheap merkle + parent-continuity checks), so it correctly rejects these blocks once the indexer's @@ -634,41 +864,54 @@ impl BlockchainSource for ProptestMockchain { return Ok((None, None, None)); }; - let (sapling, orchard) = - chain_up_to_block - .iter() - .fold((None, None), |(mut sapling, mut orchard), block| { - for transaction in &block.transactions { - for sap_commitment in transaction.sapling_note_commitments() { - let sap_commitment = - sapling_crypto::Node::from_bytes(sap_commitment.to_bytes()) - .unwrap(); - - sapling = Some(sapling.unwrap_or_else(|| { - incrementalmerkletree::frontier::Frontier::<_, 32>::empty() - })); - - sapling = sapling.map(|mut tree| { - tree.append(sap_commitment); - tree - }); - } - for orc_commitment in transaction.orchard_note_commitments() { - let orc_commitment = - zebra_chain::orchard::tree::Node::from(*orc_commitment); - - orchard = Some(orchard.unwrap_or_else(|| { - incrementalmerkletree::frontier::Frontier::<_, 32>::empty() - })); - - orchard = orchard.map(|mut tree| { - tree.append(orc_commitment); - tree - }); - } + let (sapling, orchard, ironwood) = chain_up_to_block.iter().fold( + (None, None, None), + |(mut sapling, mut orchard, mut ironwood), block| { + for transaction in &block.transactions { + for sap_commitment in transaction.sapling_note_commitments() { + let sap_commitment = + sapling_crypto::Node::from_bytes(sap_commitment.to_bytes()).unwrap(); + + sapling = Some(sapling.unwrap_or_else(|| { + incrementalmerkletree::frontier::Frontier::<_, 32>::empty() + })); + + sapling = sapling.map(|mut tree| { + tree.append(sap_commitment); + tree + }); } - (sapling, orchard) - }); + for orc_commitment in transaction.orchard_note_commitments() { + let orc_commitment = + zebra_chain::orchard::tree::Node::from(*orc_commitment); + + orchard = Some(orchard.unwrap_or_else(|| { + incrementalmerkletree::frontier::Frontier::<_, 32>::empty() + })); + + orchard = orchard.map(|mut tree| { + tree.append(orc_commitment); + tree + }); + } + // Ironwood reuses the Orchard tree/node types. + for irw_commitment in transaction.ironwood_note_commitments() { + let irw_commitment = + zebra_chain::orchard::tree::Node::from(*irw_commitment); + + ironwood = Some(ironwood.unwrap_or_else(|| { + incrementalmerkletree::frontier::Frontier::<_, 32>::empty() + })); + + ironwood = ironwood.map(|mut tree| { + tree.append(irw_commitment); + tree + }); + } + } + (sapling, orchard, ironwood) + }, + ); Ok(( sapling.map(|sap_front| { ( @@ -682,7 +925,12 @@ impl BlockchainSource for ProptestMockchain { orc_front.tree_size(), ) }), - None, + ironwood.map(|irw_front| { + ( + zebra_chain::orchard::tree::Root::from_bytes(irw_front.root().as_bytes()), + irw_front.tree_size(), + ) + }), )) } From f5a4323f573e1ef41c7615a8e440f866ced6a9cb Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 18:02:37 -0700 Subject: [PATCH 073/134] test(chain-index): demonstrate zebra's missing V6 Arbitrary generation (known red) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pin the upstream gap that forced the fake-transaction injection in the metadata-consistency walk: zebra-chain's stock Transaction strategy never generates V6 — its NU6.3/NU7 arm is prop_oneof![v4, v5], so V6 is structurally impossible from generation, not merely rare. The test pins an NU6.3 ledger state (and asserts the pin took), samples 64 transactions, and requires a V6 among them. #[should_panic] tracks the upstream gap: when a zebra upgrade starts generating V6 this flips, signalling that the #[should_panic] and the inject_ironwood_transactions workaround can be retired together. Worth flagging upstream to zebra, and to valargroup whose NU7 branch inherits the same gap. Co-Authored-By: Claude Fable 5 --- .../chain_index/tests/proptest_blockgen.rs | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs b/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs index 9d4528515..32557da2c 100644 --- a/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs +++ b/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs @@ -394,6 +394,56 @@ fn passthrough_get_block_range() { }) } +/// Upstream gap demonstration: zebra-chain's stock [`Transaction`] strategy never +/// generates V6 transactions, even for an NU6.3 ledger state — its NU6.3/NU7 arm is +/// `prop_oneof![v4_strategy, v5_strategy]` (zebra-chain `transaction/arbitrary.rs`). +/// V6 is therefore structurally impossible from the stock strategy, not merely rare, +/// which is why [`passthrough_compact_block_metadata_consistency`] must inject +/// `fake_v6_transaction` ironwood content instead of relying on generation. +/// +/// `should_panic` tracks the upstream gap: when a zebra upgrade starts generating V6, +/// this test flips, and the `#[should_panic]` should be removed together with the +/// fake-transaction injection in `inject_ironwood_transactions` (generation then covers +/// it natively). +/// +/// [`Transaction`]: zebra_chain::transaction::Transaction +#[test] +#[should_panic(expected = "zebra's stock Transaction strategy generated no V6")] +fn zebra_arbitrary_generates_v6_transactions_for_nu6_3() { + use proptest::strategy::ValueTree as _; + use proptest::test_runner::TestRunner; + use zebra_chain::parameters::NetworkUpgrade; + + let mut runner = TestRunner::default(); + + let ledger = LedgerState::arbitrary_with(LedgerStateOverride { + network_upgrade_override: Some(NetworkUpgrade::Nu6_3), + ..LedgerStateOverride::default() + }) + .new_tree(&mut runner) + .expect("ledger strategy yields a value") + .current(); + assert_eq!(ledger.network_upgrade(), NetworkUpgrade::Nu6_3); + + let transaction_strategy = + zebra_chain::transaction::Transaction::arbitrary_with(ledger.clone()); + + let mut generated_versions = std::collections::BTreeSet::new(); + for _ in 0..64 { + let transaction = transaction_strategy + .new_tree(&mut runner) + .expect("transaction strategy yields a value") + .current(); + generated_versions.insert(transaction.version()); + } + + assert!( + generated_versions.contains(&6), + "zebra's stock Transaction strategy generated no V6 transaction for an NU6.3 \ + ledger state in 64 samples (saw versions {generated_versions:?})" + ); +} + /// NU6.3 active from height 2, so post-activation generated blocks carry V6 /// transactions whose shielded data lands in the Ironwood pool. const NU6_3_ACTIVE_HEIGHTS: ActivationHeights = ActivationHeights { From 36e017f87e88198edeecce52f4e7d398059bd814 Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 18:37:43 -0700 Subject: [PATCH 074/134] test(chain-index): package-tier era triple for compact-block consistency The metadata-consistency walk becomes metadata_consistency_for_era and runs three era layouts: - orchard-only (the current zebrad default heights, NU6.3 never activates): fake Orchard content from height 2; ironwood provably never appears, since zebra's stock strategy cannot generate V6; - ironwood-only (NU6.3 from height 2): the previous walk, renamed; - orchard -> ironwood transition: the boundary is placed inside the walked non-finalised window so the walk observes both eras and the flip itself. Era-composition guards on the source chain replace the single ironwood guard, so no era's assertions can go vacuously green and no era leaks content into the other: below-boundary ironwood must be exactly zero, above-boundary ironwood and injected orchard must be nonzero. fake_orchard_transaction (V5 with a two-action Orchard bundle) joins fake_ironwood_transaction for deterministic era content; the harness segment length becomes a named constant the transition boundary is computed from. Package tier of the era triple (clientless and e2e tiers ride the gated live-test files). Full zaino-state suite green (186 tests). Co-Authored-By: Claude Fable 5 --- .../chain_index/tests/proptest_blockgen.rs | 194 ++++++++++++++---- 1 file changed, 158 insertions(+), 36 deletions(-) diff --git a/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs b/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs index 32557da2c..a56d1774a 100644 --- a/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs +++ b/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs @@ -7,7 +7,10 @@ use proptest::{ }; use rand::seq::IndexedRandom; use tokio_stream::StreamExt as _; -use zaino_common::{network::ActivationHeights, DatabaseConfig, Network, StorageConfig}; +use zaino_common::{ + network::{ActivationHeights, ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS}, + DatabaseConfig, Network, StorageConfig, +}; use zaino_fetch::jsonrpsee::response::address_deltas::{ GetAddressDeltasParams, GetAddressDeltasResponse, }; @@ -39,6 +42,12 @@ use crate::{ use zaino_proto::proto::utils::PoolTypeFilter; +/// Chain length per generated segment in the passthrough harness — long enough to +/// have some finalised blocks to play with. The best chain is twice this (genesis +/// segment plus one branch), so its expected tip height is +/// `2 * PASSTHROUGH_SEGMENT_LENGTH - 1`. +const PASSTHROUGH_SEGMENT_LENGTH: usize = OPERATIONAL_NFS_DEPTH as usize + 20; + /// Handle all the boilerplate for a passthrough fn passthrough_test( // The actual assertions. Takes as args: @@ -80,8 +89,7 @@ fn passthrough_test_on( ), ) { init_tracing(); - // Long enough to have some finalized blocks to play with - let segment_length = OPERATIONAL_NFS_DEPTH as usize + 20; + let segment_length = PASSTHROUGH_SEGMENT_LENGTH; // No need to worry about non-best chains for this test let branch_count = 1; @@ -398,7 +406,7 @@ fn passthrough_get_block_range() { /// generates V6 transactions, even for an NU6.3 ledger state — its NU6.3/NU7 arm is /// `prop_oneof![v4_strategy, v5_strategy]` (zebra-chain `transaction/arbitrary.rs`). /// V6 is therefore structurally impossible from the stock strategy, not merely rare, -/// which is why [`passthrough_compact_block_metadata_consistency`] must inject +/// which is why the `passthrough_metadata_consistency_*` walks must inject /// `fake_v6_transaction` ironwood content instead of relying on generation. /// /// `should_panic` tracks the upstream gap: when a zebra upgrade starts generating V6, @@ -472,47 +480,121 @@ const NU6_3_ACTIVE_HEIGHTS: ActivationHeights = ActivationHeights { /// explicit all-pools filter, and cross-checks served counts against the mockchain /// source of truth. #[test] -fn passthrough_compact_block_metadata_consistency() { - /// Appends one structurally-valid (cryptographically fake) V6 transaction carrying - /// a two-action Ironwood bundle to every post-activation block, since zebra's - /// stock strategy never generates V6. Heights match [`NU6_3_ACTIVE_HEIGHTS`]. - fn inject_ironwood_transactions(blocks: &mut Vec>) { - use zebra_chain::amount::Amount; - use zebra_chain::orchard::{Flags, ShieldedDataV6}; - use zebra_chain::parameters::NetworkUpgrade; - use zebra_chain::transaction::arbitrary::{ - fake_v6_orchard_shielded_data, fake_v6_transaction, - }; +fn passthrough_metadata_consistency_ironwood_only() { + metadata_consistency_for_era(NU6_3_ACTIVE_HEIGHTS, Some(2), false) +} + +/// Orchard-only era on the current zebrad default heights (NU6.3 never activates): +/// fake Orchard content from height 2, and — since zebra's stock strategy cannot +/// generate V6 — ironwood provably never appears anywhere in the chain or the served +/// form. +#[test] +fn passthrough_metadata_consistency_orchard_only() { + metadata_consistency_for_era(ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS, None, false) +} + +/// The transition: fake Orchard content below the NU6.3 boundary, fake Ironwood +/// content from it. The boundary is placed inside the walked non-finalised window so +/// both eras are actually observed by the walk. +#[test] +fn passthrough_metadata_consistency_orchard_to_ironwood_transition() { + let expected_tip = (2 * PASSTHROUGH_SEGMENT_LENGTH - 1) as u32; + let boundary = expected_tip - (OPERATIONAL_NFS_DEPTH / 2); + metadata_consistency_for_era( + ActivationHeights { + nu6_3: Some(boundary), + ..NU6_3_ACTIVE_HEIGHTS + }, + Some(boundary), + true, + ) +} +/// A structurally-valid (cryptographically fake) V6 transaction carrying a two-action +/// Ironwood bundle. Injected because zebra's stock strategy never generates V6 +/// (demonstrated by [`zebra_arbitrary_generates_v6_transactions_for_nu6_3`]). +fn fake_ironwood_transaction() -> zebra_chain::transaction::Transaction { + use zebra_chain::amount::Amount; + use zebra_chain::orchard::{Flags, ShieldedDataV6}; + use zebra_chain::parameters::NetworkUpgrade; + use zebra_chain::transaction::arbitrary::{fake_v6_orchard_shielded_data, fake_v6_transaction}; + + let ironwood = zebra_chain::ironwood::ShieldedData::new(ShieldedDataV6::new( + fake_v6_orchard_shielded_data( + Flags::ENABLE_SPENDS, + Amount::try_from(0).expect("zero is a valid amount"), + 2, + ), + )); + fake_v6_transaction(NetworkUpgrade::Nu6_3, None, Some(ironwood)) +} + +/// A structurally-valid (cryptographically fake) V5 transaction carrying a two-action +/// Orchard bundle, for deterministic orchard-era content (the stock strategy's orchard +/// data is probabilistic). +fn fake_orchard_transaction() -> zebra_chain::transaction::Transaction { + use zebra_chain::amount::Amount; + use zebra_chain::orchard::Flags; + use zebra_chain::parameters::NetworkUpgrade; + use zebra_chain::transaction::arbitrary::fake_v6_orchard_shielded_data; + use zebra_chain::transaction::{LockTime, Transaction}; + + Transaction::V5 { + network_upgrade: NetworkUpgrade::Nu5, + lock_time: LockTime::unlocked(), + expiry_height: zebra_chain::block::Height(0), + inputs: Vec::new(), + outputs: Vec::new(), + sapling_shielded_data: None, + orchard_shielded_data: Some(fake_v6_orchard_shielded_data( + Flags::ENABLE_SPENDS, + Amount::try_from(0).expect("zero is a valid amount"), + 2, + )), + } +} + +/// Runs the metadata-consistency walk on a chain whose injected shielded content +/// follows the era layout: +/// +/// - `ironwood_boundary: None` — orchard era only: fake Orchard content from height 2, +/// and ironwood must never appear anywhere; +/// - `ironwood_boundary: Some(b)` — fake Ironwood content from height `b`; when +/// `orchard_below_boundary` is set, fake Orchard content fills heights 2..b (the +/// transition layout), otherwise heights below `b` carry only generated content. +fn metadata_consistency_for_era( + heights: ActivationHeights, + ironwood_boundary: Option, + orchard_below_boundary: bool, +) { + let inject = move |blocks: &mut Vec>| { for block in blocks.iter_mut() { let height = block .coinbase_height() - .expect("generated blocks always have a coinbase height"); - if height < zebra_chain::block::Height(2) { + .expect("generated blocks always have a coinbase height") + .0; + if height < 2 { continue; } - let ironwood = zebra_chain::ironwood::ShieldedData::new(ShieldedDataV6::new( - fake_v6_orchard_shielded_data( - Flags::ENABLE_SPENDS, - Amount::try_from(0).expect("zero is a valid amount"), - 2, - ), - )); - let fake_tx = fake_v6_transaction(NetworkUpgrade::Nu6_3, None, Some(ironwood)); - + let fake_tx = match ironwood_boundary { + None => fake_orchard_transaction(), + Some(boundary) if height >= boundary => fake_ironwood_transaction(), + Some(_) if orchard_below_boundary => fake_orchard_transaction(), + Some(_) => continue, + }; let mut new_block = (**block).clone(); new_block.transactions.push(Arc::new(fake_tx)); *block = Arc::new(new_block); } - } + }; passthrough_test_on( - Network::Regtest(NU6_3_ACTIVE_HEIGHTS), + Network::Regtest(heights), // No artificial source delay: this test waits for the indexer to finish // syncing, because compact blocks are not served while the finalised state // is still syncing (get_compact_block's StillSyncingFinalizedState arm). None, - inject_ironwood_transactions, + inject, async |mockchain, index_reader, _snapshot| { // Source of truth: per-height shielded commitment counts from the mockchain // blocks themselves (single branch, so arb branch order is chain order). @@ -538,13 +620,53 @@ fn passthrough_compact_block_metadata_consistency() { }) .collect(); - let total_source_ironwood: u32 = source_counts.iter().map(|(_, _, i)| i).sum(); - assert!( - total_source_ironwood > 0, - "the generated chain carries no ironwood commitments; every ironwood \ - assertion below would be vacuous. If this recurs, force V6 generation \ - via LedgerStateOverride::transaction_version_override." - ); + // Era-composition guards on the source chain, so no assertion below can go + // vacuously green (and no era leaks content into the other). + match ironwood_boundary { + None => { + let ironwood_total: u32 = source_counts.iter().map(|(_, _, i)| i).sum(); + assert_eq!( + ironwood_total, 0, + "orchard-only era must carry no ironwood commitments" + ); + let orchard_total: u32 = source_counts.iter().map(|(_, o, _)| o).sum(); + assert!( + orchard_total > 0, + "orchard-only era carries no orchard commitments; the orchard \ + assertions below would be vacuous" + ); + } + Some(boundary) => { + let below: u32 = source_counts[..boundary as usize] + .iter() + .map(|(_, _, i)| i) + .sum(); + assert_eq!( + below, 0, + "no ironwood commitments may exist below the activation boundary" + ); + let above: u32 = source_counts[boundary as usize..] + .iter() + .map(|(_, _, i)| i) + .sum(); + assert!( + above > 0, + "no ironwood commitments above the boundary; the ironwood \ + assertions below would be vacuous" + ); + if orchard_below_boundary { + let orchard_below: u32 = source_counts[2..boundary as usize] + .iter() + .map(|(_, o, _)| o) + .sum(); + assert!( + orchard_below > 0, + "transition layout carries no orchard commitments below the \ + boundary; the orchard-era half would be vacuous" + ); + } + } + } // Compact blocks are only served once the finalised state has caught up. let snapshot = poll_until( From ee9ed33c22e94f610a3876e0831a2cff62b7f581 Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 18:40:30 -0700 Subject: [PATCH 075/134] test(live): clientless and e2e era triples with tier-exclusive predicates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Complete the three-tier era matrix (package tier landed in 36e017f8): each tier runs orchard-only, ironwood-only, and the orchard->ironwood transition, on shared fixtures from zaino-testutils (NU6_3_ACTIVE_ACTIVATION_HEIGHTS, ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS; the orchard-only fixtures reuse ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS, so those tests double as documentation of today's default world). Clientless (compact_block_consistency.rs): - is_valid_orchard_coinbase / is_valid_ironwood_coinbase predicates over raw validator blocks — pure consensus observation, zaino uninvolved — walked by assert_coinbase_routing across the three era layouts, with describe_coinbase naming the violated clause; - the metadata walk gains the tier-exclusive ORACLE-PARITY predicate: every served chainMetadata tree size must equal zebrad's own answer from verbose getblock trees. Package tests cannot express this — their source of truth is the object being served, so the comparison is circular; only a live tier has an independent implementation as oracle; - the unfiltered walk keeps its two composition guards: nonzero ironwood (anti-vacuity: the first run verifies the ironwood-coinbase fixture premise) and zero orchard (from NU6.3 an orchard-receiver coinbase must be routed to Ironwood), which together distinguish pool-swap from pool-drop failures. E2e (compact_block_wire.rs): - the tier-exclusive WIRE-FIDELITY predicate: compact blocks received by a real tonic client from a running zainod must equal, block for block, the in-process subscriber's blocks for the same unfiltered request — the protobuf encode/decode and gRPC-server layer that clientless tests can never observe; - era composition asserted on the wire-received stream across the same three layouts. Both tiers exercise the empty-poolTypes request shape real (including pre-Ironwood) clients send. Compile-verified; runtime verification is the container flow, where the anti-vacuity guards make the first run double as the fixture-premise check. Co-Authored-By: Claude Fable 5 --- .../tests/compact_block_consistency.rs | 339 ++++++++++++++++++ live-tests/e2e/tests/compact_block_wire.rs | 202 +++++++++++ live-tests/zaino-testutils/src/lib.rs | 38 ++ 3 files changed, 579 insertions(+) create mode 100644 live-tests/clientless/tests/compact_block_consistency.rs create mode 100644 live-tests/e2e/tests/compact_block_wire.rs diff --git a/live-tests/clientless/tests/compact_block_consistency.rs b/live-tests/clientless/tests/compact_block_consistency.rs new file mode 100644 index 000000000..7f2274f8b --- /dev/null +++ b/live-tests/clientless/tests/compact_block_consistency.rs @@ -0,0 +1,339 @@ +//! Per-block consistency between served compact-block content and its chain metadata. +//! +//! A compact block's `chainMetadata` commitment-tree sizes are cumulative counts of the +//! note commitments the chain has produced. A scanning wallet advances its trees by the +//! actions/outputs each served block carries, so whenever a served block's tree-size +//! delta disagrees with its served commitment count the wallet observes a tree-size +//! discontinuity and treats it as a chain reorg. This walk pins that invariant per +//! block, per pool, for the request shape real (including pre-Ironwood) light clients +//! send: an empty `poolTypes` filter. + +use zaino_common::network::ActivationHeights; +use zaino_fetch::jsonrpsee::response::GetBlockResponse; +#[allow(deprecated)] +use zaino_state::FetchService; +use zaino_state::ZcashIndexer as _; +use zaino_testutils::{ + MinerPool, TestManager, ValidatorKind, NU6_3_ACTIVE_ACTIVATION_HEIGHTS, + NU6_3_TRANSITION_BOUNDARY, ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS, +}; +use zcash_local_net::validator::zebrad::Zebrad; +use zebra_chain::serialization::ZcashDeserialize as _; + +/// multi_thread required: the test manager spawns the validator and indexer services. +#[allow(deprecated)] +#[tokio::test(flavor = "multi_thread")] +async fn unfiltered_compact_blocks_match_chain_metadata_zebrad() { + let mut test_manager = TestManager::::launch_mining_to( + // Shielded mining: from NU6.3 an orchard-receiver coinbase is built as Ironwood + // actions (the coinbase's Orchard component must be empty from NU6.3), so every + // generated block carries ironwood data for the walk to check. A transparent + // miner would leave the ironwood assertions vacuous. + MinerPool::Orchard, + &ValidatorKind::Zebrad, + None, + Some(NU6_3_ACTIVE_ACTIVATION_HEIGHTS), + None, + false, + false, + false, + ) + .await + .unwrap(); + let subscriber = test_manager.subscriber().clone(); + + test_manager + .generate_blocks_and_wait_for_tip(8, &subscriber) + .await; + let tip = u64::from(subscriber.chain_height().await.unwrap().0); + + // The empty pool filter is what unfiltered (pre-Ironwood) clients send; the served + // stream must include every shielded pool's actions. + let blocks = zaino_testutils::collect_block_range(&subscriber, 0, tip, vec![]).await; + assert!(!blocks.is_empty(), "no compact blocks served"); + + let connector = test_manager.full_node_jsonrpc_connector().await; + + let (mut prev_sapling, mut prev_orchard, mut prev_ironwood) = (0u32, 0u32, 0u32); + let mut total_orchard_actions = 0usize; + let mut total_ironwood_actions = 0usize; + for (index, block) in blocks.iter().enumerate() { + assert_eq!( + block.height, index as u64, + "served blocks must be contiguous from genesis for the walk's running totals" + ); + let metadata = block + .chain_metadata + .as_ref() + .expect("every served compact block carries chain metadata"); + + let sapling_outputs: u32 = block.vtx.iter().map(|tx| tx.outputs.len() as u32).sum(); + let orchard_actions: u32 = block.vtx.iter().map(|tx| tx.actions.len() as u32).sum(); + let ironwood_actions: u32 = block + .vtx + .iter() + .map(|tx| tx.ironwood_actions.len() as u32) + .sum(); + total_orchard_actions += orchard_actions as usize; + total_ironwood_actions += ironwood_actions as usize; + + assert_eq!( + metadata.sapling_commitment_tree_size, + prev_sapling + sapling_outputs, + "sapling tree-size delta must equal the served output count at height {}", + block.height + ); + assert_eq!( + metadata.orchard_commitment_tree_size, + prev_orchard + orchard_actions, + "orchard tree-size delta must equal the served action count at height {}", + block.height + ); + // The regression this walk exists for: a served block whose metadata counts + // commitments from actions the block omits (e.g. ironwood stripped from an + // unfiltered request) reads to a scanning wallet as a phantom chain reorg. + assert_eq!( + metadata.ironwood_commitment_tree_size, + prev_ironwood + ironwood_actions, + "ironwood tree-size delta must equal the served action count at height {}", + block.height + ); + + // Clientless-exclusive predicate — oracle parity: zebrad's verbose getblock + // reports the validator's own per-block tree sizes, an independent + // implementation's answer to compare zaino's served metadata against. Package + // tests cannot express this: their "source of truth" is the object being + // served, so any such comparison is circular. + let oracle_trees = match connector + .get_block(block.height.to_string(), Some(2)) + .await + .expect("validator serves verbose blocks") + { + GetBlockResponse::Object(block_object) => block_object.trees, + other => panic!("verbosity-2 getblock must return a block object, got {other:?}"), + }; + assert_eq!( + u64::from(metadata.sapling_commitment_tree_size), + oracle_trees.sapling(), + "served sapling tree size must match the validator's own at height {}", + block.height + ); + assert_eq!( + u64::from(metadata.orchard_commitment_tree_size), + oracle_trees.orchard(), + "served orchard tree size must match the validator's own at height {}", + block.height + ); + assert_eq!( + u64::from(metadata.ironwood_commitment_tree_size), + oracle_trees.ironwood(), + "served ironwood tree size must match the validator's own at height {}", + block.height + ); + + prev_sapling = metadata.sapling_commitment_tree_size; + prev_orchard = metadata.orchard_commitment_tree_size; + prev_ironwood = metadata.ironwood_commitment_tree_size; + } + + assert!( + total_ironwood_actions > 0, + "the fixture produced no ironwood actions; the walk asserted nothing about ironwood" + ); + // The counterpart of the guard above: the miner *asked* for Orchard, and from NU6.3 + // consensus requires an empty Orchard coinbase component, routing the reward into + // Ironwood actions instead. With coinbase-only blocks, served orchard actions must + // therefore be exactly zero. Together the two totals distinguish failure modes: + // pool-swap (orchard > 0, ironwood == 0: ironwood served under the orchard field) + // vs pool-drop (both zero) vs a broken routing premise. + assert_eq!( + total_orchard_actions, 0, + "an Orchard-receiver coinbase must carry no Orchard actions from NU6.3" + ); + + test_manager.close().await; +} + +/// Class-1 (consensus) predicate: in the NU5-through-NU6.2 era, a shielded +/// (orchard-receiver) miner's coinbase carries the reward as Orchard actions. +/// +/// The action count uses `>= 1` rather than the padded exact count so the predicate +/// does not couple to the Orchard bundle-padding rule. +fn is_valid_orchard_coinbase(block: &zebra_chain::block::Block) -> bool { + let Some(coinbase) = block.transactions.first() else { + return false; + }; + coinbase.is_coinbase() + && coinbase.version() == 5 + && coinbase.sapling_outputs().count() == 0 + && coinbase.orchard_actions().count() >= 1 + && coinbase.ironwood_actions().count() == 0 +} + +/// Class-1 (consensus) predicate: from NU6.3 the same miner's coinbase must have an +/// empty Orchard component, with the reward routed to Ironwood actions instead. +fn is_valid_ironwood_coinbase(block: &zebra_chain::block::Block) -> bool { + let Some(coinbase) = block.transactions.first() else { + return false; + }; + coinbase.is_coinbase() + && coinbase.version() == 6 + && coinbase.sapling_outputs().count() == 0 + && coinbase.orchard_actions().count() == 0 + && coinbase.ironwood_actions().count() >= 1 +} + +/// One-line coinbase summary for assertion messages, so a predicate failure names the +/// violated clause instead of reporting a bare boolean. +fn describe_coinbase(block: &zebra_chain::block::Block) -> String { + match block.transactions.first() { + Some(coinbase) => format!( + "coinbase: v{}, sapling outputs {}, orchard actions {}, ironwood actions {}, txs in block {}", + coinbase.version(), + coinbase.sapling_outputs().count(), + coinbase.orchard_actions().count(), + coinbase.ironwood_actions().count(), + block.transactions.len(), + ), + None => "block has no transactions".to_string(), + } +} + +/// Which shielded pool the fixture's coinbase reward must land in at a given height. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum CoinbaseEra { + /// Below NU5: the shielded-receiver reward cannot land in either pool. + Neither, + /// NU5 through NU6.2: the orchard-receiver reward is paid as Orchard actions. + Orchard, + /// From NU6.3: the same receiver's reward is routed to Ironwood actions. + Ironwood, +} + +/// Launches an orchard-receiver mining fixture on `activation_heights`, generates +/// `blocks` blocks, and asserts each height's raw validator block satisfies exactly +/// the era predicate `expected_era` assigns it. Raw blocks come straight from the +/// validator — zaino's serving is not involved, so a failure here is a class-1 +/// (consensus/routing) or fixture fact, never a zaino one. +async fn assert_coinbase_routing( + activation_heights: ActivationHeights, + blocks: u32, + expected_era: impl Fn(u64) -> CoinbaseEra, +) { + #[allow(deprecated)] + let mut test_manager = TestManager::::launch_mining_to( + MinerPool::Orchard, + &ValidatorKind::Zebrad, + None, + Some(activation_heights), + None, + false, + false, + false, + ) + .await + .unwrap(); + let subscriber = test_manager.subscriber().clone(); + + test_manager + .generate_blocks_and_wait_for_tip(blocks, &subscriber) + .await; + let tip = u64::from(subscriber.chain_height().await.unwrap().0); + + let connector = test_manager.full_node_jsonrpc_connector().await; + for height in 0..=tip { + let block = match connector + .get_block(height.to_string(), Some(0)) + .await + .unwrap() + { + GetBlockResponse::Raw(raw) => { + zebra_chain::block::Block::zcash_deserialize(raw.as_ref()) + .expect("validator serves deserializable blocks") + } + other => panic!("verbosity-0 getblock must return a raw block, got {other:?}"), + }; + + let (want_orchard, want_ironwood) = match expected_era(height) { + CoinbaseEra::Neither => (false, false), + CoinbaseEra::Orchard => (true, false), + CoinbaseEra::Ironwood => (false, true), + }; + assert_eq!( + is_valid_orchard_coinbase(&block), + want_orchard, + "orchard-coinbase predicate mismatch at height {height} ({})", + describe_coinbase(&block) + ); + assert_eq!( + is_valid_ironwood_coinbase(&block), + want_ironwood, + "ironwood-coinbase predicate mismatch at height {height} ({})", + describe_coinbase(&block) + ); + } + + test_manager.close().await; +} + +/// Orchard-only era: NU6.3 never activates (the current zebrad default heights), so +/// every post-NU5 coinbase stays an Orchard coinbase and no ironwood ever appears. +/// +/// multi_thread required: the test manager spawns the validator and indexer services. +#[tokio::test(flavor = "multi_thread")] +async fn orchard_only_coinbase_routing_zebrad() { + assert_coinbase_routing( + zaino_common::network::ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS, + 6, + |height| { + if height >= 2 { + CoinbaseEra::Orchard + } else { + CoinbaseEra::Neither + } + }, + ) + .await; +} + +/// Ironwood-only era: NU6.3 active from height 2 (with every prior upgrade), so every +/// post-activation coinbase is an Ironwood coinbase and no Orchard coinbase ever +/// appears. +/// +/// multi_thread required: the test manager spawns the validator and indexer services. +#[tokio::test(flavor = "multi_thread")] +async fn ironwood_only_coinbase_routing_zebrad() { + assert_coinbase_routing(NU6_3_ACTIVE_ACTIVATION_HEIGHTS, 6, |height| { + if height >= 2 { + CoinbaseEra::Ironwood + } else { + CoinbaseEra::Neither + } + }) + .await; +} + +/// The transition: the same unchanged orchard-receiver miner produces Orchard +/// coinbases through NU6.2 and Ironwood coinbases from the NU6.3 activation height — +/// each predicate exactly delimiting its era, so a mis-timed flip fails on both sides +/// of the boundary. +/// +/// multi_thread required: the test manager spawns the validator and indexer services. +#[tokio::test(flavor = "multi_thread")] +async fn orchard_coinbase_routing_flips_to_ironwood_at_activation_zebrad() { + // Two blocks past the boundary, so both eras carry more than one block. + assert_coinbase_routing( + ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS, + NU6_3_TRANSITION_BOUNDARY + 2, + |height| { + if height >= u64::from(NU6_3_TRANSITION_BOUNDARY) { + CoinbaseEra::Ironwood + } else if height >= 2 { + CoinbaseEra::Orchard + } else { + CoinbaseEra::Neither + } + }, + ) + .await; +} diff --git a/live-tests/e2e/tests/compact_block_wire.rs b/live-tests/e2e/tests/compact_block_wire.rs new file mode 100644 index 000000000..22c2bcd7c --- /dev/null +++ b/live-tests/e2e/tests/compact_block_wire.rs @@ -0,0 +1,202 @@ +//! Wire-tier (zainod gRPC) era tests for compact-block serving. +//! +//! The e2e-exclusive predicate here is **wire fidelity**: the compact blocks a real +//! tonic client receives from a running zainod equal, block for block, what the +//! in-process subscriber produces for the same request. Only this tier crosses the +//! protobuf encode → network → decode boundary and zainod's gRPC server; clientless +//! tests call subscriber methods in-process and can never observe that layer. +//! +//! On top of fidelity, each test asserts the era composition of the served stream for +//! the unfiltered request shape (empty `poolTypes`): orchard-only, ironwood-only, and +//! the orchard→ironwood transition at the NU6.3 activation boundary. + +use zaino_common::network::{ActivationHeights, ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS}; +use zaino_proto::proto::compact_formats::CompactBlock; +use zaino_proto::proto::service::compact_tx_streamer_client::CompactTxStreamerClient; +use zaino_proto::proto::service::{BlockId, BlockRange}; +#[allow(deprecated)] +use zaino_state::FetchService; +use zaino_state::ZcashIndexer as _; +use zaino_testutils::{ + make_uri, MinerPool, TestManager, ValidatorKind, NU6_3_ACTIVE_ACTIVATION_HEIGHTS, + NU6_3_TRANSITION_BOUNDARY, ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS, +}; +use zcash_local_net::validator::zebrad::Zebrad; + +/// Which shielded pool the fixture's coinbase reward must land in at a given height, +/// as observed in the served compact form. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum CoinbaseEra { + /// Below NU5: the shielded-receiver reward cannot land in either pool. + Neither, + /// NU5 through NU6.2: the orchard-receiver reward is paid as Orchard actions. + Orchard, + /// From NU6.3: the same receiver's reward is routed to Ironwood actions. + Ironwood, +} + +/// Launches an orchard-receiver mining fixture with zainod enabled, generates +/// `blocks` blocks, streams `[0, tip]` through zainod's real gRPC wire with the empty +/// `poolTypes` filter, and asserts: +/// +/// 1. wire fidelity — the streamed blocks equal the in-process subscriber's blocks; +/// 2. era composition — each height's served orchard/ironwood action presence matches +/// the era `expected_era` assigns it. +async fn assert_wire_served_eras( + activation_heights: ActivationHeights, + blocks: u32, + expected_era: impl Fn(u64) -> CoinbaseEra, +) { + #[allow(deprecated)] + let mut test_manager = TestManager::::launch_mining_to( + MinerPool::Orchard, + &ValidatorKind::Zebrad, + None, + Some(activation_heights), + None, + true, + false, + false, + ) + .await + .unwrap(); + let subscriber = test_manager.subscriber().clone(); + + test_manager + .generate_blocks_and_wait_for_tip(blocks, &subscriber) + .await; + let tip = u64::from(subscriber.chain_height().await.unwrap().0); + + // The in-process serving result: the same request answered without the wire. + let in_process = zaino_testutils::collect_block_range(&subscriber, 0, tip, vec![]).await; + + // The same request through zainod's real gRPC server. + let uri = make_uri( + test_manager + .zaino_grpc_listen_address + .expect("zaino enabled") + .port(), + ); + let mut grpc_client = CompactTxStreamerClient::connect(uri) + .await + .expect("zainod grpc reachable"); + let mut stream = grpc_client + .get_block_range(BlockRange { + start: Some(BlockId { + height: 0, + hash: vec![], + }), + end: Some(BlockId { + height: tip, + hash: vec![], + }), + // The unfiltered wire request real (including pre-Ironwood) clients send. + pool_types: vec![], + }) + .await + .expect("get_block_range succeeds") + .into_inner(); + let mut wire: Vec = Vec::new(); + while let Some(block) = stream.message().await.expect("stream yields blocks") { + wire.push(block); + } + + // E2e-exclusive predicate — wire fidelity: the protobuf encode/decode and zainod's + // gRPC server must be transparent. + assert_eq!( + wire.len(), + in_process.len(), + "wire and in-process must serve the same block count" + ); + for (wire_block, in_process_block) in wire.iter().zip(&in_process) { + assert_eq!( + wire_block, in_process_block, + "wire round-trip must be transparent at height {}", + wire_block.height + ); + } + + // Era composition of the served stream. + for block in &wire { + let orchard_actions: usize = block.vtx.iter().map(|tx| tx.actions.len()).sum(); + let ironwood_actions: usize = block.vtx.iter().map(|tx| tx.ironwood_actions.len()).sum(); + let (want_orchard, want_ironwood) = match expected_era(block.height) { + CoinbaseEra::Neither => (false, false), + CoinbaseEra::Orchard => (true, false), + CoinbaseEra::Ironwood => (false, true), + }; + assert_eq!( + orchard_actions > 0, + want_orchard, + "served orchard actions mismatch at height {} (orchard {orchard_actions}, \ + ironwood {ironwood_actions})", + block.height + ); + assert_eq!( + ironwood_actions > 0, + want_ironwood, + "served ironwood actions mismatch at height {} (orchard {orchard_actions}, \ + ironwood {ironwood_actions})", + block.height + ); + } + + test_manager.close().await; +} + +/// Orchard-only era over the wire: NU6.3 never activates (the current zebrad default +/// heights), so the served stream carries Orchard actions from height 2 and no +/// ironwood anywhere. +/// +/// multi_thread required: the test manager spawns the validator, indexer, and zainod. +#[tokio::test(flavor = "multi_thread")] +async fn orchard_only_wire_serving_zebrad() { + assert_wire_served_eras(ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS, 6, |height| { + if height >= 2 { + CoinbaseEra::Orchard + } else { + CoinbaseEra::Neither + } + }) + .await; +} + +/// Ironwood-only era over the wire: NU6.3 active from height 2, so the served stream +/// carries Ironwood actions from height 2 and no Orchard actions anywhere. +/// +/// multi_thread required: the test manager spawns the validator, indexer, and zainod. +#[tokio::test(flavor = "multi_thread")] +async fn ironwood_only_wire_serving_zebrad() { + assert_wire_served_eras(NU6_3_ACTIVE_ACTIVATION_HEIGHTS, 6, |height| { + if height >= 2 { + CoinbaseEra::Ironwood + } else { + CoinbaseEra::Neither + } + }) + .await; +} + +/// The transition over the wire: the same unchanged orchard-receiver miner's served +/// stream flips from Orchard to Ironwood actions exactly at the NU6.3 activation +/// height. +/// +/// multi_thread required: the test manager spawns the validator, indexer, and zainod. +#[tokio::test(flavor = "multi_thread")] +async fn orchard_to_ironwood_transition_wire_serving_zebrad() { + // Two blocks past the boundary, so both eras carry more than one block. + assert_wire_served_eras( + ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS, + NU6_3_TRANSITION_BOUNDARY + 2, + |height| { + if height >= u64::from(NU6_3_TRANSITION_BOUNDARY) { + CoinbaseEra::Ironwood + } else if height >= 2 { + CoinbaseEra::Orchard + } else { + CoinbaseEra::Neither + } + }, + ) + .await; +} diff --git a/live-tests/zaino-testutils/src/lib.rs b/live-tests/zaino-testutils/src/lib.rs index d4f5797e6..09596e2c8 100644 --- a/live-tests/zaino-testutils/src/lib.rs +++ b/live-tests/zaino-testutils/src/lib.rs @@ -67,6 +67,44 @@ macro_rules! validator_tests { }; } +/// Zebrad regtest heights with every upgrade through NU6.3 active from height 2, so +/// generated blocks carry V6 coinbases from the first post-genesis era. +pub const NU6_3_ACTIVE_ACTIVATION_HEIGHTS: ActivationHeights = ActivationHeights { + before_overwinter: Some(1), + overwinter: Some(1), + sapling: Some(1), + blossom: Some(1), + heartwood: Some(1), + canopy: Some(1), + nu5: Some(2), + nu6: Some(2), + nu6_1: Some(2), + nu6_2: Some(2), + nu6_3: Some(2), + nu7: None, +}; + +/// Keep in sync with [`ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS`]. +pub const NU6_3_TRANSITION_BOUNDARY: u32 = 6; + +/// NU5 era from height 2, NU6.3 from [`NU6_3_TRANSITION_BOUNDARY`]: the same +/// orchard-receiver miner first produces Orchard coinbases, then — at the activation +/// boundary — Ironwood ones. +pub const ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS: ActivationHeights = ActivationHeights { + before_overwinter: Some(1), + overwinter: Some(1), + sapling: Some(1), + blossom: Some(1), + heartwood: Some(1), + canopy: Some(1), + nu5: Some(2), + nu6: Some(2), + nu6_1: Some(2), + nu6_2: Some(2), + nu6_3: Some(NU6_3_TRANSITION_BOUNDARY), + nu7: None, +}; + /// All three value pools as the `i32`s a `get_block_range` request carries — /// the "request everything" pool filter. pub fn all_pools_i32() -> Vec { From 5aa718d6792ca4e8dca28dd6babd1ab2515dfde7 Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 18:48:43 -0700 Subject: [PATCH 076/134] test(config): retries are 0 in every nextest profile Drop the live-package retries override from 2 to 0: a flaky live test is a signal to fix, not to absorb. Production unit tests already ran with retries = 0. Co-Authored-By: Claude Fable 5 --- .config/nextest.toml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.config/nextest.toml b/.config/nextest.toml index 6ef3821f0..79608e2ff 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -34,14 +34,15 @@ path = "junit.xml" [test-groups] live-validators = { max-threads = 6 } -# Live-package tuning: capped concurrency (via the group), a slow-timeout for -# validator startup, and retries to absorb RPC-timing flakiness. Scoped to the -# live packages so production unit tests keep retries = 0 and no slow-timeout. +# Live-package tuning: capped concurrency (via the group) and a slow-timeout for +# validator startup. Scoped to the live packages so production unit tests keep +# no slow-timeout. Retries are 0 everywhere: a flaky live test is a signal to +# fix, not to absorb. [[profile.default.overrides]] filter = 'package(e2e) | package(clientless)' test-group = "live-validators" slow-timeout = { period = "60s", terminate-after = "10" } -retries = 2 +retries = 0 # The canonical test-target inventory consumed by the update/validate-test-targets # scripts via `cargo nextest list` -- NOT the CI test run (that is `default`). From 63c4583f129ec9b2993c025ce810325872870932 Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 18:48:43 -0700 Subject: [PATCH 077/134] fix(live): enable zaino in the clientless era-test launches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TestManager::subscriber() is only populated when zaino is enabled at launch; both clientless launch sites passed enable_zaino: false and panicked immediately in the container run ("service_subscriber is None"). The routing walk still reads raw validator blocks — zaino runs as the block-generation pollable but is never consulted; the runner doc now says so precisely. Co-Authored-By: Claude Fable 5 --- live-tests/clientless/tests/compact_block_consistency.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/live-tests/clientless/tests/compact_block_consistency.rs b/live-tests/clientless/tests/compact_block_consistency.rs index 7f2274f8b..50a0fe82d 100644 --- a/live-tests/clientless/tests/compact_block_consistency.rs +++ b/live-tests/clientless/tests/compact_block_consistency.rs @@ -34,7 +34,7 @@ async fn unfiltered_compact_blocks_match_chain_metadata_zebrad() { None, Some(NU6_3_ACTIVE_ACTIVATION_HEIGHTS), None, - false, + true, false, false, ) @@ -213,7 +213,8 @@ enum CoinbaseEra { /// Launches an orchard-receiver mining fixture on `activation_heights`, generates /// `blocks` blocks, and asserts each height's raw validator block satisfies exactly /// the era predicate `expected_era` assigns it. Raw blocks come straight from the -/// validator — zaino's serving is not involved, so a failure here is a class-1 +/// validator — zaino runs (the harness needs its subscriber as the block-generation +/// pollable) but is never consulted, so a failure here is a class-1 /// (consensus/routing) or fixture fact, never a zaino one. async fn assert_coinbase_routing( activation_heights: ActivationHeights, @@ -227,7 +228,7 @@ async fn assert_coinbase_routing( None, Some(activation_heights), None, - false, + true, false, false, ) From 1bfe61ce059bbdde652403e1affd79794705fb15 Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 19:04:12 -0700 Subject: [PATCH 078/134] test(live): full-chain routing report keyed to the issue #1368 hypothesis map MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refine the clientless coinbase-routing walk for the issue #1368 triage: predicate mismatches are now collected across every height and reported together (with describe_coinbase per violation) instead of aborting at the first failure. One run then distinguishes the hypotheses — a boundary-only mismatch is the zebrad builder off-by-one (A1), mismatches at every post-activation height mean the routing is absent (A2), and a clean raw-block pass while the e2e wire tests fail moves the blame to a zaino pool-swap (B). Doc comments on the ironwood predicate, the routing runner, and the e2e era-composition walk now link https://github.com/zingolabs/zaino/issues/1368 with the hypothesis summary, so the failing assertions point readers at the live triage. Co-Authored-By: Claude Fable 5 --- .../tests/compact_block_consistency.rs | 47 ++++++++++++++----- live-tests/e2e/tests/compact_block_wire.rs | 5 +- 2 files changed, 38 insertions(+), 14 deletions(-) diff --git a/live-tests/clientless/tests/compact_block_consistency.rs b/live-tests/clientless/tests/compact_block_consistency.rs index 50a0fe82d..27c766fc4 100644 --- a/live-tests/clientless/tests/compact_block_consistency.rs +++ b/live-tests/clientless/tests/compact_block_consistency.rs @@ -172,6 +172,12 @@ fn is_valid_orchard_coinbase(block: &zebra_chain::block::Block) -> bool { /// Class-1 (consensus) predicate: from NU6.3 the same miner's coinbase must have an /// empty Orchard component, with the reward routed to Ironwood actions instead. +/// +/// Known open question at the activation boundary: the first NU6.3 block's coinbase +/// has been observed served as Orchard (one action, zero ironwood) — see +/// for the hypotheses (zebrad +/// builder off-by-one vs missing routing vs a zaino pool-swap). This predicate over +/// raw validator blocks is that issue's disambiguator. fn is_valid_ironwood_coinbase(block: &zebra_chain::block::Block) -> bool { let Some(coinbase) = block.transactions.first() else { return false; @@ -216,6 +222,13 @@ enum CoinbaseEra { /// validator — zaino runs (the harness needs its subscriber as the block-generation /// pollable) but is never consulted, so a failure here is a class-1 /// (consensus/routing) or fixture fact, never a zaino one. +/// +/// Mismatches are collected across the whole chain and reported together rather than +/// aborting at the first failing height, so one run distinguishes the hypotheses of +/// : a boundary-only mismatch is the +/// zebrad builder off-by-one (A1), every-post-activation-height mismatches mean the +/// routing is absent (A2), and a clean pass here while the e2e wire tests fail moves +/// the blame to a zaino pool-swap (B). async fn assert_coinbase_routing( activation_heights: ActivationHeights, blocks: u32, @@ -242,6 +255,7 @@ async fn assert_coinbase_routing( let tip = u64::from(subscriber.chain_height().await.unwrap().0); let connector = test_manager.full_node_jsonrpc_connector().await; + let mut violations: Vec = Vec::new(); for height in 0..=tip { let block = match connector .get_block(height.to_string(), Some(0)) @@ -255,25 +269,32 @@ async fn assert_coinbase_routing( other => panic!("verbosity-0 getblock must return a raw block, got {other:?}"), }; - let (want_orchard, want_ironwood) = match expected_era(height) { + let expected = expected_era(height); + let (want_orchard, want_ironwood) = match expected { CoinbaseEra::Neither => (false, false), CoinbaseEra::Orchard => (true, false), CoinbaseEra::Ironwood => (false, true), }; - assert_eq!( - is_valid_orchard_coinbase(&block), - want_orchard, - "orchard-coinbase predicate mismatch at height {height} ({})", - describe_coinbase(&block) - ); - assert_eq!( - is_valid_ironwood_coinbase(&block), - want_ironwood, - "ironwood-coinbase predicate mismatch at height {height} ({})", - describe_coinbase(&block) - ); + let got_orchard = is_valid_orchard_coinbase(&block); + let got_ironwood = is_valid_ironwood_coinbase(&block); + if got_orchard != want_orchard || got_ironwood != want_ironwood { + violations.push(format!( + "height {height}: expected {expected:?}, predicates say \ + (orchard: {got_orchard}, ironwood: {got_ironwood}) — {}", + describe_coinbase(&block) + )); + } } + assert!( + violations.is_empty(), + "coinbase routing mismatches ({} of {} heights; see \ + https://github.com/zingolabs/zaino/issues/1368 for the hypothesis map):\n{}", + violations.len(), + tip + 1, + violations.join("\n") + ); + test_manager.close().await; } diff --git a/live-tests/e2e/tests/compact_block_wire.rs b/live-tests/e2e/tests/compact_block_wire.rs index 22c2bcd7c..36d7c70de 100644 --- a/live-tests/e2e/tests/compact_block_wire.rs +++ b/live-tests/e2e/tests/compact_block_wire.rs @@ -116,7 +116,10 @@ async fn assert_wire_served_eras( ); } - // Era composition of the served stream. + // Era composition of the served stream. Observed failing at the first + // ironwood-era height (served orchard 1 / ironwood 0) — hypotheses and the + // raw-block disambiguation procedure are tracked in + // . for block in &wire { let orchard_actions: usize = block.vtx.iter().map(|tx| tx.actions.len()).sum(); let ironwood_actions: usize = block.vtx.iter().map(|tx| tx.ironwood_actions.len()).sum(); From e9cbf4cae66e56fef97389cd655a957d05e54780 Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 19:13:07 -0700 Subject: [PATCH 079/134] test(live): gate ironwood-era tests on the CI image's missing routing The five ironwood-era live tests (both routing/metadata walks in clientless and both wire walks in e2e that require ironwood coinbases) are expected red on the current CI image: its zebrad 6.0.0-rc.0 never routes the orchard-receiver coinbase to Ironwood (issue #1368, hypothesis A2, confirmed by the full-chain raw-block report). Gate them with #[ignore] naming the issue; the orchard-only tests stay live and are unaffected. should_panic self-tracking is the house style for known-red tests, but these fail via a runtime-composed panic table across an async runner, so the ignore + issue link is the practical gate; un-gate when the image ships a routing-capable zebrad. Co-Authored-By: Claude Fable 5 --- live-tests/clientless/tests/compact_block_consistency.rs | 9 +++++++++ live-tests/e2e/tests/compact_block_wire.rs | 6 ++++++ 2 files changed, 15 insertions(+) diff --git a/live-tests/clientless/tests/compact_block_consistency.rs b/live-tests/clientless/tests/compact_block_consistency.rs index 27c766fc4..3fe4795be 100644 --- a/live-tests/clientless/tests/compact_block_consistency.rs +++ b/live-tests/clientless/tests/compact_block_consistency.rs @@ -23,6 +23,9 @@ use zebra_chain::serialization::ZcashDeserialize as _; /// multi_thread required: the test manager spawns the validator and indexer services. #[allow(deprecated)] #[tokio::test(flavor = "multi_thread")] +#[ignore = "zebrad in the CI image (6.0.0-rc.0) lacks ironwood coinbase routing — \ + expected red until the image ships a routing-capable build; \ + https://github.com/zingolabs/zaino/issues/1368"] async fn unfiltered_compact_blocks_match_chain_metadata_zebrad() { let mut test_manager = TestManager::::launch_mining_to( // Shielded mining: from NU6.3 an orchard-receiver coinbase is built as Ironwood @@ -324,6 +327,9 @@ async fn orchard_only_coinbase_routing_zebrad() { /// /// multi_thread required: the test manager spawns the validator and indexer services. #[tokio::test(flavor = "multi_thread")] +#[ignore = "zebrad in the CI image (6.0.0-rc.0) lacks ironwood coinbase routing — \ + expected red until the image ships a routing-capable build; \ + https://github.com/zingolabs/zaino/issues/1368"] async fn ironwood_only_coinbase_routing_zebrad() { assert_coinbase_routing(NU6_3_ACTIVE_ACTIVATION_HEIGHTS, 6, |height| { if height >= 2 { @@ -342,6 +348,9 @@ async fn ironwood_only_coinbase_routing_zebrad() { /// /// multi_thread required: the test manager spawns the validator and indexer services. #[tokio::test(flavor = "multi_thread")] +#[ignore = "zebrad in the CI image (6.0.0-rc.0) lacks ironwood coinbase routing — \ + expected red until the image ships a routing-capable build; \ + https://github.com/zingolabs/zaino/issues/1368"] async fn orchard_coinbase_routing_flips_to_ironwood_at_activation_zebrad() { // Two blocks past the boundary, so both eras carry more than one block. assert_coinbase_routing( diff --git a/live-tests/e2e/tests/compact_block_wire.rs b/live-tests/e2e/tests/compact_block_wire.rs index 36d7c70de..bc19fc63f 100644 --- a/live-tests/e2e/tests/compact_block_wire.rs +++ b/live-tests/e2e/tests/compact_block_wire.rs @@ -169,6 +169,9 @@ async fn orchard_only_wire_serving_zebrad() { /// /// multi_thread required: the test manager spawns the validator, indexer, and zainod. #[tokio::test(flavor = "multi_thread")] +#[ignore = "zebrad in the CI image (6.0.0-rc.0) lacks ironwood coinbase routing — \ + expected red until the image ships a routing-capable build; \ + https://github.com/zingolabs/zaino/issues/1368"] async fn ironwood_only_wire_serving_zebrad() { assert_wire_served_eras(NU6_3_ACTIVE_ACTIVATION_HEIGHTS, 6, |height| { if height >= 2 { @@ -186,6 +189,9 @@ async fn ironwood_only_wire_serving_zebrad() { /// /// multi_thread required: the test manager spawns the validator, indexer, and zainod. #[tokio::test(flavor = "multi_thread")] +#[ignore = "zebrad in the CI image (6.0.0-rc.0) lacks ironwood coinbase routing — \ + expected red until the image ships a routing-capable build; \ + https://github.com/zingolabs/zaino/issues/1368"] async fn orchard_to_ironwood_transition_wire_serving_zebrad() { // Two blocks past the boundary, so both eras carry more than one block. assert_wire_served_eras( From 9606e4bc95a8519b0c75d22c39220b87f6925d48 Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 22:01:45 -0700 Subject: [PATCH 080/134] build(live): bump zcash_local_net past the NU6.3-dropping launcher; un-gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit zcash_local_net moves from tag zcash_local_net_v0.7.0 (0b38b6ec) to rev 0dc4a51f: the v0.7.0 launcher's zebrad config writer silently dropped nu6_3 activation heights, so NU6.3 never activated on test chains — the root cause behind the ironwood-era test failures (https://github.com/zingolabs/zaino/issues/1368; zebrad v6.0.0-rc.0 and zaino were both exonerated). The new rev writes the "NU6.3" TOML key when configured and carries a regression test citing the issue. Un-gate the five #[ignore]d ironwood-era tests (e9cbf4ca): their gate reason ("image's zebrad lacks routing") was the interim hypothesis; with the launcher actually delivering the activation height they are expected green on the existing image. Co-Authored-By: Claude Fable 5 --- Cargo.lock | 6 +++--- Cargo.toml | 9 +++++---- live-tests/clientless/tests/compact_block_consistency.rs | 9 --------- live-tests/e2e/tests/compact_block_wire.rs | 6 ------ 4 files changed, 8 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 278632b78..f725865bb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6183,7 +6183,7 @@ dependencies = [ [[package]] name = "zcash_local_net" version = "0.7.0" -source = "git+https://github.com/zingolabs/infrastructure.git?tag=zcash_local_net_v0.7.0#0b38b6ecc9f5fb83754c6a14b36f575949d58929" +source = "git+https://github.com/zingolabs/infrastructure.git?rev=0dc4a51f7d9666a3df7eaf28ad4f95cacb86d646#0dc4a51f7d9666a3df7eaf28ad4f95cacb86d646" dependencies = [ "hex", "reqwest 0.12.28", @@ -6734,12 +6734,12 @@ dependencies = [ [[package]] name = "zingo-consensus" version = "0.1.0" -source = "git+https://github.com/zingolabs/infrastructure.git?tag=zcash_local_net_v0.7.0#0b38b6ecc9f5fb83754c6a14b36f575949d58929" +source = "git+https://github.com/zingolabs/infrastructure.git?rev=0dc4a51f7d9666a3df7eaf28ad4f95cacb86d646#0dc4a51f7d9666a3df7eaf28ad4f95cacb86d646" [[package]] name = "zingo_test_vectors" version = "0.0.1" -source = "git+https://github.com/zingolabs/infrastructure.git?tag=zcash_local_net_v0.7.0#0b38b6ecc9f5fb83754c6a14b36f575949d58929" +source = "git+https://github.com/zingolabs/infrastructure.git?rev=0dc4a51f7d9666a3df7eaf28ad4f95cacb86d646#0dc4a51f7d9666a3df7eaf28ad4f95cacb86d646" [[package]] name = "zip32" diff --git a/Cargo.toml b/Cargo.toml index 8a06b0abe..0bf07dc94 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -136,10 +136,11 @@ zaino-serve = { path = "packages/zaino-serve", version = "0.3.1", default-featur zaino-testutils = { path = "live-tests/zaino-testutils" } # Zingo-infra (validator launcher driving the live tests; not a prod dep). -# TEMPORARY (zingolabs/infrastructure#269): pinned to an exact commit on the -# `add_client_support` branch so every machine resolves the same code; restore -# the release tag once tagged upstream. -zcash_local_net = { git = "https://github.com/zingolabs/infrastructure.git", tag = "zcash_local_net_v0.7.0" } +# Pinned to an exact commit past zcash_local_net_v0.7.0: the v0.7.0 launcher's +# zebrad config writer silently drops NU6.3 activation heights +# (https://github.com/zingolabs/zaino/issues/1368); this rev carries the +# "NU6.3" TOML plumbing. Restore a release tag once one is cut upstream. +zcash_local_net = { git = "https://github.com/zingolabs/infrastructure.git", rev = "0dc4a51f7d9666a3df7eaf28ad4f95cacb86d646" } wire_serialized_transaction_test_data = "1.0.0" config = { version = "0.15", default-features = false, features = ["toml"] } diff --git a/live-tests/clientless/tests/compact_block_consistency.rs b/live-tests/clientless/tests/compact_block_consistency.rs index 3fe4795be..27c766fc4 100644 --- a/live-tests/clientless/tests/compact_block_consistency.rs +++ b/live-tests/clientless/tests/compact_block_consistency.rs @@ -23,9 +23,6 @@ use zebra_chain::serialization::ZcashDeserialize as _; /// multi_thread required: the test manager spawns the validator and indexer services. #[allow(deprecated)] #[tokio::test(flavor = "multi_thread")] -#[ignore = "zebrad in the CI image (6.0.0-rc.0) lacks ironwood coinbase routing — \ - expected red until the image ships a routing-capable build; \ - https://github.com/zingolabs/zaino/issues/1368"] async fn unfiltered_compact_blocks_match_chain_metadata_zebrad() { let mut test_manager = TestManager::::launch_mining_to( // Shielded mining: from NU6.3 an orchard-receiver coinbase is built as Ironwood @@ -327,9 +324,6 @@ async fn orchard_only_coinbase_routing_zebrad() { /// /// multi_thread required: the test manager spawns the validator and indexer services. #[tokio::test(flavor = "multi_thread")] -#[ignore = "zebrad in the CI image (6.0.0-rc.0) lacks ironwood coinbase routing — \ - expected red until the image ships a routing-capable build; \ - https://github.com/zingolabs/zaino/issues/1368"] async fn ironwood_only_coinbase_routing_zebrad() { assert_coinbase_routing(NU6_3_ACTIVE_ACTIVATION_HEIGHTS, 6, |height| { if height >= 2 { @@ -348,9 +342,6 @@ async fn ironwood_only_coinbase_routing_zebrad() { /// /// multi_thread required: the test manager spawns the validator and indexer services. #[tokio::test(flavor = "multi_thread")] -#[ignore = "zebrad in the CI image (6.0.0-rc.0) lacks ironwood coinbase routing — \ - expected red until the image ships a routing-capable build; \ - https://github.com/zingolabs/zaino/issues/1368"] async fn orchard_coinbase_routing_flips_to_ironwood_at_activation_zebrad() { // Two blocks past the boundary, so both eras carry more than one block. assert_coinbase_routing( diff --git a/live-tests/e2e/tests/compact_block_wire.rs b/live-tests/e2e/tests/compact_block_wire.rs index bc19fc63f..36d7c70de 100644 --- a/live-tests/e2e/tests/compact_block_wire.rs +++ b/live-tests/e2e/tests/compact_block_wire.rs @@ -169,9 +169,6 @@ async fn orchard_only_wire_serving_zebrad() { /// /// multi_thread required: the test manager spawns the validator, indexer, and zainod. #[tokio::test(flavor = "multi_thread")] -#[ignore = "zebrad in the CI image (6.0.0-rc.0) lacks ironwood coinbase routing — \ - expected red until the image ships a routing-capable build; \ - https://github.com/zingolabs/zaino/issues/1368"] async fn ironwood_only_wire_serving_zebrad() { assert_wire_served_eras(NU6_3_ACTIVE_ACTIVATION_HEIGHTS, 6, |height| { if height >= 2 { @@ -189,9 +186,6 @@ async fn ironwood_only_wire_serving_zebrad() { /// /// multi_thread required: the test manager spawns the validator, indexer, and zainod. #[tokio::test(flavor = "multi_thread")] -#[ignore = "zebrad in the CI image (6.0.0-rc.0) lacks ironwood coinbase routing — \ - expected red until the image ships a routing-capable build; \ - https://github.com/zingolabs/zaino/issues/1368"] async fn orchard_to_ironwood_transition_wire_serving_zebrad() { // Two blocks past the boundary, so both eras carry more than one block. assert_wire_served_eras( From be3c87961ce125f5a2407bdfd5710c45d99aa1c0 Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 22:11:33 -0700 Subject: [PATCH 081/134] fix(live): fetch the oracle block at verbosity 1, not 2 The oracle-parity fetch requested getblock verbosity 2, whose tx array carries full transaction objects; zaino-fetch's BlockObject types tx as txid strings (the verbosity-1 shape), so deserialization failed with "invalid type: map, expected a string" before any assertion ran. Verbosity 1 carries the trees field the oracle needs. Co-Authored-By: Claude Fable 5 --- live-tests/clientless/tests/compact_block_consistency.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/live-tests/clientless/tests/compact_block_consistency.rs b/live-tests/clientless/tests/compact_block_consistency.rs index 27c766fc4..197fa59a1 100644 --- a/live-tests/clientless/tests/compact_block_consistency.rs +++ b/live-tests/clientless/tests/compact_block_consistency.rs @@ -104,8 +104,11 @@ async fn unfiltered_compact_blocks_match_chain_metadata_zebrad() { // implementation's answer to compare zaino's served metadata against. Package // tests cannot express this: their "source of truth" is the object being // served, so any such comparison is circular. + // Verbosity 1: txids-as-strings plus the `trees` field the oracle needs + // (verbosity 2 returns full transaction objects, which BlockObject's + // string-typed `tx` field rejects). let oracle_trees = match connector - .get_block(block.height.to_string(), Some(2)) + .get_block(block.height.to_string(), Some(1)) .await .expect("validator serves verbose blocks") { From 13b306a6d96eb10055ff157119427194f2cfe478 Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 22:16:24 -0700 Subject: [PATCH 082/134] fix(live): align zebrad default heights with the devtool canonical set MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bumped zcash_local_net's devtool wallet client hardcodes the canonical regtest activation heights — all upgrades through NU6.3 at height 2 — while ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS still had nu6_3 unset (the option-(b) parking). A wallet signing with the NU6.3 consensus branch id against a zebrad without NU6.3 fails validation ("incorrect consensus branch id"), which broke the devtool send-from-faucet flows. ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS now carries nu6_3: Some(2) with a comment stating the coupling and citing issue #1368. The zcashd nuparams writer ignores nu6_3, so the zcashd path is unaffected. The orchard-only era fixtures relied on the old default meaning "NU6.3 never activates"; they get explicit heights instead: ORCHARD_ONLY_ACTIVATION_HEIGHTS in zaino-testutils (documented as client-free-launches-only, since wallet clients assume the canonical set) and a local ORCHARD_ONLY_HEIGHTS in the package-tier walk. Known consequence, accepted: wallet-funding flows that mine to a shielded pool now produce Ironwood coinbases the devtool wallet cannot scan yet — the previously-tracked wallet-balance test family; those stay red for the upstream-scanning reason, not this change's correctness. Co-Authored-By: Claude Fable 5 --- .../tests/compact_block_consistency.rs | 26 +++++++--------- live-tests/e2e/tests/compact_block_wire.rs | 13 ++++---- live-tests/zaino-testutils/src/lib.rs | 20 ++++++++++++ packages/zaino-common/src/config/network.rs | 7 ++++- .../chain_index/tests/proptest_blockgen.rs | 31 +++++++++++++------ 5 files changed, 67 insertions(+), 30 deletions(-) diff --git a/live-tests/clientless/tests/compact_block_consistency.rs b/live-tests/clientless/tests/compact_block_consistency.rs index 197fa59a1..98d60daa8 100644 --- a/live-tests/clientless/tests/compact_block_consistency.rs +++ b/live-tests/clientless/tests/compact_block_consistency.rs @@ -15,7 +15,8 @@ use zaino_state::FetchService; use zaino_state::ZcashIndexer as _; use zaino_testutils::{ MinerPool, TestManager, ValidatorKind, NU6_3_ACTIVE_ACTIVATION_HEIGHTS, - NU6_3_TRANSITION_BOUNDARY, ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS, + NU6_3_TRANSITION_BOUNDARY, ORCHARD_ONLY_ACTIVATION_HEIGHTS, + ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS, }; use zcash_local_net::validator::zebrad::Zebrad; use zebra_chain::serialization::ZcashDeserialize as _; @@ -301,23 +302,20 @@ async fn assert_coinbase_routing( test_manager.close().await; } -/// Orchard-only era: NU6.3 never activates (the current zebrad default heights), so -/// every post-NU5 coinbase stays an Orchard coinbase and no ironwood ever appears. +/// Orchard-only era: NU6.3 never activates, so every post-NU5 coinbase stays an +/// Orchard coinbase and no ironwood ever appears. (The zebrad *default* heights are +/// now the canonical NU6.3-at-2 set, so this fixture is explicit.) /// /// multi_thread required: the test manager spawns the validator and indexer services. #[tokio::test(flavor = "multi_thread")] async fn orchard_only_coinbase_routing_zebrad() { - assert_coinbase_routing( - zaino_common::network::ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS, - 6, - |height| { - if height >= 2 { - CoinbaseEra::Orchard - } else { - CoinbaseEra::Neither - } - }, - ) + assert_coinbase_routing(ORCHARD_ONLY_ACTIVATION_HEIGHTS, 6, |height| { + if height >= 2 { + CoinbaseEra::Orchard + } else { + CoinbaseEra::Neither + } + }) .await; } diff --git a/live-tests/e2e/tests/compact_block_wire.rs b/live-tests/e2e/tests/compact_block_wire.rs index 36d7c70de..495de0c2d 100644 --- a/live-tests/e2e/tests/compact_block_wire.rs +++ b/live-tests/e2e/tests/compact_block_wire.rs @@ -10,7 +10,7 @@ //! the unfiltered request shape (empty `poolTypes`): orchard-only, ironwood-only, and //! the orchard→ironwood transition at the NU6.3 activation boundary. -use zaino_common::network::{ActivationHeights, ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS}; +use zaino_common::network::ActivationHeights; use zaino_proto::proto::compact_formats::CompactBlock; use zaino_proto::proto::service::compact_tx_streamer_client::CompactTxStreamerClient; use zaino_proto::proto::service::{BlockId, BlockRange}; @@ -19,7 +19,8 @@ use zaino_state::FetchService; use zaino_state::ZcashIndexer as _; use zaino_testutils::{ make_uri, MinerPool, TestManager, ValidatorKind, NU6_3_ACTIVE_ACTIVATION_HEIGHTS, - NU6_3_TRANSITION_BOUNDARY, ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS, + NU6_3_TRANSITION_BOUNDARY, ORCHARD_ONLY_ACTIVATION_HEIGHTS, + ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS, }; use zcash_local_net::validator::zebrad::Zebrad; @@ -147,14 +148,14 @@ async fn assert_wire_served_eras( test_manager.close().await; } -/// Orchard-only era over the wire: NU6.3 never activates (the current zebrad default -/// heights), so the served stream carries Orchard actions from height 2 and no -/// ironwood anywhere. +/// Orchard-only era over the wire: NU6.3 never activates (explicit fixture — the +/// zebrad default heights are now the canonical NU6.3-at-2 set), so the served +/// stream carries Orchard actions from height 2 and no ironwood anywhere. /// /// multi_thread required: the test manager spawns the validator, indexer, and zainod. #[tokio::test(flavor = "multi_thread")] async fn orchard_only_wire_serving_zebrad() { - assert_wire_served_eras(ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS, 6, |height| { + assert_wire_served_eras(ORCHARD_ONLY_ACTIVATION_HEIGHTS, 6, |height| { if height >= 2 { CoinbaseEra::Orchard } else { diff --git a/live-tests/zaino-testutils/src/lib.rs b/live-tests/zaino-testutils/src/lib.rs index 09596e2c8..1dcfd0b9a 100644 --- a/live-tests/zaino-testutils/src/lib.rs +++ b/live-tests/zaino-testutils/src/lib.rs @@ -67,6 +67,26 @@ macro_rules! validator_tests { }; } +/// Orchard-era-only fixture: every upgrade through NU6.2 active from height 2 and +/// NU6.3 never activating, so coinbases stay Orchard for the whole chain. For +/// client-free launches only — the zcash-devtool wallet hardcodes the canonical +/// regtest set (NU6.3 at 2), so wallet-built transactions on this fixture would fail +/// consensus branch-id validation. +pub const ORCHARD_ONLY_ACTIVATION_HEIGHTS: ActivationHeights = ActivationHeights { + before_overwinter: Some(1), + overwinter: Some(1), + sapling: Some(1), + blossom: Some(1), + heartwood: Some(1), + canopy: Some(1), + nu5: Some(2), + nu6: Some(2), + nu6_1: Some(2), + nu6_2: Some(2), + nu6_3: None, + nu7: None, +}; + /// Zebrad regtest heights with every upgrade through NU6.3 active from height 2, so /// generated blocks carry V6 coinbases from the first post-genesis era. pub const NU6_3_ACTIVE_ACTIVATION_HEIGHTS: ActivationHeights = ActivationHeights { diff --git a/packages/zaino-common/src/config/network.rs b/packages/zaino-common/src/config/network.rs index 5f306ee32..d518d895b 100644 --- a/packages/zaino-common/src/config/network.rs +++ b/packages/zaino-common/src/config/network.rs @@ -5,6 +5,11 @@ use std::fmt; use serde::{Deserialize, Serialize}; use zebra_chain::parameters::testnet::ConfiguredActivationHeights; +/// Must equal zcash_local_net's `supported_regtest_activation_heights`: the +/// zcash-devtool wallet client hardcodes that canonical set (NU6.3 at 2), and a +/// zebrad configured differently rejects wallet-built transactions with +/// "incorrect consensus branch id". See +/// . pub const ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS: ActivationHeights = ActivationHeights { overwinter: Some(1), before_overwinter: Some(1), @@ -16,7 +21,7 @@ pub const ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS: ActivationHeights = ActivationHeigh nu6: Some(2), nu6_1: Some(2), nu6_2: Some(2), - nu6_3: None, + nu6_3: Some(2), nu7: None, }; diff --git a/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs b/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs index a56d1774a..ddb17407a 100644 --- a/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs +++ b/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs @@ -7,10 +7,7 @@ use proptest::{ }; use rand::seq::IndexedRandom; use tokio_stream::StreamExt as _; -use zaino_common::{ - network::{ActivationHeights, ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS}, - DatabaseConfig, Network, StorageConfig, -}; +use zaino_common::{network::ActivationHeights, DatabaseConfig, Network, StorageConfig}; use zaino_fetch::jsonrpsee::response::address_deltas::{ GetAddressDeltasParams, GetAddressDeltasResponse, }; @@ -484,13 +481,29 @@ fn passthrough_metadata_consistency_ironwood_only() { metadata_consistency_for_era(NU6_3_ACTIVE_HEIGHTS, Some(2), false) } -/// Orchard-only era on the current zebrad default heights (NU6.3 never activates): -/// fake Orchard content from height 2, and — since zebra's stock strategy cannot -/// generate V6 — ironwood provably never appears anywhere in the chain or the served -/// form. +/// Orchard-only heights: every upgrade through NU6.2 at height 2, NU6.3 never +/// activating. +const ORCHARD_ONLY_HEIGHTS: ActivationHeights = ActivationHeights { + before_overwinter: Some(1), + overwinter: Some(1), + sapling: Some(1), + blossom: Some(1), + heartwood: Some(1), + canopy: Some(1), + nu5: Some(2), + nu6: Some(2), + nu6_1: Some(2), + nu6_2: Some(2), + nu6_3: None, + nu7: None, +}; + +/// Orchard-only era (NU6.3 never activates): fake Orchard content from height 2, +/// and — since zebra's stock strategy cannot generate V6 — ironwood provably never +/// appears anywhere in the chain or the served form. #[test] fn passthrough_metadata_consistency_orchard_only() { - metadata_consistency_for_era(ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS, None, false) + metadata_consistency_for_era(ORCHARD_ONLY_HEIGHTS, None, false) } /// The transition: fake Orchard content below the NU6.3 boundary, fake Ironwood From 2258a8efa2d0b70c26d0aa5eae97319057f39b5a Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 22:37:30 -0700 Subject: [PATCH 083/134] test(e2e): gate wallet-funding tests on the ironwood scanning gap; fix pool helpers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The canonical NU6.3-at-2 heights make every shielded funding coinbase an Ironwood coinbase, which the devtool wallet cannot scan yet. The escape hatches are all closed: zebrad offers no sapling mining (MinerPool::Sapling is unsupported) and devtool cannot shield its own transparent coinbase — so the wallet-funding e2e family (sends, mempool flows, mining-reward and shielding checks: 20 wrappers across fetch/state) is structurally blocked until upstream ironwood scanning lands. Gate them with #[ignore] stating exactly that. Two zaino-testutils pool helpers were stale against the b1783b9c default-filter fix and only held vacuously on ironwood-free chains: - shielded_pools_i32 claimed to equal the empty-request default but omitted Ironwood — the cause of the block_range_returns_default_pools failure (default served ironwood actions, the explicit set stripped them). It now matches PoolTypeFilter::default(), and that test stays live: its single send spends the height-1 sapling-era coinbase, the one note the wallet can still see. - all_pools_i32 ("request everything") likewise gains Ironwood. Co-Authored-By: Claude Fable 5 --- live-tests/e2e/tests/devtool.rs | 20 ++++++++++++++++++++ live-tests/zaino-testutils/src/lib.rs | 12 +++++++++--- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/live-tests/e2e/tests/devtool.rs b/live-tests/e2e/tests/devtool.rs index 7338f695f..86079dfd9 100644 --- a/live-tests/e2e/tests/devtool.rs +++ b/live-tests/e2e/tests/devtool.rs @@ -2294,6 +2294,7 @@ mod zebrad { use zaino_state::FetchService; #[tokio::test(flavor = "multi_thread")] + #[ignore = "shielded funding mines Ironwood coinbases under the canonical NU6.3-at-2 heights, and the devtool wallet cannot scan Ironwood yet (zebrad offers no sapling mining; devtool cannot shield its own transparent coinbase) — un-gate when upstream ironwood scanning lands"] async fn receives_mining_reward() { crate::receives_mining_reward::().await; } @@ -2304,26 +2305,31 @@ mod zebrad { } #[tokio::test(flavor = "multi_thread")] + #[ignore = "shielded funding mines Ironwood coinbases under the canonical NU6.3-at-2 heights, and the devtool wallet cannot scan Ironwood yet (zebrad offers no sapling mining; devtool cannot shield its own transparent coinbase) — un-gate when upstream ironwood scanning lands"] async fn send_to_orchard() { crate::send_to_pool::(e2e::Pool::Orchard).await; } #[tokio::test(flavor = "multi_thread")] + #[ignore = "shielded funding mines Ironwood coinbases under the canonical NU6.3-at-2 heights, and the devtool wallet cannot scan Ironwood yet (zebrad offers no sapling mining; devtool cannot shield its own transparent coinbase) — un-gate when upstream ironwood scanning lands"] async fn send_to_sapling() { crate::send_to_pool::(e2e::Pool::Sapling).await; } #[tokio::test(flavor = "multi_thread")] + #[ignore = "shielded funding mines Ironwood coinbases under the canonical NU6.3-at-2 heights, and the devtool wallet cannot scan Ironwood yet (zebrad offers no sapling mining; devtool cannot shield its own transparent coinbase) — un-gate when upstream ironwood scanning lands"] async fn send_to_transparent() { crate::send_to_pool::(e2e::Pool::Transparent).await; } #[tokio::test(flavor = "multi_thread")] + #[ignore = "shielded funding mines Ironwood coinbases under the canonical NU6.3-at-2 heights, and the devtool wallet cannot scan Ironwood yet (zebrad offers no sapling mining; devtool cannot shield its own transparent coinbase) — un-gate when upstream ironwood scanning lands"] async fn send_to_all() { crate::send_to_all::().await; } #[tokio::test(flavor = "multi_thread")] + #[ignore = "shielded funding mines Ironwood coinbases under the canonical NU6.3-at-2 heights, and the devtool wallet cannot scan Ironwood yet (zebrad offers no sapling mining; devtool cannot shield its own transparent coinbase) — un-gate when upstream ironwood scanning lands"] async fn shield_for_validator() { crate::shield_for_validator::().await; } @@ -2343,21 +2349,25 @@ mod zebrad { } #[tokio::test(flavor = "multi_thread")] + #[ignore = "shielded funding mines Ironwood coinbases under the canonical NU6.3-at-2 heights, and the devtool wallet cannot scan Ironwood yet (zebrad offers no sapling mining; devtool cannot shield its own transparent coinbase) — un-gate when upstream ironwood scanning lands"] async fn get_raw_mempool() { crate::get_raw_mempool::().await; } #[tokio::test(flavor = "multi_thread")] + #[ignore = "shielded funding mines Ironwood coinbases under the canonical NU6.3-at-2 heights, and the devtool wallet cannot scan Ironwood yet (zebrad offers no sapling mining; devtool cannot shield its own transparent coinbase) — un-gate when upstream ironwood scanning lands"] async fn get_mempool_tx() { crate::get_mempool_tx::().await; } #[tokio::test(flavor = "multi_thread")] + #[ignore = "shielded funding mines Ironwood coinbases under the canonical NU6.3-at-2 heights, and the devtool wallet cannot scan Ironwood yet (zebrad offers no sapling mining; devtool cannot shield its own transparent coinbase) — un-gate when upstream ironwood scanning lands"] async fn get_mempool_stream() { crate::get_mempool_stream::().await; } #[tokio::test(flavor = "multi_thread")] + #[ignore = "shielded funding mines Ironwood coinbases under the canonical NU6.3-at-2 heights, and the devtool wallet cannot scan Ironwood yet (zebrad offers no sapling mining; devtool cannot shield its own transparent coinbase) — un-gate when upstream ironwood scanning lands"] async fn get_mempool_info() { crate::get_mempool_info_fetch().await; } @@ -2435,6 +2445,7 @@ mod zebrad { } #[tokio::test(flavor = "multi_thread")] + #[ignore = "shielded funding mines Ironwood coinbases under the canonical NU6.3-at-2 heights, and the devtool wallet cannot scan Ironwood yet (zebrad offers no sapling mining; devtool cannot shield its own transparent coinbase) — un-gate when upstream ironwood scanning lands"] async fn block_range_returns_all_pools() { crate::block_range_returns_all_pools().await; } @@ -2465,6 +2476,7 @@ mod zebrad { } #[tokio::test(flavor = "multi_thread")] + #[ignore = "shielded funding mines Ironwood coinbases under the canonical NU6.3-at-2 heights, and the devtool wallet cannot scan Ironwood yet (zebrad offers no sapling mining; devtool cannot shield its own transparent coinbase) — un-gate when upstream ironwood scanning lands"] async fn get_raw_mempool_fetch_vs_state() { crate::get_raw_mempool_fetch_vs_state().await; } @@ -2547,6 +2559,7 @@ mod zebrad { use zaino_state::StateService; #[tokio::test(flavor = "multi_thread")] + #[ignore = "shielded funding mines Ironwood coinbases under the canonical NU6.3-at-2 heights, and the devtool wallet cannot scan Ironwood yet (zebrad offers no sapling mining; devtool cannot shield its own transparent coinbase) — un-gate when upstream ironwood scanning lands"] async fn receives_mining_reward() { crate::receives_mining_reward::().await; } @@ -2557,26 +2570,31 @@ mod zebrad { } #[tokio::test(flavor = "multi_thread")] + #[ignore = "shielded funding mines Ironwood coinbases under the canonical NU6.3-at-2 heights, and the devtool wallet cannot scan Ironwood yet (zebrad offers no sapling mining; devtool cannot shield its own transparent coinbase) — un-gate when upstream ironwood scanning lands"] async fn send_to_orchard() { crate::send_to_pool::(e2e::Pool::Orchard).await; } #[tokio::test(flavor = "multi_thread")] + #[ignore = "shielded funding mines Ironwood coinbases under the canonical NU6.3-at-2 heights, and the devtool wallet cannot scan Ironwood yet (zebrad offers no sapling mining; devtool cannot shield its own transparent coinbase) — un-gate when upstream ironwood scanning lands"] async fn send_to_sapling() { crate::send_to_pool::(e2e::Pool::Sapling).await; } #[tokio::test(flavor = "multi_thread")] + #[ignore = "shielded funding mines Ironwood coinbases under the canonical NU6.3-at-2 heights, and the devtool wallet cannot scan Ironwood yet (zebrad offers no sapling mining; devtool cannot shield its own transparent coinbase) — un-gate when upstream ironwood scanning lands"] async fn send_to_transparent() { crate::send_to_pool::(e2e::Pool::Transparent).await; } #[tokio::test(flavor = "multi_thread")] + #[ignore = "shielded funding mines Ironwood coinbases under the canonical NU6.3-at-2 heights, and the devtool wallet cannot scan Ironwood yet (zebrad offers no sapling mining; devtool cannot shield its own transparent coinbase) — un-gate when upstream ironwood scanning lands"] async fn send_to_all() { crate::send_to_all::().await; } #[tokio::test(flavor = "multi_thread")] + #[ignore = "shielded funding mines Ironwood coinbases under the canonical NU6.3-at-2 heights, and the devtool wallet cannot scan Ironwood yet (zebrad offers no sapling mining; devtool cannot shield its own transparent coinbase) — un-gate when upstream ironwood scanning lands"] async fn shield_for_validator() { crate::shield_for_validator::().await; } @@ -2596,11 +2614,13 @@ mod zebrad { } #[tokio::test(flavor = "multi_thread")] + #[ignore = "shielded funding mines Ironwood coinbases under the canonical NU6.3-at-2 heights, and the devtool wallet cannot scan Ironwood yet (zebrad offers no sapling mining; devtool cannot shield its own transparent coinbase) — un-gate when upstream ironwood scanning lands"] async fn get_raw_mempool() { crate::get_raw_mempool::().await; } #[tokio::test(flavor = "multi_thread")] + #[ignore = "shielded funding mines Ironwood coinbases under the canonical NU6.3-at-2 heights, and the devtool wallet cannot scan Ironwood yet (zebrad offers no sapling mining; devtool cannot shield its own transparent coinbase) — un-gate when upstream ironwood scanning lands"] async fn get_mempool_info() { crate::get_mempool_info_state().await; } diff --git a/live-tests/zaino-testutils/src/lib.rs b/live-tests/zaino-testutils/src/lib.rs index 1dcfd0b9a..9fcd5649b 100644 --- a/live-tests/zaino-testutils/src/lib.rs +++ b/live-tests/zaino-testutils/src/lib.rs @@ -125,7 +125,7 @@ pub const ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS: ActivationHeights = Activati nu7: None, }; -/// All three value pools as the `i32`s a `get_block_range` request carries — +/// All four value pools as the `i32`s a `get_block_range` request carries — /// the "request everything" pool filter. pub fn all_pools_i32() -> Vec { use zaino_proto::proto::service::PoolType; @@ -133,14 +133,20 @@ pub fn all_pools_i32() -> Vec { PoolType::Transparent as i32, PoolType::Sapling as i32, PoolType::Orchard as i32, + PoolType::Ironwood as i32, ] } /// The shielded pools as request `i32`s — what `get_block_range` defaults to -/// when a request names no pools. +/// when a request names no pools (`PoolTypeFilter::default()`: every shielded +/// pool including Ironwood, transparent excluded). pub fn shielded_pools_i32() -> Vec { use zaino_proto::proto::service::PoolType; - vec![PoolType::Sapling as i32, PoolType::Orchard as i32] + vec![ + PoolType::Sapling as i32, + PoolType::Orchard as i32, + PoolType::Ironwood as i32, + ] } /// Collect a `get_block_range` query over heights `[start, end]` for the given From d174f4fd2fa9c4f878fbdb9638d81ea193ded679 Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 22:50:17 -0700 Subject: [PATCH 084/134] update devtool --- .env.testing-artifacts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.testing-artifacts b/.env.testing-artifacts index a3c399ea2..92b2fbae2 100644 --- a/.env.testing-artifacts +++ b/.env.testing-artifacts @@ -16,4 +16,4 @@ ZEBRA_VERSION=6.0.0-rc.0 # binary — no manual SHA bump needed. Pin a tag here only to freeze it. # Pinned to the NU6.3/Ironwood support commit: tip of # `ironwood-valar-portables`, under review as zcash/zcash-devtool#208. -DEVTOOL_VERSION=252114bbfa69b1baadfa4a84d1e2e8e37d2968b3 +DEVTOOL_VERSION=2efc8e9c690c6b4a844d6bb0a49bc66ba6fcbb14 From 3554c9112e204bbf699ea12b594e2b3bc8f5fb98 Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 22:52:24 -0700 Subject: [PATCH 085/134] test(e2e): un-gate the wallet-funding family for the ironwood-capable devtool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The devtool image pin moves to zingolabs/zcash-devtool support_ironwood_scan_model (librustzcash ironwood scanning #2533 + note storage #2531, ironwood surfaced in balance/list-unspent/send-max, shield-to-Ironwood post-NU6.3), so the 20 wrappers gated on "wallet cannot scan Ironwood" come back live. receives_mining_reward follows the consensus routing: the orchard-receiver mining reward is an Ironwood note under the canonical NU6.3-at-2 heights, so the assertion moves from orchard_spendable to ironwood_spendable (the pinned zcash_local_net's WalletBalance already parses it — its parser was built against the same devtool branch). The send_to_* receipt assertions are left as written on purpose: the first run against the new devtool reveals its post-NU6.3 output pool-selection policy, and the assertions should follow observed behavior, not guesses. Co-Authored-By: Claude Fable 5 --- live-tests/e2e/tests/devtool.rs | 26 ++++---------------------- 1 file changed, 4 insertions(+), 22 deletions(-) diff --git a/live-tests/e2e/tests/devtool.rs b/live-tests/e2e/tests/devtool.rs index 86079dfd9..68f47e613 100644 --- a/live-tests/e2e/tests/devtool.rs +++ b/live-tests/e2e/tests/devtool.rs @@ -157,9 +157,11 @@ where let (mut test_manager, clients) = launch_and_fund_faucet::(1).await; let faucet_balance = dbg!(clients.faucet_balance().await); + // Under the canonical NU6.3-at-2 heights the orchard-receiver mining reward + // is routed to the Ironwood pool. assert!( - faucet_balance.orchard_spendable > 0, - "faucet should hold a spendable orchard coinbase note, got {faucet_balance:?}" + faucet_balance.ironwood_spendable > 0, + "faucet should hold a spendable ironwood coinbase note, got {faucet_balance:?}" ); test_manager.close().await; @@ -2294,7 +2296,6 @@ mod zebrad { use zaino_state::FetchService; #[tokio::test(flavor = "multi_thread")] - #[ignore = "shielded funding mines Ironwood coinbases under the canonical NU6.3-at-2 heights, and the devtool wallet cannot scan Ironwood yet (zebrad offers no sapling mining; devtool cannot shield its own transparent coinbase) — un-gate when upstream ironwood scanning lands"] async fn receives_mining_reward() { crate::receives_mining_reward::().await; } @@ -2305,31 +2306,26 @@ mod zebrad { } #[tokio::test(flavor = "multi_thread")] - #[ignore = "shielded funding mines Ironwood coinbases under the canonical NU6.3-at-2 heights, and the devtool wallet cannot scan Ironwood yet (zebrad offers no sapling mining; devtool cannot shield its own transparent coinbase) — un-gate when upstream ironwood scanning lands"] async fn send_to_orchard() { crate::send_to_pool::(e2e::Pool::Orchard).await; } #[tokio::test(flavor = "multi_thread")] - #[ignore = "shielded funding mines Ironwood coinbases under the canonical NU6.3-at-2 heights, and the devtool wallet cannot scan Ironwood yet (zebrad offers no sapling mining; devtool cannot shield its own transparent coinbase) — un-gate when upstream ironwood scanning lands"] async fn send_to_sapling() { crate::send_to_pool::(e2e::Pool::Sapling).await; } #[tokio::test(flavor = "multi_thread")] - #[ignore = "shielded funding mines Ironwood coinbases under the canonical NU6.3-at-2 heights, and the devtool wallet cannot scan Ironwood yet (zebrad offers no sapling mining; devtool cannot shield its own transparent coinbase) — un-gate when upstream ironwood scanning lands"] async fn send_to_transparent() { crate::send_to_pool::(e2e::Pool::Transparent).await; } #[tokio::test(flavor = "multi_thread")] - #[ignore = "shielded funding mines Ironwood coinbases under the canonical NU6.3-at-2 heights, and the devtool wallet cannot scan Ironwood yet (zebrad offers no sapling mining; devtool cannot shield its own transparent coinbase) — un-gate when upstream ironwood scanning lands"] async fn send_to_all() { crate::send_to_all::().await; } #[tokio::test(flavor = "multi_thread")] - #[ignore = "shielded funding mines Ironwood coinbases under the canonical NU6.3-at-2 heights, and the devtool wallet cannot scan Ironwood yet (zebrad offers no sapling mining; devtool cannot shield its own transparent coinbase) — un-gate when upstream ironwood scanning lands"] async fn shield_for_validator() { crate::shield_for_validator::().await; } @@ -2349,25 +2345,21 @@ mod zebrad { } #[tokio::test(flavor = "multi_thread")] - #[ignore = "shielded funding mines Ironwood coinbases under the canonical NU6.3-at-2 heights, and the devtool wallet cannot scan Ironwood yet (zebrad offers no sapling mining; devtool cannot shield its own transparent coinbase) — un-gate when upstream ironwood scanning lands"] async fn get_raw_mempool() { crate::get_raw_mempool::().await; } #[tokio::test(flavor = "multi_thread")] - #[ignore = "shielded funding mines Ironwood coinbases under the canonical NU6.3-at-2 heights, and the devtool wallet cannot scan Ironwood yet (zebrad offers no sapling mining; devtool cannot shield its own transparent coinbase) — un-gate when upstream ironwood scanning lands"] async fn get_mempool_tx() { crate::get_mempool_tx::().await; } #[tokio::test(flavor = "multi_thread")] - #[ignore = "shielded funding mines Ironwood coinbases under the canonical NU6.3-at-2 heights, and the devtool wallet cannot scan Ironwood yet (zebrad offers no sapling mining; devtool cannot shield its own transparent coinbase) — un-gate when upstream ironwood scanning lands"] async fn get_mempool_stream() { crate::get_mempool_stream::().await; } #[tokio::test(flavor = "multi_thread")] - #[ignore = "shielded funding mines Ironwood coinbases under the canonical NU6.3-at-2 heights, and the devtool wallet cannot scan Ironwood yet (zebrad offers no sapling mining; devtool cannot shield its own transparent coinbase) — un-gate when upstream ironwood scanning lands"] async fn get_mempool_info() { crate::get_mempool_info_fetch().await; } @@ -2445,7 +2437,6 @@ mod zebrad { } #[tokio::test(flavor = "multi_thread")] - #[ignore = "shielded funding mines Ironwood coinbases under the canonical NU6.3-at-2 heights, and the devtool wallet cannot scan Ironwood yet (zebrad offers no sapling mining; devtool cannot shield its own transparent coinbase) — un-gate when upstream ironwood scanning lands"] async fn block_range_returns_all_pools() { crate::block_range_returns_all_pools().await; } @@ -2476,7 +2467,6 @@ mod zebrad { } #[tokio::test(flavor = "multi_thread")] - #[ignore = "shielded funding mines Ironwood coinbases under the canonical NU6.3-at-2 heights, and the devtool wallet cannot scan Ironwood yet (zebrad offers no sapling mining; devtool cannot shield its own transparent coinbase) — un-gate when upstream ironwood scanning lands"] async fn get_raw_mempool_fetch_vs_state() { crate::get_raw_mempool_fetch_vs_state().await; } @@ -2559,7 +2549,6 @@ mod zebrad { use zaino_state::StateService; #[tokio::test(flavor = "multi_thread")] - #[ignore = "shielded funding mines Ironwood coinbases under the canonical NU6.3-at-2 heights, and the devtool wallet cannot scan Ironwood yet (zebrad offers no sapling mining; devtool cannot shield its own transparent coinbase) — un-gate when upstream ironwood scanning lands"] async fn receives_mining_reward() { crate::receives_mining_reward::().await; } @@ -2570,31 +2559,26 @@ mod zebrad { } #[tokio::test(flavor = "multi_thread")] - #[ignore = "shielded funding mines Ironwood coinbases under the canonical NU6.3-at-2 heights, and the devtool wallet cannot scan Ironwood yet (zebrad offers no sapling mining; devtool cannot shield its own transparent coinbase) — un-gate when upstream ironwood scanning lands"] async fn send_to_orchard() { crate::send_to_pool::(e2e::Pool::Orchard).await; } #[tokio::test(flavor = "multi_thread")] - #[ignore = "shielded funding mines Ironwood coinbases under the canonical NU6.3-at-2 heights, and the devtool wallet cannot scan Ironwood yet (zebrad offers no sapling mining; devtool cannot shield its own transparent coinbase) — un-gate when upstream ironwood scanning lands"] async fn send_to_sapling() { crate::send_to_pool::(e2e::Pool::Sapling).await; } #[tokio::test(flavor = "multi_thread")] - #[ignore = "shielded funding mines Ironwood coinbases under the canonical NU6.3-at-2 heights, and the devtool wallet cannot scan Ironwood yet (zebrad offers no sapling mining; devtool cannot shield its own transparent coinbase) — un-gate when upstream ironwood scanning lands"] async fn send_to_transparent() { crate::send_to_pool::(e2e::Pool::Transparent).await; } #[tokio::test(flavor = "multi_thread")] - #[ignore = "shielded funding mines Ironwood coinbases under the canonical NU6.3-at-2 heights, and the devtool wallet cannot scan Ironwood yet (zebrad offers no sapling mining; devtool cannot shield its own transparent coinbase) — un-gate when upstream ironwood scanning lands"] async fn send_to_all() { crate::send_to_all::().await; } #[tokio::test(flavor = "multi_thread")] - #[ignore = "shielded funding mines Ironwood coinbases under the canonical NU6.3-at-2 heights, and the devtool wallet cannot scan Ironwood yet (zebrad offers no sapling mining; devtool cannot shield its own transparent coinbase) — un-gate when upstream ironwood scanning lands"] async fn shield_for_validator() { crate::shield_for_validator::().await; } @@ -2614,13 +2598,11 @@ mod zebrad { } #[tokio::test(flavor = "multi_thread")] - #[ignore = "shielded funding mines Ironwood coinbases under the canonical NU6.3-at-2 heights, and the devtool wallet cannot scan Ironwood yet (zebrad offers no sapling mining; devtool cannot shield its own transparent coinbase) — un-gate when upstream ironwood scanning lands"] async fn get_raw_mempool() { crate::get_raw_mempool::().await; } #[tokio::test(flavor = "multi_thread")] - #[ignore = "shielded funding mines Ironwood coinbases under the canonical NU6.3-at-2 heights, and the devtool wallet cannot scan Ironwood yet (zebrad offers no sapling mining; devtool cannot shield its own transparent coinbase) — un-gate when upstream ironwood scanning lands"] async fn get_mempool_info() { crate::get_mempool_info_state().await; } From df443c9c931c87b0afb138ebeab932798843caa9 Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 23:03:07 -0700 Subject: [PATCH 086/134] =?UTF-8?q?test(e2e):=20follow=20devtool's=20NU6.3?= =?UTF-8?q?=20output=20routing=20=E2=80=94=20receipts=20land=20in=20Ironwo?= =?UTF-8?q?od?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first run against the ironwood-capable devtool revealed its post-NU6.3 output pool-selection policy: unified-address sends and shields both produce Ironwood outputs, not Orchard (matching the coinbase routing convention). The seven receipt-pool failures were all this — every deeper layer (scanning, spending, mempool, mining reward) passed. Pool gains an Ironwood variant (unified address kind; ironwood_actions in compact form; ironwood_spendable in balances). The assertions follow the era: - send_to_orchard becomes send_to_ironwood (zebrad mods only; the zcashd variant keeps Orchard — no NU6.3 there); - send_to_all expects the unified receipt in Ironwood and additionally asserts orchard stays 0, distinguishing mislabelled from misrouted; - shield_for_validator confirms the shielded balance in Ironwood; - block_range_returns_all_pools asserts the unified send carries ironwood_actions and no Orchard actions. Co-Authored-By: Claude Fable 5 --- live-tests/e2e/src/devtool.rs | 1 + live-tests/e2e/src/lib.rs | 11 ++++++--- live-tests/e2e/tests/devtool.rs | 41 ++++++++++++++++++--------------- 3 files changed, 31 insertions(+), 22 deletions(-) diff --git a/live-tests/e2e/src/devtool.rs b/live-tests/e2e/src/devtool.rs index 31b772d03..4fed529e6 100644 --- a/live-tests/e2e/src/devtool.rs +++ b/live-tests/e2e/src/devtool.rs @@ -234,6 +234,7 @@ impl Pool { pub fn spendable_balance(self, balance: &WalletBalance) -> u64 { match self { Pool::Orchard => balance.orchard_spendable, + Pool::Ironwood => balance.ironwood_spendable, Pool::Sapling => balance.sapling_spendable, Pool::Transparent => balance.transparent_spendable, } diff --git a/live-tests/e2e/src/lib.rs b/live-tests/e2e/src/lib.rs index 6ac1bc325..c2788fd0b 100644 --- a/live-tests/e2e/src/lib.rs +++ b/live-tests/e2e/src/lib.rs @@ -17,8 +17,12 @@ pub mod devtool; /// address string plus a balance-field closure. #[derive(Clone, Copy, Debug)] pub enum Pool { - /// Orchard (funds routed via a unified address). + /// Orchard (funds routed via a unified address through NU6.2; from NU6.3 + /// devtool routes unified-address outputs to Ironwood — use + /// [`Pool::Ironwood`] for the receipt pool on NU6.3-active chains). Orchard, + /// Ironwood (funds routed via a unified address from NU6.3). + Ironwood, /// Sapling. Sapling, /// Transparent. @@ -30,7 +34,7 @@ impl Pool { /// funds into this pool. pub fn address_kind(self) -> &'static str { match self { - Pool::Orchard => "unified", + Pool::Orchard | Pool::Ironwood => "unified", Pool::Sapling => "sapling", Pool::Transparent => "transparent", } @@ -38,7 +42,7 @@ impl Pool { } /// Whether the compact tx with `txid` carries no data for `pool` (transparent -/// `vout` / sapling `outputs` / orchard `actions`). +/// `vout` / sapling `outputs` / orchard `actions` / `ironwood_actions`). fn pool_tx_field_empty(block: &CompactBlock, txid: &TxId, pool: Pool) -> bool { let tx = block .vtx @@ -49,6 +53,7 @@ fn pool_tx_field_empty(block: &CompactBlock, txid: &TxId, pool: Pool) -> bool { Pool::Transparent => tx.vout.is_empty(), Pool::Sapling => tx.outputs.is_empty(), Pool::Orchard => tx.actions.is_empty(), + Pool::Ironwood => tx.ironwood_actions.is_empty(), } } diff --git a/live-tests/e2e/tests/devtool.rs b/live-tests/e2e/tests/devtool.rs index 68f47e613..e334ca441 100644 --- a/live-tests/e2e/tests/devtool.rs +++ b/live-tests/e2e/tests/devtool.rs @@ -170,8 +170,8 @@ where /// Port of the `assert_send_to_pool` family from `wallet_to_validator` /// (zebrad): send 250_000 from the faucet (spending its orchard coinbase) to /// the recipient's `pool` address, mine it in, and assert the recipient's -/// synced wallet shows the receipt in that pool. Covers `send_to_orchard` -/// (unified), `send_to_sapling`, and the basic transparent receipt — the +/// synced wallet shows the receipt in that pool. Covers `send_to_ironwood` +/// (unified, the NU6.3-era receipt pool), `send_to_sapling`, and the basic transparent receipt — the /// per-pool address wiring is the new surface under test. (The original /// `send_to_transparent` additionally mines across the finalization boundary /// under transparent mining; that variant is deferred, as it exercises the @@ -233,7 +233,11 @@ where clients.sync_recipient().await; let balance = clients.recipient_balance().await; - assert_eq!(e2e::Pool::Orchard.spendable_balance(&balance), 250_000); + // From NU6.3 devtool routes unified-address outputs to Ironwood; the + // orchard pool must stay empty (a nonzero orchard here means the receipt + // was mislabelled, not merely misrouted). + assert_eq!(e2e::Pool::Ironwood.spendable_balance(&balance), 250_000); + assert_eq!(e2e::Pool::Orchard.spendable_balance(&balance), 0); assert_eq!(e2e::Pool::Sapling.spendable_balance(&balance), 250_000); assert_eq!(e2e::Pool::Transparent.spendable_balance(&balance), 250_000); @@ -242,10 +246,10 @@ where /// Port of `wallet_to_validator::shield_for_validator` (zebrad): the faucet /// sends 250_000 to the recipient's transparent address; the recipient -/// confirms the transparent receipt, shields it to orchard, and confirms the -/// orchard balance net of the ZIP-317 fee (250_000 − 15_000 = 235_000 — the -/// first devtool exercise of `shield`, and the constant flagged as needing -/// re-verification against devtool's note selection). +/// confirms the transparent receipt, shields it (to Ironwood from NU6.3), and +/// confirms the shielded balance net of the ZIP-317 fee (250_000 − 15_000 = +/// 235_000 — the first devtool exercise of `shield`, and the constant flagged +/// as needing re-verification against devtool's note selection). async fn shield_for_validator() where Service: TestService, @@ -273,7 +277,7 @@ where clients.sync_recipient().await; assert_eq!( - e2e::Pool::Orchard.spendable_balance(&clients.recipient_balance().await), + e2e::Pool::Ironwood.spendable_balance(&clients.recipient_balance().await), 235_000 ); @@ -879,7 +883,7 @@ async fn block_range_returns_all_pools() { ) .await; - // Three orchard coinbase notes (one per send below — devtool will not + // Three ironwood coinbase notes (one per send below — devtool will not // chain unconfirmed change), then one send to each pool's recipient // address, mined into a single block. svc.generate_blocks_and_wait_for_tips(3).await; @@ -890,7 +894,7 @@ async fn block_range_returns_all_pools() { let recipient_u = clients.get_recipient_address("unified").await; let deshielding_txid = clients.send_from_faucet(&recipient_t, 250_000).await; let sapling_txid = clients.send_from_faucet(&recipient_s, 250_000).await; - let orchard_txid = clients.send_from_faucet(&recipient_u, 250_000).await; + let ironwood_txid = clients.send_from_faucet(&recipient_u, 250_000).await; svc.generate_blocks_and_wait_for_tips(1).await; let start_height: u64 = 1; @@ -928,11 +932,10 @@ async fn block_range_returns_all_pools() { &e2e::devtool::txid_from_devtool(&sapling_txid), e2e::Pool::Sapling, ); - e2e::assert_pool_present( - compact_block, - &e2e::devtool::txid_from_devtool(&orchard_txid), - e2e::Pool::Orchard, - ); + let ironwood_txid = e2e::devtool::txid_from_devtool(&ironwood_txid); + e2e::assert_pool_present(compact_block, &ironwood_txid, e2e::Pool::Ironwood); + // The unified-address send must carry no Orchard actions from NU6.3. + e2e::assert_pool_absent(compact_block, &ironwood_txid, e2e::Pool::Orchard); svc.test_manager.close().await; } @@ -2306,8 +2309,8 @@ mod zebrad { } #[tokio::test(flavor = "multi_thread")] - async fn send_to_orchard() { - crate::send_to_pool::(e2e::Pool::Orchard).await; + async fn send_to_ironwood() { + crate::send_to_pool::(e2e::Pool::Ironwood).await; } #[tokio::test(flavor = "multi_thread")] @@ -2559,8 +2562,8 @@ mod zebrad { } #[tokio::test(flavor = "multi_thread")] - async fn send_to_orchard() { - crate::send_to_pool::(e2e::Pool::Orchard).await; + async fn send_to_ironwood() { + crate::send_to_pool::(e2e::Pool::Ironwood).await; } #[tokio::test(flavor = "multi_thread")] From 6d838c05e7f1e4d69bd02167bbf3d39a2d59fe49 Mon Sep 17 00:00:00 2001 From: idky137 Date: Sun, 5 Jul 2026 15:37:34 +0100 Subject: [PATCH 087/134] merged fetchservice and stateservice into unified NodeBackedIndexerService --- CHANGELOG.md | 13 + live-tests/clientless/tests/chain_cache.rs | 171 +- live-tests/clientless/tests/fetch_service.rs | 9 +- live-tests/clientless/tests/json_server.rs | 19 +- live-tests/clientless/tests/state_service.rs | 9 +- live-tests/e2e/tests/devtool.rs | 309 ++- live-tests/e2e/tests/devtool_zcashd.rs | 12 +- live-tests/e2e/tests/test_vectors.rs | 12 +- live-tests/zaino-testutils/src/lib.rs | 308 ++- packages/zaino-state/src/backends.rs | 71 - packages/zaino-state/src/backends/state.rs | 1896 ----------------- .../chain_index/source/validator_connector.rs | 40 +- .../src/chain_index/tests/mockchain_tests.rs | 24 + packages/zaino-state/src/config.rs | 176 +- packages/zaino-state/src/error.rs | 166 +- packages/zaino-state/src/indexer.rs | 266 ++- .../node_backed_indexer.rs} | 527 +++-- packages/zaino-state/src/lib.rs | 16 +- packages/zainod/src/config.rs | 209 +- packages/zainod/src/error.rs | 27 +- packages/zainod/src/indexer.rs | 27 +- 21 files changed, 1250 insertions(+), 3057 deletions(-) delete mode 100644 packages/zaino-state/src/backends.rs delete mode 100644 packages/zaino-state/src/backends/state.rs rename packages/zaino-state/src/{backends/fetch.rs => indexer/node_backed_indexer.rs} (82%) diff --git a/CHANGELOG.md b/CHANGELOG.md index c74c79e32..ee246f9b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,19 @@ and this library adheres to Rust's notion of ## Unreleased ### Changed +- `zaino-state`: `FetchService` and `StateService` are merged into a single + generic `NodeBackedIndexerService` (module + `zaino_state::indexer::node_backed_indexer`; the former `backends` module is + gone). The validator connection is now selected at runtime rather than by type: + `NodeBackedIndexerServiceConfig { common, connection }` carries a + `ValidatorConnectionType` of either `Rpc` (JSON-RPC, formerly `Fetch`) or + `Direct(DirectConnectionConfig)` (Zebra `ReadStateService`, formerly `State`). + The per-backend `Fetch/StateServiceConfig`, `Fetch/StateServiceError`, and + `BackendConfig` types are replaced by `NodeBackedIndexerServiceConfig`, + `NodeBackedIndexerServiceError`, and `ValidatorConnectionType`. +- **Breaking** — config: `zainod.toml`'s `backend` selector is renamed + `state` → `direct` and `fetch` → `rpc`. The legacy `"state"` / `"fetch"` + values are still accepted as aliases, so existing config files keep working. - `zaino-state`: the `ChainIndex` trait is split into `ChainIndex` (the wallet-essential core: chain/tx/address/mempool access) and a `ChainIndexRpcExt: ChainIndex` extension (compact-block serving, subtree diff --git a/live-tests/clientless/tests/chain_cache.rs b/live-tests/clientless/tests/chain_cache.rs index 2bc08ea2e..cad4f6816 100644 --- a/live-tests/clientless/tests/chain_cache.rs +++ b/live-tests/clientless/tests/chain_cache.rs @@ -1,24 +1,20 @@ use zaino_common::network::ActivationHeights; use zaino_fetch::jsonrpsee::connector::JsonRpSeeConnector; -use zaino_state::{ZcashIndexer, ZcashService}; -use zaino_testutils::{TestManager, ValidatorExt, ValidatorKind}; -use zainodlib::error::IndexerError; +use zaino_testutils::{Direct, Rpc, TestManager, ValidatorExt, ValidatorKind}; #[allow(deprecated)] -async fn create_test_manager_and_connector( +async fn create_test_manager_and_connector( validator: &ValidatorKind, activation_heights: Option, chain_cache: Option, enable_zaino: bool, enable_clients: bool, -) -> (TestManager, JsonRpSeeConnector) +) -> (TestManager, JsonRpSeeConnector) where T: ValidatorExt, - Service: zaino_testutils::TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: zaino_testutils::PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { - let test_manager = TestManager::::launch( + let test_manager = TestManager::::launch( validator, None, activation_heights, @@ -50,7 +46,7 @@ mod chain_query_interface { chain_index::{ChainIndex, ChainIndexRpcExt}, ChainIndexConfig, }, - FetchService, Height, StateService, StateServiceConfig, ZcashService, + Height, NodeBackedIndexerService, NodeBackedIndexerServiceConfig, ZcashService, }; #[cfg(feature = "zcashd_support")] use zcash_local_net::validator::zcashd::Zcashd; @@ -66,26 +62,24 @@ mod chain_query_interface { use super::*; #[allow(deprecated)] - async fn create_test_manager_and_chain_index( + async fn create_test_manager_and_chain_index( validator: &ValidatorKind, chain_cache: Option, enable_zaino: bool, enable_clients: bool, ephemeral: bool, ) -> ( - TestManager, + TestManager, JsonRpSeeConnector, - Option, + Option, NodeBackedChainIndex, NodeBackedChainIndexSubscriber, ) where C: ValidatorExt, - Service: zaino_testutils::TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: zaino_testutils::PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { - let (test_manager, json_service) = create_test_manager_and_connector::( + let (test_manager, json_service) = create_test_manager_and_connector::( validator, None, chain_cache.clone(), @@ -125,42 +119,43 @@ mod chain_query_interface { NetworkKind::Testnet => zebra_chain::parameters::Network::new_default_testnet(), NetworkKind::Mainnet => zebra_chain::parameters::Network::Mainnet, }; - // FIXME: when state service is integrated into chain index this initialization must change - let state_service = StateService::spawn(StateServiceConfig::new( - zebra_state::Config { - cache_dir: state_chain_cache_dir, - ephemeral: false, - delete_old_database: true, - debug_stop_at_height: None, - debug_validity_check_interval: None, - // todo: does this matter? - should_backup_non_finalized_state: true, - debug_skip_non_finalized_state_backup_task: false, - }, - test_manager.full_node_rpc_listen_address.to_string(), - test_manager.full_node_grpc_listen_address, - false, - None, - None, - None, - ServiceConfig::default(), - StorageConfig { - cache: CacheConfig::default(), - database: DatabaseConfig { - path: test_manager - .data_dir - .as_path() - .to_path_buf() - .join("state-service-zaino"), - ..Default::default() + // FIXME: when the direct connection is integrated into chain index this initialization must change + let state_service = + NodeBackedIndexerService::spawn(NodeBackedIndexerServiceConfig::new_direct( + zebra_state::Config { + cache_dir: state_chain_cache_dir, + ephemeral: false, + delete_old_database: true, + debug_stop_at_height: None, + debug_validity_check_interval: None, + // todo: does this matter? + should_backup_non_finalized_state: true, + debug_skip_non_finalized_state_backup_task: false, }, - }, - false, - network.into(), - None, - )) - .await - .unwrap(); + test_manager.full_node_rpc_listen_address.to_string(), + test_manager.full_node_grpc_listen_address, + false, + None, + None, + None, + ServiceConfig::default(), + StorageConfig { + cache: CacheConfig::default(), + database: DatabaseConfig { + path: test_manager + .data_dir + .as_path() + .to_path_buf() + .join("state-service-zaino"), + ..Default::default() + }, + }, + false, + network.into(), + None, + )) + .await + .unwrap(); let config = ChainIndexConfig { storage: StorageConfig { database: DatabaseConfig { @@ -240,25 +235,23 @@ mod chain_query_interface { // #[ignore = "prone to timeouts and hangs, to be fixed in chain index integration"] #[tokio::test(flavor = "multi_thread")] async fn get_block_range_zebrad() { - get_block_range::(&ValidatorKind::Zebrad).await + get_block_range::(&ValidatorKind::Zebrad).await } #[cfg(feature = "zcashd_support")] #[ignore = "prone to timeouts and hangs, to be fixed in chain index integration"] #[tokio::test(flavor = "multi_thread")] async fn get_block_range_zcashd() { - get_block_range::(&ValidatorKind::Zcashd).await + get_block_range::(&ValidatorKind::Zcashd).await } - async fn get_block_range(validator: &ValidatorKind) + async fn get_block_range(validator: &ValidatorKind) where C: ValidatorExt, - Service: zaino_testutils::TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: zaino_testutils::PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { let (test_manager, _json_service, _option_state_service, _chain_index, indexer) = - create_test_manager_and_chain_index::(validator, None, false, false, false) + create_test_manager_and_chain_index::(validator, None, false, false, false) .await; test_manager @@ -280,7 +273,7 @@ mod chain_query_interface { #[tokio::test(flavor = "multi_thread")] async fn ephemeral_serves_finalised_blocks_zebrad() { - ephemeral_serves_finalised_blocks::(&ValidatorKind::Zebrad).await + ephemeral_serves_finalised_blocks::(&ValidatorKind::Zebrad).await } /// Ephemeral mode on regtest: the chain index opens no persistent @@ -296,17 +289,15 @@ mod chain_query_interface { /// its hash, and asserts the two are identical; /// - streams compact blocks across the finalised / non-finalised boundary; /// - asserts nothing was persisted to disk. - async fn ephemeral_serves_finalised_blocks(validator: &ValidatorKind) + async fn ephemeral_serves_finalised_blocks(validator: &ValidatorKind) where C: ValidatorExt, - Service: zaino_testutils::TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: zaino_testutils::PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { use zaino_proto::proto::utils::PoolTypeFilter; let (test_manager, json_service, _option_state_service, _chain_index, indexer) = - create_test_manager_and_chain_index::(validator, None, false, false, true) + create_test_manager_and_chain_index::(validator, None, false, false, true) .await; // Generate well past MAX_NFS_DEPTH (110) so low heights are evicted from @@ -375,25 +366,23 @@ mod chain_query_interface { #[ignore = "prone to timeouts and hangs, to be fixed in chain index integration"] #[tokio::test(flavor = "multi_thread")] async fn sync_large_chain_zebrad() { - sync_large_chain::(&ValidatorKind::Zebrad).await + sync_large_chain::(&ValidatorKind::Zebrad).await } #[cfg(feature = "zcashd_support")] #[ignore = "prone to timeouts and hangs, to be fixed in chain index integration"] #[tokio::test(flavor = "multi_thread")] async fn sync_large_chain_zcashd() { - sync_large_chain::(&ValidatorKind::Zcashd).await + sync_large_chain::(&ValidatorKind::Zcashd).await } - async fn sync_large_chain(validator: &ValidatorKind) + async fn sync_large_chain(validator: &ValidatorKind) where C: ValidatorExt, - Service: zaino_testutils::TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: zaino_testutils::PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { let (test_manager, json_service, option_state_service, _chain_index, indexer) = - create_test_manager_and_chain_index::(validator, None, false, false, false) + create_test_manager_and_chain_index::(validator, None, false, false, false) .await; test_manager @@ -451,25 +440,23 @@ mod chain_query_interface { // #[ignore = "prone to timeouts and hangs, to be fixed in chain index integration"] #[tokio::test(flavor = "multi_thread")] async fn get_subtree_roots_zebrad() { - get_subtree_roots::(&ValidatorKind::Zebrad).await + get_subtree_roots::(&ValidatorKind::Zebrad).await } #[cfg(feature = "zcashd_support")] #[ignore = "prone to timeouts and hangs, to be fixed in chain index integration"] #[tokio::test(flavor = "multi_thread")] async fn get_subtree_roots_zcashd() { - get_subtree_roots::(&ValidatorKind::Zcashd).await + get_subtree_roots::(&ValidatorKind::Zcashd).await } - async fn get_subtree_roots(validator: &ValidatorKind) + async fn get_subtree_roots(validator: &ValidatorKind) where C: ValidatorExt, - Service: zaino_testutils::TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: zaino_testutils::PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { let (test_manager, json_service, _option_state_service, _chain_index, indexer) = - create_test_manager_and_chain_index::(validator, None, false, false, false) + create_test_manager_and_chain_index::(validator, None, false, false, false) .await; test_manager @@ -556,30 +543,26 @@ mod chain_query_interface { // #[ignore = "prone to timeouts and hangs, to be fixed in chain index integration"] #[tokio::test(flavor = "multi_thread")] async fn get_mempool_stream_fresh_snapshot_repeated_zebrad() { - get_mempool_stream_fresh_snapshot_repeated::(&ValidatorKind::Zebrad) - .await + get_mempool_stream_fresh_snapshot_repeated::(&ValidatorKind::Zebrad).await } #[cfg(feature = "zcashd_support")] #[ignore = "prone to timeouts and hangs, to be fixed in chain index integration"] #[tokio::test(flavor = "multi_thread")] async fn get_mempool_stream_fresh_snapshot_repeated_zcashd() { - get_mempool_stream_fresh_snapshot_repeated::(&ValidatorKind::Zcashd) - .await + get_mempool_stream_fresh_snapshot_repeated::(&ValidatorKind::Zcashd).await } - async fn get_mempool_stream_fresh_snapshot_repeated(validator: &ValidatorKind) + async fn get_mempool_stream_fresh_snapshot_repeated(validator: &ValidatorKind) where C: ValidatorExt, - Service: zaino_testutils::TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: zaino_testutils::PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { use futures::StreamExt as _; use tokio::time::{timeout, Duration}; let (test_manager, _json_service, _option_state_service, _chain_index, indexer) = - create_test_manager_and_chain_index::(validator, None, false, false, false) + create_test_manager_and_chain_index::(validator, None, false, false, false) .await; test_manager @@ -615,28 +598,26 @@ mod chain_query_interface { // #[ignore = "prone to timeouts and hangs, to be fixed in chain index integration"] #[tokio::test(flavor = "multi_thread")] async fn zallet_like_steady_state_loop_zebrad() { - zallet_like_steady_state_loop::(&ValidatorKind::Zebrad).await + zallet_like_steady_state_loop::(&ValidatorKind::Zebrad).await } #[cfg(feature = "zcashd_support")] #[ignore = "prone to timeouts and hangs, to be fixed in chain index integration"] #[tokio::test(flavor = "multi_thread")] async fn zallet_like_steady_state_loop_zcashd() { - zallet_like_steady_state_loop::(&ValidatorKind::Zcashd).await + zallet_like_steady_state_loop::(&ValidatorKind::Zcashd).await } - async fn zallet_like_steady_state_loop(validator: &ValidatorKind) + async fn zallet_like_steady_state_loop(validator: &ValidatorKind) where C: ValidatorExt, - Service: zaino_testutils::TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: zaino_testutils::PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { use futures::{StreamExt as _, TryStreamExt as _}; use tokio::time::{timeout, Duration}; let (test_manager, _json_service, _option_state_service, _chain_index, indexer) = - create_test_manager_and_chain_index::(validator, None, false, false, false) + create_test_manager_and_chain_index::(validator, None, false, false, false) .await; test_manager diff --git a/live-tests/clientless/tests/fetch_service.rs b/live-tests/clientless/tests/fetch_service.rs index 53ea44785..f4ac94f37 100644 --- a/live-tests/clientless/tests/fetch_service.rs +++ b/live-tests/clientless/tests/fetch_service.rs @@ -5,11 +5,10 @@ use futures::StreamExt as _; use zaino_fetch::jsonrpsee::connector::JsonRpSeeConnector; use zaino_proto::proto::compact_formats::CompactBlock; use zaino_proto::proto::service::{BlockId, BlockRange, GetSubtreeRootsArg}; -#[allow(deprecated)] use zaino_state::{ - FetchService, FetchServiceSubscriber, LightWalletIndexer, Status, StatusType, ZcashIndexer, + LightWalletIndexer, NodeBackedIndexerServiceSubscriber, Status, StatusType, ZcashIndexer, }; -use zaino_testutils::{TestManager, ValidatorExt, ValidatorKind}; +use zaino_testutils::{Rpc, TestManager, ValidatorExt, ValidatorKind}; use zebra_chain::parameters::subsidy::ParameterSubsidy as _; use zebra_rpc::client::ValidateAddressResponse; use zebra_rpc::methods::{GetBlock, GetBlockHash}; @@ -95,7 +94,7 @@ async fn fetch_service_get_latest_block(validator: &ValidatorKi #[allow(deprecated)] async fn assert_subscriber_matches_rpc( validator: &ValidatorKind, - fetch_query: impl FnOnce(FetchServiceSubscriber) -> FFut, + fetch_query: impl FnOnce(NodeBackedIndexerServiceSubscriber) -> FFut, rpc_query: impl FnOnce(JsonRpSeeConnector) -> RFut, ) where V: ValidatorExt, @@ -321,7 +320,7 @@ async fn fetch_service_get_block_header(validator: &ValidatorKi #[allow(deprecated)] async fn launch_and_mine_five_blocks( validator: &ValidatorKind, -) -> (TestManager, FetchServiceSubscriber) { +) -> (TestManager, NodeBackedIndexerServiceSubscriber) { let (test_manager, fetch_service_subscriber) = zaino_testutils::launch_with_fetch_subscriber::(validator, None).await; diff --git a/live-tests/clientless/tests/json_server.rs b/live-tests/clientless/tests/json_server.rs index 199e0709f..72df376c5 100644 --- a/live-tests/clientless/tests/json_server.rs +++ b/live-tests/clientless/tests/json_server.rs @@ -5,9 +5,8 @@ //! docs/adr/0001-zcashd-support-feature-gate.md. #![cfg(feature = "zcashd_support")] -#[allow(deprecated)] -use zaino_state::{FetchService, FetchServiceSubscriber, ZcashIndexer}; -use zaino_testutils::TestManager; +use zaino_state::{NodeBackedIndexerServiceSubscriber, ZcashIndexer}; +use zaino_testutils::{Rpc, TestManager}; use zcash_local_net::validator::zcashd::Zcashd; /// Assert that `query` returns the same value from the zcashd-backed and @@ -16,11 +15,11 @@ use zcash_local_net::validator::zcashd::Zcashd; /// problem. The pure-compare building block of the json_server tests. #[allow(deprecated)] async fn assert_subscribers_agree( - zcashd_subscriber: &FetchServiceSubscriber, - zaino_subscriber: &FetchServiceSubscriber, + zcashd_subscriber: &NodeBackedIndexerServiceSubscriber, + zaino_subscriber: &NodeBackedIndexerServiceSubscriber, query: Q, ) where - Q: Fn(FetchServiceSubscriber) -> Fut, + Q: Fn(NodeBackedIndexerServiceSubscriber) -> Fut, Fut: std::future::Future, T: std::fmt::Debug + PartialEq, { @@ -34,13 +33,13 @@ async fn assert_subscribers_agree( /// tests that check an invariant holds across the chain. #[allow(deprecated)] async fn compare_over_blocks( - test_manager: &TestManager, - zcashd_subscriber: &FetchServiceSubscriber, - zaino_subscriber: &FetchServiceSubscriber, + test_manager: &TestManager, + zcashd_subscriber: &NodeBackedIndexerServiceSubscriber, + zaino_subscriber: &NodeBackedIndexerServiceSubscriber, blocks: u32, query: Q, ) where - Q: Fn(FetchServiceSubscriber) -> Fut, + Q: Fn(NodeBackedIndexerServiceSubscriber) -> Fut, Fut: std::future::Future, T: std::fmt::Debug + PartialEq, { diff --git a/live-tests/clientless/tests/state_service.rs b/live-tests/clientless/tests/state_service.rs index 5655dced3..9a4190a12 100644 --- a/live-tests/clientless/tests/state_service.rs +++ b/live-tests/clientless/tests/state_service.rs @@ -1,9 +1,6 @@ use zaino_fetch::jsonrpsee::response::address_deltas::GetAddressDeltasParams; -#[allow(deprecated)] -use zaino_state::{ - FetchServiceSubscriber, LightWalletIndexer, StateServiceSubscriber, ZcashIndexer, -}; +use zaino_state::{LightWalletIndexer, NodeBackedIndexerServiceSubscriber, ZcashIndexer}; use zaino_testutils::{StateAndFetchServices, ValidatorExt}; use zaino_testutils::{ValidatorKind, ZEBRAD_TESTNET_CACHE_DIR}; use zcash_local_net::validator::zebrad::Zebrad; @@ -42,8 +39,8 @@ async fn launch_testnet_cached() -> StateAndFetchServices { #[allow(deprecated)] async fn assert_subscribers_agree( services: &StateAndFetchServices, - fetch_query: impl FnOnce(FetchServiceSubscriber) -> FFut, - state_query: impl FnOnce(StateServiceSubscriber) -> SFut, + fetch_query: impl FnOnce(NodeBackedIndexerServiceSubscriber) -> FFut, + state_query: impl FnOnce(NodeBackedIndexerServiceSubscriber) -> SFut, ) -> T where T: std::fmt::Debug + PartialEq, diff --git a/live-tests/e2e/tests/devtool.rs b/live-tests/e2e/tests/devtool.rs index 6252148ab..0fa75c8e0 100644 --- a/live-tests/e2e/tests/devtool.rs +++ b/live-tests/e2e/tests/devtool.rs @@ -44,9 +44,8 @@ use e2e::devtool::DevtoolClients; use zaino_proto::proto::service::{ AddressList, BlockId, BlockRange, GetAddressUtxosArg, TransparentAddressBlockFilter, TxFilter, }; -use zaino_state::{LightWalletIndexer, ZcashIndexer, ZcashService}; -use zaino_testutils::{PollableTip, TestManager, TestService, ValidatorKind}; -use zainodlib::error::IndexerError; +use zaino_state::{LightWalletIndexer, ZcashIndexer}; +use zaino_testutils::{PollableTip, Rpc, TestManager, ValidatorKind}; use zcash_local_net::validator::zebrad::Zebrad; use zebra_chain::subtree::NoteCommitmentSubtreeIndex; use zebra_rpc::methods::{GetAddressBalanceRequest, GetAddressTxIdsRequest}; @@ -59,15 +58,13 @@ use zebra_rpc::methods::{GetAddressBalanceRequest, GetAddressTxIdsRequest}; /// spendable orchard notes the faucet ends with — one per send a test makes /// before mining (devtool will not chain unconfirmed change). The shared /// launch+fund preamble of the ports below. -async fn launch_and_fund_faucet( +async fn launch_and_fund_faucet( coinbase_blocks: u32, -) -> (TestManager, DevtoolClients) +) -> (TestManager, DevtoolClients) where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { - let test_manager = TestManager::::launch_mining_to( + let test_manager = TestManager::::launch_mining_to( zaino_testutils::SHIELDED_FUNDING_POOL, &ValidatorKind::Zebrad, None, @@ -99,13 +96,11 @@ where /// Launch an orchard-mining zebrad + Zaino on the `Service` backend and build /// devtool faucet/recipient wallets against it, without mining or syncing — the /// minimal preamble for tests that only exercise wallet↔server connectivity. -async fn launch_and_build_clients() -> (TestManager, DevtoolClients) +async fn launch_and_build_clients() -> (TestManager, DevtoolClients) where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { - let test_manager = TestManager::::launch_mining_to( + let test_manager = TestManager::::launch_mining_to( zaino_testutils::SHIELDED_FUNDING_POOL, &ValidatorKind::Zebrad, None, @@ -133,13 +128,11 @@ where /// and recipient wallets can report node/server info without erroring. Smoke /// test — the original discards the result. (`chain_name` is "test" for regtest /// and `server_uri` has no trailing slash, so this asserts neither.) -async fn connect_to_node_get_info() +async fn connect_to_node_get_info() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { - let (mut test_manager, clients) = launch_and_build_clients::().await; + let (mut test_manager, clients) = launch_and_build_clients::().await; clients.get_info_faucet().await; clients.get_info_recipient().await; @@ -149,13 +142,11 @@ where /// Port of `wallet_to_validator::check_received_mining_reward` (zebrad): the /// faucet's synced wallet sees the orchard coinbase note. -async fn receives_mining_reward() +async fn receives_mining_reward() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { - let (mut test_manager, clients) = launch_and_fund_faucet::(1).await; + let (mut test_manager, clients) = launch_and_fund_faucet::(1).await; let faucet_balance = dbg!(clients.faucet_balance().await); assert!( @@ -175,13 +166,11 @@ where /// `send_to_transparent` additionally mines across the finalization boundary /// under transparent mining; that variant is deferred, as it exercises the /// transparent-coinbase detection path devtool has not yet been run against.) -async fn send_to_pool(pool: e2e::Pool) +async fn send_to_pool(pool: e2e::Pool) where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { - let (mut test_manager, mut clients) = launch_and_fund_faucet::(1).await; + let (mut test_manager, mut clients) = launch_and_fund_faucet::(1).await; let recipient = clients.get_recipient_address(pool.address_kind()).await; let txid = clients.send_from_faucet(&recipient, 250_000).await; @@ -210,14 +199,12 @@ where /// balance assertions — the sends confirm in one block, and received funds are /// regular (non-coinbase) outputs with no maturity rule — and 100 orchard /// blocks would cost 100 halo2 proofs against the net-speedup criterion. -async fn send_to_all() +async fn send_to_all() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { // Three orchard notes — one per send (devtool will not chain unconfirmed change). - let (mut test_manager, mut clients) = launch_and_fund_faucet::(3).await; + let (mut test_manager, mut clients) = launch_and_fund_faucet::(3).await; let recipient_ua = clients.get_recipient_address("unified").await; let recipient_zaddr = clients.get_recipient_address("sapling").await; @@ -245,13 +232,11 @@ where /// orchard balance net of the ZIP-317 fee (250_000 − 15_000 = 235_000 — the /// first devtool exercise of `shield`, and the constant flagged as needing /// re-verification against devtool's note selection). -async fn shield_for_validator() +async fn shield_for_validator() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { - let (mut test_manager, mut clients) = launch_and_fund_faucet::(1).await; + let (mut test_manager, mut clients) = launch_and_fund_faucet::(1).await; let recipient_taddr = clients.get_recipient_address("transparent").await; clients.send_from_faucet(&recipient_taddr, 250_000).await; @@ -283,14 +268,12 @@ where /// mining) one transparent and one unified send into the mempool. Returns the /// manager and the two broadcast txids (transparent, then unified). The /// devtool analogue of `fund_and_fill_mempool`. -async fn fund_and_fill_mempool() -> (TestManager, String, String) +async fn fund_and_fill_mempool() -> (TestManager, String, String) where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { // Two orchard notes — one per unmined send. - let (test_manager, mut clients) = launch_and_fund_faucet::(2).await; + let (test_manager, mut clients) = launch_and_fund_faucet::(2).await; let recipient_taddr = clients.get_recipient_address("transparent").await; let recipient_ua = clients.get_recipient_address("unified").await; @@ -308,15 +291,13 @@ where /// txid hex. The txid is in display (reversed) order — devtool prints it that /// way, which matches the txid strings zaino's address queries return, so no /// conversion is needed for those comparisons. -async fn fund_and_send_to( +async fn fund_and_send_to( pool: e2e::Pool, -) -> (TestManager, DevtoolClients, String) +) -> (TestManager, DevtoolClients, String) where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { - let (test_manager, mut clients) = launch_and_fund_faucet::(1).await; + let (test_manager, mut clients) = launch_and_fund_faucet::(1).await; let recipient = clients.get_recipient_address(pool.address_kind()).await; let txid_hex = clients.send_from_faucet(&recipient, 250_000).await; @@ -331,14 +312,12 @@ where /// Port of `fetch_service_get_address_tx_ids` (zebrad): after a transparent /// send to the recipient, `get_address_tx_ids` over the recipient's taddr /// returns the send's txid. -async fn get_address_tx_ids() +async fn get_address_tx_ids() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip + LightWalletIndexer, + Conn: zaino_testutils::ValidatorConnectionMarker, { let (mut test_manager, clients, txid_hex) = - fund_and_send_to::(e2e::Pool::Transparent).await; + fund_and_send_to::(e2e::Pool::Transparent).await; let recipient_taddr = clients.get_recipient_address("transparent").await; // A range start a couple of blocks below the tip, covering the send block. @@ -362,14 +341,12 @@ where /// Port of `fetch_service_get_address_utxos` (zebrad): after a transparent /// send to the recipient, `z_get_address_utxos` over the recipient's taddr /// returns a utxo whose txid is the send's. -async fn get_address_utxos() +async fn get_address_utxos() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip + LightWalletIndexer, + Conn: zaino_testutils::ValidatorConnectionMarker, { let (mut test_manager, clients, txid_hex) = - fund_and_send_to::(e2e::Pool::Transparent).await; + fund_and_send_to::(e2e::Pool::Transparent).await; let recipient_taddr = clients.get_recipient_address("transparent").await; let utxos = test_manager @@ -387,14 +364,12 @@ where /// Port of `fetch_service_z_get_treestate` (zebrad, smoke): `z_get_treestate` /// at the tip height succeeds. -async fn z_get_treestate() +async fn z_get_treestate() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip + LightWalletIndexer, + Conn: zaino_testutils::ValidatorConnectionMarker, { let (mut test_manager, _clients, _txid_hex) = - fund_and_send_to::(e2e::Pool::Orchard).await; + fund_and_send_to::(e2e::Pool::Orchard).await; let tip = test_manager.subscriber().tip_height().await; dbg!(test_manager @@ -408,14 +383,12 @@ where /// Port of `fetch_service_z_get_subtrees_by_index` (zebrad, smoke): /// `z_get_subtrees_by_index` for orchard succeeds. -async fn z_get_subtrees_by_index() +async fn z_get_subtrees_by_index() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip + LightWalletIndexer, + Conn: zaino_testutils::ValidatorConnectionMarker, { let (mut test_manager, _clients, _txid_hex) = - fund_and_send_to::(e2e::Pool::Orchard).await; + fund_and_send_to::(e2e::Pool::Orchard).await; dbg!(test_manager .subscriber() @@ -428,14 +401,11 @@ where /// Port of `fetch_service_get_raw_transaction` (zebrad, smoke): /// `get_raw_transaction` for the orchard send's txid succeeds. -async fn get_raw_transaction() +async fn get_raw_transaction() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip + LightWalletIndexer, + Conn: zaino_testutils::ValidatorConnectionMarker, { - let (mut test_manager, _clients, txid_hex) = - fund_and_send_to::(e2e::Pool::Orchard).await; + let (mut test_manager, _clients, txid_hex) = fund_and_send_to::(e2e::Pool::Orchard).await; dbg!(test_manager .subscriber() @@ -449,16 +419,14 @@ where /// Port of `fetch_service_get_taddress_txids` (zebrad, smoke): /// `get_taddress_txids` over the recipient's taddr and a height range around /// the send succeeds. -async fn get_taddress_txids() +async fn get_taddress_txids() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip + LightWalletIndexer, + Conn: zaino_testutils::ValidatorConnectionMarker, { use futures::StreamExt as _; let (mut test_manager, clients, _txid_hex) = - fund_and_send_to::(e2e::Pool::Transparent).await; + fund_and_send_to::(e2e::Pool::Transparent).await; let recipient_taddr = clients.get_recipient_address("transparent").await; let tip = test_manager.subscriber().tip_height().await; @@ -492,14 +460,12 @@ where /// Port of `fetch_service_get_taddress_utxos` (zebrad, smoke): /// `get_address_utxos` over the recipient's taddr succeeds. -async fn get_taddress_utxos() +async fn get_taddress_utxos() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip + LightWalletIndexer, + Conn: zaino_testutils::ValidatorConnectionMarker, { let (mut test_manager, clients, _txid_hex) = - fund_and_send_to::(e2e::Pool::Transparent).await; + fund_and_send_to::(e2e::Pool::Transparent).await; let recipient_taddr = clients.get_recipient_address("transparent").await; let utxos_arg = GetAddressUtxosArg { @@ -518,16 +484,14 @@ where /// Port of `fetch_service_get_taddress_utxos_stream` (zebrad, smoke): /// `get_address_utxos_stream` over the recipient's taddr succeeds. -async fn get_taddress_utxos_stream() +async fn get_taddress_utxos_stream() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip + LightWalletIndexer, + Conn: zaino_testutils::ValidatorConnectionMarker, { use futures::StreamExt as _; let (mut test_manager, clients, _txid_hex) = - fund_and_send_to::(e2e::Pool::Transparent).await; + fund_and_send_to::(e2e::Pool::Transparent).await; let recipient_taddr = clients.get_recipient_address("transparent").await; let utxos_arg = GetAddressUtxosArg { @@ -551,14 +515,12 @@ where /// Port of `fetch_service_get_raw_mempool` (zebrad): with two transactions /// broadcast but unmined, the indexer's `get_raw_mempool` matches the /// validator's own JSON-RPC `getrawmempool`. -async fn get_raw_mempool() +async fn get_raw_mempool() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip + LightWalletIndexer, + Conn: zaino_testutils::ValidatorConnectionMarker, { let (mut test_manager, _transparent_txid, _unified_txid) = - fund_and_fill_mempool::().await; + fund_and_fill_mempool::().await; let json_service = test_manager.full_node_jsonrpc_connector().await; @@ -577,17 +539,14 @@ where /// Port of `fetch_service_get_mempool_tx` (zebrad): the `get_mempool_tx` /// stream returns the two unmined transactions keyed by txid (internal byte /// order), and the exclude-by-txid-suffix filter drops the named one. -async fn get_mempool_tx() +async fn get_mempool_tx() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip + LightWalletIndexer, + Conn: zaino_testutils::ValidatorConnectionMarker, { use futures::StreamExt as _; use zaino_proto::proto::service::GetMempoolTxRequest; - let (mut test_manager, transparent_txid, unified_txid) = - fund_and_fill_mempool::().await; + let (mut test_manager, transparent_txid, unified_txid) = fund_and_fill_mempool::().await; let to_bytes = |hex: &str| -> [u8; 32] { e2e::devtool::txid_internal_bytes(hex) @@ -638,16 +597,14 @@ where /// Port of `fetch_service_get_mempool_stream` (zebrad): a `get_mempool_stream` /// subscription opened before two sends observes those unmined transactions. /// Smoke test — the original only `dbg!`s the collected stream. -async fn get_mempool_stream() +async fn get_mempool_stream() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip + LightWalletIndexer, + Conn: zaino_testutils::ValidatorConnectionMarker, { use futures::StreamExt as _; // Two orchard notes — one per unmined send. - let (mut test_manager, mut clients) = launch_and_fund_faucet::(2).await; + let (mut test_manager, mut clients) = launch_and_fund_faucet::(2).await; // Subscribe before the sends so the stream observes them entering the mempool. let subscriber = test_manager.subscriber().clone(); @@ -687,13 +644,11 @@ where /// order. The devtool analogue of `fund_and_send(Pool::Orchard)`; the wallet /// clients are dropped once the transaction is mined (the queries below hit /// zaino, not the wallet). -async fn fund_and_send_orchard() -> (TestManager, Vec) +async fn fund_and_send_orchard() -> (TestManager, Vec) where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { - let (test_manager, mut clients) = launch_and_fund_faucet::(1).await; + let (test_manager, mut clients) = launch_and_fund_faucet::(1).await; let recipient_ua = clients.get_recipient_address("unified").await; let txid_hex = clients.send_from_faucet(&recipient_ua, 250_000).await; @@ -708,13 +663,11 @@ where /// Port of `fetch_service_get_transaction_mined` (zebrad): the indexer serves /// `get_transaction` for the mined orchard send, keyed by its txid. Also /// confirms `txid_internal_bytes` yields the order the indexer matches on. -async fn get_transaction_mined() +async fn get_transaction_mined() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip + LightWalletIndexer, + Conn: zaino_testutils::ValidatorConnectionMarker, { - let (mut test_manager, txid_bytes) = fund_and_send_orchard::().await; + let (mut test_manager, txid_bytes) = fund_and_send_orchard::().await; let tx_filter = TxFilter { block: None, @@ -734,13 +687,11 @@ where /// Port of `fetch_service_get_transaction_mempool` (zebrad): the indexer /// serves `get_transaction` for an orchard send that is broadcast but not /// mined, keyed by its txid — i.e. from the mempool. -async fn get_transaction_mempool() +async fn get_transaction_mempool() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip + LightWalletIndexer, + Conn: zaino_testutils::ValidatorConnectionMarker, { - let (mut test_manager, mut clients) = launch_and_fund_faucet::(1).await; + let (mut test_manager, mut clients) = launch_and_fund_faucet::(1).await; let recipient_ua = clients.get_recipient_address("unified").await; let txid_hex = clients.send_from_faucet(&recipient_ua, 250_000).await; @@ -944,14 +895,12 @@ async fn block_range_returns_all_pools() { /// Port of `fetch_service_get_address_balance` (zebrad): after a transparent /// send of 250_000 to the recipient, `z_get_address_balance` over that taddr /// reports exactly 250_000. -async fn get_address_balance() +async fn get_address_balance() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip + LightWalletIndexer, + Conn: zaino_testutils::ValidatorConnectionMarker, { let (mut test_manager, clients, _txid_hex) = - fund_and_send_to::(e2e::Pool::Transparent).await; + fund_and_send_to::(e2e::Pool::Transparent).await; let recipient_taddr = clients.get_recipient_address("transparent").await; let balance = test_manager @@ -970,14 +919,12 @@ where /// Port of `fetch_service_get_taddress_balance` (zebrad): after a transparent /// send of 250_000 to the recipient, `get_taddress_balance` over that taddr /// reports value_zat 250_000. -async fn get_taddress_balance() +async fn get_taddress_balance() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip + LightWalletIndexer, + Conn: zaino_testutils::ValidatorConnectionMarker, { let (mut test_manager, clients, _txid_hex) = - fund_and_send_to::(e2e::Pool::Transparent).await; + fund_and_send_to::(e2e::Pool::Transparent).await; let recipient_taddr = clients.get_recipient_address("transparent").await; let address_list = AddressList { @@ -1590,13 +1537,11 @@ async fn get_block_range_out_of_range_lower_bound() { /// per-call override, so the advance can't be cheap transparent filler until /// per-call filler mining (round-3 P2) lands; un-ignore and mine the advance to /// a transparent throwaway then. -async fn send_to_transparent_finalization() +async fn send_to_transparent_finalization() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { - let (mut test_manager, mut clients) = launch_and_fund_faucet::(1).await; + let (mut test_manager, mut clients) = launch_and_fund_faucet::(1).await; let recipient_taddr = clients.get_recipient_address("transparent").await; clients.send_from_faucet(&recipient_taddr, 250_000).await; @@ -2148,13 +2093,11 @@ async fn get_outpoint_spenders_fetch_vs_state() { /// sync is block-based so it never scans the mempool). Restore the assertions /// (using `Pool::spendable_balance` for the confirmed ones) and un-ignore when /// devtool surfaces unconfirmed balances. -async fn monitor_unverified_mempool() +async fn monitor_unverified_mempool() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { - let (mut test_manager, mut clients) = launch_and_fund_faucet::(2).await; + let (mut test_manager, mut clients) = launch_and_fund_faucet::(2).await; let recipient_ua = clients.get_recipient_address("unified").await; let txid_1 = clients.send_from_faucet(&recipient_ua, 250_000).await; @@ -2221,13 +2164,11 @@ where /// Port of `test_get_mempool_info` (fetch_service, zebrad): `get_mempool_info` /// matches values recomputed from the fetch subscriber's mempool internals. /// FetchService-only — the recompute reads `FetchServiceSubscriber.indexer`. -#[allow(deprecated)] // FetchService is a deprecated re-export. async fn get_mempool_info_fetch() { use hex::ToHex as _; use zaino_state::ChainIndex as _; - let (mut test_manager, _transparent_txid, _unified_txid) = - fund_and_fill_mempool::().await; + let (mut test_manager, _transparent_txid, _unified_txid) = fund_and_fill_mempool::().await; let subscriber = test_manager.subscriber(); let info = subscriber.get_mempool_info().await.unwrap(); @@ -2284,45 +2225,42 @@ async fn get_mempool_info_state() { } mod zebrad { - // FetchService is a deprecated re-export; the deprecation fires at the - // turbofish use sites below, so the allow covers the whole module. - #[allow(deprecated)] mod fetch_service { - use zaino_state::FetchService; + use zaino_testutils::Rpc; #[tokio::test(flavor = "multi_thread")] async fn receives_mining_reward() { - crate::receives_mining_reward::().await; + crate::receives_mining_reward::().await; } #[tokio::test(flavor = "multi_thread")] async fn connect_to_node_get_info() { - crate::connect_to_node_get_info::().await; + crate::connect_to_node_get_info::().await; } #[tokio::test(flavor = "multi_thread")] async fn send_to_orchard() { - crate::send_to_pool::(e2e::Pool::Orchard).await; + crate::send_to_pool::(e2e::Pool::Orchard).await; } #[tokio::test(flavor = "multi_thread")] async fn send_to_sapling() { - crate::send_to_pool::(e2e::Pool::Sapling).await; + crate::send_to_pool::(e2e::Pool::Sapling).await; } #[tokio::test(flavor = "multi_thread")] async fn send_to_transparent() { - crate::send_to_pool::(e2e::Pool::Transparent).await; + crate::send_to_pool::(e2e::Pool::Transparent).await; } #[tokio::test(flavor = "multi_thread")] async fn send_to_all() { - crate::send_to_all::().await; + crate::send_to_all::().await; } #[tokio::test(flavor = "multi_thread")] async fn shield_for_validator() { - crate::shield_for_validator::().await; + crate::shield_for_validator::().await; } #[tokio::test(flavor = "multi_thread")] @@ -2331,27 +2269,27 @@ mod zebrad { ignore = "heavy: 99-block orchard advance (~99 halo2 proofs); un-ignore + transparent filler when round-3 P2 lands" )] async fn send_to_transparent_finalization() { - crate::send_to_transparent_finalization::().await; + crate::send_to_transparent_finalization::().await; } #[tokio::test(flavor = "multi_thread")] async fn get_transaction_mined() { - crate::get_transaction_mined::().await; + crate::get_transaction_mined::().await; } #[tokio::test(flavor = "multi_thread")] async fn get_raw_mempool() { - crate::get_raw_mempool::().await; + crate::get_raw_mempool::().await; } #[tokio::test(flavor = "multi_thread")] async fn get_mempool_tx() { - crate::get_mempool_tx::().await; + crate::get_mempool_tx::().await; } #[tokio::test(flavor = "multi_thread")] async fn get_mempool_stream() { - crate::get_mempool_stream::().await; + crate::get_mempool_stream::().await; } #[tokio::test(flavor = "multi_thread")] @@ -2365,62 +2303,62 @@ mod zebrad { ignore = "devtool WalletBalance has no unconfirmed_*/confirmed_* fields; balance asserts are commented out — restore + un-ignore when devtool surfaces unconfirmed balances" )] async fn monitor_unverified_mempool() { - crate::monitor_unverified_mempool::().await; + crate::monitor_unverified_mempool::().await; } #[tokio::test(flavor = "multi_thread")] async fn get_address_tx_ids() { - crate::get_address_tx_ids::().await; + crate::get_address_tx_ids::().await; } #[tokio::test(flavor = "multi_thread")] async fn get_address_utxos() { - crate::get_address_utxos::().await; + crate::get_address_utxos::().await; } #[tokio::test(flavor = "multi_thread")] async fn z_get_treestate() { - crate::z_get_treestate::().await; + crate::z_get_treestate::().await; } #[tokio::test(flavor = "multi_thread")] async fn z_get_subtrees_by_index() { - crate::z_get_subtrees_by_index::().await; + crate::z_get_subtrees_by_index::().await; } #[tokio::test(flavor = "multi_thread")] async fn get_raw_transaction() { - crate::get_raw_transaction::().await; + crate::get_raw_transaction::().await; } #[tokio::test(flavor = "multi_thread")] async fn get_taddress_txids() { - crate::get_taddress_txids::().await; + crate::get_taddress_txids::().await; } #[tokio::test(flavor = "multi_thread")] async fn get_taddress_utxos() { - crate::get_taddress_utxos::().await; + crate::get_taddress_utxos::().await; } #[tokio::test(flavor = "multi_thread")] async fn get_taddress_utxos_stream() { - crate::get_taddress_utxos_stream::().await; + crate::get_taddress_utxos_stream::().await; } #[tokio::test(flavor = "multi_thread")] async fn get_transaction_mempool() { - crate::get_transaction_mempool::().await; + crate::get_transaction_mempool::().await; } #[tokio::test(flavor = "multi_thread")] async fn get_address_balance() { - crate::get_address_balance::().await; + crate::get_address_balance::().await; } #[tokio::test(flavor = "multi_thread")] async fn get_taddress_balance() { - crate::get_taddress_balance::().await; + crate::get_taddress_balance::().await; } } @@ -2540,42 +2478,41 @@ mod zebrad { } mod state_service { - #[allow(deprecated)] - use zaino_state::StateService; + use zaino_testutils::Direct; #[tokio::test(flavor = "multi_thread")] async fn receives_mining_reward() { - crate::receives_mining_reward::().await; + crate::receives_mining_reward::().await; } #[tokio::test(flavor = "multi_thread")] async fn connect_to_node_get_info() { - crate::connect_to_node_get_info::().await; + crate::connect_to_node_get_info::().await; } #[tokio::test(flavor = "multi_thread")] async fn send_to_orchard() { - crate::send_to_pool::(e2e::Pool::Orchard).await; + crate::send_to_pool::(e2e::Pool::Orchard).await; } #[tokio::test(flavor = "multi_thread")] async fn send_to_sapling() { - crate::send_to_pool::(e2e::Pool::Sapling).await; + crate::send_to_pool::(e2e::Pool::Sapling).await; } #[tokio::test(flavor = "multi_thread")] async fn send_to_transparent() { - crate::send_to_pool::(e2e::Pool::Transparent).await; + crate::send_to_pool::(e2e::Pool::Transparent).await; } #[tokio::test(flavor = "multi_thread")] async fn send_to_all() { - crate::send_to_all::().await; + crate::send_to_all::().await; } #[tokio::test(flavor = "multi_thread")] async fn shield_for_validator() { - crate::shield_for_validator::().await; + crate::shield_for_validator::().await; } #[tokio::test(flavor = "multi_thread")] @@ -2584,17 +2521,17 @@ mod zebrad { ignore = "heavy: 99-block orchard advance (~99 halo2 proofs); un-ignore + transparent filler when round-3 P2 lands" )] async fn send_to_transparent_finalization() { - crate::send_to_transparent_finalization::().await; + crate::send_to_transparent_finalization::().await; } #[tokio::test(flavor = "multi_thread")] async fn get_transaction_mined() { - crate::get_transaction_mined::().await; + crate::get_transaction_mined::().await; } #[tokio::test(flavor = "multi_thread")] async fn get_raw_mempool() { - crate::get_raw_mempool::().await; + crate::get_raw_mempool::().await; } #[tokio::test(flavor = "multi_thread")] @@ -2608,7 +2545,7 @@ mod zebrad { ignore = "devtool WalletBalance has no unconfirmed_*/confirmed_* fields; balance asserts are commented out — restore + un-ignore when devtool surfaces unconfirmed balances" )] async fn monitor_unverified_mempool() { - crate::monitor_unverified_mempool::().await; + crate::monitor_unverified_mempool::().await; } // No get_mempool_tx here: the state backend returns mempool-tx txids diff --git a/live-tests/e2e/tests/devtool_zcashd.rs b/live-tests/e2e/tests/devtool_zcashd.rs index bcc002bd1..cb2edbb6e 100644 --- a/live-tests/e2e/tests/devtool_zcashd.rs +++ b/live-tests/e2e/tests/devtool_zcashd.rs @@ -22,11 +22,9 @@ // launchers, all gated behind `zcashd_support`. Gate the whole binary so it // compiles out under `--no-default-features` (mirrors the clientless partition's json_server.rs). #![cfg(feature = "zcashd_support")] -#![allow(deprecated)] // FetchService is a deprecated re-export. - use e2e::devtool::DevtoolClients; -use zaino_state::{ChainIndex, FetchService, ZcashIndexer}; -use zaino_testutils::{TestManager, ValidatorKind, ZcashdDualFetchServices}; +use zaino_state::{ChainIndex, ZcashIndexer}; +use zaino_testutils::{Rpc, TestManager, ValidatorKind, ZcashdDualFetchServices}; use zcash_local_net::validator::zcashd::Zcashd; use zebra_chain::subtree::NoteCommitmentSubtreeIndex; use zebra_rpc::client::GetAddressBalanceRequest; @@ -38,8 +36,8 @@ use zebra_rpc::methods::GetAddressTxIdsRequest; /// faucet/recipient wallets against the resulting Zaino, without mining or /// syncing. The zcashd analogue of devtool.rs's `launch_and_build_clients`, /// concrete on zcashd (which has no StateService backend). -async fn launch_zcashd_and_build_clients() -> (TestManager, DevtoolClients) { - let test_manager = TestManager::::launch_mining_to( +async fn launch_zcashd_and_build_clients() -> (TestManager, DevtoolClients) { + let test_manager = TestManager::::launch_mining_to( zaino_testutils::SHIELDED_FUNDING_POOL, // ORCHARD &ValidatorKind::Zcashd, None, // network -> Regtest @@ -70,7 +68,7 @@ async fn launch_zcashd_and_build_clients() -> (TestManager /// height 2). The send/shield analogue of devtool.rs's `launch_and_fund_faucet`. async fn launch_and_fund_zcashd_faucet( orchard_notes: u32, -) -> (TestManager, DevtoolClients) { +) -> (TestManager, DevtoolClients) { let (test_manager, mut clients) = launch_zcashd_and_build_clients().await; test_manager .generate_blocks_and_wait_for_tip(orchard_notes + 1, test_manager.subscriber()) diff --git a/live-tests/e2e/tests/test_vectors.rs b/live-tests/e2e/tests/test_vectors.rs index 36128df39..11cfa336c 100644 --- a/live-tests/e2e/tests/test_vectors.rs +++ b/live-tests/e2e/tests/test_vectors.rs @@ -14,11 +14,9 @@ use zaino_state::read_u64_le; use zaino_state::write_u32_le; use zaino_state::write_u64_le; use zaino_state::CompactSize; -#[allow(deprecated)] -use zaino_state::StateService; use zaino_state::ZcashIndexer; use zaino_state::{ChainWork, IndexedBlock}; -use zaino_testutils::{TestManager, ValidatorKind}; +use zaino_testutils::{Direct, TestManager, ValidatorKind}; use zcash_local_net::validator::zebrad::Zebrad; use zebra_chain::serialization::{ZcashDeserialize, ZcashSerialize}; use zebra_rpc::methods::GetAddressUtxos; @@ -47,7 +45,7 @@ async fn create_200_block_regtest_chain_vectors() { // The committed unit-test vectors encode a mixed-pool chain built by // repeatedly shielding transparent coinbase — mining must stay transparent // so regenerated vectors keep that shape. - let mut test_manager = TestManager::::launch_mining_to( + let mut test_manager = TestManager::::launch_mining_to( zaino_testutils::PoolType::Transparent, &ValidatorKind::Zebrad, None, @@ -242,7 +240,7 @@ async fn create_200_block_regtest_chain_vectors() { .await .and_then(|response| match response { zebra_rpc::methods::GetBlock::Raw(_) => { - Err(zaino_state::StateServiceError::Custom( + Err(zaino_state::NodeBackedIndexerServiceError::Custom( "Found transaction of `Raw` type, expected only `Object` types." .to_string(), )) @@ -254,7 +252,7 @@ async fn create_200_block_regtest_chain_vectors() { match item { GetBlockTransaction::Hash(h) => Ok(h.0.to_vec()), GetBlockTransaction::Object(_) => Err( - zaino_state::StateServiceError::Custom( + zaino_state::NodeBackedIndexerServiceError::Custom( "Found transaction of `Object` type, expected only `Hash` types." .to_string(), ), @@ -273,7 +271,7 @@ async fn create_200_block_regtest_chain_vectors() { .await .and_then(|response| match response { zebra_rpc::methods::GetBlock::Object { .. } => { - Err(zaino_state::StateServiceError::Custom( + Err(zaino_state::NodeBackedIndexerServiceError::Custom( "Found transaction of `Object` type, expected only `Raw` types." .to_string(), )) diff --git a/live-tests/zaino-testutils/src/lib.rs b/live-tests/zaino-testutils/src/lib.rs index 6ebb6298f..44969008f 100644 --- a/live-tests/zaino-testutils/src/lib.rs +++ b/live-tests/zaino-testutils/src/lib.rs @@ -7,6 +7,7 @@ use futures::StreamExt as _; use once_cell::sync::Lazy; use std::{ future::Future, + marker::PhantomData, net::{IpAddr, Ipv4Addr, SocketAddr}, path::PathBuf, }; @@ -24,15 +25,12 @@ use zaino_fetch::jsonrpsee::connector::{test_node_and_return_url, JsonRpSeeConne use zaino_proto::proto::compact_formats::CompactBlock; use zaino_proto::proto::service::{BlockId, BlockRange}; use zaino_serve::server::config::{GrpcServerConfig, JsonRpcServerConfig}; -#[cfg(feature = "zcashd_support")] -use zaino_state::BackendType; use zaino_state::{ - BlockchainSource, ChainIndex, FetchServiceSubscriber, LightWalletIndexer, LightWalletService, - NodeBackedChainIndexSubscriber, StateServiceSubscriber, ZcashIndexer, ZcashService, + BlockchainSource, ChainIndex, LightWalletIndexer, NodeBackedChainIndexSubscriber, + NodeBackedIndexerService, NodeBackedIndexerServiceConfig, NodeBackedIndexerServiceSubscriber, + ZcashService, }; -#[allow(deprecated)] -use zaino_state::{FetchService, FetchServiceConfig, StateService, StateServiceConfig}; -use zainodlib::{config::ZainodConfig, error::IndexerError, indexer::Indexer}; +use zainodlib::{config::BackendType, error::IndexerError, indexer::Indexer}; pub use zcash_local_net as services; use zcash_local_net::error::LaunchError; #[cfg(feature = "zcashd_support")] @@ -221,20 +219,11 @@ pub trait PollableTip: Status + Sync { fn tip_height(&self) -> impl std::future::Future; } -impl PollableTip for FetchServiceSubscriber { +impl PollableTip for NodeBackedIndexerServiceSubscriber { async fn tip_height(&self) -> u64 { self.get_latest_block() .await - .expect("PollableTip: FetchServiceSubscriber::get_latest_block failed") - .height - } -} - -impl PollableTip for StateServiceSubscriber { - async fn tip_height(&self) -> u64 { - self.get_latest_block() - .await - .expect("PollableTip: StateServiceSubscriber::get_latest_block failed") + .expect("PollableTip: NodeBackedIndexerServiceSubscriber::get_latest_block failed") .height } } @@ -254,39 +243,32 @@ impl PollableTip for NodeBackedChainIndexSubscriber` -/// - `Service::Subscriber: PollableTip` -/// -/// **Not** bundled: the reverse bound -/// `IndexerError: From`. Rust does not -/// propagate non-`Self` bounds declared in a trait's `where`-clause -/// through a `T: TestService` constraint, so call sites that touch -/// `TestManager::launch` (or anything else exercising -/// `Indexer::launch_inner`'s `?` propagation) must still restate that -/// one bound explicitly. Everything else collapses to `TestService`. -pub trait TestService: - LightWalletService, Subscriber: PollableTip> - + Send - + Sync - + 'static -{ +/// Replaces the former `FetchService` / `StateService` `Service` type parameters: +/// there is now one service, parameterized by connection. Implementors are the +/// zero-sized markers [`Rpc`] and [`Direct`]. +pub trait ValidatorConnectionMarker: Send + Sync + 'static { + /// The on-disk backend selector this connection maps to. + const BACKEND: BackendType; } -impl TestService for T where - T: LightWalletService< - Config: TryFrom, - Subscriber: PollableTip, - > + Send - + Sync - + 'static -{ +/// JSON-RPC validator connection (formerly `FetchService`). +#[derive(Debug, Clone, Copy)] +pub struct Rpc; + +impl ValidatorConnectionMarker for Rpc { + const BACKEND: BackendType = BackendType::Rpc; +} + +/// Direct Zebra `ReadStateService` validator connection (formerly `StateService`). +#[derive(Debug, Clone, Copy)] +pub struct Direct; + +impl ValidatorConnectionMarker for Direct { + const BACKEND: BackendType = BackendType::Direct; } // temporary until activation heights are unified to zebra-chain type. @@ -399,7 +381,11 @@ pub enum ValidatorTestConfig { } /// Configuration data for Zaino Tests. -pub struct TestManager { +/// +/// Generic over the validator control plane `C` and the [`ValidatorConnectionMarker`] +/// `Conn` — the validator connection ([`Rpc`] or [`Direct`]) the single +/// [`NodeBackedIndexerService`] uses. +pub struct TestManager { /// Control plane for a validator pub local_net: C, /// Data directory for the validator. @@ -417,9 +403,11 @@ pub struct TestManager, /// Service subscriber. - pub service_subscriber: Option, + pub service_subscriber: Option, /// JsonRPC server cookie dir. pub json_server_cookie_dir: Option, + /// Selected validator connection marker. + _connection: PhantomData, } /// Needed validator functionality that is not implemented in infrastructure @@ -503,19 +491,17 @@ pub fn default_mining_pool(validator: &ValidatorKind) -> PoolType { } } -impl TestManager +impl TestManager where C: ValidatorExt, - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip, + Conn: ValidatorConnectionMarker, { /// Returns the service subscriber, panicking if zaino wasn't enabled at launch. /// /// Convenience for tests that always pass `enable_zaino: true` and want to /// hand the subscriber to [`Self::generate_blocks_and_wait_for_tip`] without /// repeating `.service_subscriber.as_ref().unwrap()` at every call site. - pub fn subscriber(&self) -> &Service::Subscriber { + pub fn subscriber(&self) -> &NodeBackedIndexerServiceSubscriber { self.service_subscriber .as_ref() .expect("TestManager::subscriber called but service_subscriber is None (zaino disabled at launch?)") @@ -599,9 +585,9 @@ where enable_clients: bool, ) -> Result { #[cfg(feature = "zcashd_support")] - if (validator == &ValidatorKind::Zcashd) && (Service::BACKEND_TYPE == BackendType::State) { + if (validator == &ValidatorKind::Zcashd) && (Conn::BACKEND == BackendType::Direct) { return Err(std::io::Error::other( - "Cannot use state backend with zcashd.", + "Cannot use the direct (ReadStateService) connection with zcashd.", )); } zaino_common::logging::try_init(); @@ -677,8 +663,7 @@ where "[TEST] Launching Zaino indexer" ); let indexer_config = zainodlib::config::ZainodConfig { - // TODO: Make configurable. - backend: Service::BACKEND_TYPE, + backend: Conn::BACKEND, json_server_settings: if enable_zaino_jsonrpc_server { Some(JsonRpcServerConfig { json_rpc_listen_address: zaino_json_listen_address, @@ -707,15 +692,16 @@ where metrics_endpoint: None, }; - let (handle, service_subscriber) = Indexer::::launch_inner_with_listeners( - Service::Config::try_from(indexer_config.clone()) - .expect("Failed to convert ZainodConfig to service config"), - indexer_config, - zaino_grpc_listener, - zaino_json_listener, - ) - .await - .unwrap(); + let (handle, service_subscriber) = + Indexer::::launch_inner_with_listeners( + NodeBackedIndexerServiceConfig::try_from(indexer_config.clone()) + .expect("Failed to convert ZainodConfig to service config"), + indexer_config, + zaino_grpc_listener, + zaino_json_listener, + ) + .await + .unwrap(); ( Some(handle), @@ -752,6 +738,7 @@ where zaino_grpc_listen_address, service_subscriber: zaino_service_subscriber, json_server_cookie_dir: zaino_json_server_cookie_dir, + _connection: PhantomData, }; test_manager.activate_nu5_nu6(enable_zaino).await; @@ -938,25 +925,27 @@ where } } -/// Handles from [`launch_state_and_fetch_services`]: a state-backend -/// [`TestManager`] plus standalone fetch and state services pointed at the -/// same validator. The owned services exist only to keep their subscribers -/// alive — tests interact through the manager and the subscribers. -#[allow(deprecated)] +/// Handles from [`launch_state_and_fetch_services`]: a direct-connection +/// [`TestManager`] plus standalone `Rpc`- and `Direct`-connection +/// [`NodeBackedIndexerService`]s pointed at the same validator. The owned services +/// exist only to keep their subscribers alive — tests interact through the manager and +/// the subscribers. +/// +/// The `fetch_*` fields carry the `Rpc` (JSON-RPC) connection; the `state_*` fields carry +/// the `Direct` (`ReadStateService`) connection. pub struct StateAndFetchServices { - /// The launched validator + Zaino test manager. - pub test_manager: TestManager, - /// Owned fetch service; keeps `fetch_subscriber` alive. - pub fetch_service: FetchService, - /// Subscriber to the standalone fetch service's chain index. - pub fetch_subscriber: FetchServiceSubscriber, - /// Owned state service; keeps `state_subscriber` alive. - pub state_service: StateService, - /// Subscriber to the standalone state service's chain index. - pub state_subscriber: StateServiceSubscriber, + /// The launched validator + Zaino test manager (direct connection). + pub test_manager: TestManager, + /// Owned `Rpc`-connection service; keeps `fetch_subscriber` alive. + pub fetch_service: NodeBackedIndexerService, + /// Subscriber to the standalone `Rpc`-connection service's chain index. + pub fetch_subscriber: NodeBackedIndexerServiceSubscriber, + /// Owned `Direct`-connection service; keeps `state_subscriber` alive. + pub state_service: NodeBackedIndexerService, + /// Subscriber to the standalone `Direct`-connection service's chain index. + pub state_subscriber: NodeBackedIndexerServiceSubscriber, } -#[allow(deprecated)] impl StateAndFetchServices { /// Mine `n` blocks and wait for both the fetch and state subscribers to /// observe the new tip. @@ -975,7 +964,6 @@ impl StateAndFetchServices { /// harness used by both the clientless and e2e live-test partitions. /// Wallet callers wrap this and additionally build lightclients from the /// returned manager's gRPC address. -#[allow(deprecated)] pub async fn launch_state_and_fetch_services( validator: &ValidatorKind, chain_cache: Option, @@ -996,7 +984,6 @@ pub async fn launch_state_and_fetch_services( /// caller instead of [`default_mining_pool`]: [`SHIELDED_FUNDING_POOL`] for /// sessions funding wallets from coinbase, or a pinned pool for tests whose /// subject is the miner's coinbase footprint. -#[allow(deprecated)] pub async fn launch_state_and_fetch_services_mining_to( mine_to_pool: PoolType, validator: &ValidatorKind, @@ -1004,7 +991,7 @@ pub async fn launch_state_and_fetch_services_mining_to( enable_zaino: bool, network: Option, ) -> StateAndFetchServices { - let test_manager = TestManager::::launch_mining_to( + let test_manager = TestManager::::launch_mining_to( mine_to_pool, validator, network, @@ -1067,41 +1054,42 @@ pub async fn launch_state_and_fetch_services_mining_to( None => test_manager.data_dir.clone(), }; - let state_service = StateService::spawn(StateServiceConfig::new( - zebra_state::Config { - cache_dir: state_chain_cache_dir, - ephemeral: false, - delete_old_database: true, - debug_stop_at_height: None, - debug_validity_check_interval: None, - should_backup_non_finalized_state: false, - debug_skip_non_finalized_state_backup_task: false, - }, - test_manager.full_node_rpc_listen_address.to_string(), - test_manager.full_node_grpc_listen_address, - false, - None, - None, - None, - ServiceConfig::default(), - StorageConfig { - database: DatabaseConfig { - path: test_manager - .local_net - .data_dir() - .path() - .to_path_buf() - .join("state-srvice-zaino"), + let state_service = + NodeBackedIndexerService::spawn(NodeBackedIndexerServiceConfig::new_direct( + zebra_state::Config { + cache_dir: state_chain_cache_dir, + ephemeral: false, + delete_old_database: true, + debug_stop_at_height: None, + debug_validity_check_interval: None, + should_backup_non_finalized_state: false, + debug_skip_non_finalized_state_backup_task: false, + }, + test_manager.full_node_rpc_listen_address.to_string(), + test_manager.full_node_grpc_listen_address, + false, + None, + None, + None, + ServiceConfig::default(), + StorageConfig { + database: DatabaseConfig { + path: test_manager + .local_net + .data_dir() + .path() + .to_path_buf() + .join("state-srvice-zaino"), + ..Default::default() + }, ..Default::default() }, - ..Default::default() - }, - false, // ephemeral_finalised_state: tests use a persistent finalised DB - network_type, - None, - )) - .await - .unwrap(); + false, // ephemeral_finalised_state: tests use a persistent finalised DB + network_type, + None, + )) + .await + .unwrap(); let state_subscriber = state_service.get_subscriber().inner(); @@ -1116,16 +1104,15 @@ pub async fn launch_state_and_fetch_services_mining_to( } } -/// Spawn a standalone [`FetchService`] pointed at `rpc_url`, storing its index -/// under `db_path`. Shared by the launch helpers below. -#[allow(deprecated)] +/// Spawn a standalone `Rpc`-connection [`NodeBackedIndexerService`] pointed at `rpc_url`, +/// storing its index under `db_path`. Shared by the launch helpers below. async fn spawn_fetch_service( rpc_url: String, cookie_dir: Option, db_path: PathBuf, network: Network, -) -> FetchService { - FetchService::spawn(FetchServiceConfig::new( +) -> NodeBackedIndexerService { + NodeBackedIndexerService::spawn(NodeBackedIndexerServiceConfig::new_rpc( rpc_url, cookie_dir, None, @@ -1152,18 +1139,17 @@ async fn spawn_fetch_service( /// server — for tests that compare the two. The owned services exist only to /// keep their subscribers alive. #[cfg(feature = "zcashd_support")] -#[allow(deprecated)] pub struct ZcashdDualFetchServices { /// The launched zcashd + Zaino test manager. - pub test_manager: TestManager, - /// Owned zcashd-direct fetch service; keeps `zcashd_subscriber` alive. - pub zcashd_fetch_service: FetchService, + pub test_manager: TestManager, + /// Owned zcashd-direct `Rpc` service; keeps `zcashd_subscriber` alive. + pub zcashd_fetch_service: NodeBackedIndexerService, /// Subscriber whose answers come from zcashd directly. - pub zcashd_subscriber: FetchServiceSubscriber, - /// Owned Zaino-pointed fetch service; keeps `zaino_subscriber` alive. - pub zaino_fetch_service: FetchService, + pub zcashd_subscriber: NodeBackedIndexerServiceSubscriber, + /// Owned Zaino-pointed `Rpc` service; keeps `zaino_subscriber` alive. + pub zaino_fetch_service: NodeBackedIndexerService, /// Subscriber whose answers come through Zaino's JSON-RPC server. - pub zaino_subscriber: FetchServiceSubscriber, + pub zaino_subscriber: NodeBackedIndexerServiceSubscriber, } #[cfg(feature = "zcashd_support")] @@ -1202,7 +1188,7 @@ pub async fn launch_zcashd_dual_fetch_services() -> ZcashdDualFetchServices { pub async fn launch_zcashd_dual_fetch_services_at( activation_heights: ActivationHeights, ) -> ZcashdDualFetchServices { - let test_manager = TestManager::::launch( + let test_manager = TestManager::::launch( &ValidatorKind::Zcashd, None, Some(activation_heights), @@ -1299,11 +1285,10 @@ pub fn get_info_with_zeroed_timestamp(info: GetInfo) -> GetInfo { /// harness in both live-test partitions (`clientless` and `e2e`). The `e2e` /// partition wraps this and builds lightclients from the returned manager's /// gRPC address. -#[allow(deprecated)] pub async fn launch_with_fetch_subscriber( validator: &ValidatorKind, chain_cache: Option, -) -> (TestManager, FetchServiceSubscriber) { +) -> (TestManager, NodeBackedIndexerServiceSubscriber) { launch_with_fetch_subscriber_mining_to::( default_mining_pool(validator), validator, @@ -1316,13 +1301,12 @@ pub async fn launch_with_fetch_subscriber( /// caller instead of [`default_mining_pool`]: [`SHIELDED_FUNDING_POOL`] for /// sessions funding wallets from coinbase, or a pinned pool for tests whose /// subject is the miner's coinbase footprint. -#[allow(deprecated)] pub async fn launch_with_fetch_subscriber_mining_to( mine_to_pool: PoolType, validator: &ValidatorKind, chain_cache: Option, -) -> (TestManager, FetchServiceSubscriber) { - let mut test_manager = TestManager::::launch_mining_to( +) -> (TestManager, NodeBackedIndexerServiceSubscriber) { + let mut test_manager = TestManager::::launch_mining_to( mine_to_pool, validator, None, @@ -1338,9 +1322,7 @@ pub async fn launch_with_fetch_subscriber_mining_to( (test_manager, fetch_service_subscriber) } -impl Drop - for TestManager -{ +impl Drop for TestManager { fn drop(&mut self) { debug!("[TEST] Shutting down test environment"); if let Some(handle) = &self.zaino_handle { @@ -1362,24 +1344,20 @@ async fn build_client( #[cfg(test)] mod launch_testmanager { use super::*; - #[allow(deprecated)] - use zaino_state::FetchService; /// Launch with no network/heights overrides and the optional servers off — /// the smoke tests' shared launch shape; only the chain cache and the /// zaino toggle vary. - async fn launch_minimal( + async fn launch_minimal( validator: &ValidatorKind, chain_cache: Option, enable_zaino: bool, - ) -> TestManager + ) -> TestManager where C: ValidatorExt, - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip, + Conn: ValidatorConnectionMarker, { - TestManager::::launch( + TestManager::::launch( validator, None, None, @@ -1402,7 +1380,7 @@ mod launch_testmanager { #[allow(deprecated)] pub(crate) async fn basic() { let mut test_manager = - launch_minimal::(&ValidatorKind::Zcashd, None, false).await; + launch_minimal::(&ValidatorKind::Zcashd, None, false).await; assert_eq!(2, (test_manager.local_net.get_chain_height().await)); test_manager.close().await; } @@ -1411,7 +1389,7 @@ mod launch_testmanager { #[allow(deprecated)] pub(crate) async fn generate_blocks() { let mut test_manager = - launch_minimal::(&ValidatorKind::Zcashd, None, false).await; + launch_minimal::(&ValidatorKind::Zcashd, None, false).await; assert_eq!(2, (test_manager.local_net.get_chain_height().await)); test_manager.local_net.generate_blocks(1).await.unwrap(); assert_eq!(3, (test_manager.local_net.get_chain_height().await)); @@ -1422,7 +1400,7 @@ mod launch_testmanager { #[tokio::test(flavor = "multi_thread")] #[allow(deprecated)] pub(crate) async fn with_chain() { - let mut test_manager = launch_minimal::( + let mut test_manager = launch_minimal::( &ValidatorKind::Zcashd, ZCASHD_CHAIN_CACHE_DIR.clone(), false, @@ -1436,7 +1414,7 @@ mod launch_testmanager { #[allow(deprecated)] pub(crate) async fn zaino() { let mut test_manager = - launch_minimal::(&ValidatorKind::Zcashd, None, true).await; + launch_minimal::(&ValidatorKind::Zcashd, None, true).await; let _grpc_client = build_client(test_manager.grpc_socket_to_uri()) .await @@ -1458,8 +1436,7 @@ mod launch_testmanager { #[allow(deprecated)] pub(crate) async fn basic() { let mut test_manager = - launch_minimal::(&ValidatorKind::Zebrad, None, false) - .await; + launch_minimal::(&ValidatorKind::Zebrad, None, false).await; assert_eq!(2, (test_manager.local_net.get_chain_height().await)); test_manager.close().await; } @@ -1468,8 +1445,7 @@ mod launch_testmanager { #[allow(deprecated)] pub(crate) async fn generate_blocks() { let mut test_manager = - launch_minimal::(&ValidatorKind::Zebrad, None, false) - .await; + launch_minimal::(&ValidatorKind::Zebrad, None, false).await; assert_eq!(2, (test_manager.local_net.get_chain_height().await)); test_manager.local_net.generate_blocks(1).await.unwrap(); assert_eq!(3, (test_manager.local_net.get_chain_height().await)); @@ -1480,7 +1456,7 @@ mod launch_testmanager { #[tokio::test(flavor = "multi_thread")] #[allow(deprecated)] pub(crate) async fn with_chain() { - let mut test_manager = launch_minimal::( + let mut test_manager = launch_minimal::( &ValidatorKind::Zebrad, ZEBRAD_CHAIN_CACHE_DIR.clone(), false, @@ -1494,8 +1470,7 @@ mod launch_testmanager { #[allow(deprecated)] pub(crate) async fn zaino() { let mut test_manager = - launch_minimal::(&ValidatorKind::Zebrad, None, true) - .await; + launch_minimal::(&ValidatorKind::Zebrad, None, true).await; let _grpc_client = build_client(test_manager.grpc_socket_to_uri()) .await .unwrap(); @@ -1505,15 +1480,12 @@ mod launch_testmanager { mod state_service { use super::*; - #[allow(deprecated)] - use zaino_state::StateService; #[tokio::test(flavor = "multi_thread")] #[allow(deprecated)] pub(crate) async fn basic() { let mut test_manager = - launch_minimal::(&ValidatorKind::Zebrad, None, false) - .await; + launch_minimal::(&ValidatorKind::Zebrad, None, false).await; assert_eq!(2, (test_manager.local_net.get_chain_height().await)); test_manager.close().await; } @@ -1522,8 +1494,7 @@ mod launch_testmanager { #[allow(deprecated)] pub(crate) async fn generate_blocks() { let mut test_manager = - launch_minimal::(&ValidatorKind::Zebrad, None, false) - .await; + launch_minimal::(&ValidatorKind::Zebrad, None, false).await; assert_eq!(2, (test_manager.local_net.get_chain_height().await)); test_manager.local_net.generate_blocks(1).await.unwrap(); assert_eq!(3, (test_manager.local_net.get_chain_height().await)); @@ -1534,7 +1505,7 @@ mod launch_testmanager { #[tokio::test(flavor = "multi_thread")] #[allow(deprecated)] pub(crate) async fn with_chain() { - let mut test_manager = launch_minimal::( + let mut test_manager = launch_minimal::( &ValidatorKind::Zebrad, ZEBRAD_CHAIN_CACHE_DIR.clone(), false, @@ -1548,8 +1519,7 @@ mod launch_testmanager { #[allow(deprecated)] pub(crate) async fn zaino() { let mut test_manager = - launch_minimal::(&ValidatorKind::Zebrad, None, true) - .await; + launch_minimal::(&ValidatorKind::Zebrad, None, true).await; let _grpc_client = build_client(test_manager.grpc_socket_to_uri()) .await .unwrap(); diff --git a/packages/zaino-state/src/backends.rs b/packages/zaino-state/src/backends.rs deleted file mode 100644 index 019cfc5f1..000000000 --- a/packages/zaino-state/src/backends.rs +++ /dev/null @@ -1,71 +0,0 @@ -//! Zaino's chain fetch and tx submission backend services. - -pub mod fetch; - -pub mod state; - -fn latest_network_upgrade( - upgrades: &indexmap::IndexMap< - zebra_rpc::methods::ConsensusBranchIdHex, - zebra_rpc::methods::NetworkUpgradeInfo, - >, -) -> Result<&zebra_rpc::methods::NetworkUpgradeInfo, tonic::Status> { - upgrades.last().map(|(_, upgrade)| upgrade).ok_or_else(|| { - tonic::Status::failed_precondition("validator returned no network upgrade metadata") - }) -} - -/// Maximum number of addresses a single `get_address_utxos` / `get_address_utxos_stream` -/// request may carry. -/// -/// Both backends resolve the full backend UTXO set before applying `max_entries` / -/// `start_height` (issue #974). A complete pushdown fix needs upstream interface changes -/// the caller-supplied entry cap cannot reach today, so until then this bounds the one -/// input the service controls locally: the address fan-out. It stops an unauthenticated -/// caller forcing an unbounded number of backend address lookups in a single request, and -/// is set well above realistic wallet usage. -/// -/// TODO: make this deployment-configurable rather than a fixed constant. -const UTXO_MAX_ADDRESSES: usize = 1000; - -/// Reject a `get_address_utxos` request whose address list exceeds [`UTXO_MAX_ADDRESSES`]. -/// -/// `max_entries` bounds the response size, not the backend work; this guard bounds the -/// address fan-out, the part the service can cap without upstream changes. -fn validate_utxo_address_count(count: usize) -> Result<(), tonic::Status> { - if count > UTXO_MAX_ADDRESSES { - return Err(tonic::Status::invalid_argument(format!( - "Error: too many addresses in request: {count} exceeds the maximum of {UTXO_MAX_ADDRESSES}." - ))); - } - Ok(()) -} - -#[cfg(test)] -mod tests { - #[test] - fn latest_network_upgrade_rejects_empty_metadata() { - let upgrades = indexmap::IndexMap::new(); - let err = super::latest_network_upgrade(&upgrades).expect_err("empty upgrades must fail"); - - assert_eq!(err.code(), tonic::Code::FailedPrecondition); - assert_eq!( - err.message(), - "validator returned no network upgrade metadata" - ); - } - - #[test] - fn utxo_address_count_within_limit_is_accepted() { - assert!(super::validate_utxo_address_count(0).is_ok()); - assert!(super::validate_utxo_address_count(super::UTXO_MAX_ADDRESSES).is_ok()); - } - - #[test] - fn utxo_address_count_over_limit_is_rejected() { - let err = super::validate_utxo_address_count(super::UTXO_MAX_ADDRESSES + 1) - .expect_err("over-limit address count must fail"); - - assert_eq!(err.code(), tonic::Code::InvalidArgument); - } -} diff --git a/packages/zaino-state/src/backends/state.rs b/packages/zaino-state/src/backends/state.rs deleted file mode 100644 index cc6f4b609..000000000 --- a/packages/zaino-state/src/backends/state.rs +++ /dev/null @@ -1,1896 +0,0 @@ -//! Zcash chain fetch and tx submission service backed by Zebras [`ReadStateService`]. - -#[allow(deprecated)] -use crate::{ - chain_index::{ - chain_tips_from_nonfinalized_snapshot, source::ValidatorConnector, types as chain_types, - ChainIndex, ChainIndexRpcExt, - }, - config::{DonationAddress, StateServiceConfig}, - error::{BlockCacheError, StateServiceError}, - indexer::{ - handle_raw_transaction, IndexerSubscriber, LightWalletIndexer, ZcashIndexer, ZcashService, - }, - status::{Status, StatusType}, - stream::{ - AddressStream, CompactBlockStream, CompactTransactionStream, RawTransactionStream, - UtxoReplyStream, - }, - utils::{get_build_info, ServiceMetadata}, - BackendType, NodeBackedChainIndex, NodeBackedChainIndexSubscriber, -}; -use crate::{ - chain_index::{types::BestChainLocation, NonFinalizedSnapshot}, - TransactionHash, -}; -use tokio_stream::StreamExt as _; -use zaino_fetch::{ - chain::{transaction::FullTransaction, utils::ParseFromSlice}, - jsonrpsee::{ - connector::RpcError, - response::{ - address_deltas::{GetAddressDeltasParams, GetAddressDeltasResponse}, - block_deltas::BlockDeltas, - block_header::GetBlockHeader, - block_subsidy::GetBlockSubsidy, - chain_tips::GetChainTipsResponse, - mining_info::GetMiningInfoWire, - peer_info::GetPeerInfo, - z_validate_address::ZValidateAddressResponse, - GetMempoolInfoResponse, GetNetworkSolPsResponse, GetSpentInfoRequest, - GetSpentInfoResponse, GetTxOutResponse, GetTxOutSetInfoResponse, - }, - }, -}; -use zaino_proto::proto::utils::{ - blockid_to_hashorheight, compact_block_to_nullifiers, GetBlockRangeError, PoolTypeError, - PoolTypeFilter, ValidatedBlockRangeRequest, -}; -use zaino_proto::proto::{ - compact_formats::CompactBlock, - service::{ - AddressList, Balance, BlockId, BlockRange, GetAddressUtxosArg, GetAddressUtxosReply, - GetAddressUtxosReplyList, GetMempoolTxRequest, LightdInfo, PingResponse, RawTransaction, - SendResponse, TransparentAddressBlockFilter, TreeState, TxFilter, - }, -}; -use zebra_chain::{ - block::Height, serialization::ZcashDeserialize as _, subtree::NoteCommitmentSubtreeIndex, -}; -use zebra_rpc::{ - client::{ - GetAddressBalanceRequest, GetSubtreesByIndexResponse, GetTreestateResponse, - TransactionObject, ValidateAddressResponse, - }, - methods::{ - AddressBalance, GetAddressTxIdsRequest, GetAddressUtxos, GetBlock, GetBlockHash, - GetBlockchainInfoResponse, GetInfo, GetRawTransaction, SentTransactionHash, - }, - server::error::LegacyCode, -}; -use zebra_state::HashOrHeight; - -use hex::{FromHex as _, ToHex}; -use std::str::FromStr; -use tokio::{ - sync::mpsc, - time::{self, timeout}, -}; -use tracing::{info, instrument, warn}; - -/// Chain fetch service backed by Zebra's `ReadStateService` and `TrustedChainSync`. -/// -/// NOTE: We currently dop not implement clone for chain fetch services -/// as this service is responsible for maintaining and closing its child processes. -/// ServiceSubscribers are used to create separate chain fetch processes -/// while allowing central state processes to be managed in a single place. -/// If we want the ability to clone Service all JoinHandle's should be -/// converted to Arc\. -#[derive(Debug)] -// #[deprecated = "Will be eventually replaced by `BlockchainSource"] -pub struct StateService { - /// Core indexer. - indexer: NodeBackedChainIndex, - - /// Service metadata. - data: ServiceMetadata, - - /// StateService config data. - #[allow(deprecated)] - config: StateServiceConfig, -} - -impl Status for StateService { - fn status(&self) -> StatusType { - self.indexer.status() - } -} - -// #[allow(deprecated)] -impl ZcashService for StateService { - const BACKEND_TYPE: BackendType = BackendType::State; - - type Subscriber = StateServiceSubscriber; - type Config = StateServiceConfig; - - /// Initializes a new StateService instance and starts sync process. - #[instrument(name = "StateService::spawn", skip(config), fields(network = %config.common.network))] - async fn spawn(config: StateServiceConfig) -> Result { - info!( - rpc_address = %config.common.validator_rpc_address, - network = %config.common.network, - "Spawning State Service" - ); - - let (source, zebra_build_data) = ValidatorConnector::spawn_state(&config) - .await - .map_err(|error| StateServiceError::Critical(error.to_string()))?; - - let data = ServiceMetadata::new( - get_build_info(config.common.indexer_version.clone()), - config.common.network.to_zebra_network(), - zebra_build_data.build, - zebra_build_data.subversion, - ); - info!(build = %data.zebra_build(), subversion = %data.zebra_subversion(), "Connected to Zcash node"); - - let indexer = NodeBackedChainIndex::new(source, config.clone().into()) - .await - .map_err(|error| StateServiceError::Critical(error.to_string()))?; - - let state_service = Self { - indexer, - data, - config, - }; - - // wait for sync to complete, return error on sync fail. - loop { - match state_service.status() { - StatusType::Ready | StatusType::Closing => break, - StatusType::CriticalError => { - return Err(StateServiceError::Critical( - "Chain index sync failed".to_string(), - )); - } - _ => { - tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; - } - } - } - - Ok(state_service) - } - - fn get_subscriber(&self) -> IndexerSubscriber { - IndexerSubscriber::new(StateServiceSubscriber { - indexer: self.indexer.subscriber(), - data: self.data.clone(), - config: self.config.clone(), - }) - } - - /// Shuts down the StateService. - /// - /// Delegates to the indexer, which cancels its sync loop, tears down the - /// finalised DB and mempool, and aborts the source-owned Zebra chain-syncer - /// task via [`crate::chain_index::source::BlockchainSource::shutdown`]. - fn close(&mut self) { - tokio::task::block_in_place(|| { - tokio::runtime::Handle::current().block_on(async { - let _ = self.indexer.shutdown().await; - }); - }); - } -} - -#[allow(deprecated)] -impl Drop for StateService { - fn drop(&mut self) { - self.close() - } -} - -/// A fetch service subscriber. -/// -/// Subscribers should be -#[derive(Debug, Clone)] -// #[deprecated] -pub struct StateServiceSubscriber { - /// Core indexer. - pub indexer: NodeBackedChainIndexSubscriber, - - /// Service metadata. - pub data: ServiceMetadata, - - /// StateService config data. - #[allow(deprecated)] - config: StateServiceConfig, -} - -impl StateServiceSubscriber { - /// The backing Zebra [`ReadStateService`]. - /// - /// Test-only escape hatch: live tests recompute expected chain data (e.g. - /// treestate roots) directly off the `ReadStateService`. Production code goes - /// through the `ChainIndex` API. - #[cfg(feature = "test_dependencies")] - pub fn read_state_service(&self) -> zebra_state::ReadStateService { - self.indexer - .source() - .read_state_service() - .expect("StateServiceSubscriber is always State-backed") - .clone() - } - - /// The indexer's mempool subscriber. - /// - /// Test-only escape hatch: live tests recompute expected `getmempoolinfo` - /// values directly off the mempool's entries. Production code goes through the - /// `ChainIndex` mempool API. - #[cfg(feature = "test_dependencies")] - pub fn mempool(&self) -> &crate::chain_index::mempool::MempoolSubscriber { - self.indexer.mempool_subscriber() - } -} - -impl Status for StateServiceSubscriber { - fn status(&self) -> StatusType { - self.indexer.status() - } -} - -/// A subscriber to any chaintip updates -#[derive(Clone)] -pub struct ChainTipSubscriber { - monitor: zebra_state::ChainTipChange, -} - -impl ChainTipSubscriber { - /// Waits until the tip hash has changed (relative to the last time this method - /// was called), then returns the best tip's block hash. - pub async fn next_tip_hash( - &mut self, - ) -> Result { - self.monitor - .wait_for_tip_change() - .await - .map(|tip| tip.best_tip_hash()) - } -} - -/// Private RPC methods, which are used as helper methods by the public ones -/// -/// These would be simple to add to the public interface if -/// needed, there are currently no plans to do so. -// #[allow(deprecated)] -impl StateServiceSubscriber { - /// Gets a Subscriber to any updates to the latest chain tip - pub fn chaintip_update_subscriber(&self) -> ChainTipSubscriber { - ChainTipSubscriber { - monitor: self - .indexer - .source() - .chain_tip_change() - .expect("StateServiceSubscriber is always State-backed"), - } - } - /// Return a list of consecutive compact blocks. - #[allow(dead_code, deprecated)] - async fn get_block_range_inner( - &self, - request: BlockRange, - nullifiers_only: bool, - ) -> Result { - let validated_request = ValidatedBlockRangeRequest::new_from_block_range(&request) - .map_err(StateServiceError::from)?; - - let pool_type_filter = PoolTypeFilter::new_from_pool_types(&validated_request.pool_types()) - .map_err(GetBlockRangeError::PoolTypeArgumentError) - .map_err(StateServiceError::from)?; - - // Note conversion here is safe due to the use of [`ValidatedBlockRangeRequest::new_from_block_range`] - let start = validated_request.start() as u32; - let end = validated_request.end() as u32; - - let state_service_clone = self.clone(); - let service_timeout = self.config.common.service.timeout; - let (channel_tx, channel_rx) = - mpsc::channel(self.config.common.service.channel_size as usize); - let snapshot = state_service_clone - .indexer - .snapshot_nonfinalized_state() - .await?; - - tokio::spawn(async move { - let timeout_result = timeout( - time::Duration::from_secs((service_timeout * 4) as u64), - async { - // This method does not support passthrough. Just return. - let Some(non_finalized_snapshot) = snapshot.get_nfs_snapshot() else {return}; - let chain_height = non_finalized_snapshot.best_tip.height.0; - - match state_service_clone - .indexer - .get_compact_block_stream( - &snapshot, - chain_types::Height(start), - chain_types::Height(end), - pool_type_filter.clone(), - ) - .await - { - Ok(Some(mut compact_block_stream)) => { - if nullifiers_only { - while let Some(stream_item) = compact_block_stream.next().await { - match stream_item { - Ok(block) => { - if channel_tx - .send(Ok(compact_block_to_nullifiers(block))) - .await - .is_err() - { - break; - } - } - Err(status) => { - if channel_tx.send(Err(status)).await.is_err() { - break; - } - } - } - } - } else { - while let Some(stream_item) = compact_block_stream.next().await { - if channel_tx.send(stream_item).await.is_err() { - break; - } - } - } - } - Ok(None) => { - // Per `get_compact_block_stream` semantics: `None` means at least one bound is above the tip. - let offending_height = if start > chain_height { start } else { end }; - - match channel_tx - .send(Err(tonic::Status::out_of_range(format!( - "Error: Height out of range [{offending_height}]. Height requested is greater than the best chain tip [{chain_height}].", - )))) - .await - { - Ok(_) => {} - Err(e) => { - warn!(%e, "GetBlockRange channel closed unexpectedly"); - } - } - } - Err(e) => { - // Preserve previous behaviour: if the request is above tip, surface OutOfRange; - // otherwise return the error (currently exposed for dev). - if start > chain_height || end > chain_height { - let offending_height = if start > chain_height { start } else { end }; - - match channel_tx - .send(Err(tonic::Status::out_of_range(format!( - "Error: Height out of range [{offending_height}]. Height requested is greater than the best chain tip [{chain_height}].", - )))) - .await - { - Ok(_) => {} - Err(e) => { - warn!(%e, "GetBlockRange channel closed unexpectedly"); - } - } - } else { - // TODO: Hide server error from clients before release. Currently useful for dev purposes. - if channel_tx - .send(Err(tonic::Status::unknown(e.to_string()))) - .await - .is_err() - { - warn!(%e, "GetBlockRangeStream closed unexpectedly"); - } - } - } - } - }, - ) - .await; - - if timeout_result.is_err() { - channel_tx - .send(Err(tonic::Status::deadline_exceeded( - "Error: get_block_range gRPC request timed out.", - ))) - .await - .ok(); - } - }); - - Ok(CompactBlockStream::new(channel_rx)) - } - - async fn error_get_block( - &self, - e: BlockCacheError, - height: u32, - ) -> Result { - let snapshot = self.indexer.snapshot_nonfinalized_state().await?; - let chain_height = snapshot.max_serviceable_height().0; - Err(if height >= chain_height { - StateServiceError::TonicStatusError(tonic::Status::out_of_range(format!( - "Error: Height out of range [{height}]. Height requested \ - is greater than Zaino's best chain tip [{chain_height}].", - ))) - } else { - // TODO: Hide server error from clients before release. - // Currently useful for dev purposes. - StateServiceError::TonicStatusError(tonic::Status::unknown(format!( - "Error: Failed to retrieve block from node. Server Error: {e}", - ))) - }) - } - - /// Returns the network type running. - #[allow(deprecated)] - pub fn network(&self) -> zaino_common::Network { - self.config.common.network - } -} - -// #[allow(deprecated)] -impl ZcashIndexer for StateServiceSubscriber { - type Error = StateServiceError; - - async fn get_info(&self) -> Result { - Ok(self.indexer.get_info().await?) - } - - /// Returns all changes for an address. - /// - /// Returns information about all changes to the given transparent addresses within the given (inclusive) - /// - /// block height range, default is the full blockchain. - /// If start or end are not specified, they default to zero. - /// If start is greater than the latest block height, it's interpreted as that height. - /// - /// If end is zero, it's interpreted as the latest block height. - /// - /// [Original zcashd implementation](https://github.com/zcash/zcash/blob/18238d90cd0b810f5b07d5aaa1338126aa128c06/src/rpc/misc.cpp#L881) - /// - /// zcashd reference: [`getaddressdeltas`](https://zcash.github.io/rpc/getaddressdeltas.html) - /// method: post - /// tags: address - async fn get_address_deltas( - &self, - params: GetAddressDeltasParams, - ) -> Result { - Ok(self.indexer.get_address_deltas(params).await?) - } - - async fn get_difficulty(&self) -> Result { - Ok(self.indexer.get_difficulty().await?) - } - - async fn get_block_subsidy(&self, height: u32) -> Result { - Ok(self.indexer.get_block_subsidy(height).await?) - } - - async fn get_blockchain_info(&self) -> Result { - Ok(self.indexer.get_blockchain_info().await?) - } - - /// Returns details on the active state of the TX memory pool. - /// In Zaino, this RPC call information is gathered from the local Zaino state instead of directly reflecting the full node's mempool. This state is populated from a gRPC stream, sourced from the full node. - /// There are no request parameters. - /// The Zcash source code is considered canonical: - /// [from the rpc definition](), [this function is called to produce the return value](>). - /// There are no required or optional parameters. - /// the `size` field is called by [this line of code](), and returns an int64. - /// `size` represents the number of transactions currently in the mempool. - /// the `bytes` field is called by [this line of code](), and returns an int64 from [this variable](). - /// `bytes` is the sum memory size in bytes of all transactions in the mempool: the sum of all transaction byte sizes. - /// the `usage` field is called by [this line of code](), and returns an int64 derived from the return of this function(), which includes a number of elements. - /// `usage` is the total memory usage for the mempool, in bytes. - /// the [optional `fullyNotified` field](), is only utilized for zcashd regtests, is deprecated, and is not included. - async fn get_mempool_info(&self) -> Result { - Ok(self.indexer.get_mempool_info().await.into()) - } - - async fn get_peer_info(&self) -> Result { - Ok(self.indexer.get_peer_info().await?) - } - - async fn z_get_address_balance( - &self, - address_strings: GetAddressBalanceRequest, - ) -> Result { - Ok(self.indexer.get_address_balance(address_strings).await?) - } - - async fn send_raw_transaction( - &self, - raw_transaction_hex: String, - ) -> Result { - Ok(self - .indexer - .send_raw_transaction(raw_transaction_hex) - .await?) - } - - async fn get_block_header( - &self, - hash: String, - verbose: bool, - ) -> Result { - Ok(self.indexer.get_block_header(hash, verbose).await?) - } - - async fn z_get_block( - &self, - hash_or_height_string: String, - verbosity: Option, - ) -> Result { - Ok(self - .indexer - .z_get_block(hash_or_height_string, verbosity) - .await?) - } - - async fn get_block_deltas(&self, hash: String) -> Result { - Ok(self.indexer.get_block_deltas(hash).await?) - } - - async fn get_raw_mempool(&self) -> Result, Self::Error> { - Ok(self - .indexer - .get_mempool_txids() - .await? - .into_iter() - .map(|txid| txid.to_rpc_hex()) - .collect()) - } - - /// NOTE: This method currently has to fetch data from 2 places (get_treestate and get_indexed_block_by_*), - /// If `ValidatorConnector::GetTreeState` was updated to return the additional information - /// required, this second call could be removed, improving the performance of this method. - // Pre-existing lint: `StateServiceError` is a large error type; returning it by value here is - // flagged by `result_large_err`. Suppressed to satisfy `-D warnings` without an invasive - // boxing refactor of the shared error enum. - #[allow(clippy::result_large_err)] - async fn z_get_treestate( - &self, - hash_or_height: String, - ) -> Result { - let fallback_hash_or_height = hash_or_height.clone(); - let local_result: Result = async { - let hash_or_height_struct: HashOrHeight = HashOrHeight::from_str(&hash_or_height)?; - let snapshot = self.indexer.snapshot_nonfinalized_state().await?; - - let block_data = match hash_or_height_struct { - HashOrHeight::Hash(hash) => self - .indexer - .get_indexed_block_by_hash(&snapshot, &hash.into()) - .await? - .ok_or(StateServiceError::RpcError(RpcError::new_from_legacycode( - zebra_rpc::server::error::LegacyCode::InvalidParameter, - "Failed to fetch block data.", - )))?, - HashOrHeight::Height(height) => self - .indexer - .get_indexed_block_by_height(&snapshot, &height.into()) - .await? - .ok_or(StateServiceError::RpcError(RpcError::new_from_legacycode( - zebra_rpc::server::error::LegacyCode::InvalidParameter, - "Failed to fetch block data.", - )))?, - }; - - let (sapling, orchard) = self.indexer.get_treestate(block_data.hash()).await?; - let time: u32 = block_data.data().time().try_into().map_err(|_error| { - StateServiceError::RpcError(RpcError::new_from_legacycode( - zebra_rpc::server::error::LegacyCode::InvalidParameter, - "Block time is out of range for u32.", - )) - })?; - - #[allow(deprecated)] - Ok(GetTreestateResponse::from_parts( - (*block_data.hash()).into(), - block_data.height().into(), - time, - sapling, - orchard, - )) - } - .await; - - if let Ok(response) = local_result { - return Ok(response); - } - - let snapshot = self.indexer.snapshot_nonfinalized_state().await?; - if !self - .indexer - .hash_or_height_known_for_treestate(&snapshot, &fallback_hash_or_height) - .await? - { - return local_result; - } - - Ok(self - .indexer - .get_treestate_by_id(fallback_hash_or_height) - .await?) - } - - async fn get_mining_info(&self) -> Result { - Ok(self.indexer.get_mining_info().await?) - } - - /// Returns statistics about the unspent transaction output set. - /// - /// zcashd reference: [`gettxoutsetinfo`](https://zcash.github.io/rpc/gettxoutsetinfo.html) - /// method: post - /// tags: blockchain - async fn get_tx_out_set_info(&self) -> Result { - Ok(self.indexer.get_tx_out_set_info().await?) - } - - // No request parameters. - /// Return the hex encoded hash of the best (tip) block, in the longest block chain. - /// The Zcash source code is considered canonical: - /// [In the rpc definition](https://github.com/zcash/zcash/blob/654a8be2274aa98144c80c1ac459400eaf0eacbe/src/rpc/common.h#L48) there are no required params, or optional params. - /// [The function in rpc/blockchain.cpp](https://github.com/zcash/zcash/blob/654a8be2274aa98144c80c1ac459400eaf0eacbe/src/rpc/blockchain.cpp#L325) - /// where `return chainActive.Tip()->GetBlockHash().GetHex();` is the [return expression](https://github.com/zcash/zcash/blob/654a8be2274aa98144c80c1ac459400eaf0eacbe/src/rpc/blockchain.cpp#L339)returning a `std::string` - async fn get_best_blockhash(&self) -> Result { - let snapshot = self.indexer.snapshot_nonfinalized_state().await?; - let tip = self.indexer.best_chaintip(&snapshot).await?; - Ok(GetBlockHash::new(tip.hash.into())) - } - - /// Returns the current block count in the best valid block chain. - /// - /// zcashd reference: [`getblockcount`](https://zcash.github.io/rpc/getblockcount.html) - /// method: post - /// tags: blockchain - async fn get_block_count(&self) -> Result { - let snapshot = self.indexer.snapshot_nonfinalized_state().await?; - let tip = self.indexer.best_chaintip(&snapshot).await?; - Ok(tip.height.into()) - } - - async fn get_chain_tips(&self) -> Result { - let snapshot = self.indexer.snapshot_nonfinalized_state().await?; - let Some(non_finalized_snapshot) = snapshot.get_nfs_snapshot() else { - return Err(StateServiceError::UnavailableNotSyncedEnough); - }; - Ok(chain_tips_from_nonfinalized_snapshot( - non_finalized_snapshot, - )) - } - - async fn validate_address( - &self, - raw_address: String, - ) -> Result { - let network = self.config.common.network.to_zebra_network(); - Ok(crate::indexer::validate_address(raw_address, &network)) - } - - #[allow(deprecated)] - async fn z_validate_address( - &self, - address: String, - ) -> Result { - let network = self.config.common.network.to_zebra_network(); - Ok(crate::indexer::z_validate_address(address, &network)) - } - - async fn z_get_subtrees_by_index( - &self, - pool: String, - start_index: NoteCommitmentSubtreeIndex, - limit: Option, - ) -> Result { - let shielded_pool = match pool.as_str() { - "sapling" => crate::chain_index::ShieldedPool::Sapling, - "orchard" => crate::chain_index::ShieldedPool::Orchard, - otherwise => { - return Err(StateServiceError::RpcError(RpcError::new_from_legacycode( - LegacyCode::Misc, - format!( - "invalid pool name \"{otherwise}\", must be \"sapling\" or \"orchard\"" - ), - ))) - } - }; - let roots = self - .indexer - .get_subtree_roots(shielded_pool, start_index.0, limit.map(|index| index.0)) - .await?; - Ok(crate::indexer::build_subtrees_by_index_response( - pool, - start_index, - roots, - )) - } - - async fn get_raw_transaction( - &self, - txid_hex: String, - verbose: Option, - ) -> Result { - #[allow(deprecated)] - let txid = TransactionHash::from_hex(&txid_hex).map_err(|error| { - StateServiceError::RpcError(RpcError::new_from_legacycode( - zebra_rpc::server::error::LegacyCode::InvalidAddressOrKey, - error.to_string(), - )) - })?; - - #[allow(deprecated)] - let not_found_error = || { - StateServiceError::RpcError(RpcError::new_from_legacycode( - zebra_rpc::server::error::LegacyCode::InvalidAddressOrKey, - "No such mempool or main chain transaction", - )) - }; - - let snapshot = self.indexer.snapshot_nonfinalized_state().await?; - - let Some((serialized_transaction, _consensus_branch_id)) = - self.indexer.get_raw_transaction(&snapshot, &txid).await? - else { - return Err(not_found_error()); - }; - - if verbose.is_none() { - return Ok(GetRawTransaction::Raw( - zebra_chain::transaction::SerializedTransaction::from(serialized_transaction), - )); - } - - let transaction = zebra_chain::transaction::Transaction::zcash_deserialize( - serialized_transaction.as_slice(), - ) - .map_err(|_| not_found_error())?; - - let (best_chain_location, _non_best_chain_locations) = self - .indexer - .get_transaction_status(&snapshot, &txid) - .await?; - - let (height, confirmations, block_hash, in_best_chain) = match best_chain_location { - Some(BestChainLocation::Block(block_hash, height)) => { - let confirmations = snapshot - .max_serviceable_height() - .0 - .saturating_sub(height.0) - .saturating_add(1); - - ( - Some(zebra_chain::block::Height::from(height)), - Some(confirmations), - Some(zebra_chain::block::Hash::from(block_hash)), - Some(true), - ) - } - Some(BestChainLocation::Mempool(_height)) => (None, Some(0), None, Some(false)), - None => (None, None, None, Some(false)), - }; - - Ok(GetRawTransaction::Object(Box::new( - TransactionObject::from_transaction( - transaction.into(), - height, - confirmations, - #[allow(deprecated)] - &self.config.common.network.to_zebra_network(), - None, - block_hash, - in_best_chain, - zebra_chain::transaction::Hash::from(txid), - ), - ))) - } - - /// Returns details about an unspent transaction output. - /// - /// zcashd reference: [`gettxout`](https://zcash.github.io/rpc/gettxout.html) - /// method: post - /// tags: transaction - async fn get_tx_out( - &self, - txid: String, - n: u32, - include_mempool: Option, - ) -> Result { - Ok(self.indexer.get_tx_out(txid, n, include_mempool).await?) - } - - async fn get_spent_info( - &self, - request: GetSpentInfoRequest, - ) -> Result { - Ok(self.indexer.get_spent_info(request).await?) - } - - async fn get_address_tx_ids( - &self, - request: GetAddressTxIdsRequest, - ) -> Result, Self::Error> { - Ok(self - .indexer - .get_address_txids(request) - .await? - .into_iter() - .map(|transaction_hash| transaction_hash.to_rpc_hex()) - .collect()) - } - - async fn z_get_address_utxos( - &self, - address_strings: GetAddressBalanceRequest, - ) -> Result, Self::Error> { - Ok(self.indexer.get_address_utxos(address_strings).await?) - } - - /// Returns the estimated network solutions per second based on the last n blocks. - /// - /// zcashd reference: [`getnetworksolps`](https://zcash.github.io/rpc/getnetworksolps.html) - /// method: post - /// tags: blockchain - /// - /// This RPC is implemented in the [mining.cpp](https://github.com/zcash/zcash/blob/d00fc6f4365048339c83f463874e4d6c240b63af/src/rpc/mining.cpp#L104) - /// file of the Zcash repository. The Zebra implementation can be found [here](https://github.com/ZcashFoundation/zebra/blob/19bca3f1159f9cb9344c9944f7e1cb8d6a82a07f/zebra-rpc/src/methods.rs#L2687). - /// - /// # Parameters - /// - /// - `blocks`: (number, optional, default=120) Number of blocks, or -1 for blocks over difficulty averaging window. - /// - `height`: (number, optional, default=-1) To estimate network speed at the time of a specific block height. - async fn get_network_sol_ps( - &self, - blocks: Option, - height: Option, - ) -> Result { - Ok(self.indexer.get_network_sol_ps(blocks, height).await?) - } - - // Helper function, to get the chain height in rpc implementations - async fn chain_height(&self) -> Result { - let snapshot = self.indexer.snapshot_nonfinalized_state().await?; - Ok(self.indexer.best_chaintip(&snapshot).await?.height.into()) - } -} - -// #[allow(deprecated)] -impl LightWalletIndexer for StateServiceSubscriber { - /// Return the height of the tip of the best chain - async fn get_latest_block(&self) -> Result { - let snapshot = self.indexer.snapshot_nonfinalized_state().await?; - let Some(non_finalized_snapshot) = snapshot.get_nfs_snapshot() else { - // TODO: This probably shouldn't be an error. - // this is an improvement over previous behaviour of - // acting as if we are only synced to the genesis block - return Err(StateServiceError::UnavailableNotSyncedEnough); - }; - Ok(non_finalized_snapshot.best_tip.to_wire()) - } - - /// Return the compact block corresponding to the given block identifier - async fn get_block(&self, request: BlockId) -> Result { - let hash_or_height = blockid_to_hashorheight(request).ok_or( - StateServiceError::TonicStatusError(tonic::Status::invalid_argument( - "Error: Invalid hash and/or height out of range. Failed to convert to u32.", - )), - )?; - - let snapshot = self.indexer.snapshot_nonfinalized_state().await?; - - // Convert HashOrHeight to chain_types::Height - let block_height = match hash_or_height { - HashOrHeight::Height(h) => chain_types::Height(h.0), - HashOrHeight::Hash(h) => self - .indexer - .get_block_height(&snapshot, chain_types::BlockHash(h.0)) - .await - .map_err(StateServiceError::ChainIndexError)? - .ok_or_else(|| { - StateServiceError::TonicStatusError(tonic::Status::not_found( - "Error: Block not found for given hash.", - )) - })?, - }; - - match self - .indexer - .get_compact_block(&snapshot, block_height, PoolTypeFilter::includes_all()) - .await - { - Ok(Some(block)) => Ok(block), - Ok(None) => { - let Some(non_finalized_snapshot) = snapshot.get_nfs_snapshot() else { - // TODO: This probably shouldn't be an error. - // this is an improvement over previous behaviour of - // acting as if we are only synced to the genesis block - return Err(StateServiceError::UnavailableNotSyncedEnough); - }; - let chain_height = non_finalized_snapshot.best_tip.height.0; - match hash_or_height { - HashOrHeight::Height(Height(height)) if height >= chain_height => Err( - StateServiceError::TonicStatusError(tonic::Status::out_of_range(format!( - "Error: Height out of range [{hash_or_height}]. Height requested \ - is greater than the best chain tip [{chain_height}].", - ))), - ), - _otherwise => Err(StateServiceError::TonicStatusError(tonic::Status::unknown( - "Error: Failed to retrieve block from state.", - ))), - } - } - Err(e) => { - let Some(non_finalized_snapshot) = snapshot.get_nfs_snapshot() else { - // TODO: This probably shouldn't be an error. - // this is an improvement over previous behaviour of - // acting as if we are only synced to the genesis block - return Err(StateServiceError::UnavailableNotSyncedEnough); - }; - let chain_height = non_finalized_snapshot.best_tip.height.0; - match hash_or_height { - HashOrHeight::Height(Height(height)) if height >= chain_height => Err( - StateServiceError::TonicStatusError(tonic::Status::out_of_range(format!( - "Error: Height out of range [{hash_or_height}]. Height requested \ - is greater than the best chain tip [{chain_height}].", - ))), - ), - _otherwise => - // TODO: Hide server error from clients before release. Currently useful for dev purposes. - { - Err(StateServiceError::TonicStatusError(tonic::Status::unknown( - format!("Error: Failed to retrieve block from node. Server Error: {e}",), - ))) - } - } - } - } - } - - /// Same as GetBlock except actions contain only nullifiers, - /// and saling outputs are not returned (Sapling spends still are) - async fn get_block_nullifiers(&self, request: BlockId) -> Result { - let height: u32 = request.height.try_into().map_err(|_| { - StateServiceError::TonicStatusError(tonic::Status::invalid_argument( - "Error: Height out of range. Failed to convert to u32.", - )) - })?; - - let snapshot = self.indexer.snapshot_nonfinalized_state().await?; - let block_height = chain_types::Height(height); - - match self - .indexer - .get_compact_block(&snapshot, block_height, PoolTypeFilter::includes_all()) - .await - { - Ok(Some(block)) => Ok(compact_block_to_nullifiers(block)), - Ok(None) => { - self.error_get_block( - BlockCacheError::Custom("Block not found".to_string()), - height, - ) - .await - } - Err(e) => Err(StateServiceError::ChainIndexError(e)), - } - } - - /// Return a list of consecutive compact blocks - async fn get_block_range( - &self, - blockrange: BlockRange, - ) -> Result { - self.get_block_range_inner(blockrange, false).await - } - /// Same as GetBlockRange except actions contain only nullifiers - async fn get_block_range_nullifiers( - &self, - request: BlockRange, - ) -> Result { - self.get_block_range_inner(request, true).await - } - - /// Return the requested full (not compact) transaction (as from zcashd) - async fn get_transaction(&self, request: TxFilter) -> Result { - let hash = zebra_chain::transaction::Hash::from( - <[u8; 32]>::try_from(request.hash).map_err(|_| { - StateServiceError::TonicStatusError(tonic::Status::invalid_argument( - "Error: Transaction hash incorrect", - )) - })?, - ); - let hex = hash.encode_hex(); - - // explicit over method call syntax to make it clear where this method is coming from - #[allow(clippy::result_large_err)] - ::get_raw_transaction(self, hex, Some(1)) - .await - .and_then(|grt| match grt { - GetRawTransaction::Raw(_serialized_transaction) => Err(StateServiceError::Custom( - "unreachable, verbose transaction expected".to_string(), - )), - GetRawTransaction::Object(transaction_object) => Ok(RawTransaction { - data: transaction_object.hex().as_ref().to_vec(), - height: transaction_object.height().unwrap_or(0) as u64, - }), - }) - } - - /// Submit the given transaction to the Zcash network - async fn send_transaction(&self, request: RawTransaction) -> Result { - let hex_tx = hex::encode(request.data); - let tx_output = self.send_raw_transaction(hex_tx).await?; - - Ok(SendResponse { - error_code: 0, - error_message: tx_output.hash().to_string(), - }) - } - - /// Return the transactions corresponding to the given t-address within the given block range - async fn get_taddress_transactions( - &self, - request: TransparentAddressBlockFilter, - ) -> Result { - let chain_height = self.chain_height().await?; - let txids = self.get_taddress_txids_helper(request).await?; - let fetch_service_clone = self.clone(); - let service_timeout = self.config.common.service.timeout; - let (transmitter, receiver) = - mpsc::channel(self.config.common.service.channel_size as usize); - tokio::spawn(async move { - let timeout = timeout( - time::Duration::from_secs((service_timeout * 4) as u64), - async { - for txid in txids { - let transaction = - fetch_service_clone.get_raw_transaction(txid, Some(1)).await; - if handle_raw_transaction::( - chain_height.0 as u64, - transaction, - transmitter.clone(), - ) - .await - .is_err() - { - break; - } - } - }, - ) - .await; - match timeout { - Ok(_) => {} - Err(_) => { - transmitter - .send(Err(tonic::Status::internal( - "Error: get_taddress_txids gRPC request timed out", - ))) - .await - .ok(); - } - } - }); - Ok(RawTransactionStream::new(receiver)) - } - - /// Return the txids corresponding to the given t-address within the given block range - /// This function is deprecated. Use `get_taddress_transactions`. - async fn get_taddress_txids( - &self, - request: TransparentAddressBlockFilter, - ) -> Result { - self.get_taddress_transactions(request).await - } - - /// Returns the total balance for a list of taddrs - async fn get_taddress_balance( - &self, - request: AddressList, - ) -> Result { - let taddrs = GetAddressBalanceRequest::new(request.addresses); - let balance = self.z_get_address_balance(taddrs).await?; - let checked_balance: i64 = match i64::try_from(balance.balance()) { - Ok(balance) => balance, - Err(_) => { - return Err(StateServiceError::TonicStatusError(tonic::Status::unknown( - "Error: Error converting balance from u64 to i64.", - ))); - } - }; - Ok(Balance { - value_zat: checked_balance, - }) - } - - /// Returns the total balance for a list of taddrs - /// - /// TODO: This is taken from fetch.rs, we could / probably should reconfigure into a trait implementation. - async fn get_taddress_balance_stream( - &self, - mut request: AddressStream, - ) -> Result { - let fetch_service_clone = self.clone(); - let service_timeout = self.config.common.service.timeout; - let (channel_tx, mut channel_rx) = - mpsc::channel::(self.config.common.service.channel_size as usize); - let fetcher_task_handle = tokio::spawn(async move { - let fetcher_timeout = timeout( - time::Duration::from_secs((service_timeout * 4) as u64), - async { - let mut total_balance: u64 = 0; - loop { - match channel_rx.recv().await { - Some(taddr) => { - let taddrs = GetAddressBalanceRequest::new(vec![taddr]); - let balance = - fetch_service_clone.z_get_address_balance(taddrs).await?; - total_balance += balance.balance(); - } - None => { - return Ok(total_balance); - } - } - } - }, - ) - .await; - match fetcher_timeout { - Ok(result) => result, - Err(_) => Err(tonic::Status::deadline_exceeded( - "Error: get_taddress_balance_stream request timed out.", - )), - } - }); - // NOTE: This timeout is so slow due to the blockcache not - // being implemented. This should be reduced to 30s once functionality is in place. - // TODO: Make [rpc_timout] a configurable system variable - // with [default = 30s] and [mempool_rpc_timout = 4*rpc_timeout] - let addr_recv_timeout = timeout( - time::Duration::from_secs((service_timeout * 4) as u64), - async { - while let Some(address_result) = request.next().await { - // TODO: Hide server error from clients before release. - // Currently useful for dev purposes. - let address = address_result.map_err(|e| { - tonic::Status::unknown(format!("Failed to read from stream: {e}")) - })?; - if channel_tx.send(address.address).await.is_err() { - // TODO: Hide server error from clients before release. - // Currently useful for dev purposes. - return Err(tonic::Status::unknown( - "Error: Failed to send address to balance task.", - )); - } - } - drop(channel_tx); - Ok::<(), tonic::Status>(()) - }, - ) - .await; - match addr_recv_timeout { - Ok(Ok(())) => {} - Ok(Err(e)) => { - fetcher_task_handle.abort(); - return Err(StateServiceError::TonicStatusError(e)); - } - Err(_) => { - fetcher_task_handle.abort(); - return Err(StateServiceError::TonicStatusError( - tonic::Status::deadline_exceeded( - "Error: get_taddress_balance_stream request timed out in address loop.", - ), - )); - } - } - match fetcher_task_handle.await { - Ok(Ok(total_balance)) => { - let checked_balance: i64 = match i64::try_from(total_balance) { - Ok(balance) => balance, - Err(_) => { - // TODO: Hide server error from clients before release. - // Currently useful for dev purposes. - return Err(StateServiceError::TonicStatusError(tonic::Status::unknown( - "Error: Error converting balance from u64 to i64.", - ))); - } - }; - Ok(Balance { - value_zat: checked_balance, - }) - } - Ok(Err(e)) => Err(StateServiceError::TonicStatusError(e)), - // TODO: Hide server error from clients before release. - // Currently useful for dev purposes. - Err(e) => Err(StateServiceError::TonicStatusError(tonic::Status::unknown( - format!("Fetcher Task failed: {e}"), - ))), - } - } - - /// Return the compact transactions currently in the mempool; the results - /// can be a few seconds out of date. If the Exclude list is empty, return - /// all transactions; otherwise return all *except* those in the Exclude list - /// (if any); this allows the client to avoid receiving transactions that it - /// already has (from an earlier call to this rpc). The transaction IDs in the - /// Exclude list can be shortened to any number of bytes to make the request - /// more bandwidth-efficient; if two or more transactions in the mempool - /// match a shortened txid, they are all sent (none is excluded). Transactions - /// in the exclude list that don't exist in the mempool are ignored. - async fn get_mempool_tx( - &self, - request: GetMempoolTxRequest, - ) -> Result { - let mut exclude_txids: Vec = vec![]; - - for (i, excluded_id) in request.exclude_txid_suffixes.iter().enumerate() { - if excluded_id.len() > 32 { - return Err(StateServiceError::TonicStatusError( - tonic::Status::invalid_argument(format!( - "Error: excluded txid {} is larger than 32 bytes", - i - )), - )); - } - - // NOTE: the TransactionHash methods cannot be used for this hex encoding as exclusions could be truncated to less than 32 bytes - let reversed_txid_bytes: Vec = excluded_id.iter().cloned().rev().collect(); - let hex_string_txid: String = hex::encode(&reversed_txid_bytes); - exclude_txids.push(hex_string_txid); - } - - let pool_types = match PoolTypeFilter::new_from_slice(&request.pool_types) { - Ok(pool_type_filter) => pool_type_filter, - Err(PoolTypeError::InvalidPoolType) => { - return Err(StateServiceError::TonicStatusError( - tonic::Status::invalid_argument( - "Error: An invalid `PoolType' was found".to_string(), - ), - )) - } - Err(PoolTypeError::UnknownPoolType(unknown_pool_type)) => { - return Err(StateServiceError::TonicStatusError( - tonic::Status::invalid_argument(format!( - "Error: Unknown `PoolType' {} was found", - unknown_pool_type - )), - )) - } - }; - - let indexer = self.indexer.clone(); - let service_timeout = self.config.common.service.timeout; - let (channel_tx, channel_rx) = - mpsc::channel(self.config.common.service.channel_size as usize); - tokio::spawn(async move { - let timeout = timeout( - time::Duration::from_secs((service_timeout * 4) as u64), - async { - let transactions = match indexer.get_mempool_transactions(exclude_txids).await { - Ok(transactions) => transactions, - Err(e) => { - channel_tx - .send(Err(tonic::Status::unknown(e.to_string()))) - .await - .ok(); - return; - } - }; - for serialized_transaction_bytes in transactions { - let txid = match zebra_chain::transaction::Transaction::zcash_deserialize( - &mut std::io::Cursor::new(&serialized_transaction_bytes), - ) { - Ok(transaction) => transaction.hash().0.to_vec(), - Err(error) => { - if channel_tx - .send(Err(tonic::Status::unknown(error.to_string()))) - .await - .is_err() - { - break; - } else { - continue; - } - } - }; - match ::parse_from_slice( - &serialized_transaction_bytes, - Some(vec![txid]), - None, - ) { - Ok(transaction) => { - // ParseFromSlice returns any data left after the conversion to a - // FullTransaction, If the conversion has succeeded this should be empty. - if transaction.0.is_empty() { - if channel_tx - .send( - transaction - .1 - .to_compact_tx(None, &pool_types) - .map_err(|e| tonic::Status::unknown(e.to_string())), - ) - .await - .is_err() - { - break; - } - } else { - // TODO: Hide server error from clients before release. Currently useful for dev purposes. - if channel_tx - .send(Err(tonic::Status::unknown("Error: "))) - .await - .is_err() - { - break; - } - } - } - Err(e) => { - // TODO: Hide server error from clients before release. Currently useful for dev purposes. - if channel_tx - .send(Err(tonic::Status::unknown(e.to_string()))) - .await - .is_err() - { - break; - } - } - } - } - }, - ) - .await; - match timeout { - Ok(_) => {} - Err(_) => { - channel_tx - .send(Err(tonic::Status::internal( - "Error: get_mempool_tx gRPC request timed out", - ))) - .await - .ok(); - } - } - }); - - Ok(CompactTransactionStream::new(channel_rx)) - } - - /// Return a stream of current Mempool transactions. This will keep the output stream open while - /// there are mempool transactions. It will close the returned stream when a new block is mined. - async fn get_mempool_stream(&self) -> Result { - let indexer = self.indexer.clone(); - let service_timeout = self.config.common.service.timeout; - let (channel_tx, channel_rx) = - mpsc::channel(self.config.common.service.channel_size as usize); - let snapshot = indexer.snapshot_nonfinalized_state().await?; - tokio::spawn(async move { - let timeout = timeout( - time::Duration::from_secs((service_timeout * 6) as u64), - async { - let Some(non_finalized_snapshot) = snapshot.get_nfs_snapshot() else { - // TODO: This probably shouldn't be an error. - // this is an improvement over previous behaviour of - // acting as if we are only synced to the genesis block - if let Err(e) = channel_tx - .send(Err(tonic::Status::failed_precondition( - "zaino not yet synced".to_string(), - ))) - .await - { - warn!(%e, "GetMempoolStream channel closed unexpectedly"); - }; - return; - }; - let mempool_height = non_finalized_snapshot.best_tip.height.0; - match indexer.get_mempool_stream(None) { - Some(mut mempool_stream) => { - while let Some(result) = mempool_stream.next().await { - match result { - Ok(transaction_bytes) => { - if channel_tx - .send(Ok(RawTransaction { - data: transaction_bytes, - height: mempool_height as u64, - })) - .await - .is_err() - { - break; - } - } - Err(e) => { - channel_tx - .send(Err(tonic::Status::internal(format!( - "Error in mempool stream: {e:?}" - )))) - .await - .ok(); - break; - } - } - } - } - None => { - warn!("Error fetching stream from mempool, Incorrect chain tip!"); - channel_tx - .send(Err(tonic::Status::internal("Error getting mempool stream"))) - .await - .ok(); - } - }; - }, - ) - .await; - match timeout { - Ok(_) => {} - Err(_) => { - channel_tx - .send(Err(tonic::Status::internal( - "Error: get_mempool_stream gRPC request timed out", - ))) - .await - .ok(); - } - } - }); - Ok(RawTransactionStream::new(channel_rx)) - } - - /// GetTreeState returns the note commitment tree state corresponding to the given block. - /// See section 3.7 of the Zcash protocol specification. It returns several other useful - /// values also (even though they can be obtained using GetBlock). - /// The block can be specified by either height or hash. - async fn get_tree_state(&self, request: BlockId) -> Result { - let hash_or_height = blockid_to_hashorheight(request).ok_or( - crate::error::StateServiceError::TonicStatusError(tonic::Status::invalid_argument( - "Invalid hash or height", - )), - )?; - #[allow(deprecated)] - let (hash, height, time, sapling, orchard) = - ::z_get_treestate( - self, - hash_or_height.to_string(), - ) - .await? - .into_parts(); - Ok(TreeState { - network: self - .config - .common - .network - .to_zebra_network() - .bip70_network_name(), - height: height.0 as u64, - hash: hash.to_string(), - time, - sapling_tree: sapling.map(hex::encode).unwrap_or_default(), - orchard_tree: orchard.map(hex::encode).unwrap_or_default(), - }) - } - - /// GetLatestTreeState returns the note commitment tree state corresponding to the chain tip. - async fn get_latest_tree_state(&self) -> Result { - let latest_block = self.chain_height().await?; - self.get_tree_state(BlockId { - height: latest_block.0 as u64, - hash: vec![], - }) - .await - } - - fn timeout_channel_size(&self) -> (u32, u32) { - ( - self.config.common.service.timeout, - self.config.common.service.channel_size, - ) - } - - /// Returns all unspent outputs for a list of addresses. - /// - /// Ignores all utxos below block height [GetAddressUtxosArg.start_height]. - /// Returns max [GetAddressUtxosArg.max_entries] utxos, or unrestricted if - /// [GetAddressUtxosArg.max_entries] = 0. - /// Utxos are collected and returned as a single Vec. - async fn get_address_utxos( - &self, - request: GetAddressUtxosArg, - ) -> Result { - super::validate_utxo_address_count(request.addresses.len())?; - let taddrs = GetAddressBalanceRequest::new(request.addresses); - let utxos = self.z_get_address_utxos(taddrs).await?; - let mut address_utxos: Vec = Vec::new(); - let mut entries: u32 = 0; - for utxo in utxos { - let (address, tx_hash, output_index, script, satoshis, height) = utxo.into_parts(); - if (height.0 as u64) < request.start_height { - continue; - } - entries += 1; - if request.max_entries > 0 && entries > request.max_entries { - break; - } - let checked_index = match i32::try_from(output_index.index()) { - Ok(index) => index, - Err(_) => { - return Err(StateServiceError::TonicStatusError(tonic::Status::unknown( - "Error: Index out of range. Failed to convert to i32.", - ))); - } - }; - let checked_satoshis = match i64::try_from(satoshis) { - Ok(satoshis) => satoshis, - Err(_) => { - return Err(StateServiceError::TonicStatusError(tonic::Status::unknown( - "Error: Satoshis out of range. Failed to convert to i64.", - ))); - } - }; - let utxo_reply = GetAddressUtxosReply { - address: address.to_string(), - txid: tx_hash.0.to_vec(), - index: checked_index, - script: script.as_raw_bytes().to_vec(), - value_zat: checked_satoshis, - height: height.0 as u64, - }; - address_utxos.push(utxo_reply) - } - Ok(GetAddressUtxosReplyList { address_utxos }) - } - - /// Returns all unspent outputs for a list of addresses. - /// - /// Ignores all utxos below block height [GetAddressUtxosArg.start_height]. - /// Returns max [GetAddressUtxosArg.max_entries] utxos, or unrestricted if - /// [GetAddressUtxosArg.max_entries] = 0. - /// Utxos are returned in a stream. - async fn get_address_utxos_stream( - &self, - request: GetAddressUtxosArg, - ) -> Result { - super::validate_utxo_address_count(request.addresses.len())?; - let taddrs = GetAddressBalanceRequest::new(request.addresses); - let utxos = self.z_get_address_utxos(taddrs).await?; - let service_timeout = self.config.common.service.timeout; - let (channel_tx, channel_rx) = - mpsc::channel(self.config.common.service.channel_size as usize); - tokio::spawn(async move { - let timeout = timeout( - time::Duration::from_secs((service_timeout * 4) as u64), - async { - let mut entries: u32 = 0; - for utxo in utxos { - let (address, tx_hash, output_index, script, satoshis, height) = - utxo.into_parts(); - if (height.0 as u64) < request.start_height { - continue; - } - entries += 1; - if request.max_entries > 0 && entries > request.max_entries { - break; - } - let checked_index = match i32::try_from(output_index.index()) { - Ok(index) => index, - Err(_) => { - let _ = channel_tx - .send(Err(tonic::Status::unknown( - "Error: Index out of range. Failed to convert to i32.", - ))) - .await; - return; - } - }; - let checked_satoshis = match i64::try_from(satoshis) { - Ok(satoshis) => satoshis, - Err(_) => { - let _ = channel_tx - .send(Err(tonic::Status::unknown( - "Error: Satoshis out of range. Failed to convert to i64.", - ))) - .await; - return; - } - }; - let utxo_reply = GetAddressUtxosReply { - address: address.to_string(), - txid: tx_hash.0.to_vec(), - index: checked_index, - script: script.as_raw_bytes().to_vec(), - value_zat: checked_satoshis, - height: height.0 as u64, - }; - if channel_tx.send(Ok(utxo_reply)).await.is_err() { - return; - } - } - }, - ) - .await; - match timeout { - Ok(_) => {} - Err(_) => { - channel_tx - .send(Err(tonic::Status::deadline_exceeded( - "Error: get_mempool_stream gRPC request timed out", - ))) - .await - .ok(); - } - } - }); - Ok(UtxoReplyStream::new(channel_rx)) - } - - /// Return information about this lightwalletd instance and the blockchain - /// - /// TODO: This could be made more efficient by fetching data directly (not using self.get_blockchain_info()) - async fn get_lightd_info(&self) -> Result { - let blockchain_info = self.get_blockchain_info().await?; - let sapling_id = zebra_rpc::methods::ConsensusBranchIdHex::new( - zebra_chain::parameters::ConsensusBranchId::from_hex("76b809bb") - .map_err(|_e| { - tonic::Status::internal( - "Internal Error - Consesnsus Branch ID hex conversion failed", - ) - })? - .into(), - ); - let sapling_activation_height = blockchain_info - .upgrades() - .get(&sapling_id) - .map_or(Height(1), |sapling_json| sapling_json.into_parts().1); - - let consensus_branch_id = zebra_chain::parameters::ConsensusBranchId::from( - blockchain_info.consensus().into_parts().0, - ) - .to_string(); - - let latest_upgrade = super::latest_network_upgrade(blockchain_info.upgrades()) - .map_err(StateServiceError::TonicStatusError)? - .into_parts(); - - let nu_name = latest_upgrade.0; - let nu_height = latest_upgrade.1; - - Ok(LightdInfo { - version: self.data.build_info().version(), - vendor: "ZingoLabs ZainoD".to_string(), - taddr_support: true, - chain_name: blockchain_info.chain().clone(), - sapling_activation_height: sapling_activation_height.0 as u64, - consensus_branch_id, - block_height: blockchain_info.blocks().0 as u64, - git_commit: self.data.build_info().commit_hash(), - branch: self.data.build_info().branch(), - build_date: self.data.build_info().build_date(), - build_user: self.data.build_info().build_user(), - estimated_height: blockchain_info.estimated_height().0 as u64, - zcashd_build: self.data.zebra_build(), - zcashd_subversion: self.data.zebra_subversion(), - donation_address: self - .config - .common - .donation_address - .as_ref() - .map(DonationAddress::encode) - .unwrap_or_default(), - upgrade_name: nu_name.to_string(), - upgrade_height: nu_height.0 as u64, - lightwallet_protocol_version: "v0.4.0".to_string(), - }) - } - - /// Testing-only, requires lightwalletd --ping-very-insecure (do not enable in production) - /// - /// NOTE: Currently unimplemented in Zaino. - async fn ping( - &self, - _request: zaino_proto::proto::service::Duration, - ) -> Result { - Err(crate::error::StateServiceError::TonicStatusError( - tonic::Status::unimplemented( - "Ping not yet implemented. If you require this RPC please open an \ - issue or PR at the Zaino github (https://github.com/zingolabs/zaino.git).", - ), - )) - } -} - -#[cfg(test)] -mod tests { - /// Classifies the byte-level relationship between two slices. - #[derive(Debug, PartialEq)] - enum ByteRelation { - /// The slices are identical. - Equal, - /// `actual` fully byte-reversed equals `expected` (endian swap). - FullByteReversal, - /// Each byte's bits reversed maps `actual` to `expected`. - PerByteBitReversal, - /// Reversing bytes within 16-bit chunks maps `actual` to `expected`. - ChunkSwap16, - /// Reversing bytes within 32-bit chunks maps `actual` to `expected`. - ChunkSwap32, - /// Reversing bytes within 64-bit chunks maps `actual` to `expected`. - ChunkSwap64, - /// No recognized transformation. - Unrecognized, - } - - impl std::fmt::Display for ByteRelation { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::Equal => write!(f, "equal"), - Self::FullByteReversal => write!(f, "full byte-reversal (endian swap)"), - Self::PerByteBitReversal => write!(f, "per-byte bit-reversal"), - Self::ChunkSwap16 => write!(f, "16-bit pairwise byte-swap"), - Self::ChunkSwap32 => write!(f, "32-bit chunk byte-reversal"), - Self::ChunkSwap64 => write!(f, "64-bit chunk byte-reversal"), - Self::Unrecognized => write!(f, "unrecognized mismatch"), - } - } - } - - /// Applies each candidate byte transformation to `actual` and returns - /// the first that produces `expected`, or [`ByteRelation::Unrecognized`]. - // `u32::is_multiple_of` is only stable from Rust 1.87; keep `% n == 0` for our older MSRV. - #[allow(clippy::manual_is_multiple_of)] - fn classify_byte_relation(actual: &[u8], expected: &[u8]) -> ByteRelation { - if actual == expected { - return ByteRelation::Equal; - } - - let chunk_swap = |size: usize| -> Vec { - actual - .chunks(size) - .flat_map(|c| c.iter().rev()) - .copied() - .collect() - }; - - let mut reversed = actual.to_vec(); - reversed.reverse(); - if reversed == expected { - return ByteRelation::FullByteReversal; - } - - let bit_reversed: Vec = actual.iter().map(|b| b.reverse_bits()).collect(); - if bit_reversed == expected { - return ByteRelation::PerByteBitReversal; - } - - if actual.len() % 2 == 0 && chunk_swap(2) == expected { - return ByteRelation::ChunkSwap16; - } - if actual.len() % 4 == 0 && chunk_swap(4) == expected { - return ByteRelation::ChunkSwap32; - } - if actual.len() % 8 == 0 && chunk_swap(8) == expected { - return ByteRelation::ChunkSwap64; - } - - ByteRelation::Unrecognized - } - - /// Verifies that our Sapling address parsing logic produces the same - /// diversifier and diversified transmission key (pk_d) hex strings as - /// zcashd's `z_validateaddress` RPC. - /// - /// # Guarantees - /// - /// - Exercises the production `sapling_key_bytes` function directly. - /// - The 11-byte diversifier matches the zcashd-derived test vector. - /// - The 32-byte pk_d (after the endian reversal inside `sapling_key_bytes`) - /// matches the zcashd-derived test vector. - /// - If the upstream serialization changes, the failure message - /// classifies the mismatch (endian swap, bit-reversal, chunk swap, - /// or unrecognized) to aid diagnosis. - /// - /// # Non-guarantees - /// - /// - Does not prove the test vector constants themselves are correct; - /// they were captured from zcashd and are trusted as ground truth. - /// - Does not exercise the full `z_validate_address` RPC path through - /// `StateService` — only the `sapling_key_bytes` extraction function. - /// - Does not verify behavior for malformed Sapling addresses or - /// addresses on other networks (mainnet, testnet). - #[test] - fn sapling_pk_d_byte_order_matches_test_vector() { - use crate::indexer::sapling_key_bytes; - use zcash_keys::address::Address; - use zcash_protocol::consensus::NetworkType; - - // Canonical source: live-tests/clientless/src/lib.rs::rpc::json_rpc - // Tracked for DRY consolidation: https://github.com/zingolabs/zaino/issues/988 - const SAPLING_ADDRESS: &str = "zregtestsapling1jalqhycwumq3unfxlzyzcktq3n478n82k2wacvl8gwfxk6ahshkxmtp2034qj28n7gl92ka5wca"; - const EXPECTED_DIVERSIFIER: &str = "977e0b930ee6c11e4d26f8"; - const EXPECTED_PK_D: &str = - "553ef2f328096a7c2aac6dec85b76b6b9243e733dc9db2eacce3eb8c60592c88"; - - let parsed: zcash_address::ZcashAddress = SAPLING_ADDRESS.parse().unwrap(); - let converted = parsed - .convert_if_network::
(NetworkType::Regtest) - .unwrap(); - - let Address::Sapling(s) = converted else { - panic!("expected Sapling address"); - }; - - let (diversifier, pk_d) = sapling_key_bytes(&s); - - let expected_diversifier = hex::decode(EXPECTED_DIVERSIFIER).unwrap(); - let expected_pk_d = hex::decode(EXPECTED_PK_D).unwrap(); - - // Diversifier - match classify_byte_relation(&diversifier, &expected_diversifier) { - ByteRelation::Equal => {} - relation => panic!( - "diversifier mismatch.\n relation: {relation}\n actual: {}\n expected: {}", - hex::encode(diversifier), - hex::encode(expected_diversifier), - ), - } - - // pk_d (sapling_key_bytes already applies the endian reversal) - match classify_byte_relation(&pk_d, &expected_pk_d) { - ByteRelation::Equal => {} - relation => panic!( - "pk_d mismatch — upstream serialization may have changed.\ - \n relation: {relation}\n actual: {}\n expected: {}", - hex::encode(pk_d), - hex::encode(expected_pk_d), - ), - } - } - - #[test] - fn classify_byte_relation_detects_known_transforms() { - let original = [0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08]; - - assert_eq!( - classify_byte_relation(&original, &original), - ByteRelation::Equal, - ); - - let mut reversed = original.to_vec(); - reversed.reverse(); - assert_eq!( - classify_byte_relation(&original, &reversed), - ByteRelation::FullByteReversal, - ); - - let bit_rev: Vec = original.iter().map(|b| b.reverse_bits()).collect(); - assert_eq!( - classify_byte_relation(&original, &bit_rev), - ByteRelation::PerByteBitReversal, - ); - - let swapped_16: Vec = original - .chunks(2) - .flat_map(|c| c.iter().rev()) - .copied() - .collect(); - assert_eq!( - classify_byte_relation(&original, &swapped_16), - ByteRelation::ChunkSwap16, - ); - - let garbage = [0xFF; 8]; - assert_eq!( - classify_byte_relation(&original, &garbage), - ByteRelation::Unrecognized, - ); - } -} diff --git a/packages/zaino-state/src/chain_index/source/validator_connector.rs b/packages/zaino-state/src/chain_index/source/validator_connector.rs index d8b3bb50e..231b27e06 100644 --- a/packages/zaino-state/src/chain_index/source/validator_connector.rs +++ b/packages/zaino-state/src/chain_index/source/validator_connector.rs @@ -19,7 +19,7 @@ use zaino_fetch::jsonrpsee::response::{ }; use zebra_rpc::sync::init_read_state_with_syncer; -use crate::config::{CommonBackendConfig, StateServiceConfig}; +use crate::config::{CommonBackendConfig, DirectConnectionConfig}; use zebra_chain::{ amount::{Amount, NonNegative}, block::{Header, SerializedBlock}, @@ -124,26 +124,28 @@ impl ValidatorConnector { Ok((ValidatorConnector::Fetch(fetcher), info)) } - /// Spawns a `ReadStateService`-backed [`ValidatorConnector::State`] from the state - /// backend config, returning the connector plus the validator's `getinfo` response. + /// Spawns a `ReadStateService`-backed [`ValidatorConnector::State`] from the common + /// backend config plus the [`DirectConnectionConfig`], returning the connector plus + /// the validator's `getinfo` response. /// - /// Owns the JSON-RPC + Zebra chain-syncer setup that previously lived in - /// `StateService::spawn`: builds the mempool JSON-RPC fetcher, launches the syncer - /// and `ReadStateService`, then blocks until the syncer has caught up to the - /// validator's best chain tip (comparing tip *hash* as well as height so a reorg - /// mid-sync cannot report a false match). The syncer task handle is retained on the - /// `State` so [`ValidatorConnector::shutdown`] can abort it. + /// Owns the JSON-RPC + Zebra chain-syncer setup for the `Direct` connection: builds + /// the mempool JSON-RPC fetcher, launches the syncer and `ReadStateService`, then + /// blocks until the syncer has caught up to the validator's best chain tip (comparing + /// tip *hash* as well as height so a reorg mid-sync cannot report a false match). The + /// syncer task handle is retained on the `State` so [`ValidatorConnector::shutdown`] + /// can abort it. pub(crate) async fn spawn_state( - config: &StateServiceConfig, + common: &CommonBackendConfig, + direct: &DirectConnectionConfig, ) -> Result<(Self, GetInfoResponse), BlockchainSourceError> { let map_err = |error: &dyn std::fmt::Display| BlockchainSourceError::Unrecoverable(error.to_string()); let rpc_client = JsonRpSeeConnector::new_from_config_parts( - &config.common.validator_rpc_address, - config.common.validator_rpc_user.clone(), - config.common.validator_rpc_password.clone(), - config.common.validator_cookie_path.clone(), + &common.validator_rpc_address, + common.validator_rpc_user.clone(), + common.validator_rpc_password.clone(), + common.validator_cookie_path.clone(), ) .await .map_err(|error| map_err(&error))?; @@ -154,14 +156,14 @@ impl ValidatorConnector { .map_err(|error| map_err(&error))?; info!( - grpc_address = %config.validator_grpc_address, + grpc_address = %direct.validator_grpc_address, "Launching Chain Syncer" ); let (mut read_state_service, _latest_chain_tip, chain_tip_change, sync_task_handle) = init_read_state_with_syncer( - config.validator_state_config.clone(), - &config.common.network.to_zebra_network(), - config.validator_grpc_address, + direct.validator_state_config.clone(), + &common.network.to_zebra_network(), + direct.validator_grpc_address, ) .await .map_err(|error| map_err(&error))? @@ -213,7 +215,7 @@ impl ValidatorConnector { let source = ValidatorConnector::State(State { read_state_service, mempool_fetcher: rpc_client, - network: config.common.network, + network: common.network, chain_tip_change, sync_task_handle: Some(Arc::new(sync_task_handle)), }); diff --git a/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs b/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs index c287b22d3..60e3091d4 100644 --- a/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs +++ b/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs @@ -1048,3 +1048,27 @@ async fn get_difficulty() { "difficulty should be positive, got {via_index}" ); } + +/// Drives the merged [`NodeBackedIndexerServiceSubscriber`] RPC layer over a +/// `MockchainSource`, confirming the service delegates to its chain index: the +/// service's `get_latest_block` reports the same tip the mockchain was synced to. +#[tokio::test(flavor = "multi_thread")] +async fn node_backed_indexer_service_serves_latest_block() { + use crate::indexer::node_backed_indexer::NodeBackedIndexerServiceSubscriber; + use crate::LightWalletIndexer as _; + use zaino_common::{network::ActivationHeights, Network}; + + let (blocks, _indexer, index_reader, _mockchain) = + load_test_vectors_and_sync_chain_index(MockchainMode::Static).await; + + let expected_tip = (blocks.len() as u32) - 1; + wait_for_indexer_tip(&index_reader, expected_tip).await; + + let service = NodeBackedIndexerServiceSubscriber::new_for_test( + index_reader, + Network::Regtest(ActivationHeights::default()), + ); + + let latest = service.get_latest_block().await.unwrap(); + assert_eq!(latest.height, expected_tip as u64); +} diff --git a/packages/zaino-state/src/config.rs b/packages/zaino-state/src/config.rs index 0083dbcd4..522c2be13 100644 --- a/packages/zaino-state/src/config.rs +++ b/packages/zaino-state/src/config.rs @@ -42,31 +42,37 @@ impl std::fmt::Display for DonationAddress { } } -/// Type of backend to be used. +/// How the [`NodeBackedIndexerService`](crate::NodeBackedIndexerService) connects to its +/// validator to source blockchain data. /// -/// Determines how Zaino fetches blockchain data from the validator. -#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, serde::Deserialize, serde::Serialize)] -#[serde(rename_all = "lowercase")] -pub enum BackendType { - /// Uses Zebra's ReadStateService for direct state access. - /// - /// More efficient but requires running on the same machine as Zebra. - State, - /// Uses JSON-RPC client to fetch data. +/// Carries the connection-specific configuration inline: the `Direct` variant owns the +/// Zebra `ReadStateService` settings, while `Rpc` needs only the JSON-RPC connection bits +/// already held in [`CommonBackendConfig`]. +#[derive(Debug, Clone)] +pub enum ValidatorConnectionType { + /// JSON-RPC connection (formerly `Fetch`). /// /// Compatible with Zcashd, Zebra, or another Zaino instance. - #[default] - Fetch, + Rpc, + /// Direct Zebra `ReadStateService` connection (formerly `State`). + /// + /// More efficient but requires running alongside a Zebra whose state DB and gRPC + /// sync endpoint we own. + Direct(DirectConnectionConfig), } -/// Unified backend configuration enum. +/// Connection parameters for the [`ValidatorConnectionType::Direct`] backend. +/// +/// Bundles the Zebra `ReadStateService` DB config with the gRPC sync endpoint used to +/// drive it, so the whole "direct connection" description travels as one value. #[derive(Debug, Clone)] -#[allow(deprecated)] -pub enum BackendConfig { - /// StateService config. - State(StateServiceConfig), - /// Fetchservice config. - Fetch(FetchServiceConfig), +pub struct DirectConnectionConfig { + /// Zebra [`zebra_state::ReadStateService`] config data (DB cache dir etc.). + pub validator_state_config: zebra_state::Config, + /// Validator gRPC address (requires ip:port format for Zebra state sync). + pub validator_grpc_address: std::net::SocketAddr, + /// Whether validator cookie authentication is enabled. + pub validator_cookie_auth: bool, } /// Configuration shared by every backend variant. @@ -109,29 +115,55 @@ pub struct CommonBackendConfig { pub indexer_version: String, } -/// Holds config data for [crate::StateService]. +impl CommonBackendConfig { + /// Builds a [`CommonBackendConfig`], applying the default RPC user/password + /// placeholder (`"xxxxxx"`) and this crate's `CARGO_PKG_VERSION` as the + /// indexer version. Shared by the [`NodeBackedIndexerServiceConfig`] constructors. + #[allow(clippy::too_many_arguments)] + pub fn new( + validator_rpc_address: String, + validator_cookie_path: Option, + validator_rpc_user: Option, + validator_rpc_password: Option, + service: ServiceConfig, + storage: StorageConfig, + ephemeral_finalised_state: bool, + network: Network, + donation_address: Option, + ) -> Self { + CommonBackendConfig { + validator_rpc_address, + validator_cookie_path, + validator_rpc_user: validator_rpc_user.unwrap_or_else(|| "xxxxxx".to_string()), + validator_rpc_password: validator_rpc_password.unwrap_or_else(|| "xxxxxx".to_string()), + service, + storage, + ephemeral_finalised_state, + network, + donation_address, + indexer_version: env!("CARGO_PKG_VERSION").to_string(), + } + } +} + +/// Holds config data for [`crate::NodeBackedIndexerService`]. +/// +/// Replaces the former per-backend `FetchServiceConfig` / `StateServiceConfig`: the +/// shared bits live in [`CommonBackendConfig`] and the backend-specific bits (only the +/// `Direct` backend has any) live in [`ValidatorConnectionType`]. #[derive(Debug, Clone)] -// #[deprecated] -pub struct StateServiceConfig { - /// Settings shared with [`FetchServiceConfig`]. +pub struct NodeBackedIndexerServiceConfig { + /// Connection-independent settings (validator RPC, storage, network, ...). pub common: CommonBackendConfig, - /// Zebra [`zebra_state::ReadStateService`] config data - pub validator_state_config: zebra_state::Config, - /// Validator gRPC address (requires ip:port format for Zebra state sync). - pub validator_grpc_address: std::net::SocketAddr, - /// Validator cookie auth. - pub validator_cookie_auth: bool, + /// Which validator connection to use, and its connection-specific config. + pub connection: ValidatorConnectionType, } -#[allow(deprecated)] -impl StateServiceConfig { - /// Returns a new instance of [`StateServiceConfig`]. +impl NodeBackedIndexerServiceConfig { + /// Returns a JSON-RPC (`Rpc`) service config (formerly `FetchServiceConfig::new`). #[allow(clippy::too_many_arguments)] - pub fn new( - validator_state_config: zebra_state::Config, + pub fn new_rpc( validator_rpc_address: String, - validator_grpc_address: std::net::SocketAddr, - validator_cookie_auth: bool, validator_cookie_path: Option, validator_rpc_user: Option, validator_rpc_password: Option, @@ -141,44 +173,30 @@ impl StateServiceConfig { network: Network, donation_address: Option, ) -> Self { - tracing::trace!( - activations = ?network.to_zebra_network().full_activation_list(), - "state service expecting NU activations" - ); - StateServiceConfig { - common: CommonBackendConfig { + NodeBackedIndexerServiceConfig { + common: CommonBackendConfig::new( validator_rpc_address, validator_cookie_path, - validator_rpc_user: validator_rpc_user.unwrap_or("xxxxxx".to_string()), - validator_rpc_password: validator_rpc_password.unwrap_or("xxxxxx".to_string()), + validator_rpc_user, + validator_rpc_password, service, storage, ephemeral_finalised_state, network, donation_address, - indexer_version: env!("CARGO_PKG_VERSION").to_string(), - }, - validator_state_config, - validator_grpc_address, - validator_cookie_auth, + ), + connection: ValidatorConnectionType::Rpc, } } -} - -/// Holds config data for [crate::FetchService]. -#[derive(Debug, Clone)] -#[deprecated] -pub struct FetchServiceConfig { - /// Settings shared with [`StateServiceConfig`]. - pub common: CommonBackendConfig, -} -#[allow(deprecated)] -impl FetchServiceConfig { - /// Returns a new instance of [`FetchServiceConfig`]. + /// Returns a direct-`ReadStateService` (`Direct`) service config + /// (formerly `StateServiceConfig::new`). #[allow(clippy::too_many_arguments)] - pub fn new( + pub fn new_direct( + validator_state_config: zebra_state::Config, validator_rpc_address: String, + validator_grpc_address: std::net::SocketAddr, + validator_cookie_auth: bool, validator_cookie_path: Option, validator_rpc_user: Option, validator_rpc_password: Option, @@ -188,19 +206,27 @@ impl FetchServiceConfig { network: Network, donation_address: Option, ) -> Self { - FetchServiceConfig { - common: CommonBackendConfig { + tracing::trace!( + activations = ?network.to_zebra_network().full_activation_list(), + "direct-connection service expecting NU activations" + ); + NodeBackedIndexerServiceConfig { + common: CommonBackendConfig::new( validator_rpc_address, validator_cookie_path, - validator_rpc_user: validator_rpc_user.unwrap_or("xxxxxx".to_string()), - validator_rpc_password: validator_rpc_password.unwrap_or("xxxxxx".to_string()), + validator_rpc_user, + validator_rpc_password, service, storage, ephemeral_finalised_state, network, donation_address, - indexer_version: env!("CARGO_PKG_VERSION").to_string(), - }, + ), + connection: ValidatorConnectionType::Direct(DirectConnectionConfig { + validator_state_config, + validator_grpc_address, + validator_cookie_auth, + }), } } } @@ -249,17 +275,9 @@ impl From for ChainIndexConfig { } } -#[allow(deprecated)] -impl From for ChainIndexConfig { - fn from(value: StateServiceConfig) -> Self { - value.common.into() - } -} - -#[allow(deprecated)] -impl From for ChainIndexConfig { - fn from(value: FetchServiceConfig) -> Self { - value.common.into() +impl From<&NodeBackedIndexerServiceConfig> for ChainIndexConfig { + fn from(value: &NodeBackedIndexerServiceConfig) -> Self { + value.common.clone().into() } } diff --git a/packages/zaino-state/src/error.rs b/packages/zaino-state/src/error.rs index 839c17045..2093ca68b 100644 --- a/packages/zaino-state/src/error.rs +++ b/packages/zaino-state/src/error.rs @@ -10,11 +10,11 @@ use std::{any::type_name, fmt::Display}; use zaino_fetch::jsonrpsee::connector::RpcRequestError; use zaino_proto::proto::utils::GetBlockRangeError; -/// Errors related to the `StateService`. -// #[deprecated] +/// Errors returned by the [`NodeBackedIndexerService`](crate::NodeBackedIndexerService) +/// subscriber's `ZcashIndexer` / `LightWalletIndexer` methods. #[derive(Debug, thiserror::Error)] #[allow(clippy::result_large_err)] -pub enum StateServiceError { +pub enum NodeBackedIndexerServiceError { /// Critical Errors, Restart Zaino. #[error("Critical error: {0}")] Critical(String), @@ -88,7 +88,7 @@ pub enum StateServiceError { UnavailableNotSyncedEnough, } -impl From for StateServiceError { +impl From for NodeBackedIndexerServiceError { fn from(value: GetBlockRangeError) -> Self { match value { GetBlockRangeError::StartHeightOutOfRange => { @@ -115,55 +115,59 @@ impl From for StateServiceError { } #[allow(deprecated)] -impl From for tonic::Status { - fn from(error: StateServiceError) -> Self { +impl From for tonic::Status { + fn from(error: NodeBackedIndexerServiceError) -> Self { match error { - StateServiceError::Critical(message) => tonic::Status::internal(message), - StateServiceError::Custom(message) => tonic::Status::internal(message), - StateServiceError::JoinError(err) => { + NodeBackedIndexerServiceError::Critical(message) => tonic::Status::internal(message), + NodeBackedIndexerServiceError::Custom(message) => tonic::Status::internal(message), + NodeBackedIndexerServiceError::JoinError(err) => { tonic::Status::internal(format!("Join error: {err}")) } - StateServiceError::JsonRpcConnectorError(err) => { + NodeBackedIndexerServiceError::JsonRpcConnectorError(err) => { tonic::Status::internal(format!("JsonRpcConnector error: {err}")) } - StateServiceError::RpcError(err) => { + NodeBackedIndexerServiceError::RpcError(err) => { tonic::Status::internal(format!("RPC error: {err:?}")) } - StateServiceError::ChainIndexError(err) => match err.kind { + NodeBackedIndexerServiceError::ChainIndexError(err) => match err.kind { ChainIndexErrorKind::InternalServerError => tonic::Status::internal(err.message), ChainIndexErrorKind::InvalidSnapshot => { tonic::Status::failed_precondition(err.message) } }, - StateServiceError::BlockCacheError(err) => { + NodeBackedIndexerServiceError::BlockCacheError(err) => { tonic::Status::internal(format!("BlockCache error: {err:?}")) } - StateServiceError::MempoolError(err) => { + NodeBackedIndexerServiceError::MempoolError(err) => { tonic::Status::internal(format!("Mempool error: {err:?}")) } - StateServiceError::TonicStatusError(err) => err, - StateServiceError::SerializationError(err) => { + NodeBackedIndexerServiceError::TonicStatusError(err) => err, + NodeBackedIndexerServiceError::SerializationError(err) => { tonic::Status::internal(format!("Serialization error: {err}")) } - StateServiceError::TryFromIntError(err) => { + NodeBackedIndexerServiceError::TryFromIntError(err) => { tonic::Status::internal(format!("Integer conversion error: {err}")) } - StateServiceError::IoError(err) => tonic::Status::internal(format!("IO error: {err}")), - StateServiceError::Generic(err) => { + NodeBackedIndexerServiceError::IoError(err) => { + tonic::Status::internal(format!("IO error: {err}")) + } + NodeBackedIndexerServiceError::Generic(err) => { tonic::Status::internal(format!("Generic error: {err}")) } - ref err @ StateServiceError::ZebradVersionMismatch { .. } => { + ref err @ NodeBackedIndexerServiceError::ZebradVersionMismatch { .. } => { tonic::Status::internal(err.to_string()) } - StateServiceError::UnhandledRpcError(e) => tonic::Status::internal(e.to_string()), - StateServiceError::UnavailableNotSyncedEnough => { + NodeBackedIndexerServiceError::UnhandledRpcError(e) => { + tonic::Status::internal(e.to_string()) + } + NodeBackedIndexerServiceError::UnavailableNotSyncedEnough => { tonic::Status::failed_precondition("zaino not yet synced".to_string()) } } } } -impl From> for StateServiceError { +impl From> for NodeBackedIndexerServiceError { fn from(value: RpcRequestError) -> Self { match value { RpcRequestError::Transport(transport_error) => { @@ -184,122 +188,6 @@ impl From> for StateServiceError { } } -/// Errors related to the `FetchService`. -#[deprecated] -#[derive(Debug, thiserror::Error)] -pub enum FetchServiceError { - /// Critical Errors, Restart Zaino. - #[error("Critical error: {0}")] - Critical(String), - - /// Error from JsonRpcConnector. - #[error("JsonRpcConnector error: {0}")] - JsonRpcConnectorError(#[from] zaino_fetch::jsonrpsee::error::TransportError), - - /// Chain index error. - #[error("Chain index error: {0}")] - ChainIndexError(#[from] ChainIndexError), - - /// RPC error in compatibility with zcashd. - #[error("RPC error: {0:?}")] - RpcError(#[from] zaino_fetch::jsonrpsee::connector::RpcError), - - /// Tonic gRPC error. - #[error("Tonic status error: {0}")] - TonicStatusError(#[from] tonic::Status), - - /// Serialization error. - #[error("Serialization error: {0}")] - SerializationError(#[from] zebra_chain::serialization::SerializationError), - #[error("Zaino has not synced high enough to serve this data")] - /// Zaino has not yet synced. - UnavailableNotSyncedEnough, -} - -impl From for tonic::Status { - fn from(error: FetchServiceError) -> Self { - match error { - FetchServiceError::Critical(message) => tonic::Status::internal(message), - FetchServiceError::JsonRpcConnectorError(err) => { - tonic::Status::internal(format!("JsonRpcConnector error: {err}")) - } - FetchServiceError::ChainIndexError(err) => match err.kind { - ChainIndexErrorKind::InternalServerError => tonic::Status::internal(err.message), - ChainIndexErrorKind::InvalidSnapshot => { - tonic::Status::failed_precondition(err.message) - } - }, - FetchServiceError::RpcError(err) => { - tonic::Status::internal(format!("RPC error: {err:?}")) - } - FetchServiceError::TonicStatusError(err) => err, - FetchServiceError::SerializationError(err) => { - tonic::Status::internal(format!("Serialization error: {err}")) - } - FetchServiceError::UnavailableNotSyncedEnough => { - tonic::Status::failed_precondition("zaino not yet synced".to_string()) - } - } - } -} - -impl From> for FetchServiceError { - fn from(value: RpcRequestError) -> Self { - match value { - RpcRequestError::Transport(transport_error) => { - FetchServiceError::JsonRpcConnectorError(transport_error) - } - RpcRequestError::JsonRpc(error) => { - FetchServiceError::Critical(format!("argument failed to serialze: {error}")) - } - RpcRequestError::InternalUnrecoverable(e) => { - FetchServiceError::Critical(format!("Internal unrecoverable error: {e}")) - } - RpcRequestError::ServerWorkQueueFull => FetchServiceError::Critical( - "Server queue full. Handling for this not yet implemented".to_string(), - ), - RpcRequestError::Method(e) => FetchServiceError::Critical(format!( - "unhandled rpc-specific {} error: {}", - type_name::(), - e.to_string() - )), - RpcRequestError::UnexpectedErrorResponse(error) => { - FetchServiceError::Critical(format!( - "unhandled rpc-specific {} error: {}", - type_name::(), - error - )) - } - } - } -} - -impl From for FetchServiceError { - fn from(value: GetBlockRangeError) -> Self { - match value { - GetBlockRangeError::StartHeightOutOfRange => { - FetchServiceError::TonicStatusError(tonic::Status::out_of_range( - "Error: Start height out of range. Failed to convert to u32.", - )) - } - GetBlockRangeError::NoStartHeightProvided => FetchServiceError::TonicStatusError( - tonic::Status::out_of_range("Error: No start height given"), - ), - GetBlockRangeError::EndHeightOutOfRange => { - FetchServiceError::TonicStatusError(tonic::Status::out_of_range( - "Error: End height out of range. Failed to convert to u32.", - )) - } - GetBlockRangeError::NoEndHeightProvided => FetchServiceError::TonicStatusError( - tonic::Status::out_of_range("Error: No end height given."), - ), - GetBlockRangeError::PoolTypeArgumentError(_) => FetchServiceError::TonicStatusError( - tonic::Status::invalid_argument("Error: invalid pool type"), - ), - } - } -} - /// These aren't the best conversions, but the MempoolError should go away /// in favor of a new type with the new chain cache is complete impl From> for MempoolError { diff --git a/packages/zaino-state/src/indexer.rs b/packages/zaino-state/src/indexer.rs index d2ffca07b..a2ef9d61d 100644 --- a/packages/zaino-state/src/indexer.rs +++ b/packages/zaino-state/src/indexer.rs @@ -1,5 +1,10 @@ -//! Holds the Indexer trait containing the zcash RPC definitions served by zaino -//! and generic wrapper structs for the various backend options available. +//! Zaino's indexer frontend: the `ZcashIndexer` / `LightWalletIndexer` RPC trait +//! definitions served by zaino, the generic [`IndexerService`] / [`IndexerSubscriber`] +//! wrappers, and the concrete [`node_backed_indexer::NodeBackedIndexerService`] — the +//! single validator-backed service (JSON-RPC `Rpc` or direct `ReadStateService` +//! connection, selected at runtime). + +pub(crate) mod node_backed_indexer; use crate::SendFut; use tokio::{sync::mpsc, time::timeout}; @@ -48,10 +53,10 @@ use crate::{ AddressStream, CompactBlockStream, CompactTransactionStream, RawTransactionStream, SubtreeRootReplyStream, UtxoReplyStream, }, - BackendType, }; -/// Wrapper Struct for a ZainoState chain-fetch service (StateService, FetchService) +/// Wrapper struct for a ZainoState chain-fetch service (currently the single +/// [`node_backed_indexer::NodeBackedIndexerService`]). /// /// The future plan is to also add a TonicService and DarksideService to this to enable /// wallets to use a single unified chain fetch service. @@ -92,9 +97,6 @@ where /// Implementors automatically gain [`Liveness`](zaino_common::probing::Liveness) and /// [`Readiness`](zaino_common::probing::Readiness) via the [`Status`] supertrait. pub trait ZcashService: Sized + Status { - /// Backend type. Read state or fetch service. - const BACKEND_TYPE: BackendType; - /// A subscriber to the service, used to fetch chain data. type Subscriber: Clone + ZcashIndexer + LightWalletIndexer + Status; @@ -113,7 +115,8 @@ pub trait ZcashService: Sized + Status { fn close(&mut self); } -/// Wrapper Struct for a ZainoState chain-fetch service subscriber (StateServiceSubscriber, FetchServiceSubscriber) +/// Wrapper struct for a ZainoState chain-fetch service subscriber (currently the single +/// [`node_backed_indexer::NodeBackedIndexerServiceSubscriber`]). /// /// The future plan is to also add a TonicServiceSubscriber and DarksideServiceSubscriber to this to enable wallets to use a single unified chain fetch service. #[derive(Clone)] @@ -1154,10 +1157,73 @@ pub(crate) fn build_subtrees_by_index_response( .into() } +fn latest_network_upgrade( + upgrades: &indexmap::IndexMap< + zebra_rpc::methods::ConsensusBranchIdHex, + zebra_rpc::methods::NetworkUpgradeInfo, + >, +) -> Result<&zebra_rpc::methods::NetworkUpgradeInfo, tonic::Status> { + upgrades.last().map(|(_, upgrade)| upgrade).ok_or_else(|| { + tonic::Status::failed_precondition("validator returned no network upgrade metadata") + }) +} + +/// Maximum number of addresses a single `get_address_utxos` / `get_address_utxos_stream` +/// request may carry. +/// +/// The service resolves the full backend UTXO set before applying `max_entries` / +/// `start_height` (issue #974). A complete pushdown fix needs upstream interface changes +/// the caller-supplied entry cap cannot reach today, so until then this bounds the one +/// input the service controls locally: the address fan-out. It stops an unauthenticated +/// caller forcing an unbounded number of backend address lookups in a single request, and +/// is set well above realistic wallet usage. +/// +/// TODO: make this deployment-configurable rather than a fixed constant. +const UTXO_MAX_ADDRESSES: usize = 1000; + +/// Reject a `get_address_utxos` request whose address list exceeds [`UTXO_MAX_ADDRESSES`]. +/// +/// `max_entries` bounds the response size, not the backend work; this guard bounds the +/// address fan-out, the part the service can cap without upstream changes. +fn validate_utxo_address_count(count: usize) -> Result<(), tonic::Status> { + if count > UTXO_MAX_ADDRESSES { + return Err(tonic::Status::invalid_argument(format!( + "Error: too many addresses in request: {count} exceeds the maximum of {UTXO_MAX_ADDRESSES}." + ))); + } + Ok(()) +} + #[cfg(test)] mod tests { use super::*; + #[test] + fn latest_network_upgrade_rejects_empty_metadata() { + let upgrades = indexmap::IndexMap::new(); + let err = super::latest_network_upgrade(&upgrades).expect_err("empty upgrades must fail"); + + assert_eq!(err.code(), tonic::Code::FailedPrecondition); + assert_eq!( + err.message(), + "validator returned no network upgrade metadata" + ); + } + + #[test] + fn utxo_address_count_within_limit_is_accepted() { + assert!(super::validate_utxo_address_count(0).is_ok()); + assert!(super::validate_utxo_address_count(super::UTXO_MAX_ADDRESSES).is_ok()); + } + + #[test] + fn utxo_address_count_over_limit_is_rejected() { + let err = super::validate_utxo_address_count(super::UTXO_MAX_ADDRESSES + 1) + .expect_err("over-limit address count must fail"); + + assert_eq!(err.code(), tonic::Code::InvalidArgument); + } + #[test] fn build_subtrees_by_index_response_hex_encodes_roots() { let roots = vec![([0xabu8; 32], 100u32), ([0xcdu8; 32], 200u32)]; @@ -1177,4 +1243,188 @@ mod tests { assert_eq!(subtrees[1].root, hex::encode([0xcdu8; 32])); assert_eq!(subtrees[1].end_height, Height(200)); } + + /// Classifies the byte-level relationship between two slices. + #[derive(Debug, PartialEq)] + enum ByteRelation { + /// The slices are identical. + Equal, + /// `actual` fully byte-reversed equals `expected` (endian swap). + FullByteReversal, + /// Each byte's bits reversed maps `actual` to `expected`. + PerByteBitReversal, + /// Reversing bytes within 16-bit chunks maps `actual` to `expected`. + ChunkSwap16, + /// Reversing bytes within 32-bit chunks maps `actual` to `expected`. + ChunkSwap32, + /// Reversing bytes within 64-bit chunks maps `actual` to `expected`. + ChunkSwap64, + /// No recognized transformation. + Unrecognized, + } + + impl std::fmt::Display for ByteRelation { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Equal => write!(f, "equal"), + Self::FullByteReversal => write!(f, "full byte-reversal (endian swap)"), + Self::PerByteBitReversal => write!(f, "per-byte bit-reversal"), + Self::ChunkSwap16 => write!(f, "16-bit pairwise byte-swap"), + Self::ChunkSwap32 => write!(f, "32-bit chunk byte-reversal"), + Self::ChunkSwap64 => write!(f, "64-bit chunk byte-reversal"), + Self::Unrecognized => write!(f, "unrecognized mismatch"), + } + } + } + + /// Applies each candidate byte transformation to `actual` and returns + /// the first that produces `expected`, or [`ByteRelation::Unrecognized`]. + // `u32::is_multiple_of` is only stable from Rust 1.87; keep `% n == 0` for our older MSRV. + #[allow(clippy::manual_is_multiple_of)] + fn classify_byte_relation(actual: &[u8], expected: &[u8]) -> ByteRelation { + if actual == expected { + return ByteRelation::Equal; + } + + let chunk_swap = |size: usize| -> Vec { + actual + .chunks(size) + .flat_map(|c| c.iter().rev()) + .copied() + .collect() + }; + + let mut reversed = actual.to_vec(); + reversed.reverse(); + if reversed == expected { + return ByteRelation::FullByteReversal; + } + + let bit_reversed: Vec = actual.iter().map(|b| b.reverse_bits()).collect(); + if bit_reversed == expected { + return ByteRelation::PerByteBitReversal; + } + + if actual.len() % 2 == 0 && chunk_swap(2) == expected { + return ByteRelation::ChunkSwap16; + } + if actual.len() % 4 == 0 && chunk_swap(4) == expected { + return ByteRelation::ChunkSwap32; + } + if actual.len() % 8 == 0 && chunk_swap(8) == expected { + return ByteRelation::ChunkSwap64; + } + + ByteRelation::Unrecognized + } + + /// Verifies that our Sapling address parsing logic produces the same + /// diversifier and diversified transmission key (pk_d) hex strings as + /// zcashd's `z_validateaddress` RPC. + /// + /// # Guarantees + /// + /// - Exercises the production `sapling_key_bytes` function directly. + /// - The 11-byte diversifier matches the zcashd-derived test vector. + /// - The 32-byte pk_d (after the endian reversal inside `sapling_key_bytes`) + /// matches the zcashd-derived test vector. + /// - If the upstream serialization changes, the failure message + /// classifies the mismatch (endian swap, bit-reversal, chunk swap, + /// or unrecognized) to aid diagnosis. + /// + /// # Non-guarantees + /// + /// - Does not prove the test vector constants themselves are correct; + /// they were captured from zcashd and are trusted as ground truth. + /// - Does not exercise the full `z_validate_address` RPC path through + /// `StateService` — only the `sapling_key_bytes` extraction function. + /// - Does not verify behavior for malformed Sapling addresses or + /// addresses on other networks (mainnet, testnet). + #[test] + fn sapling_pk_d_byte_order_matches_test_vector() { + use crate::indexer::sapling_key_bytes; + use zcash_keys::address::Address; + use zcash_protocol::consensus::NetworkType; + + // Canonical source: live-tests/clientless/src/lib.rs::rpc::json_rpc + // Tracked for DRY consolidation: https://github.com/zingolabs/zaino/issues/988 + const SAPLING_ADDRESS: &str = "zregtestsapling1jalqhycwumq3unfxlzyzcktq3n478n82k2wacvl8gwfxk6ahshkxmtp2034qj28n7gl92ka5wca"; + const EXPECTED_DIVERSIFIER: &str = "977e0b930ee6c11e4d26f8"; + const EXPECTED_PK_D: &str = + "553ef2f328096a7c2aac6dec85b76b6b9243e733dc9db2eacce3eb8c60592c88"; + + let parsed: zcash_address::ZcashAddress = SAPLING_ADDRESS.parse().unwrap(); + let converted = parsed + .convert_if_network::
(NetworkType::Regtest) + .unwrap(); + + let Address::Sapling(s) = converted else { + panic!("expected Sapling address"); + }; + + let (diversifier, pk_d) = sapling_key_bytes(&s); + + let expected_diversifier = hex::decode(EXPECTED_DIVERSIFIER).unwrap(); + let expected_pk_d = hex::decode(EXPECTED_PK_D).unwrap(); + + // Diversifier + match classify_byte_relation(&diversifier, &expected_diversifier) { + ByteRelation::Equal => {} + relation => panic!( + "diversifier mismatch.\n relation: {relation}\n actual: {}\n expected: {}", + hex::encode(diversifier), + hex::encode(expected_diversifier), + ), + } + + // pk_d (sapling_key_bytes already applies the endian reversal) + match classify_byte_relation(&pk_d, &expected_pk_d) { + ByteRelation::Equal => {} + relation => panic!( + "pk_d mismatch — upstream serialization may have changed.\ + \n relation: {relation}\n actual: {}\n expected: {}", + hex::encode(pk_d), + hex::encode(expected_pk_d), + ), + } + } + + #[test] + fn classify_byte_relation_detects_known_transforms() { + let original = [0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08]; + + assert_eq!( + classify_byte_relation(&original, &original), + ByteRelation::Equal, + ); + + let mut reversed = original.to_vec(); + reversed.reverse(); + assert_eq!( + classify_byte_relation(&original, &reversed), + ByteRelation::FullByteReversal, + ); + + let bit_rev: Vec = original.iter().map(|b| b.reverse_bits()).collect(); + assert_eq!( + classify_byte_relation(&original, &bit_rev), + ByteRelation::PerByteBitReversal, + ); + + let swapped_16: Vec = original + .chunks(2) + .flat_map(|c| c.iter().rev()) + .copied() + .collect(); + assert_eq!( + classify_byte_relation(&original, &swapped_16), + ByteRelation::ChunkSwap16, + ); + + let garbage = [0xFF; 8]; + assert_eq!( + classify_byte_relation(&original, &garbage), + ByteRelation::Unrecognized, + ); + } } diff --git a/packages/zaino-state/src/backends/fetch.rs b/packages/zaino-state/src/indexer/node_backed_indexer.rs similarity index 82% rename from packages/zaino-state/src/backends/fetch.rs rename to packages/zaino-state/src/indexer/node_backed_indexer.rs index 273851768..1619a3693 100644 --- a/packages/zaino-state/src/backends/fetch.rs +++ b/packages/zaino-state/src/indexer/node_backed_indexer.rs @@ -57,9 +57,15 @@ use zaino_proto::proto::{ #[allow(deprecated)] use crate::{ chain_index::chain_tips_from_nonfinalized_snapshot, - chain_index::{source::ValidatorConnector, types}, - config::{DonationAddress, FetchServiceConfig}, - error::FetchServiceError, + chain_index::{ + source::{BlockchainSource, ValidatorConnector}, + types, + }, + config::{ + CommonBackendConfig, DonationAddress, NodeBackedIndexerServiceConfig, + ValidatorConnectionType, + }, + error::NodeBackedIndexerServiceError, indexer::{ handle_raw_transaction, IndexerSubscriber, LightWalletIndexer, ZcashIndexer, ZcashService, }, @@ -69,60 +75,80 @@ use crate::{ UtxoReplyStream, }, utils::{get_build_info, ServiceMetadata}, - BackendType, }; use crate::{ chain_index::{non_finalised_state::ChainIndexSnapshot, NonFinalizedSnapshot}, ChainIndex, ChainIndexRpcExt, NodeBackedChainIndex, NodeBackedChainIndexSubscriber, }; -/// Chain fetch service backed by Zcashd's JsonRPC engine. +/// A single node-backed chain-fetch + tx-submission service, generic over its +/// [`BlockchainSource`]. /// -/// This service is a central service, [`FetchServiceSubscriber`] should be created to fetch data. -/// This is done to enable large numbers of concurrent subscribers without significant slowdowns. +/// Replaces the former `FetchService` / `StateService` split: the JSON-RPC (`Rpc`) and +/// direct-`ReadStateService` (`Direct`) backends are now one type selected at runtime +/// via [`NodeBackedIndexerServiceConfig`]. Production instantiates the default +/// `Source = ValidatorConnector` (which itself carries the `Rpc`/`Direct` arm); tests +/// may instantiate over a mock source. /// -/// NOTE: We currently do not implement clone for chain fetch services as this service is responsible for maintaining and closing its child processes. -/// ServiceSubscribers are used to create separate chain fetch processes while allowing central state processes to be managed in a single place. -/// If we want the ability to clone Service all JoinHandle's should be converted to Arc\. +/// This service is a central service — create a [`NodeBackedIndexerServiceSubscriber`] +/// (via [`ZcashService::get_subscriber`]) to fetch data. This lets large numbers of +/// concurrent subscribers run without contending on the single central process. +/// +/// NOTE: We do not implement `Clone` for the central service: it owns and closes its +/// child processes. Subscribers are the clone-safe read handles. #[derive(Debug)] -#[deprecated = "Will be eventually replaced by `BlockchainSource`"] -pub struct FetchService { +pub struct NodeBackedIndexerService { /// Core indexer. - indexer: NodeBackedChainIndex, + indexer: NodeBackedChainIndex, /// Service metadata. data: ServiceMetadata, - - /// StateService config data. - #[allow(deprecated)] - config: FetchServiceConfig, + /// Connection-independent config (validator RPC, storage, network, ...). + config: CommonBackendConfig, } -#[allow(deprecated)] -impl Status for FetchService { +impl Status for NodeBackedIndexerService { fn status(&self) -> StatusType { self.indexer.status() } } -#[allow(deprecated)] -impl ZcashService for FetchService { - const BACKEND_TYPE: BackendType = BackendType::Fetch; +impl NodeBackedIndexerService { + /// Tears down the indexer (sync loop, finalised DB, mempool, and any source-owned + /// syncer task) from a synchronous context. Shared by [`ZcashService::close`] and + /// [`Drop`]. + fn shutdown_blocking(&self) { + tokio::task::block_in_place(|| { + tokio::runtime::Handle::current().block_on(async { + let _ = self.indexer.shutdown().await; + }); + }); + } +} - type Subscriber = FetchServiceSubscriber; - type Config = FetchServiceConfig; +impl ZcashService for NodeBackedIndexerService { + type Subscriber = NodeBackedIndexerServiceSubscriber; + type Config = NodeBackedIndexerServiceConfig; - /// Initializes a new FetchService instance and starts sync process. - #[instrument(name = "FetchService::spawn", skip(config), fields(network = %config.common.network))] - async fn spawn(config: FetchServiceConfig) -> Result { + /// Initializes a new [`NodeBackedIndexerService`] and starts its sync process. + #[instrument(name = "NodeBackedIndexerService::spawn", skip(config), fields(network = %config.common.network))] + async fn spawn( + config: NodeBackedIndexerServiceConfig, + ) -> Result { info!( rpc_address = %config.common.validator_rpc_address, network = %config.common.network, - "Launching Fetch Service" + "Launching NodeBackedIndexerService" ); - let (source, zebra_build_data) = ValidatorConnector::spawn_fetch(&config.common) - .await - .map_err(|error| FetchServiceError::Critical(error.to_string()))?; + // Select the validator connection from config; both arms return the built source + // plus the validator `getinfo` used for service metadata. + let (source, zebra_build_data) = match &config.connection { + ValidatorConnectionType::Rpc => ValidatorConnector::spawn_fetch(&config.common).await, + ValidatorConnectionType::Direct(direct) => { + ValidatorConnector::spawn_state(&config.common, direct).await + } + } + .map_err(|error| NodeBackedIndexerServiceError::Critical(error.to_string()))?; let data = ServiceMetadata::new( get_build_info(config.common.indexer_version.clone()), @@ -132,22 +158,22 @@ impl ZcashService for FetchService { ); info!(build = %data.zebra_build(), subversion = %data.zebra_subversion(), "Connected to Zcash node"); - let indexer = NodeBackedChainIndex::new(source, config.clone().into()) + let indexer = NodeBackedChainIndex::new(source, (&config).into()) .await - .map_err(|error| FetchServiceError::Critical(error.to_string()))?; + .map_err(|error| NodeBackedIndexerServiceError::Critical(error.to_string()))?; - let fetch_service = Self { + let service = Self { indexer, data, - config, + config: config.common, }; // wait for sync to complete, return error on sync fail. loop { - match fetch_service.status() { + match service.status() { StatusType::Ready | StatusType::Closing => break, StatusType::CriticalError => { - return Err(FetchServiceError::Critical( + return Err(NodeBackedIndexerServiceError::Critical( "ChainIndex initial sync failed, check full log for details.".to_string(), )); } @@ -157,57 +183,51 @@ impl ZcashService for FetchService { } } - Ok(fetch_service) + Ok(service) } - /// Returns a [`FetchServiceSubscriber`]. - fn get_subscriber(&self) -> IndexerSubscriber { - IndexerSubscriber::new(FetchServiceSubscriber { + /// Returns a [`NodeBackedIndexerServiceSubscriber`]. + fn get_subscriber( + &self, + ) -> IndexerSubscriber> { + IndexerSubscriber::new(NodeBackedIndexerServiceSubscriber { indexer: self.indexer.subscriber(), data: self.data.clone(), config: self.config.clone(), }) } - /// Shuts down the StateService. + /// Shuts down the service, tearing down its indexer (sync loop, finalised DB, + /// mempool, and any source-owned syncer task). fn close(&mut self) { - tokio::task::block_in_place(|| { - tokio::runtime::Handle::current().block_on(async { - let _ = self.indexer.shutdown().await; - }); - }); + self.shutdown_blocking(); } } -#[allow(deprecated)] -impl Drop for FetchService { +impl Drop for NodeBackedIndexerService { fn drop(&mut self) { - self.close() + self.shutdown_blocking(); } } -/// A fetch service subscriber. -/// -/// Subscribers should be +/// A clone-safe, read-only subscriber to a [`NodeBackedIndexerService`]. #[derive(Debug, Clone)] -#[allow(deprecated)] -pub struct FetchServiceSubscriber { +pub struct NodeBackedIndexerServiceSubscriber { /// Core indexer. - pub indexer: NodeBackedChainIndexSubscriber, + pub indexer: NodeBackedChainIndexSubscriber, /// Service metadata. pub data: ServiceMetadata, - /// StateService config data. - #[allow(deprecated)] - config: FetchServiceConfig, + /// Connection-independent config (validator RPC, storage, network, ...). + config: CommonBackendConfig, } -impl Status for FetchServiceSubscriber { +impl Status for NodeBackedIndexerServiceSubscriber { fn status(&self) -> StatusType { self.indexer.status() } } -impl FetchServiceSubscriber { +impl NodeBackedIndexerServiceSubscriber { /// Fetches the current status #[deprecated(note = "Use the Status trait method instead")] pub fn get_status(&self) -> StatusType { @@ -215,15 +235,106 @@ impl FetchServiceSubscriber { } /// Returns the network type running. - #[allow(deprecated)] pub fn network(&self) -> zaino_common::Network { - self.config.common.network + self.config.network } } -impl ZcashIndexer for FetchServiceSubscriber { - #[allow(deprecated)] - type Error = FetchServiceError; +#[cfg(test)] +impl NodeBackedIndexerServiceSubscriber { + /// Wraps a chain-index subscriber in a service subscriber for tests, with placeholder + /// metadata/config. Lets unit tests drive the service RPC layer over a mock source + /// (no real validator). Production builds go through [`ZcashService::get_subscriber`]. + pub(crate) fn new_for_test( + indexer: NodeBackedChainIndexSubscriber, + network: zaino_common::Network, + ) -> Self { + Self { + data: ServiceMetadata::new( + get_build_info("test".to_string()), + network.to_zebra_network(), + "test-build".to_string(), + "test-subversion".to_string(), + ), + config: CommonBackendConfig::new( + "127.0.0.1:0".to_string(), + None, + None, + None, + zaino_common::ServiceConfig::default(), + zaino_common::StorageConfig::default(), + true, + network, + None, + ), + indexer, + } + } +} + +/// A subscriber to any chaintip updates. +#[derive(Clone)] +pub struct ChainTipSubscriber { + monitor: zebra_state::ChainTipChange, +} + +impl ChainTipSubscriber { + /// Waits until the tip hash has changed (relative to the last time this method + /// was called), then returns the best tip's block hash. + pub async fn next_tip_hash( + &mut self, + ) -> Result { + self.monitor + .wait_for_tip_change() + .await + .map(|tip| tip.best_tip_hash()) + } +} + +/// Methods available only on the production (`ValidatorConnector`-backed) subscriber: +/// the `Direct` connection owns a Zebra chain-tip stream and `ReadStateService` that the +/// generic source abstraction does not expose. +impl NodeBackedIndexerServiceSubscriber { + /// Gets a subscriber to any updates to the latest chain tip. + /// + /// Only meaningful for the `Direct` connection; the `Rpc` connection has no local + /// tip-change stream. + pub fn chaintip_update_subscriber(&self) -> ChainTipSubscriber { + ChainTipSubscriber { + monitor: self + .indexer + .source() + .chain_tip_change() + .expect("chaintip_update_subscriber requires the Direct connection"), + } + } + + /// The backing Zebra [`zebra_state::ReadStateService`] (`Direct` connection only). + /// + /// Test-only escape hatch: live tests recompute expected chain data directly off the + /// `ReadStateService`. Production code goes through the `ChainIndex` API. + #[cfg(feature = "test_dependencies")] + pub fn read_state_service(&self) -> zebra_state::ReadStateService { + self.indexer + .source() + .read_state_service() + .expect("read_state_service requires the Direct connection") + .clone() + } + + /// The indexer's mempool subscriber. + /// + /// Test-only escape hatch: live tests recompute expected `getmempoolinfo` values + /// directly off the mempool's entries. Production code goes through the `ChainIndex` + /// mempool API. + #[cfg(feature = "test_dependencies")] + pub fn mempool(&self) -> &crate::chain_index::mempool::MempoolSubscriber { + self.indexer.mempool_subscriber() + } +} + +impl ZcashIndexer for NodeBackedIndexerServiceSubscriber { + type Error = NodeBackedIndexerServiceError; /// Returns information about all changes to the given transparent addresses within the given inclusive block-height range. /// @@ -468,7 +579,7 @@ impl ZcashIndexer for FetchServiceSubscriber { async fn get_chain_tips(&self) -> Result { let snapshot = self.indexer.snapshot_nonfinalized_state().await?; let Some(non_finalized_snapshot) = snapshot.get_nfs_snapshot() else { - return Err(FetchServiceError::UnavailableNotSyncedEnough); + return Err(NodeBackedIndexerServiceError::UnavailableNotSyncedEnough); }; Ok(chain_tips_from_nonfinalized_snapshot( non_finalized_snapshot, @@ -488,7 +599,7 @@ impl ZcashIndexer for FetchServiceSubscriber { address: String, ) -> Result { #[allow(deprecated)] - let network = self.config.common.network.to_zebra_network(); + let network = self.config.network.to_zebra_network(); Ok(crate::indexer::validate_address(address, &network)) } @@ -498,7 +609,7 @@ impl ZcashIndexer for FetchServiceSubscriber { address: String, ) -> Result { #[allow(deprecated)] - let network = self.config.common.network.to_zebra_network(); + let network = self.config.network.to_zebra_network(); Ok(crate::indexer::z_validate_address(address, &network)) } @@ -538,7 +649,7 @@ impl ZcashIndexer for FetchServiceSubscriber { /// NOTE: This method currently has to fetch data from 2 places (get_treestate and get_indexed_block_by_*), /// If `ValidatorConnector::GetTreeState` was updated to return the additional information /// required, this second call could be removed, improving the performance of this method. - // Pre-existing lint: `FetchServiceError` is a large error type; returning it by value here is + // Pre-existing lint: `NodeBackedIndexerServiceError` is a large error type; returning it by value here is // flagged by `result_large_err`. Suppressed to satisfy `-D warnings` without an invasive // boxing refactor of the shared error enum. #[allow(clippy::result_large_err)] @@ -558,7 +669,7 @@ impl ZcashIndexer for FetchServiceSubscriber { .await? .ok_or( #[allow(deprecated)] - FetchServiceError::RpcError(RpcError::new_from_legacycode( + NodeBackedIndexerServiceError::RpcError(RpcError::new_from_legacycode( zebra_rpc::server::error::LegacyCode::InvalidParameter, "Failed to fetch block data.", )), @@ -569,7 +680,7 @@ impl ZcashIndexer for FetchServiceSubscriber { .await? .ok_or( #[allow(deprecated)] - FetchServiceError::RpcError(RpcError::new_from_legacycode( + NodeBackedIndexerServiceError::RpcError(RpcError::new_from_legacycode( zebra_rpc::server::error::LegacyCode::InvalidParameter, "Failed to fetch block data.", )), @@ -579,7 +690,7 @@ impl ZcashIndexer for FetchServiceSubscriber { let (sapling, orchard) = self.indexer.get_treestate(block_data.hash()).await?; let time: u32 = block_data.data().time().try_into().map_err(|_error| { #[allow(deprecated)] - FetchServiceError::RpcError(RpcError::new_from_legacycode( + NodeBackedIndexerServiceError::RpcError(RpcError::new_from_legacycode( zebra_rpc::server::error::LegacyCode::InvalidParameter, "Block time is out of range for u32.", )) @@ -644,12 +755,14 @@ impl ZcashIndexer for FetchServiceSubscriber { "sapling" => crate::chain_index::ShieldedPool::Sapling, "orchard" => crate::chain_index::ShieldedPool::Orchard, otherwise => { - return Err(FetchServiceError::RpcError(RpcError::new_from_legacycode( - zebra_rpc::server::error::LegacyCode::Misc, - format!( - "invalid pool name \"{otherwise}\", must be \"sapling\" or \"orchard\"" + return Err(NodeBackedIndexerServiceError::RpcError( + RpcError::new_from_legacycode( + zebra_rpc::server::error::LegacyCode::Misc, + format!( + "invalid pool name \"{otherwise}\", must be \"sapling\" or \"orchard\"" + ), ), - ))) + )) } }; let roots = self @@ -689,7 +802,7 @@ impl ZcashIndexer for FetchServiceSubscriber { ) -> Result { #[allow(deprecated)] let txid = types::TransactionHash::from_hex(&txid_hex).map_err(|error| { - FetchServiceError::RpcError(RpcError::new_from_legacycode( + NodeBackedIndexerServiceError::RpcError(RpcError::new_from_legacycode( zebra_rpc::server::error::LegacyCode::InvalidAddressOrKey, error.to_string(), )) @@ -697,7 +810,7 @@ impl ZcashIndexer for FetchServiceSubscriber { #[allow(deprecated)] let not_found_error = || { - FetchServiceError::RpcError(RpcError::new_from_legacycode( + NodeBackedIndexerServiceError::RpcError(RpcError::new_from_legacycode( zebra_rpc::server::error::LegacyCode::InvalidAddressOrKey, "No such mempool or main chain transaction", )) @@ -752,7 +865,7 @@ impl ZcashIndexer for FetchServiceSubscriber { height, confirmations, #[allow(deprecated)] - &self.config.common.network.to_zebra_network(), + &self.config.network.to_zebra_network(), None, block_hash, in_best_chain, @@ -860,7 +973,7 @@ impl ZcashIndexer for FetchServiceSubscriber { } #[allow(deprecated)] -impl LightWalletIndexer for FetchServiceSubscriber { +impl LightWalletIndexer for NodeBackedIndexerServiceSubscriber { /// Return the height of the tip of the best chain async fn get_latest_block(&self) -> Result { match self.indexer.snapshot_nonfinalized_state().await? { @@ -871,7 +984,7 @@ impl LightWalletIndexer for FetchServiceSubscriber { // TODO: This probably shouldn't be an error. // this is an improvement over previous behaviour of reporting // the genesis block - Err(FetchServiceError::UnavailableNotSyncedEnough) + Err(NodeBackedIndexerServiceError::UnavailableNotSyncedEnough) } } // dbg!(&tip); @@ -880,7 +993,7 @@ impl LightWalletIndexer for FetchServiceSubscriber { /// Return the compact block corresponding to the given block identifier async fn get_block(&self, request: BlockId) -> Result { let hash_or_height = blockid_to_hashorheight(request).ok_or( - FetchServiceError::TonicStatusError(tonic::Status::invalid_argument( + NodeBackedIndexerServiceError::TonicStatusError(tonic::Status::invalid_argument( "Error: Invalid hash and/or height out of range. Failed to convert to u32.", )), )?; @@ -892,12 +1005,12 @@ impl LightWalletIndexer for FetchServiceSubscriber { match self.indexer.get_block_height(&snapshot, hash.into()).await { Ok(Some(height)) => height.0, Ok(None) => { - return Err(FetchServiceError::TonicStatusError(tonic::Status::invalid_argument( + return Err(NodeBackedIndexerServiceError::TonicStatusError(tonic::Status::invalid_argument( "Error: Invalid hash and/or height out of range. Hash not founf in chain", ))); } Err(_e) => { - return Err(FetchServiceError::TonicStatusError( + return Err(NodeBackedIndexerServiceError::TonicStatusError( tonic::Status::internal("Error: Internal db error."), )); } @@ -909,7 +1022,7 @@ impl LightWalletIndexer for FetchServiceSubscriber { // TODO: This probably shouldn't be an error. // this is an improvement over previous behaviour of // acting as if we are only synced to the genesis block - return Err(FetchServiceError::UnavailableNotSyncedEnough); + return Err(NodeBackedIndexerServiceError::UnavailableNotSyncedEnough); }; match self @@ -925,32 +1038,38 @@ impl LightWalletIndexer for FetchServiceSubscriber { Ok(None) => { let chain_height = non_finalized_snapshot.best_tip.height.0; match hash_or_height { - HashOrHeight::Height(Height(height)) if height >= chain_height => Err( - FetchServiceError::TonicStatusError(tonic::Status::out_of_range(format!( - "Error: Height out of range [{hash_or_height}]. Height requested \ + HashOrHeight::Height(Height(height)) if height >= chain_height => { + Err(NodeBackedIndexerServiceError::TonicStatusError( + tonic::Status::out_of_range(format!( + "Error: Height out of range [{hash_or_height}]. Height requested \ is greater than the best chain tip [{chain_height}].", - ))), - ), - _otherwise => Err(FetchServiceError::TonicStatusError(tonic::Status::unknown( - "Error: Failed to retrieve block from state.", - ))), + )), + )) + } + _otherwise => Err(NodeBackedIndexerServiceError::TonicStatusError( + tonic::Status::unknown("Error: Failed to retrieve block from state."), + )), } } Err(e) => { let chain_height = non_finalized_snapshot.best_tip.height.0; match hash_or_height { - HashOrHeight::Height(Height(height)) if height >= chain_height => Err( - FetchServiceError::TonicStatusError(tonic::Status::out_of_range(format!( - "Error: Height out of range [{hash_or_height}]. Height requested \ + HashOrHeight::Height(Height(height)) if height >= chain_height => { + Err(NodeBackedIndexerServiceError::TonicStatusError( + tonic::Status::out_of_range(format!( + "Error: Height out of range [{hash_or_height}]. Height requested \ is greater than the best chain tip [{chain_height}].", - ))), - ), + )), + )) + } _otherwise => // TODO: Hide server error from clients before release. Currently useful for dev purposes. { - Err(FetchServiceError::TonicStatusError(tonic::Status::unknown( - format!("Error: Failed to retrieve block from node. Server Error: {e}",), - ))) + Err(NodeBackedIndexerServiceError::TonicStatusError( + tonic::Status::unknown(format!( + "Error: Failed to retrieve block from node. Server Error: {e}", + )), + )) } } } @@ -962,7 +1081,7 @@ impl LightWalletIndexer for FetchServiceSubscriber { /// NOTE: Currently this only returns Orchard nullifiers to follow Lightwalletd functionality but Sapling could be added if required by wallets. async fn get_block_nullifiers(&self, request: BlockId) -> Result { let hash_or_height = blockid_to_hashorheight(request).ok_or( - FetchServiceError::TonicStatusError(tonic::Status::invalid_argument( + NodeBackedIndexerServiceError::TonicStatusError(tonic::Status::invalid_argument( "Error: Invalid hash and/or height out of range. Failed to convert to u32.", )), )?; @@ -973,12 +1092,12 @@ impl LightWalletIndexer for FetchServiceSubscriber { match self.indexer.get_block_height(&snapshot, hash.into()).await { Ok(Some(height)) => height.0, Ok(None) => { - return Err(FetchServiceError::TonicStatusError(tonic::Status::invalid_argument( + return Err(NodeBackedIndexerServiceError::TonicStatusError(tonic::Status::invalid_argument( "Error: Invalid hash and/or height out of range. Hash not founf in chain", ))); } Err(_e) => { - return Err(FetchServiceError::TonicStatusError( + return Err(NodeBackedIndexerServiceError::TonicStatusError( tonic::Status::internal("Error: Internal db error."), )); } @@ -989,7 +1108,7 @@ impl LightWalletIndexer for FetchServiceSubscriber { // TODO: This probably shouldn't be an error. // this is an improvement over previous behaviour of // acting as if we are only synced to the genesis block - return Err(FetchServiceError::UnavailableNotSyncedEnough); + return Err(NodeBackedIndexerServiceError::UnavailableNotSyncedEnough); }; match self .indexer @@ -1004,40 +1123,44 @@ impl LightWalletIndexer for FetchServiceSubscriber { Ok(None) => { let chain_height = non_finalized_snapshot.best_tip.height.0; match hash_or_height { - HashOrHeight::Height(Height(height)) if height >= chain_height => Err( - FetchServiceError::TonicStatusError(tonic::Status::out_of_range(format!( - "Error: Height out of range [{hash_or_height}]. Height requested \ + HashOrHeight::Height(Height(height)) if height >= chain_height => { + Err(NodeBackedIndexerServiceError::TonicStatusError( + tonic::Status::out_of_range(format!( + "Error: Height out of range [{hash_or_height}]. Height requested \ is greater than the best chain tip [{chain_height}].", - ))), - ), + )), + )) + } HashOrHeight::Height(height) if height > self.data.network().sapling_activation_height() => { - Err(FetchServiceError::TonicStatusError( + Err(NodeBackedIndexerServiceError::TonicStatusError( tonic::Status::out_of_range(format!( "Error: Height out of range [{hash_or_height}]. Height requested \ is below sapling activation height [{chain_height}].", )), )) } - _otherwise => Err(FetchServiceError::TonicStatusError(tonic::Status::unknown( - "Error: Failed to retrieve block from state.", - ))), + _otherwise => Err(NodeBackedIndexerServiceError::TonicStatusError( + tonic::Status::unknown("Error: Failed to retrieve block from state."), + )), } } Err(e) => { let chain_height = non_finalized_snapshot.best_tip.height.0; match hash_or_height { - HashOrHeight::Height(Height(height)) if height >= chain_height => Err( - FetchServiceError::TonicStatusError(tonic::Status::out_of_range(format!( - "Error: Height out of range [{hash_or_height}]. Height requested \ + HashOrHeight::Height(Height(height)) if height >= chain_height => { + Err(NodeBackedIndexerServiceError::TonicStatusError( + tonic::Status::out_of_range(format!( + "Error: Height out of range [{hash_or_height}]. Height requested \ is greater than the best chain tip [{chain_height}].", - ))), - ), + )), + )) + } HashOrHeight::Height(height) if height > self.data.network().sapling_activation_height() => { - Err(FetchServiceError::TonicStatusError( + Err(NodeBackedIndexerServiceError::TonicStatusError( tonic::Status::out_of_range(format!( "Error: Height out of range [{hash_or_height}]. Height requested \ is below sapling activation height [{chain_height}].", @@ -1047,9 +1170,11 @@ impl LightWalletIndexer for FetchServiceSubscriber { _otherwise => // TODO: Hide server error from clients before release. Currently useful for dev purposes. { - Err(FetchServiceError::TonicStatusError(tonic::Status::unknown( - format!("Error: Failed to retrieve block from node. Server Error: {e}",), - ))) + Err(NodeBackedIndexerServiceError::TonicStatusError( + tonic::Status::unknown(format!( + "Error: Failed to retrieve block from node. Server Error: {e}", + )), + )) } } } @@ -1063,24 +1188,20 @@ impl LightWalletIndexer for FetchServiceSubscriber { request: BlockRange, ) -> Result { let validated_request = ValidatedBlockRangeRequest::new_from_block_range(&request) - .map_err(FetchServiceError::from)?; + .map_err(NodeBackedIndexerServiceError::from)?; let pool_type_filter = PoolTypeFilter::new_from_pool_types(&validated_request.pool_types()) .map_err(GetBlockRangeError::PoolTypeArgumentError) - .map_err(FetchServiceError::from)?; + .map_err(NodeBackedIndexerServiceError::from)?; // Note conversion here is safe due to the use of [`ValidatedBlockRangeRequest::new_from_block_range`] let start = validated_request.start() as u32; let end = validated_request.end() as u32; - let fetch_service_clone = self.clone(); - let service_timeout = self.config.common.service.timeout; - let (channel_tx, channel_rx) = - mpsc::channel(self.config.common.service.channel_size as usize); - let snapshot = fetch_service_clone - .indexer - .snapshot_nonfinalized_state() - .await?; + let service_clone = self.clone(); + let service_timeout = self.config.service.timeout; + let (channel_tx, channel_rx) = mpsc::channel(self.config.service.channel_size as usize); + let snapshot = service_clone.indexer.snapshot_nonfinalized_state().await?; tokio::spawn(async move { let timeout_result = timeout( @@ -1103,7 +1224,7 @@ impl LightWalletIndexer for FetchServiceSubscriber { // Use the snapshot tip directly, as this function doesn't support passthrough let chain_height = non_finalized_snapshot.best_tip.height.0; - match fetch_service_clone + match service_clone .indexer .get_compact_block_stream( &snapshot, @@ -1196,24 +1317,20 @@ impl LightWalletIndexer for FetchServiceSubscriber { request: BlockRange, ) -> Result { let validated_request = ValidatedBlockRangeRequest::new_from_block_range(&request) - .map_err(FetchServiceError::from)?; + .map_err(NodeBackedIndexerServiceError::from)?; let pool_type_filter = PoolTypeFilter::new_from_pool_types(&validated_request.pool_types()) .map_err(GetBlockRangeError::PoolTypeArgumentError) - .map_err(FetchServiceError::from)?; + .map_err(NodeBackedIndexerServiceError::from)?; // Note conversion here is safe due to the use of [`ValidatedBlockRangeRequest::new_from_block_range`] let start = validated_request.start() as u32; let end = validated_request.end() as u32; - let fetch_service_clone = self.clone(); - let service_timeout = self.config.common.service.timeout; - let (channel_tx, channel_rx) = - mpsc::channel(self.config.common.service.channel_size as usize); - let snapshot = fetch_service_clone - .indexer - .snapshot_nonfinalized_state() - .await?; + let service_clone = self.clone(); + let service_timeout = self.config.service.timeout; + let (channel_tx, channel_rx) = mpsc::channel(self.config.service.channel_size as usize); + let snapshot = service_clone.indexer.snapshot_nonfinalized_state().await?; tokio::spawn(async move { let timeout_result = timeout( @@ -1237,7 +1354,7 @@ impl LightWalletIndexer for FetchServiceSubscriber { // Use the snapshot tip directly, as this function doesn't support passthrough let chain_height = non_finalized_snapshot.best_tip.height.0; - match fetch_service_clone + match service_clone .indexer .get_compact_block_stream( &snapshot, @@ -1347,7 +1464,7 @@ impl LightWalletIndexer for FetchServiceSubscriber { let (hex, height) = if let GetRawTransaction::Object(tx_object) = tx { (tx_object.hex().clone(), tx_object.height()) } else { - return Err(FetchServiceError::TonicStatusError( + return Err(NodeBackedIndexerServiceError::TonicStatusError( tonic::Status::not_found("Error: Transaction not received"), )); }; @@ -1360,7 +1477,7 @@ impl LightWalletIndexer for FetchServiceSubscriber { // TODO: This probably shouldn't be an error. // this is an improvement over previous behaviour of // acting as if we are only synced to the genesis block - return Err(FetchServiceError::UnavailableNotSyncedEnough); + return Err(NodeBackedIndexerServiceError::UnavailableNotSyncedEnough); }; non_finalized_snapshot.best_tip.height.0 as u64 } @@ -1371,7 +1488,7 @@ impl LightWalletIndexer for FetchServiceSubscriber { height, }) } else { - Err(FetchServiceError::TonicStatusError( + Err(NodeBackedIndexerServiceError::TonicStatusError( tonic::Status::invalid_argument("Error: Transaction hash incorrect"), )) } @@ -1395,17 +1512,15 @@ impl LightWalletIndexer for FetchServiceSubscriber { ) -> Result { let chain_height = self.chain_height().await?; let txids = self.get_taddress_txids_helper(request).await?; - let fetch_service_clone = self.clone(); - let service_timeout = self.config.common.service.timeout; - let (transmitter, receiver) = - mpsc::channel(self.config.common.service.channel_size as usize); + let service_clone = self.clone(); + let service_timeout = self.config.service.timeout; + let (transmitter, receiver) = mpsc::channel(self.config.service.channel_size as usize); tokio::spawn(async move { let timeout = timeout( time::Duration::from_secs((service_timeout * 4) as u64), async { for txid in txids { - let transaction = - fetch_service_clone.get_raw_transaction(txid, Some(1)).await; + let transaction = service_clone.get_raw_transaction(txid, Some(1)).await; if handle_raw_transaction::( chain_height.0 as u64, transaction, @@ -1452,9 +1567,9 @@ impl LightWalletIndexer for FetchServiceSubscriber { let checked_balance: i64 = match i64::try_from(balance.balance()) { Ok(balance) => balance, Err(_) => { - return Err(FetchServiceError::TonicStatusError(tonic::Status::unknown( - "Error: Error converting balance from u64 to i64.", - ))); + return Err(NodeBackedIndexerServiceError::TonicStatusError( + tonic::Status::unknown("Error: Error converting balance from u64 to i64."), + )); } }; Ok(Balance { @@ -1468,10 +1583,10 @@ impl LightWalletIndexer for FetchServiceSubscriber { &self, mut request: AddressStream, ) -> Result { - let fetch_service_clone = self.clone(); - let service_timeout = self.config.common.service.timeout; + let service_clone = self.clone(); + let service_timeout = self.config.service.timeout; let (channel_tx, mut channel_rx) = - mpsc::channel::(self.config.common.service.channel_size as usize); + mpsc::channel::(self.config.service.channel_size as usize); let fetcher_task_handle = tokio::spawn(async move { let fetcher_timeout = timeout( time::Duration::from_secs((service_timeout * 4) as u64), @@ -1481,8 +1596,7 @@ impl LightWalletIndexer for FetchServiceSubscriber { match channel_rx.recv().await { Some(taddr) => { let taddrs = GetAddressBalanceRequest::new(vec![taddr]); - let balance = - fetch_service_clone.z_get_address_balance(taddrs).await?; + let balance = service_clone.z_get_address_balance(taddrs).await?; total_balance += balance.balance(); } None => { @@ -1530,11 +1644,11 @@ impl LightWalletIndexer for FetchServiceSubscriber { Ok(Ok(())) => {} Ok(Err(e)) => { fetcher_task_handle.abort(); - return Err(FetchServiceError::TonicStatusError(e)); + return Err(NodeBackedIndexerServiceError::TonicStatusError(e)); } Err(_) => { fetcher_task_handle.abort(); - return Err(FetchServiceError::TonicStatusError( + return Err(NodeBackedIndexerServiceError::TonicStatusError( tonic::Status::deadline_exceeded( "Error: get_taddress_balance_stream request timed out in address loop.", ), @@ -1548,21 +1662,23 @@ impl LightWalletIndexer for FetchServiceSubscriber { Err(_) => { // TODO: Hide server error from clients before release. // Currently useful for dev purposes. - return Err(FetchServiceError::TonicStatusError(tonic::Status::unknown( - "Error: Error converting balance from u64 to i64.", - ))); + return Err(NodeBackedIndexerServiceError::TonicStatusError( + tonic::Status::unknown( + "Error: Error converting balance from u64 to i64.", + ), + )); } }; Ok(Balance { value_zat: checked_balance, }) } - Ok(Err(e)) => Err(FetchServiceError::TonicStatusError(e)), + Ok(Err(e)) => Err(NodeBackedIndexerServiceError::TonicStatusError(e)), // TODO: Hide server error from clients before release. // Currently useful for dev purposes. - Err(e) => Err(FetchServiceError::TonicStatusError(tonic::Status::unknown( - format!("Fetcher Task failed: {e}"), - ))), + Err(e) => Err(NodeBackedIndexerServiceError::TonicStatusError( + tonic::Status::unknown(format!("Fetcher Task failed: {e}")), + )), } } @@ -1587,7 +1703,7 @@ impl LightWalletIndexer for FetchServiceSubscriber { for (i, excluded_id) in request.exclude_txid_suffixes.iter().enumerate() { if excluded_id.len() > 32 { - return Err(FetchServiceError::TonicStatusError( + return Err(NodeBackedIndexerServiceError::TonicStatusError( tonic::Status::invalid_argument(format!( "Error: excluded txid {} is larger than 32 bytes", i @@ -1603,9 +1719,8 @@ impl LightWalletIndexer for FetchServiceSubscriber { } let mempool = self.indexer.clone(); - let service_timeout = self.config.common.service.timeout; - let (channel_tx, channel_rx) = - mpsc::channel(self.config.common.service.channel_size as usize); + let service_timeout = self.config.service.timeout; + let (channel_tx, channel_rx) = mpsc::channel(self.config.service.channel_size as usize); tokio::spawn(async move { let timeout = timeout( @@ -1704,9 +1819,8 @@ impl LightWalletIndexer for FetchServiceSubscriber { #[allow(deprecated)] async fn get_mempool_stream(&self) -> Result { let indexer = self.indexer.clone(); - let service_timeout = self.config.common.service.timeout; - let (channel_tx, channel_rx) = - mpsc::channel(self.config.common.service.channel_size as usize); + let service_timeout = self.config.service.timeout; + let (channel_tx, channel_rx) = mpsc::channel(self.config.service.channel_size as usize); let snapshot = indexer.snapshot_nonfinalized_state().await?; tokio::spawn(async move { let timeout = timeout( @@ -1787,26 +1901,17 @@ impl LightWalletIndexer for FetchServiceSubscriber { /// The block can be specified by either height or hash. async fn get_tree_state(&self, request: BlockId) -> Result { let hash_or_height = blockid_to_hashorheight(request).ok_or( - crate::error::FetchServiceError::TonicStatusError(tonic::Status::invalid_argument( - "Invalid hash or height", - )), + crate::error::NodeBackedIndexerServiceError::TonicStatusError( + tonic::Status::invalid_argument("Invalid hash or height"), + ), )?; - #[allow(deprecated)] let (hash, height, time, sapling, orchard) = - ::z_get_treestate( - self, - hash_or_height.to_string(), - ) - .await? - .into_parts(); + ::z_get_treestate(self, hash_or_height.to_string()) + .await? + .into_parts(); Ok(TreeState { - network: self - .config - .common - .network - .to_zebra_network() - .bip70_network_name(), + network: self.config.network.to_zebra_network().bip70_network_name(), height: height.0 as u64, hash: hash.to_string(), time, @@ -1828,8 +1933,8 @@ impl LightWalletIndexer for FetchServiceSubscriber { #[allow(deprecated)] fn timeout_channel_size(&self) -> (u32, u32) { ( - self.config.common.service.timeout, - self.config.common.service.channel_size, + self.config.service.timeout, + self.config.service.channel_size, ) } @@ -1860,17 +1965,21 @@ impl LightWalletIndexer for FetchServiceSubscriber { let checked_index = match i32::try_from(output_index.index()) { Ok(index) => index, Err(_) => { - return Err(FetchServiceError::TonicStatusError(tonic::Status::unknown( - "Error: Index out of range. Failed to convert to i32.", - ))); + return Err(NodeBackedIndexerServiceError::TonicStatusError( + tonic::Status::unknown( + "Error: Index out of range. Failed to convert to i32.", + ), + )); } }; let checked_satoshis = match i64::try_from(satoshis) { Ok(satoshis) => satoshis, Err(_) => { - return Err(FetchServiceError::TonicStatusError(tonic::Status::unknown( - "Error: Satoshis out of range. Failed to convert to i64.", - ))); + return Err(NodeBackedIndexerServiceError::TonicStatusError( + tonic::Status::unknown( + "Error: Satoshis out of range. Failed to convert to i64.", + ), + )); } }; let utxo_reply = GetAddressUtxosReply { @@ -1900,9 +2009,8 @@ impl LightWalletIndexer for FetchServiceSubscriber { super::validate_utxo_address_count(request.addresses.len())?; let taddrs = GetAddressBalanceRequest::new(request.addresses); let utxos = self.z_get_address_utxos(taddrs).await?; - let service_timeout = self.config.common.service.timeout; - let (channel_tx, channel_rx) = - mpsc::channel(self.config.common.service.channel_size as usize); + let service_timeout = self.config.service.timeout; + let (channel_tx, channel_rx) = mpsc::channel(self.config.service.channel_size as usize); tokio::spawn(async move { let timeout = timeout( time::Duration::from_secs((service_timeout * 4) as u64), @@ -1994,7 +2102,7 @@ impl LightWalletIndexer for FetchServiceSubscriber { .to_string(); let latest_upgrade = super::latest_network_upgrade(blockchain_info.upgrades()) - .map_err(FetchServiceError::TonicStatusError)? + .map_err(NodeBackedIndexerServiceError::TonicStatusError)? .into_parts(); let nu_name = latest_upgrade.0; @@ -2017,7 +2125,6 @@ impl LightWalletIndexer for FetchServiceSubscriber { zcashd_subversion: self.data.zebra_subversion(), donation_address: self .config - .common .donation_address .as_ref() .map(DonationAddress::encode) @@ -2032,7 +2139,7 @@ impl LightWalletIndexer for FetchServiceSubscriber { /// /// NOTE: Currently unimplemented in Zaino. async fn ping(&self, _request: Duration) -> Result { - Err(FetchServiceError::TonicStatusError( + Err(NodeBackedIndexerServiceError::TonicStatusError( tonic::Status::unimplemented( "Ping not yet implemented. If you require this RPC \ please open an issue or PR at the Zaino github \ diff --git a/packages/zaino-state/src/lib.rs b/packages/zaino-state/src/lib.rs index 22cb0121a..a2c0d67a6 100644 --- a/packages/zaino-state/src/lib.rs +++ b/packages/zaino-state/src/lib.rs @@ -56,12 +56,8 @@ pub use indexer::{ ZcashService, }; -pub(crate) mod backends; - -#[allow(deprecated)] -pub use backends::{ - fetch::{FetchService, FetchServiceSubscriber}, - state::{StateService, StateServiceSubscriber}, +pub use indexer::node_backed_indexer::{ + ChainTipSubscriber, NodeBackedIndexerService, NodeBackedIndexerServiceSubscriber, }; pub mod chain_index; @@ -104,16 +100,14 @@ pub mod test_dependencies { pub(crate) mod config; -#[allow(deprecated)] pub use config::{ - BackendConfig, BackendType, ChainIndexConfig, CommonBackendConfig, DonationAddress, - FetchServiceConfig, StateServiceConfig, + ChainIndexConfig, CommonBackendConfig, DirectConnectionConfig, DonationAddress, + NodeBackedIndexerServiceConfig, ValidatorConnectionType, }; pub(crate) mod error; -#[allow(deprecated)] -pub use error::{FetchServiceError, StateServiceError}; +pub use error::NodeBackedIndexerServiceError; pub(crate) mod status; diff --git a/packages/zainod/src/config.rs b/packages/zainod/src/config.rs index aeb97dbbe..a5bd0a720 100644 --- a/packages/zainod/src/config.rs +++ b/packages/zainod/src/config.rs @@ -21,11 +21,32 @@ use zaino_common::{ try_resolve_address, AddressResolution, Network, ServiceConfig, StorageConfig, ValidatorConfig, }; use zaino_serve::server::config::{GrpcServerConfig, JsonRpcServerConfig}; -#[allow(deprecated)] use zaino_state::{ - BackendType, CommonBackendConfig, DonationAddress, FetchServiceConfig, StateServiceConfig, + CommonBackendConfig, DirectConnectionConfig, DonationAddress, NodeBackedIndexerServiceConfig, + ValidatorConnectionType, }; +/// On-disk selector for the validator connection (`backend = "direct" | "rpc"` in the +/// config file), mapped to [`zaino_state::ValidatorConnectionType`] at spawn. +/// +/// The legacy values `"state"` / `"fetch"` remain accepted as aliases for backward +/// compatibility with existing `zainod.toml` files. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum BackendType { + /// Direct Zebra `ReadStateService` access (formerly `state`). + /// + /// More efficient but requires running on the same machine as Zebra. + #[serde(alias = "state")] + Direct, + /// JSON-RPC access (formerly `fetch`). + /// + /// Compatible with Zcashd, Zebra, or another Zaino instance. + #[default] + #[serde(alias = "fetch")] + Rpc, +} + /// Header for generated configuration files. pub const GENERATED_CONFIG_HEADER: &str = r#"# Zaino Configuration # @@ -363,59 +384,56 @@ pub fn load_config_with_env( Ok(parsed_config) } -#[allow(deprecated)] -impl TryFrom for StateServiceConfig { +impl TryFrom for NodeBackedIndexerServiceConfig { type Error = IndexerError; fn try_from(cfg: ZainodConfig) -> Result { - let grpc_listen_address = cfg - .validator_settings - .validator_grpc_listen_address - .as_ref() - .ok_or_else(|| { - IndexerError::ConfigError( - "Missing validator_grpc_listen_address in configuration".to_string(), - ) - })?; - - let validator_grpc_address = - fetch_socket_addr_from_hostname(grpc_listen_address).map_err(|e| { - let msg = match e { - IndexerError::ConfigError(msg) => msg, - other => other.to_string(), + let connection = match cfg.backend { + BackendType::Rpc => ValidatorConnectionType::Rpc, + BackendType::Direct => { + let grpc_listen_address = cfg + .validator_settings + .validator_grpc_listen_address + .as_ref() + .ok_or_else(|| { + IndexerError::ConfigError( + "Missing validator_grpc_listen_address in configuration".to_string(), + ) + })?; + + let validator_grpc_address = fetch_socket_addr_from_hostname(grpc_listen_address) + .map_err(|e| { + let msg = match e { + IndexerError::ConfigError(msg) => msg, + other => other.to_string(), + }; + IndexerError::ConfigError(format!( + "Invalid validator_grpc_listen_address '{grpc_listen_address}': {msg}" + )) + })?; + + let validator_state_config = zebra_state::Config { + cache_dir: cfg.zebra_db_path.clone(), + ephemeral: false, + delete_old_database: true, + debug_stop_at_height: None, + debug_validity_check_interval: None, + should_backup_non_finalized_state: true, + debug_skip_non_finalized_state_backup_task: false, }; - IndexerError::ConfigError(format!( - "Invalid validator_grpc_listen_address '{grpc_listen_address}': {msg}" - )) - })?; - - let validator_state_config = zebra_state::Config { - cache_dir: cfg.zebra_db_path.clone(), - ephemeral: false, - delete_old_database: true, - debug_stop_at_height: None, - debug_validity_check_interval: None, - should_backup_non_finalized_state: true, - debug_skip_non_finalized_state_backup_task: false, - }; - let validator_cookie_auth = cfg.validator_settings.validator_cookie_path.is_some(); - - Ok(StateServiceConfig { - common: build_common(cfg), - validator_state_config, - validator_grpc_address, - validator_cookie_auth, - }) - } -} + let validator_cookie_auth = cfg.validator_settings.validator_cookie_path.is_some(); -#[allow(deprecated)] -impl TryFrom for FetchServiceConfig { - type Error = IndexerError; + ValidatorConnectionType::Direct(DirectConnectionConfig { + validator_state_config, + validator_grpc_address, + validator_cookie_auth, + }) + } + }; - fn try_from(cfg: ZainodConfig) -> Result { - Ok(FetchServiceConfig { + Ok(NodeBackedIndexerServiceConfig { common: build_common(cfg), + connection, }) } } @@ -555,7 +573,8 @@ key_path = "{}" let config_path = create_test_config_file(&temp_dir, &toml_content, "full_config.toml"); let config = load_config(&config_path).expect("load_config failed"); - assert_eq!(config.backend, BackendType::Fetch); + // legacy `backend = "fetch"` still parses via the serde alias + assert_eq!(config.backend, BackendType::Rpc); assert!(config.json_server_settings.is_some()); assert_eq!( config @@ -605,7 +624,8 @@ listen_address = "127.0.0.1:8137" let config = load_config(&config_path).expect("load_config failed"); let default_values = ZainodConfig::default(); - assert_eq!(config.backend, BackendType::State); + // legacy `backend = "state"` still parses via the serde alias + assert_eq!(config.backend, BackendType::Direct); assert!(config.json_server_settings.is_none()); assert_eq!( config.validator_settings.validator_user, @@ -1138,66 +1158,56 @@ listen_address = "127.0.0.1:8137" /// `CARGO_PKG_VERSION` at the boundary so the wire reflects the /// deployed binary, not zaino-state's library version. #[test] - #[allow(deprecated)] fn indexer_version_is_zainod_pkg_version() { let _guard = EnvGuard::new(); - let cfg = ZainodConfig::default(); - - let state_cfg = StateServiceConfig::try_from(cfg.clone()) - .expect("StateServiceConfig conversion should succeed for default ZainodConfig"); - assert_eq!(state_cfg.common.indexer_version, env!("CARGO_PKG_VERSION")); - - let fetch_cfg = FetchServiceConfig::try_from(cfg) - .expect("FetchServiceConfig conversion should succeed for default ZainodConfig"); - assert_eq!(fetch_cfg.common.indexer_version, env!("CARGO_PKG_VERSION")); + let service_cfg = NodeBackedIndexerServiceConfig::try_from(ZainodConfig::default()) + .expect("service config conversion should succeed for default ZainodConfig"); + assert_eq!( + service_cfg.common.indexer_version, + env!("CARGO_PKG_VERSION") + ); } - /// `StateServiceConfig::try_from` and `FetchServiceConfig::try_from` - /// share a single `build_common` helper, so the two backends can - /// never quietly disagree on the common payload they hand to a - /// service. Locks that property in across every field: a future - /// hand-rolled divergence (e.g. one path stops applying the - /// missing-credentials sentinel, or a new common field gets - /// populated on only one side) makes this fail. Pretty-Debug - /// equality is used because not every constituent of - /// `CommonBackendConfig` derives `PartialEq`, and a single - /// stringified compare future-proofs the test against fields added - /// later. + /// The `Rpc` and `Direct` connections share a single `build_common` helper, so the + /// common payload handed to the service is connection-independent. Locks that in + /// across every field: a future divergence (e.g. one path stops applying the + /// missing-credentials sentinel, or a new common field gets populated on only one + /// side) makes this fail. Pretty-Debug equality is used because not every constituent + /// of `CommonBackendConfig` derives `PartialEq`, and a single stringified compare + /// future-proofs the test against fields added later. #[test] - #[allow(deprecated)] - fn state_and_fetch_common_payloads_agree() { + fn common_payload_is_connection_independent() { let _guard = EnvGuard::new(); - let cfg = ZainodConfig::default(); + let rpc_cfg = NodeBackedIndexerServiceConfig::try_from(ZainodConfig { + backend: BackendType::Rpc, + ..ZainodConfig::default() + }) + .expect("Rpc conversion should succeed for default ZainodConfig"); + let direct_cfg = NodeBackedIndexerServiceConfig::try_from(ZainodConfig { + backend: BackendType::Direct, + ..ZainodConfig::default() + }) + .expect("Direct conversion should succeed for default ZainodConfig"); - let state_config = StateServiceConfig::try_from(cfg.clone()) - .expect("StateServiceConfig conversion should succeed for default ZainodConfig"); - let fetch_config = FetchServiceConfig::try_from(cfg) - .expect("FetchServiceConfig conversion should succeed for default ZainodConfig"); + assert!(matches!(rpc_cfg.connection, ValidatorConnectionType::Rpc)); + assert!(matches!( + direct_cfg.connection, + ValidatorConnectionType::Direct(_) + )); assert_eq!( - format!("{:#?}", state_config.common), - format!("{:#?}", fetch_config.common), + format!("{:#?}", rpc_cfg.common), + format!("{:#?}", direct_cfg.common), ); - let cfg = ZainodConfig { + let ephemeral_cfg = NodeBackedIndexerServiceConfig::try_from(ZainodConfig { ephemeral_finalised_state: true, ..ZainodConfig::default() - }; - - let state_config = StateServiceConfig::try_from(cfg.clone()) - .expect("StateServiceConfig conversion should succeed for ephemeral finalised state"); - let fetch_config = FetchServiceConfig::try_from(cfg) - .expect("FetchServiceConfig conversion should succeed for ephemeral finalised state"); - - assert!(state_config.common.ephemeral_finalised_state); - assert!(fetch_config.common.ephemeral_finalised_state); - - assert_eq!( - format!("{:#?}", state_config.common), - format!("{:#?}", fetch_config.common), - ); + }) + .expect("conversion should succeed for ephemeral finalised state"); + assert!(ephemeral_cfg.common.ephemeral_finalised_state); } /// Builds a default config with the JSON-RPC server bound to `addr`. @@ -1305,12 +1315,9 @@ listen_address = "127.0.0.1:8137" assert!(config.ephemeral_finalised_state); - #[allow(deprecated)] - { - let fetch_config = FetchServiceConfig::try_from(config) - .expect("FetchServiceConfig conversion should succeed"); + let service_config = NodeBackedIndexerServiceConfig::try_from(config) + .expect("service config conversion should succeed"); - assert!(fetch_config.common.ephemeral_finalised_state); - } + assert!(service_config.common.ephemeral_finalised_state); } } diff --git a/packages/zainod/src/error.rs b/packages/zainod/src/error.rs index 38a2433e4..0c27cecac 100644 --- a/packages/zainod/src/error.rs +++ b/packages/zainod/src/error.rs @@ -3,12 +3,10 @@ use zaino_fetch::jsonrpsee::error::TransportError; use zaino_serve::server::error::ServerError; -#[allow(deprecated)] -use zaino_state::{FetchServiceError, StateServiceError}; +use zaino_state::NodeBackedIndexerServiceError; /// Zingo-Indexer errors. #[derive(Debug, thiserror::Error)] -#[allow(deprecated)] pub enum IndexerError { /// Server based errors. #[error("Server error: {0}")] @@ -19,12 +17,9 @@ pub enum IndexerError { /// JSON RPSee connector errors. #[error("JSON RPSee connector error: {0}")] TransportError(#[from] TransportError), - /// FetchService errors. - #[error("FetchService error: {0}")] - FetchServiceError(Box), - /// FetchService errors. - #[error("StateService error: {0}")] - StateServiceError(Box), + /// NodeBackedIndexerService errors. + #[error("NodeBackedIndexerService error: {0}")] + NodeBackedIndexerServiceError(Box), /// HTTP related errors due to invalid URI. #[error("HTTP error: Invalid URI {0}")] HttpError(#[from] http::Error), @@ -43,16 +38,8 @@ pub enum IndexerError { Restart, } -#[allow(deprecated)] -impl From for IndexerError { - fn from(value: StateServiceError) -> Self { - IndexerError::StateServiceError(Box::new(value)) - } -} - -#[allow(deprecated)] -impl From for IndexerError { - fn from(value: FetchServiceError) -> Self { - IndexerError::FetchServiceError(Box::new(value)) +impl From for IndexerError { + fn from(value: NodeBackedIndexerServiceError) -> Self { + IndexerError::NodeBackedIndexerServiceError(Box::new(value)) } } diff --git a/packages/zainod/src/indexer.rs b/packages/zainod/src/indexer.rs index 1b23120a8..cfcdc2064 100644 --- a/packages/zainod/src/indexer.rs +++ b/packages/zainod/src/indexer.rs @@ -8,10 +8,9 @@ use zaino_serve::{ rpc::grpc_routes, server::{config::GrpcServerConfig, grpc::TonicServer, jsonrpc::JsonRpcServer}, }; -#[allow(deprecated)] use zaino_state::{ - BackendType, FetchService, FetchServiceConfig, IndexerService, LightWalletService, - StateService, StateServiceConfig, StatusType, ZcashIndexer, ZcashService, + IndexerService, LightWalletService, NodeBackedIndexerService, NodeBackedIndexerServiceConfig, + StatusType, ZcashIndexer, ZcashService, }; use crate::{config::ZainodConfig, error::IndexerError}; @@ -61,21 +60,13 @@ pub async fn spawn_indexer( info!(uri = %zebrad_uri, "Connected to node via JsonRPSee"); - #[allow(deprecated)] - match config.backend { - BackendType::State => { - let state_config = StateServiceConfig::try_from(config.clone())?; - Indexer::::launch_inner(state_config, config) - .await - .map(|res| res.0) - } - BackendType::Fetch => { - let fetch_config = FetchServiceConfig::try_from(config.clone())?; - Indexer::::launch_inner(fetch_config, config) - .await - .map(|res| res.0) - } - } + // Both the JSON-RPC (`Rpc`) and direct-`ReadStateService` (`Direct`) connections are + // now served by the single `NodeBackedIndexerService`; the connection is selected + // inside the config conversion from `config.backend`. + let service_config = NodeBackedIndexerServiceConfig::try_from(config.clone())?; + Indexer::::launch_inner(service_config, config) + .await + .map(|res| res.0) } impl Indexer From ec44cff337654edd64a86d6ea5c4aeb0f344cd4a Mon Sep 17 00:00:00 2001 From: zancas Date: Sun, 5 Jul 2026 14:33:05 -0700 Subject: [PATCH 088/134] refactor(zaino-common): DRY logging and activation-height conversions Deduplicate zaino-common and delete its dead public API; net -247 lines of Rust, behavior preserved. logging.rs (386 -> 187 lines): - Collapse the six init_*/try_init_* bodies into one private try_install(LogConfig); init() panics if a global subscriber is already set, try_init() ignores that case. - Delete unused DisplayHash/DisplayHexStr and the unreachable programmatic config surface (LogConfig builder methods, show_span_events, the *_with_config entry points); LogConfig and LogFormat are now private. - The ZAINOLOG_FORMAT/ZAINOLOG_COLOR/RUST_LOG interface and all three output formats are unchanged. config/network.rs (371 -> 308 lines): - New activation_heights_mirror! macro records the NetworkUpgrade variant <-> ActivationHeights field correspondence once and expands to the two mirror From impls plus from_zebra_pairs (all-None init + upgrade-list walk). Exhaustive destructures and match keep full compile-time drift detection; a future network upgrade lands as a single (Variant, field) pair. - Delete Network::zaino_regtest_heights (no callers workspace-wide). CHANGELOG: breaking removals and internal refactors recorded under Unreleased; version bump deferred to release. Verified: cargo check --workspace, cargo clippy --all-targets, cargo fmt, cargo nextest run -p zaino-common (19/19 passed), doctests (8/8 passed). Co-Authored-By: Claude Fable 5 --- packages/zaino-common/CHANGELOG.md | 15 + packages/zaino-common/src/config/network.rs | 211 +++++-------- packages/zaino-common/src/logging.rs | 309 ++++---------------- 3 files changed, 144 insertions(+), 391 deletions(-) diff --git a/packages/zaino-common/CHANGELOG.md b/packages/zaino-common/CHANGELOG.md index e2356e551..d5f413d0e 100644 --- a/packages/zaino-common/CHANGELOG.md +++ b/packages/zaino-common/CHANGELOG.md @@ -31,8 +31,23 @@ and this library adheres to Rust's notion of unrecognized key under `[storage.database]` (e.g. a stale `sync_write_batch_bytes`) is a hard parse error instead of being silently ignored and falling back to the default budget — the silent fallback previously OOM-killed nodes. +- The `ActivationHeights` ↔ `ConfiguredActivationHeights` conversions and the + zebra-network → `ActivationHeights` extraction are now generated from a single + variant/field pair list (internal refactor; conversion behavior unchanged). +- `logging::init` / `logging::try_init` build their subscriber through one shared + installer (internal refactor). The `ZAINOLOG_FORMAT` / `ZAINOLOG_COLOR` / + `RUST_LOG` runtime interface and output formats are unchanged. ### Deprecated ### Removed +- **Breaking** — `Network::zaino_regtest_heights` (unused; regtest heights come + from `ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS` or an explicit `ActivationHeights`). +- **Breaking** — `logging::DisplayHash` and `logging::DisplayHexStr` (unused). +- **Breaking** — the unused programmatic logging-configuration surface: + `logging::init_with_config` / `logging::try_init_with_config` and the `LogConfig` + / `LogFormat` types (including the `LogConfig` builder methods and + `show_span_events`, which no caller could reach). Logging is configured via the + `ZAINOLOG_FORMAT` / `ZAINOLOG_COLOR` / `RUST_LOG` environment variables, whose + behavior is unchanged. ### Fixed ## [0.2.0] - 2026-06-17 diff --git a/packages/zaino-common/src/config/network.rs b/packages/zaino-common/src/config/network.rs index d518d895b..da2e85862 100644 --- a/packages/zaino-common/src/config/network.rs +++ b/packages/zaino-common/src/config/network.rs @@ -141,97 +141,87 @@ impl Default for ActivationHeights { } } -impl From for ActivationHeights { - fn from( - ConfiguredActivationHeights { - before_overwinter, - overwinter, - sapling, - blossom, - heartwood, - canopy, - nu5, - nu6, - nu6_1, - nu6_2, - nu6_3, - nu7, - }: ConfiguredActivationHeights, - ) -> Self { - Self { - before_overwinter, - overwinter, - sapling, - blossom, - heartwood, - canopy, - nu5, - nu6, - nu6_1, - nu6_2, - nu6_3, - nu7, +/// Records the `NetworkUpgrade`-variant ↔ `ActivationHeights`-field correspondence +/// exactly once, generating everything derived from it: the two field-by-field +/// `From` conversions between [`ActivationHeights`] and zebra's +/// [`ConfiguredActivationHeights`] (the structs share field names), and +/// `ActivationHeights::from_zebra_pairs`, which walks zebra's +/// `(height, upgrade)` activation list. +/// +/// A declarative macro rather than functions because plain `fn`s cannot abstract +/// over struct fields, and the variant/field spellings (`Nu5`/`nu5`) differ only +/// by casing, which `macro_rules!` cannot derive — hence explicit pairs. +/// +/// Zebra's side of these conversions is structurally stable; the recurring edit +/// here is a new network upgrade, which lands as a single `(Variant, field)` +/// entry in the invocation below (after adding the struct field). The exhaustive +/// destructures and match keep full compile-time drift detection: a new zebra +/// field or variant fails the build until its pair is added. +macro_rules! activation_heights_mirror { + ($(($variant:ident, $field:ident)),* $(,)?) => { + impl From for ActivationHeights { + fn from( + ConfiguredActivationHeights { $($field),* }: ConfiguredActivationHeights, + ) -> Self { + Self { $($field),* } + } } - } -} -impl From for ConfiguredActivationHeights { - fn from( - ActivationHeights { - before_overwinter, - overwinter, - sapling, - blossom, - heartwood, - canopy, - nu5, - nu6, - nu6_1, - nu6_2, - nu6_3, - nu7, - }: ActivationHeights, - ) -> Self { - Self { - before_overwinter, - overwinter, - sapling, - blossom, - heartwood, - canopy, - nu5, - nu6, - nu6_1, - nu6_2, - nu6_3, - nu7, + + impl From for ConfiguredActivationHeights { + fn from(ActivationHeights { $($field),* }: ActivationHeights) -> Self { + Self { $($field),* } + } } - } + + impl ActivationHeights { + /// Builds heights from zebra's `(height, upgrade)` activation list; + /// upgrades absent from the list stay `None`. + fn from_zebra_pairs<'a>( + pairs: impl IntoIterator< + Item = ( + &'a zebra_chain::block::Height, + &'a zebra_chain::parameters::NetworkUpgrade, + ), + >, + ) -> Self { + let mut heights = Self { $($field: None),* }; + for (height, upgrade) in pairs { + match upgrade { + zebra_chain::parameters::NetworkUpgrade::Genesis => (), + $( + zebra_chain::parameters::NetworkUpgrade::$variant => { + heights.$field = Some(height.0) + } + )* + } + } + heights + } + } + }; } +activation_heights_mirror!( + (BeforeOverwinter, before_overwinter), + (Overwinter, overwinter), + (Sapling, sapling), + (Blossom, blossom), + (Heartwood, heartwood), + (Canopy, canopy), + (Nu5, nu5), + (Nu6, nu6), + (Nu6_1, nu6_1), + (Nu6_2, nu6_2), + (Nu6_3, nu6_3), + (Nu7, nu7), +); + impl Network { /// Convert to Zebra's network type for internal use (alias for to_zebra_default). pub fn to_zebra_network(&self) -> zebra_chain::parameters::Network { self.into() } - /// Get the standard regtest activation heights used by Zaino. - pub fn zaino_regtest_heights() -> ConfiguredActivationHeights { - ConfiguredActivationHeights { - before_overwinter: Some(1), - overwinter: Some(1), - sapling: Some(1), - blossom: Some(1), - heartwood: Some(1), - canopy: Some(1), - nu5: Some(1), - nu6: Some(1), - nu6_1: None, - nu6_2: None, - nu6_3: None, - nu7: None, - } - } - /// Determines if we should wait for the server to fully sync. Used for testing /// /// - Mainnet/Testnet: Skip sync (false) because we don't want to sync real chains in tests @@ -261,62 +251,9 @@ impl From for Network { zebra_chain::parameters::Network::Mainnet => Network::Mainnet, zebra_chain::parameters::Network::Testnet(parameters) => { if parameters.is_regtest() { - let mut activation_heights = ActivationHeights { - before_overwinter: None, - overwinter: None, - sapling: None, - blossom: None, - heartwood: None, - canopy: None, - nu5: None, - nu6: None, - nu6_1: None, - nu6_2: None, - nu6_3: None, - nu7: None, - }; - for (height, upgrade) in parameters.activation_heights().iter() { - match upgrade { - zebra_chain::parameters::NetworkUpgrade::Genesis => (), - zebra_chain::parameters::NetworkUpgrade::BeforeOverwinter => { - activation_heights.before_overwinter = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Overwinter => { - activation_heights.overwinter = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Sapling => { - activation_heights.sapling = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Blossom => { - activation_heights.blossom = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Heartwood => { - activation_heights.heartwood = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Canopy => { - activation_heights.canopy = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Nu5 => { - activation_heights.nu5 = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Nu6 => { - activation_heights.nu6 = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Nu6_1 => { - activation_heights.nu6_1 = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Nu6_2 => { - activation_heights.nu6_2 = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Nu6_3 => { - activation_heights.nu6_3 = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Nu7 => { - activation_heights.nu7 = Some(height.0) - } - } - } - Network::Regtest(activation_heights) + Network::Regtest(ActivationHeights::from_zebra_pairs( + parameters.activation_heights().iter(), + )) } else { Network::Testnet } diff --git a/packages/zaino-common/src/logging.rs b/packages/zaino-common/src/logging.rs index e97703bd3..7bf812671 100644 --- a/packages/zaino-common/src/logging.rs +++ b/packages/zaino-common/src/logging.rs @@ -18,23 +18,19 @@ //! ```no_run //! use zaino_common::logging; //! -//! // Initialize with defaults (tree format, info level) +//! // Initialize logging, configured via the environment variables above. //! logging::init(); -//! -//! // Or with custom configuration -//! logging::init_with_config(logging::LogConfig::default().format(logging::LogFormat::Json)); //! ``` use std::env; -use std::fmt; use std::io::IsTerminal; use time::macros::format_description; use tracing::Level; use tracing_subscriber::{ - fmt::{format::FmtSpan, time::UtcTime}, + fmt::time::UtcTime, layer::SubscriberExt, - util::SubscriberInitExt, + util::{SubscriberInitExt, TryInitError}, EnvFilter, }; use tracing_tree::HierarchicalLayer; @@ -45,7 +41,7 @@ const TIME_FORMAT: &[time::format_description::FormatItem<'static>] = /// Log output format. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -pub enum LogFormat { +enum LogFormat { /// Hierarchical tree view showing span nesting. Tree, /// Flat chronological stream (default). @@ -57,7 +53,7 @@ pub enum LogFormat { impl LogFormat { /// Parse from string (case-insensitive). - pub fn parse_str(s: &str) -> Option { + fn parse_str(s: &str) -> Option { match s.to_lowercase().as_str() { "tree" => Some(LogFormat::Tree), "stream" => Some(LogFormat::Stream), @@ -67,7 +63,7 @@ impl LogFormat { } /// Get from ZAINOLOG_FORMAT environment variable. - pub fn from_env() -> Self { + fn from_env() -> Self { env::var("ZAINOLOG_FORMAT") .ok() .and_then(|s| Self::parse_str(&s)) @@ -75,27 +71,15 @@ impl LogFormat { } } -impl fmt::Display for LogFormat { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - LogFormat::Tree => write!(f, "tree"), - LogFormat::Stream => write!(f, "stream"), - LogFormat::Json => write!(f, "json"), - } - } -} - -/// Logging configuration. +/// Logging configuration, read from the environment. #[derive(Debug, Clone)] -pub struct LogConfig { +struct LogConfig { /// Output format (tree, stream, or json). - pub format: LogFormat, + format: LogFormat, /// Enable ANSI colors. - pub color: bool, + color: bool, /// Default log level. - pub level: Level, - /// Show span events (enter/exit). - pub show_span_events: bool, + level: Level, } impl Default for LogConfig { @@ -119,228 +103,72 @@ impl Default for LogConfig { format: LogFormat::from_env(), color, level: Level::INFO, - show_span_events: false, } } } -impl LogConfig { - /// Create a new config with defaults. - pub fn new() -> Self { - Self::default() - } - - /// Set the log format. - pub fn format(mut self, format: LogFormat) -> Self { - self.format = format; - self - } - - /// Enable or disable colors. - pub fn color(mut self, color: bool) -> Self { - self.color = color; - self - } - - /// Set the default log level. - pub fn level(mut self, level: Level) -> Self { - self.level = level; - self - } - - /// Show span enter/exit events. - pub fn show_span_events(mut self, show: bool) -> Self { - self.show_span_events = show; - self - } -} - -/// Initialize logging with default configuration. +/// Initialize logging, configured from the environment (see module docs). +/// +/// # Panics /// -/// Reads `ZAINOLOG_FORMAT` environment variable to determine format: -/// - "stream" (default): Flat chronological output with timestamps -/// - "tree": Hierarchical span-based output -/// - "json": Machine-parseable JSON +/// Panics if a global tracing subscriber has already been set. pub fn init() { - init_with_config(LogConfig::default()); -} - -/// Initialize logging with custom configuration. -pub fn init_with_config(config: LogConfig) { - // If RUST_LOG is set, use it directly. Otherwise, default to zaino crates only. - // Users can set RUST_LOG=info to see all crates including zebra. - let env_filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| { - EnvFilter::new(format!( - "zaino={level},zainod={level},zainodlib={level}", - level = config.level.as_str() - )) - }); - - match config.format { - LogFormat::Tree => init_tree(env_filter, config), - LogFormat::Stream => init_stream(env_filter, config), - LogFormat::Json => init_json(env_filter), - } + try_install(LogConfig::default()).expect("global tracing subscriber already set"); } /// Try to initialize logging (won't fail if already initialized). /// /// Useful for tests where multiple test functions may try to initialize logging. pub fn try_init() { - try_init_with_config(LogConfig::default()); + let _ = try_install(LogConfig::default()); } -/// Try to initialize logging with custom configuration. -pub fn try_init_with_config(config: LogConfig) { +/// Build the subscriber described by `config` and install it as the global +/// default, erroring if one is already set. +fn try_install(config: LogConfig) -> Result<(), TryInitError> { + // If RUST_LOG is set, use it directly. Otherwise, default to zaino crates only. + // Users can set RUST_LOG=info to see all crates including zebra. let env_filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| { EnvFilter::new(format!( "zaino={level},zainod={level},zainodlib={level}", level = config.level.as_str() )) }); + let registry = tracing_subscriber::registry().with(env_filter); match config.format { - LogFormat::Tree => { - let _ = try_init_tree(env_filter, config); - } - LogFormat::Stream => { - let _ = try_init_stream(env_filter, config); - } - LogFormat::Json => { - let _ = try_init_json(env_filter); - } - } -} - -fn init_tree(env_filter: EnvFilter, config: LogConfig) { - let layer = HierarchicalLayer::new(2) - .with_ansi(config.color) - .with_targets(true) - .with_bracketed_fields(true) - .with_indent_lines(true) - .with_thread_ids(false) - .with_thread_names(false) - .with_deferred_spans(true) // Only show spans when they have events - .with_verbose_entry(false) // Don't repeat span info on entry - .with_verbose_exit(false); // Don't repeat span info on exit - - tracing_subscriber::registry() - .with(env_filter) - .with(layer) - .init(); -} - -fn try_init_tree( - env_filter: EnvFilter, - config: LogConfig, -) -> Result<(), tracing_subscriber::util::TryInitError> { - let layer = HierarchicalLayer::new(2) - .with_ansi(config.color) - .with_targets(true) - .with_bracketed_fields(true) - .with_indent_lines(true) - .with_thread_ids(false) - .with_thread_names(false) - .with_deferred_spans(true) // Only show spans when they have events - .with_verbose_entry(false) // Don't repeat span info on entry - .with_verbose_exit(false); // Don't repeat span info on exit - - tracing_subscriber::registry() - .with(env_filter) - .with(layer) - .try_init() -} - -fn init_stream(env_filter: EnvFilter, config: LogConfig) { - let span_events = if config.show_span_events { - FmtSpan::ENTER | FmtSpan::EXIT - } else { - FmtSpan::NONE - }; - - tracing_subscriber::fmt() - .with_env_filter(env_filter) - .with_timer(UtcTime::new(TIME_FORMAT)) - .with_target(true) - .with_ansi(config.color) - .with_span_events(span_events) - .pretty() - .init(); -} - -fn try_init_stream( - env_filter: EnvFilter, - config: LogConfig, -) -> Result<(), tracing_subscriber::util::TryInitError> { - let span_events = if config.show_span_events { - FmtSpan::ENTER | FmtSpan::EXIT - } else { - FmtSpan::NONE - }; - - let fmt_layer = tracing_subscriber::fmt::layer() - .with_timer(UtcTime::new(TIME_FORMAT)) - .with_target(true) - .with_ansi(config.color) - .with_span_events(span_events) - .pretty(); - - tracing_subscriber::registry() - .with(env_filter) - .with(fmt_layer) - .try_init() -} - -fn init_json(env_filter: EnvFilter) { - // JSON format keeps full RFC3339 timestamps for machine parsing - tracing_subscriber::fmt() - .with_env_filter(env_filter) - .json() - .with_timer(UtcTime::rfc_3339()) - .with_target(true) - .init(); -} - -fn try_init_json(env_filter: EnvFilter) -> Result<(), tracing_subscriber::util::TryInitError> { - // JSON format keeps full RFC3339 timestamps for machine parsing - let fmt_layer = tracing_subscriber::fmt::layer() - .json() - .with_timer(UtcTime::rfc_3339()) - .with_target(true); - - tracing_subscriber::registry() - .with(env_filter) - .with(fmt_layer) - .try_init() -} - -/// Wrapper for displaying hashes in a compact format. -/// -/// Shows the first 4 bytes (8 hex chars) followed by "..". -pub struct DisplayHash<'a>(pub &'a [u8]); - -impl fmt::Display for DisplayHash<'_> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - if self.0.len() >= 4 { - write!(f, "{}..", hex::encode(&self.0[..4])) - } else { - write!(f, "{}", hex::encode(self.0)) - } - } -} - -/// Wrapper for displaying hex strings in a compact format. -/// -/// Shows the first 8 chars followed by "..". -pub struct DisplayHexStr<'a>(pub &'a str); - -impl fmt::Display for DisplayHexStr<'_> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - if self.0.len() > 8 { - write!(f, "{}..", &self.0[..8]) - } else { - write!(f, "{}", self.0) - } + LogFormat::Tree => registry + .with( + HierarchicalLayer::new(2) + .with_ansi(config.color) + .with_targets(true) + .with_bracketed_fields(true) + .with_indent_lines(true) + .with_thread_ids(false) + .with_thread_names(false) + .with_deferred_spans(true) // Only show spans when they have events + .with_verbose_entry(false) // Don't repeat span info on entry + .with_verbose_exit(false), // Don't repeat span info on exit + ) + .try_init(), + LogFormat::Stream => registry + .with( + tracing_subscriber::fmt::layer() + .with_timer(UtcTime::new(TIME_FORMAT)) + .with_target(true) + .with_ansi(config.color) + .pretty(), + ) + .try_init(), + LogFormat::Json => registry + .with( + // JSON format keeps full RFC3339 timestamps for machine parsing + tracing_subscriber::fmt::layer() + .json() + .with_timer(UtcTime::rfc_3339()) + .with_target(true), + ) + .try_init(), } } @@ -356,31 +184,4 @@ mod tests { assert_eq!(LogFormat::parse_str("json"), Some(LogFormat::Json)); assert_eq!(LogFormat::parse_str("unknown"), None); } - - #[test] - fn test_display_hash() { - let hash = [0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc, 0xde, 0xf0]; - assert_eq!(format!("{}", DisplayHash(&hash)), "12345678.."); - } - - #[test] - fn test_display_hex_str() { - let hex_str = "1234567890abcdef"; - assert_eq!(format!("{}", DisplayHexStr(hex_str)), "12345678.."); - - let short_hex = "1234"; - assert_eq!(format!("{}", DisplayHexStr(short_hex)), "1234"); - } - - #[test] - fn test_config_builder() { - let config = LogConfig::new() - .format(LogFormat::Json) - .color(false) - .level(Level::DEBUG); - - assert_eq!(config.format, LogFormat::Json); - assert!(!config.color); - assert_eq!(config.level, Level::DEBUG); - } } From 7d19e0e91b8a75b2b59504e2cccd99bf13ab1b2e Mon Sep 17 00:00:00 2001 From: zancas Date: Sun, 5 Jul 2026 14:45:44 -0700 Subject: [PATCH 089/134] build(zaino-common): drop unused deps thiserror, nu-ansi-term, hex MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Compiler-verified sweep: each dependency in zaino-common's manifest was deleted one at a time and the crate re-checked with `cargo check -p zaino-common --all-targets`; a passing check kept the deletion, a failing one restored the dependency. Dropped (zero references, check passes without them): - thiserror — never used by this crate - nu-ansi-term — never used directly; tracing-tree pulls its own copy transitively - hex — last consumers were DisplayHash/DisplayHexStr, removed in the preceding DRY commit Kept (check fails without them): zebra-chain, serde, tracing, tracing-subscriber, tracing-tree, time. The root [workspace.dependencies] nu-ansi-term entry is now unused by every member but left in place; removing it is a separate root-manifest decision. Verified: cargo check --workspace --all-targets clean. Co-Authored-By: Claude Fable 5 --- Cargo.lock | 3 --- packages/zaino-common/CHANGELOG.md | 4 ++++ packages/zaino-common/Cargo.toml | 5 ----- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f725865bb..05f34f4d3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5942,10 +5942,7 @@ dependencies = [ name = "zaino-common" version = "0.2.0" dependencies = [ - "hex", - "nu-ansi-term", "serde", - "thiserror 2.0.18", "time", "tracing", "tracing-subscriber", diff --git a/packages/zaino-common/CHANGELOG.md b/packages/zaino-common/CHANGELOG.md index d5f413d0e..efdf02ad7 100644 --- a/packages/zaino-common/CHANGELOG.md +++ b/packages/zaino-common/CHANGELOG.md @@ -39,6 +39,10 @@ and this library adheres to Rust's notion of `RUST_LOG` runtime interface and output formats are unchanged. ### Deprecated ### Removed +- Unused dependencies `thiserror`, `nu-ansi-term`, and `hex` (`hex`'s last + consumers were the display wrappers removed below). Verified empirically: + each dependency was deleted in turn and the crate re-checked with + `cargo check --all-targets`. - **Breaking** — `Network::zaino_regtest_heights` (unused; regtest heights come from `ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS` or an explicit `ActivationHeights`). - **Breaking** — `logging::DisplayHash` and `logging::DisplayHexStr` (unused). diff --git a/packages/zaino-common/Cargo.toml b/packages/zaino-common/Cargo.toml index 5eed93aba..578f58489 100644 --- a/packages/zaino-common/Cargo.toml +++ b/packages/zaino-common/Cargo.toml @@ -15,13 +15,8 @@ zebra-chain = { workspace = true } # Serialization serde = { workspace = true, features = ["derive"] } -# Error handling -thiserror = { workspace = true } - # Logging tracing = { workspace = true } tracing-subscriber = { workspace = true } tracing-tree = { workspace = true } -nu-ansi-term = { workspace = true } -hex = { workspace = true } time = { workspace = true } From 24c31aa5da9f98fa01eb3475fee8909ce82f37ad Mon Sep 17 00:00:00 2001 From: zancas Date: Fri, 3 Jul 2026 19:24:38 -0700 Subject: [PATCH 090/134] fix: install rustls CryptoProvider explicitly and pin provider to ring zainod 0.5.0 panicked at startup when gRPC TLS was enabled: the zebra bump added a second rustls provider path (zebra-rpc -> zebra-node-services/rpc-client -> reqwest/rustls-tls -> ring) alongside zaino's own reqwest (aws-lc-rs), and rustls 0.23 refuses to auto-select a provider when both features are enabled. Reproduced and confirmed by a two-arm experiment recorded on the issue. Fixes zingolabs/zaino#1360. - workspace reqwest: "rustls" -> "rustls-no-provider" (drops aws-lc-rs from the graph entirely) - zaino-serve: tonic gains "tls-ring" so zaino chooses ring itself - zaino-common: new crypto::ensure_default_crypto_provider(), a guarded first-install-wins ring install that respects an embedder's provider; this is the load-bearing fix (Arm B: dual-provider graph passes with it), the feature pinning is defense-in-depth and build hygiene - called at the two boundaries that build rustls configs: the outbound reqwest client factory in zaino-fetch (new build_rpc_client, DRYing three hand-rolled ClientBuilder chains) and the gRPC TLS acceptor in zaino-serve - regression test: TLS-enabled spawn + real handshake against a committed test-CA/leaf fixture (CI previously ran plaintext only); fails with the exact reported panic under the pre-fix feature graph Co-Authored-By: Claude Fable 5 --- Cargo.lock | 49 +---------- Cargo.toml | 2 +- packages/zaino-common/Cargo.toml | 1 + packages/zaino-common/src/crypto.rs | 27 ++++++ packages/zaino-common/src/lib.rs | 1 + .../zaino-fetch/src/jsonrpsee/connector.rs | 48 ++++++----- packages/zaino-serve/Cargo.toml | 3 +- packages/zaino-serve/src/server/grpc.rs | 3 + packages/zaino-serve/src/server/grpc/tests.rs | 1 + .../grpc/tests/fixtures/tls_test_ca_cert.pem | 12 +++ .../grpc/tests/fixtures/tls_test_cert.pem | 12 +++ .../grpc/tests/fixtures/tls_test_key.pem | 5 ++ .../zaino-serve/src/server/grpc/tests/tls.rs | 82 +++++++++++++++++++ 13 files changed, 178 insertions(+), 68 deletions(-) create mode 100644 packages/zaino-common/src/crypto.rs create mode 100644 packages/zaino-serve/src/server/grpc/tests/fixtures/tls_test_ca_cert.pem create mode 100644 packages/zaino-serve/src/server/grpc/tests/fixtures/tls_test_cert.pem create mode 100644 packages/zaino-serve/src/server/grpc/tests/fixtures/tls_test_key.pem create mode 100644 packages/zaino-serve/src/server/grpc/tests/tls.rs diff --git a/Cargo.lock b/Cargo.lock index f725865bb..77ba5a672 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -171,28 +171,6 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" -[[package]] -name = "aws-lc-rs" -version = "1.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" -dependencies = [ - "aws-lc-sys", - "zeroize", -] - -[[package]] -name = "aws-lc-sys" -version = "0.41.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" -dependencies = [ - "cc", - "cmake", - "dunce", - "fs_extra", -] - [[package]] name = "axum" version = "0.8.9" @@ -710,15 +688,6 @@ dependencies = [ "zip32", ] -[[package]] -name = "cmake" -version = "0.1.58" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" -dependencies = [ - "cc", -] - [[package]] name = "cobs" version = "0.3.0" @@ -1190,12 +1159,6 @@ dependencies = [ "syn 2.0.118", ] -[[package]] -name = "dunce" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" - [[package]] name = "dyn-clone" version = "1.0.20" @@ -1467,12 +1430,6 @@ dependencies = [ "rand_core 0.6.4", ] -[[package]] -name = "fs_extra" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" - [[package]] name = "funty" version = "2.0.0" @@ -3487,7 +3444,6 @@ version = "0.11.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e" dependencies = [ - "aws-lc-rs", "bytes", "getrandom 0.3.4", "lru-slab", @@ -3877,7 +3833,6 @@ dependencies = [ "log", "percent-encoding", "pin-project-lite", - "quinn", "rustls", "rustls-pki-types", "rustls-platform-verifier", @@ -4003,7 +3958,6 @@ version = "0.23.41" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" dependencies = [ - "aws-lc-rs", "log", "once_cell", "ring", @@ -4068,7 +4022,6 @@ version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ - "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -5944,6 +5897,7 @@ version = "0.2.0" dependencies = [ "hex", "nu-ansi-term", + "rustls", "serde", "thiserror 2.0.18", "time", @@ -6006,6 +5960,7 @@ dependencies = [ "tonic", "tower 0.4.13", "tracing", + "zaino-common", "zaino-fetch", "zaino-proto", "zaino-state", diff --git a/Cargo.toml b/Cargo.toml index 0bf07dc94..de1292cf9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -77,7 +77,7 @@ http = "1.1" url = "2.5" reqwest = { version = "0.13", default-features = false, features = [ "cookies", - "rustls", + "rustls-no-provider", "webpki-roots", ] } tower = { version = "0.4", features = ["buffer", "util"] } diff --git a/packages/zaino-common/Cargo.toml b/packages/zaino-common/Cargo.toml index 5eed93aba..3ce75c1d6 100644 --- a/packages/zaino-common/Cargo.toml +++ b/packages/zaino-common/Cargo.toml @@ -25,3 +25,4 @@ tracing-tree = { workspace = true } nu-ansi-term = { workspace = true } hex = { workspace = true } time = { workspace = true } +rustls = { version = "0.23", default-features = false, features = ["ring", "std"] } diff --git a/packages/zaino-common/src/crypto.rs b/packages/zaino-common/src/crypto.rs new file mode 100644 index 000000000..8791bbae2 --- /dev/null +++ b/packages/zaino-common/src/crypto.rs @@ -0,0 +1,27 @@ +//! Process-level rustls `CryptoProvider` management. + +/// Installs rustls's `ring` provider as the process-level default if no +/// provider is installed yet. +/// +/// Call this before constructing anything that builds a rustls config — +/// a `reqwest` client (the workspace enables reqwest's +/// `rustls-no-provider` feature, which never auto-selects a provider) or +/// a TLS-enabled tonic server. rustls's own auto-selection is also +/// unreliable here: it works only when exactly one of rustls's `ring` / +/// `aws-lc-rs` features is enabled, and dependency feature-unification +/// can silently enable both, which panicked at runtime on TLS-enabled +/// deployments (zingolabs/zaino#1360). Installing explicitly removes +/// that dependence on the feature graph. +/// +/// The process default is first-install-wins and is not constrained by +/// our crates' rustls features, so an embedder (e.g. zallet) that has +/// already installed a provider keeps it: zaino then handshakes through +/// that provider instead of ring, which is fine for any provider +/// implementing the standard TLS suites (ring and aws-lc-rs both do). +pub fn ensure_default_crypto_provider() { + if rustls::crypto::CryptoProvider::get_default().is_none() { + // A racing concurrent install is the only error case; either + // provider serves. + let _ = rustls::crypto::ring::default_provider().install_default(); + } +} diff --git a/packages/zaino-common/src/lib.rs b/packages/zaino-common/src/lib.rs index e3f62182c..d2c5fcce5 100644 --- a/packages/zaino-common/src/lib.rs +++ b/packages/zaino-common/src/lib.rs @@ -5,6 +5,7 @@ pub mod config; pub mod consensus; +pub mod crypto; pub mod logging; pub mod net; pub mod probing; diff --git a/packages/zaino-fetch/src/jsonrpsee/connector.rs b/packages/zaino-fetch/src/jsonrpsee/connector.rs index cc8090150..056821afa 100644 --- a/packages/zaino-fetch/src/jsonrpsee/connector.rs +++ b/packages/zaino-fetch/src/jsonrpsee/connector.rs @@ -210,6 +210,29 @@ fn record_outbound_rpc_metrics(method: &'static str, start: std::time::Instant, } } +/// Whether the outbound JSON-RPC client keeps a cookie store (required +/// for cookie-authenticated validators). +enum CookieSupport { + Enabled, + Disabled, +} + +/// Builds the outbound JSON-RPC HTTP client shared by every connector +/// entry point: 2s connect timeout, 5s request timeout, no redirects. +fn build_rpc_client(cookies: CookieSupport) -> Result { + // reqwest is built with `rustls-no-provider`, which never + // auto-selects a rustls CryptoProvider (zingolabs/zaino#1360). + zaino_common::crypto::ensure_default_crypto_provider(); + let mut builder = ClientBuilder::new() + .connect_timeout(Duration::from_secs(2)) + .timeout(Duration::from_secs(5)) + .redirect(reqwest::redirect::Policy::none()); + if matches!(cookies, CookieSupport::Enabled) { + builder = builder.cookie_store(true); + } + builder.build() +} + impl JsonRpSeeConnector { /// Creates a new JsonRpSeeConnector with Basic Authentication. pub fn new_with_basic_auth( @@ -217,12 +240,8 @@ impl JsonRpSeeConnector { username: String, password: String, ) -> Result { - let client = ClientBuilder::new() - .connect_timeout(Duration::from_secs(2)) - .timeout(Duration::from_secs(5)) - .redirect(reqwest::redirect::Policy::none()) - .build() - .map_err(TransportError::ReqwestError)?; + let client = + build_rpc_client(CookieSupport::Disabled).map_err(TransportError::ReqwestError)?; Ok(Self { url, @@ -236,13 +255,8 @@ impl JsonRpSeeConnector { pub fn new_with_cookie_auth(url: Url, cookie_path: &Path) -> Result { let cookie_password = read_and_parse_cookie_token(cookie_path)?; - let client = ClientBuilder::new() - .connect_timeout(Duration::from_secs(2)) - .timeout(Duration::from_secs(5)) - .redirect(reqwest::redirect::Policy::none()) - .cookie_store(true) - .build() - .map_err(TransportError::ReqwestError)?; + let client = + build_rpc_client(CookieSupport::Enabled).map_err(TransportError::ReqwestError)?; Ok(Self { url, @@ -979,12 +993,8 @@ async fn test_node_connection( url: Url, auth_method: AuthMethod, ) -> Result<(), TestNodeConnectionError> { - let client = Client::builder() - .connect_timeout(std::time::Duration::from_secs(2)) - .timeout(std::time::Duration::from_secs(5)) - .redirect(reqwest::redirect::Policy::none()) - .build() - .map_err(TestNodeConnectionError::ClientBuild)?; + let client = + build_rpc_client(CookieSupport::Disabled).map_err(TestNodeConnectionError::ClientBuild)?; let request_body = r#"{"jsonrpc":"2.0","method":"getinfo","params":[],"id":1}"#; let mut request_builder = client diff --git a/packages/zaino-serve/Cargo.toml b/packages/zaino-serve/Cargo.toml index b5cee4792..cef85e961 100644 --- a/packages/zaino-serve/Cargo.toml +++ b/packages/zaino-serve/Cargo.toml @@ -53,9 +53,10 @@ metrics = { workspace = true, optional = true } # Miscellaneous Workspace tokio = { workspace = true, features = ["full"] } -tonic = { workspace = true, features = ["tls-native-roots"] } +tonic = { workspace = true, features = ["tls-native-roots", "tls-ring"] } thiserror = { workspace = true } futures = { workspace = true } jsonrpsee = { workspace = true, features = ["server", "macros"] } serde = { workspace = true, features = ["derive"] } tower = { workspace = true } +zaino-common.workspace = true diff --git a/packages/zaino-serve/src/server/grpc.rs b/packages/zaino-serve/src/server/grpc.rs index 7d77b9ad5..e0854aea1 100644 --- a/packages/zaino-serve/src/server/grpc.rs +++ b/packages/zaino-serve/src/server/grpc.rs @@ -74,6 +74,9 @@ impl TonicServer { let mut server_builder = Server::builder(); if let Some(tls_config) = server_config.get_valid_tls().await? { + // Building the TLS acceptor requires a process-level rustls + // CryptoProvider (zingolabs/zaino#1360). + zaino_common::crypto::ensure_default_crypto_provider(); server_builder = server_builder.tls_config(tls_config).map_err(|e| { ServerError::ServerConfigError(format!("TLS configuration error: {e}")) })?; diff --git a/packages/zaino-serve/src/server/grpc/tests.rs b/packages/zaino-serve/src/server/grpc/tests.rs index 05eb1db71..946cbaa9a 100644 --- a/packages/zaino-serve/src/server/grpc/tests.rs +++ b/packages/zaino-serve/src/server/grpc/tests.rs @@ -1,3 +1,4 @@ //! Zaino-Serve gRPC server unit tests. mod spawn; +mod tls; diff --git a/packages/zaino-serve/src/server/grpc/tests/fixtures/tls_test_ca_cert.pem b/packages/zaino-serve/src/server/grpc/tests/fixtures/tls_test_ca_cert.pem new file mode 100644 index 000000000..8c6998f49 --- /dev/null +++ b/packages/zaino-serve/src/server/grpc/tests/fixtures/tls_test_ca_cert.pem @@ -0,0 +1,12 @@ +-----BEGIN CERTIFICATE----- +MIIBzzCCAXWgAwIBAgIUKx27V49MTw1PXcKunTwIjHzXWuswCgYIKoZIzj0EAwIw +PDE6MDgGA1UEAwwxemFpbm8tc2VydmUgVExTIHJlZ3Jlc3Npb24tdGVzdCBDQSAo +Tk9UIEEgU0VDUkVUKTAgFw0yNjA3MDQwMTQ0MTNaGA8yMTI2MDYxMDAxNDQxM1ow +PDE6MDgGA1UEAwwxemFpbm8tc2VydmUgVExTIHJlZ3Jlc3Npb24tdGVzdCBDQSAo +Tk9UIEEgU0VDUkVUKTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABGT74i0fh1Vj +gYb0rPW36IYlyvvO/SdT0Mf8dJiFnKmoyqcXtORcw8antcjqe0myghHayx6xEOte +6blNT72knn+jUzBRMB0GA1UdDgQWBBSJ4axoCJxNd1mPygmrBgTI8IosCDAfBgNV +HSMEGDAWgBSJ4axoCJxNd1mPygmrBgTI8IosCDAPBgNVHRMBAf8EBTADAQH/MAoG +CCqGSM49BAMCA0gAMEUCIGhNgBQ5G/pars7fOccRedyOegikzDlMbdM8g4NIDErg +AiEAjlmMXJTBDW5Y/E1rLZgcC/dEdLodnTet3vatf6Vq1lM= +-----END CERTIFICATE----- diff --git a/packages/zaino-serve/src/server/grpc/tests/fixtures/tls_test_cert.pem b/packages/zaino-serve/src/server/grpc/tests/fixtures/tls_test_cert.pem new file mode 100644 index 000000000..f78071a86 --- /dev/null +++ b/packages/zaino-serve/src/server/grpc/tests/fixtures/tls_test_cert.pem @@ -0,0 +1,12 @@ +-----BEGIN CERTIFICATE----- +MIIB1TCCAXqgAwIBAgIBATAKBggqhkjOPQQDAjA8MTowOAYDVQQDDDF6YWluby1z +ZXJ2ZSBUTFMgcmVncmVzc2lvbi10ZXN0IENBIChOT1QgQSBTRUNSRVQpMCAXDTI2 +MDcwNDAxNDQxM1oYDzIxMjYwNjEwMDE0NDEzWjAUMRIwEAYDVQQDDAlsb2NhbGhv +c3QwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAASvy0yCNcEp5I28O0EOd7bJOZ2f +tTYE3E1AHSaMFmiiN/Foxw5KZzF7tRDFZxl0CANx+o+igLjQaerNk/bhoZepo4GS +MIGPMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATAMBgNVHRMBAf8EAjAAMA4G +A1UdDwEB/wQEAwIHgDATBgNVHSUEDDAKBggrBgEFBQcDATAdBgNVHQ4EFgQUtHTz +QTMs8vfbh3m4IVJIah8jAt8wHwYDVR0jBBgwFoAUieGsaAicTXdZj8oJqwYEyPCK +LAgwCgYIKoZIzj0EAwIDSQAwRgIhAIIkUaCFwsFMyAsZOasP/fVQSiU8C5jU8gmt +1xmx/2XvAiEApJiQJ1Kxdd/vjpJ5H1JYktPwoqUpZDmDKJMF1O7zMDM= +-----END CERTIFICATE----- diff --git a/packages/zaino-serve/src/server/grpc/tests/fixtures/tls_test_key.pem b/packages/zaino-serve/src/server/grpc/tests/fixtures/tls_test_key.pem new file mode 100644 index 000000000..290fd34eb --- /dev/null +++ b/packages/zaino-serve/src/server/grpc/tests/fixtures/tls_test_key.pem @@ -0,0 +1,5 @@ +-----BEGIN PRIVATE KEY----- +MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQg6+O2+tTz29iMo5yr +SS8d+lb3nG8G7x3s4bHfvC52G3KhRANCAASvy0yCNcEp5I28O0EOd7bJOZ2ftTYE +3E1AHSaMFmiiN/Foxw5KZzF7tRDFZxl0CANx+o+igLjQaerNk/bhoZep +-----END PRIVATE KEY----- diff --git a/packages/zaino-serve/src/server/grpc/tests/tls.rs b/packages/zaino-serve/src/server/grpc/tests/tls.rs new file mode 100644 index 000000000..02b8287ad --- /dev/null +++ b/packages/zaino-serve/src/server/grpc/tests/tls.rs @@ -0,0 +1,82 @@ +//! Regression test for zingolabs/zaino#1360. +//! +//! Spawning the gRPC server with TLS enabled must succeed and serve a +//! working TLS handshake. In 0.5.0, dependency feature-unification +//! enabled both of rustls's `ring` and `aws-lc-rs` provider features, +//! which defeats rustls's automatic provider selection: building the +//! TLS acceptor panicked at runtime ("Could not automatically determine +//! the process-level CryptoProvider") — but only on TLS-enabled +//! deployments, a path no test exercised because every existing test +//! ran plaintext. The fix pins the feature graph to `ring` and installs +//! the provider explicitly (`ensure_crypto_provider`); this test covers +//! the full TLS startup + handshake path so a future provider-selection +//! or certificate-wiring regression fails in CI instead of production. +//! +//! The certificate fixtures were minted once with openssl (EC P-256, +//! ~100-year expiry): a self-signed test CA plus a `localhost` leaf +//! (SAN `DNS:localhost` + `IP:127.0.0.1`) signed by it — rustls-webpki +//! rejects a single self-signed CA cert presented as the server's +//! end-entity cert (`CaUsedAsEndEntity`). The committed leaf key is not +//! a secret; the CA key was discarded, so regeneration remints all +//! three files. The server reads the leaf pair from disk (the +//! production `GrpcTls` path); the client trusts the CA as its root. + +use std::net::{Ipv4Addr, SocketAddr}; +use std::path::PathBuf; + +use tonic::service::Routes; +use tonic::transport::{server::TcpIncoming, Certificate, ClientTlsConfig, Endpoint}; + +use crate::server::config::{GrpcServerConfig, GrpcTls}; +use crate::server::grpc::TonicServer; + +fn fixture_path(name: &str) -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("src/server/grpc/tests/fixtures") + .join(name) +} + +#[tokio::test] +async fn tls_spawn_serves_a_handshake() { + // Pre-bind so the OS-assigned port is known before spawning; hand the + // open socket to `spawn_inner` directly (this module is a child of + // `grpc`, so the private constructor is reachable without the + // `test_dependencies`-gated `spawn_from_listener` wrapper). + let listener = tokio::net::TcpListener::bind(SocketAddr::from((Ipv4Addr::LOCALHOST, 0))) + .await + .expect("bind on ephemeral port must succeed"); + let local_addr = listener + .local_addr() + .expect("local_addr on a bound listener is infallible"); + + let mut server = TonicServer::spawn_inner( + Routes::default(), + GrpcServerConfig { + listen_address: local_addr, + tls: Some(GrpcTls { + cert_path: fixture_path("tls_test_cert.pem"), + key_path: fixture_path("tls_test_key.pem"), + }), + }, + TcpIncoming::from(listener), + ) + .await + .expect("TLS-enabled spawn must succeed (zingolabs/zaino#1360)"); + + let channel = Endpoint::try_from(format!("https://127.0.0.1:{}", local_addr.port())) + .expect("loopback endpoint URI is valid") + .tls_config( + ClientTlsConfig::new() + .ca_certificate(Certificate::from_pem(include_str!( + "fixtures/tls_test_ca_cert.pem" + ))) + .domain_name("localhost"), + ) + .expect("client TLS config from the fixture cert must be valid") + .connect() + .await + .expect("TLS handshake against the spawned server must succeed"); + + drop(channel); + server.close().await; +} From 13163ea0e9d40d74f15648a755797d5295b14a3e Mon Sep 17 00:00:00 2001 From: zancas Date: Fri, 3 Jul 2026 19:53:08 -0700 Subject: [PATCH 091/134] build: drop unnecessary rustls "std" feature from zaino-common MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two functions zaino-common calls — CryptoProvider::install_default and get_default — have no-std implementations in rustls 0.23 (the process default falls back from std::sync::OnceLock to once_cell::race::OnceBox), so "std" is not required by our usage. It is also enabled in any full build regardless, because tokio-rustls hard-enables rustls/std. Minimum-necessary-scope: declare only "ring". Co-Authored-By: Claude Fable 5 --- packages/zaino-common/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/zaino-common/Cargo.toml b/packages/zaino-common/Cargo.toml index 3ce75c1d6..437927085 100644 --- a/packages/zaino-common/Cargo.toml +++ b/packages/zaino-common/Cargo.toml @@ -25,4 +25,4 @@ tracing-tree = { workspace = true } nu-ansi-term = { workspace = true } hex = { workspace = true } time = { workspace = true } -rustls = { version = "0.23", default-features = false, features = ["ring", "std"] } +rustls = { version = "0.23", default-features = false, features = ["ring"] } From 30681b93ed4882433f734ea67b63fe0014f3b495 Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 10:33:23 -0700 Subject: [PATCH 092/134] ci: report advisory publish dry-run failures as warnings, not a red X continue-on-error makes the run succeed but still shows the failed job as a red X in the PR checks list, which reads as a blocking failure. Move the advisory decision into the step: on non-rc/non-stable contexts a failure emits a ::warning annotation and exits 0; rc/** and stable keep the hard failure. Also generalize the failure epilogue: same-name version divergence from crates.io (new cross-crate API or features, resolved by release-time bumps) is as common a cause as patch overrides. Co-Authored-By: Claude Fable 5 --- .github/workflows/publish-dry-run.yml | 37 ++++++++++++--------------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/.github/workflows/publish-dry-run.yml b/.github/workflows/publish-dry-run.yml index 403de3465..8add6d556 100644 --- a/.github/workflows/publish-dry-run.yml +++ b/.github/workflows/publish-dry-run.yml @@ -14,19 +14,12 @@ jobs: name: cargo publish --dry-run runs-on: ubuntu-latest # Blocking context (pushes to rc/** or stable, and PRs targeting them): - # this job MUST pass. Advisory context (everything else): findings inform - # but the run stays green. Keep this predicate the exact inverse of the - # "Compute check mode" step below — job-level continue-on-error cannot - # read step outputs, so the two encode the same rule twice. - continue-on-error: >- - ${{ - github.event_name == 'pull_request' - && !startsWith(github.base_ref, 'rc/') - && github.base_ref != 'stable' - || github.event_name != 'pull_request' - && !startsWith(github.ref_name, 'rc/') - && github.ref_name != 'stable' - }} + # this job MUST pass. Advisory context (everything else): failures exit 0 + # after emitting a warning annotation, so they do not paint a red X on + # PRs (a continue-on-error'd failing job still shows as failed in the PR + # checks list even though the run succeeds — hence no job-level + # continue-on-error here). The context is computed once in the + # "Compute check mode" step below. steps: - name: Checkout repository uses: actions/checkout@v7 @@ -37,7 +30,6 @@ jobs: - name: Compute check mode id: mode run: | - # Inverse of the job's continue-on-error predicate above. if [[ "${{ github.event_name }}" == "pull_request" ]]; then target="${{ github.base_ref }}" else @@ -96,13 +88,18 @@ jobs: echo "============================================" echo "" echo "The workspace does not publish cleanly as a set. Common causes:" - echo " - a [patch.crates-io] override pulling in unreleased upstream code" + echo " - a [patch.crates-io] override pulling in unreleased upstream" + echo " code (published crates cannot depend on unpublished revisions)" echo " - a dependency on a git revision or path outside the workspace" echo " - a crate that does not compile from its packaged form" + echo " - new cross-crate API or features that exist in the workspace but" + echo " not in the published dependency; resolved by the version bump +" + echo " publish of that dependency at the next release" + echo "" + if [[ "${{ steps.mode.outputs.mode }}" == "advisory" ]]; then + echo "::warning::publish dry-run failed — the workspace would not release cleanly. Advisory on this branch, but it must be resolved before the next release." + exit 0 + fi + echo "::error::publish dry-run failed — the workspace would not release cleanly." exit 1 fi - - - name: Annotate on failure (advisory context only) - if: failure() && steps.mode.outputs.mode == 'advisory' - run: | - echo "::warning::publish dry-run failed — the workspace would not release cleanly. This must be resolved before the next release." From 90522a2e4154e5481fc6ea4383b126b99b70cc47 Mon Sep 17 00:00:00 2001 From: zancas Date: Sun, 5 Jul 2026 15:43:04 -0700 Subject: [PATCH 093/134] =?UTF-8?q?docs:=20ADR-0006=20=E2=80=94=20prefer?= =?UTF-8?q?=20aws-lc-rs=20CryptoProvider;=20start=20CONTEXT.md=20glossary?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Record the decision reversing zaino's rustls provider from ring to aws-lc-rs, per Daira Emma's post-quantum argument: - aws-lc-rs is the preferred (first-install-wins, not mandated) provider - rustls prefer-post-quantum enabled: X25519MLKEM768 leads on the client leg; rustls servers follow client group preference, so inbound hybrid uptake is client-driven - classical key exchange accepted but deprecating, gated on wallet stacks (zingolib is ring-pinned; ZODL rides mobile platform TLS) - ring tolerated in Cargo.lock via the zebra-node-services reqwest path, fenced by an exact-snapshot CI lint, until the upstream zebra fix References: - https://crates.io/crates/rustls-post-quantum - https://www.reddit.com/r/rust/comments/1de13y6/comment/l89pbmc/ CONTEXT.md begins the project glossary: Preferred CryptoProvider, Hybrid key exchange, Classical key exchange. Co-Authored-By: Claude Fable 5 --- CONTEXT.md | 24 +++++ ...006-aws-lc-rs-preferred-crypto-provider.md | 90 +++++++++++++++++++ 2 files changed, 114 insertions(+) create mode 100644 docs/adr/0006-aws-lc-rs-preferred-crypto-provider.md diff --git a/CONTEXT.md b/CONTEXT.md index 4d40dd72f..11abf3640 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -29,3 +29,27 @@ its packaged content differs. The tree cannot be released until that crate's version is bumped. An unchanged crate keeping its published version is not a violation. _Avoid_: stale version, forgotten bump + +### TLS and cryptography + +**Preferred CryptoProvider**: +The TLS cryptography provider zaino installs as the process-wide default +when no provider is installed yet. A preference, not a mandate: an +embedder (e.g. zallet) that installs a provider before zaino keeps its +choice, and zaino handshakes through it. +_Avoid_: "enforced provider" (implies zaino overrides an embedder's +already-installed provider; it never does) + +**Hybrid key exchange**: +A TLS 1.3 key-exchange group combining a classical curve with a +post-quantum KEM (key-encapsulation mechanism), e.g. X25519MLKEM768. +Secure if either component holds. Hybrids are the post-quantum +deployment vehicle, not a classical fallback. +_Avoid_: calling hybrids "classical" because they contain a classical +component + +**Classical key exchange**: +A key-exchange group with no post-quantum component (X25519, SECP256R1, +SECP384R1). Deprecated in zaino: still accepted for client +compatibility, slated for refusal once major wallet stacks negotiate +hybrids. diff --git a/docs/adr/0006-aws-lc-rs-preferred-crypto-provider.md b/docs/adr/0006-aws-lc-rs-preferred-crypto-provider.md new file mode 100644 index 000000000..13306503c --- /dev/null +++ b/docs/adr/0006-aws-lc-rs-preferred-crypto-provider.md @@ -0,0 +1,90 @@ +# aws-lc-rs is the preferred CryptoProvider; classical key exchange is deprecating + +## Status + +accepted (reverses the ring choice made on zingolabs/zaino#1366 before it +merged) + +## Context and decision + +Zaino installs a rustls `CryptoProvider` as the process-wide default at +its two TLS boundaries (`zaino-serve`'s gRPC server, `zaino-fetch`'s +jsonrpsee connector) via `zaino_common::crypto::ensure_default_crypto_provider` +(zingolabs/zaino#1360). The provider was initially pinned to **ring** for +two reasons: zebra used ring (shared attack surface across the deployed +stack), and ring avoided the `aws-lc-sys` + `cmake` build dependencies. + +Daira Emma argued for **aws-lc-rs** instead: post-quantum TLS key +exchange in the rustls ecosystem requires the aws-lc-rs provider — the +ring provider has no ML-KEM support at all — and the provider +architecture makes the swap cheap +(, +). + +We decide: + +1. **aws-lc-rs is zaino's preferred CryptoProvider** — a *preference, + not a mandate*: installation stays first-install-wins, so an embedder + (e.g. zallet) that installs a provider before zaino keeps its choice. + Enforcement lives in the feature graph (`rustls` `aws_lc_rs` feature, + tonic `tls-aws-lc`), not at runtime. +2. **rustls's `prefer-post-quantum` feature is enabled**, reordering the + provider's default key-exchange groups so the hybrid X25519MLKEM768 + leads (same membership; in rustls 0.23 the hybrid is in the default + set either way). Verified effect in rustls 0.23.41: on zaino's + *outbound* (client-role) TLS the hybrid share is offered upfront, and + rustls adds the hybrid's X25519 component as a free second key share, + so classical-only servers negotiate without a HelloRetryRequest — the + cost is ~1.2 KB of ClientHello, not a round trip. On the *server* + side rustls follows the client's group preference, so inbound + negotiation outcomes are unchanged by the feature; hybrid uptake on + inbound connections is entirely client-driven, which is why the + deprecation below is gated on client stacks. +3. **Classical key exchange (X25519, SECP256R1, SECP384R1) remains + accepted but is deprecated.** Refusing it today would break both + flagship wallets: zingolib's TLS stack is feature-pinned to ring (no + ML-KEM possible), and ZODL rides mobile platform TLS where hybrid + support is still rolling out. The refusal precondition — zingolib on + aws-lc-rs with hybrid preferred, and ZODL's Android/iOS platform + stacks negotiating X25519MLKEM768 — is tracked in a dedicated issue; + deprecation is documented in the changelog rather than signalled at + runtime. +4. **Ring stays in `Cargo.lock`, tolerated and fenced, until zebra drops + it.** `zebra-rpc` hard-enables `zebra-node-services/rpc-client`, + whose reqwest 0.12 `rustls-tls` feature compiles ring in — reqwest + 0.12 offers no aws-lc alternative, only `*-no-provider` variants. This + is dormant code, not a live handshake path: reqwest consults the + process-default provider first and falls back to ring only when none + is installed, so both zaino (which installs aws-lc-rs early) and + embedders (whose earlier install wins) preempt it. A CI lint pins + `cargo tree --invert ring` to exactly this one known path and fails on + drift in *either* direction: a new ring consumer is a regression to + investigate; the path vanishing means zebra dropped ring, the + allowlist gets deleted, and the guard tightens to "ring absent". + +## Considered options + +- **Stay on ring, aligned with zebra.** Rejected: forecloses post-quantum + key exchange entirely; the harvest-now-decrypt-later argument applies + to wallet↔zaino TLS traffic today. +- **Refuse classical key exchange (hybrid-only).** Rejected for now: + hard-locks out every wallet whose TLS stack lacks X25519MLKEM768, + which today is all of them. Revisit when the tracked precondition is + met. +- **Fork-pin zebra to purge ring from the lock immediately.** Rejected: + the fork tax (pin churn here and in the wallet-tests workspace) buys + only the removal of dormant code; downstream is already protected by + provider preemption. Instead we upstream the fix to zebra — + `zebra-node-services` switches reqwest to a `*-no-provider` feature + and zebrad installs its own default provider (the same pattern as + zaino's #1360 fix) — and pick it up at the next zebra bump. + +## Consequences + +- `aws-lc-sys` and `cmake` return to the build graph (both build images + already install cmake). +- Zaino diverges from zebra's provider until zebra moves; the CI lint + makes the eventual convergence loud instead of silent. +- TLS 1.2 remains available only as long as classical key exchange + does; the refusal that ends the classical deprecation also implies + TLS 1.3-only. From a62dcd3bbe9ce6730fd50bb4ae1fcf6f833b1c0c Mon Sep 17 00:00:00 2001 From: zancas Date: Sun, 5 Jul 2026 16:18:28 -0700 Subject: [PATCH 094/134] build: prefer aws-lc-rs CryptoProvider with post-quantum key exchange MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Switch zaino's rustls provider from ring to aws-lc-rs and enable rustls's prefer-post-quantum feature, so the X25519MLKEM768 hybrid key exchange leads zaino's outbound handshakes (rustls servers follow the client's group preference, so inbound hybrid uptake is client-driven). Installation stays first-install-wins: an embedder that installs a provider before zaino keeps its choice. Full rationale in ADR-0006. - zaino-common: rustls features ring → aws_lc_rs + prefer-post-quantum; ensure_default_crypto_provider installs aws-lc-rs - zaino-serve: tonic tls-ring → tls-aws-lc - ring remains in Cargo.lock solely via zebra-rpc → zebra-node-services rpc-client → reqwest 0.12 "rustls-tls" (reqwest 0.12 offers no aws-lc alternative); it is compiled but dormant — reqwest consults the process-default provider first. Removal tracks an upstream zebra fix. - new workbench guard check-crypto-provider (makers lint-crypto-provider) pins the ring-selecting feature edges to exactly that tolerated path, failing loud on new edges and on ring's disappearance alike - classical key exchange (X25519, SECP256R1, SECP384R1) is deprecated: still accepted for wallet compatibility, slated for refusal once major wallet TLS stacks negotiate hybrids References: - https://crates.io/crates/rustls-post-quantum - https://www.reddit.com/r/rust/comments/1de13y6/comment/l89pbmc/ Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 11 + Cargo.lock | 46 ++++ packages/zaino-common/CHANGELOG.md | 4 + packages/zaino-common/Cargo.toml | 5 +- packages/zaino-common/src/crypto.rs | 17 +- packages/zaino-serve/CHANGELOG.md | 2 + packages/zaino-serve/Cargo.toml | 2 +- .../zaino-serve/src/server/grpc/tests/tls.rs | 5 +- tools/makefiles/lints.toml | 6 + .../src/bin/check-crypto-provider.rs | 206 ++++++++++++++++++ 10 files changed, 296 insertions(+), 8 deletions(-) create mode 100644 tools/workbench/src/bin/check-crypto-provider.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 621a927ce..dba661008 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,17 @@ and this library adheres to Rust's notion of ## Unreleased ### Changed +- TLS: zaino now installs rustls's **aws-lc-rs** CryptoProvider as its + preferred process-level default (was ring) and enables rustls's + `prefer-post-quantum` feature, so the X25519MLKEM768 hybrid key exchange + leads zaino's outbound handshakes (ADR-0006). Installation remains + first-install-wins: an embedder that installs a provider before zaino + keeps its choice. + +### Deprecated +- Classical TLS key exchange (X25519, SECP256R1, SECP384R1) is deprecated: + still offered and accepted for wallet compatibility, slated for refusal + once major wallet stacks negotiate hybrid key exchange (ADR-0006). - **Breaking** — config: `storage.database.sync_write_batch_bytes` (bytes) is renamed to `sync_write_batch_size` and given in **GiB** (default raised from 4 GiB to 32 GiB); this budget now also bounds the txout-set accumulator diff --git a/Cargo.lock b/Cargo.lock index 77ba5a672..5b4174254 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -171,6 +171,29 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "aws-lc-rs" +version = "1.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4342d8937fc7e5dd9b1c60292261c0670c882a2cd1719cfc11b1af41731e32ad" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d9ceb1da931507a12f4fccea479dccd00da1943e1b4ae72d8e502d707361444" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", + "pkg-config", +] + [[package]] name = "axum" version = "0.8.9" @@ -688,6 +711,15 @@ dependencies = [ "zip32", ] +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + [[package]] name = "cobs" version = "0.3.0" @@ -1159,6 +1191,12 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "dyn-clone" version = "1.0.20" @@ -1430,6 +1468,12 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "funty" version = "2.0.0" @@ -3958,6 +4002,7 @@ version = "0.23.41" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" dependencies = [ + "aws-lc-rs", "log", "once_cell", "ring", @@ -4022,6 +4067,7 @@ version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", diff --git a/packages/zaino-common/CHANGELOG.md b/packages/zaino-common/CHANGELOG.md index e2356e551..42890004b 100644 --- a/packages/zaino-common/CHANGELOG.md +++ b/packages/zaino-common/CHANGELOG.md @@ -21,6 +21,10 @@ and this library adheres to Rust's notion of `sync_write_batch_size` so the two operations cannot inflate each other's peak memory. ### Changed +- `crypto::ensure_default_crypto_provider` now installs rustls's + **aws-lc-rs** provider (was ring) as the process-level default, and the + crate's rustls features become `aws_lc_rs` + `prefer-post-quantum` + (ADR-0006). First-install-wins semantics are unchanged. - **Breaking** — `StorageConfig::database.sync_write_batch_bytes` (raw bytes) is renamed to `sync_write_batch_size` and now expressed in **GiB** (new `SyncWriteBatchSize` newtype, mirroring `DatabaseSize`); the default is 8 GiB. diff --git a/packages/zaino-common/Cargo.toml b/packages/zaino-common/Cargo.toml index 437927085..6c368e5ff 100644 --- a/packages/zaino-common/Cargo.toml +++ b/packages/zaino-common/Cargo.toml @@ -25,4 +25,7 @@ tracing-tree = { workspace = true } nu-ansi-term = { workspace = true } hex = { workspace = true } time = { workspace = true } -rustls = { version = "0.23", default-features = false, features = ["ring"] } +rustls = { version = "0.23", default-features = false, features = [ + "aws_lc_rs", + "prefer-post-quantum", +] } diff --git a/packages/zaino-common/src/crypto.rs b/packages/zaino-common/src/crypto.rs index 8791bbae2..c24dc59d9 100644 --- a/packages/zaino-common/src/crypto.rs +++ b/packages/zaino-common/src/crypto.rs @@ -1,7 +1,7 @@ //! Process-level rustls `CryptoProvider` management. -/// Installs rustls's `ring` provider as the process-level default if no -/// provider is installed yet. +/// Installs rustls's `aws-lc-rs` provider as the process-level default +/// if no provider is installed yet. /// /// Call this before constructing anything that builds a rustls config — /// a `reqwest` client (the workspace enables reqwest's @@ -13,15 +13,24 @@ /// deployments (zingolabs/zaino#1360). Installing explicitly removes /// that dependence on the feature graph. /// +/// aws-lc-rs is the preferred provider (ADR-0006): it is the only rustls +/// provider with post-quantum key exchange, and the workspace enables +/// rustls's `prefer-post-quantum` feature so the X25519MLKEM768 hybrid +/// group leads this provider's defaults. That ordering governs zaino's +/// outbound (client-role) handshakes; rustls servers follow the client's +/// group preference, so inbound hybrid uptake is decided by the +/// connecting wallet's TLS stack. Classical key-exchange groups remain +/// offered and accepted, but are deprecated (see the ADR). +/// /// The process default is first-install-wins and is not constrained by /// our crates' rustls features, so an embedder (e.g. zallet) that has /// already installed a provider keeps it: zaino then handshakes through -/// that provider instead of ring, which is fine for any provider +/// that provider instead of aws-lc-rs, which is fine for any provider /// implementing the standard TLS suites (ring and aws-lc-rs both do). pub fn ensure_default_crypto_provider() { if rustls::crypto::CryptoProvider::get_default().is_none() { // A racing concurrent install is the only error case; either // provider serves. - let _ = rustls::crypto::ring::default_provider().install_default(); + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); } } diff --git a/packages/zaino-serve/CHANGELOG.md b/packages/zaino-serve/CHANGELOG.md index 0df7dc8c7..8a6296e20 100644 --- a/packages/zaino-serve/CHANGELOG.md +++ b/packages/zaino-serve/CHANGELOG.md @@ -9,6 +9,8 @@ and this library adheres to Rust's notion of ### Added ### Changed +- tonic's TLS provider feature switches from `tls-ring` to `tls-aws-lc`, + following the workspace's aws-lc-rs preferred CryptoProvider (ADR-0006). ### Deprecated ### Removed ### Fixed diff --git a/packages/zaino-serve/Cargo.toml b/packages/zaino-serve/Cargo.toml index cef85e961..c421c8a4e 100644 --- a/packages/zaino-serve/Cargo.toml +++ b/packages/zaino-serve/Cargo.toml @@ -53,7 +53,7 @@ metrics = { workspace = true, optional = true } # Miscellaneous Workspace tokio = { workspace = true, features = ["full"] } -tonic = { workspace = true, features = ["tls-native-roots", "tls-ring"] } +tonic = { workspace = true, features = ["tls-native-roots", "tls-aws-lc"] } thiserror = { workspace = true } futures = { workspace = true } jsonrpsee = { workspace = true, features = ["server", "macros"] } diff --git a/packages/zaino-serve/src/server/grpc/tests/tls.rs b/packages/zaino-serve/src/server/grpc/tests/tls.rs index 02b8287ad..9b60a7567 100644 --- a/packages/zaino-serve/src/server/grpc/tests/tls.rs +++ b/packages/zaino-serve/src/server/grpc/tests/tls.rs @@ -7,8 +7,9 @@ //! TLS acceptor panicked at runtime ("Could not automatically determine //! the process-level CryptoProvider") — but only on TLS-enabled //! deployments, a path no test exercised because every existing test -//! ran plaintext. The fix pins the feature graph to `ring` and installs -//! the provider explicitly (`ensure_crypto_provider`); this test covers +//! ran plaintext. The fix pins the feature graph to a single provider — +//! `aws-lc-rs`, the preferred provider per ADR-0006 — and installs it +//! explicitly (`ensure_default_crypto_provider`); this test covers //! the full TLS startup + handshake path so a future provider-selection //! or certificate-wiring regression fails in CI instead of production. //! diff --git a/tools/makefiles/lints.toml b/tools/makefiles/lints.toml index cac50b8d3..e6824f01f 100644 --- a/tools/makefiles/lints.toml +++ b/tools/makefiles/lints.toml @@ -82,6 +82,11 @@ description = "Assert Dockerfile.deterministic's stagex/pallet-rust tag matches command = "cargo" args = ["run", "-q", "--manifest-path", "tools/workbench/Cargo.toml", "--bin", "check-toolchain-pin"] +[tasks.lint-crypto-provider] +description = "Assert rustls resolves with aws-lc-rs + prefer-post-quantum and that the only ring-selecting feature edges are the tolerated zebra path (ADR-0006). Fails loud in both directions: new edges and vanished edges." +command = "cargo" +args = ["run", "-q", "--manifest-path", "tools/workbench/Cargo.toml", "--bin", "check-crypto-provider"] + [tasks.lint-no-openssl-apt] description = "Fail if any Dockerfile installs a libssl*/openssl* apt package. TLS is rustls; the system-package companion to deny.toml's openssl crate ban." command = "cargo" @@ -106,6 +111,7 @@ dependencies = [ "lint-boundary-conversions", "lint-workbench", "lint-toolchain-pin", + "lint-crypto-provider", "lint-no-openssl-apt", "deny", "lint-shell", diff --git a/tools/workbench/src/bin/check-crypto-provider.rs b/tools/workbench/src/bin/check-crypto-provider.rs new file mode 100644 index 000000000..13c62a881 --- /dev/null +++ b/tools/workbench/src/bin/check-crypto-provider.rs @@ -0,0 +1,206 @@ +//! Guard: one rustls CryptoProvider — aws-lc-rs — plus the single tolerated +//! ring path through zebra (ADR-0006). +//! +//! Two checks against the production workspace's resolved dependency graph: +//! +//! 1. rustls must resolve with `aws_lc_rs` and `prefer-post-quantum`. +//! 2. The set of feature edges that select ring anywhere in the graph must +//! equal [`RING_EDGE_ALLOWLIST`] — today, the single chain rooted in +//! zebra-node-services' `rpc-client` feature (its reqwest `rustls-tls`; +//! reqwest 0.12 offers no aws-lc alternative). That ring is compiled but +//! dormant: reqwest consults the process-default provider first and zaino +//! installs aws-lc-rs at both TLS boundaries. +//! +//! Loud in both directions. A NEW edge means a second provider path is +//! creeping back in via feature unification — the regression this guard +//! exists to catch (zingolabs/zaino#1360). A MISSING edge — in particular +//! ring vanishing from the graph entirely — means zebra dropped its ring +//! dependence: delete [`RING_EDGE_ALLOWLIST`], make this check assert ring's +//! absence, and close the classical-deprecation tracking issue's zebra item. + +use std::collections::BTreeSet; +use std::path::Path; +use std::process::Command; + +use workbench::{repo_root, run}; + +/// Feature names that select the ring provider when enabled on any crate. +const RING_SELECTING_FEATURES: [&str; 3] = ["ring", "__rustls-ring", "tls-ring"]; + +/// Every feature edge expected to select ring, all downstream of +/// zebra-node-services' reqwest `rustls-tls` (see module docs). +const RING_EDGE_ALLOWLIST: [&str; 5] = [ + "hyper-rustls feature \"ring\"", + "reqwest feature \"__rustls-ring\"", + "rustls feature \"ring\"", + "rustls-webpki feature \"ring\"", + "tokio-rustls feature \"ring\"", +]; + +fn main() { + run("check-crypto-provider", check, |()| { + println!( + "check-crypto-provider: ok — rustls resolves aws-lc-rs + prefer-post-quantum; \ + ring edges match the tolerated zebra path" + ); + }) +} + +fn check() -> Result<(), Vec> { + let root = repo_root()?; + + let rustls_features = cargo_tree(&root, &["--package", "rustls", "--depth", "0"], "{f}")?; + check_preferred_provider(&rustls_features)?; + + let ring_tree = match cargo_tree(&root, &["--invert", "ring", "--edges", "features"], "{p}") { + Ok(out) => out, + Err(diag) + if diag + .iter() + .any(|l| l.contains("did not match any packages")) => + { + return Err(vec![ + "ring is GONE from the dependency graph — zebra dropped it!".to_string(), + "Tighten this guard: delete RING_EDGE_ALLOWLIST and assert ring's absence," + .to_string(), + "and close the zebra item on the classical-deprecation tracking issue.".to_string(), + ]); + } + Err(diag) => return Err(diag), + }; + check_ring_edges(&ring_tree) +} + +/// Assert the resolved rustls feature set contains the preferred-provider pair. +fn check_preferred_provider(features_line: &str) -> Result<(), Vec> { + let features: BTreeSet<&str> = features_line.trim().split(',').collect(); + let missing: Vec<&str> = ["aws_lc_rs", "prefer-post-quantum"] + .into_iter() + .filter(|f| !features.contains(f)) + .collect(); + if missing.is_empty() { + Ok(()) + } else { + Err(vec![format!( + "rustls resolved without {} (got: {}) — the preferred provider per \ + ADR-0006 is aws-lc-rs with prefer-post-quantum; check \ + packages/zaino-common/Cargo.toml's rustls features", + missing.join(" + "), + features_line.trim(), + )]) + } +} + +/// Assert the ring-selecting feature edges equal the tolerated allowlist. +fn check_ring_edges(ring_tree: &str) -> Result<(), Vec> { + let found = ring_selecting_edges(ring_tree); + let expected: BTreeSet = RING_EDGE_ALLOWLIST + .iter() + .map(ToString::to_string) + .collect(); + + if found == expected { + return Ok(()); + } + + let mut msg = Vec::new(); + for new in found.difference(&expected) { + msg.push(format!( + "NEW ring-selecting feature edge: {new} — a second CryptoProvider path is \ + creeping back in (zingolabs/zaino#1360); remove the enabling feature", + )); + } + for gone in expected.difference(&found) { + msg.push(format!( + "tolerated ring edge disappeared: {gone} — if zebra dropped ring, tighten \ + this guard (see the module docs in check-crypto-provider.rs)", + )); + } + msg.push("see docs/adr/0006-aws-lc-rs-preferred-crypto-provider.md".to_string()); + Err(msg) +} + +/// The deduplicated ` feature ""` edges in a +/// `cargo tree --edges features` output whose feature name selects ring. +fn ring_selecting_edges(tree: &str) -> BTreeSet { + tree.lines() + .map(|line| { + line.trim_start_matches(|c| matches!(c, '│' | '├' | '└' | '─' | ' ')) + .trim_end_matches(" (*)") + }) + .filter(|node| { + node.split_once(" feature ").is_some_and(|(_, feature)| { + RING_SELECTING_FEATURES + .iter() + .any(|f| feature == format!("\"{f}\"")) + }) + }) + .map(ToString::to_string) + .collect() +} + +/// Run `cargo tree --format ` against the production workspace +/// at `root`, returning stdout; diagnostics carry stderr so callers can +/// distinguish "package not in graph" from other failures. +fn cargo_tree(root: &Path, args: &[&str], format: &str) -> Result> { + let output = Command::new("cargo") + .arg("tree") + .args(args) + .args(["--format", format]) + .current_dir(root) + .output() + .map_err(|e| vec![format!("failed to run cargo tree: {e}")])?; + if !output.status.success() { + return Err(vec![format!( + "`cargo tree {}` failed: {}", + args.join(" "), + String::from_utf8_lossy(&output.stderr).trim(), + )]); + } + String::from_utf8(output.stdout).map_err(|e| vec![format!("cargo output not utf-8: {e}")]) +} + +#[cfg(test)] +mod ring_selecting_edges { + use super::*; + + #[test] + fn extracts_and_dedups_ring_edges_from_a_feature_tree() { + let tree = "\ +ring v0.17.14 +├── rustls v0.23.41 +│ ├── rustls feature \"ring\" +│ │ └── reqwest feature \"__rustls-ring\" +│ └── rustls feature \"ring\" (*) +└── rustls-webpki v0.103.13 + └── rustls-webpki feature \"ring\" +"; + let edges = ring_selecting_edges(tree); + assert_eq!( + edges.into_iter().collect::>(), + [ + "reqwest feature \"__rustls-ring\"", + "rustls feature \"ring\"", + "rustls-webpki feature \"ring\"", + ] + ); + } + + #[test] + fn ignores_package_nodes_and_non_ring_features() { + let tree = "\ +rustls v0.23.41 +├── rustls feature \"aws_lc_rs\" +├── reqwest feature \"rustls-tls\" +└── tokio-rustls feature \"logging\" +"; + assert!(ring_selecting_edges(tree).is_empty()); + } + + #[test] + fn matches_feature_names_exactly_not_by_substring() { + // "tls-ring-extra" must not match the "tls-ring" selector. + let tree = "└── tonic feature \"tls-ring-extra\"\n"; + assert!(ring_selecting_edges(tree).is_empty()); + } +} From 94cccea9714fbc0d5e6c08b87b29b9a316b498fa Mon Sep 17 00:00:00 2001 From: zancas Date: Sun, 5 Jul 2026 16:58:44 -0700 Subject: [PATCH 095/134] fix(workbench): satisfy clippy manual_pattern_char_comparison Replace the matches! closure in check-crypto-provider's tree-prefix trimming with the char-array pattern clippy suggests. The lint escaped local verification because a chained `fmt --check && clippy` short- circuited on the fmt diff, so clippy never ran. Co-Authored-By: Claude Fable 5 --- tools/workbench/src/bin/check-crypto-provider.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/workbench/src/bin/check-crypto-provider.rs b/tools/workbench/src/bin/check-crypto-provider.rs index 13c62a881..b3926a1cf 100644 --- a/tools/workbench/src/bin/check-crypto-provider.rs +++ b/tools/workbench/src/bin/check-crypto-provider.rs @@ -125,7 +125,7 @@ fn check_ring_edges(ring_tree: &str) -> Result<(), Vec> { fn ring_selecting_edges(tree: &str) -> BTreeSet { tree.lines() .map(|line| { - line.trim_start_matches(|c| matches!(c, '│' | '├' | '└' | '─' | ' ')) + line.trim_start_matches(['│', '├', '└', '─', ' ']) .trim_end_matches(" (*)") }) .filter(|node| { From 5e4d45e7ddb87e913889cf2b70fdfb27e55da5b4 Mon Sep 17 00:00:00 2001 From: idky137 Date: Mon, 6 Jul 2026 15:58:16 +0100 Subject: [PATCH 096/134] make persistent db validation wait on migration --- .../src/chain_index/finalised_state.rs | 28 +++++++++++++--- .../finalised_state/finalised_source.rs | 12 +++++++ .../finalised_state/finalised_source/v1.rs | 33 ++++++++++++++----- 3 files changed, 60 insertions(+), 13 deletions(-) diff --git a/packages/zaino-state/src/chain_index/finalised_state.rs b/packages/zaino-state/src/chain_index/finalised_state.rs index 11c824ec1..2873e6e49 100644 --- a/packages/zaino-state/src/chain_index/finalised_state.rs +++ b/packages/zaino-state/src/chain_index/finalised_state.rs @@ -543,12 +543,25 @@ impl FinalisedState { source: migration_source, }; - if let Err(error) = migration_manager.migrate().await { - tracing::error!("FinalisedState migration failed: {error}"); + match migration_manager.migrate().await { + Ok(()) => { + // Start the background validator only now that every migration has + // finished: its initial scan reads tables a migration populates (e.g. + // `commitment_tree_data_1_3_0`), so starting it earlier would race the + // migration and fail on a not-yet-written row. + migration_router.primary_backend().start_validator(); + } + Err(error) => { + tracing::error!("FinalisedState migration failed: {error}"); - migration_router.store_primary_status(StatusType::CriticalError); + migration_router.store_primary_status(StatusType::CriticalError); + } } }); + } else { + // No migration to run, so the on-disk tables the validator scans are already at the + // current schema: start it immediately. + router.primary_backend().start_validator(); } Ok(Self { db: router, cfg }) @@ -1053,6 +1066,14 @@ impl FinalisedState { migration_manager.migrate().await?; } + // This test helper builds a fixture at an arbitrary (often intermediate) version for + // inspection, so it deliberately does NOT start the validator: the validator only validates + // against the current schema and would fail on an intermediate-version database. The + // foreground migration is already complete, so mark the primary `Ready` directly (as + // `spawn_v1_0_0` does) to give callers a settled backend. Validation is exercised through + // the production `FinalisedState::spawn` path, which always targets the current schema. + router.store_primary_status(StatusType::Ready); + let metadata = router.get_metadata().await?; if metadata.version() != target_version { return Err(FinalisedStateError::Custom(format!( @@ -1084,7 +1105,6 @@ impl FinalisedState { source: T, ) -> Result, FinalisedStateError> { let db = FinalisedSource::spawn_v1_0_0(cfg).await?; - db.wait_until_ready().await; let tip = source.get_best_block_height().await?.ok_or_else(|| { FinalisedStateError::BlockchainSourceError(BlockchainSourceError::Unrecoverable( diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source.rs index cae4c96e3..e2ce0793e 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source.rs @@ -283,6 +283,18 @@ impl FinalisedSource { } } + /// Start the background validator on the primary v1 backend (no-op for the ephemeral + /// passthrough). + /// + /// Called by the orchestrator only once all pending migrations have completed, so the + /// validator never races a migration that populates the tables its initial scan reads. + pub(crate) fn start_validator(&self) { + match self { + Self::V1(db) => db.start_validator(), + Self::Ephemeral(_) => {} + } + } + /// Stores a new runtime status in the concrete backend. /// /// This is used by router-level background orchestration, for example to report an asynchronous diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1.rs index 73ed8b717..f75eacf21 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1.rs @@ -413,8 +413,15 @@ impl DbV1 { /// - opens or creates all V1 named databases, /// - validates or initializes the `"metadata"` record (schema hash + version), and /// - spawns the background validator / maintenance task. + /// Opens (and heals) the v1 database **without** starting the background validator. + /// + /// The validator is started separately via [`DbV1::start_validator`]. This split exists so the + /// orchestrator can guarantee that any pending schema migration finishes *before* validation + /// runs: the validator's `initial_block_scan` reads tables (e.g. `commitment_tree_data_1_3_0`) + /// that a migration populates, so starting it concurrently with a migration races the migration + /// and can fail on a not-yet-written row. pub(crate) async fn spawn(config: &ChainIndexConfig) -> Result { - let mut zaino_db = Self::open_env_and_dbs(config).await?; + let zaino_db = Self::open_env_and_dbs(config).await?; // Validate (or initialise) the metadata entry before we touch any tables. zaino_db.check_schema_version().await?; @@ -424,9 +431,6 @@ impl DbV1 { // on a quiescent database. zaino_db.reconcile_alpha_txid_location_index().await?; - // Spawn handler task to perform background validation and trailing tx cleanup. - zaino_db.spawn_handler().await?; - Ok(zaino_db) } @@ -598,7 +602,12 @@ impl DbV1 { /// `initial_block_scan`). /// - **Steady state:** periodically attempts to validate the next height after `validated_tip`. /// Separately, it performs periodic trailing-reader cleanup via `clean_trailing()`. - async fn spawn_handler(&mut self) -> Result<(), FinalisedStateError> { + /// + /// Kept separate from [`DbV1::spawn`] so the orchestrator starts it only once all pending + /// migrations have finished (the validator reads tables a migration populates). Takes `&self` + /// (the join handle lives behind a `Mutex`) so it can be driven through the shared + /// `Arc` the router holds after spawn. + pub(crate) fn start_validator(&self) { // Clone everything the task needs so we can move it into the async block. let zaino_db = self.detached_handle(); @@ -700,7 +709,6 @@ impl DbV1 { }); *self.db_handler.lock().expect("db_handler mutex poisoned") = Some(handle); - Ok(()) } /// Validates every stored spent-outpoint entry (`Outpoint` -> `TxLocation`) by checksum. @@ -1016,9 +1024,16 @@ impl DbV1 { // (see the method doc) — that is the behavioural difference from `spawn`. zaino_db.write_v1_0_0_metadata()?; - // Spawn handler task to perform background validation and trailing tx cleanup. - let mut zaino_db = zaino_db; - zaino_db.spawn_handler().await?; + // Deliberately does NOT start the background validator. This builds a *pre-migration* + // v1.0.0 fixture; the validator validates against the current (v1.3.0) schema — it reads + // `commitment_tree_data_1_3_0` and the `ironwood` table — so it must run only after the + // database has been migrated to the newest schema. Callers build the fixture with direct + // v1.0.0 writes, shut it down, then reopen through `FinalisedState::spawn`, which migrates + // first and starts the validator afterwards. + // + // With no validator to advance it, mark the empty fixture `Ready` directly so callers see a + // settled backend. + zaino_db.status.store(StatusType::Ready); Ok(zaino_db) } From b15510121c248e5e4a9383cf229fcf56851c5eef Mon Sep 17 00:00:00 2001 From: idky137 Date: Mon, 6 Jul 2026 21:33:58 +0100 Subject: [PATCH 097/134] fixed ironwood migration --- .../finalised_state/finalised_source.rs | 6 + .../finalised_state/finalised_source/v1.rs | 15 +- .../finalised_source/v1/write_core.rs | 30 ++- .../chain_index/finalised_state/migrations.rs | 211 ++++++++++++------ .../tests/finalised_state/migrations.rs | 2 + .../migrations/v1_2_to_v1_3.rs | 99 ++++++++ 6 files changed, 281 insertions(+), 82 deletions(-) create mode 100644 packages/zaino-state/src/chain_index/tests/finalised_state/migrations/v1_2_to_v1_3.rs diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source.rs index e2ce0793e..70a9c7137 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source.rs @@ -380,6 +380,12 @@ impl FinalisedSource { .require_v1("v1 commitment_tree_data db")? .commitment_tree_data_db()) } + + /// Provides access to the (v1.3.0) `ironwood` table, required for Migration1_2_1To1_3_0 to + /// backfill ironwood rows from validator-fetched block data. + pub(crate) fn ironwood_db(&self) -> Result { + Ok(self.require_v1("v1 ironwood db")?.ironwood_db()) + } } impl From for FinalisedSource { diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1.rs index f75eacf21..835579784 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1.rs @@ -405,15 +405,14 @@ pub(crate) struct DbV1 { /// - validated read fetchers used by the capability trait implementations, and /// - internal validation / indexing helpers. impl DbV1 { - /// Spawns a new [`DbV1`] and opens (or creates) the LMDB environment for the configured network. + /// Opens (and heals) the v1 database for the configured network **without** starting the + /// background validator. /// /// This method: /// - chooses a versioned path suffix (`...//v1`), /// - configures LMDB map size and reader slots, - /// - opens or creates all V1 named databases, - /// - validates or initializes the `"metadata"` record (schema hash + version), and - /// - spawns the background validator / maintenance task. - /// Opens (and heals) the v1 database **without** starting the background validator. + /// - opens or creates all V1 named databases, and + /// - validates or initializes the `"metadata"` record (schema hash + version). /// /// The validator is started separately via [`DbV1::start_validator`]. This split exists so the /// orchestrator can guarantee that any pending schema migration finishes *before* validation @@ -851,6 +850,12 @@ impl DbV1 { self.commitment_tree_data } + /// Provides access to the (v1.3.0) `ironwood` table, required for Migration1_2_1To1_3_0 to + /// backfill ironwood rows for post-NU6.3 blocks from validator-fetched block data. + pub(crate) fn ironwood_db(&self) -> Database { + self.ironwood + } + /// Provudes access to the spent DB table, required for Migration1_1_0To1_2_0. pub(crate) fn spent_db(&self) -> Database { self.spent diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/write_core.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/write_core.rs index 212ac75d3..5c3cfc07e 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/write_core.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/write_core.rs @@ -188,13 +188,35 @@ fn build_block_row_entries( ), sapling_entry: StoredEntryVar::new(block_height_bytes, SaplingTxList::new(sapling)), orchard_entry: StoredEntryVar::new(block_height_bytes, OrchardTxList::new(orchard)), - ironwood_entry: ironwood - .iter() - .any(Option::is_some) - .then(|| StoredEntryVar::new(block_height_bytes, OrchardTxList::new(ironwood))), + ironwood_entry: ironwood_entry(ironwood, block_height_bytes), } } +/// Builds the sparse ironwood row entry for a block's per-transaction ironwood list: `Some` only +/// when at least one transaction carries ironwood data, so a block with none stores no ironwood row +/// (readers treat an absent row as "no ironwood data"). +fn ironwood_entry( + ironwood: Vec>, + block_height_bytes: &[u8], +) -> Option> { + ironwood + .iter() + .any(Option::is_some) + .then(|| StoredEntryVar::new(block_height_bytes, OrchardTxList::new(ironwood))) +} + +/// Builds the sparse ironwood row entry for `block` (the same value the write path stores). Used by +/// the v1.2.1 → v1.3.0 migration to backfill the ironwood table from validator-fetched blocks. +pub(crate) fn build_block_ironwood_entry( + block: &IndexedBlock, + block_height_bytes: &[u8], +) -> Result>, FinalisedStateError> { + Ok(ironwood_entry( + extract_block_pool_lists(block)?.ironwood, + block_height_bytes, + )) +} + impl DbWrite for DbV1 { async fn write_block(&self, block: IndexedBlock) -> Result<(), FinalisedStateError> { self.write_block(block).await diff --git a/packages/zaino-state/src/chain_index/finalised_state/migrations.rs b/packages/zaino-state/src/chain_index/finalised_state/migrations.rs index b88009194..ef414f9d1 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/migrations.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/migrations.rs @@ -1017,27 +1017,28 @@ impl Migration for Migration1_2_0To1_2_1 { /// Minor migration: v1.2.1 → v1.3.0. /// /// Introduces the Ironwood (NU6.3) shielded pool to the finalised state: -/// - a new per-height `ironwood` table (`ironwood_1_3_0`), created empty by `DbV1::spawn`; new -/// blocks populate it via the write path. No backfill is needed because every block already on -/// disk predates NU6.3 (see the guard below). +/// - a new per-height `ironwood` table (`ironwood_1_3_0`), created empty by `DbV1::spawn`. New +/// blocks populate it via the write path; here it is backfilled for any stored block at or above +/// NU6.3 activation (see below). /// - the `commitment_tree_data` table is rebuilt from the legacy fixed-length /// `StoredEntryFixed` (V1) rows into the variable-length /// `StoredEntryVar` (V2) layout, which carries the optional Ironwood root and -/// size. The rebuild reads the legacy `commitment_tree_data_1_0_0` table and writes the new -/// `commitment_tree_data_1_3_0` table (opened as the primary's `commitment_tree_data` handle), -/// then clears the legacy table. +/// size. The new rows are written to `commitment_tree_data_1_3_0` (the primary's +/// `commitment_tree_data` handle), then the legacy table is cleared. /// -/// This is a **rebuild from existing on-disk data** (no validator refetch), correct only while every -/// stored block predates Ironwood. The migration therefore asserts the database tip is below NU6.3 -/// activation before running; a database already synced past NU6.3 under the old schema would be -/// missing ironwood data that cannot be reconstructed from existing rows and must be re-indexed from -/// the validator instead. +/// Per-height rebuild strategy, branching on NU6.3 activation: +/// - **Below activation:** rebuilt in place from the legacy on-disk commitment row (Ironwood +/// root/size default to `None`/`0` via `CommitmentTreeData` V1 decode). No validator access. +/// - **At or above activation:** the legacy data predates Ironwood, so both the commitment row +/// (now carrying the Ironwood root/size) and the sparse ironwood row are rebuilt from +/// validator-fetched block data via [`build_indexed_block_from_source`]. This lets the migration +/// run on a database already synced past NU6.3, rather than forcing a full re-index. /// /// Safety and resumability: -/// - Deterministic: each rebuilt row is derived only from the matching legacy row (Ironwood -/// root/size default to `None`/`0` via `CommitmentTreeData` V1 decode). +/// - Deterministic: a below-activation row is derived only from its legacy row; an at/above-activation +/// row is refetched from immutable finalised history, so a resumed rebuild reproduces the same bytes. /// - Resumable: the next height to rebuild is stored in the metadata DB under a temporary key. -/// - Crash-safe: each height's rebuilt row and the progress update commit in the same transaction; +/// - Crash-safe: each height's rebuilt rows and the progress update commit in the same transaction; /// idempotent on resume (`NO_OVERWRITE` + verify-match). struct Migration1_2_1To1_3_0; @@ -1062,10 +1063,16 @@ impl Migration for Migration1_2_1To1_3_0 { &self, router: Arc>, cfg: ChainIndexConfig, - _source: T, + source: T, ) -> Result<(), FinalisedStateError> { use lmdb::DatabaseFlags; + use crate::chain_index::finalised_state::{ + build_indexed_block_from_source, + finalised_source::v1::write_core::build_block_ironwood_entry, + }; + use crate::chain_index::ShieldedPool; + // Temporary metadata entry recording the next height to rebuild, removed on completion. const MIGRATION_CTD_PROGRESS_KEY: &[u8] = b"_migration_commitment_tree_data_progress_1_3_0_next_height"; @@ -1077,8 +1084,10 @@ impl Migration for Migration1_2_1To1_3_0 { let backend = router.primary_backend(); let env = backend.env()?; let metadata_db = backend.metadata_db()?; - // The primary's `commitment_tree_data` handle is the new `commitment_tree_data_1_3_0` table. + // The primary's `commitment_tree_data` handle is the new `commitment_tree_data_1_3_0` table; + // `ironwood_db` is the new (v1.3.0) sparse ironwood table backfilled below. let new_ctd_db = backend.commitment_tree_data_db()?; + let ironwood_db = backend.ironwood_db()?; // Open the legacy fixed-length commitment table by name. On a pre-v1.3.0 database it already // exists; `open_or_create_db` creating it empty on an unexpected fresh DB is harmless (the @@ -1091,10 +1100,19 @@ impl Migration for Migration1_2_1To1_3_0 { ) .await?; - // Guard: Ironwood data is only expected from NU6.3 activation. A rebuild-from-existing-data - // migration cannot reconstruct ironwood roots/sizes for post-NU6.3 blocks. - let zebra_network = cfg.network.to_zebra_network(); - let nu6_3_activation_height = crate::chain_index::ShieldedPool::Ironwood + // Network-upgrade activation heights (mirrors `write_blocks_to_height`). Blocks at or above + // NU6.3 have their ironwood root/size and ironwood tx list rebuilt from the validator (the + // legacy on-disk data predates ironwood); blocks below it are rebuilt in place from the + // legacy commitment row, with no ironwood. + let network = cfg.network; + let zebra_network = network.to_zebra_network(); + let sapling_activation_height = ShieldedPool::Sapling + .activation_upgrade() + .activation_height(&zebra_network); + let nu5_activation_height = ShieldedPool::Orchard + .activation_upgrade() + .activation_height(&zebra_network); + let nu6_3_activation_height = ShieldedPool::Ironwood .activation_upgrade() .activation_height(&zebra_network); @@ -1133,17 +1151,6 @@ impl Migration for Migration1_2_1To1_3_0 { if let Some(db_tip) = backend.db_height().await? { let db_tip = db_tip.0; - if let Some(activation) = nu6_3_activation_height { - if db_tip >= activation.0 { - return Err(FinalisedStateError::Custom(format!( - "cannot migrate finalised state to v1.3.0: tip {db_tip} is at or above NU6.3 \ - activation {} but was built without Ironwood data; wipe and re-index the \ - finalised state from the validator", - activation.0 - ))); - } - } - let mut next_height = read_progress(MIGRATION_CTD_PROGRESS_KEY)?.unwrap_or(GENESIS_HEIGHT.0); @@ -1154,58 +1161,116 @@ impl Migration for Migration1_2_1To1_3_0 { ); let started = std::time::Instant::now(); + // Idempotent write: on resume an already-written row must match byte-for-byte (the + // legacy rebuild is deterministic and the validator refetch is over immutable history). + fn put_idempotent( + txn: &mut lmdb::RwTransaction<'_>, + db: lmdb::Database, + key: &[u8], + value: &[u8], + what: &str, + ) -> Result<(), FinalisedStateError> { + match txn.put(db, &key, &value, WriteFlags::NO_OVERWRITE) { + Ok(()) => Ok(()), + Err(lmdb::Error::KeyExist) => { + let existing = txn.get(db, &key).map_err(FinalisedStateError::LmdbError)?; + if existing != value { + return Err(FinalisedStateError::Custom(format!( + "conflicting rebuilt {what}" + ))); + } + Ok(()) + } + Err(error) => Err(FinalisedStateError::LmdbError(error)), + } + } + while next_height <= db_tip { let height = Height::try_from(next_height) .map_err(|error| FinalisedStateError::Custom(error.to_string()))?; let height_bytes = height.to_bytes()?; - // Read + verify the legacy fixed-length row. - let commitment_tree_data: CommitmentTreeData = { - let txn = env.begin_ro_txn()?; - let raw = txn - .get(legacy_ctd_db, &height_bytes) - .map_err(FinalisedStateError::LmdbError)?; - let entry = StoredEntryFixed::::from_bytes(raw).map_err( - |error| { - FinalisedStateError::Custom(format!( - "legacy commitment_tree_data corrupt data: {error}" - )) - }, - )?; - if !entry.verify(&height_bytes) { - return Err(FinalisedStateError::Custom( - "legacy commitment_tree_data checksum mismatch".to_string(), - )); - } - *entry.inner() - }; - - // Re-wrap in the V2 `StoredEntryVar` layout and advance progress atomically. - let new_entry_bytes = - StoredEntryVar::new(&height_bytes, commitment_tree_data).to_bytes()?; + let ironwood_active = + nu6_3_activation_height.is_some_and(|activation| next_height >= activation.0); + + // Prepare the rows to write. Reads / validator fetches happen before the write txn + // so the commit stays short. + let commitment_bytes: Vec; + let ironwood_bytes: Option>; + + if ironwood_active { + // Post-NU6.3: the legacy row carries no ironwood root/size and the ironwood + // table has no row, so rebuild both from validator-fetched block data. + let sapling_activation_height = sapling_activation_height.ok_or_else(|| { + FinalisedStateError::Custom( + "Sapling activation height must be set to backfill ironwood" + .to_string(), + ) + })?; + let block = build_indexed_block_from_source( + &source, + network, + sapling_activation_height, + nu5_activation_height, + nu6_3_activation_height, + next_height, + // Chainwork is irrelevant here: only the commitment-tree and ironwood rows + // are extracted, and neither depends on it. + None, + ) + .await?; + + commitment_bytes = + StoredEntryVar::new(&height_bytes, *block.commitment_tree_data()) + .to_bytes()?; + ironwood_bytes = match build_block_ironwood_entry(&block, &height_bytes)? { + Some(entry) => Some(entry.to_bytes()?), + None => None, + }; + } else { + // Pre-NU6.3: rebuild the commitment row in place from the legacy fixed-length + // row (ironwood defaults to none). + let commitment_tree_data: CommitmentTreeData = { + let txn = env.begin_ro_txn()?; + let raw = txn + .get(legacy_ctd_db, &height_bytes) + .map_err(FinalisedStateError::LmdbError)?; + let entry = StoredEntryFixed::::from_bytes(raw) + .map_err(|error| { + FinalisedStateError::Custom(format!( + "legacy commitment_tree_data corrupt data: {error}" + )) + })?; + if !entry.verify(&height_bytes) { + return Err(FinalisedStateError::Custom( + "legacy commitment_tree_data checksum mismatch".to_string(), + )); + } + *entry.inner() + }; + commitment_bytes = + StoredEntryVar::new(&height_bytes, commitment_tree_data).to_bytes()?; + ironwood_bytes = None; + } + // Write commitment (+ ironwood) and advance progress atomically. { let mut txn = env.begin_rw_txn()?; - match txn.put( + put_idempotent( + &mut txn, new_ctd_db, &height_bytes, - &new_entry_bytes, - WriteFlags::NO_OVERWRITE, - ) { - Ok(()) => {} - // Idempotent on resume: an existing rebuilt row must match exactly. - Err(lmdb::Error::KeyExist) => { - let existing = txn - .get(new_ctd_db, &height_bytes) - .map_err(FinalisedStateError::LmdbError)?; - if existing != new_entry_bytes.as_slice() { - return Err(FinalisedStateError::Custom(format!( - "conflicting rebuilt commitment_tree_data at height {}", - height.0 - ))); - } - } - Err(error) => return Err(FinalisedStateError::LmdbError(error)), + &commitment_bytes, + &format!("commitment_tree_data at height {}", height.0), + )?; + if let Some(bytes) = &ironwood_bytes { + put_idempotent( + &mut txn, + ironwood_db, + &height_bytes, + bytes, + &format!("ironwood at height {}", height.0), + )?; } let progress = StoredEntryFixed::new(MIGRATION_CTD_PROGRESS_KEY, height + 1); diff --git a/packages/zaino-state/src/chain_index/tests/finalised_state/migrations.rs b/packages/zaino-state/src/chain_index/tests/finalised_state/migrations.rs index e0058faf6..11b399a6e 100644 --- a/packages/zaino-state/src/chain_index/tests/finalised_state/migrations.rs +++ b/packages/zaino-state/src/chain_index/tests/finalised_state/migrations.rs @@ -10,3 +10,5 @@ mod v1_0_to_v1_1; #[cfg(not(feature = "transparent_address_history_experimental"))] mod v1_1_to_v1_2; +#[cfg(not(feature = "transparent_address_history_experimental"))] +mod v1_2_to_v1_3; diff --git a/packages/zaino-state/src/chain_index/tests/finalised_state/migrations/v1_2_to_v1_3.rs b/packages/zaino-state/src/chain_index/tests/finalised_state/migrations/v1_2_to_v1_3.rs new file mode 100644 index 000000000..5d215be9c --- /dev/null +++ b/packages/zaino-state/src/chain_index/tests/finalised_state/migrations/v1_2_to_v1_3.rs @@ -0,0 +1,99 @@ +//! V1.2.1 to V1.3.0 migration tests. +//! +//! Coverage note: these tests use `ActivationHeights::default()`, whose NU6.3 activation is `None`, +//! so the migration takes the below-activation branch (rebuild each commitment row in place from the +//! legacy fixed-length table, no ironwood). The at/above-activation *ironwood backfill* branch — +//! which refetches block data from the validator — cannot be exercised yet: `MockchainSource` +//! serves no ironwood commitment roots (see its `get_commitment_tree_roots` TODO) and the test +//! vectors carry no ironwood actions, so building a post-NU6.3 block would fail resolving the +//! (required) ironwood root. That path needs ironwood-capable test vectors before it can be tested. + +use std::path::PathBuf; +use tempfile::TempDir; +use zaino_common::network::ActivationHeights; +use zaino_common::{DatabaseConfig, Network, StorageConfig}; + +use crate::chain_index::finalised_state::capability::{DbVersion, MigrationStatus}; +use crate::chain_index::finalised_state::finalised_source::v1::DB_VERSION_V1; +use crate::chain_index::finalised_state::FinalisedState; +use crate::chain_index::tests::init_tracing; +use crate::chain_index::tests::vectors::{ + build_active_mockchain_source, load_test_vectors, TestVectorData, +}; +use crate::{ChainIndexConfig, Height, StatusType}; + +fn v1_2_1() -> DbVersion { + DbVersion { + major: 1, + minor: 2, + patch: 1, + } +} + +/// Regression test for the startup ordering bug: opening a pre-1.3.0 cache used to start the +/// background validator concurrently with the v1.2.1 → v1.3.0 migration. The validator's +/// `initial_block_scan` read the freshly-created-empty `commitment_tree_data_1_3_0` table before the +/// migration rebuilt it, failing with `MDB_NOTFOUND` ("block scan") and latching `CriticalError`. +/// +/// The fix defers the validator until every migration finishes. This test builds an on-disk v1.2.1 +/// database, then reopens it through the production `FinalisedState::spawn` path (which migrates to +/// the current schema and only then starts the validator) and asserts it reaches `Ready` — i.e. the +/// migration completed *before* validation, and validation then passed. +#[tokio::test(flavor = "multi_thread")] +async fn v1_2_1_cache_migrates_to_current_then_validates() { + init_tracing(); + + let TestVectorData { blocks, .. } = load_test_vectors().unwrap(); + let active_height = Height(150); + + let temporary_directory: TempDir = tempfile::tempdir().unwrap(); + let database_path: PathBuf = temporary_directory.path().to_path_buf(); + + let database_config = ChainIndexConfig { + storage: StorageConfig { + database: DatabaseConfig { + path: database_path, + ..Default::default() + }, + ..Default::default() + }, + ephemeral: false, + db_version: 1, + network: Network::Regtest(ActivationHeights::default()), + }; + + let source = build_active_mockchain_source(active_height.0, blocks.clone()); + + // Build an on-disk v1.2.1 database (the pre-1.3.0 cache shape), then release it. + let old_database = + FinalisedState::build_db_to_version(database_config.clone(), source.clone(), v1_2_1()) + .await + .unwrap(); + old_database.wait_until_synced().await; + assert_eq!(old_database.get_metadata().await.unwrap().version, v1_2_1()); + old_database.shutdown().await.unwrap(); + drop(old_database); + + // Reopen through the production spawn path: it runs the v1.2.1 → v1.3.0 migration and only then + // starts the validator. Before the ordering fix this raced the migration and latched + // `CriticalError`; now it must migrate first and validate cleanly. + let migrated_database = FinalisedState::spawn(database_config.clone(), source.clone()) + .await + .unwrap(); + migrated_database.wait_until_synced().await; + + assert_eq!( + migrated_database.status(), + StatusType::Ready, + "the validator must run only after the migration completes, and then reach Ready" + ); + + let migrated_metadata = migrated_database.get_metadata().await.unwrap(); + assert_eq!(migrated_metadata.version, DB_VERSION_V1); + assert_eq!(migrated_metadata.migration_status, MigrationStatus::Empty); + + let migrated_height = migrated_database.db_height().await.unwrap().unwrap(); + assert_eq!(migrated_height, active_height); + + migrated_database.shutdown().await.unwrap(); +} From 7180e3c6eb59f38338048244178560c4d21290dd Mon Sep 17 00:00:00 2001 From: zancas Date: Mon, 6 Jul 2026 22:01:02 -0700 Subject: [PATCH 098/134] feat!: enforce the Validator as the sole source of activation-height truth MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Validator's configured activation heights are now the only source of truth for every component, enforced at compile time on both sides of the zcash_local_net seam (zaino#1076; infrastructure ADR 0003 and PR #278; the inbound spec is committed as zainod-heights-from-validator-spec.md). On the indexer side, zaino_common::Network is a payload-free kind, so a configuration value cannot carry heights and a pre-handshake height read does not typecheck. Both backends construct the runtime zebra_chain::parameters::Network in exactly one place: the spawn handshake, which uses zebra's compiled parameters for the public networks and adopts the validator's reported schedule from getblockchaininfo.upgrades for regtest. An upgrade absent from the validator's map is never-activated, nothing is backfilled from defaults, and a failed or unparseable handshake aborts startup with an error naming the validator endpoint. Every downstream consumer — the chain-index config, the connectors, the finalised state, and the subscribers via ServiceMetadata — receives that runtime network and can no longer manufacture its own; the conversions that let a config value impersonate chain truth (to_zebra_network and the From impls into zebra's type) are deleted, along with the dead accessors that invited a bypass. Test fixtures that are their own chain construct runtime networks explicitly through ActivationHeights::to_regtest_network. On the wallet side, the zcash_local_net pin moves to 78511f7, where the devtool client's network is a WalletNetwork whose regtest variant only WalletNetwork::from_validator can construct. The e2e client builder becomes build_clients(port, &validator) and derives the wallet schedule from the running validator, so: *wallet/validator height drift is unrepresentable* and a fixture's heights are typed exactly once, in the zebrad launch config. The harness no longer mirrors fixture heights into zainod: every custom-heights launch doubles as a regression test of the adoption path. The canonical launch set moves to zaino-testutils as ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS, which is validator-launch vocabulary rather than anyone's schedule. The uniqueness regressions live in clientless/tests/validator_heights.rs: a kind-only-configured zainod must sync a NU6.3-at-6 schedule its configuration never saw, and a golden test pins the upgrades map a live zebrad reports. Three unit tests beside activation_heights_from_upgrades pin the mapping itself. The production workspace, all live-test crates, and the unit suites compile and pass clean. Co-Authored-By: Claude Fable 5 --- Cargo.lock | 6 +- Cargo.toml | 9 +- live-tests/clientless/tests/chain_cache.rs | 22 ++- live-tests/clientless/tests/fetch_service.rs | 1 - .../clientless/tests/validator_heights.rs | 156 +++++++++++++++ live-tests/e2e/src/devtool.rs | 22 ++- live-tests/e2e/tests/devtool.rs | 10 + live-tests/e2e/tests/devtool_zcashd.rs | 5 +- live-tests/e2e/tests/test_vectors.rs | 5 +- live-tests/zaino-testutils/src/lib.rs | 62 +++--- packages/zaino-common/src/config/network.rs | 183 +++--------------- packages/zaino-common/src/lib.rs | 2 +- packages/zaino-state/src/backends.rs | 171 ++++++++++++++++ packages/zaino-state/src/backends/fetch.rs | 59 ++++-- packages/zaino-state/src/backends/state.rs | 162 ++++++++-------- packages/zaino-state/src/chain_index.rs | 2 +- .../src/chain_index/finalised_state.rs | 16 +- .../finalised_state/finalised_source/v1.rs | 6 +- .../v1/tx_out_set_accumulator.rs | 6 +- .../finalised_source/v1/write_core.rs | 9 +- .../chain_index/finalised_state/migrations.rs | 4 +- .../zaino-state/src/chain_index/source.rs | 1 - .../chain_index/source/mockchain_source.rs | 2 +- .../chain_index/source/validator_connector.rs | 12 +- packages/zaino-state/src/chain_index/tests.rs | 6 +- .../tests/finalised_state/ephemeral.rs | 4 +- .../migrations/v1_0_to_v1_1.rs | 6 +- .../migrations/v1_1_to_v1_2.rs | 8 +- .../chain_index/tests/finalised_state/v1.rs | 10 +- .../chain_index/tests/proptest_blockgen.rs | 22 +-- packages/zaino-state/src/config.rs | 52 +++-- packages/zainod/src/config.rs | 5 - zainod-heights-from-validator-spec.md | 170 ++++++++++++++++ 33 files changed, 821 insertions(+), 395 deletions(-) create mode 100644 live-tests/clientless/tests/validator_heights.rs create mode 100644 zainod-heights-from-validator-spec.md diff --git a/Cargo.lock b/Cargo.lock index 5b4174254..3d0124b57 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6184,7 +6184,7 @@ dependencies = [ [[package]] name = "zcash_local_net" version = "0.7.0" -source = "git+https://github.com/zingolabs/infrastructure.git?rev=0dc4a51f7d9666a3df7eaf28ad4f95cacb86d646#0dc4a51f7d9666a3df7eaf28ad4f95cacb86d646" +source = "git+https://github.com/zingolabs/infrastructure.git?rev=78511f7304ba27895967be860016ede923acec08#78511f7304ba27895967be860016ede923acec08" dependencies = [ "hex", "reqwest 0.12.28", @@ -6735,12 +6735,12 @@ dependencies = [ [[package]] name = "zingo-consensus" version = "0.1.0" -source = "git+https://github.com/zingolabs/infrastructure.git?rev=0dc4a51f7d9666a3df7eaf28ad4f95cacb86d646#0dc4a51f7d9666a3df7eaf28ad4f95cacb86d646" +source = "git+https://github.com/zingolabs/infrastructure.git?rev=78511f7304ba27895967be860016ede923acec08#78511f7304ba27895967be860016ede923acec08" [[package]] name = "zingo_test_vectors" version = "0.0.1" -source = "git+https://github.com/zingolabs/infrastructure.git?rev=0dc4a51f7d9666a3df7eaf28ad4f95cacb86d646#0dc4a51f7d9666a3df7eaf28ad4f95cacb86d646" +source = "git+https://github.com/zingolabs/infrastructure.git?rev=78511f7304ba27895967be860016ede923acec08#78511f7304ba27895967be860016ede923acec08" [[package]] name = "zip32" diff --git a/Cargo.toml b/Cargo.toml index de1292cf9..bacac4ad1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -138,9 +138,12 @@ zaino-testutils = { path = "live-tests/zaino-testutils" } # Zingo-infra (validator launcher driving the live tests; not a prod dep). # Pinned to an exact commit past zcash_local_net_v0.7.0: the v0.7.0 launcher's # zebrad config writer silently drops NU6.3 activation heights -# (https://github.com/zingolabs/zaino/issues/1368); this rev carries the -# "NU6.3" TOML plumbing. Restore a release tag once one is cut upstream. -zcash_local_net = { git = "https://github.com/zingolabs/infrastructure.git", rev = "0dc4a51f7d9666a3df7eaf28ad4f95cacb86d646" } +# (https://github.com/zingolabs/zaino/issues/1368). This rev additionally +# makes the Validator the sole compile-time source of activation-height +# truth (infrastructure ADR 0003, PR #278): wallet clients derive their +# heights via `WalletNetwork::from_validator`, and `ZainodConfig` carries a +# payload-free network kind. Restore a release tag once one is cut upstream. +zcash_local_net = { git = "https://github.com/zingolabs/infrastructure.git", rev = "78511f7304ba27895967be860016ede923acec08" } wire_serialized_transaction_test_data = "1.0.0" config = { version = "0.15", default-features = false, features = ["toml"] } diff --git a/live-tests/clientless/tests/chain_cache.rs b/live-tests/clientless/tests/chain_cache.rs index a12cb7291..8d0b4b058 100644 --- a/live-tests/clientless/tests/chain_cache.rs +++ b/live-tests/clientless/tests/chain_cache.rs @@ -173,11 +173,12 @@ mod chain_query_interface { }, ephemeral, db_version: 1, - network: zaino_common::Network::Regtest( - zaino_testutils::from_local_net_activation_heights( - &test_manager.local_net.get_activation_heights().await, - ), - ), + // This fixture derives its runtime network from the + // heights the harness launched the validator with. + network: zaino_testutils::from_local_net_activation_heights( + &test_manager.local_net.get_activation_heights().await, + ) + .to_regtest_network(), }; // **NOTE** The "fetch" backend is currently the backend used in the wild, and @@ -219,11 +220,12 @@ mod chain_query_interface { }, ephemeral, db_version: 1, - network: zaino_common::Network::Regtest( - zaino_testutils::from_local_net_activation_heights( - &test_manager.local_net.get_activation_heights().await, - ), - ), + // This fixture derives its runtime network from the + // heights the harness launched the validator with. + network: zaino_testutils::from_local_net_activation_heights( + &test_manager.local_net.get_activation_heights().await, + ) + .to_regtest_network(), }; let chain_index = NodeBackedChainIndex::new( ValidatorConnector::Fetch(json_service.clone()), diff --git a/live-tests/clientless/tests/fetch_service.rs b/live-tests/clientless/tests/fetch_service.rs index 53ea44785..0a74f69d3 100644 --- a/live-tests/clientless/tests/fetch_service.rs +++ b/live-tests/clientless/tests/fetch_service.rs @@ -214,7 +214,6 @@ async fn fetch_service_get_block_subsidy(validator: &ValidatorK // first halving boundary plus a margin on both validators. let height_limit = fetch_service_subscriber .network() - .to_zebra_network() .height_for_first_halving() .0 + 10; diff --git a/live-tests/clientless/tests/validator_heights.rs b/live-tests/clientless/tests/validator_heights.rs new file mode 100644 index 000000000..a7bbc434a --- /dev/null +++ b/live-tests/clientless/tests/validator_heights.rs @@ -0,0 +1,156 @@ +//! Regression tests for the single source of truth for activation heights +//! (zaino#1076, `zainod-heights-from-validator-spec.md`). +//! +//! The invariant: the validator's configured activation heights are +//! authoritative. zainod's config carries only a network kind — its regtest +//! placeholder is the canonical `ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS` +//! regardless of the fixture — and both backends adopt the real schedule +//! from `getblockchaininfo.upgrades` at spawn. +//! +//! Two halves, matching the spec's acceptance criteria: +//! +//! 1. [`zainod_syncs_a_schedule_its_config_never_saw`] — boundary sync and +//! the no-recompile proof in one: the same zainod build and (kind-only) +//! configuration that every canonical-heights test runs is here pointed +//! at a validator on the NU6.3-at-6 transition schedule, and must sync +//! across the boundary and serve era-correct compact blocks. Before +//! adoption this exact misalignment killed the chain-index sync with +//! `InvalidData("Block commitment could not be computed")`. +//! 2. [`getblockchaininfo_reports_the_configured_schedule`] — the input +//! contract: what zebrad actually puts in the `upgrades` map for a +//! configured schedule, pinned against a live node rather than assumed. +//! The mapping from that shape to adopted heights is unit-tested next to +//! `activation_heights_from_upgrades` in zaino-state. + +#[allow(deprecated)] +use zaino_state::FetchService; +use zaino_state::ZcashIndexer as _; +use zaino_testutils::{ + all_pools_i32, collect_block_range, MinerPool, TestManager, ValidatorKind, + NU6_3_TRANSITION_BOUNDARY, ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS, + ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS, +}; +use zcash_local_net::validator::zebrad::Zebrad; + +/// Launches an orchard-receiver-mining zebrad on the transition schedule +/// with zainod enabled. The harness hands zainod only the canonical +/// placeholder (see `launch_mining_to`), so the launch itself is the +/// deliberate config/validator misalignment under test. +#[allow(deprecated)] +async fn launch_transition_validator() -> TestManager { + assert_ne!( + ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS, ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS, + "premise: zainod's config placeholder must differ from the fixture \ + schedule, or this proves nothing" + ); + + TestManager::::launch_mining_to( + MinerPool::Orchard, + &ValidatorKind::Zebrad, + None, + Some(ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS), + None, + true, + false, + false, + ) + .await + .expect("launch TestManager") +} + +/// Boundary sync + no-recompile proof: a kind-only-configured zainod adopts +/// the NU6.3-at-6 schedule from the validator, syncs across the boundary, +/// and serves era-correct compact blocks for both eras. +/// +/// multi_thread required: the test manager spawns the validator and indexer +/// services. +#[tokio::test(flavor = "multi_thread")] +async fn zainod_syncs_a_schedule_its_config_never_saw() { + let mut test_manager = launch_transition_validator().await; + let subscriber = test_manager.subscriber().clone(); + + // Two blocks past the boundary, so both eras carry more than one block. + // Reaching the tip at all is the core regression: pre-adoption, the + // chain-index sync died on the first block whose commitment scheme the + // misconfigured heights got wrong. + test_manager + .generate_blocks_and_wait_for_tip(NU6_3_TRANSITION_BOUNDARY + 1, &subscriber) + .await; + let tip = u64::from(subscriber.chain_height().await.expect("chain height").0); + assert!( + tip > u64::from(NU6_3_TRANSITION_BOUNDARY), + "sync must cross the boundary, tip is {tip}" + ); + + // Era composition of the served chain proves the adopted schedule is the + // validator's, not the placeholder: under the placeholder (NU6.3 at 2) + // the pre-boundary orchard coinbases would be misread as ironwood-era. + let blocks = collect_block_range(&subscriber, 2, tip, all_pools_i32()).await; + for block in &blocks { + let height = block.height; + let has_orchard = block.vtx.iter().any(|tx| !tx.actions.is_empty()); + let has_ironwood = block.vtx.iter().any(|tx| !tx.ironwood_actions.is_empty()); + if height >= u64::from(NU6_3_TRANSITION_BOUNDARY) { + assert!( + has_ironwood && !has_orchard, + "height {height} must be ironwood-era, got orchard={has_orchard} ironwood={has_ironwood}" + ); + } else { + assert!( + has_orchard && !has_ironwood, + "height {height} must be orchard-era, got orchard={has_orchard} ironwood={has_ironwood}" + ); + } + } + + test_manager.close().await; +} + +/// The input contract for adoption: the `upgrades` map a live zebrad reports +/// for the transition schedule, pinned exactly — upgrade set, order, and +/// heights. Establishes (from real output, not reasoning) that nothing +/// pre-Overwinter appears: the map is keyed by consensus branch ID, which +/// pre-Overwinter eras don't have. +/// +/// multi_thread required: the test manager spawns the validator and indexer +/// services. +#[tokio::test(flavor = "multi_thread")] +async fn getblockchaininfo_reports_the_configured_schedule() { + use zebra_chain::parameters::NetworkUpgrade; + + let mut test_manager = launch_transition_validator().await; + + let blockchain_info = test_manager + .full_node_jsonrpc_connector() + .await + .get_blockchain_info() + .await + .expect("getblockchaininfo"); + + let reported: Vec<(NetworkUpgrade, u32)> = blockchain_info + .upgrades + .values() + .map(|upgrade_info| { + let (upgrade, height, _status) = upgrade_info.into_parts(); + (upgrade, height.0) + }) + .collect(); + + assert_eq!( + reported, + vec![ + (NetworkUpgrade::Overwinter, 1), + (NetworkUpgrade::Sapling, 1), + (NetworkUpgrade::Blossom, 1), + (NetworkUpgrade::Heartwood, 1), + (NetworkUpgrade::Canopy, 1), + (NetworkUpgrade::Nu5, 2), + (NetworkUpgrade::Nu6, 2), + (NetworkUpgrade::Nu6_1, 2), + (NetworkUpgrade::Nu6_2, 2), + (NetworkUpgrade::Nu6_3, 6), + ], + ); + + test_manager.close().await; +} diff --git a/live-tests/e2e/src/devtool.rs b/live-tests/e2e/src/devtool.rs index 4fed529e6..e987a386a 100644 --- a/live-tests/e2e/src/devtool.rs +++ b/live-tests/e2e/src/devtool.rs @@ -39,17 +39,27 @@ pub struct DevtoolClients { } /// Launch faucet + recipient devtool wallets against a running Zaino gRPC -/// listener. The devtool analogue of [`crate::build_clients`]; Zaino must -/// already be serving (wallet initialization fetches the chain tip and -/// birthday tree state from it). -pub async fn build_clients(zaino_grpc_listen_port: u16) -> DevtoolClients { - let mut faucet_config = ZcashDevtoolConfig::faucet(); +/// listener, deriving the wallet network from the running `validator`. That +/// derivation is the only construction the client API offers +/// (infrastructure ADR 0003: the Validator is the single source of truth +/// for activation heights), so wallet/validator height drift is +/// unrepresentable — the wallets follow whatever schedule the validator was +/// launched with, fixture or canonical. The devtool analogue of +/// [`crate::build_clients`]; Zaino must already be serving (wallet +/// initialization fetches the chain tip and birthday tree state from it). +pub async fn build_clients( + zaino_grpc_listen_port: u16, + validator: &V, +) -> DevtoolClients { + let network = zcash_local_net::client::WalletNetwork::from_validator(validator).await; + + let mut faucet_config = ZcashDevtoolConfig::faucet(network); faucet_config.indexer_port = zaino_grpc_listen_port; let faucet = ZcashDevtool::launch(faucet_config) .await .expect("launch devtool faucet wallet"); - let mut recipient_config = ZcashDevtoolConfig::recipient(); + let mut recipient_config = ZcashDevtoolConfig::recipient(network); recipient_config.indexer_port = zaino_grpc_listen_port; let recipient = ZcashDevtool::launch(recipient_config) .await diff --git a/live-tests/e2e/tests/devtool.rs b/live-tests/e2e/tests/devtool.rs index e334ca441..9f8db292f 100644 --- a/live-tests/e2e/tests/devtool.rs +++ b/live-tests/e2e/tests/devtool.rs @@ -84,6 +84,7 @@ where .zaino_grpc_listen_address .expect("zaino enabled") .port(), + &test_manager.local_net, ) .await; @@ -122,6 +123,7 @@ where .zaino_grpc_listen_address .expect("zaino enabled") .port(), + &test_manager.local_net, ) .await; @@ -789,6 +791,7 @@ async fn block_range_returns_default_pools() { .zaino_grpc_listen_address .expect("zaino enabled") .port(), + &svc.test_manager.local_net, ) .await; @@ -880,6 +883,7 @@ async fn block_range_returns_all_pools() { .zaino_grpc_listen_address .expect("zaino enabled") .port(), + &svc.test_manager.local_net, ) .await; @@ -1021,6 +1025,7 @@ async fn fund_and_send_dual( .zaino_grpc_listen_address .expect("zaino enabled") .port(), + &svc.test_manager.local_net, ) .await; @@ -1172,6 +1177,7 @@ async fn fund_and_fill_mempool_dual() -> zaino_testutils::StateAndFetchServices< .zaino_grpc_listen_address .expect("zaino enabled") .port(), + &svc.test_manager.local_net, ) .await; @@ -1346,6 +1352,7 @@ async fn launch_transparent_and_faucet_taddr( .zaino_grpc_listen_address .expect("zaino enabled") .port(), + &svc.test_manager.local_net, ) .await; @@ -1679,6 +1686,7 @@ async fn address_deltas() { .zaino_grpc_listen_address .expect("zaino enabled") .port(), + &svc.test_manager.local_net, ) .await; @@ -1838,6 +1846,7 @@ async fn get_block_deltas_resolves_transparent_spend() { .zaino_grpc_listen_address .expect("zaino enabled") .port(), + &svc.test_manager.local_net, ) .await; @@ -2093,6 +2102,7 @@ async fn get_outpoint_spenders_fetch_vs_state() { .zaino_grpc_listen_address .expect("zaino enabled") .port(), + &svc.test_manager.local_net, ) .await; diff --git a/live-tests/e2e/tests/devtool_zcashd.rs b/live-tests/e2e/tests/devtool_zcashd.rs index 4405ff344..3713509fc 100644 --- a/live-tests/e2e/tests/devtool_zcashd.rs +++ b/live-tests/e2e/tests/devtool_zcashd.rs @@ -44,7 +44,7 @@ async fn launch_zcashd_and_build_clients() -> (TestManager &ValidatorKind::Zcashd, None, // network -> Regtest // The heights the devtool wallet accepts (same as the zebrad path). - Some(zaino_common::network::ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS), + Some(zaino_testutils::ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS), None, // no chain cache: build fresh at these heights true, // enable zaino false, // no json-rpc server @@ -58,6 +58,7 @@ async fn launch_zcashd_and_build_clients() -> (TestManager .zaino_grpc_listen_address .expect("zaino enabled") .port(), + &test_manager.local_net, ) .await; @@ -110,7 +111,7 @@ async fn faucet_receives_zcashd_orchard_reward() { /// of json_server's `create_zcashd_test_manager_and_fetch_services`. async fn create_zcashd_devtool_services() -> (ZcashdDualFetchServices, DevtoolClients) { let services = zaino_testutils::launch_zcashd_dual_fetch_services_at( - zaino_common::network::ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS, + zaino_testutils::ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS, ) .await; let clients = e2e::devtool::build_clients( diff --git a/live-tests/e2e/tests/test_vectors.rs b/live-tests/e2e/tests/test_vectors.rs index aeee240f4..a44dffa40 100644 --- a/live-tests/e2e/tests/test_vectors.rs +++ b/live-tests/e2e/tests/test_vectors.rs @@ -67,6 +67,7 @@ async fn create_200_block_regtest_chain_vectors() { .zaino_grpc_listen_address .expect("zaino enabled") .port(), + &test_manager.local_net, ) .await; @@ -319,7 +320,7 @@ async fn create_200_block_regtest_chain_vectors() { }; let sapling_treestate = match zebra_chain::parameters::NetworkUpgrade::Sapling - .activation_height(&state_service_subscriber.network().to_zebra_network()) + .activation_height(&state_service_subscriber.network()) { Some(activation_height) if height >= activation_height.0 => Some( state @@ -342,7 +343,7 @@ async fn create_200_block_regtest_chain_vectors() { }) .unwrap(); let orchard_treestate = match zebra_chain::parameters::NetworkUpgrade::Nu5 - .activation_height(&state_service_subscriber.network().to_zebra_network()) + .activation_height(&state_service_subscriber.network()) { Some(activation_height) if height >= activation_height.0 => Some( state diff --git a/live-tests/zaino-testutils/src/lib.rs b/live-tests/zaino-testutils/src/lib.rs index 9fcd5649b..6e8a342e1 100644 --- a/live-tests/zaino-testutils/src/lib.rs +++ b/live-tests/zaino-testutils/src/lib.rs @@ -14,7 +14,7 @@ use std::{ use tonic::transport::Channel; use tracing::{debug, info, instrument}; use zaino_common::{ - network::{ActivationHeights, ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS}, + network::ActivationHeights, probing::{Liveness, Readiness}, status::Status, validator::ValidatorConfig, @@ -67,6 +67,29 @@ macro_rules! validator_tests { }; } +/// The canonical regtest activation heights the harness *launches* validators +/// with when a test names no others (everything through NU6.3 active from +/// height 2). This is validator-launch vocabulary — the configuring side of +/// the truth source — and is never zainod's own schedule: zainod's config +/// carries only a network kind, and both backends adopt the runtime schedule +/// from the running validator's `getblockchaininfo.upgrades` (zaino#1076). +/// It matches zcash_local_net's `supported_regtest_activation_heights`, which +/// the devtool wallet client currently requires (zaino#1368). +pub const ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS: ActivationHeights = ActivationHeights { + before_overwinter: Some(1), + overwinter: Some(1), + sapling: Some(1), + blossom: Some(1), + heartwood: Some(1), + canopy: Some(1), + nu5: Some(2), + nu6: Some(2), + nu6_1: Some(2), + nu6_2: Some(2), + nu6_3: Some(2), + nu7: None, +}; + /// Orchard-era-only fixture: every upgrade through NU6.2 active from height 2 and /// NU6.3 never activating, so coinbases stay Orchard for the whole chain. For /// client-free launches only — the zcash-devtool wallet hardcodes the canonical @@ -719,8 +742,17 @@ where let activation_heights = activation_heights.unwrap_or_else(|| validator.default_activation_heights()); let network_kind = network.unwrap_or(NetworkKind::Regtest); - let zaino_network_kind = - Network::from_network_kind_and_activation_heights(&network_kind, &activation_heights); + // The validator is the single source of truth for activation heights + // (zaino#1076). zainod's config is a payload-free network kind: the + // fixture heights configure the validator alone, and zainod adopts + // the real schedule from the validator at spawn. Every + // custom-heights launch is therefore a standing regression test of + // that adoption. + let zaino_network_kind = match network_kind { + NetworkKind::Mainnet => Network::Mainnet, + NetworkKind::Testnet => Network::Testnet, + NetworkKind::Regtest => Network::Regtest, + }; if enable_clients && !enable_zaino { return Err(std::io::Error::other( @@ -1142,23 +1174,9 @@ pub async fn launch_state_and_fetch_services_mining_to( tokio::time::sleep(std::time::Duration::from_millis(5000)).await; Network::Testnet } - _ => Network::Regtest({ - let activation_heights = test_manager.local_net.get_activation_heights().await; - ActivationHeights { - before_overwinter: activation_heights.overwinter(), - overwinter: activation_heights.overwinter(), - sapling: activation_heights.sapling(), - blossom: activation_heights.blossom(), - heartwood: activation_heights.heartwood(), - canopy: activation_heights.canopy(), - nu5: activation_heights.nu5(), - nu6: activation_heights.nu6(), - nu6_1: activation_heights.nu6_1(), - nu6_2: activation_heights.nu6_2(), - nu6_3: activation_heights.nu6_3(), - nu7: activation_heights.nu7(), - } - }), + // The kind suffices: zainod adopts the schedule from the validator + // at spawn (zaino#1076). + _ => Network::Regtest, }; test_manager.local_net.print_stdout(); @@ -1339,7 +1357,7 @@ pub async fn launch_zcashd_dual_fetch_services_at( .data_dir() .path() .join("zcashd-fetch-service-zaino"), - Network::Regtest(activation_heights), + Network::Regtest, ) .await; let zcashd_subscriber = zcashd_fetch_service.get_subscriber().inner(); @@ -1357,7 +1375,7 @@ pub async fn launch_zcashd_dual_fetch_services_at( .data_dir() .path() .join("zaino-fetch-service-zaino"), - Network::Regtest(activation_heights), + Network::Regtest, ) .await; let zaino_subscriber = zaino_fetch_service.get_subscriber().inner(); diff --git a/packages/zaino-common/src/config/network.rs b/packages/zaino-common/src/config/network.rs index d518d895b..398908ca1 100644 --- a/packages/zaino-common/src/config/network.rs +++ b/packages/zaino-common/src/config/network.rs @@ -5,36 +5,21 @@ use std::fmt; use serde::{Deserialize, Serialize}; use zebra_chain::parameters::testnet::ConfiguredActivationHeights; -/// Must equal zcash_local_net's `supported_regtest_activation_heights`: the -/// zcash-devtool wallet client hardcodes that canonical set (NU6.3 at 2), and a -/// zebrad configured differently rejects wallet-built transactions with -/// "incorrect consensus branch id". See -/// . -pub const ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS: ActivationHeights = ActivationHeights { - overwinter: Some(1), - before_overwinter: Some(1), - sapling: Some(1), - blossom: Some(1), - heartwood: Some(1), - canopy: Some(1), - nu5: Some(2), - nu6: Some(2), - nu6_1: Some(2), - nu6_2: Some(2), - nu6_3: Some(2), - nu7: None, -}; - -/// Network type for Zaino configuration. +/// The network *kind* zaino is configured for. Deliberately payload-free: +/// activation heights are chain facts the validator owns, so a config value +/// cannot carry them — the backends adopt the runtime schedule from the +/// validator's `getblockchaininfo.upgrades` at spawn and hold it as a +/// `zebra_chain::parameters::Network` +/// (). A pre-adoption +/// height read is unrepresentable: this type has no heights to read. #[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Deserialize, serde::Serialize)] -#[serde(from = "NetworkSerde", into = "NetworkSerde")] pub enum Network { /// Mainnet network Mainnet, /// Testnet network Testnet, /// Regtest network (for local testing) - Regtest(ActivationHeights), + Regtest, } impl fmt::Display for Network { @@ -42,38 +27,7 @@ impl fmt::Display for Network { match self { Network::Mainnet => write!(f, "Mainnet"), Network::Testnet => write!(f, "Testnet"), - Network::Regtest(_) => write!(f, "Regtest"), - } - } -} - -/// Helper type for Network serialization/deserialization. -/// -/// This allows Network to serialize as simple strings ("Mainnet", "Testnet", "Regtest") -/// while the actual Network::Regtest variant carries activation heights internally. -#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Deserialize, serde::Serialize)] -enum NetworkSerde { - Mainnet, - Testnet, - Regtest, -} - -impl From for Network { - fn from(value: NetworkSerde) -> Self { - match value { - NetworkSerde::Mainnet => Network::Mainnet, - NetworkSerde::Testnet => Network::Testnet, - NetworkSerde::Regtest => Network::Regtest(ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS), - } - } -} - -impl From for NetworkSerde { - fn from(value: Network) -> Self { - match value { - Network::Mainnet => NetworkSerde::Mainnet, - Network::Testnet => NetworkSerde::Testnet, - Network::Regtest(_) => NetworkSerde::Regtest, + Network::Regtest => write!(f, "Regtest"), } } } @@ -209,29 +163,6 @@ impl From for ConfiguredActivationHeights { } impl Network { - /// Convert to Zebra's network type for internal use (alias for to_zebra_default). - pub fn to_zebra_network(&self) -> zebra_chain::parameters::Network { - self.into() - } - - /// Get the standard regtest activation heights used by Zaino. - pub fn zaino_regtest_heights() -> ConfiguredActivationHeights { - ConfiguredActivationHeights { - before_overwinter: Some(1), - overwinter: Some(1), - sapling: Some(1), - blossom: Some(1), - heartwood: Some(1), - canopy: Some(1), - nu5: Some(1), - nu6: Some(1), - nu6_1: None, - nu6_2: None, - nu6_3: None, - nu7: None, - } - } - /// Determines if we should wait for the server to fully sync. Used for testing /// /// - Mainnet/Testnet: Skip sync (false) because we don't want to sync real chains in tests @@ -239,18 +170,7 @@ impl Network { pub fn wait_on_server_sync(&self) -> bool { match self { Network::Mainnet | Network::Testnet => false, // Real networks - don't try to sync the whole chain - Network::Regtest(_) => true, // Local network - safe and fast to sync - } - } - - pub fn from_network_kind_and_activation_heights( - network: &zebra_chain::parameters::NetworkKind, - activation_heights: &ActivationHeights, - ) -> Self { - match network { - zebra_chain::parameters::NetworkKind::Mainnet => Network::Mainnet, - zebra_chain::parameters::NetworkKind::Testnet => Network::Testnet, - zebra_chain::parameters::NetworkKind::Regtest => Network::Regtest(*activation_heights), + Network::Regtest => true, // Local network - safe and fast to sync } } } @@ -261,62 +181,7 @@ impl From for Network { zebra_chain::parameters::Network::Mainnet => Network::Mainnet, zebra_chain::parameters::Network::Testnet(parameters) => { if parameters.is_regtest() { - let mut activation_heights = ActivationHeights { - before_overwinter: None, - overwinter: None, - sapling: None, - blossom: None, - heartwood: None, - canopy: None, - nu5: None, - nu6: None, - nu6_1: None, - nu6_2: None, - nu6_3: None, - nu7: None, - }; - for (height, upgrade) in parameters.activation_heights().iter() { - match upgrade { - zebra_chain::parameters::NetworkUpgrade::Genesis => (), - zebra_chain::parameters::NetworkUpgrade::BeforeOverwinter => { - activation_heights.before_overwinter = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Overwinter => { - activation_heights.overwinter = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Sapling => { - activation_heights.sapling = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Blossom => { - activation_heights.blossom = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Heartwood => { - activation_heights.heartwood = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Canopy => { - activation_heights.canopy = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Nu5 => { - activation_heights.nu5 = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Nu6 => { - activation_heights.nu6 = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Nu6_1 => { - activation_heights.nu6_1 = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Nu6_2 => { - activation_heights.nu6_2 = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Nu6_3 => { - activation_heights.nu6_3 = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Nu7 => { - activation_heights.nu7 = Some(height.0) - } - } - } - Network::Regtest(activation_heights) + Network::Regtest } else { Network::Testnet } @@ -325,21 +190,17 @@ impl From for Network { } } -impl From for zebra_chain::parameters::Network { - fn from(val: Network) -> Self { - match val { - Network::Regtest(activation_heights) => zebra_chain::parameters::Network::new_regtest( - Into::::into(activation_heights).into(), - ), - Network::Testnet => zebra_chain::parameters::Network::new_default_testnet(), - Network::Mainnet => zebra_chain::parameters::Network::Mainnet, - } - } -} - -impl From<&Network> for zebra_chain::parameters::Network { - fn from(val: &Network) -> Self { - (*val).into() +impl ActivationHeights { + /// Builds the runtime regtest network for a chain whose schedule is + /// known first-hand: a validator being *launched* with these heights, or + /// a test fixture that is its own chain (mockchain sources, proptest + /// block generators). Production indexer code never calls this with + /// configured values — it adopts the runtime network from the + /// validator's reported schedule instead (zaino#1076). + pub fn to_regtest_network(&self) -> zebra_chain::parameters::Network { + zebra_chain::parameters::Network::new_regtest( + Into::::into(*self).into(), + ) } } diff --git a/packages/zaino-common/src/lib.rs b/packages/zaino-common/src/lib.rs index d2c5fcce5..818f23935 100644 --- a/packages/zaino-common/src/lib.rs +++ b/packages/zaino-common/src/lib.rs @@ -17,7 +17,7 @@ pub use net::{resolve_socket_addr, try_resolve_address, AddressResolution}; // Re-export commonly used config types at crate root for backward compatibility. // This allows existing code using `use zaino_common::Network` to continue working. -pub use config::network::{ActivationHeights, Network, ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS}; +pub use config::network::{ActivationHeights, Network}; pub use config::service::ServiceConfig; pub use config::storage::{ AccumulatorRebuildMemorySize, CacheConfig, DatabaseConfig, DatabaseSize, StorageConfig, diff --git a/packages/zaino-state/src/backends.rs b/packages/zaino-state/src/backends.rs index 7af6467ef..c5461bb2d 100644 --- a/packages/zaino-state/src/backends.rs +++ b/packages/zaino-state/src/backends.rs @@ -89,6 +89,62 @@ fn build_treestate_response( ) } +/// Builds the regtest activation heights from the validator's reported +/// upgrade schedule (`getblockchaininfo.upgrades`). +/// +/// The validator's configured activation heights are authoritative: the +/// config type is a payload-free kind, so both backends construct the +/// runtime network here at first contact, before anything consumes a +/// `Network` (zaino#1076). An upgrade absent from the validator's map is +/// never-activated — nothing is backfilled from defaults. Mainnet and +/// Testnet use zebra's compiled parameters and never take this path. +fn activation_heights_from_upgrades( + upgrades: &indexmap::IndexMap< + zebra_rpc::methods::ConsensusBranchIdHex, + zebra_rpc::methods::NetworkUpgradeInfo, + >, +) -> Result { + use zebra_chain::parameters::NetworkUpgrade; + + let mut heights = zaino_common::config::network::ActivationHeights { + before_overwinter: None, + overwinter: None, + sapling: None, + blossom: None, + heartwood: None, + canopy: None, + nu5: None, + nu6: None, + nu6_1: None, + nu6_2: None, + nu6_3: None, + nu7: None, + }; + for upgrade_info in upgrades.values() { + let (upgrade, height, _status) = upgrade_info.into_parts(); + let slot = match upgrade { + // Genesis is height 0 by definition; it has no configuration slot. + NetworkUpgrade::Genesis => continue, + NetworkUpgrade::BeforeOverwinter => &mut heights.before_overwinter, + NetworkUpgrade::Overwinter => &mut heights.overwinter, + NetworkUpgrade::Sapling => &mut heights.sapling, + NetworkUpgrade::Blossom => &mut heights.blossom, + NetworkUpgrade::Heartwood => &mut heights.heartwood, + NetworkUpgrade::Canopy => &mut heights.canopy, + NetworkUpgrade::Nu5 => &mut heights.nu5, + NetworkUpgrade::Nu6 => &mut heights.nu6, + NetworkUpgrade::Nu6_1 => &mut heights.nu6_1, + NetworkUpgrade::Nu6_2 => &mut heights.nu6_2, + NetworkUpgrade::Nu6_3 => &mut heights.nu6_3, + NetworkUpgrade::Nu7 => &mut heights.nu7, + }; + if slot.replace(height.0).is_some() { + return Err(format!("validator reported {upgrade:?} twice")); + } + } + Ok(heights) +} + fn latest_network_upgrade( upgrades: &indexmap::IndexMap< zebra_rpc::methods::ConsensusBranchIdHex, @@ -128,6 +184,121 @@ fn validate_utxo_address_count(count: usize) -> Result<(), tonic::Status> { #[cfg(test)] mod tests { + use zaino_common::config::network::ActivationHeights; + + /// All-`None` heights: the starting point adoption fills from the + /// validator's map, and the expected value for every absent upgrade. + const NEVER_ACTIVATED: ActivationHeights = ActivationHeights { + before_overwinter: None, + overwinter: None, + sapling: None, + blossom: None, + heartwood: None, + canopy: None, + nu5: None, + nu6: None, + nu6_1: None, + nu6_2: None, + nu6_3: None, + nu7: None, + }; + + fn upgrades_map( + json: &str, + ) -> indexmap::IndexMap< + zebra_rpc::methods::ConsensusBranchIdHex, + zebra_rpc::methods::NetworkUpgradeInfo, + > { + serde_json::from_str(json).expect("upgrades fixture parses") + } + + fn adopted_heights( + upgrades: &indexmap::IndexMap< + zebra_rpc::methods::ConsensusBranchIdHex, + zebra_rpc::methods::NetworkUpgradeInfo, + >, + ) -> ActivationHeights { + super::activation_heights_from_upgrades(upgrades).expect("valid schedule") + } + + /// An upgrade absent from the validator's map is never-activated — + /// nothing is backfilled from any default schedule. + #[test] + fn regtest_network_from_upgrades_leaves_absent_upgrades_never_activated() { + let upgrades = upgrades_map( + r#"{ + "c2d6d0b4": { "name": "NU5", "activationheight": 2, "status": "active" }, + "c8e71055": { "name": "NU6", "activationheight": 2, "status": "active" } + }"#, + ); + + assert_eq!( + adopted_heights(&upgrades), + ActivationHeights { + nu5: Some(2), + nu6: Some(2), + ..NEVER_ACTIVATED + } + ); + } + + /// The ORCHARD_THEN_IRONWOOD transition shape: everything through NU6.2 + /// at 1–2, NU6.3 at 6 — the schedule the ironwood_activation fixtures + /// launch validators with. + #[test] + fn regtest_network_from_upgrades_reads_a_transition_schedule() { + let upgrades = upgrades_map( + r#"{ + "5ba81b19": { "name": "Overwinter", "activationheight": 1, "status": "active" }, + "76b809bb": { "name": "Sapling", "activationheight": 1, "status": "active" }, + "2bb40e60": { "name": "Blossom", "activationheight": 1, "status": "active" }, + "f5b9230b": { "name": "Heartwood", "activationheight": 1, "status": "active" }, + "e9ff75a6": { "name": "Canopy", "activationheight": 1, "status": "active" }, + "c2d6d0b4": { "name": "NU5", "activationheight": 2, "status": "active" }, + "c8e71055": { "name": "NU6", "activationheight": 2, "status": "active" }, + "4dec4df0": { "name": "NU6.1", "activationheight": 2, "status": "active" }, + "5437f330": { "name": "NU6.2", "activationheight": 2, "status": "active" }, + "37a5165b": { "name": "NU6.3", "activationheight": 6, "status": "pending" } + }"#, + ); + + assert_eq!( + adopted_heights(&upgrades), + ActivationHeights { + overwinter: Some(1), + sapling: Some(1), + blossom: Some(1), + heartwood: Some(1), + canopy: Some(1), + nu5: Some(2), + nu6: Some(2), + nu6_1: Some(2), + nu6_2: Some(2), + nu6_3: Some(6), + ..NEVER_ACTIVATED + } + ); + } + + /// A validator reporting the same upgrade twice is nonsense; adoption + /// must fail loudly rather than pick a height. + #[test] + fn regtest_network_from_upgrades_rejects_a_duplicate_upgrade() { + let upgrades = upgrades_map( + r#"{ + "c2d6d0b4": { "name": "NU5", "activationheight": 2, "status": "active" }, + "c8e71055": { "name": "NU5", "activationheight": 3, "status": "pending" } + }"#, + ); + + let reason = + super::activation_heights_from_upgrades(&upgrades).expect_err("duplicate must fail"); + assert!( + reason.contains("twice"), + "error should name the duplication, got: {reason}" + ); + } + #[test] fn latest_network_upgrade_rejects_empty_metadata() { let upgrades = indexmap::IndexMap::new(); diff --git a/packages/zaino-state/src/backends/fetch.rs b/packages/zaino-state/src/backends/fetch.rs index aefbb97fb..b452b03f2 100644 --- a/packages/zaino-state/src/backends/fetch.rs +++ b/packages/zaino-state/src/backends/fetch.rs @@ -61,7 +61,7 @@ use zaino_proto::proto::{ use crate::{ chain_index::chain_tips_from_nonfinalized_snapshot, chain_index::{source::ValidatorConnector, types}, - config::{DonationAddress, FetchServiceConfig}, + config::{ChainIndexConfig, DonationAddress, FetchServiceConfig}, error::FetchServiceError, indexer::{ handle_raw_transaction, IndexerSubscriber, LightWalletIndexer, ZcashIndexer, ZcashService, @@ -135,19 +135,52 @@ impl ZcashService for FetchService { ) .await?; + // The validator is the single source of truth for activation heights + // (zaino#1076): the config carries only a network kind, and the + // runtime network is constructed here before anything consumes one — + // from zebra's compiled parameters for the public networks, from the + // validator's reported schedule for regtest. There is no fallback: a + // silently wrong schedule is the failure mode this removes. + let network = match config.common.network { + zaino_common::Network::Mainnet => zebra_chain::parameters::Network::Mainnet, + zaino_common::Network::Testnet => { + zebra_chain::parameters::Network::new_default_testnet() + } + zaino_common::Network::Regtest => { + let blockchain_info = fetcher.get_blockchain_info().await.map_err(|error| { + FetchServiceError::Critical(format!( + "cannot fetch activation heights from the validator at {}: {error}", + config.common.validator_rpc_address + )) + })?; + let heights = super::activation_heights_from_upgrades(&blockchain_info.upgrades) + .map_err(|reason| { + FetchServiceError::Critical(format!( + "cannot adopt activation heights from the validator at {}: {reason}", + config.common.validator_rpc_address + )) + })?; + info!(?heights, "Adopted activation heights from the validator"); + heights.to_regtest_network() + } + }; + let zebra_build_data = fetcher.get_info().await?; let data = ServiceMetadata::new( get_build_info(config.common.indexer_version.clone()), - config.common.network.to_zebra_network(), + network.clone(), zebra_build_data.build, zebra_build_data.subversion, ); info!(build = %data.zebra_build(), subversion = %data.zebra_subversion(), "Connected to Zcash node"); let source = ValidatorConnector::Fetch(fetcher.clone()); - let indexer = NodeBackedChainIndex::new(source, config.clone().into()) - .await - .unwrap(); + let indexer = NodeBackedChainIndex::new( + source, + ChainIndexConfig::from_backend_config(&config.common, network), + ) + .await + .unwrap(); let fetch_service = Self { fetcher, @@ -233,10 +266,10 @@ impl FetchServiceSubscriber { self.indexer.status() } - /// Returns the network type running. - #[allow(deprecated)] - pub fn network(&self) -> zaino_common::Network { - self.config.common.network + /// Returns the runtime network, carrying the activation schedule + /// adopted from the validator (zaino#1076). + pub fn network(&self) -> zebra_chain::parameters::Network { + self.data.network() } } @@ -778,7 +811,7 @@ impl ZcashIndexer for FetchServiceSubscriber { height, confirmations, #[allow(deprecated)] - &self.config.common.network.to_zebra_network(), + &self.data.network(), None, block_hash, in_best_chain, @@ -1830,11 +1863,7 @@ impl LightWalletIndexer for FetchServiceSubscriber { .await?; Ok(super::tree_state_from_treestate_response( - self.config - .common - .network - .to_zebra_network() - .bip70_network_name(), + self.data.network().bip70_network_name(), treestate_response, )) } diff --git a/packages/zaino-state/src/backends/state.rs b/packages/zaino-state/src/backends/state.rs index 33c19c87c..321a1ddd0 100644 --- a/packages/zaino-state/src/backends/state.rs +++ b/packages/zaino-state/src/backends/state.rs @@ -7,7 +7,7 @@ use crate::{ source::ValidatorConnector, types as chain_types, ChainIndex, }, - config::{DonationAddress, StateServiceConfig}, + config::{ChainIndexConfig, DonationAddress, StateServiceConfig}, error::{BlockCacheError, StateServiceError}, indexer::{ handle_raw_transaction, IndexerSubscriber, LightWalletIndexer, ZcashIndexer, ZcashService, @@ -195,11 +195,41 @@ impl ZcashService for StateService { ) .await?; + // The validator is the single source of truth for activation heights + // (zaino#1076): the config carries only a network kind, and the + // runtime network is constructed here before anything consumes one — + // from zebra's compiled parameters for the public networks, from the + // validator's reported schedule for regtest. There is no fallback: a + // silently wrong schedule is the failure mode this removes. + let network = match config.common.network { + zaino_common::Network::Mainnet => zebra_chain::parameters::Network::Mainnet, + zaino_common::Network::Testnet => { + zebra_chain::parameters::Network::new_default_testnet() + } + zaino_common::Network::Regtest => { + let blockchain_info = rpc_client.get_blockchain_info().await.map_err(|error| { + StateServiceError::Critical(format!( + "cannot fetch activation heights from the validator at {}: {error}", + config.common.validator_rpc_address + )) + })?; + let heights = super::activation_heights_from_upgrades(&blockchain_info.upgrades) + .map_err(|reason| { + StateServiceError::Critical(format!( + "cannot adopt activation heights from the validator at {}: {reason}", + config.common.validator_rpc_address + )) + })?; + info!(?heights, "Adopted activation heights from the validator"); + heights.to_regtest_network() + } + }; + let zebra_build_data = rpc_client.get_info().await?; let data = ServiceMetadata::new( get_build_info(config.common.indexer_version.clone()), - config.common.network.to_zebra_network(), + network.clone(), zebra_build_data.build, zebra_build_data.subversion, ); @@ -212,7 +242,7 @@ impl ZcashService for StateService { let (mut read_state_service, _latest_chain_tip, chain_tip_change, sync_task_handle) = init_read_state_with_syncer( config.validator_state_config.clone(), - &config.common.network.to_zebra_network(), + &network, config.validator_grpc_address, ) .await??; @@ -259,7 +289,7 @@ impl ZcashService for StateService { let mempool_source = ValidatorConnector::State(crate::chain_index::source::State { read_state_service: read_state_service.clone(), mempool_fetcher: rpc_client.clone(), - network: config.common.network, + network: network.clone(), }); let mempool = Mempool::spawn(mempool_source, None).await?; @@ -268,9 +298,9 @@ impl ZcashService for StateService { ValidatorConnector::State(State { read_state_service: read_state_service.clone(), mempool_fetcher: rpc_client.clone(), - network: config.common.network, + network: network.clone(), }), - config.clone().into(), + ChainIndexConfig::from_backend_config(&config.common, network), ) .await .unwrap(); @@ -907,10 +937,10 @@ impl StateServiceSubscriber { } } - /// Returns the network type running. - #[allow(deprecated)] - pub fn network(&self) -> zaino_common::Network { - self.config.common.network + /// Returns the runtime network, carrying the activation schedule + /// adopted from the validator (zaino#1076). + pub fn network(&self) -> zebra_chain::parameters::Network { + self.data.network() } /// Returns the median time of the last 11 blocks. @@ -1032,18 +1062,14 @@ impl ZcashIndexer for StateServiceSubscriber { } async fn get_difficulty(&self) -> Result { - chain_tip_difficulty( - self.config.common.network.to_zebra_network(), - self.read_state_service.clone(), - false, - ) - .await - .map_err(|e| { - StateServiceError::RpcError(RpcError::new_from_errorobject( - e, - "failed to get difficulty", - )) - }) + chain_tip_difficulty(self.data.network(), self.read_state_service.clone(), false) + .await + .map_err(|e| { + StateServiceError::RpcError(RpcError::new_from_errorobject( + e, + "failed to get difficulty", + )) + }) } async fn get_block_subsidy(&self, height: u32) -> Result { @@ -1090,12 +1116,9 @@ impl ZcashIndexer for StateServiceSubscriber { }; let now = Utc::now(); - let zebra_estimated_height = NetworkChainTipHeightEstimator::new( - header.time, - height, - &self.config.common.network.into(), - ) - .estimate_height_at(now); + let zebra_estimated_height = + NetworkChainTipHeightEstimator::new(header.time, height, &self.data.network()) + .estimate_height_at(now); let estimated_height = if header.time > now || zebra_estimated_height < height { height } else { @@ -1103,10 +1126,8 @@ impl ZcashIndexer for StateServiceSubscriber { }; let upgrades = IndexMap::from_iter( - self.config - .common - .network - .to_zebra_network() + self.data + .network() .full_activation_list() .into_iter() .filter_map(|(activation_height, network_upgrade)| { @@ -1140,14 +1161,14 @@ impl ZcashIndexer for StateServiceSubscriber { (height + 1).expect("valid chain tips are a lot less than Height::MAX"); let consensus = TipConsensusBranch::from_parts( ConsensusBranchIdHex::new( - NetworkUpgrade::current(&self.config.common.network.into(), height) + NetworkUpgrade::current(&self.data.network(), height) .branch_id() .unwrap_or(ConsensusBranchId::RPC_MISSING_ID) .into(), ) .inner(), ConsensusBranchIdHex::new( - NetworkUpgrade::current(&self.config.common.network.into(), next_block_height) + NetworkUpgrade::current(&self.data.network(), next_block_height) .branch_id() .unwrap_or(ConsensusBranchId::RPC_MISSING_ID) .into(), @@ -1156,22 +1177,15 @@ impl ZcashIndexer for StateServiceSubscriber { ); // TODO: Remove unwrap() - let difficulty = chain_tip_difficulty( - self.config.common.network.to_zebra_network(), - self.read_state_service.clone(), - false, - ) - .await - .unwrap(); + let difficulty = + chain_tip_difficulty(self.data.network(), self.read_state_service.clone(), false) + .await + .unwrap(); let verification_progress = f64::from(height.0) / f64::from(zebra_estimated_height.0); Ok(GetBlockchainInfoResponse::new( - self.config - .common - .network - .to_zebra_network() - .bip70_network_name(), + self.data.network().bip70_network_name(), height, hash, estimated_height, @@ -1621,19 +1635,18 @@ impl ZcashIndexer for StateServiceSubscriber { return Ok(ValidateAddressResponse::invalid()); }; - let address = match address.convert_if_network::
( - match self.config.common.network.to_zebra_network().kind() { + let address = + match address.convert_if_network::
(match self.data.network().kind() { NetworkKind::Mainnet => NetworkType::Main, NetworkKind::Testnet => NetworkType::Test, NetworkKind::Regtest => NetworkType::Regtest, - }, - ) { - Ok(address) => address, - Err(err) => { - tracing::debug!(?err, "conversion error"); - return Ok(ValidateAddressResponse::invalid()); - } - }; + }) { + Ok(address) => address, + Err(err) => { + tracing::debug!(?err, "conversion error"); + return Ok(ValidateAddressResponse::invalid()); + } + }; Ok(match address { Address::Transparent(taddr) => ValidateAddressResponse::new( @@ -1658,21 +1671,20 @@ impl ZcashIndexer for StateServiceSubscriber { )); }; - let converted_address = match parsed_address.convert_if_network::
( - match self.config.common.network.to_zebra_network().kind() { + let converted_address = + match parsed_address.convert_if_network::
(match self.data.network().kind() { NetworkKind::Mainnet => NetworkType::Main, NetworkKind::Testnet => NetworkType::Test, NetworkKind::Regtest => NetworkType::Regtest, - }, - ) { - Ok(address) => address, - Err(err) => { - tracing::debug!(?err, "conversion error"); - return Ok(ZValidateAddressResponse::Known( - KnownZValidateAddress::Invalid(InvalidZValidateAddress::new()), - )); - } - }; + }) { + Ok(address) => address, + Err(err) => { + tracing::debug!(?err, "conversion error"); + return Ok(ZValidateAddressResponse::Known( + KnownZValidateAddress::Invalid(InvalidZValidateAddress::new()), + )); + } + }; // Note: It could be the case that Zaino needs to support Sprout. For now, it's been disabled. match converted_address { @@ -1682,13 +1694,11 @@ impl ZcashIndexer for StateServiceSubscriber { } TransparentAddress::ScriptHash(_) => Ok(ZValidateAddressResponse::p2sh(address)), }, - Address::Unified(u) => Ok(ZValidateAddressResponse::unified( - u.encode(&self.network().to_zebra_network()), - )), + Address::Unified(u) => Ok(ZValidateAddressResponse::unified(u.encode(&self.network()))), Address::Sapling(s) => { let (diversifier, pk_d) = sapling_key_bytes(&s); Ok(ZValidateAddressResponse::sapling( - s.encode(&self.network().to_zebra_network()), + s.encode(&self.network()), Some(hex::encode(diversifier)), Some(hex::encode(pk_d)), )) @@ -1836,7 +1846,7 @@ impl ZcashIndexer for StateServiceSubscriber { height, confirmations, #[allow(deprecated)] - &self.config.common.network.to_zebra_network(), + &self.data.network(), None, block_hash, in_best_chain, @@ -2515,11 +2525,7 @@ impl LightWalletIndexer for StateServiceSubscriber { .await?; Ok(super::tree_state_from_treestate_response( - self.config - .common - .network - .to_zebra_network() - .bip70_network_name(), + self.data.network().bip70_network_name(), treestate_response, )) } diff --git a/packages/zaino-state/src/chain_index.rs b/packages/zaino-state/src/chain_index.rs index 4609c0011..8d53d0d20 100644 --- a/packages/zaino-state/src/chain_index.rs +++ b/packages/zaino-state/src/chain_index.rs @@ -809,7 +809,7 @@ impl NodeBackedChainIndex { finalized_db, sync_loop_handle: None, status: NamedAtomicStatus::new("ChainIndex", StatusType::Spawning), - network: config.network.to_zebra_network(), + network: config.network.clone(), source, sync_timings, cancel_token: CancellationToken::new(), diff --git a/packages/zaino-state/src/chain_index/finalised_state.rs b/packages/zaino-state/src/chain_index/finalised_state.rs index 11c824ec1..e57875d8a 100644 --- a/packages/zaino-state/src/chain_index/finalised_state.rs +++ b/packages/zaino-state/src/chain_index/finalised_state.rs @@ -251,7 +251,7 @@ use tokio::time::{interval, MissedTickBehavior}; /// owns the loop. pub(crate) async fn build_indexed_block_from_source( source: &S, - network: zaino_common::Network, + network: zebra_chain::parameters::Network, sapling_activation_height: zebra_chain::block::Height, nu5_activation_height: Option, nu6_3_activation_height: Option, @@ -312,7 +312,7 @@ pub(crate) async fn build_indexed_block_from_source( orchard_size, ironwood, parent_chainwork, - network.to_zebra_network(), + network, ); let block_with_metadata = BlockWithMetadata::new(block.as_ref(), metadata); @@ -464,7 +464,7 @@ impl FinalisedState { return Ok(Self { db: Arc::new(Router::new(Arc::new(FinalisedSource::ephemeral( source, - cfg.network.into(), + cfg.network.clone(), None, )))), cfg, @@ -658,7 +658,7 @@ impl FinalisedState { /// - `Some(version)` if a compatible database directory is found, /// - `None` if no database is detected (fresh DB creation case). async fn try_find_current_db_version(cfg: &ChainIndexConfig) -> Option { - let legacy_dir = match cfg.network.to_zebra_network().kind() { + let legacy_dir = match cfg.network.kind() { NetworkKind::Mainnet => "live", NetworkKind::Testnet => "test", NetworkKind::Regtest => "local", @@ -668,7 +668,7 @@ impl FinalisedState { return Some(0); } - let net_dir = match cfg.network.to_zebra_network().kind() { + let net_dir = match cfg.network.kind() { NetworkKind::Mainnet => "mainnet", NetworkKind::Testnet => "testnet", NetworkKind::Regtest => "regtest", @@ -786,7 +786,7 @@ impl FinalisedState { let ephemeral_reference = router .init_or_take_ephemeral( source.clone(), - cfg.network.to_zebra_network(), + cfg.network.clone(), EphemeralMode::ReadOnly, db_height_opt, ) @@ -1098,7 +1098,7 @@ impl FinalisedState { // defaults — mirroring `build_indexed_block_from_source`. let nu6_3_activation_height = super::ShieldedPool::Ironwood .activation_upgrade() - .activation_height(&cfg.network.to_zebra_network()); + .activation_height(&cfg.network); let mut parent_chainwork: Option = None; @@ -1145,7 +1145,7 @@ impl FinalisedState { orchard_size, ironwood, parent_chainwork, - cfg.network.to_zebra_network(), + cfg.network.clone(), ); let block_with_metadata = BlockWithMetadata::new(block.as_ref(), metadata); let chain_block = IndexedBlock::try_from(block_with_metadata).map_err(|_| { diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1.rs index 73ed8b717..1d0ed091e 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1.rs @@ -453,7 +453,7 @@ impl DbV1 { // Prepare database details and path. let db_size_bytes = config.storage.database.size.to_byte_count(); - let db_path_dir = match config.network.to_zebra_network().kind() { + let db_path_dir = match config.network.kind() { NetworkKind::Mainnet => "mainnet", NetworkKind::Testnet => "testnet", NetworkKind::Regtest => "regtest", @@ -1108,7 +1108,7 @@ mod tests { async fn initial_spent_scan_reports_corrupt_value() { use lmdb::{Transaction as _, WriteFlags}; use zaino_common::network::ActivationHeights; - use zaino_common::{DatabaseConfig, Network, StorageConfig}; + use zaino_common::{DatabaseConfig, StorageConfig}; let temp_dir = tempfile::tempdir().expect("tempdir"); let config = ChainIndexConfig { @@ -1121,7 +1121,7 @@ mod tests { }, ephemeral: false, db_version: 1, - network: Network::Regtest(ActivationHeights::default()), + network: ActivationHeights::default().to_regtest_network(), }; let db = DbV1::spawn(&config).await.expect("spawn empty v1 db"); diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/tx_out_set_accumulator.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/tx_out_set_accumulator.rs index e254e70f0..df3aad9ca 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/tx_out_set_accumulator.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/tx_out_set_accumulator.rs @@ -1710,7 +1710,7 @@ mod tests { use std::sync::Arc; use tempfile::TempDir; use zaino_common::network::ActivationHeights; - use zaino_common::{DatabaseConfig, Network, StorageConfig, SyncWriteBatchSize}; + use zaino_common::{DatabaseConfig, StorageConfig, SyncWriteBatchSize}; fn p2pkh_out(value: u64) -> TxOutCompact { TxOutCompact::new(value, [0x11; 20], 0).expect("P2PKH script_type should be valid") @@ -2111,7 +2111,7 @@ mod tests { }, ephemeral: false, db_version: 1, - network: Network::Regtest(ActivationHeights::default()), + network: ActivationHeights::default().to_regtest_network(), }; let zaino_db = FinalisedState::spawn(config, source.clone()).await.unwrap(); @@ -2186,7 +2186,7 @@ mod tests { }, ephemeral: false, db_version: 1, - network: Network::Regtest(ActivationHeights::default()), + network: ActivationHeights::default().to_regtest_network(), }; let zaino_db = FinalisedState::spawn(config, source.clone()).await.unwrap(); diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/write_core.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/write_core.rs index 212ac75d3..347c0ba8a 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/write_core.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/write_core.rs @@ -211,8 +211,7 @@ impl DbWrite for DbV1 { ) -> Result<(), FinalisedStateError> { use crate::chain_index::finalised_state::build_indexed_block_from_source; - let network = self.config.network; - let zebra_network = network.to_zebra_network(); + let zebra_network = self.config.network.clone(); let sapling_activation_height = ShieldedPool::Sapling .activation_upgrade() .activation_height(&zebra_network) @@ -264,7 +263,7 @@ impl DbWrite for DbV1 { info!( start_height, target = height.0, - ?network, + ?zebra_network, "write_blocks_to_height: syncing finalised blocks" ); @@ -309,7 +308,7 @@ impl DbWrite for DbV1 { let build_start = std::time::Instant::now(); let block = build_indexed_block_from_source( source, - network, + zebra_network.clone(), sapling_activation_height, nu5_activation_height, nu6_3_activation_height, @@ -336,7 +335,7 @@ impl DbWrite for DbV1 { info!( current = next - 1, target = height.0, - ?network, + ?zebra_network, "write_blocks_to_height: syncing" ); last_progress_log = std::time::Instant::now(); diff --git a/packages/zaino-state/src/chain_index/finalised_state/migrations.rs b/packages/zaino-state/src/chain_index/finalised_state/migrations.rs index b88009194..efae33e16 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/migrations.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/migrations.rs @@ -343,7 +343,7 @@ impl MigrationManager { .router .init_or_take_ephemeral( self.source.clone(), - self.cfg.network.to_zebra_network(), + self.cfg.network.clone(), EphemeralMode::Full, db_height, ) @@ -1093,7 +1093,7 @@ impl Migration for Migration1_2_1To1_3_0 { // Guard: Ironwood data is only expected from NU6.3 activation. A rebuild-from-existing-data // migration cannot reconstruct ironwood roots/sizes for post-NU6.3 blocks. - let zebra_network = cfg.network.to_zebra_network(); + let zebra_network = cfg.network.clone(); let nu6_3_activation_height = crate::chain_index::ShieldedPool::Ironwood .activation_upgrade() .activation_height(&zebra_network); diff --git a/packages/zaino-state/src/chain_index/source.rs b/packages/zaino-state/src/chain_index/source.rs index 9b5ea63f0..3acf8f34b 100644 --- a/packages/zaino-state/src/chain_index/source.rs +++ b/packages/zaino-state/src/chain_index/source.rs @@ -10,7 +10,6 @@ use crate::SendFut; use futures::TryFutureExt as _; use incrementalmerkletree::frontier::CommitmentTree; use tower::{Service, ServiceExt as _}; -use zaino_common::Network; use zaino_fetch::jsonrpsee::{ connector::{JsonRpSeeConnector, RpcRequestError}, response::{ diff --git a/packages/zaino-state/src/chain_index/source/mockchain_source.rs b/packages/zaino-state/src/chain_index/source/mockchain_source.rs index 00cab444d..8379bf0f3 100644 --- a/packages/zaino-state/src/chain_index/source/mockchain_source.rs +++ b/packages/zaino-state/src/chain_index/source/mockchain_source.rs @@ -125,7 +125,7 @@ fn normalize_requested_addresses_for_network( /// transparent address prefixes, so output-derived transparent addresses use /// `NetworkKind::Testnet`. fn mockchain_network() -> zebra_chain::parameters::Network { - zaino_common::Network::Regtest(ActivationHeights::default()).to_zebra_network() + ActivationHeights::default().to_regtest_network() } /// A test-only mock implementation of BlockchainReader using ordered lists by height. diff --git a/packages/zaino-state/src/chain_index/source/validator_connector.rs b/packages/zaino-state/src/chain_index/source/validator_connector.rs index cf7908a94..51816b606 100644 --- a/packages/zaino-state/src/chain_index/source/validator_connector.rs +++ b/packages/zaino-state/src/chain_index/source/validator_connector.rs @@ -34,8 +34,8 @@ pub struct State { pub read_state_service: ReadStateService, /// Temporarily used to fetch mempool data. pub mempool_fetcher: JsonRpSeeConnector, - /// Current network type being run. - pub network: Network, + /// The runtime network (activation schedule adopted from the validator). + pub network: zebra_chain::parameters::Network, } /// A connection to a validator. @@ -436,7 +436,7 @@ impl BlockchainSource for ValidatorConnector { let sapling = match ShieldedPool::Sapling .activation_upgrade() - .activation_height(&state.network.to_zebra_network()) + .activation_height(&state.network) { Some(activation_height) if height >= activation_height => Some( state @@ -467,7 +467,7 @@ impl BlockchainSource for ValidatorConnector { let orchard = match ShieldedPool::Orchard .activation_upgrade() - .activation_height(&state.network.to_zebra_network()) + .activation_height(&state.network) { Some(activation_height) if height >= activation_height => Some( state @@ -498,7 +498,7 @@ impl BlockchainSource for ValidatorConnector { let ironwood = match ShieldedPool::Ironwood .activation_upgrade() - .activation_height(&state.network.to_zebra_network()) + .activation_height(&state.network) { Some(activation_height) if height >= activation_height => Some( state @@ -860,7 +860,7 @@ impl BlockchainSource for ValidatorConnector { transaction.clone(), height, None, - &state.network.to_zebra_network(), + &state.network, None, None, Some(matches!( diff --git a/packages/zaino-state/src/chain_index/tests.rs b/packages/zaino-state/src/chain_index/tests.rs index d2cd1d58e..47e954214 100644 --- a/packages/zaino-state/src/chain_index/tests.rs +++ b/packages/zaino-state/src/chain_index/tests.rs @@ -26,7 +26,7 @@ use std::path::{Path, PathBuf}; use tempfile::TempDir; use tokio::sync::OnceCell; use tokio::time::Duration; -use zaino_common::{network::ActivationHeights, DatabaseConfig, Network, StorageConfig}; +use zaino_common::{network::ActivationHeights, DatabaseConfig, StorageConfig}; use crate::{ chain_index::{ @@ -134,7 +134,7 @@ async fn load_with_settings( }, ephemeral: false, db_version: 1, - network: Network::Regtest(ActivationHeights::default()), + network: ActivationHeights::default().to_regtest_network(), }; let indexer = NodeBackedChainIndex::new_with_sync_timings(source.clone(), config, sync_timings) @@ -222,7 +222,7 @@ async fn v1_finalised_seed_dir(mode: MockchainMode) -> &'static Path { }, ephemeral: false, db_version: 1, - network: Network::Regtest(ActivationHeights::default()), + network: ActivationHeights::default().to_regtest_network(), }; let zaino_db = FinalisedState::spawn(config, source).await.unwrap(); diff --git a/packages/zaino-state/src/chain_index/tests/finalised_state/ephemeral.rs b/packages/zaino-state/src/chain_index/tests/finalised_state/ephemeral.rs index d2b455b22..34475c4ef 100644 --- a/packages/zaino-state/src/chain_index/tests/finalised_state/ephemeral.rs +++ b/packages/zaino-state/src/chain_index/tests/finalised_state/ephemeral.rs @@ -15,7 +15,7 @@ use std::sync::Arc; use tempfile::TempDir; use zaino_common::network::ActivationHeights; -use zaino_common::{DatabaseConfig, Network, StorageConfig}; +use zaino_common::{DatabaseConfig, StorageConfig}; use zaino_proto::proto::utils::{compact_block_with_pool_types, PoolTypeFilter}; use crate::chain_index::finalised_state::FinalisedState; @@ -45,7 +45,7 @@ pub(crate) async fn spawn_ephemeral_finalised_state( }, ephemeral: true, db_version: 1, - network: Network::Regtest(ActivationHeights::default()), + network: ActivationHeights::default().to_regtest_network(), }; let finalised_state = FinalisedState::spawn(config, source).await?; diff --git a/packages/zaino-state/src/chain_index/tests/finalised_state/migrations/v1_0_to_v1_1.rs b/packages/zaino-state/src/chain_index/tests/finalised_state/migrations/v1_0_to_v1_1.rs index e2db7587b..3f70ca3aa 100644 --- a/packages/zaino-state/src/chain_index/tests/finalised_state/migrations/v1_0_to_v1_1.rs +++ b/packages/zaino-state/src/chain_index/tests/finalised_state/migrations/v1_0_to_v1_1.rs @@ -3,7 +3,7 @@ use std::path::PathBuf; use tempfile::TempDir; use zaino_common::network::ActivationHeights; -use zaino_common::{DatabaseConfig, Network, StorageConfig}; +use zaino_common::{DatabaseConfig, StorageConfig}; use crate::chain_index::finalised_state::capability::{ DbCore as _, DbRead as _, DbVersion, MigrationStatus, @@ -34,7 +34,7 @@ async fn v1_0_to_v1_1_metadata_migration() { }, ephemeral: false, db_version: 1, - network: Network::Regtest(ActivationHeights::default()), + network: ActivationHeights::default().to_regtest_network(), }; let source = build_active_mockchain_source(150, blocks.clone()); @@ -100,7 +100,7 @@ async fn v1_0_to_v1_1_mixed_blockheaderdata_formats() { }, ephemeral: false, db_version: 1, - network: Network::Regtest(ActivationHeights::default()), + network: ActivationHeights::default().to_regtest_network(), }; let source = build_active_mockchain_source(initial_active_height.0, blocks.clone()); diff --git a/packages/zaino-state/src/chain_index/tests/finalised_state/migrations/v1_1_to_v1_2.rs b/packages/zaino-state/src/chain_index/tests/finalised_state/migrations/v1_1_to_v1_2.rs index 882763cdc..bfa0a32ef 100644 --- a/packages/zaino-state/src/chain_index/tests/finalised_state/migrations/v1_1_to_v1_2.rs +++ b/packages/zaino-state/src/chain_index/tests/finalised_state/migrations/v1_1_to_v1_2.rs @@ -4,7 +4,7 @@ use lmdb::{Cursor as _, Transaction as _, WriteFlags}; use std::path::PathBuf; use tempfile::TempDir; use zaino_common::network::ActivationHeights; -use zaino_common::{DatabaseConfig, Network, StorageConfig}; +use zaino_common::{DatabaseConfig, StorageConfig}; use crate::chain_index::finalised_state::capability::{ BlockCoreExt as _, CapabilityRequest, DbRead as _, DbVersion, MigrationStatus, @@ -426,7 +426,7 @@ async fn v1_1_to_v1_2_spent_index_backfill_from_old_version() { }, ephemeral: false, db_version: 1, - network: Network::Regtest(ActivationHeights::default()), + network: ActivationHeights::default().to_regtest_network(), }; let source = build_active_mockchain_source(initial_active_height.0, blocks.clone()); @@ -498,7 +498,7 @@ async fn v1_1_to_v1_2_spent_index_migration_resumes_after_crash() { }, ephemeral: false, db_version: 1, - network: Network::Regtest(ActivationHeights::default()), + network: ActivationHeights::default().to_regtest_network(), }; let source = build_active_mockchain_source(initial_active_height.0, blocks.clone()); @@ -597,7 +597,7 @@ async fn v1_2_0_cache_missing_txid_location_index_is_rebuilt() { }, ephemeral: false, db_version: 1, - network: Network::Regtest(ActivationHeights::default()), + network: ActivationHeights::default().to_regtest_network(), }; let source = build_active_mockchain_source(initial_active_height.0, blocks.clone()); diff --git a/packages/zaino-state/src/chain_index/tests/finalised_state/v1.rs b/packages/zaino-state/src/chain_index/tests/finalised_state/v1.rs index 1ef3c02ce..d3febab5b 100644 --- a/packages/zaino-state/src/chain_index/tests/finalised_state/v1.rs +++ b/packages/zaino-state/src/chain_index/tests/finalised_state/v1.rs @@ -4,7 +4,7 @@ use hex::ToHex; use std::path::PathBuf; use tempfile::TempDir; use zaino_common::network::ActivationHeights; -use zaino_common::{DatabaseConfig, Network, StorageConfig}; +use zaino_common::{DatabaseConfig, StorageConfig}; use zaino_proto::proto::utils::{compact_block_with_pool_types, PoolTypeFilter}; use crate::chain_index::finalised_state::finalised_source::FinalisedSource; @@ -44,7 +44,7 @@ pub(crate) async fn spawn_v1_zaino_db( }, ephemeral: false, db_version: 1, - network: Network::Regtest(ActivationHeights::default()), + network: ActivationHeights::default().to_regtest_network(), }; let zaino_db = FinalisedState::spawn(config, source).await.unwrap(); @@ -193,7 +193,7 @@ async fn save_db_to_file_and_reload() { }, ephemeral: false, db_version: 1, - network: Network::Regtest(ActivationHeights::default()), + network: ActivationHeights::default().to_regtest_network(), }; let source = build_mockchain_source(blocks.clone()); @@ -272,7 +272,7 @@ async fn load_db_backend_from_file() { }, ephemeral: false, db_version: 1, - network: Network::Regtest(ActivationHeights::default()), + network: ActivationHeights::default().to_regtest_network(), }; let finalized_state_backend: FinalisedSource = FinalisedSource::spawn_v1(&config).await.unwrap(); @@ -338,7 +338,7 @@ async fn try_write_invalid_block() { orchard_tree_size as u32, None, None, // no parent chainwork for this test - zaino_common::Network::Regtest(ActivationHeights::default()).to_zebra_network(), + ActivationHeights::default().to_regtest_network(), ); let mut chain_block = diff --git a/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs b/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs index ddb17407a..54f273917 100644 --- a/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs +++ b/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs @@ -7,7 +7,7 @@ use proptest::{ }; use rand::seq::IndexedRandom; use tokio_stream::StreamExt as _; -use zaino_common::{network::ActivationHeights, DatabaseConfig, Network, StorageConfig}; +use zaino_common::{network::ActivationHeights, DatabaseConfig, StorageConfig}; use zaino_fetch::jsonrpsee::response::address_deltas::{ GetAddressDeltasParams, GetAddressDeltasResponse, }; @@ -58,7 +58,7 @@ fn passthrough_test( ), ) { passthrough_test_on( - Network::Regtest(ActivationHeights::default()), + ActivationHeights::default().to_regtest_network(), // Slow the source enough to hold the indexer in passthrough while the // assertions run, without slowing passthrough more than necessary. Some(Duration::from_millis(100)), @@ -76,7 +76,7 @@ fn passthrough_test( /// header's merkle root is already arbitrary — the passthrough path tolerates that by /// construction. fn passthrough_test_on( - network: Network, + network: zebra_chain::parameters::Network, source_delay: Option, mutate_segment: impl Fn(&mut Vec>), test: impl AsyncFn( @@ -92,7 +92,7 @@ fn passthrough_test_on( // from this line to `runtime.block_on(async {` are all // copy-pasted. Could a macro get rid of some of this boilerplate? - proptest::proptest!(proptest::test_runner::Config::with_cases(1), |(segments in make_branching_chain(branch_count, segment_length, network))| { + proptest::proptest!(proptest::test_runner::Config::with_cases(1), |(segments in make_branching_chain(branch_count, segment_length, network.clone()))| { let runtime = tokio::runtime::Builder::new_multi_thread().worker_threads(2).enable_time().build().unwrap(); runtime.block_on(async { let (mut genesis_segment, mut branching_segments) = segments; @@ -120,7 +120,7 @@ fn passthrough_test_on( }, ephemeral: true, db_version: 1, - network, + network: network.clone(), }; @@ -602,7 +602,7 @@ fn metadata_consistency_for_era( }; passthrough_test_on( - Network::Regtest(heights), + heights.to_regtest_network(), // No artificial source delay: this test waits for the indexer to finish // syncing, because compact blocks are not served while the finalised state // is still syncing (get_compact_block's StillSyncingFinalizedState arm). @@ -791,14 +791,14 @@ fn metadata_consistency_for_era( #[test] fn make_chain() { init_tracing(); - let network = Network::Regtest(ActivationHeights::default()); + let network = ActivationHeights::default().to_regtest_network(); let segment_length = 12; let branch_count = 2; // default is 256. As each case takes multiple seconds, this seems too many. // TODO: this should be higher than 1. Currently set to 1 for ease of iteration - proptest::proptest!(proptest::test_runner::Config::with_cases(1), |(segments in make_branching_chain(branch_count, segment_length, network))| { + proptest::proptest!(proptest::test_runner::Config::with_cases(1), |(segments in make_branching_chain(branch_count, segment_length, network.clone()))| { let runtime = tokio::runtime::Builder::new_multi_thread().worker_threads(2).enable_time().build().unwrap(); runtime.block_on(async { let (genesis_segment, branching_segments) = segments; @@ -822,7 +822,7 @@ fn make_chain() { }, ephemeral: true, db_version: 1, - network, + network: network.clone(), }; @@ -1257,9 +1257,9 @@ fn make_branching_chain( // The length of the initial segment, and of the branches // TODO: it would be useful to allow branches of different lengths. chain_size: usize, - network_override: Network, + network_override: zebra_chain::parameters::Network, ) -> BoxedStrategy<(ChainSegment, Vec)> { - let network_override = Some(network_override.to_zebra_network()); + let network_override = Some(network_override); add_segment( SummaryDebug(Vec::new()), network_override.clone(), diff --git a/packages/zaino-state/src/config.rs b/packages/zaino-state/src/config.rs index 0083dbcd4..282af0a93 100644 --- a/packages/zaino-state/src/config.rs +++ b/packages/zaino-state/src/config.rs @@ -141,10 +141,8 @@ impl StateServiceConfig { network: Network, donation_address: Option, ) -> Self { - tracing::trace!( - activations = ?network.to_zebra_network().full_activation_list(), - "state service expecting NU activations" - ); + // The config carries only the network kind; the activation schedule + // is adopted from the validator at spawn and logged there (#1076). StateServiceConfig { common: CommonBackendConfig { validator_rpc_address, @@ -212,8 +210,10 @@ pub struct ChainIndexConfig { pub storage: StorageConfig, /// Database version selected to be run. pub db_version: u32, - /// Network type. - pub network: Network, + /// The runtime network, carrying the activation schedule adopted from + /// the validator (zaino#1076) — or, in fixtures that are their own + /// chain, the fixture's schedule. + pub network: zebra_chain::parameters::Network, /// Ephemeral finalised state: /// /// If true, FinalisedState does not write data to disk, @@ -225,9 +225,14 @@ pub struct ChainIndexConfig { } impl ChainIndexConfig { - /// Returns a new instance of [`BlockCacheConfig`]. + /// Returns a new instance of [`ChainIndexConfig`]. #[allow(dead_code)] - pub fn new(storage: StorageConfig, db_version: u32, network: Network, ephemeral: bool) -> Self { + pub fn new( + storage: StorageConfig, + db_version: u32, + network: zebra_chain::parameters::Network, + ephemeral: bool, + ) -> Self { ChainIndexConfig { storage, db_version, @@ -235,34 +240,25 @@ impl ChainIndexConfig { ephemeral, } } -} -impl From for ChainIndexConfig { - fn from(value: CommonBackendConfig) -> Self { + /// Builds the chain-index config from the backend config plus the + /// runtime network. The backend config carries only a network kind, so + /// the adopted runtime network arrives as its own argument — there is + /// no conversion from a service config alone. + pub fn from_backend_config( + common: &CommonBackendConfig, + network: zebra_chain::parameters::Network, + ) -> Self { Self { - storage: value.storage, + storage: common.storage.clone(), // TODO: update zaino configs to include db version. db_version: 1, - network: value.network, - ephemeral: value.ephemeral_finalised_state, + network, + ephemeral: common.ephemeral_finalised_state, } } } -#[allow(deprecated)] -impl From for ChainIndexConfig { - fn from(value: StateServiceConfig) -> Self { - value.common.into() - } -} - -#[allow(deprecated)] -impl From for ChainIndexConfig { - fn from(value: FetchServiceConfig) -> Self { - value.common.into() - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/packages/zainod/src/config.rs b/packages/zainod/src/config.rs index aeb97dbbe..1a1ff77fd 100644 --- a/packages/zainod/src/config.rs +++ b/packages/zainod/src/config.rs @@ -222,11 +222,6 @@ impl ZainodConfig { Ok(()) } - - /// Returns the network type currently being used by the server. - pub fn get_network(&self) -> Result { - Ok(self.network.to_zebra_network()) - } } impl Default for ZainodConfig { diff --git a/zainod-heights-from-validator-spec.md b/zainod-heights-from-validator-spec.md new file mode 100644 index 000000000..66725a501 --- /dev/null +++ b/zainod-heights-from-validator-spec.md @@ -0,0 +1,170 @@ +# Spec: zainod learns regtest activation heights from the Validator + +Solves zingolabs/zaino#1076. Requested by infras (`zingolabs/infrastructure#278`, +`bump_to_NU6.3`) in coordination with zaino's `ironwood_activation` e2e suite +(zingolabs/zaino#1368). Companion spec on the infras side: +`infras/dev/zaino-ironwood-activation-infra-spec.md` (devtool wallets on +arbitrary regtest heights — "Provider heights"). + +Governing invariant, decided 2026-07-06 (recorded as infras ADR 0003): + +> **The Validator's configured activation heights are authoritative. Every +> other component mirrors them by the strongest mechanism its protocol +> allows.** No component may treat compiled-in or config-file regtest +> heights as chain truth. + +Concretely, per component: + +- **Indexer (this spec)**: queries the Validator's `getblockchaininfo` at + startup and adopts the reported schedule. +- **Wallet client (infras side, same release as the guard lift)**: the + light-client protocol doesn't expose the schedule, so the wallet can't + ask the Validator itself — the harness queries `getblockchaininfo` on + the wallet's behalf and writes the *derived* heights into the wallet's + `activation-heights.toml`. The derivation is enforced at compile time: + the wallet config's regtest variant demands an opaque + `ValidatorHeights`, whose only public constructor is + `WalletNetwork::from_validator(&validator)`. There is no + caller-supplied-heights escape hatch; writing a height vector into a + wallet config is unrepresentable. + +Consequence for zaino's e2e suite: once both sides land, your +`ironwood_activation` fixture heights get typed in exactly one place — the +zebrad launch config. The wallet derives them via +`WalletNetwork::from_validator`; zainod adopts them via this spec; no +test-side constant needs to mirror anything. Expect the zcash_local_net +pin bump to be loud: `ZainodConfig.network` narrows to a payload-free +kind (Mainnet/Testnet/Regtest), `ZcashDevtoolConfig::faucet()/recipient()` +take a `WalletNetwork` parameter, and any test that assigned heights into +a wallet config stops compiling — the fix is always to launch the +validator first and derive. + +## Problem + +zainod's regtest activation heights are a compiled-in constant, silently +assumed to match whatever zebrad was launched with: + +- The zainod TOML carries only the string `network = "Regtest"`. + Deserialization inflates it to + `Network::Regtest(ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS)` + (`packages/zaino-common/src/config/network.rs:66`); the constant + (`network.rs:13-26`) documents itself as "must equal zcash_local_net's + `supported_regtest_activation_heights`" — a cross-repo mirror maintained + by hand across three repos (zebra config, zaino constant, infras canonical + heights). +- That `Network` flows into everything downstream as truth: + `StateService::spawn` passes `config.common.network.to_zebra_network()` + into `init_read_state_with_syncer` (`zaino-state/src/backends/state.rs:213-218`) + and into `NodeBackedChainIndex::new` (`state.rs:267-276`); + `FetchService::spawn` does the equivalent (`backends/fetch.rs:122+`). +- When the running zebrad's heights differ, block commitment verification + computes against the wrong upgrade schedule and the chain-index sync loop + dies with `InvalidData("Block commitment could not be computed")` + (`zaino-state/src/chain_index/types/helpers.rs:183`, + `block.commitment(network)`). + +The mismatch is not hypothetical: infras is lifting its client-side guard so +devtool wallets run on **arbitrary** regtest heights, and zaino's own +`ironwood_activation` fixture (`ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS`) +puts NU6.3 at height 6 with everything else at 1–2. No compiled constant can +ever match caller-chosen heights. The only correct source is the running +zebrad — and it already publishes its schedule: `getblockchaininfo` returns +an `upgrades` map with per-upgrade `activationheight`, which zaino already +parses (`GetBlockchainInfoResponse.upgrades`, +`zaino-fetch/src/jsonrpsee/response.rs:356-359`, round-trip tested at +`response.rs:530+`). Both backends already call the validator at startup +(`get_info` at `state.rs:198`, `get_blockchain_info` in the tip-wait loop at +`state.rs:226`). The handshake exists; the heights are simply never adopted +from it. + +## Requested change + +1. **Learn heights at first contact.** When the configured network kind is + Regtest, each backend fetches `getblockchaininfo` from the validator + **before** constructing anything that consumes a + `zebra_chain::parameters::Network` — the ReadStateService syncer, the + chain index, the mempool source. Build the runtime regtest `Network` from + the reported `upgrades` map. The RPC client is already constructed first + (`state.rs:190-196`), so the ordering is achievable without restructuring. +2. **Learned heights are the only source.** `ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS` + ceases to exist as a truth source. Recommended shape: the *config-level* + network type degrades to a kind (Mainnet / Testnet / Regtest, no payload), + and only the *runtime* type constructed after the handshake carries + heights — making pre-handshake height reads unrepresentable. A minimal + variant (populate the existing payload from the handshake before first + use) is acceptable if the type split is too invasive, but then nothing may + read the payload before adoption. Behavior is mandated; shape is + implementer's choice. +3. **An upgrade absent from the validator's map is never-activated.** This + matches zebra's own semantics and the devtool's absent-key encoding. + Do not backfill absent upgrades from any default. +4. **Failure is loud, never defaulted.** If the network kind is Regtest and + the upgrades map cannot be fetched or parsed, startup fails with an error + naming the validator endpoint. Under no circumstances fall back to a + compiled or configured height set — a silently wrong schedule is the + exact failure mode this spec removes. + +## Acceptance + +The fixture that must work end-to-end (as zebrad regtest config heights): + +``` +overwinter = 1, sapling = 1, blossom = 1, heartwood = 1, canopy = 1, +nu5 = 2, nu6 = 2, nu6_1 = 2, nu6_2 = 2, nu6_3 = 6 +``` + +1. **Boundary sync**: launch zebrad regtest with the fixture; launch zainod + against it with only `network = "Regtest"` in its TOML. Mine past height + 6. zainod's chain index syncs across the NU6.3 boundary with no + commitment error, and serves compact blocks for both eras (Orchard + coinbase at heights 2–5, Ironwood from 6). +2. **No-recompile proof**: the same zainod binary and config, pointed at a + zebrad running a *different* schedule (the canonical all-at-2 set), also + syncs — demonstrating heights come from the validator, not the build. +3. **Contract test on the upgrades map**: pin the `getblockchaininfo → + heights` mapping with a test against real zebrad output (live regtest or + a recorded golden response) — not hand-written assertions about what the + RPC is believed to return. In particular, establish from the real output + (not from reasoning) which upgrades zebrad includes in the map (e.g. + whether anything pre-Overwinter appears) and encode exactly that. +4. **No residual truth source**: no non-test code path consumes + `ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS` or any other compiled regtest + schedule as chain truth. Test-only fixtures that *are* their own chain + (mockchain sources, proptest block generators) legitimately keep explicit + heights — there, the test is the validator. + +## Known unknowns to check while implementing + +- **StateService init ordering**: `init_read_state_with_syncer` receives the + network at construction and the ReadStateService validates blocks with it. + Confirm nothing else needs a `Network` before the validator RPC is + reachable; if something does, that is a design conflict to surface, not + paper over. +- **zcashd as validator**: the legacy e2e path (`devtool_zcashd.rs`) drives + zcashd, whose `getblockchaininfo` also reports an `upgrades` map. Prefer + making the adoption path validator-generic so zcashd rides along; if its + map shape differs materially, scope this spec to zebrad and record the + divergence in the zcashd test docs. +- **zaino-testutils**: `live-tests/zaino-testutils/src/lib.rs:497` uses the + constant to *launch zebrad* — that is legitimate harness-side + configuration of the truth source, not adoption of it. Keep the height + set, but move/rename it so it can no longer be mistaken for (or imported + as) zainod's own schedule. +- **Reorg/restart**: heights are learned once at startup. A validator + restarted mid-session with different heights invalidates the indexer's + world; document that zainod must be restarted with its validator (regtest + harnesses already do this), rather than attempting live re-adoption. + +## Out of scope + +- Mainnet / Testnet: compiled zebra network parameters are correct there; no + handshake is needed and none should be added. +- The infras-side client guard lift (Provider heights) — lands on + `infrastructure#278` independently. + +## Delivery + +A rev on a zaino branch that `live-tests` can exercise. Downstream effects +when it lands: zaino#1368's `ironwood_activation` tests gain a zainod that +can actually serve their fixture, and infras' nu6_3-at-6 integration test +(currently `#[ignore]`d naming #1076) flips live. From d445446640bb2c02f3a2d248aa0fcfe50d09df45 Mon Sep 17 00:00:00 2001 From: zancas Date: Mon, 6 Jul 2026 22:44:55 -0700 Subject: [PATCH 099/134] test(e2e): wallet-tier Ironwood activation suite on the transition fixture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ironwood_activation suite exercises the wallet-observable predicates across the Orchard-to-Ironwood boundary on ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS, the hermetic replay of the public testnet's NU6.3 activation at height 4,134,000. The wallets derive their schedule from the running validator and zainod adopts the same schedule over getblockchaininfo, so the fixture heights are typed exactly once, in the zebrad launch config. unified_receipt_lands_in_orchard_before_boundary pins Orchard-era receipt semantics. orchard_note_spends_to_ironwood_across_boundary exercises the ZIP 318 migration shape: an Orchard note minted before the boundary, spent after activation to a freshly generated unified address, with the receipt asserted in the Ironwood pool. It also pins that the send actually exits the Orchard pool, which is sound under the cross-address restriction because change can return only to the spent note's own receiver. Both cells register on the fetch and state backends. The public testnet can never host the migration cell: its pre-NU6.3 epoch is closed and no pre-activation Orchard TAZ is obtainable, so this fixture is the only controlled venue. Renames apply the pool-naming convention (NU6_3_ACTIVE_* becomes IRONWOOD_ONLY_*: pool names pair with pool names, and NU6.3 names only the upgrade itself). CONTEXT.md gains the supporting glossary entries — testnet versus regtest nets, the era-naming convention, and the cross-address restriction with its normative constraint quoted verbatim. docs/notes/ironwood-activation-plan.md records the design decisions and their sources. Co-Authored-By: Claude Fable 5 --- CONTEXT.md | 42 +++ docs/notes/ironwood-activation-plan.md | 227 ++++++++++++++++ .../tests/compact_block_consistency.rs | 6 +- live-tests/e2e/tests/compact_block_wire.rs | 4 +- live-tests/e2e/tests/ironwood_activation.rs | 243 ++++++++++++++++++ live-tests/zaino-testutils/src/lib.rs | 2 +- .../chain_index/tests/proptest_blockgen.rs | 6 +- 7 files changed, 521 insertions(+), 9 deletions(-) create mode 100644 docs/notes/ironwood-activation-plan.md create mode 100644 live-tests/e2e/tests/ironwood_activation.rs diff --git a/CONTEXT.md b/CONTEXT.md index 11abf3640..4b4e11ba8 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -30,6 +30,48 @@ version is bumped. An unchanged crate keeping its published version is not a violation. _Avoid_: stale version, forgotten bump +### Chains and networks + +**Testnet**: +The public Zcash test network, and only that. Testnet regimes are +non-hermetic — state is shared with other participants, and an epoch +the public chain has left (e.g. pre-NU6.3 once NU6.3 activates there) +cannot be re-entered. +_Avoid_: "testnet" for any locally-launched chain, even one launched +under a testnet network kind + +**Regtest net**: +A hermetic, locally-launched chain whose activation heights the +launcher chooses. Every hermetic local net is a regtest net, whatever +network-kind flag it runs under. +_Avoid_: local testnet, custom testnet + +### Pools and upgrades + +**Ironwood / Orchard (era naming)**: +Eras, fixtures, and predicates that speak of shielded pools are named by +pool — Orchard, Ironwood — and a name that mentions one pool pairs with +the other pool's name, never with the upgrade's. **NU6.3** names only the +network upgrade itself: activation heights, consensus branch ID, +consensus rules. +_Avoid_: mixing vocabularies in one name or one sibling set (e.g. an +`ORCHARD_ONLY_*` fixture whose sibling is `NU6_3_ACTIVE_*` — the sibling +is `IRONWOOD_ONLY_*`) + +**Cross-address restriction**: +The post-NU6.3 rule the Orchard Action circuit enforces: "(g_d, pk_d) +of the output note must equal (g_d, pk_d) of the spent note" — the +output note must carry the same expanded receiver (diversified base +g_d, diversified transmission key pk_d) as its spent note, so each +Orchard action is either change to the spent note's own address or a +withdrawal (positive value balance). Orchard-to-Orchard transfers to +any other address — including another address of the same wallet — are +prohibited. A companion transaction-level rule forbids new value +entering the pool. Source: + +_Avoid_: "exit-only" (overclaims — same-receiver change still lands in +the pool and its commitment tree still grows) + ### TLS and cryptography **Preferred CryptoProvider**: diff --git a/docs/notes/ironwood-activation-plan.md b/docs/notes/ironwood-activation-plan.md new file mode 100644 index 000000000..abb2d3b97 --- /dev/null +++ b/docs/notes/ironwood-activation-plan.md @@ -0,0 +1,227 @@ +# Ironwood activation: test design and heights source-of-truth roadmap + +State capture, 2026-07-06. Design session artifacts live in this tree +(uncommitted at time of writing); coordination artifacts live in +`zingolabs/infrastructure` (PR #278 worktree). This note is the durable +record of the decisions and the facts they rest on. + +## Domain facts, with sources + +- **Ironwood is a new shielded pool** activated by NU6.3 + (). It shares the Action-circuit shape + with Orchard but is a distinct pool. +- **Public testnet activated NU6.3 at height 4,134,000, ~2026-07-04.** + Defined identically in zebra-chain 11.0.0 + (`parameters/constants.rs::activation_heights::testnet::NU6_3`) and + zcash_protocol 0.10.0-pre.0 (`consensus.rs`), consensus branch ID + `0x37a5165b` in both. Wallet stack and validator stack therefore flip eras + at the same height with no drift. **Mainnet has no NU6.3 height** in those + pins. (Gotcha: public explorers can mislabel testnet/mainnet heights — + verify against a synced node.) +- **ZIP 318 (zcash/zips PR #1317), "Orchard to Ironwood migration"**: wallet + best-practices for a two-phase scheduled migration. Normative anchor: "The + user MUST be able to migrate Orchard-pool funds to the Ironwood pool, and + MUST be informed that doing so is necessary to retain access to those + funds." The wallet-visible migration transaction is an Orchard spend with + Ironwood outputs. +- **Cross-address restriction** + (): + post-NU6.3 the Orchard Action circuit requires "(g_d, pk_d) of the output + note must equal (g_d, pk_d) of the spent note" — every Orchard action is + either change to the spent note's own address or a withdrawal (positive + value balance). Cross-address Orchard transfers are circuit-impossible, + including between two addresses of one wallet. A companion + transaction-level rule forbids new value entering the pool. + - Do **not** describe Orchard as "exit-only": same-receiver change still + lands in the pool, so the Orchard note-commitment tree keeps growing + after activation. There is **no frozen-finalRoot predicate**; the + correct chain-walk predicate is **Orchard pool value non-increasing from + the boundary** (observable via `valuePools`). +- **No Orchard TAZ is obtainable** (and the testnet pre-epoch is closed): + post-activation nothing new can enter Orchard, so pre-activation Orchard + notes exist only in wallets that held them before the flip. Zaino holds + none. Consequence: the migration cell is exercisable **only on hermetic + regtest transition chains** — permanently. + +## Language decisions (canonical in CONTEXT.md) + +- **Testnet** names only the public test network. Every hermetic local net + is a **regtest net**, whatever network-kind flag it runs under. +- **Pool names pair with pool names** (Orchard ↔ Ironwood); **NU6.3** names + only the upgrade itself (activation heights, branch ID, consensus rules). + Applied renames: `NU6_3_ACTIVE_ACTIVATION_HEIGHTS` → + `IRONWOOD_ONLY_ACTIVATION_HEIGHTS` (zaino-testutils), + `NU6_3_ACTIVE_HEIGHTS` → `IRONWOOD_ONLY_HEIGHTS` + (`proptest_blockgen.rs`). `NU6_3_TRANSITION_BOUNDARY` keeps the upgrade + vocabulary: an activation height is the upgrade's concept. + +## Design model: predicates × regimes + +A **regime** is a (network, era) pair; a **predicate** is a +wallet-observable claim whose truth is fixed per regime. The suite is the +truth table; tests instantiate predicates in every reachable regime. + +| predicate | Orchard era | Ironwood era | +|---|---|---| +| unified-address receipt lands in Orchard | true | false | +| unified-address receipt lands in Ironwood | false (pool inactive) | true | +| shielded-receiver coinbase pays the era pool | Orchard | Ironwood | +| Orchard note spends into an Ironwood receipt (ZIP 318 migration) | n/a | true | +| `shield` deposits into the era pool | Orchard | Ironwood | +| anything can land in Orchard from outside | true | false (no-new-value rule) | +| Orchard pool value non-increasing | n/a | true (cross-address restriction) | + +Regime venues: + +- **Regtest, Ironwood-only** (`IRONWOOD_ONLY_ACTIVATION_HEIGHTS`, canonical + NU6.3-at-2): the existing devtool wallet tests (`send_to_ironwood`, + `shield_for_validator`, `receives_mining_reward`, `send_to_all` in + `live-tests/e2e/tests/devtool.rs`). +- **Regtest, Orchard-only** (`ORCHARD_ONLY_ACTIVATION_HEIGHTS`): clientless + / wire mining fixtures only (devtool wallet cannot run there today). +- **Regtest, transition** (`ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS`, + NU6.3 at `NU6_3_TRANSITION_BOUNDARY` = 6): coinbase-routing and era + composition covered clientless + (`clientless/tests/compact_block_consistency.rs`) and over the wire + (`e2e/tests/compact_block_wire.rs`); **wallet cells live in + `e2e/tests/ironwood_activation.rs`** (see below). +- **Public testnet**: observational boundary walk across height 4,134,000 — + durable forever, historical blocks don't expire; third-party ZIP 318 + migrations supply the Orchard-spend traffic zaino must index but cannot + author. Active cells post-flip only, Orchard-free: faucet TAZ → + transparent receipt → `shield` → Ironwood send/receive. Env-gated manual + runs; record txids/heights in a dated evidence log + (`docs/notes/ironwood-activation-evidence.md`, created on first run). + `ZEBRAD_TESTNET_CACHE_DIR` (zaino-testutils) anticipates the cached sync. + +## Landed artifacts (this tree) + +- **`live-tests/e2e/tests/ironwood_activation.rs`** — the transition-regime + wallet cells, full real bodies, gated + `#[should_panic(expected = "UnsupportedActivationHeights")]`: + - `unified_receipt_lands_in_orchard_before_boundary` + - `orchard_note_spends_to_ironwood_across_boundary` — the migration cell; + recipient unified address generated *after* activation; asserts the + faucet's Orchard balance strictly shrinks, guarding against devtool note + selection dodging the migration by spending the boundary-height Ironwood + coinbase instead (sound under the cross-address restriction: change + returns only to the spent note's receiver, so a genuine Orchard spend + nets sent-plus-fee out of the pool). + - Verified: both pass as should_panic in ~5 s each + (`cargo nextest run -p e2e -E 'binary(ironwood_activation)'`). + - Registered on one backend (FetchService) while known-red: the panic + fires before the backend is exercised; mirror the fetch/state matrix + when the cells go live. +- **`e2e::devtool::build_clients_at(port, &heights)`** — devtool wallets on + caller-chosen regtest heights; `build_clients` delegates with the + canonical set. +- CONTEXT.md glossary entries; the renames above. + +## The blocker, and the infra handoff + +`zcash_local_net`'s devtool client rejects every regtest heights set except +its canonical NU6.3-at-2 one at wallet launch +(`ClientError::UnsupportedActivationHeights`; the guard exists because +heights drift between wallet and validator produces "incorrect consensus +branch id" — see zaino#1368). The guard predates the current devtool, which +reads heights at `init` from the `--activation-heights` TOML the client +already writes. + +Handoff: **`~/src/zingolabs/infras/dev/zaino-ironwood-activation-infra-spec.md`** +(for zingolabs/infrastructure PR #278, branch `bump_to_NU6.3`; zaino pins +rev `0dc4a51f...`). Asks: lift the equality guard (canonical set stays the +default), retire the error variant (zaino's known-red tests key on its name +so the pin bump flips them loudly), keep the caller-responsibility doc +contract, plus an infra-side integration test proving era-correct scanning +and branch-ID selection from file heights. Known unknown flagged: devtool +note-selection policy across pools. + +Scope addition (confirmed): infra also narrows `ZainodConfig.network` from +`NetworkType` to a payload-free `NetworkKind` in the same release — the +heights payload there is dead code (discarded at +`network_type_to_string()`; the Indexer never received heights that way). +Validator configs keep full `NetworkType`: they configure the source of +truth. + +## Roadmap: source of truth first + +Current directive (supersedes "unblock the suite next"): **first** make the +Validator the single source of truth for activation heights, with +regression tests that keep that source unique, and fix zaino#1076. + +Invariant: *the Indexer learns activation heights from the Validator at +startup and accepts them from nowhere else; configs carry network kind +only.* + +Audit facts grounding the implementation: + +- zainod's TOML is already kind-only: `NetworkSerde` serializes + "Mainnet"/"Testnet"/"Regtest", and deserializing "Regtest" injects the + compiled-in `ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS` + (`zaino-common/src/config/network.rs`). The real second channel is the + library seam: `Network::Regtest(ActivationHeights)` constructed + programmatically (e.g. by `TestManager`) and consumed via + `config.get_network()` → `to_zebra_network()`. +- The validator channel already exists in zaino's types: + `GetBlockchainInfoResponse.upgrades` is parsed (zaino-fetch + `jsonrpsee/response.rs`) and consumed (`backends/fetch.rs`, + `latest_network_upgrade`). The work is deriving the runtime + `zebra_chain::parameters::Network` from that response at startup and + narrowing the config type to kind-only. +- Regression tests that maintain uniqueness: launch the validator at + non-canonical heights (the transition fixture) with a kind-only-configured + zainod and assert era-correct serving (fails if anyone re-couples zainod + to a compiled-in set); the narrowed config type itself is the compile-time + regression test; a serde test pins the TOML shape. +- Payoff: the `InvalidData("Block commitment could not be computed")` + config-drift failure class becomes structurally impossible — the same + invariant the infra guard lift expresses on the wallet side (wallets get + heights from a file matched to the validator; the indexer gets them from + the validator directly). +- **#1076 residue** (issue text predates the all-at-2 canonical set, which + already replaced the NU6.1-at-1000 sentinel): wire + `lockbox_disbursements` into zebrad-launching fixtures (plumbing landed + at pinned rev, infra#243); add one NU6.1 boundary-crossing test (same + transition-fixture pattern as the Ironwood suite); watch for + block-commitment regressions. + +## Sequencing + +1. Zaino: heights source-of-truth implementation + uniqueness regression + tests (**landed in this tree**: `regtest_network_from_upgrades` + + spawn-time adoption in the zaino-state backends, de-mirrored testutils, + unit tests beside the mapping, live regressions in + `clientless/tests/validator_heights.rs`); remaining from this item: the + config-type split (kind-only config type, heights only post-handshake) + and the #1076 residue (lockbox fixtures, NU6.1 boundary test). +2. Infra: `NetworkKind` narrowing plus the strict form of the wallet-side + change on `bump_to_NU6.3`. The standing directive (2026-07-06) is that + the Validator is the *only* source of truth, enforced at compile time: + the harness derives the wallet's `activation-heights.toml` from + `Validator::get_activation_heights()`, and no client API accepts + caller-supplied heights (the spec's original "lift the guard, accept + caller heights" shape is superseded — a caller heights parameter is + itself a second source). +3. Zaino pin bump: adapt to the narrowed `ZainodConfig`; the two + `ironwood_activation` tests flip red (their expected + `UnsupportedActivationHeights` panic disappears along with the error + variant); replace `e2e::devtool::build_clients_at(port, &heights)` — + which exists only to express the caller-supplied shape — with a + derivation-based builder, so a fixture's heights are typed in exactly + one place: the zebrad launch config. First live runs settle devtool + note selection and the compact form of Orchard-spend data. +4. Deferred cells: testnet observational walk + post-flip Ironwood active + cells (env-gated, evidence log); chain-walk cells for the cross-address + restriction (Orchard pool-value monotonicity, same-receiver-change-only + commitments); the Orchard spend-window/freeze sub-regime waits on a + consensus source. + +## Unrelated finding from the same session + +The red `Integration tests / RPC tests shard-2/3` statuses on zaino commits +(zcash/integration-tests runs) are an upstream zallet break, not zaino's: +every run since 2026-07-03 ~23:25 UTC fails `zcashd_key_import{,_db}.py` in +zallet's `migrate-zcashd-wallet` with `UNIQUE constraint failed: +addresses.cached_transparent_receiver_address`, immediately after zallet +main's librustzcash/NU6.3 dependency bumps. No zallet issue existed for it +as of 2026-07-06. diff --git a/live-tests/clientless/tests/compact_block_consistency.rs b/live-tests/clientless/tests/compact_block_consistency.rs index 98d60daa8..753456026 100644 --- a/live-tests/clientless/tests/compact_block_consistency.rs +++ b/live-tests/clientless/tests/compact_block_consistency.rs @@ -14,7 +14,7 @@ use zaino_fetch::jsonrpsee::response::GetBlockResponse; use zaino_state::FetchService; use zaino_state::ZcashIndexer as _; use zaino_testutils::{ - MinerPool, TestManager, ValidatorKind, NU6_3_ACTIVE_ACTIVATION_HEIGHTS, + MinerPool, TestManager, ValidatorKind, IRONWOOD_ONLY_ACTIVATION_HEIGHTS, NU6_3_TRANSITION_BOUNDARY, ORCHARD_ONLY_ACTIVATION_HEIGHTS, ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS, }; @@ -33,7 +33,7 @@ async fn unfiltered_compact_blocks_match_chain_metadata_zebrad() { MinerPool::Orchard, &ValidatorKind::Zebrad, None, - Some(NU6_3_ACTIVE_ACTIVATION_HEIGHTS), + Some(IRONWOOD_ONLY_ACTIVATION_HEIGHTS), None, true, false, @@ -326,7 +326,7 @@ async fn orchard_only_coinbase_routing_zebrad() { /// multi_thread required: the test manager spawns the validator and indexer services. #[tokio::test(flavor = "multi_thread")] async fn ironwood_only_coinbase_routing_zebrad() { - assert_coinbase_routing(NU6_3_ACTIVE_ACTIVATION_HEIGHTS, 6, |height| { + assert_coinbase_routing(IRONWOOD_ONLY_ACTIVATION_HEIGHTS, 6, |height| { if height >= 2 { CoinbaseEra::Ironwood } else { diff --git a/live-tests/e2e/tests/compact_block_wire.rs b/live-tests/e2e/tests/compact_block_wire.rs index 495de0c2d..565869d3f 100644 --- a/live-tests/e2e/tests/compact_block_wire.rs +++ b/live-tests/e2e/tests/compact_block_wire.rs @@ -18,7 +18,7 @@ use zaino_proto::proto::service::{BlockId, BlockRange}; use zaino_state::FetchService; use zaino_state::ZcashIndexer as _; use zaino_testutils::{ - make_uri, MinerPool, TestManager, ValidatorKind, NU6_3_ACTIVE_ACTIVATION_HEIGHTS, + make_uri, MinerPool, TestManager, ValidatorKind, IRONWOOD_ONLY_ACTIVATION_HEIGHTS, NU6_3_TRANSITION_BOUNDARY, ORCHARD_ONLY_ACTIVATION_HEIGHTS, ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS, }; @@ -171,7 +171,7 @@ async fn orchard_only_wire_serving_zebrad() { /// multi_thread required: the test manager spawns the validator, indexer, and zainod. #[tokio::test(flavor = "multi_thread")] async fn ironwood_only_wire_serving_zebrad() { - assert_wire_served_eras(NU6_3_ACTIVE_ACTIVATION_HEIGHTS, 6, |height| { + assert_wire_served_eras(IRONWOOD_ONLY_ACTIVATION_HEIGHTS, 6, |height| { if height >= 2 { CoinbaseEra::Ironwood } else { diff --git a/live-tests/e2e/tests/ironwood_activation.rs b/live-tests/e2e/tests/ironwood_activation.rs new file mode 100644 index 000000000..6e04096de --- /dev/null +++ b/live-tests/e2e/tests/ironwood_activation.rs @@ -0,0 +1,243 @@ +//! Wallet-tier predicates across the Orchard→Ironwood activation boundary. +//! +//! Every test here runs a devtool wallet on +//! [`ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS`] — the hermetic replay of what +//! the public testnet did once at height 4,134,000: heights 2 through 5 are +//! Orchard era, [`NU6_3_TRANSITION_BOUNDARY`] (6) onward is Ironwood era. +//! The wallets derive their activation schedule from the running validator +//! (`WalletNetwork::from_validator`, infrastructure ADR 0003), so the +//! fixture heights are typed in exactly one place: the zebrad launch +//! config. Height drift between wallet, indexer, and validator is +//! unrepresentable — zainod adopts the same schedule over +//! `getblockchaininfo` (zaino#1076). +//! +//! # The predicates, and where each era's cell is covered +//! +//! | predicate (wallet-observable) | Orchard era | Ironwood era | +//! |----------------------------------------------|-------------|--------------| +//! | unified-address receipt lands in Orchard | here | false — `devtool.rs` `send_to_ironwood` asserts the Orchard pool stays empty | +//! | unified-address receipt lands in Ironwood | false (pool inactive) | `devtool.rs` `send_to_ironwood` | +//! | shielded-receiver coinbase pays the era pool | here (wallet view); wire tier in `compact_block_wire.rs` | `devtool.rs` `receives_mining_reward`; wire tier in `compact_block_wire.rs` | +//! | an Orchard note spends into an Ironwood receipt (ZIP 318 migration) | n/a (nothing to exit) | here | +//! +//! Era composition of the *served* chain (coinbase routing, compact-block +//! action fields) is covered clientless in +//! `clientless/tests/compact_block_consistency.rs` and over the real gRPC +//! wire in `compact_block_wire.rs`; this file owns the cells that need a +//! wallet on both sides of the boundary. +//! +//! The public testnet cannot host the migration cell for us: its pre-NU6.3 +//! epoch closed at height 4,134,000, no new value may enter Orchard from +//! there (post-activation Orchard actions permit only same-receiver change +//! or withdrawal — the cross-address restriction, +//! ), +//! and we hold no pre-activation Orchard TAZ — so this hermetic fixture is +//! the only controlled venue for it. +//! +//! Deferred cells the cross-address restriction implies (chain-walk tier, +//! not wallet tier): the Orchard pool value is non-increasing from the +//! boundary, and post-activation Orchard commitments exist only as +//! same-receiver change — note the Orchard note-commitment tree therefore +//! still grows after activation; do not encode a frozen-finalRoot predicate. +//! +//! Requires a `zcash-devtool` binary built with `--features regtest_support` +//! in `TEST_BINARIES_DIR`/`PATH`, alongside the usual validator binaries. + +use e2e::devtool::DevtoolClients; +use zaino_state::{ZcashIndexer, ZcashService}; +use zaino_testutils::{ + PollableTip, TestManager, TestService, ValidatorKind, NU6_3_TRANSITION_BOUNDARY, + ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS, +}; +use zainodlib::error::IndexerError; +use zcash_local_net::validator::zebrad::Zebrad; + +/// Launch an orchard-receiver-mining zebrad + Zaino on the transition +/// heights, build devtool faucet/recipient wallets (their schedule derived +/// from the launched validator), mine one block (height 2: the first +/// Orchard-era coinbase note), and sync the faucet. The transition-fixture +/// analogue of `devtool.rs::launch_and_fund_faucet`. +async fn launch_transition_chain_and_fund_faucet( +) -> (TestManager, DevtoolClients) +where + Service: TestService, + IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, + ::Subscriber: PollableTip, +{ + let test_manager = TestManager::::launch_mining_to( + zaino_testutils::SHIELDED_FUNDING_POOL, + &ValidatorKind::Zebrad, + None, + Some(ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS), + None, + true, + false, + false, + ) + .await + .expect("launch TestManager"); + + let mut clients = e2e::devtool::build_clients( + test_manager + .zaino_grpc_listen_address + .expect("zaino enabled") + .port(), + &test_manager.local_net, + ) + .await; + + test_manager + .generate_blocks_and_wait_for_tip(1, test_manager.subscriber()) + .await; + clients.sync_faucet().await; + + (test_manager, clients) +} + +/// Orchard-era receipt: with the tip still below the boundary, the faucet's +/// coinbase note is an Orchard note (not Ironwood), and a unified-address +/// send received before the boundary lands in the recipient's Orchard pool +/// with the Ironwood pool exactly empty — the era-mirror of +/// `devtool.rs::send_to_pool(Ironwood)`. +async fn unified_receipt_lands_in_orchard_before_boundary() +where + Service: TestService, + IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, + ::Subscriber: PollableTip, +{ + let (mut test_manager, mut clients) = + launch_transition_chain_and_fund_faucet::().await; + + // Tip is 2: inside the Orchard era, with room to confirm the send at + // height 3 while staying below the boundary at 6. + let faucet_balance = clients.faucet_balance().await; + assert!( + faucet_balance.orchard_spendable > 0, + "pre-boundary coinbase should be an orchard note, got {faucet_balance:?}" + ); + assert_eq!( + faucet_balance.ironwood_spendable, 0, + "no ironwood note can exist below the boundary, got {faucet_balance:?}" + ); + + let recipient = clients.get_recipient_address("unified").await; + let txid = clients.send_from_faucet(&recipient, 250_000).await; + dbg!(txid); + + test_manager + .generate_blocks_and_wait_for_tip(1, test_manager.subscriber()) + .await; + clients.sync_recipient().await; + + let balance = clients.recipient_balance().await; + assert_eq!(e2e::Pool::Orchard.spendable_balance(&balance), 250_000); + assert_eq!(e2e::Pool::Ironwood.spendable_balance(&balance), 0); + + test_manager.close().await; +} + +/// The ZIP 318 migration shape: an Orchard note minted before the boundary +/// is spent after it, to a unified address generated after activation, and +/// the receipt lands in the Ironwood pool with the recipient's Orchard pool +/// exactly empty. The faucet's Orchard balance must shrink: from the +/// boundary, the cross-address restriction limits each Orchard action to +/// same-receiver change or withdrawal +/// (), +/// so a genuine Orchard spend nets sent-amount-plus-fee out of the pool even +/// when change returns to the spent note's address. +async fn orchard_note_spends_to_ironwood_across_boundary() +where + Service: TestService, + IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, + ::Subscriber: PollableTip, +{ + let (mut test_manager, mut clients) = + launch_transition_chain_and_fund_faucet::().await; + + let pre_boundary_balance = clients.faucet_balance().await; + assert!( + pre_boundary_balance.orchard_spendable > 0, + "pre-boundary coinbase should be an orchard note, got {pre_boundary_balance:?}" + ); + + // Tip is 2; mine to the boundary itself. Heights 3–5 add more Orchard + // coinbase notes, height 6 is the first Ironwood-era block (its coinbase + // is the faucet's first Ironwood note). + test_manager + .generate_blocks_and_wait_for_tip(NU6_3_TRANSITION_BOUNDARY - 2, test_manager.subscriber()) + .await; + clients.sync_faucet().await; + let crossed_balance = clients.faucet_balance().await; + let orchard_before_send = crossed_balance.orchard_spendable; + assert!( + crossed_balance.ironwood_spendable > 0, + "the boundary coinbase should be an ironwood note, got {crossed_balance:?}" + ); + + // Generated only now — after activation — per the migration shape under + // test: old-pool note, new-era address. + let recipient = clients.get_recipient_address("unified").await; + let txid = clients.send_from_faucet(&recipient, 250_000).await; + dbg!(txid); + + test_manager + .generate_blocks_and_wait_for_tip(1, test_manager.subscriber()) + .await; + clients.sync_faucet().await; + clients.sync_recipient().await; + + let balance = clients.recipient_balance().await; + assert_eq!(e2e::Pool::Ironwood.spendable_balance(&balance), 250_000); + assert_eq!(e2e::Pool::Orchard.spendable_balance(&balance), 0); + + // Pins that the send actually exited the Orchard pool rather than + // spending the boundary-height Ironwood coinbase — the note-selection + // question the first live runs of this suite exist to answer. + assert!( + clients.faucet_balance().await.orchard_spendable < orchard_before_send, + "the migration send must spend an orchard note" + ); + + test_manager.close().await; +} + +mod zebrad { + // FetchService is a deprecated re-export; the deprecation fires at the + // turbofish use sites below, so the allow covers the whole module. + #[allow(deprecated)] + mod fetch_service { + use zaino_state::FetchService; + + /// multi_thread required: the test manager spawns the validator and + /// indexer services. + #[tokio::test(flavor = "multi_thread")] + async fn unified_receipt_lands_in_orchard_before_boundary() { + crate::unified_receipt_lands_in_orchard_before_boundary::().await; + } + + /// multi_thread required: the test manager spawns the validator and + /// indexer services. + #[tokio::test(flavor = "multi_thread")] + async fn orchard_note_spends_to_ironwood_across_boundary() { + crate::orchard_note_spends_to_ironwood_across_boundary::().await; + } + } + + mod state_service { + use zaino_state::StateService; + + /// multi_thread required: the test manager spawns the validator and + /// indexer services. + #[tokio::test(flavor = "multi_thread")] + async fn unified_receipt_lands_in_orchard_before_boundary() { + crate::unified_receipt_lands_in_orchard_before_boundary::().await; + } + + /// multi_thread required: the test manager spawns the validator and + /// indexer services. + #[tokio::test(flavor = "multi_thread")] + async fn orchard_note_spends_to_ironwood_across_boundary() { + crate::orchard_note_spends_to_ironwood_across_boundary::().await; + } + } +} diff --git a/live-tests/zaino-testutils/src/lib.rs b/live-tests/zaino-testutils/src/lib.rs index 6e8a342e1..f6c8ba692 100644 --- a/live-tests/zaino-testutils/src/lib.rs +++ b/live-tests/zaino-testutils/src/lib.rs @@ -112,7 +112,7 @@ pub const ORCHARD_ONLY_ACTIVATION_HEIGHTS: ActivationHeights = ActivationHeights /// Zebrad regtest heights with every upgrade through NU6.3 active from height 2, so /// generated blocks carry V6 coinbases from the first post-genesis era. -pub const NU6_3_ACTIVE_ACTIVATION_HEIGHTS: ActivationHeights = ActivationHeights { +pub const IRONWOOD_ONLY_ACTIVATION_HEIGHTS: ActivationHeights = ActivationHeights { before_overwinter: Some(1), overwinter: Some(1), sapling: Some(1), diff --git a/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs b/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs index 54f273917..93843c6db 100644 --- a/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs +++ b/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs @@ -451,7 +451,7 @@ fn zebra_arbitrary_generates_v6_transactions_for_nu6_3() { /// NU6.3 active from height 2, so post-activation generated blocks carry V6 /// transactions whose shielded data lands in the Ironwood pool. -const NU6_3_ACTIVE_HEIGHTS: ActivationHeights = ActivationHeights { +const IRONWOOD_ONLY_HEIGHTS: ActivationHeights = ActivationHeights { before_overwinter: Some(1), overwinter: Some(1), sapling: Some(1), @@ -478,7 +478,7 @@ const NU6_3_ACTIVE_HEIGHTS: ActivationHeights = ActivationHeights { /// source of truth. #[test] fn passthrough_metadata_consistency_ironwood_only() { - metadata_consistency_for_era(NU6_3_ACTIVE_HEIGHTS, Some(2), false) + metadata_consistency_for_era(IRONWOOD_ONLY_HEIGHTS, Some(2), false) } /// Orchard-only heights: every upgrade through NU6.2 at height 2, NU6.3 never @@ -516,7 +516,7 @@ fn passthrough_metadata_consistency_orchard_to_ironwood_transition() { metadata_consistency_for_era( ActivationHeights { nu6_3: Some(boundary), - ..NU6_3_ACTIVE_HEIGHTS + ..IRONWOOD_ONLY_HEIGHTS }, Some(boundary), true, From c6ec6796d1ada0d784247e81e4ce16556dc0eb4b Mon Sep 17 00:00:00 2001 From: zancas Date: Mon, 6 Jul 2026 22:45:16 -0700 Subject: [PATCH 100/134] test: enforce the Ironwood boundary invariants at its edges, and on testnet MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ironwood_activation suite gains the boundary-edge cells the plan deferred. receipts_flip_pools_exactly_at_the_boundary confirms one send in the last Orchard-era block and a second in the activation block itself: the receipt pool flips between adjacent heights, the pre-boundary receipt survives the flip unchanged, and the second send — built while the tip was still Orchard-era, spending an Orchard note — is a migration transaction inside the activation block, pinning the wallet's era anticipation at the edge. shield_deposits_to_orchard_before_boundary is the era-mirror of the Ironwood-era shield cell. The migration test additionally walks the served per-height value pools and enforces the chain-tier consequences of the cross-address restriction: the Ironwood pool holds no value below the boundary, the Orchard pool grows only below it, holds exactly steady across the 5-to-6 edge, and shrinks at the migration block. All cells register on both backends (eight tests). The same invariants are enforced against the real activation on the public testnet in clientless/tests/testnet_ironwood_boundary.rs: an env-gated walk (skips unless ~/.cache/zebra holds a testnet chain synced past the boundary) that launches a public-testnet zebrad directly through ZebradConfig, reads the activation height from the validator's getblockchaininfo.upgrades, cross-checks it against the observed activation at 4,134,000, and asserts the Ironwood pool is empty below the boundary and the Orchard pool never grows from it. zaino-fetch's BlockObject gains a value_pools accessor (and ChainBalance a balance accessor) so the walk can read the validator's verbosity-2 per-height pool balances. Co-Authored-By: Claude Fable 5 --- .../tests/testnet_ironwood_boundary.rs | 155 +++++++++++++ live-tests/e2e/tests/ironwood_activation.rs | 218 +++++++++++++++++- .../zaino-fetch/src/jsonrpsee/response.rs | 15 ++ 3 files changed, 385 insertions(+), 3 deletions(-) create mode 100644 live-tests/clientless/tests/testnet_ironwood_boundary.rs diff --git a/live-tests/clientless/tests/testnet_ironwood_boundary.rs b/live-tests/clientless/tests/testnet_ironwood_boundary.rs new file mode 100644 index 000000000..08b5a7f7f --- /dev/null +++ b/live-tests/clientless/tests/testnet_ironwood_boundary.rs @@ -0,0 +1,155 @@ +//! Observational walk of the real NU6.3 activation boundary on the public +//! testnet (glossary: "testnet" means the public test network and nothing +//! else). Historical blocks are permanent, so these invariants are checkable +//! forever, long after the epoch that produced them closed. +//! +//! The invariants, enforced one block below the boundary and at it: +//! +//! - Below the activation height the Ironwood pool holds no value. +//! - From the activation height the Orchard pool's value never increases: +//! the no-new-value rule admits only withdrawals and same-receiver change +//! (the cross-address restriction, +//! ). +//! +//! The activation height itself is read from the running validator's +//! `getblockchaininfo.upgrades` — the single source of truth for activation +//! heights — and cross-checked against the observed public-testnet +//! activation at height 4,134,000 (~2026-07-04). +//! +//! Non-hermetic and therefore env-gated: the test needs a zebrad chain +//! cache at `~/.cache/zebra` (`ZEBRAD_TESTNET_CACHE_DIR`) synced past the +//! activation height, and skips with a message when the cache is absent or +//! short. Run it manually, or from a job that maintains the cache. +//! +//! The validator is launched through `ZebradConfig` directly rather than +//! `TestManager`: the manager's launch path always writes regtest test +//! parameters into the config, while this test needs the public testnet +//! (`network_type: NetworkType::Testnet`, which makes zebrad ignore +//! `activation_heights` and `miner_address`). + +use zaino_fetch::jsonrpsee::connector::{test_node_and_return_url, JsonRpSeeConnector}; +use zaino_fetch::jsonrpsee::response::GetBlockResponse; +use zaino_testutils::ZEBRAD_TESTNET_CACHE_DIR; +use zcash_local_net::process::Process as _; +use zcash_local_net::protocol::NetworkType; +use zcash_local_net::validator::zebrad::{Zebrad, ZebradConfig}; +use zcash_local_net::validator::Validator as _; +use zebra_chain::parameters::NetworkUpgrade; + +/// The height the public testnet activated NU6.3 at, recorded from the real +/// activation. A mismatch means either a testnet reset or a wrong validator +/// pin — both worth failing loudly over. +const OBSERVED_TESTNET_NU6_3_ACTIVATION: u32 = 4_134_000; + +/// The chain value of `pool_id` as of `height`, from the validator's +/// verbosity-2 block object. +async fn pool_zats(connector: &JsonRpSeeConnector, height: u32, pool_id: &str) -> i64 { + let response = connector + .get_block(height.to_string(), Some(2)) + .await + .expect("getblock verbosity 2"); + let GetBlockResponse::Object(block) = response else { + panic!("verbosity-2 getblock must return a block object"); + }; + block + .value_pools() + .expect("verbosity-2 block object carries value pools") + .iter() + .map(|pool| pool.balance()) + .find(|pool| pool.id() == pool_id) + .unwrap_or_else(|| panic!("value pools must include {pool_id}")) + .chain_value_zat() + .zatoshis() +} + +/// multi_thread required: the test launches the validator process and polls +/// it over RPC. +#[tokio::test(flavor = "multi_thread")] +async fn value_pools_respect_the_boundary_on_testnet() { + let Some(cache_dir) = ZEBRAD_TESTNET_CACHE_DIR.clone() else { + eprintln!("skipping: no testnet cache dir configured"); + return; + }; + if !cache_dir.exists() { + eprintln!( + "skipping: no zebrad testnet chain cache at {}", + cache_dir.display() + ); + return; + } + + let config = ZebradConfig { + network_type: NetworkType::Testnet, + chain_cache: Some(cache_dir), + ..ZebradConfig::default() + }; + let mut zebrad = Zebrad::launch(config).await.expect("launch testnet zebrad"); + + let rpc_address = format!("127.0.0.1:{}", zebrad.get_port()); + let connector = JsonRpSeeConnector::new_with_basic_auth( + test_node_and_return_url( + &rpc_address, + None, + Some("xxxxxx".to_string()), + Some("xxxxxx".to_string()), + ) + .await + .expect("validator RPC reachable"), + "xxxxxx".to_string(), + "xxxxxx".to_string(), + ) + .expect("connect to the validator RPC"); + + let blockchain_info = connector + .get_blockchain_info() + .await + .expect("getblockchaininfo"); + + // The validator's reported schedule is the source of truth for the + // boundary; the recorded constant pins the real public-testnet history. + let boundary = blockchain_info + .upgrades + .values() + .find_map(|upgrade_info| { + let (upgrade, height, _status) = upgrade_info.into_parts(); + (upgrade == NetworkUpgrade::Nu6_3).then_some(height.0) + }) + .expect("the testnet validator must report an NU6.3 activation height"); + assert_eq!( + boundary, OBSERVED_TESTNET_NU6_3_ACTIVATION, + "the validator's NU6.3 height must match the observed testnet activation" + ); + + let tip = blockchain_info.blocks.0; + if tip <= boundary { + eprintln!( + "skipping: testnet cache tip {tip} has not crossed the NU6.3 boundary {boundary}" + ); + zebrad.stop(); + return; + } + + // One block below the boundary and at it, plus a block of margin on + // each side: the Ironwood pool holds no value below the activation + // height, and the Orchard pool never grows from it. + for height in (boundary - 2)..boundary { + assert_eq!( + pool_zats(&connector, height, "ironwood").await, + 0, + "the ironwood pool must hold no value at height {height}, below the boundary" + ); + } + let walk_end = boundary + 1; + let mut previous_orchard = pool_zats(&connector, boundary - 1, "orchard").await; + for height in boundary..=walk_end { + let orchard = pool_zats(&connector, height, "orchard").await; + assert!( + orchard <= previous_orchard, + "the orchard pool must never grow from the boundary; \ + height {height} holds {orchard} zats after {previous_orchard}" + ); + previous_orchard = orchard; + } + + zebrad.stop(); +} diff --git a/live-tests/e2e/tests/ironwood_activation.rs b/live-tests/e2e/tests/ironwood_activation.rs index 6e04096de..eba7c5ef1 100644 --- a/live-tests/e2e/tests/ironwood_activation.rs +++ b/live-tests/e2e/tests/ironwood_activation.rs @@ -19,6 +19,9 @@ //! | unified-address receipt lands in Ironwood | false (pool inactive) | `devtool.rs` `send_to_ironwood` | //! | shielded-receiver coinbase pays the era pool | here (wallet view); wire tier in `compact_block_wire.rs` | `devtool.rs` `receives_mining_reward`; wire tier in `compact_block_wire.rs` | //! | an Orchard note spends into an Ironwood receipt (ZIP 318 migration) | n/a (nothing to exit) | here | +//! | `shield` deposits into the era pool | here | `devtool.rs` `shield_for_validator` | +//! | receipt pool flips between the last Orchard block and the activation block | here (boundary − 1) | here (exactly at the boundary) | +//! | Orchard pool value grows only below the boundary; Ironwood holds value only from it | here (per-height value pools) | here | //! //! Era composition of the *served* chain (coinbase routing, compact-block //! action fields) is covered clientless in @@ -44,10 +47,10 @@ //! in `TEST_BINARIES_DIR`/`PATH`, alongside the usual validator binaries. use e2e::devtool::DevtoolClients; -use zaino_state::{ZcashIndexer, ZcashService}; +use zaino_state::{LightWalletIndexer, ZcashIndexer, ZcashService}; use zaino_testutils::{ - PollableTip, TestManager, TestService, ValidatorKind, NU6_3_TRANSITION_BOUNDARY, - ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS, + all_pools_i32, collect_block_range, PollableTip, TestManager, TestService, ValidatorKind, + NU6_3_TRANSITION_BOUNDARY, ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS, }; use zainodlib::error::IndexerError; use zcash_local_net::validator::zebrad::Zebrad; @@ -94,6 +97,34 @@ where (test_manager, clients) } +/// The chain value (in zatoshis) of the pool named `pool_id` as of +/// `height`, read from the served verbosity-2 block object — the same +/// per-height `valuePools` a zcashd `getblock` reports. +async fn pool_zats_at_height(subscriber: &S, height: u32, pool_id: &str) -> i64 +where + S: ZcashIndexer, + IndexerError: From, +{ + let response = subscriber + .z_get_block(height.to_string(), Some(2)) + .await + .map_err(IndexerError::from) + .expect("z_get_block verbosity 2"); + let zebra_rpc::methods::GetBlock::Object(block) = response else { + panic!("verbosity-2 getblock must return a block object"); + }; + let pools = block + .value_pools() + .as_ref() + .expect("verbosity-2 block object carries value pools"); + pools + .iter() + .find(|pool| pool.id() == pool_id) + .unwrap_or_else(|| panic!("value pools must include {pool_id}")) + .chain_value_zat() + .zatoshis() +} + /// Orchard-era receipt: with the tip still below the boundary, the faucet's /// coinbase note is an Orchard note (not Ironwood), and a unified-address /// send received before the boundary lands in the recipient's Orchard pool @@ -198,6 +229,159 @@ where "the migration send must spend an orchard note" ); + // Chain-tier consequences, read from the served per-height value pools. + // The Ironwood pool holds no value below the activation height; the + // Orchard pool grows only below it (its coinbases), holds exactly + // steady across the boundary edge (the activation block's coinbase pays + // Ironwood, and no new value may enter Orchard), and shrinks at the + // migration block by the withdrawn amount plus fee. + let boundary = NU6_3_TRANSITION_BOUNDARY; + let migration_height = boundary + 1; + let subscriber = test_manager.subscriber(); + let mut orchard_at = Vec::new(); + for height in 1..=migration_height { + let ironwood = pool_zats_at_height(subscriber, height, "ironwood").await; + let orchard = pool_zats_at_height(subscriber, height, "orchard").await; + if height < boundary { + assert_eq!( + ironwood, 0, + "the ironwood pool must hold no value at height {height}, below the boundary" + ); + } + orchard_at.push(orchard); + } + let orchard = |height: u32| orchard_at[height as usize - 1]; + for height in 2..boundary { + assert!( + orchard(height) > orchard(height - 1), + "each pre-boundary coinbase must grow the orchard pool (height {height})" + ); + } + assert_eq!( + orchard(boundary), + orchard(boundary - 1), + "the orchard pool must hold exactly steady across the boundary edge" + ); + assert!( + pool_zats_at_height(subscriber, boundary, "ironwood").await > 0, + "the activation block's coinbase must give the ironwood pool its first value" + ); + assert!( + orchard(migration_height) < orchard(boundary), + "the migration block must shrink the orchard pool" + ); + + test_manager.close().await; +} + +/// The receipt pool flips between adjacent blocks: a send confirmed in the +/// last Orchard-era block (boundary − 1) lands in Orchard, and a send built +/// one block below the boundary but confirmed in the activation block +/// itself lands in Ironwood. The second send also pins the wallet's era +/// anticipation at the edge: it is constructed while the tip is still +/// Orchard-era, targeting the first Ironwood-era height, and it spends an +/// Orchard note (the faucet holds no Ironwood note until the activation +/// block is mined) — a migration transaction in the activation block. +async fn receipts_flip_pools_exactly_at_the_boundary() +where + Service: TestService, + IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, + ::Subscriber: PollableTip + LightWalletIndexer, +{ + let (mut test_manager, mut clients) = + launch_transition_chain_and_fund_faucet::().await; + + // Tip is 2. Mine heights 3 and 4 (a second spendable Orchard note and a + // filler), so the two sends below confirm at exactly boundary − 1 and + // the boundary. + test_manager + .generate_blocks_and_wait_for_tip(NU6_3_TRANSITION_BOUNDARY - 4, test_manager.subscriber()) + .await; + clients.sync_faucet().await; + + let recipient = clients.get_recipient_address("unified").await; + + // Confirmed in block 5: the last Orchard-era block. + let last_orchard_txid = clients.send_from_faucet(&recipient, 250_000).await; + test_manager + .generate_blocks_and_wait_for_tip(1, test_manager.subscriber()) + .await; + clients.sync_faucet().await; + clients.sync_recipient().await; + let balance = clients.recipient_balance().await; + assert_eq!(e2e::Pool::Orchard.spendable_balance(&balance), 250_000); + assert_eq!(e2e::Pool::Ironwood.spendable_balance(&balance), 0); + + // Built at tip boundary − 1, confirmed in block 6: the activation block. + let first_ironwood_txid = clients.send_from_faucet(&recipient, 250_000).await; + test_manager + .generate_blocks_and_wait_for_tip(1, test_manager.subscriber()) + .await; + clients.sync_recipient().await; + let balance = clients.recipient_balance().await; + assert_eq!(e2e::Pool::Ironwood.spendable_balance(&balance), 250_000); + assert_eq!( + e2e::Pool::Orchard.spendable_balance(&balance), + 250_000, + "the pre-boundary receipt must survive the flip unchanged" + ); + + // Era composition of the two served edge blocks. + let subscriber = test_manager.subscriber(); + let boundary = u64::from(NU6_3_TRANSITION_BOUNDARY); + let blocks = collect_block_range(subscriber, boundary - 1, boundary, all_pools_i32()).await; + let [last_orchard_block, activation_block] = blocks.as_slice() else { + panic!("expected exactly the two edge blocks, got {}", blocks.len()); + }; + assert_eq!(last_orchard_block.height, boundary - 1); + assert_eq!(activation_block.height, boundary); + let last_orchard_txid = e2e::devtool::txid_from_devtool(&last_orchard_txid); + e2e::assert_pool_present(last_orchard_block, &last_orchard_txid, e2e::Pool::Orchard); + e2e::assert_pool_absent(last_orchard_block, &last_orchard_txid, e2e::Pool::Ironwood); + let first_ironwood_txid = e2e::devtool::txid_from_devtool(&first_ironwood_txid); + e2e::assert_pool_present(activation_block, &first_ironwood_txid, e2e::Pool::Ironwood); + e2e::assert_pool_absent(activation_block, &first_ironwood_txid, e2e::Pool::Orchard); + + test_manager.close().await; +} + +/// Below the boundary, `shield` deposits into the Orchard pool: the faucet +/// funds the recipient's transparent address, the recipient shields, and +/// the shielded balance (net of the ZIP-317 fee, mirroring +/// `devtool.rs::shield_for_validator`) lands in Orchard with the Ironwood +/// pool exactly empty — the era-mirror of the Ironwood-era shield cell. +async fn shield_deposits_to_orchard_before_boundary() +where + Service: TestService, + IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, + ::Subscriber: PollableTip, +{ + let (mut test_manager, mut clients) = + launch_transition_chain_and_fund_faucet::().await; + + // Tip is 2; the transparent receipt confirms at 3 and the shield at 4, + // all below the boundary at 6. + let recipient_t = clients.get_recipient_address("transparent").await; + clients.send_from_faucet(&recipient_t, 250_000).await; + test_manager + .generate_blocks_and_wait_for_tip(1, test_manager.subscriber()) + .await; + clients.sync_recipient().await; + assert_eq!( + e2e::Pool::Transparent.spendable_balance(&clients.recipient_balance().await), + 250_000 + ); + + clients.shield_recipient().await; + test_manager + .generate_blocks_and_wait_for_tip(1, test_manager.subscriber()) + .await; + clients.sync_recipient().await; + + let balance = clients.recipient_balance().await; + assert_eq!(e2e::Pool::Orchard.spendable_balance(&balance), 235_000); + assert_eq!(e2e::Pool::Ironwood.spendable_balance(&balance), 0); + test_manager.close().await; } @@ -221,6 +405,20 @@ mod zebrad { async fn orchard_note_spends_to_ironwood_across_boundary() { crate::orchard_note_spends_to_ironwood_across_boundary::().await; } + + /// multi_thread required: the test manager spawns the validator and + /// indexer services. + #[tokio::test(flavor = "multi_thread")] + async fn receipts_flip_pools_exactly_at_the_boundary() { + crate::receipts_flip_pools_exactly_at_the_boundary::().await; + } + + /// multi_thread required: the test manager spawns the validator and + /// indexer services. + #[tokio::test(flavor = "multi_thread")] + async fn shield_deposits_to_orchard_before_boundary() { + crate::shield_deposits_to_orchard_before_boundary::().await; + } } mod state_service { @@ -239,5 +437,19 @@ mod zebrad { async fn orchard_note_spends_to_ironwood_across_boundary() { crate::orchard_note_spends_to_ironwood_across_boundary::().await; } + + /// multi_thread required: the test manager spawns the validator and + /// indexer services. + #[tokio::test(flavor = "multi_thread")] + async fn receipts_flip_pools_exactly_at_the_boundary() { + crate::receipts_flip_pools_exactly_at_the_boundary::().await; + } + + /// multi_thread required: the test manager spawns the validator and + /// indexer services. + #[tokio::test(flavor = "multi_thread")] + async fn shield_deposits_to_orchard_before_boundary() { + crate::shield_deposits_to_orchard_before_boundary::().await; + } } } diff --git a/packages/zaino-fetch/src/jsonrpsee/response.rs b/packages/zaino-fetch/src/jsonrpsee/response.rs index b63fe58f5..86a6fe4be 100644 --- a/packages/zaino-fetch/src/jsonrpsee/response.rs +++ b/packages/zaino-fetch/src/jsonrpsee/response.rs @@ -1245,6 +1245,21 @@ pub struct BlockObject { pub next_block_hash: Option, } +impl BlockObject { + /// Per-pool chain value balances as of this block, when the validator + /// reports them (`getblock` verbosity 2). + pub fn value_pools(&self) -> Option<&[ChainBalance]> { + self.value_pools.as_deref() + } +} + +impl ChainBalance { + /// The underlying per-pool balance entry. + pub fn balance(&self) -> &GetBlockchainInfoBalance { + &self.0 + } +} + impl TryFrom for zebra_rpc::methods::GetBlock { type Error = zebra_chain::serialization::SerializationError; From 888d0ccb432865390d9c5440168971b0e5e71906 Mon Sep 17 00:00:00 2001 From: zancas Date: Tue, 7 Jul 2026 08:05:13 -0700 Subject: [PATCH 101/134] build(makers): add an ironwood set to the test front door `makers test ironwood` runs the ironwood tests of every set, package then clientless then e2e, reusing the same engines as the other sets with the `all` set's continue-on-failure accumulation. The selection is defined once, as a nextest filter expression naming the two dedicated test binaries plus every ironwood-named test; the CONTEXT.md era-naming convention keeps new ironwood tests inside that net. The engines are invoked directly rather than through the `live` summary runner, which forwards no nextest arguments. Co-Authored-By: Claude Fable 5 --- Makefile.toml | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/Makefile.toml b/Makefile.toml index f38a60907..73f540efe 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -150,17 +150,18 @@ script.main = 'source "./tools/scripts/container-nextest-workspace.sh"' # clear = true fully replaces cargo-make's built-in `test` task (which runs # `cargo test`); without it our `script` would merge with the builtin `command`. clear = true -description = "Run a test set: `makers test [package|e2e|clientless|live|all]` (default: package). Pass --with-zcashd for the zcashd-backed tests." +description = "Run a test set: `makers test [package|e2e|clientless|live|all|ironwood]` (default: package). Pass --with-zcashd for the zcashd-backed tests." script_runner = "bash" script = ''' set -uo pipefail usage() { - echo "usage: makers test [package|e2e|clientless|live|all] [-- nextest args]" >&2 + echo "usage: makers test [package|e2e|clientless|live|all|ironwood] [-- nextest args]" >&2 echo " package (default) packages/* tests, no live validator" >&2 echo " e2e | clientless that single live partition" >&2 echo " live both live partitions + combined summary" >&2 echo " all package then live (everything)" >&2 + echo " ironwood the ironwood tests of every set, package then clientless then e2e" >&2 } # A non-flag first arg selects the set and must be a known name; a flag first @@ -170,7 +171,7 @@ usage() { set="package" if [ "$#" -gt 0 ] && [ "${1#-}" = "$1" ]; then case "$1" in - package|e2e|clientless|live|all) set="$1"; shift ;; + package|e2e|clientless|live|all|ironwood) set="$1"; shift ;; *) echo "makers test: unknown set '$1'" >&2; usage; exit 2 ;; esac fi @@ -187,6 +188,21 @@ case "$set" in makers container-test "$@" || rc=1 cargo run -q --manifest-path tools/test-runner/Cargo.toml --bin live-summary -- "$@" || rc=1 exit "$rc" ;; + ironwood) + # The one definition of the ironwood selection: the two dedicated test + # binaries, plus every test whose name carries the pool's name (the + # CONTEXT.md era-naming convention keeps new ironwood tests inside this + # net). The same expression serves every engine — a selector naming a + # binary a partition lacks simply matches nothing there. The engines are + # invoked directly rather than through the `live` summary runner, which + # forwards no nextest args. Run every set even if an earlier one fails, + # as in `all`. + ironwood_filter='binary(ironwood_activation) | binary(testnet_ironwood_boundary) | test(ironwood)' + rc=0 + makers container-test -E "$ironwood_filter" "$@" || rc=1 + makers live-clientless -E "$ironwood_filter" "$@" || rc=1 + makers live-e2e -E "$ironwood_filter" "$@" || rc=1 + exit "$rc" ;; esac ''' From 8aeb2d339dccdf9fd3f5a715aea25ea2f0478148 Mon Sep 17 00:00:00 2001 From: zancas Date: Tue, 7 Jul 2026 08:29:56 -0700 Subject: [PATCH 102/134] fix(e2e): position the boundary cells from the observed tip and pin the observed migration facts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first container run of the boundary-edge cells falsified two of my assumptions and confirmed a third. All three corrections follow from one root cause plus two observed chain facts. The root cause: TestManager's launch mines an NU-activation block after block 1, so every launch returns at tip 2, and the suite's hand-counted heights sat one block high. The flip test's sends confirmed one era late, and the migration test's pool-value walk stopped one block short of the real migration block — making its "the orchard pool does not shrink" verdict vacuous. Both tests now mine to absolute heights computed from the observed tip, immune to launch-preamble changes. With the walk actually spanning the migration block, the original predicate proves correct and now passes on both backends: the migration block debits the Orchard pool by the full spent note while the Ironwood pool absorbs the coinbase plus the migrated value. The walk reads verbosity-1 block objects rather than verbosity 2, which the fetch backend cannot deserialize (its `tx` entries arrive as maps where zaino-fetch expects txid strings); the value pools ride along at verbosity 1. The flip test's final expectation inverts: a send built one block below the boundary is itself migration-shaped — it spends an Orchard note, the faucet's only spendable kind at that tip — so its compact form correctly carries the Orchard spend's data alongside the Ironwood receipt. The walk and balance assertions now carry their observed values in their failure messages, which is what made this diagnosis one-pass. Co-Authored-By: Claude Fable 5 --- live-tests/e2e/tests/ironwood_activation.rs | 92 +++++++++++++++------ 1 file changed, 68 insertions(+), 24 deletions(-) diff --git a/live-tests/e2e/tests/ironwood_activation.rs b/live-tests/e2e/tests/ironwood_activation.rs index eba7c5ef1..fd9f86b82 100644 --- a/live-tests/e2e/tests/ironwood_activation.rs +++ b/live-tests/e2e/tests/ironwood_activation.rs @@ -57,8 +57,12 @@ use zcash_local_net::validator::zebrad::Zebrad; /// Launch an orchard-receiver-mining zebrad + Zaino on the transition /// heights, build devtool faucet/recipient wallets (their schedule derived -/// from the launched validator), mine one block (height 2: the first -/// Orchard-era coinbase note), and sync the faucet. The transition-fixture +/// from the launched validator), mine one block, and sync the faucet. The +/// launch itself already leaves the tip at 2 (`TestManager` mines an +/// NU-activation block after block 1), so the helper returns at tip 3 with +/// the faucet holding the Orchard coinbase notes of heights 2 and 3. Tests +/// that need exact boundary positioning mine to absolute heights from the +/// observed tip rather than counting from here. The transition-fixture /// analogue of `devtool.rs::launch_and_fund_faucet`. async fn launch_transition_chain_and_fund_faucet( ) -> (TestManager, DevtoolClients) @@ -98,25 +102,28 @@ where } /// The chain value (in zatoshis) of the pool named `pool_id` as of -/// `height`, read from the served verbosity-2 block object — the same -/// per-height `valuePools` a zcashd `getblock` reports. +/// `height`, read from the served verbosity-1 block object — the same +/// per-height `valuePools` a zcashd `getblock` reports. Verbosity 1, not 2: +/// the fetch backend cannot deserialize a verbosity-2 block object (its +/// `tx` entries are maps where zaino-fetch expects txid strings), and the +/// value pools ride along at verbosity 1. async fn pool_zats_at_height(subscriber: &S, height: u32, pool_id: &str) -> i64 where S: ZcashIndexer, IndexerError: From, { let response = subscriber - .z_get_block(height.to_string(), Some(2)) + .z_get_block(height.to_string(), Some(1)) .await .map_err(IndexerError::from) - .expect("z_get_block verbosity 2"); + .expect("z_get_block verbosity 1"); let zebra_rpc::methods::GetBlock::Object(block) = response else { - panic!("verbosity-2 getblock must return a block object"); + panic!("verbosity-1 getblock must return a block object"); }; let pools = block .value_pools() .as_ref() - .expect("verbosity-2 block object carries value pools"); + .expect("verbosity-1 block object carries value pools"); pools .iter() .find(|pool| pool.id() == pool_id) @@ -139,8 +146,8 @@ where let (mut test_manager, mut clients) = launch_transition_chain_and_fund_faucet::().await; - // Tip is 2: inside the Orchard era, with room to confirm the send at - // height 3 while staying below the boundary at 6. + // Tip is 3: inside the Orchard era, with room to confirm the send at + // height 4 while staying below the boundary at 6. let faucet_balance = clients.faucet_balance().await; assert!( faucet_balance.orchard_spendable > 0, @@ -191,11 +198,19 @@ where "pre-boundary coinbase should be an orchard note, got {pre_boundary_balance:?}" ); - // Tip is 2; mine to the boundary itself. Heights 3–5 add more Orchard - // coinbase notes, height 6 is the first Ironwood-era block (its coinbase - // is the faucet's first Ironwood note). + // Mine to the boundary itself, from the observed tip rather than a + // hand-count. The blocks below the boundary add more Orchard coinbase + // notes; the boundary block is the first Ironwood-era block, and its + // coinbase is the faucet's first Ironwood note. + let tip = u32::try_from(test_manager.subscriber().tip_height().await) + .expect("regtest tips fit in u32"); test_manager - .generate_blocks_and_wait_for_tip(NU6_3_TRANSITION_BOUNDARY - 2, test_manager.subscriber()) + .generate_blocks_and_wait_for_tip( + NU6_3_TRANSITION_BOUNDARY + .checked_sub(tip) + .expect("the launch preamble must leave room below the boundary"), + test_manager.subscriber(), + ) .await; clients.sync_faucet().await; let crossed_balance = clients.faucet_balance().await; @@ -250,11 +265,18 @@ where } orchard_at.push(orchard); } + for (index, orchard) in orchard_at.iter().enumerate() { + let height = index + 1; + let ironwood = pool_zats_at_height(subscriber, height as u32, "ironwood").await; + eprintln!("pool values at height {height}: orchard={orchard} ironwood={ironwood}"); + } let orchard = |height: u32| orchard_at[height as usize - 1]; for height in 2..boundary { assert!( orchard(height) > orchard(height - 1), - "each pre-boundary coinbase must grow the orchard pool (height {height})" + "each pre-boundary coinbase must grow the orchard pool (height {height}: {} after {})", + orchard(height), + orchard(height - 1) ); } assert_eq!( @@ -268,7 +290,9 @@ where ); assert!( orchard(migration_height) < orchard(boundary), - "the migration block must shrink the orchard pool" + "the migration block must shrink the orchard pool ({} at the boundary, {} at the migration block)", + orchard(boundary), + orchard(migration_height) ); test_manager.close().await; @@ -291,11 +315,18 @@ where let (mut test_manager, mut clients) = launch_transition_chain_and_fund_faucet::().await; - // Tip is 2. Mine heights 3 and 4 (a second spendable Orchard note and a - // filler), so the two sends below confirm at exactly boundary − 1 and - // the boundary. + // Position the tip at exactly boundary − 2, from the observed tip + // rather than a hand-count, so the two sends below confirm at exactly + // boundary − 1 and the boundary. + let tip = u32::try_from(test_manager.subscriber().tip_height().await) + .expect("regtest tips fit in u32"); test_manager - .generate_blocks_and_wait_for_tip(NU6_3_TRANSITION_BOUNDARY - 4, test_manager.subscriber()) + .generate_blocks_and_wait_for_tip( + NU6_3_TRANSITION_BOUNDARY + .checked_sub(2 + tip) + .expect("the launch preamble must leave room below the boundary"), + test_manager.subscriber(), + ) .await; clients.sync_faucet().await; @@ -309,8 +340,16 @@ where clients.sync_faucet().await; clients.sync_recipient().await; let balance = clients.recipient_balance().await; - assert_eq!(e2e::Pool::Orchard.spendable_balance(&balance), 250_000); - assert_eq!(e2e::Pool::Ironwood.spendable_balance(&balance), 0); + assert_eq!( + e2e::Pool::Orchard.spendable_balance(&balance), + 250_000, + "receipt confirmed at boundary - 1 must be orchard, got {balance:?}" + ); + assert_eq!( + e2e::Pool::Ironwood.spendable_balance(&balance), + 0, + "no ironwood receipt below the boundary, got {balance:?}" + ); // Built at tip boundary − 1, confirmed in block 6: the activation block. let first_ironwood_txid = clients.send_from_faucet(&recipient, 250_000).await; @@ -340,7 +379,12 @@ where e2e::assert_pool_absent(last_orchard_block, &last_orchard_txid, e2e::Pool::Ironwood); let first_ironwood_txid = e2e::devtool::txid_from_devtool(&first_ironwood_txid); e2e::assert_pool_present(activation_block, &first_ironwood_txid, e2e::Pool::Ironwood); - e2e::assert_pool_absent(activation_block, &first_ironwood_txid, e2e::Pool::Orchard); + // The second send is itself migration-shaped: built one block below the + // boundary, it spends an Orchard note (the faucet's only spendable kind + // at that tip), so its compact form carries the Orchard spend's data + // alongside the Ironwood receipt. The receipt's pool routing is what + // flips at the boundary, and that is asserted at the wallet tier above. + e2e::assert_pool_present(activation_block, &first_ironwood_txid, e2e::Pool::Orchard); test_manager.close().await; } @@ -359,7 +403,7 @@ where let (mut test_manager, mut clients) = launch_transition_chain_and_fund_faucet::().await; - // Tip is 2; the transparent receipt confirms at 3 and the shield at 4, + // Tip is 3; the transparent receipt confirms at 4 and the shield at 5, // all below the boundary at 6. let recipient_t = clients.get_recipient_address("transparent").await; clients.send_from_faucet(&recipient_t, 250_000).await; From 6402edd708e6e1c5333e6a10a4d865f17ffa5d7a Mon Sep 17 00:00:00 2001 From: zancas Date: Tue, 7 Jul 2026 10:49:16 -0700 Subject: [PATCH 103/134] test(zaino-fetch): pin the verbosity-2 getblock transaction-object shape (known red) A verbosity-2 `getblock` response carries full transaction objects in its `tx` array, where a verbosity-1 response carries txid strings. zaino-fetch's `BlockObject` types the array for the verbosity-1 shape only, so the object form fails to deserialize and the fetch backend cannot serve verbosity-2 blocks at all; the state backend serves them correctly, so the defect also breaks backend parity. This pins both shapes: the verbosity-1 test guards the behavior the fix must preserve, and the verbosity-2 test is gated `should_panic` on the failure until the fix lands (issue #1380). The verbosity-2 fixture is built by round-tripping a real test-vector transaction through zebra-rpc's own response types, so it reproduces zebrad's wire shape by construction and cannot drift from it. Co-Authored-By: Claude Fable 5 --- .../zaino-fetch/src/jsonrpsee/response.rs | 86 +++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/packages/zaino-fetch/src/jsonrpsee/response.rs b/packages/zaino-fetch/src/jsonrpsee/response.rs index b63fe58f5..1944a5b50 100644 --- a/packages/zaino-fetch/src/jsonrpsee/response.rs +++ b/packages/zaino-fetch/src/jsonrpsee/response.rs @@ -2028,3 +2028,89 @@ mod get_transaction_response { ); } } + +#[cfg(test)] +mod get_block_response { + use super::GetBlockResponse; + + /// Builds the `tx` entry of a verbosity-2 `getblock` response exactly as + /// zebrad serializes it, by round-tripping a real test-vector + /// transaction through zebra-rpc's own response types. Constructing the + /// fixture from zebra's serializer keeps it faithful to the wire across + /// zebra upgrades, with no hand-maintained JSON to drift. + fn verbosity_2_transaction_json() -> serde_json::Value { + use zebra_chain::serialization::ZcashDeserializeInto as _; + + let vector = wire_serialized_transaction_test_data::transactions::get_test_vectors() + .into_iter() + .next() + .expect("the test-vector crate provides at least one transaction"); + let transaction: zebra_chain::transaction::Transaction = vector + .tx + .as_slice() + .zcash_deserialize_into() + .expect("test-vector transactions deserialize"); + let txid = transaction.hash(); + let tx_object = zebra_rpc::client::TransactionObject::from_transaction( + std::sync::Arc::new(transaction), + Some(zebra_chain::block::Height(2)), + Some(1), + &zebra_chain::parameters::Network::Mainnet, + None, + None, + None, + txid, + ); + serde_json::to_value(zebra_rpc::methods::GetBlockTransaction::Object(Box::new( + tx_object, + ))) + .expect("zebra's response types serialize") + } + + fn block_json_with_tx(tx: serde_json::Value) -> serde_json::Value { + serde_json::json!({ + "hash": "0000000000000000000000000000000000000000000000000000000000000002", + "confirmations": 1, + "tx": [tx], + "trees": {}, + }) + } + + /// A verbosity-1 `getblock` response carries txid strings in `tx`. This + /// pins the shape the type already handles, so the verbosity-2 fix below + /// cannot regress it. + #[test] + fn parses_verbosity_1_txid_strings() { + let block_json = block_json_with_tx(serde_json::json!( + "e85e5729b34c34e8e62aa639302cb3434bb961666e13191da6dc39fcf775b320" + )); + + let parsed: GetBlockResponse = + serde_json::from_value(block_json).expect("verbosity-1 block objects deserialize"); + let GetBlockResponse::Object(block) = parsed else { + panic!("a JSON block object must parse as the object variant"); + }; + assert_eq!(block.tx.len(), 1); + } + + /// A verbosity-2 `getblock` response carries full transaction objects in + /// `tx`; the response type must deserialize that shape as well. (known + /// red: the `tx` field is typed for the verbosity-1 shape only, so the + /// object form fails and the fetch backend cannot serve verbosity-2 + /// blocks at all; issue #1380. The connector surfaces the failure as + /// `invalid type: map, expected a string` against `Box`; + /// through the untagged response enum it appears as the variant + /// mismatch below.) + #[test] + #[should_panic(expected = "data did not match any variant of untagged enum GetBlockResponse")] + fn parses_verbosity_2_transaction_objects() { + let block_json = block_json_with_tx(verbosity_2_transaction_json()); + + let parsed: GetBlockResponse = + serde_json::from_value(block_json).expect("verbosity-2 block objects deserialize"); + let GetBlockResponse::Object(block) = parsed else { + panic!("a JSON block object must parse as the object variant"); + }; + assert_eq!(block.tx.len(), 1); + } +} From 576b72df07f4d753a9c46fa31c0fd3d37e281e65 Mon Sep 17 00:00:00 2001 From: zancas Date: Tue, 7 Jul 2026 10:51:16 -0700 Subject: [PATCH 104/134] fix(zaino-fetch): deserialize verbosity-2 getblock transaction objects The `tx` field of zaino-fetch's `BlockObject` becomes a vector of zebra-rpc's `GetBlockTransaction`, the untagged enum that already models both wire shapes: hex-encoded transaction IDs at verbosity 1 and full transaction objects at verbosity 2. The fetch backend previously typed the field for the verbosity-1 shape only, so a verbosity-2 response failed to deserialize and the backend could not serve it at all, while the state backend served the same request correctly through zebra's own types (issue #1380). The change is DRY by deletion. Reusing zebra's enum removes zaino's parallel string representation of the same wire data, and the `TryFrom` conversion shrinks from a fallible txid-by-txid re-parse into a straight pass-through, taking its only error path with it. Verbosity-1 behavior is unchanged: the enum's `Hash` variant serializes to the same hex string the field produced before, and the precursor commit's verbosity-1 test pins that shape. The verbosity-2 test sheds its known-red gate and now also asserts the round trip: the parsed entry is a transaction object, and the conversion into zebra-rpc's `GetBlock` passes the object through intact. Closes #1380. Co-Authored-By: Claude Fable 5 --- .../zaino-fetch/src/jsonrpsee/response.rs | 54 +++++++++++-------- 1 file changed, 31 insertions(+), 23 deletions(-) diff --git a/packages/zaino-fetch/src/jsonrpsee/response.rs b/packages/zaino-fetch/src/jsonrpsee/response.rs index 1944a5b50..18248f5c8 100644 --- a/packages/zaino-fetch/src/jsonrpsee/response.rs +++ b/packages/zaino-fetch/src/jsonrpsee/response.rs @@ -1213,8 +1213,10 @@ pub struct BlockObject { #[serde(default, skip_serializing_if = "Option::is_none")] pub difficulty: Option, - /// List of transaction IDs in block order, hex-encoded. - pub tx: Vec, + /// The block's transactions in block order: hex-encoded transaction IDs + /// at verbosity 1, full transaction objects at verbosity 2. Both shapes + /// share zebra-rpc's untagged enum. + pub tx: Vec, /// Chain supply balance #[serde(default)] @@ -1254,15 +1256,7 @@ impl TryFrom for zebra_rpc::methods::GetBlock { Ok(zebra_rpc::methods::GetBlock::Raw(serialized_block.0)) } GetBlockResponse::Object(block) => { - let tx: Result, _> = block - .tx - .into_iter() - .map(|txid| { - txid.parse::() - .map(zebra_rpc::methods::GetBlockTransaction::Hash) - }) - .collect(); - let tx = tx?; + let tx = block.tx; Ok(zebra_rpc::methods::GetBlock::Object(Box::new( zebra_rpc::client::BlockObject::new( @@ -2042,8 +2036,8 @@ mod get_block_response { use zebra_chain::serialization::ZcashDeserializeInto as _; let vector = wire_serialized_transaction_test_data::transactions::get_test_vectors() - .into_iter() - .next() + .first() + .cloned() .expect("the test-vector crate provides at least one transaction"); let transaction: zebra_chain::transaction::Transaction = vector .tx @@ -2094,23 +2088,37 @@ mod get_block_response { } /// A verbosity-2 `getblock` response carries full transaction objects in - /// `tx`; the response type must deserialize that shape as well. (known - /// red: the `tx` field is typed for the verbosity-1 shape only, so the - /// object form fails and the fetch backend cannot serve verbosity-2 - /// blocks at all; issue #1380. The connector surfaces the failure as - /// `invalid type: map, expected a string` against `Box`; - /// through the untagged response enum it appears as the variant - /// mismatch below.) + /// `tx`; the response type must deserialize that shape as well, and the + /// conversion into zebra-rpc's `GetBlock` must preserve the objects + /// (issue #1380). #[test] - #[should_panic(expected = "data did not match any variant of untagged enum GetBlockResponse")] fn parses_verbosity_2_transaction_objects() { let block_json = block_json_with_tx(verbosity_2_transaction_json()); let parsed: GetBlockResponse = serde_json::from_value(block_json).expect("verbosity-2 block objects deserialize"); - let GetBlockResponse::Object(block) = parsed else { + let GetBlockResponse::Object(ref block) = parsed else { panic!("a JSON block object must parse as the object variant"); }; - assert_eq!(block.tx.len(), 1); + assert!( + matches!( + block.tx.as_slice(), + [zebra_rpc::methods::GetBlockTransaction::Object(_)] + ), + "the verbosity-2 entry must parse as a transaction object" + ); + + let converted: zebra_rpc::methods::GetBlock = + parsed.try_into().expect("the object form converts"); + let zebra_rpc::methods::GetBlock::Object(converted) = converted else { + panic!("the conversion must keep the object variant"); + }; + assert!( + matches!( + converted.tx().as_slice(), + [zebra_rpc::methods::GetBlockTransaction::Object(_)] + ), + "the conversion must pass the transaction object through" + ); } } From 63dc84ce2ac6a060352b071b91e179cf93077fe1 Mon Sep 17 00:00:00 2001 From: zancas Date: Tue, 7 Jul 2026 12:06:06 -0700 Subject: [PATCH 105/134] refactor(state): apply ironwood-migration polish from review of #1379 This applies the six non-blocking follow-ups tracked in #1382. Backfill errors now carry remediation advice: when the v1.2.1 to v1.3.0 migration cannot refetch a post-NU6.3 block, the error tells the operator to ensure the backing validator serves the range from NU6.3 activation through the database tip, or to wipe and re-index. The idempotent-write helper is hoisted to module scope and reused at all three migration sites that previously inlined the NO_OVERWRITE plus verify-match pattern (the spent-batch flush, the txid_location rebuild, and the v1.3.0 commitment and ironwood writes). It now takes a lazy describe closure, so the per-row success path allocates nothing; the previous version formatted two messages per height on the success path. At the txid_location site the byte-for-byte comparison subsumes the old decode-and-checksum check, since a corrupt existing row cannot match the rebuilt bytes. The three shielded-pool activation heights are resolved through a new PoolActivationHeights struct in finalised_state.rs, shared by write_blocks_to_height and the migration so the two call sites cannot drift apart. The accessors introduced by #1379 (FinalisedSource::start_validator and the two ironwood_db accessors, plus DbV1::start_validator) narrow from pub(crate) to pub(super), the minimum visibility that compiles. The v1_2_1_cache_migrates_to_current_then_validates test gains a comment justifying its multi_thread flavor: the background validator runs blocking LMDB validation inline on its task, which would starve the test's status polling on a current-thread runtime. Fixes #1382. Co-Authored-By: Claude Fable 5 --- .../src/chain_index/finalised_state.rs | 27 +++ .../finalised_state/finalised_source.rs | 4 +- .../finalised_state/finalised_source/v1.rs | 4 +- .../finalised_source/v1/write_core.rs | 20 +- .../chain_index/finalised_state/migrations.rs | 175 ++++++++---------- .../migrations/v1_2_to_v1_3.rs | 3 + 6 files changed, 115 insertions(+), 118 deletions(-) diff --git a/packages/zaino-state/src/chain_index/finalised_state.rs b/packages/zaino-state/src/chain_index/finalised_state.rs index 2873e6e49..a44aa7e63 100644 --- a/packages/zaino-state/src/chain_index/finalised_state.rs +++ b/packages/zaino-state/src/chain_index/finalised_state.rs @@ -243,6 +243,33 @@ use crate::{ use std::{sync::Arc, time::Duration}; use tokio::time::{interval, MissedTickBehavior}; +/// The activation heights of the three shielded pools whose data +/// [`build_indexed_block_from_source`] assembles, resolved once per run. +/// +/// Both the ingestion loop ([`capability::DbWrite::write_blocks_to_height`]) and the v1.2.1 → +/// v1.3.0 migration backfill need exactly this set of heights; resolving them in one place keeps +/// the two call sites from drifting apart. A `None` height means the pool's network upgrade has no +/// activation height on the given network. +struct PoolActivationHeights { + sapling: Option, + nu5: Option, + nu6_3: Option, +} + +impl PoolActivationHeights { + /// Resolves the Sapling, NU5 (Orchard), and NU6.3 (Ironwood) activation heights on + /// `zebra_network`. + fn resolve(zebra_network: &zebra_chain::parameters::Network) -> Self { + let activation_height = + |pool: super::ShieldedPool| pool.activation_upgrade().activation_height(zebra_network); + Self { + sapling: activation_height(super::ShieldedPool::Sapling), + nu5: activation_height(super::ShieldedPool::Orchard), + nu6_3: activation_height(super::ShieldedPool::Ironwood), + } + } +} + /// Fetches the block at `height_int` from `source` and builds its [`IndexedBlock`], threading /// `parent_chainwork` into the block metadata. /// diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source.rs index 70a9c7137..d2aeb1cce 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source.rs @@ -288,7 +288,7 @@ impl FinalisedSource { /// /// Called by the orchestrator only once all pending migrations have completed, so the /// validator never races a migration that populates the tables its initial scan reads. - pub(crate) fn start_validator(&self) { + pub(super) fn start_validator(&self) { match self { Self::V1(db) => db.start_validator(), Self::Ephemeral(_) => {} @@ -383,7 +383,7 @@ impl FinalisedSource { /// Provides access to the (v1.3.0) `ironwood` table, required for Migration1_2_1To1_3_0 to /// backfill ironwood rows from validator-fetched block data. - pub(crate) fn ironwood_db(&self) -> Result { + pub(super) fn ironwood_db(&self) -> Result { Ok(self.require_v1("v1 ironwood db")?.ironwood_db()) } } diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1.rs index 835579784..56256f242 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1.rs @@ -606,7 +606,7 @@ impl DbV1 { /// migrations have finished (the validator reads tables a migration populates). Takes `&self` /// (the join handle lives behind a `Mutex`) so it can be driven through the shared /// `Arc` the router holds after spawn. - pub(crate) fn start_validator(&self) { + pub(super) fn start_validator(&self) { // Clone everything the task needs so we can move it into the async block. let zaino_db = self.detached_handle(); @@ -852,7 +852,7 @@ impl DbV1 { /// Provides access to the (v1.3.0) `ironwood` table, required for Migration1_2_1To1_3_0 to /// backfill ironwood rows for post-NU6.3 blocks from validator-fetched block data. - pub(crate) fn ironwood_db(&self) -> Database { + pub(super) fn ironwood_db(&self) -> Database { self.ironwood } diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/write_core.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/write_core.rs index 5c3cfc07e..8ca13c12e 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/write_core.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/write_core.rs @@ -2,8 +2,6 @@ use super::*; -use crate::chain_index::ShieldedPool; - #[cfg(feature = "prometheus")] use crate::metric_names::*; @@ -231,20 +229,18 @@ impl DbWrite for DbV1 { height: Height, source: &S, ) -> Result<(), FinalisedStateError> { - use crate::chain_index::finalised_state::build_indexed_block_from_source; + use crate::chain_index::finalised_state::{ + build_indexed_block_from_source, PoolActivationHeights, + }; let network = self.config.network; let zebra_network = network.to_zebra_network(); - let sapling_activation_height = ShieldedPool::Sapling - .activation_upgrade() - .activation_height(&zebra_network) + let pool_activations = PoolActivationHeights::resolve(&zebra_network); + let sapling_activation_height = pool_activations + .sapling .expect("Sapling activation height must be set"); - let nu5_activation_height = ShieldedPool::Orchard - .activation_upgrade() - .activation_height(&zebra_network); - let nu6_3_activation_height = ShieldedPool::Ironwood - .activation_upgrade() - .activation_height(&zebra_network); + let nu5_activation_height = pool_activations.nu5; + let nu6_3_activation_height = pool_activations.nu6_3; // Seed `parent_chainwork` from the current tip header (the block before the first one we // write). On an empty database this is genesis with zero chainwork. Read raw rather than via diff --git a/packages/zaino-state/src/chain_index/finalised_state/migrations.rs b/packages/zaino-state/src/chain_index/finalised_state/migrations.rs index ef414f9d1..57b4cd00d 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/migrations.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/migrations.rs @@ -491,6 +491,34 @@ impl Migration for Migration1_0_0To1_1_0 { /// - No shadow database. struct Migration1_1_0To1_2_0; +/// Writes `value` under `key` with `NO_OVERWRITE`, tolerating an existing byte-identical row. +/// +/// Migrations use this to stay idempotent on crash-resume: a resumed pass may revisit rows it has +/// already committed, and each such row must match the rebuilt bytes exactly. Any difference — +/// a conflicting rebuild or a corrupt existing row — aborts the migration with the message +/// `describe` produces. `describe` runs only on that error path, so the per-row success path +/// allocates nothing. +fn put_idempotent( + txn: &mut lmdb::RwTransaction<'_>, + db: lmdb::Database, + key: &[u8], + value: &[u8], + describe: impl FnOnce() -> String, +) -> Result<(), FinalisedStateError> { + match txn.put(db, &key, &value, WriteFlags::NO_OVERWRITE) { + Ok(()) => Ok(()), + Err(lmdb::Error::KeyExist) => { + let existing = txn.get(db, &key).map_err(FinalisedStateError::LmdbError)?; + if existing == value { + Ok(()) + } else { + Err(FinalisedStateError::Custom(describe())) + } + } + Err(error) => Err(FinalisedStateError::LmdbError(error)), + } +} + /// Flushes a buffered batch of `spent` index entries in sorted key order, then commits them /// together with the Stage B progress watermark and fsyncs. /// @@ -511,26 +539,12 @@ fn flush_migration_spent_batch( let mut txn = env.begin_rw_txn()?; for (outpoint_bytes, tx_location) in buffer.iter() { let entry_bytes = StoredEntryFixed::new(outpoint_bytes, *tx_location).to_bytes()?; - match txn.put( - spent_db, - outpoint_bytes, - &entry_bytes, - WriteFlags::NO_OVERWRITE, - ) { - Ok(()) => {} - Err(lmdb::Error::KeyExist) => { - let existing = txn - .get(spent_db, outpoint_bytes) - .map_err(FinalisedStateError::LmdbError)?; - if existing != entry_bytes { - return Err(FinalisedStateError::Custom(format!( - "conflicting existing spent entry during batched migration for outpoint {}", - hex::encode(outpoint_bytes) - ))); - } - } - Err(error) => return Err(FinalisedStateError::LmdbError(error)), - } + put_idempotent(&mut txn, spent_db, outpoint_bytes, &entry_bytes, || { + format!( + "conflicting existing spent entry during batched migration for outpoint {}", + hex::encode(outpoint_bytes) + ) + })?; } let progress = StoredEntryFixed::new(progress_key, up_to_height + 1); @@ -689,42 +703,20 @@ impl Migration for Migration1_1_0To1_2_0 { let entry_bytes = StoredEntryFixed::new(txid_bytes, *tx_location).to_bytes()?; - match txn.put( + // Idempotent on resume: an existing entry must match byte-for-byte, which + // also rejects a corrupt existing row (its checksum bytes would differ). + put_idempotent( + &mut txn, txid_location_db, txid_bytes, &entry_bytes, - WriteFlags::NO_OVERWRITE, - ) { - Ok(()) => {} - - // Idempotent on resume: an existing entry must match exactly. - Err(lmdb::Error::KeyExist) => { - let existing_bytes = txn - .get(txid_location_db, txid_bytes) - .map_err(FinalisedStateError::LmdbError)?; - let existing_entry = - StoredEntryFixed::::from_bytes(existing_bytes) - .map_err(|error| { - FinalisedStateError::Custom(format!( - "corrupt existing txid_location entry: {error}" - )) - })?; - if !existing_entry.verify(txid_bytes) { - return Err(FinalisedStateError::Custom( - "existing txid_location entry checksum mismatch" - .to_string(), - )); - } - if existing_entry.inner() != tx_location { - return Err(FinalisedStateError::Custom(format!( - "conflicting txid_location entry at height {}", - height.0 - ))); - } - } - - Err(error) => return Err(FinalisedStateError::LmdbError(error)), - } + || { + format!( + "conflicting or corrupt existing txid_location entry at height {}", + height.0 + ) + }, + )?; } let progress = @@ -1069,9 +1061,8 @@ impl Migration for Migration1_2_1To1_3_0 { use crate::chain_index::finalised_state::{ build_indexed_block_from_source, - finalised_source::v1::write_core::build_block_ironwood_entry, + finalised_source::v1::write_core::build_block_ironwood_entry, PoolActivationHeights, }; - use crate::chain_index::ShieldedPool; // Temporary metadata entry recording the next height to rebuild, removed on completion. const MIGRATION_CTD_PROGRESS_KEY: &[u8] = @@ -1100,21 +1091,15 @@ impl Migration for Migration1_2_1To1_3_0 { ) .await?; - // Network-upgrade activation heights (mirrors `write_blocks_to_height`). Blocks at or above - // NU6.3 have their ironwood root/size and ironwood tx list rebuilt from the validator (the - // legacy on-disk data predates ironwood); blocks below it are rebuilt in place from the - // legacy commitment row, with no ironwood. + // Network-upgrade activation heights (shared with `write_blocks_to_height`). Blocks at or + // above NU6.3 have their ironwood root/size and ironwood tx list rebuilt from the validator + // (the legacy on-disk data predates ironwood); blocks below it are rebuilt in place from + // the legacy commitment row, with no ironwood. let network = cfg.network; - let zebra_network = network.to_zebra_network(); - let sapling_activation_height = ShieldedPool::Sapling - .activation_upgrade() - .activation_height(&zebra_network); - let nu5_activation_height = ShieldedPool::Orchard - .activation_upgrade() - .activation_height(&zebra_network); - let nu6_3_activation_height = ShieldedPool::Ironwood - .activation_upgrade() - .activation_height(&zebra_network); + let pool_activations = PoolActivationHeights::resolve(&network.to_zebra_network()); + let sapling_activation_height = pool_activations.sapling; + let nu5_activation_height = pool_activations.nu5; + let nu6_3_activation_height = pool_activations.nu6_3; // Mark migration in progress (observability only; resumption uses the progress key). { @@ -1161,30 +1146,6 @@ impl Migration for Migration1_2_1To1_3_0 { ); let started = std::time::Instant::now(); - // Idempotent write: on resume an already-written row must match byte-for-byte (the - // legacy rebuild is deterministic and the validator refetch is over immutable history). - fn put_idempotent( - txn: &mut lmdb::RwTransaction<'_>, - db: lmdb::Database, - key: &[u8], - value: &[u8], - what: &str, - ) -> Result<(), FinalisedStateError> { - match txn.put(db, &key, &value, WriteFlags::NO_OVERWRITE) { - Ok(()) => Ok(()), - Err(lmdb::Error::KeyExist) => { - let existing = txn.get(db, &key).map_err(FinalisedStateError::LmdbError)?; - if existing != value { - return Err(FinalisedStateError::Custom(format!( - "conflicting rebuilt {what}" - ))); - } - Ok(()) - } - Err(error) => Err(FinalisedStateError::LmdbError(error)), - } - } - while next_height <= db_tip { let height = Height::try_from(next_height) .map_err(|error| FinalisedStateError::Custom(error.to_string()))?; @@ -1218,7 +1179,16 @@ impl Migration for Migration1_2_1To1_3_0 { // are extracted, and neither depends on it. None, ) - .await?; + .await + .map_err(|error| { + FinalisedStateError::Custom(format!( + "v1.3.0 ironwood backfill failed at height {next_height}: {error}. \ + This backfill refetches every stored block from NU6.3 activation \ + through the database tip; ensure the backing validator serves that \ + range, or wipe the finalised-state directory and re-index from the \ + validator." + )) + })?; commitment_bytes = StoredEntryVar::new(&height_bytes, *block.commitment_tree_data()) @@ -1261,16 +1231,17 @@ impl Migration for Migration1_2_1To1_3_0 { new_ctd_db, &height_bytes, &commitment_bytes, - &format!("commitment_tree_data at height {}", height.0), + || { + format!( + "conflicting rebuilt commitment_tree_data at height {}", + height.0 + ) + }, )?; if let Some(bytes) = &ironwood_bytes { - put_idempotent( - &mut txn, - ironwood_db, - &height_bytes, - bytes, - &format!("ironwood at height {}", height.0), - )?; + put_idempotent(&mut txn, ironwood_db, &height_bytes, bytes, || { + format!("conflicting rebuilt ironwood at height {}", height.0) + })?; } let progress = StoredEntryFixed::new(MIGRATION_CTD_PROGRESS_KEY, height + 1); diff --git a/packages/zaino-state/src/chain_index/tests/finalised_state/migrations/v1_2_to_v1_3.rs b/packages/zaino-state/src/chain_index/tests/finalised_state/migrations/v1_2_to_v1_3.rs index 5d215be9c..0c7756e06 100644 --- a/packages/zaino-state/src/chain_index/tests/finalised_state/migrations/v1_2_to_v1_3.rs +++ b/packages/zaino-state/src/chain_index/tests/finalised_state/migrations/v1_2_to_v1_3.rs @@ -39,6 +39,9 @@ fn v1_2_1() -> DbVersion { /// database, then reopens it through the production `FinalisedState::spawn` path (which migrates to /// the current schema and only then starts the validator) and asserts it reaches `Ready` — i.e. the /// migration completed *before* validation, and validation then passed. +// multi_thread required: the background validator runs blocking LMDB validation +// (`validate_block_blocking`) inline on its task, which would starve this test's status polling on +// a current-thread runtime. #[tokio::test(flavor = "multi_thread")] async fn v1_2_1_cache_migrates_to_current_then_validates() { init_tracing(); From 083ced1efc5284218bde941e4b0188434edd3d3a Mon Sep 17 00:00:00 2001 From: zancas Date: Tue, 7 Jul 2026 21:52:25 -0700 Subject: [PATCH 106/134] test!: expect derived Sapling keys from z_validateaddress on all connections The merged NodeBackedIndexerService computes z_validateaddress locally on both connection types, so the zebrad Rpc path now returns the derived diversifier and diversified transmission key instead of zebrad's passthrough response, which omitted them. This failed the pinned expectation in clientless::fetch_service zebrad::validation::z_validate_address deterministically. Adopt the local-computation behavior as the single expectation: fold the Sapling assertion (keys present) into run_z_validate_suite, delete the SaplingSuite enum with its ZebradPassthroughFetchService variant and the passthrough-specific suite function, and drop the suite parameter from run_z_validate_for and its four call sites. The old "safer behavior" rationale for omitting the keys addressed delegation to a remote actor and does not apply to a local derivation; the RPC's deprecation signal is unaffected because the shared helper zaino_state::indexer::z_validate_address still warns with Z_VALIDATE_DEPRECATION on every call. See https://github.com/zingolabs/zaino/pull/1361#issuecomment-4908129940 for the failure analysis. Co-Authored-By: Claude Fable 5 --- live-tests/clientless/src/lib.rs | 75 ++++---------------- live-tests/clientless/tests/fetch_service.rs | 14 ++-- live-tests/clientless/tests/json_server.rs | 7 +- live-tests/clientless/tests/state_service.rs | 7 +- 4 files changed, 24 insertions(+), 79 deletions(-) diff --git a/live-tests/clientless/src/lib.rs b/live-tests/clientless/src/lib.rs index 9e7314757..12d93f9be 100644 --- a/live-tests/clientless/src/lib.rs +++ b/live-tests/clientless/src/lib.rs @@ -83,7 +83,17 @@ pub mod rpc { // assert_eq!(fs_sprout, expected_sprout); - // Sapling (differs by validator) + // Sapling + let expected_sapling = ValidZValidateAddress::sapling( + VALID_SAPLING_ADDRESS.to_string(), + Some(VALID_DIVERSIFIER.to_string()), + Some(VALID_DIVERSIFIED_TRANSMISSION_KEY.to_string()), + ); + assert_known_valid_eq( + rpc_call(VALID_SAPLING_ADDRESS.to_string()).await, + expected_sapling, + "Sapling", + ); // Unified (differs by validator) let expected_unified = @@ -101,70 +111,15 @@ pub mod rpc { assert_eq!(all_zeroes, ZValidateAddressResponse::invalid()); } - pub async fn run_z_validate_sapling(rpc_call: &F) - where - F: Fn(String) -> Fut, - Fut: Future, - { - let expected_sapling = ValidZValidateAddress::sapling( - VALID_SAPLING_ADDRESS.to_string(), - Some(VALID_DIVERSIFIER.to_string()), - Some(VALID_DIVERSIFIED_TRANSMISSION_KEY.to_string()), - ); - assert_known_valid_eq( - rpc_call(VALID_SAPLING_ADDRESS.to_string()).await, - expected_sapling, - "Sapling", - ); - } - - /// zebrad's JSON-RPC passthrough (via FetchService) omits `diversifier` - /// and `diversifiedtransmissionkey` from the Sapling response. This is - /// the safer behavior: address component extraction should happen - /// client-side, not by delegating to a remote actor. - /// - /// See [`DEPRECATION_NOTICE`](zaino_fetch::jsonrpsee::response::z_validate_address::DEPRECATION_NOTICE). - pub async fn run_z_validate_sapling_zebrad_passthrough_fetchservice(rpc_call: &F) - where - F: Fn(String) -> Fut, - Fut: Future, - { - let expected_sapling = ValidZValidateAddress::sapling( - VALID_SAPLING_ADDRESS.to_string(), - None::, - None::, - ); - assert_known_valid_eq( - rpc_call(VALID_SAPLING_ADDRESS.to_string()).await, - expected_sapling, - "Sapling (zebrad passthrough via FetchService — keys omitted)", - ); - } - - /// Which sapling suite to run after the shared suite. - pub enum SaplingSuite { - /// Full response — diversifier and diversifiedtransmissionkey present. - Standard, - /// zebrad's JSON-RPC passthrough (via FetchService) omits those keys. - ZebradPassthroughFetchService, - } - /// Build the `z_validate_address` rpc-call closure from `subscriber` and - /// run the shared validation suite plus the chosen sapling suite. Factors - /// the identical closure + suite-call preamble shared by the four - /// `z_validate_address` tests (fetch_service zcashd/zebrad, state_service, - /// json_server). + /// run the shared validation suite. Factors the identical closure + + /// suite-call preamble shared by the four `z_validate_address` tests + /// (fetch_service zcashd/zebrad, state_service, json_server). #[allow(deprecated)] - pub async fn run_z_validate_for(subscriber: &S, sapling: SaplingSuite) { + pub async fn run_z_validate_for(subscriber: &S) { let rpc_call = |addr: String| async move { subscriber.z_validate_address(addr).await.unwrap() }; run_z_validate_suite(&rpc_call).await; - match sapling { - SaplingSuite::Standard => run_z_validate_sapling(&rpc_call).await, - SaplingSuite::ZebradPassthroughFetchService => { - run_z_validate_sapling_zebrad_passthrough_fetchservice(&rpc_call).await - } - } } } } diff --git a/live-tests/clientless/tests/fetch_service.rs b/live-tests/clientless/tests/fetch_service.rs index f4ac94f37..ac80060df 100644 --- a/live-tests/clientless/tests/fetch_service.rs +++ b/live-tests/clientless/tests/fetch_service.rs @@ -1,6 +1,6 @@ //! These tests compare the output of `FetchService` with the output of `JsonRpcConnector`. -use clientless::rpc::z_validate_address::{run_z_validate_for, SaplingSuite}; +use clientless::rpc::z_validate_address::run_z_validate_for; use futures::StreamExt as _; use zaino_fetch::jsonrpsee::connector::JsonRpSeeConnector; use zaino_proto::proto::compact_formats::CompactBlock; @@ -416,11 +416,11 @@ async fn fetch_service_validate_address(validator: &ValidatorKi /// Launch a fetch-backend manager and run the shared `z_validate_address` /// suite against its subscriber. -async fn z_validate(validator: &ValidatorKind, suite: SaplingSuite) { +async fn z_validate(validator: &ValidatorKind) { let (mut test_manager, fetch_service_subscriber) = zaino_testutils::launch_with_fetch_subscriber::(validator, None).await; - run_z_validate_for(&fetch_service_subscriber, suite).await; + run_z_validate_for(&fetch_service_subscriber).await; test_manager.close().await; } @@ -648,7 +648,7 @@ mod zcashd { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] pub(crate) async fn z_validate_address() { - z_validate::(&ValidatorKind::Zcashd, SaplingSuite::Standard).await; + z_validate::(&ValidatorKind::Zcashd).await; } } @@ -719,11 +719,7 @@ mod zebrad { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] pub(crate) async fn z_validate_address() { - z_validate::( - &ValidatorKind::Zebrad, - SaplingSuite::ZebradPassthroughFetchService, - ) - .await; + z_validate::(&ValidatorKind::Zebrad).await; } } diff --git a/live-tests/clientless/tests/json_server.rs b/live-tests/clientless/tests/json_server.rs index 72df376c5..b8c17443c 100644 --- a/live-tests/clientless/tests/json_server.rs +++ b/live-tests/clientless/tests/json_server.rs @@ -429,11 +429,8 @@ mod zcashd { async fn z_validate_address() { let mut services = zaino_testutils::launch_zcashd_dual_fetch_services().await; - clientless::rpc::z_validate_address::run_z_validate_for( - &services.zcashd_subscriber, - clientless::rpc::z_validate_address::SaplingSuite::Standard, - ) - .await; + clientless::rpc::z_validate_address::run_z_validate_for(&services.zcashd_subscriber) + .await; services.test_manager.close().await; } diff --git a/live-tests/clientless/tests/state_service.rs b/live-tests/clientless/tests/state_service.rs index 9a4190a12..ec6257f9f 100644 --- a/live-tests/clientless/tests/state_service.rs +++ b/live-tests/clientless/tests/state_service.rs @@ -573,11 +573,8 @@ mod zebra { ) .await; - clientless::rpc::z_validate_address::run_z_validate_for( - &services.state_subscriber, - clientless::rpc::z_validate_address::SaplingSuite::Standard, - ) - .await; + clientless::rpc::z_validate_address::run_z_validate_for(&services.state_subscriber) + .await; services.test_manager.close().await; } From 5f2ac737b27853c3714974a7b1d31b0729cace62 Mon Sep 17 00:00:00 2001 From: zancas Date: Tue, 7 Jul 2026 22:22:43 -0700 Subject: [PATCH 107/134] fix(zaino-state): preserve typed error causes across the BlockchainSource boundary zaino-serve recovers zcashd-compatible RPC error codes by downcast-walking std::error::Error::source() chains (getblock_error_object_from_indexer_error looks for TransportError::ErrorStatusCode(500) to return legacy code -8; sendrawtransaction_error_object_from_indexer_error forwards RpcError codes). The connector boundary flattened every validator and read-state error to BlockchainSourceError::Unrecoverable(error.to_string()), which severed the chain, so those walks fell through to a generic -32603 internal error. Add BlockchainSourceError::UnrecoverableWithSource, which boxes the typed cause as #[source] while rendering the same top-level Display message as the old stringified form, with constructors unrecoverable(error) and unrecoverable_context(context, error) accepting both concrete and already-boxed errors (zebra's BoxError). Convert the 84 connector sites that wrapped a typed error; sites that only carry a rendered message keep the string-only Unrecoverable variant. The regression test constructs a Fetch connector against a closed port and asserts the TransportError stays reachable through the source() chain of the get_block_verbose error; it fails on the previous stringifying code. Addresses finding 1 of the PR #1361 review. Co-Authored-By: Claude Fable 5 --- .../zaino-state/src/chain_index/source.rs | 45 ++- .../chain_index/source/mockchain_source.rs | 14 +- .../chain_index/source/validator_connector.rs | 312 ++++++++++-------- 3 files changed, 232 insertions(+), 139 deletions(-) diff --git a/packages/zaino-state/src/chain_index/source.rs b/packages/zaino-state/src/chain_index/source.rs index 982b6bfe2..c06294122 100644 --- a/packages/zaino-state/src/chain_index/source.rs +++ b/packages/zaino-state/src/chain_index/source.rs @@ -379,11 +379,54 @@ pub(super) async fn wait_or_source_change( /// An error originating from a blockchain source. #[derive(Debug, thiserror::Error)] pub enum BlockchainSourceError { - /// Unrecoverable error. + /// Unrecoverable error described only by a message (no underlying + /// typed error exists, e.g. an unexpected response shape). // TODO: Add logic for handling recoverable errors if any are identified // one candidate may be ephemerable network hiccoughs #[error("critical error in backing block source: {0}")] Unrecoverable(String), + /// Unrecoverable error whose typed cause is preserved as + /// [`std::error::Error::source`]. zaino-serve recovers zcashd-compatible + /// RPC error codes by downcast-walking `source()` chains, so errors that + /// wrap a typed transport or RPC error must use this variant rather than + /// [`Self::Unrecoverable`] with a stringified cause. + #[error("critical error in backing block source: {message}")] + UnrecoverableWithSource { + /// Rendered description, including the cause's `Display` output so + /// top-level log lines match the previous stringified form. + message: String, + /// The typed cause, available to `source()`-chain walks. + #[source] + source: Box, + }, +} + +impl BlockchainSourceError { + /// Wraps a typed error, preserving it for `source()`-chain recovery. + /// Accepts both concrete error types and already-boxed errors (e.g. + /// zebra's `BoxError`). + pub(crate) fn unrecoverable( + error: impl Into>, + ) -> Self { + let source = error.into(); + Self::UnrecoverableWithSource { + message: source.to_string(), + source, + } + } + + /// Wraps a typed error with a context prefix, preserving the error for + /// `source()`-chain recovery. + pub(crate) fn unrecoverable_context( + context: impl std::fmt::Display, + error: impl Into>, + ) -> Self { + let source = error.into(); + Self::UnrecoverableWithSource { + message: format!("{context}: {source}"), + source, + } + } } /// Error type returned when invalid data is returned by the validator. diff --git a/packages/zaino-state/src/chain_index/source/mockchain_source.rs b/packages/zaino-state/src/chain_index/source/mockchain_source.rs index 74eb31d20..1bec4de8e 100644 --- a/packages/zaino-state/src/chain_index/source/mockchain_source.rs +++ b/packages/zaino-state/src/chain_index/source/mockchain_source.rs @@ -370,7 +370,7 @@ impl MockchainSource { return Ok(GetBlockHeaderResponse::Raw(HexData( header .zcash_serialize_to_vec() - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?, + .map_err(BlockchainSourceError::unrecoverable)?, ))); } @@ -592,7 +592,7 @@ impl BlockchainSource for MockchainSource { // full block, so serialize it to measure. let size = block .zcash_serialize_to_vec() - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))? + .map_err(BlockchainSourceError::unrecoverable)? .len() as i64; // `chain_supply` / `value_pools` are cumulative pool balances that the test @@ -622,8 +622,8 @@ impl BlockchainSource for MockchainSource { hash: String, verbose: bool, ) -> BlockchainSourceResult { - let hash_or_height = HashOrHeight::from_str(&hash) - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + let hash_or_height = + HashOrHeight::from_str(&hash).map_err(BlockchainSourceError::unrecoverable)?; let height_index = self.resolve_index(&hash_or_height).ok_or_else(|| { BlockchainSourceError::Unrecoverable("block height not in best chain".to_string()) })?; @@ -632,8 +632,8 @@ impl BlockchainSource for MockchainSource { } async fn get_block_deltas(&self, hash: String) -> BlockchainSourceResult { - let hash_or_height = HashOrHeight::from_str(&hash) - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + let hash_or_height = + HashOrHeight::from_str(&hash).map_err(BlockchainSourceError::unrecoverable)?; let GetBlock::Object(object) = self.get_block_verbose(hash_or_height, Some(2)).await? else { return Err(BlockchainSourceError::Unrecoverable( @@ -655,7 +655,7 @@ impl BlockchainSource for MockchainSource { continue; }; let prev_hash = zebra_chain::transaction::Hash::from_str(prevtxid) - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + .map_err(BlockchainSourceError::unrecoverable)?; if prevtx_cache.contains_key(&prev_hash) { continue; } diff --git a/packages/zaino-state/src/chain_index/source/validator_connector.rs b/packages/zaino-state/src/chain_index/source/validator_connector.rs index 603db354a..22d9a01a0 100644 --- a/packages/zaino-state/src/chain_index/source/validator_connector.rs +++ b/packages/zaino-state/src/chain_index/source/validator_connector.rs @@ -115,12 +115,12 @@ impl ValidatorConnector { common.validator_cookie_path.clone(), ) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + .map_err(BlockchainSourceError::unrecoverable)?; let info = fetcher .get_info() .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + .map_err(BlockchainSourceError::unrecoverable)?; Ok((ValidatorConnector::Fetch(fetcher), info)) } @@ -187,7 +187,7 @@ impl ValidatorConnector { .ready() .and_then(|service| service.call(ReadRequest::Tip)) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + .map_err(BlockchainSourceError::unrecoverable)?; let (syncer_height, syncer_tip_hash) = expected_read_response!(syncer_response, Tip) .ok_or_else(|| { BlockchainSourceError::Unrecoverable("no blocks in chain".to_string()) @@ -309,7 +309,7 @@ impl BlockchainSource for ValidatorConnector { { Ok(GetBlockResponse::Raw(raw_block)) => Ok(Some(Arc::new( zebra_chain::block::Block::zcash_deserialize(raw_block.as_ref()) - .map_err(|e| BlockchainSourceError::Unrecoverable(e.to_string()))?, + .map_err(BlockchainSourceError::unrecoverable)?, ))), Ok(_) => unreachable!(), Err(e) => match e { @@ -317,7 +317,7 @@ impl BlockchainSource for ValidatorConnector { // TODO/FIX: zcashd returns this transport error when a block is requested higher than current chain. is this correct? RpcRequestError::Transport(zaino_fetch::jsonrpsee::error::TransportError::ErrorStatusCode(500)) => Ok(None), RpcRequestError::ServerWorkQueueFull => Err(BlockchainSourceError::Unrecoverable("Work queue full. not yet implemented: handling of ephemeral network errors.".to_string())), - _ => Err(BlockchainSourceError::Unrecoverable(e.to_string())), + _ => Err(BlockchainSourceError::unrecoverable(e)), }, } } @@ -325,7 +325,7 @@ impl BlockchainSource for ValidatorConnector { "Read Request of Block returned Read Response of {otherwise:#?} \n\ This should be deterministically unreachable" ), - Err(e) => Err(BlockchainSourceError::Unrecoverable(e.to_string())), + Err(e) => Err(BlockchainSourceError::unrecoverable(e)), }, ValidatorConnector::Fetch(fetch) => { match fetch @@ -334,7 +334,7 @@ impl BlockchainSource for ValidatorConnector { { Ok(GetBlockResponse::Raw(raw_block)) => Ok(Some(Arc::new( zebra_chain::block::Block::zcash_deserialize(raw_block.as_ref()) - .map_err(|e| BlockchainSourceError::Unrecoverable(e.to_string()))?, + .map_err(BlockchainSourceError::unrecoverable)?, ))), Ok(_) => unreachable!(), Err(e) => match e { @@ -342,7 +342,7 @@ impl BlockchainSource for ValidatorConnector { // TODO/FIX: zcashd returns this transport error when a block is requested higher than current chain. is this correct? RpcRequestError::Transport(zaino_fetch::jsonrpsee::error::TransportError::ErrorStatusCode(500)) => Ok(None), RpcRequestError::ServerWorkQueueFull => Err(BlockchainSourceError::Unrecoverable("Work queue full. not yet implemented: handling of ephemeral network errors.".to_string())), - _ => Err(BlockchainSourceError::Unrecoverable(e.to_string())), + _ => Err(BlockchainSourceError::unrecoverable(e)), }, } } @@ -361,10 +361,10 @@ impl BlockchainSource for ValidatorConnector { ValidatorConnector::Fetch(fetch) => fetch .get_block(hash_or_height.to_string(), verbosity) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))? + .map_err(BlockchainSourceError::unrecoverable)? .try_into() .map_err(|error: zebra_chain::serialization::SerializationError| { - BlockchainSourceError::Unrecoverable(error.to_string()) + BlockchainSourceError::unrecoverable(error) }), } } @@ -376,8 +376,8 @@ impl BlockchainSource for ValidatorConnector { ) -> BlockchainSourceResult { match self { ValidatorConnector::State(state) => { - let hash_or_height = HashOrHeight::from_str(&hash) - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + let hash_or_height = + HashOrHeight::from_str(&hash).map_err(BlockchainSourceError::unrecoverable)?; let header = state .get_block_header_inner(hash_or_height, Some(verbose)) .await?; @@ -386,7 +386,7 @@ impl BlockchainSource for ValidatorConnector { ValidatorConnector::Fetch(fetch) => fetch .get_block_header(hash, verbose) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string())), + .map_err(BlockchainSourceError::unrecoverable), } } @@ -396,7 +396,7 @@ impl BlockchainSource for ValidatorConnector { ValidatorConnector::Fetch(fetch) => fetch .get_block_deltas(hash) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string())), + .map_err(BlockchainSourceError::unrecoverable), } } @@ -410,11 +410,11 @@ impl BlockchainSource for ValidatorConnector { false, ) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string())), + .map_err(BlockchainSourceError::unrecoverable), ValidatorConnector::Fetch(fetch) => Ok(fetch .get_difficulty() .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))? + .map_err(BlockchainSourceError::unrecoverable)? .0), } } @@ -425,7 +425,7 @@ impl BlockchainSource for ValidatorConnector { ValidatorConnector::Fetch(fetch) => fetch .get_blockchain_info() .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))? + .map_err(BlockchainSourceError::unrecoverable)? .try_into() .map_err(|_error| { BlockchainSourceError::Unrecoverable( @@ -442,7 +442,7 @@ impl BlockchainSource for ValidatorConnector { .json_rpc_connector() .get_info() .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))? + .map_err(BlockchainSourceError::unrecoverable)? .into()) } @@ -450,21 +450,21 @@ impl BlockchainSource for ValidatorConnector { self.json_rpc_connector() .get_peer_info() .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string())) + .map_err(BlockchainSourceError::unrecoverable) } async fn get_block_subsidy(&self, height: u32) -> BlockchainSourceResult { self.json_rpc_connector() .get_block_subsidy(height) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string())) + .map_err(BlockchainSourceError::unrecoverable) } async fn get_mining_info(&self) -> BlockchainSourceResult { self.json_rpc_connector() .get_mining_info() .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string())) + .map_err(BlockchainSourceError::unrecoverable) } async fn get_tx_out( @@ -476,7 +476,7 @@ impl BlockchainSource for ValidatorConnector { self.json_rpc_connector() .get_tx_out(txid, n, include_mempool) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string())) + .map_err(BlockchainSourceError::unrecoverable) } async fn get_spent_info( @@ -486,7 +486,7 @@ impl BlockchainSource for ValidatorConnector { self.json_rpc_connector() .get_spent_info(request) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string())) + .map_err(BlockchainSourceError::unrecoverable) } async fn get_network_sol_ps( @@ -497,7 +497,7 @@ impl BlockchainSource for ValidatorConnector { self.json_rpc_connector() .get_network_sol_ps(blocks, height) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string())) + .map_err(BlockchainSourceError::unrecoverable) } async fn send_raw_transaction( @@ -510,7 +510,7 @@ impl BlockchainSource for ValidatorConnector { .send_raw_transaction(raw_transaction_hex) .await .map(SentTransactionHash::from) - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string())) + .map_err(BlockchainSourceError::unrecoverable) } async fn get_treestate_by_id( @@ -520,7 +520,7 @@ impl BlockchainSource for ValidatorConnector { self.json_rpc_connector() .get_treestate(hash_or_height) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))? + .map_err(BlockchainSourceError::unrecoverable)? .try_into() .map_err(|_error| { BlockchainSourceError::Unrecoverable("failed to parse treestate".to_string()) @@ -559,7 +559,7 @@ impl BlockchainSource for ValidatorConnector { }) .await .map_err(|e| { - BlockchainSourceError::Unrecoverable(format!("state read failed: {e}")) + BlockchainSourceError::unrecoverable_context("state read failed", e) })?; if let zebra_state::ReadResponse::AnyChainTransaction(opt) = response { @@ -596,9 +596,10 @@ impl BlockchainSource for ValidatorConnector { .get_raw_transaction(zebra_txid.to_string(), Some(0)) .await .map_err(|e| { - BlockchainSourceError::Unrecoverable(format!( - "could not fetch transaction data: {e}" - )) + BlockchainSourceError::unrecoverable_context( + "could not fetch transaction data", + e, + ) })? { serialized_transaction } else { @@ -611,9 +612,10 @@ impl BlockchainSource for ValidatorConnector { std::io::Cursor::new(serialized_transaction.as_ref()), ) .map_err(|e| { - BlockchainSourceError::Unrecoverable(format!( - "could not deserialize transaction data: {e}" - )) + BlockchainSourceError::unrecoverable_context( + "could not deserialize transaction data", + e, + ) })?; Ok(Some((transaction.into(), GetTransactionLocation::Mempool))) } else { @@ -626,9 +628,10 @@ impl BlockchainSource for ValidatorConnector { .get_raw_transaction(txid.to_rpc_hex(), Some(1)) .await .map_err(|e| { - BlockchainSourceError::Unrecoverable(format!( - "could not fetch transaction data: {e}" - )) + BlockchainSourceError::unrecoverable_context( + "could not fetch transaction data", + e, + ) })? { transaction_object } else { @@ -641,9 +644,10 @@ impl BlockchainSource for ValidatorConnector { transaction_object.hex().as_ref(), )) .map_err(|e| { - BlockchainSourceError::Unrecoverable(format!( - "could not deserialize transaction data: {e}" - )) + BlockchainSourceError::unrecoverable_context( + "could not deserialize transaction data", + e, + ) })?; let location = match transaction_object.height() { Some(-1) => GetTransactionLocation::NonbestChain, @@ -673,7 +677,7 @@ impl BlockchainSource for ValidatorConnector { .get_raw_mempool() .await .map_err(|e| { - BlockchainSourceError::Unrecoverable(format!("could not fetch mempool data: {e}")) + BlockchainSourceError::unrecoverable_context("could not fetch mempool data", e) })? .transactions; @@ -681,9 +685,10 @@ impl BlockchainSource for ValidatorConnector { .into_iter() .map(|txid_str| { zebra_chain::transaction::Hash::from_str(&txid_str).map_err(|e| { - BlockchainSourceError::Unrecoverable(format!( - "invalid transaction id '{txid_str}': {e}" - )) + BlockchainSourceError::unrecoverable_context( + format!("invalid transaction id '{txid_str}'"), + e, + ) }) }) .collect::>()?; @@ -711,9 +716,10 @@ impl BlockchainSource for ValidatorConnector { .get_best_blockhash() .await .map_err(|e| { - BlockchainSourceError::Unrecoverable(format!( - "could not fetch best block hash from validator: {e}" - )) + BlockchainSourceError::unrecoverable_context( + "could not fetch best block hash from validator", + e, + ) })? .0, )) @@ -725,9 +731,10 @@ impl BlockchainSource for ValidatorConnector { .get_best_blockhash() .await .map_err(|e| { - BlockchainSourceError::Unrecoverable(format!( - "could not fetch best block hash from validator: {e}" - )) + BlockchainSourceError::unrecoverable_context( + "could not fetch best block hash from validator", + e, + ) })? .0, )), @@ -753,9 +760,10 @@ impl BlockchainSource for ValidatorConnector { .get_block_count() .await .map_err(|e| { - BlockchainSourceError::Unrecoverable(format!( - "could not fetch best block hash from validator: {e}" - )) + BlockchainSourceError::unrecoverable_context( + "could not fetch best block hash from validator", + e, + ) })? .into(), )) @@ -767,9 +775,10 @@ impl BlockchainSource for ValidatorConnector { .get_block_count() .await .map_err(|e| { - BlockchainSourceError::Unrecoverable(format!( - "could not fetch best block hash from validator: {e}" - )) + BlockchainSourceError::unrecoverable_context( + "could not fetch best block hash from validator", + e, + ) })? .into(), )), @@ -988,9 +997,10 @@ impl BlockchainSource for ValidatorConnector { } }) .map_err(|e| { - BlockchainSourceError::Unrecoverable(format!( - "could not get subtrees from validator: {e}" - )) + BlockchainSourceError::unrecoverable_context( + "could not get subtrees from validator", + e, + ) }) } @@ -999,9 +1009,10 @@ impl BlockchainSource for ValidatorConnector { .get_subtrees_by_index(pool.pool_string(), start_index, max_entries) .await .map_err(|e| { - BlockchainSourceError::Unrecoverable(format!( - "could not get subtrees from validator: {e}" - )) + BlockchainSourceError::unrecoverable_context( + "could not get subtrees from validator", + e, + ) })?; Ok(subtrees @@ -1022,9 +1033,10 @@ impl BlockchainSource for ValidatorConnector { }) .collect::, _>>() .map_err(|e| { - BlockchainSourceError::Unrecoverable(format!( - "could not get subtrees from validator: {e}" - )) + BlockchainSourceError::unrecoverable_context( + "could not get subtrees from validator", + e, + ) })?) } } @@ -1056,12 +1068,9 @@ impl BlockchainSource for ValidatorConnector { .await; let (sapling_tree, orchard_tree, ironwood_tree) = match ( //TODO: Better readstateservice error handling - sapling_tree_response - .map_err(|e| BlockchainSourceError::Unrecoverable(e.to_string()))?, - orchard_tree_response - .map_err(|e| BlockchainSourceError::Unrecoverable(e.to_string()))?, - ironwood_tree_response - .map_err(|e| BlockchainSourceError::Unrecoverable(e.to_string()))?, + sapling_tree_response.map_err(BlockchainSourceError::unrecoverable)?, + orchard_tree_response.map_err(BlockchainSourceError::unrecoverable)?, + ironwood_tree_response.map_err(BlockchainSourceError::unrecoverable)?, ) { ( ReadResponse::SaplingTree(saptree), @@ -1097,7 +1106,7 @@ impl BlockchainSource for ValidatorConnector { .to_string(), ) } - _ => BlockchainSourceError::Unrecoverable(e.to_string()), + _ => BlockchainSourceError::unrecoverable(e), })?; let GetTreestateResponse { sapling, @@ -1117,7 +1126,7 @@ impl BlockchainSource for ValidatorConnector { }, ) .transpose() - .map_err(|e| BlockchainSourceError::Unrecoverable(format!("io error: {e}")))?; + .map_err(|e| BlockchainSourceError::unrecoverable_context("io error", e))?; let orchard_frontier = orchard .map_or_else( || Some(Ok(CommitmentTree::empty())), @@ -1130,7 +1139,7 @@ impl BlockchainSource for ValidatorConnector { }, ) .transpose() - .map_err(|e| BlockchainSourceError::Unrecoverable(format!("io error: {e}")))?; + .map_err(|e| BlockchainSourceError::unrecoverable_context("io error", e))?; let ironwood_frontier = ironwood .map_or_else( || Some(Ok(CommitmentTree::empty())), @@ -1143,7 +1152,7 @@ impl BlockchainSource for ValidatorConnector { }, ) .transpose() - .map_err(|e| BlockchainSourceError::Unrecoverable(format!("io error: {e}")))?; + .map_err(|e| BlockchainSourceError::unrecoverable_context("io error", e))?; let sapling_root = sapling_frontier .map(|tree| { zebra_chain::sapling::tree::Root::try_from(tree.root().to_bytes()) @@ -1151,7 +1160,7 @@ impl BlockchainSource for ValidatorConnector { }) .transpose() .map_err(|e| { - BlockchainSourceError::Unrecoverable(format!("could not deser: {e}")) + BlockchainSourceError::unrecoverable_context("could not deser", e) })?; let orchard_root = orchard_frontier .map(|tree| { @@ -1160,7 +1169,7 @@ impl BlockchainSource for ValidatorConnector { }) .transpose() .map_err(|e| { - BlockchainSourceError::Unrecoverable(format!("could not deser: {e}")) + BlockchainSourceError::unrecoverable_context("could not deser", e) })?; let ironwood_root = ironwood_frontier .map(|tree| { @@ -1169,7 +1178,7 @@ impl BlockchainSource for ValidatorConnector { }) .transpose() .map_err(|e| { - BlockchainSourceError::Unrecoverable(format!("could not deser: {e}")) + BlockchainSourceError::unrecoverable_context("could not deser", e) })?; Ok((sapling_root, orchard_root, ironwood_root)) } @@ -1268,14 +1277,10 @@ impl BlockchainSource for ValidatorConnector { let response = read_state .ready() .await - .map_err(|error| { - BlockchainSourceError::Unrecoverable(error.to_string()) - })? + .map_err(BlockchainSourceError::unrecoverable)? .call(ReadRequest::BlockHeader(hash_or_height)) .await - .map_err(|error| { - BlockchainSourceError::Unrecoverable(error.to_string()) - })?; + .map_err(BlockchainSourceError::unrecoverable)?; match response { ReadResponse::BlockHeader { hash, .. } => Ok(BlockInfo::new( @@ -1296,14 +1301,10 @@ impl BlockchainSource for ValidatorConnector { let response = read_state .ready() .await - .map_err(|error| { - BlockchainSourceError::Unrecoverable(error.to_string()) - })? + .map_err(BlockchainSourceError::unrecoverable)? .call(ReadRequest::BlockHeader(hash_or_height)) .await - .map_err(|error| { - BlockchainSourceError::Unrecoverable(error.to_string()) - })?; + .map_err(BlockchainSourceError::unrecoverable)?; match response { ReadResponse::BlockHeader { hash, .. } => Ok(BlockInfo::new( @@ -1330,7 +1331,7 @@ impl BlockchainSource for ValidatorConnector { ValidatorConnector::Fetch(fetch) => fetch .get_address_deltas(params) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string())), + .map_err(BlockchainSourceError::unrecoverable), } } @@ -1343,14 +1344,14 @@ impl BlockchainSource for ValidatorConnector { let mut state = state.read_state_service.clone(); let strings_set = address_strings.valid_addresses().map_err(|error| { - BlockchainSourceError::Unrecoverable(format!("invalid address: {error}")) + BlockchainSourceError::unrecoverable_context("invalid address", error) })?; let response = state .ready() .and_then(|service| service.call(ReadRequest::AddressBalance(strings_set))) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + .map_err(BlockchainSourceError::unrecoverable)?; let (balance, received) = match response { ReadResponse::AddressBalance { balance, received } => (balance, received), @@ -1375,7 +1376,7 @@ impl BlockchainSource for ValidatorConnector { .collect(), ) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))? + .map_err(BlockchainSourceError::unrecoverable)? .into()), } } @@ -1393,7 +1394,7 @@ impl BlockchainSource for ValidatorConnector { .ready() .and_then(|service| service.call(ReadRequest::Tip)) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + .map_err(BlockchainSourceError::unrecoverable)?; let (chain_height, _chain_hash) = expected_read_response!(response, Tip) .ok_or_else(|| { @@ -1420,9 +1421,7 @@ impl BlockchainSource for ValidatorConnector { addresses: GetAddressBalanceRequest::new(addresses) .valid_addresses() .map_err(|error| { - BlockchainSourceError::Unrecoverable(format!( - "invalid address: {error}" - )) + BlockchainSourceError::unrecoverable_context("invalid address", error) })?, height_range: zebra_chain::block::Height(start) @@ -1432,7 +1431,7 @@ impl BlockchainSource for ValidatorConnector { .ready() .and_then(|service| service.call(request)) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + .map_err(BlockchainSourceError::unrecoverable)?; let hashes = expected_read_response!(response, AddressesTransactionIds); @@ -1461,14 +1460,15 @@ impl BlockchainSource for ValidatorConnector { fetch .get_address_txids(addresses, start, end) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))? + .map_err(BlockchainSourceError::unrecoverable)? .transactions .iter() .map(|txid_string| { TransactionHash::from_hex(txid_string.as_bytes()).map_err(|error| { - BlockchainSourceError::Unrecoverable(format!( - "invalid txid from getaddresstxids `{txid_string}`: {error}" - )) + BlockchainSourceError::unrecoverable_context( + format!("invalid txid from getaddresstxids `{txid_string}`"), + error, + ) }) }) .collect::, BlockchainSourceError>>() @@ -1485,7 +1485,7 @@ impl BlockchainSource for ValidatorConnector { let mut state = state.read_state_service.clone(); let valid_addresses = addresses.valid_addresses().map_err(|error| { - BlockchainSourceError::Unrecoverable(format!("invalid address: {error}")) + BlockchainSourceError::unrecoverable_context("invalid address", error) })?; let request = ReadRequest::UtxosByAddresses(valid_addresses); @@ -1493,7 +1493,7 @@ impl BlockchainSource for ValidatorConnector { .ready() .and_then(|service| service.call(request)) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + .map_err(BlockchainSourceError::unrecoverable)?; let utxos = expected_read_response!(response, AddressUtxos); let mut last_output_location = @@ -1536,7 +1536,7 @@ impl BlockchainSource for ValidatorConnector { .collect(), ) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))? + .map_err(BlockchainSourceError::unrecoverable)? .into_iter() .map(|utxos| utxos.into()) .collect()), @@ -1635,14 +1635,14 @@ impl State { GetBlockHeaderResponse::Raw(HexData( header .zcash_serialize_to_vec() - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?, + .map_err(BlockchainSourceError::unrecoverable)?, )) } else { let ReadResponse::SaplingTree(sapling_tree) = state .ready() .and_then(|service| service.call(ReadRequest::SaplingTree(hash_or_height))) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))? + .map_err(BlockchainSourceError::unrecoverable)? else { return Err(BlockchainSourceError::Unrecoverable( "Unexpected response to SaplingTree request".to_string(), @@ -1657,7 +1657,7 @@ impl State { .ready() .and_then(|service| service.call(ReadRequest::Depth(hash))) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))? + .map_err(BlockchainSourceError::unrecoverable)? else { return Err(BlockchainSourceError::Unrecoverable( "Unexpected response to Depth request".to_string(), @@ -1707,7 +1707,7 @@ impl State { .ready() .and_then(|service| service.call(request)) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + .map_err(BlockchainSourceError::unrecoverable)?; let block = expected_read_response!(response, Block); block .map(SerializedBlock::from) @@ -1787,19 +1787,19 @@ impl State { unreachable!("Unexpected responses from state service: {unexpected:?} {unexpected2:?}") } (Err(e), _) | (_, Err(e)) => { - return Err(BlockchainSourceError::Unrecoverable(e.to_string())); + return Err(BlockchainSourceError::unrecoverable(e)); } }; - let orchard_tree_response = orchard_tree_response - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + let orchard_tree_response = + orchard_tree_response.map_err(BlockchainSourceError::unrecoverable)?; let orchard_tree = expected_read_response!(orchard_tree_response, OrchardTree) .ok_or_else(|| { BlockchainSourceError::Unrecoverable("missing orchard tree".to_string()) })?; - let ironwood_tree_response = ironwood_tree_response - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + let ironwood_tree_response = + ironwood_tree_response.map_err(BlockchainSourceError::unrecoverable)?; let ironwood_tree = expected_read_response!(ironwood_tree_response, IronwoodTree) .ok_or_else(|| { BlockchainSourceError::Unrecoverable("missing ironwood tree".to_string()) @@ -1843,8 +1843,8 @@ impl State { /// spent prevout via a best-chain `ReadRequest::Transaction` (finalised + /// non-finalised), then hands off to the shared [`assemble_block_deltas`]. async fn get_block_deltas(&self, hash: String) -> BlockchainSourceResult { - let hash_or_height = HashOrHeight::from_str(&hash) - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + let hash_or_height = + HashOrHeight::from_str(&hash).map_err(BlockchainSourceError::unrecoverable)?; let GetBlock::Object(object) = self.get_block_inner(hash_or_height, Some(2)).await? else { return Err(BlockchainSourceError::Unrecoverable( "getblockdeltas: unexpected raw block".to_string(), @@ -1866,7 +1866,7 @@ impl State { continue; }; let prev_hash = zebra_chain::transaction::Hash::from_str(prevtxid) - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + .map_err(BlockchainSourceError::unrecoverable)?; if prevtx_cache.contains_key(&prev_hash) { continue; } @@ -1875,7 +1875,7 @@ impl State { .ready() .and_then(|service| service.call(ReadRequest::Transaction(prev_hash))) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + .map_err(BlockchainSourceError::unrecoverable)?; let mined_tx = expected_read_response!(response, Transaction).ok_or_else(|| { BlockchainSourceError::Unrecoverable(format!( "getblockdeltas: prevout tx {prevtxid} not in best chain" @@ -1938,7 +1938,7 @@ impl State { .ready() .and_then(|service| service.call(ReadRequest::TipPoolValues)) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + .map_err(BlockchainSourceError::unrecoverable)?; let (height, hash, balance) = match response { ReadResponse::TipPoolValues { tip_height, @@ -1954,14 +1954,14 @@ impl State { .ready() .and_then(|service| service.call(ReadRequest::UsageInfo)) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + .map_err(BlockchainSourceError::unrecoverable)?; let size_on_disk = expected_read_response!(usage_response, UsageInfo); let response = state .ready() .and_then(|service| service.call(ReadRequest::BlockHeader(hash.into()))) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + .map_err(BlockchainSourceError::unrecoverable)?; let header = match response { ReadResponse::BlockHeader { header, .. } => header, unexpected => { @@ -2020,7 +2020,7 @@ impl State { let difficulty = chain_tip_difficulty(network.clone(), self.read_state_service.clone(), false) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + .map_err(BlockchainSourceError::unrecoverable)?; let verification_progress = f64::from(height.0) / f64::from(zebra_estimated_height.0); @@ -2211,7 +2211,7 @@ pub(crate) fn header_to_block_commitments( final_sapling_root: [u8; 32], ) -> BlockchainSourceResult<[u8; 32]> { let hash = match header.commitment(network, height).map_err(|error| { - BlockchainSourceError::Unrecoverable(format!("invalid block commitment: {error}")) + BlockchainSourceError::unrecoverable_context("invalid block commitment", error) })? { zebra_chain::block::Commitment::PreSaplingReserved(bytes) => bytes, zebra_chain::block::Commitment::FinalSaplingRoot(_root) => final_sapling_root, @@ -2232,10 +2232,8 @@ pub(crate) fn header_to_block_commitments( pub(crate) fn zebra_block_header_to_wire( header: GetBlockHeaderResponse, ) -> BlockchainSourceResult { - let value = serde_json::to_value(&header) - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; - serde_json::from_value(value) - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string())) + let value = serde_json::to_value(&header).map_err(BlockchainSourceError::unrecoverable)?; + serde_json::from_value(value).map_err(BlockchainSourceError::unrecoverable) } /// Returns the median of a non-empty set of block times. @@ -2286,7 +2284,7 @@ pub(crate) fn assemble_block_deltas( }; let prev_hash = zebra_chain::transaction::Hash::from_str(prevtxid) - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + .map_err(BlockchainSourceError::unrecoverable)?; let prev_tx = prevtx_cache.get(&prev_hash).ok_or_else(|| { BlockchainSourceError::Unrecoverable(format!( "getblockdeltas: prevout tx {prevtxid} not resolved" @@ -2307,9 +2305,10 @@ pub(crate) fn assemble_block_deltas( // Inputs are debits, so the amount leaves the address. let satoshis: Amount = (-output.value().zatoshis()).try_into().map_err(|error| { - BlockchainSourceError::Unrecoverable(format!( - "getblockdeltas: input amount out of range for {prevtxid}:{prevout}: {error}" - )) + BlockchainSourceError::unrecoverable_context( + format!("getblockdeltas: input amount out of range for {prevtxid}:{prevout}"), + error, + ) })?; inputs.push(InputDelta { @@ -2436,6 +2435,57 @@ mod fetch_pool_treestate_slot { } } +#[cfg(test)] +mod get_block_verbose { + use super::*; + + /// zaino-serve recovers zcashd-compatible RPC error codes by downcast-walking + /// [`std::error::Error::source`] chains (see + /// `getblock_error_object_from_indexer_error` and + /// `sendrawtransaction_error_object_from_indexer_error` in + /// `zaino-serve/src/rpc/jsonrpc/service.rs`). The connector boundary must + /// therefore preserve the typed cause instead of flattening it to a string, + /// or `getblock` failures surface as generic internal errors rather than the + /// legacy `-8` code lightwalletd-family clients key on. + #[tokio::test] + async fn fetch_error_keeps_transport_error_in_source_chain() { + // Port 1 refuses connections, so the request fails at the transport + // layer without contacting any validator. + let connector = zaino_fetch::jsonrpsee::connector::JsonRpSeeConnector::new_with_basic_auth( + "http://127.0.0.1:1/" + .parse() + .expect("static url literal parses"), + "user".to_string(), + "password".to_string(), + ) + .expect("connector construction is network-free"); + let source = ValidatorConnector::Fetch(connector); + + let error = source + .get_block_verbose(HashOrHeight::Height(zebra_chain::block::Height(1)), Some(1)) + .await + .expect_err("a request to a closed port must fail"); + + let mut current: Option<&(dyn std::error::Error + 'static)> = Some(&error); + let mut transport_error_reachable = false; + while let Some(source_error) = current { + if source_error + .downcast_ref::() + .is_some() + { + transport_error_reachable = true; + break; + } + current = source_error.source(); + } + assert!( + transport_error_reachable, + "the typed TransportError must stay reachable via the source() chain; \ + stringifying it strips the RPC error code the serve layer recovers" + ); + } +} + #[cfg(test)] mod ironwood_treestate_slot { /// Regression test: when the validator reported no ironwood treestate (below NU6.3 From fd322a174b065f174596b7502959b80478410ca0 Mon Sep 17 00:00:00 2001 From: zancas Date: Tue, 7 Jul 2026 22:26:42 -0700 Subject: [PATCH 108/134] fix(zaino-state): keep the typed RpcError on sendrawtransaction hex rejection validate_raw_transaction_hex returns an RpcError carrying zcashd's legacy InvalidParameter code (-8), but send_raw_transaction stringified it through ChainIndexError::internal, so zaino-serve's source()-chain walk (sendrawtransaction_error_object_from_indexer_error) could not find it and wallets submitting malformed hex received a generic -32603 internal error. Add ChainIndexError::internal_from, which preserves the typed error as source while rendering the same message, and use it at the validation site. The regression test drives ChainIndexRpcExt::send_raw_transaction over a MockchainSource with invalid hex and asserts the -8 code is recoverable from the source() chain; it fails on the stringifying code. The mock's send_raw_transaction stub would panic if reached, so the test also pins that rejection happens before any validator round-trip. Addresses finding 2 of the PR #1361 review. Co-Authored-By: Claude Fable 5 --- packages/zaino-state/src/chain_index.rs | 2 +- .../src/chain_index/tests/mockchain_tests.rs | 37 +++++++++++++++++++ packages/zaino-state/src/error.rs | 12 ++++++ 3 files changed, 50 insertions(+), 1 deletion(-) diff --git a/packages/zaino-state/src/chain_index.rs b/packages/zaino-state/src/chain_index.rs index 83dce333d..008461c3d 100644 --- a/packages/zaino-state/src/chain_index.rs +++ b/packages/zaino-state/src/chain_index.rs @@ -2865,7 +2865,7 @@ impl ChainIndexRpcExt for NodeBackedChainIndexSubscrib raw_transaction_hex: String, ) -> Result { validate_raw_transaction_hex(&raw_transaction_hex) - .map_err(|error| ChainIndexError::internal(error.to_string()))?; + .map_err(ChainIndexError::internal_from)?; self.source() .send_raw_transaction(raw_transaction_hex) .await diff --git a/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs b/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs index 95f8e489b..76a7d6107 100644 --- a/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs +++ b/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs @@ -1072,3 +1072,40 @@ async fn node_backed_indexer_service_serves_latest_block() { let latest = service.get_latest_block().await.unwrap(); assert_eq!(latest.height, expected_tip as u64); } + +/// `sendrawtransaction` rejections must carry zcashd's legacy error code: +/// zaino-serve forwards the code by downcast-walking the `source()` chain for +/// the typed `RpcError` (`sendrawtransaction_error_object_from_indexer_error`), +/// so stringifying it downgrades the legacy `-8` "invalid hex" rejection to a +/// generic `-32603` internal error. +/// +/// Invalid hex fails in local validation before the source is consulted, so +/// this also pins that the rejection happens without a validator round-trip +/// (the mock's `send_raw_transaction` would panic if reached). +#[tokio::test(flavor = "multi_thread")] +async fn send_raw_transaction_invalid_hex_keeps_legacy_error_code() { + let (_blocks, _indexer, index_reader, _mockchain) = + load_test_vectors_and_sync_chain_index(MockchainMode::Static).await; + + let error = index_reader + .send_raw_transaction("notahexstring".to_string()) + .await + .expect_err("invalid hex must be rejected"); + + let mut current: Option<&(dyn std::error::Error + 'static)> = Some(&error); + let mut rpc_error_code = None; + while let Some(source_error) = current { + if let Some(rpc_error) = + source_error.downcast_ref::() + { + rpc_error_code = Some(rpc_error.code); + break; + } + current = source_error.source(); + } + assert_eq!( + rpc_error_code, + Some(zebra_rpc::server::error::LegacyCode::InvalidParameter as i64), + "the typed RpcError (legacy code -8) must stay reachable via the source() chain" + ); +} diff --git a/packages/zaino-state/src/error.rs b/packages/zaino-state/src/error.rs index 2093ca68b..b0eb87b31 100644 --- a/packages/zaino-state/src/error.rs +++ b/packages/zaino-state/src/error.rs @@ -510,6 +510,18 @@ impl ChainIndexError { } } + /// Constructs an `InternalServerError`-kind error from a typed error, + /// preserving it as `source` so zaino-serve's RPC-error-code recovery + /// walks can downcast to it (e.g. the legacy `-8` code carried by an + /// [`RpcError`](zaino_fetch::jsonrpsee::connector::RpcError)). + pub(crate) fn internal_from(error: impl std::error::Error + Send + Sync + 'static) -> Self { + Self { + kind: ChainIndexErrorKind::InternalServerError, + message: error.to_string(), + source: Some(Box::new(error)), + } + } + pub(crate) fn backing_validator(value: impl std::error::Error + Send + Sync + 'static) -> Self { Self { kind: ChainIndexErrorKind::InternalServerError, From 39053f16816ac441888ac85382c6499d990d7beb Mon Sep 17 00:00:00 2001 From: zancas Date: Tue, 7 Jul 2026 22:31:07 -0700 Subject: [PATCH 109/134] fix(zaino-state)!: return Option from chaintip_update_subscriber instead of panicking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Before the service merge the chain-tip subscriber existed only on the State-backed subscriber type, so requesting one from an RPC-backed service was a compile error. The merged NodeBackedIndexerServiceSubscriber exposed it as an ungated pub method whose .expect() panicked at runtime whenever the service was configured with backend = "rpc" — a caller-dependent condition, not a program invariant, so the panic also violated the repository's .expect() policy. Lift chain_tip_change onto the BlockchainSource trait with a None default (only the Direct connection drives its own Zebra syncer and owns a local tip-change stream), make chaintip_update_subscriber generic over the source, and return Option. The live-test caller now unwraps explicitly on the Direct connection it launches. The regression test builds the service over a MockchainSource (a source with no tip stream, like the Rpc connection) and asserts the subscriber request yields None; before the fix the non-panicking form did not exist and the only reachable behavior was the runtime panic. Addresses finding 3 of the PR #1361 review. Co-Authored-By: Claude Fable 5 --- live-tests/clientless/tests/state_service.rs | 5 +++- .../zaino-state/src/chain_index/source.rs | 8 +++++ .../chain_index/source/validator_connector.rs | 21 +++++++------ .../src/chain_index/tests/mockchain_tests.rs | 26 ++++++++++++++++ .../src/indexer/node_backed_indexer.rs | 30 +++++++++---------- 5 files changed, 62 insertions(+), 28 deletions(-) diff --git a/live-tests/clientless/tests/state_service.rs b/live-tests/clientless/tests/state_service.rs index ec6257f9f..c1ccbd6c5 100644 --- a/live-tests/clientless/tests/state_service.rs +++ b/live-tests/clientless/tests/state_service.rs @@ -383,7 +383,10 @@ mod zebra { #[tokio::test(flavor = "multi_thread")] async fn state_service_chaintip_update_subscriber() { let services = launch_regtest(true).await; - let mut chaintip_subscriber = services.state_subscriber.chaintip_update_subscriber(); + let mut chaintip_subscriber = services + .state_subscriber + .chaintip_update_subscriber() + .expect("the Direct connection exposes a local tip-change stream"); services .test_manager .generate_blocks_and_check_each( diff --git a/packages/zaino-state/src/chain_index/source.rs b/packages/zaino-state/src/chain_index/source.rs index c06294122..5080c367e 100644 --- a/packages/zaino-state/src/chain_index/source.rs +++ b/packages/zaino-state/src/chain_index/source.rs @@ -145,6 +145,14 @@ pub trait BlockchainSource: Clone + Send + Sync + 'static { /// minimum difficulty (the `getdifficulty` RPC value). fn get_difficulty(&self) -> impl SendFut>; + /// A watch stream of the source's chain-tip changes, when the source owns + /// one locally. Only the `Direct` validator connection (which drives its + /// own Zebra syncer) does; every other source observes tips by polling, + /// and inherits this `None` default. + fn chain_tip_change(&self) -> Option { + None + } + /// Returns the `getblockchaininfo` response. fn get_blockchain_info( &self, diff --git a/packages/zaino-state/src/chain_index/source/validator_connector.rs b/packages/zaino-state/src/chain_index/source/validator_connector.rs index 22d9a01a0..5dc94d43e 100644 --- a/packages/zaino-state/src/chain_index/source/validator_connector.rs +++ b/packages/zaino-state/src/chain_index/source/validator_connector.rs @@ -224,17 +224,6 @@ impl ValidatorConnector { Ok((source, info)) } - /// Watches the Zebra syncer's chain-tip changes, when this connector owns one. - /// - /// `Some` for the `State` variant (which drives its own syncer); `None` for the - /// JSON-RPC `Fetch` variant, which has no local tip-change stream. - pub(crate) fn chain_tip_change(&self) -> Option { - match self { - ValidatorConnector::State(state) => Some(state.chain_tip_change.clone()), - ValidatorConnector::Fetch(_) => None, - } - } - /// The backing [`ReadStateService`], when this connector is `State`-backed. /// /// Test-only escape hatch: live tests recompute expected chain data directly off @@ -288,6 +277,16 @@ fn fetch_pool_treestate_slot(treestate: zebra_rpc::client::Treestate) -> Option< } impl BlockchainSource for ValidatorConnector { + /// `Some` for the `State` variant, which drives its own Zebra syncer; the + /// JSON-RPC `Fetch` variant has no local tip-change stream and keeps the + /// trait's `None` default semantics. + fn chain_tip_change(&self) -> Option { + match self { + ValidatorConnector::State(state) => Some(state.chain_tip_change.clone()), + ValidatorConnector::Fetch(_) => None, + } + } + // ********** Block methods ********** async fn get_block( diff --git a/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs b/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs index 76a7d6107..b3aa8a1b5 100644 --- a/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs +++ b/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs @@ -1073,6 +1073,32 @@ async fn node_backed_indexer_service_serves_latest_block() { assert_eq!(latest.height, expected_tip as u64); } +/// The `Rpc` connection has no local chain-tip-change stream, so requesting a +/// chain-tip subscriber over such a source must yield `None` rather than +/// panic. Before this method returned `Option`, it existed only in a +/// panicking form (`.expect("chaintip_update_subscriber requires the Direct +/// connection")`) reachable by any embedder configured with `backend = "rpc"`; +/// pre-merge the misuse was a compile error because only the State-backed +/// subscriber type had the method. +#[tokio::test(flavor = "multi_thread")] +async fn chaintip_update_subscriber_absent_without_tip_stream() { + use crate::indexer::node_backed_indexer::NodeBackedIndexerServiceSubscriber; + use zaino_common::{network::ActivationHeights, Network}; + + let (_blocks, _indexer, index_reader, _mockchain) = + load_test_vectors_and_sync_chain_index(MockchainMode::Static).await; + + let service = NodeBackedIndexerServiceSubscriber::new_for_test( + index_reader, + Network::Regtest(ActivationHeights::default()), + ); + + assert!( + service.chaintip_update_subscriber().is_none(), + "a source with no local tip-change stream must yield no subscriber, not panic" + ); +} + /// `sendrawtransaction` rejections must carry zcashd's legacy error code: /// zaino-serve forwards the code by downcast-walking the `source()` chain for /// the typed `RpcError` (`sendrawtransaction_error_object_from_indexer_error`), diff --git a/packages/zaino-state/src/indexer/node_backed_indexer.rs b/packages/zaino-state/src/indexer/node_backed_indexer.rs index bd804312a..27e666eaa 100644 --- a/packages/zaino-state/src/indexer/node_backed_indexer.rs +++ b/packages/zaino-state/src/indexer/node_backed_indexer.rs @@ -291,24 +291,22 @@ impl ChainTipSubscriber { } } -/// Methods available only on the production (`ValidatorConnector`-backed) subscriber: -/// the `Direct` connection owns a Zebra chain-tip stream and `ReadStateService` that the -/// generic source abstraction does not expose. -impl NodeBackedIndexerServiceSubscriber { - /// Gets a subscriber to any updates to the latest chain tip. - /// - /// Only meaningful for the `Direct` connection; the `Rpc` connection has no local - /// tip-change stream. - pub fn chaintip_update_subscriber(&self) -> ChainTipSubscriber { - ChainTipSubscriber { - monitor: self - .indexer - .source() - .chain_tip_change() - .expect("chaintip_update_subscriber requires the Direct connection"), - } +impl NodeBackedIndexerServiceSubscriber { + /// A subscriber to chain-tip updates, when the backing source exposes a + /// local tip-change stream. `Some` only on the `Direct` connection; the + /// `Rpc` connection (and any other stream-less source) observes tips by + /// polling the validator and yields `None`. + pub fn chaintip_update_subscriber(&self) -> Option { + Some(ChainTipSubscriber { + monitor: self.indexer.source().chain_tip_change()?, + }) } +} +/// Methods available only on the production (`ValidatorConnector`-backed) subscriber: +/// the `Direct` connection owns a `ReadStateService` that the generic source +/// abstraction does not expose. +impl NodeBackedIndexerServiceSubscriber { /// The backing Zebra [`zebra_state::ReadStateService`] (`Direct` connection only). /// /// Test-only escape hatch: live tests recompute expected chain data directly off the From dfc6d98668d0f5b3e0c2fee0ae574859175e6994 Mon Sep 17 00:00:00 2001 From: zancas Date: Tue, 7 Jul 2026 22:33:25 -0700 Subject: [PATCH 110/134] fix(zaino-state): replace the getaddressdeltas tip unwrap with a typed error The Direct arm of get_address_deltas clamped its height range with self.get_best_block_height().await?.unwrap(), panicking whenever the source reports no best height (nothing indexed yet). Production code disallows .unwrap() without exception. Extract the range clamp into clamp_deltas_range_to_tip, which turns the absent-tip case into a BlockchainSourceError and preserves the existing clamping semantics (end == 0 means "to the tip"; both bounds clamp down to the tip). The tests pin the absent-tip error, which had no non-panicking form before the fix, and the clamping behavior verbatim. Addresses finding 4 of the PR #1361 review. Co-Authored-By: Claude Fable 5 --- .../chain_index/source/validator_connector.rs | 75 ++++++++++++++++--- 1 file changed, 66 insertions(+), 9 deletions(-) diff --git a/packages/zaino-state/src/chain_index/source/validator_connector.rs b/packages/zaino-state/src/chain_index/source/validator_connector.rs index 5dc94d43e..8974271cd 100644 --- a/packages/zaino-state/src/chain_index/source/validator_connector.rs +++ b/packages/zaino-state/src/chain_index/source/validator_connector.rs @@ -1204,15 +1204,11 @@ impl BlockchainSource for ValidatorConnector { GetAddressDeltasParams::Address(a) => (vec![a.clone()], 0, 0, false), }; - let tip = self.get_best_block_height().await?.unwrap().into(); - let mut start = Height(start_raw); - let mut end = Height(end_raw); - if end == Height(0) || end > tip { - end = tip; - } - if start > tip { - start = tip; - } + let (start, end) = clamp_deltas_range_to_tip( + self.get_best_block_height().await?, + start_raw, + end_raw, + )?; let transactions: Vec> = { let tx_ids_request = @@ -2434,6 +2430,67 @@ mod fetch_pool_treestate_slot { } } +/// Clamps a `getaddressdeltas` height range to the current best tip: +/// `end == 0` means "to the tip", and both bounds clamp down to it. A source +/// that reports no best height (nothing indexed yet) is a typed error. +fn clamp_deltas_range_to_tip( + tip: Option, + start_raw: u32, + end_raw: u32, +) -> BlockchainSourceResult<(Height, Height)> { + let tip: Height = tip + .ok_or_else(|| { + BlockchainSourceError::Unrecoverable( + "getaddressdeltas: the source reports no best block height".to_string(), + ) + })? + .into(); + let mut start = Height(start_raw); + let mut end = Height(end_raw); + if end == Height(0) || end > tip { + end = tip; + } + if start > tip { + start = tip; + } + Ok((start, end)) +} + +#[cfg(test)] +mod clamp_deltas_range_to_tip { + use super::*; + + /// A source that reports no best height (nothing indexed yet, or the + /// RPC fallback found no tip) must yield a typed error, not a panic: + /// this range clamp previously lived inline in `get_address_deltas` + /// behind `get_best_block_height().await?.unwrap()`. + #[test] + fn absent_tip_is_a_typed_error() { + assert!( + clamp_deltas_range_to_tip(None, 0, 0).is_err(), + "an absent tip must surface as a BlockchainSourceError" + ); + } + + /// `end == 0` means "to the tip", and both bounds clamp down to the tip. + #[test] + fn bounds_clamp_to_tip() { + let tip = Some(zebra_chain::block::Height(100)); + + let (start, end) = + clamp_deltas_range_to_tip(tip, 5, 0).expect("a present tip clamps successfully"); + assert_eq!((start, end), (Height(5), Height(100))); + + let (start, end) = + clamp_deltas_range_to_tip(tip, 5, 400).expect("a present tip clamps successfully"); + assert_eq!((start, end), (Height(5), Height(100))); + + let (start, end) = + clamp_deltas_range_to_tip(tip, 300, 50).expect("a present tip clamps successfully"); + assert_eq!((start, end), (Height(100), Height(50))); + } +} + #[cfg(test)] mod get_block_verbose { use super::*; From e094e56660991e7c7ec4c7e213af4b8a64895037 Mon Sep 17 00:00:00 2001 From: zancas Date: Tue, 7 Jul 2026 22:38:09 -0700 Subject: [PATCH 111/134] fix(zaino-state): make the service teardown safe off the multi-thread runtime NodeBackedIndexerService's shutdown_blocking (shared by ZcashService::close and Drop) entered block_in_place + Handle::current().block_on unconditionally. block_in_place panics on a current-thread runtime and Handle::current() panics on a thread with no runtime; either way a panic inside Drop during unwind aborts the process, so merely dropping the service in the wrong context was fatal. The old StateService drop was a plain JoinHandle::abort with no runtime requirement. Guard the blocking path with Handle::try_current() and a runtime-flavor check. Contexts that cannot block fall back to NodeBackedChainIndex::shutdown_sync_best_effort, a synchronous teardown that cancels the sync worker, closes the mempool, and releases source-owned background work, skipping only the finalised DB's async shutdown step (the DB's own Drop still releases its resources). The regression tests build the service over a MockchainSource and drop it on a plain thread with no runtime and inside a current-thread runtime; both panicked before the fix. Both wrap the drop in a spawned thread because the mock harness itself requires the multi-thread flavor (its finalised-state validation uses block_in_place). Addresses finding 5 of the PR #1361 review. Co-Authored-By: Claude Fable 5 --- packages/zaino-state/src/chain_index.rs | 13 +++ .../src/chain_index/tests/mockchain_tests.rs | 55 ++++++++++++ .../src/indexer/node_backed_indexer.rs | 87 ++++++++++++++----- 3 files changed, 133 insertions(+), 22 deletions(-) diff --git a/packages/zaino-state/src/chain_index.rs b/packages/zaino-state/src/chain_index.rs index 008461c3d..47d3a9463 100644 --- a/packages/zaino-state/src/chain_index.rs +++ b/packages/zaino-state/src/chain_index.rs @@ -986,6 +986,19 @@ impl NodeBackedChainIndex { Ok(()) } + /// Synchronous best-effort teardown for contexts that cannot run async + /// work (a `Drop` on a current-thread runtime, or on a thread with no + /// runtime at all): cancels the sync worker, closes the mempool, and + /// releases source-owned background work. The finalised DB's async + /// [`shutdown`](Self::shutdown) step is skipped; the DB's own `Drop` + /// remains responsible for releasing its resources. + pub(crate) fn shutdown_sync_best_effort(&self) { + self.cancel_token.cancel(); + self.status.store(StatusType::Closing); + self.mempool.close(); + self.source.shutdown(); + } + /// Displays the status of the chain_index pub fn status(&self) -> StatusType { let finalized_status = self.finalized_db.status(); diff --git a/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs b/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs index b3aa8a1b5..6d0f0a611 100644 --- a/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs +++ b/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs @@ -1073,6 +1073,61 @@ async fn node_backed_indexer_service_serves_latest_block() { assert_eq!(latest.height, expected_tip as u64); } +/// Dropping the service must not panic on a thread with no Tokio runtime. +/// `Drop` runs the blocking teardown, which previously called +/// `Handle::current()` unconditionally; outside a runtime that panics, and a +/// panic inside `Drop` during unwind aborts the process. +/// +/// multi_thread required: the harness's finalised-state validation uses +/// `block_in_place`. The drop under test happens on a separate plain thread. +#[tokio::test(flavor = "multi_thread")] +async fn service_drop_survives_thread_without_runtime() { + use crate::indexer::node_backed_indexer::NodeBackedIndexerService; + use zaino_common::{network::ActivationHeights, Network}; + + let (_blocks, indexer, _index_reader, _mockchain) = + load_test_vectors_and_sync_chain_index(MockchainMode::Static).await; + + let service = NodeBackedIndexerService::new_for_test( + indexer, + Network::Regtest(ActivationHeights::default()), + ); + + std::thread::spawn(move || drop(service)) + .join() + .expect("dropping the service off-runtime must not panic"); +} + +/// Dropping the service must not panic on a current-thread Tokio runtime, +/// where `block_in_place` (the old unconditional teardown entry) aborts. +/// +/// multi_thread required: the harness's finalised-state validation uses +/// `block_in_place`. The drop under test happens inside a current-thread +/// runtime built on a separate thread. +#[tokio::test(flavor = "multi_thread")] +async fn service_drop_survives_current_thread_runtime() { + use crate::indexer::node_backed_indexer::NodeBackedIndexerService; + use zaino_common::{network::ActivationHeights, Network}; + + let (_blocks, indexer, _index_reader, _mockchain) = + load_test_vectors_and_sync_chain_index(MockchainMode::Static).await; + + let service = NodeBackedIndexerService::new_for_test( + indexer, + Network::Regtest(ActivationHeights::default()), + ); + + std::thread::spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("current-thread runtime builds") + .block_on(async move { drop(service) }); + }) + .join() + .expect("dropping the service on a current-thread runtime must not panic"); +} + /// The `Rpc` connection has no local chain-tip-change stream, so requesting a /// chain-tip subscriber over such a source must yield `None` rather than /// panic. Before this method returned `Option`, it existed only in a diff --git a/packages/zaino-state/src/indexer/node_backed_indexer.rs b/packages/zaino-state/src/indexer/node_backed_indexer.rs index 27e666eaa..9fac9ef83 100644 --- a/packages/zaino-state/src/indexer/node_backed_indexer.rs +++ b/packages/zaino-state/src/indexer/node_backed_indexer.rs @@ -116,12 +116,26 @@ impl NodeBackedIndexerService { /// Tears down the indexer (sync loop, finalised DB, mempool, and any source-owned /// syncer task) from a synchronous context. Shared by [`ZcashService::close`] and /// [`Drop`]. + /// + /// `block_in_place` is only legal on a multi-thread runtime; on a + /// current-thread runtime or a thread with no runtime it panics, and a + /// panic inside `Drop` during unwind aborts the process. Those contexts + /// fall back to the synchronous best-effort teardown, which skips only + /// the finalised DB's async shutdown step (the DB's own `Drop` still + /// releases its resources). fn shutdown_blocking(&self) { - tokio::task::block_in_place(|| { - tokio::runtime::Handle::current().block_on(async { - let _ = self.indexer.shutdown().await; - }); - }); + use tokio::runtime::{Handle, RuntimeFlavor}; + + match Handle::try_current() { + Ok(handle) if handle.runtime_flavor() == RuntimeFlavor::MultiThread => { + tokio::task::block_in_place(|| { + handle.block_on(async { + let _ = self.indexer.shutdown().await; + }); + }); + } + _ => self.indexer.shutdown_sync_best_effort(), + } } } @@ -240,6 +254,49 @@ impl NodeBackedIndexerServiceSubscriber { } } +/// Placeholder metadata and config for test-only service construction. +#[cfg(test)] +fn test_service_parts(network: zaino_common::Network) -> (ServiceMetadata, CommonBackendConfig) { + ( + ServiceMetadata::new( + get_build_info("test".to_string()), + network.to_zebra_network(), + "test-build".to_string(), + "test-subversion".to_string(), + ), + CommonBackendConfig::new( + "127.0.0.1:0".to_string(), + None, + None, + None, + zaino_common::ServiceConfig::default(), + zaino_common::StorageConfig::default(), + true, + network, + None, + ), + ) +} + +#[cfg(test)] +impl NodeBackedIndexerService { + /// Wraps a chain index in a service for tests, with placeholder + /// metadata/config. Lets unit tests exercise the service lifecycle over a + /// mock source (no real validator). Production builds go through + /// [`ZcashService::spawn`]. + pub(crate) fn new_for_test( + indexer: NodeBackedChainIndex, + network: zaino_common::Network, + ) -> Self { + let (data, config) = test_service_parts(network); + Self { + data, + config, + indexer, + } + } +} + #[cfg(test)] impl NodeBackedIndexerServiceSubscriber { /// Wraps a chain-index subscriber in a service subscriber for tests, with placeholder @@ -249,24 +306,10 @@ impl NodeBackedIndexerServiceSubscriber { indexer: NodeBackedChainIndexSubscriber, network: zaino_common::Network, ) -> Self { + let (data, config) = test_service_parts(network); Self { - data: ServiceMetadata::new( - get_build_info("test".to_string()), - network.to_zebra_network(), - "test-build".to_string(), - "test-subversion".to_string(), - ), - config: CommonBackendConfig::new( - "127.0.0.1:0".to_string(), - None, - None, - None, - zaino_common::ServiceConfig::default(), - zaino_common::StorageConfig::default(), - true, - network, - None, - ), + data, + config, indexer, } } From 61d11924ab2e8106896bcad2b38ced7ffb483339 Mon Sep 17 00:00:00 2001 From: zancas Date: Tue, 7 Jul 2026 22:42:30 -0700 Subject: [PATCH 112/134] fix(zaino-state): omit multi-address outputs from getblockdeltas again MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit assemble_block_deltas attributed each output's full value to addresses.first(), where the pre-merge state backend required exactly one derivable address. An output with multiple derivable addresses (e.g. bare multisig) has no single owning address: zcashd's getblockdeltas omits it, and crediting the first address fabricates balance for it — and makes the Direct and Rpc connections disagree, since the Rpc arm proxies the validator's response. Extract the output mapping into output_delta_from_verbose and restore the exactly-one-address rule there. The extraction landed first with the first-address behavior intact, so the new multi-address test failed against it and flipped green only with the rule restored; single-address attribution and nonstandard-script omission are pinned alongside. Addresses finding 6 of the PR #1361 review. Co-Authored-By: Claude Fable 5 --- .../chain_index/source/validator_connector.rs | 91 ++++++++++++++++--- 1 file changed, 78 insertions(+), 13 deletions(-) diff --git a/packages/zaino-state/src/chain_index/source/validator_connector.rs b/packages/zaino-state/src/chain_index/source/validator_connector.rs index 8974271cd..d7c6bb9c5 100644 --- a/packages/zaino-state/src/chain_index/source/validator_connector.rs +++ b/packages/zaino-state/src/chain_index/source/validator_connector.rs @@ -2242,6 +2242,29 @@ pub(crate) fn median_of_block_times(mut times: Vec) -> BlockchainSourceResu Ok(times[times.len() / 2]) } +/// Maps a verbose-transaction output to its `getblockdeltas` output delta, +/// or `None` when the output does not participate in address deltas: only +/// outputs with exactly one derivable address are attributed. Zero means no +/// address to credit (nonstandard script); more than one (e.g. bare multisig) +/// has no single owning address, and zcashd's getblockdeltas omits such +/// outputs rather than crediting the first address. +fn output_delta_from_verbose(vout: &zebra_rpc::client::Output) -> Option { + let address = + vout.script_pub_key() + .addresses() + .as_ref() + .and_then(|addresses| match addresses.as_slice() { + [address] => Some(address.clone()), + _ => None, + })?; + let satoshis: Amount = vout.value_zat().try_into().ok()?; + Some(OutputDelta { + address, + satoshis, + index: vout.n(), + }) +} + /// Assembles a `getblockdeltas` response from a verbosity-2 `getblock` object plus a /// cache of previous transactions (keyed by txid) needed to resolve each spend's address /// and value, its median time, and the running network. @@ -2318,19 +2341,7 @@ pub(crate) fn assemble_block_deltas( let outputs: Vec = txo .outputs() .iter() - .filter_map(|vout| { - let address = vout - .script_pub_key() - .addresses() - .as_ref() - .and_then(|addresses| addresses.first().cloned())?; - let satoshis: Amount = vout.value_zat().try_into().ok()?; - Some(OutputDelta { - address, - satoshis, - index: vout.n(), - }) - }) + .filter_map(output_delta_from_verbose) .collect(); deltas.push(BlockDelta { @@ -2456,6 +2467,60 @@ fn clamp_deltas_range_to_tip( Ok((start, end)) } +#[cfg(test)] +mod output_delta_from_verbose { + use super::*; + + fn output_with_addresses(addresses: Option>) -> zebra_rpc::client::Output { + zebra_rpc::client::Output::new( + 0.00000001, + 1, + 0, + zebra_rpc::client::ScriptPubKey::new( + "asm".to_string(), + zebra_chain::transparent::Script::new(&[0u8]), + addresses.as_ref().map(|a| a.len() as u32), + "pubkeyhash".to_string(), + addresses, + ), + ) + } + + /// An output with exactly one derivable address is attributed to it. + #[test] + fn single_address_output_is_attributed() { + let delta = + output_delta_from_verbose(&output_with_addresses(Some(vec!["t1address".to_string()]))) + .expect("a single-address output participates in deltas"); + assert_eq!(delta.address, "t1address"); + } + + /// An output with multiple derivable addresses (e.g. bare multisig) has no + /// single owning address: zcashd's getblockdeltas omits it, and crediting + /// the first address would fabricate balance for it. The pre-fix code took + /// `addresses.first()`. + #[test] + fn multi_address_output_is_omitted() { + assert_eq!( + output_delta_from_verbose(&output_with_addresses(Some(vec![ + "t1first".to_string(), + "t1second".to_string(), + ]))), + None, + "a multi-address output must be omitted, not attributed to the first address" + ); + } + + /// No derivable address (nonstandard script) means no delta. + #[test] + fn addressless_output_is_omitted() { + assert_eq!( + output_delta_from_verbose(&output_with_addresses(None)), + None + ); + } +} + #[cfg(test)] mod clamp_deltas_range_to_tip { use super::*; From d198d9dbcf13bd66dd6c16b671321e6b22d8f90b Mon Sep 17 00:00:00 2001 From: zancas Date: Tue, 7 Jul 2026 22:49:53 -0700 Subject: [PATCH 113/134] fix(zaino-state): proxy getchaintips to the validator while still syncing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both pre-merge backends answered getchaintips during the initial finalised-state build (no non-finalised snapshot yet — hours on mainnet) by proxying the validator's own JSON-RPC response. The merged service returned UnavailableNotSyncedEnough for that whole window instead. getbestblockhash and getblockcount are unaffected: they already reach the source through best_chaintip's syncing-window arm. Add a get_chain_tips node-passthrough to the BlockchainSource trait (both connector arms already hold the JSON-RPC client the old backends used) and extract the snapshot-or-fallback choice into chain_tips_for_snapshot, which the service method now delegates to. MockchainSource serves a single active tip at its active height, matching a validator with no side branches; the proptest mock stubs the method like its other passthroughs. The regression test drives chain_tips_for_snapshot with a StillSyncingFinalizedState snapshot over the mock and asserts the validator's tips come back; before the fix the fallback path did not exist and the only behavior was the UnavailableNotSyncedEnough error. Addresses finding 7 of the PR #1361 review. Co-Authored-By: Claude Fable 5 --- .../zaino-state/src/chain_index/source.rs | 9 +++++ .../chain_index/source/mockchain_source.rs | 28 +++++++++++++++ .../chain_index/source/validator_connector.rs | 10 ++++++ .../src/chain_index/tests/mockchain_tests.rs | 35 +++++++++++++++++++ .../chain_index/tests/proptest_blockgen.rs | 7 ++++ .../src/indexer/node_backed_indexer.rs | 26 ++++++++++---- 6 files changed, 109 insertions(+), 6 deletions(-) diff --git a/packages/zaino-state/src/chain_index/source.rs b/packages/zaino-state/src/chain_index/source.rs index 5080c367e..0a4af2e1c 100644 --- a/packages/zaino-state/src/chain_index/source.rs +++ b/packages/zaino-state/src/chain_index/source.rs @@ -169,6 +169,15 @@ pub trait BlockchainSource: Clone + Send + Sync + 'static { /// Returns the `getpeerinfo` response. fn get_peer_info(&self) -> impl SendFut>; + /// Returns the validator's `getchaintips` response. Serves as the + /// `getchaintips` fallback while the local index is still building its + /// finalised state and has no non-finalised snapshot to answer from. + fn get_chain_tips( + &self, + ) -> impl SendFut< + BlockchainSourceResult, + >; + /// Returns the `getblocksubsidy` response at the given height. fn get_block_subsidy( &self, diff --git a/packages/zaino-state/src/chain_index/source/mockchain_source.rs b/packages/zaino-state/src/chain_index/source/mockchain_source.rs index 1bec4de8e..cc41537b1 100644 --- a/packages/zaino-state/src/chain_index/source/mockchain_source.rs +++ b/packages/zaino-state/src/chain_index/source/mockchain_source.rs @@ -712,6 +712,34 @@ impl BlockchainSource for MockchainSource { unimplemented!("MockchainSource cannot serve get_peer_info until test vectors are extended") } + /// A single active tip at the mockchain's current active height, matching + /// what a validator with no side branches reports. + async fn get_chain_tips( + &self, + ) -> BlockchainSourceResult + { + if self + .force_requests_against_source_to_fail + .load(Ordering::SeqCst) + { + return Err(BlockchainSourceError::Unrecoverable( + "forced source failure".into(), + )); + } + let height = self.active_height(); + let Some(index) = self.valid_height(height) else { + return Ok(vec![]); + }; + Ok(vec![ + zaino_fetch::jsonrpsee::response::chain_tips::ChainTip::new( + height, + self.blocks[index].hash().to_string(), + 0, + zaino_fetch::jsonrpsee::response::chain_tips::ChainTipStatus::Active, + ), + ]) + } + async fn get_block_subsidy( &self, _height: u32, diff --git a/packages/zaino-state/src/chain_index/source/validator_connector.rs b/packages/zaino-state/src/chain_index/source/validator_connector.rs index d7c6bb9c5..94119dbde 100644 --- a/packages/zaino-state/src/chain_index/source/validator_connector.rs +++ b/packages/zaino-state/src/chain_index/source/validator_connector.rs @@ -452,6 +452,16 @@ impl BlockchainSource for ValidatorConnector { .map_err(BlockchainSourceError::unrecoverable) } + async fn get_chain_tips( + &self, + ) -> BlockchainSourceResult + { + self.json_rpc_connector() + .get_chain_tips() + .await + .map_err(BlockchainSourceError::unrecoverable) + } + async fn get_block_subsidy(&self, height: u32) -> BlockchainSourceResult { self.json_rpc_connector() .get_block_subsidy(height) diff --git a/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs b/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs index 6d0f0a611..d7ffa6e6f 100644 --- a/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs +++ b/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs @@ -1073,6 +1073,41 @@ async fn node_backed_indexer_service_serves_latest_block() { assert_eq!(latest.height, expected_tip as u64); } +/// During the initial finalised-state build there is no non-finalised +/// snapshot. Both pre-merge backends answered `getchaintips` in that window by +/// proxying the validator's own response; the merged service must fall back to +/// the source the same way, rather than serving UnavailableNotSyncedEnough for +/// the whole build (hours on mainnet). +#[tokio::test] +async fn get_chain_tips_falls_back_to_source_while_syncing() { + use crate::chain_index::non_finalised_state::ChainIndexSnapshot; + use crate::chain_index::tests::vectors::build_mockchain_source; + use crate::indexer::node_backed_indexer::chain_tips_for_snapshot; + + let blocks = load_test_vectors().unwrap().blocks; + let tip_height = (blocks.len() as u32) - 1; + let expected_tip_hash = blocks[tip_height as usize].zebra_block.hash().to_string(); + let mock = build_mockchain_source(blocks); + + let syncing_snapshot = ChainIndexSnapshot::StillSyncingFinalizedState { + validator_finalized_height: crate::Height(tip_height), + }; + + let tips = chain_tips_for_snapshot(&syncing_snapshot, &mock) + .await + .expect("the syncing window must proxy the source's chain tips"); + + assert_eq!( + tips, + vec![zaino_fetch::jsonrpsee::response::chain_tips::ChainTip::new( + tip_height, + expected_tip_hash, + 0, + zaino_fetch::jsonrpsee::response::chain_tips::ChainTipStatus::Active, + )] + ); +} + /// Dropping the service must not panic on a thread with no Tokio runtime. /// `Drop` runs the blocking teardown, which previously called /// `Handle::current()` unconditionally; outside a runtime that panics, and a diff --git a/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs b/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs index bdb5dc3b9..ac4255e64 100644 --- a/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs +++ b/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs @@ -1080,6 +1080,13 @@ impl BlockchainSource for ProptestMockchain { unimplemented!() } + async fn get_chain_tips( + &self, + ) -> BlockchainSourceResult + { + unimplemented!() + } + async fn get_block_subsidy( &self, _height: u32, diff --git a/packages/zaino-state/src/indexer/node_backed_indexer.rs b/packages/zaino-state/src/indexer/node_backed_indexer.rs index 9fac9ef83..913c14862 100644 --- a/packages/zaino-state/src/indexer/node_backed_indexer.rs +++ b/packages/zaino-state/src/indexer/node_backed_indexer.rs @@ -254,6 +254,25 @@ impl NodeBackedIndexerServiceSubscriber { } } +/// `getchaintips` served from the non-finalised snapshot when it exists, +/// falling back to the validator's own response during the initial +/// finalised-state build — matching both pre-merge backends, which proxied +/// the validator for that window instead of erroring. +pub(crate) async fn chain_tips_for_snapshot( + snapshot: &ChainIndexSnapshot, + source: &Source, +) -> Result { + match snapshot.get_nfs_snapshot() { + Some(non_finalized_snapshot) => Ok(chain_tips_from_nonfinalized_snapshot( + non_finalized_snapshot, + )), + None => Ok(source + .get_chain_tips() + .await + .map_err(crate::error::ChainIndexError::backing_validator)?), + } +} + /// Placeholder metadata and config for test-only service construction. #[cfg(test)] fn test_service_parts(network: zaino_common::Network) -> (ServiceMetadata, CommonBackendConfig) { @@ -619,12 +638,7 @@ impl ZcashIndexer for NodeBackedIndexerServiceSubscrib #[allow(deprecated)] async fn get_chain_tips(&self) -> Result { let snapshot = self.indexer.snapshot_nonfinalized_state().await?; - let Some(non_finalized_snapshot) = snapshot.get_nfs_snapshot() else { - return Err(NodeBackedIndexerServiceError::UnavailableNotSyncedEnough); - }; - Ok(chain_tips_from_nonfinalized_snapshot( - non_finalized_snapshot, - )) + chain_tips_for_snapshot(&snapshot, self.indexer.source()).await } /// Return information about the given Zcash address. From 0d62a5af90c75b6fcbb70b6a4b002fbc832062e2 Mon Sep 17 00:00:00 2001 From: zancas Date: Tue, 7 Jul 2026 22:52:48 -0700 Subject: [PATCH 114/134] fix(zaino-state): resolve tip-relative getblock heights, keep the -8 code z_get_block pre-parsed its identifier with HashOrHeight::from_str, which rejects the negative tip-relative heights zebra's own getblock accepts ("-1" is the tip), and flattened the rejection into an internal-error string. The old Rpc backend forwarded the raw string to the validator, so negative heights worked there; the merge regressed them, and unparsable identifiers lost zcashd's legacy InvalidParameter code. Parse with HashOrHeight::new against the best chaintip instead, matching zebra's semantics on both connection types, and attach the rejection as a typed RpcError (legacy code -8) source via ChainIndexError::internal_from so the serve layer's downcast walk can recover it. The regression tests drive z_get_block over a MockchainSource: "-1" must serve the same block as the explicit tip height, and "notablockid" must carry the -8 RpcError in its source() chain. Both failed before the fix. Addresses finding 8 of the PR #1361 review. Co-Authored-By: Claude Fable 5 --- packages/zaino-state/src/chain_index.rs | 17 +++++- .../src/chain_index/tests/mockchain_tests.rs | 55 +++++++++++++++++++ 2 files changed, 70 insertions(+), 2 deletions(-) diff --git a/packages/zaino-state/src/chain_index.rs b/packages/zaino-state/src/chain_index.rs index 47d3a9463..88b5a5c78 100644 --- a/packages/zaino-state/src/chain_index.rs +++ b/packages/zaino-state/src/chain_index.rs @@ -2762,8 +2762,21 @@ impl ChainIndexRpcExt for NodeBackedChainIndexSubscrib hash_or_height: String, verbosity: Option, ) -> Result { - let id = HashOrHeight::from_str(&hash_or_height) - .map_err(|error| ChainIndexError::internal(error.to_string()))?; + // Resolve tip-relative negative heights against the best chaintip, + // matching zebra's own `getblock` semantics (`-1` is the tip). A + // rejected identifier carries zcashd's legacy InvalidParameter code + // as a typed `RpcError` source, which the serve layer recovers by + // downcast-walking the error chain. + let snapshot = self.snapshot_nonfinalized_state().await?; + let tip = self.best_chaintip(&snapshot).await?; + let id = HashOrHeight::new(&hash_or_height, Some(tip.height.into())).map_err(|error| { + ChainIndexError::internal_from( + zaino_fetch::jsonrpsee::connector::RpcError::new_from_legacycode( + zebra_rpc::server::error::LegacyCode::InvalidParameter, + error, + ), + ) + })?; self.source() .get_block_verbose(id, verbosity) .await diff --git a/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs b/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs index d7ffa6e6f..7adc31b7b 100644 --- a/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs +++ b/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs @@ -1073,6 +1073,61 @@ async fn node_backed_indexer_service_serves_latest_block() { assert_eq!(latest.height, expected_tip as u64); } +/// zebra's `getblock` resolves negative heights against the tip (`-1` is the +/// tip block). The old Rpc backend forwarded the raw identifier string to the +/// validator, so `getblock "-1"` worked; the merged pre-parse used +/// `HashOrHeight::from_str`, which rejects negative heights. +#[tokio::test(flavor = "multi_thread")] +async fn z_get_block_resolves_negative_heights_against_the_tip() { + let (blocks, _indexer, index_reader, _mockchain) = + load_test_vectors_and_sync_chain_index(MockchainMode::Static).await; + let tip_height = (blocks.len() as u32) - 1; + wait_for_indexer_tip(&index_reader, tip_height).await; + + let by_negative_height = index_reader + .z_get_block("-1".to_string(), Some(1)) + .await + .expect("height -1 must resolve to the tip block"); + let by_tip_height = index_reader + .z_get_block(tip_height.to_string(), Some(1)) + .await + .expect("the tip height resolves"); + assert_eq!(by_negative_height, by_tip_height); +} + +/// An unparsable `getblock` identifier must carry zcashd's legacy +/// InvalidParameter code (-8) as a typed `RpcError` in the `source()` chain, +/// not be flattened into an internal-error string: the serve layer recovers +/// legacy codes by downcast-walking the chain. +#[tokio::test(flavor = "multi_thread")] +async fn z_get_block_invalid_identifier_keeps_legacy_error_code() { + let (blocks, _indexer, index_reader, _mockchain) = + load_test_vectors_and_sync_chain_index(MockchainMode::Static).await; + wait_for_indexer_tip(&index_reader, (blocks.len() as u32) - 1).await; + + let error = index_reader + .z_get_block("notablockid".to_string(), Some(1)) + .await + .expect_err("an unparsable identifier must be rejected"); + + let mut current: Option<&(dyn std::error::Error + 'static)> = Some(&error); + let mut rpc_error_code = None; + while let Some(source_error) = current { + if let Some(rpc_error) = + source_error.downcast_ref::() + { + rpc_error_code = Some(rpc_error.code); + break; + } + current = source_error.source(); + } + assert_eq!( + rpc_error_code, + Some(zebra_rpc::server::error::LegacyCode::InvalidParameter as i64), + "the typed RpcError (legacy code -8) must stay reachable via the source() chain" + ); +} + /// During the initial finalised-state build there is no non-finalised /// snapshot. Both pre-merge backends answered `getchaintips` in that window by /// proxying the validator's own response; the merged service must fall back to From 493032b5761ac85183b79adb343a53f953f24631 Mon Sep 17 00:00:00 2001 From: zancas Date: Tue, 7 Jul 2026 22:56:15 -0700 Subject: [PATCH 115/134] fix(zaino-state): never let the source syncer task outlive its chain index NodeBackedChainIndex::shutdown() released source-owned background work (the Direct connection's Zebra syncer task) only after the fallible finalised-DB shutdown, so a DB shutdown error skipped the release via the `?`; and Drop only cancelled the sync worker, never releasing the source at all. Either way the syncer task could keep running against a torn-down index. Route both paths through shutdown_sync_best_effort (introduced with the teardown-safety fix): shutdown() runs the synchronous teardown before the fallible DB step, and Drop runs it in place of the bare cancel. The cancel-before-DB ordering the shutdown doc calls out is preserved. The regression test gives MockchainSource a shutdown-recording flag, drops the index without calling shutdown(), and asserts the source was released; before the fix Drop never invoked the release. Addresses finding 9 of the PR #1361 review. Co-Authored-By: Claude Fable 5 --- packages/zaino-state/src/chain_index.rs | 19 +++++++++-------- .../chain_index/source/mockchain_source.rs | 16 ++++++++++++++ .../src/chain_index/tests/mockchain_tests.rs | 21 +++++++++++++++++++ 3 files changed, 47 insertions(+), 9 deletions(-) diff --git a/packages/zaino-state/src/chain_index.rs b/packages/zaino-state/src/chain_index.rs index 88b5a5c78..23e5dd710 100644 --- a/packages/zaino-state/src/chain_index.rs +++ b/packages/zaino-state/src/chain_index.rs @@ -976,14 +976,11 @@ impl NodeBackedChainIndex { /// the design we have. Cancelling first just removes the wasted /// failure-path round trip. pub async fn shutdown(&self) -> Result<(), FinalisedStateError> { - self.cancel_token.cancel(); - self.status.store(StatusType::Closing); - self.finalized_db.shutdown().await?; - self.mempool.close(); - // Release any source-owned background work (e.g. the Zebra chain-syncer - // task behind the `State` connector). No-op for poll-only sources. - self.source.shutdown(); - Ok(()) + // The synchronous teardown (cancellation, mempool close, source + // release) runs before the fallible DB shutdown so a DB error cannot + // skip it — the source's Zebra syncer task must not outlive the index. + self.shutdown_sync_best_effort(); + self.finalized_db.shutdown().await } /// Synchronous best-effort teardown for contexts that cannot run async @@ -1222,7 +1219,11 @@ impl Drop for NodeBackedChainIndex { /// on `cancel_token.cancelled()` wrapping the iter body), the worker /// exits at its next await checkpoint instead. fn drop(&mut self) { - self.cancel_token.cancel(); + // The full synchronous teardown, not just the cancellation: the + // source release in particular must run here, or the `State` + // connector's Zebra syncer task outlives an index that is dropped + // without an explicit `shutdown()` call. + self.shutdown_sync_best_effort(); } } diff --git a/packages/zaino-state/src/chain_index/source/mockchain_source.rs b/packages/zaino-state/src/chain_index/source/mockchain_source.rs index cc41537b1..68cf04afe 100644 --- a/packages/zaino-state/src/chain_index/source/mockchain_source.rs +++ b/packages/zaino-state/src/chain_index/source/mockchain_source.rs @@ -180,6 +180,9 @@ pub(crate) struct MockchainSource { /// neither know nor care how many `mine_blocks` events occurred /// between wakes. blocks_received_broadcaster: tokio::sync::watch::Sender<()>, + /// Records whether [`BlockchainSource::shutdown`] ran, so teardown tests + /// can assert the index releases its source. Shared across clones. + shutdown_called: Arc, } impl MockchainSource { @@ -246,9 +249,16 @@ impl MockchainSource { )), get_block_hook: Arc::new(Mutex::new(None)), blocks_received_broadcaster: tokio::sync::watch::channel(()).0, + shutdown_called: Arc::new(std::sync::atomic::AtomicBool::new(false)), } } + /// Whether [`BlockchainSource::shutdown`] has run on this source (or any + /// clone of it). + pub(crate) fn shutdown_called(&self) -> bool { + self.shutdown_called.load(Ordering::SeqCst) + } + /// When set to true, `get_best_block_height` and `get_best_block_hash` /// return `BlockchainSourceError::Unrecoverable`. pub(crate) fn set_failing(&self, fail: bool) { @@ -712,6 +722,12 @@ impl BlockchainSource for MockchainSource { unimplemented!("MockchainSource cannot serve get_peer_info until test vectors are extended") } + /// Records the release so teardown tests can assert the index shuts its + /// source down (the mock owns no real background work). + fn shutdown(&self) { + self.shutdown_called.store(true, Ordering::SeqCst); + } + /// A single active tip at the mockchain's current active height, matching /// what a validator with no side branches reports. async fn get_chain_tips( diff --git a/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs b/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs index 7adc31b7b..6c391552a 100644 --- a/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs +++ b/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs @@ -1073,6 +1073,27 @@ async fn node_backed_indexer_service_serves_latest_block() { assert_eq!(latest.height, expected_tip as u64); } +/// Dropping the chain index without an explicit `shutdown()` call must still +/// release source-owned background work: `Drop` previously only cancelled the +/// sync worker, and the async `shutdown()`'s `?` on the DB teardown skipped +/// the source release when the DB shutdown errored — either way the Direct +/// connection's Zebra syncer task could outlive its index. +#[tokio::test(flavor = "multi_thread")] +async fn dropping_the_chain_index_releases_the_source() { + let (_blocks, indexer, _index_reader, mockchain) = + load_test_vectors_and_sync_chain_index(MockchainMode::Static).await; + + assert!( + !mockchain.shutdown_called(), + "the source must not be shut down while the index is live" + ); + drop(indexer); + assert!( + mockchain.shutdown_called(), + "dropping the index must release source-owned background work" + ); +} + /// zebra's `getblock` resolves negative heights against the tip (`-1` is the /// tip block). The old Rpc backend forwarded the raw identifier string to the /// validator, so `getblock "-1"` worked; the merged pre-parse used From 06fd01fa4ed1c2864c7a93bdb66c04fab4130c3e Mon Sep 17 00:00:00 2001 From: zancas Date: Tue, 7 Jul 2026 23:00:36 -0700 Subject: [PATCH 116/134] test(zaino-state): pin the getblockheader raw-to-wire serde round-trip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Direct connection's non-verbose getblockheader builds a GetBlockHeaderResponse::Raw and crosses to zaino-fetch's untagged GetBlockHeader through a serde JSON round-trip with no test at the conversion's definition: a shape regression on either side would break every non-verbose getblockheader on a Direct connection at runtime and only surface in live tests. The conversion is currently correct — the mockchain get_block_header test exercises it end to end — so this is a coverage pin rather than a fix. The test asserts Raw hex lands in the Compact variant with identical bytes; a mutation check (nulling the intermediate serde value) confirmed it fails against a broken conversion. Addresses finding 10 of the PR #1361 review. Co-Authored-By: Claude Fable 5 --- .../chain_index/source/validator_connector.rs | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/packages/zaino-state/src/chain_index/source/validator_connector.rs b/packages/zaino-state/src/chain_index/source/validator_connector.rs index 94119dbde..551fd30ce 100644 --- a/packages/zaino-state/src/chain_index/source/validator_connector.rs +++ b/packages/zaino-state/src/chain_index/source/validator_connector.rs @@ -2415,6 +2415,29 @@ pub(crate) fn assemble_block_deltas( }) } +#[cfg(test)] +mod zebra_block_header_to_wire { + use super::*; + + /// The Direct connection's non-verbose `getblockheader` builds a + /// `GetBlockHeaderResponse::Raw` and crosses to the wire type through the + /// serde round-trip under test: the raw hex must land in the untagged + /// `Compact` variant with the same bytes, or every non-verbose + /// `getblockheader` on a Direct connection errors at runtime. The + /// mockchain `get_block_header` test covers the same conversion end to + /// end; this pins it at its definition. + #[test] + fn raw_header_round_trips_to_compact_hex() { + let header_bytes = vec![0x01, 0x02, 0xab, 0xcd]; + let raw = GetBlockHeaderResponse::Raw(HexData(header_bytes.clone())); + + let wire = zebra_block_header_to_wire(raw) + .expect("the raw header shape must convert to the wire type"); + + assert_eq!(wire, GetBlockHeader::Compact(hex::encode(header_bytes))); + } +} + #[cfg(test)] mod fetch_pool_treestate_slot { /// Regression test: the validator's finalRoot must pass through to the pool slot. From 064b1775257ec07795f806aee24a6e191c3f26b6 Mon Sep 17 00:00:00 2001 From: Mark Henderson Date: Thu, 9 Jul 2026 09:14:15 +0200 Subject: [PATCH 117/134] fix: wait for the validator's first block in the connector spawn paths A freshly started validator answers JSON-RPC before it has fetched and committed its first block: zebra serves getblockchaininfo (reporting a genesis-hash placeholder on an empty state) and an empty getrawmempool while getbestblockhash returns "No blocks in state" until genesis arrives from the network, which can take minutes of peer discovery on slow networks. Starting zainod in that window fails spawn on both connector paths and the process exit-loops under any supervisor: - spawn_fetch builds the connector without checking for a servable tip, so NodeBackedChainIndex::new fails (Mempool::spawn returns Critical when get_best_block_hash errors). - spawn_state's sync-wait loop gets None from expected_read_response!(_, Tip) on an empty state and fails with "no blocks in chain". Poll get_best_blockhash in spawn_fetch until it succeeds before returning the connector, and treat the missing syncer tip in spawn_state as wait-and-retry; both log progress every 3 seconds. The validator is reachable and version-checked at this point, so waiting matches the RPC-liveness standard operators use to sequence zainod after the validator. Co-Authored-By: Claude Fable 5 --- .../chain_index/source/validator_connector.rs | 30 ++++++++++++++++--- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/packages/zaino-state/src/chain_index/source/validator_connector.rs b/packages/zaino-state/src/chain_index/source/validator_connector.rs index 551fd30ce..b0a912ed5 100644 --- a/packages/zaino-state/src/chain_index/source/validator_connector.rs +++ b/packages/zaino-state/src/chain_index/source/validator_connector.rs @@ -122,6 +122,19 @@ impl ValidatorConnector { .await .map_err(BlockchainSourceError::unrecoverable)?; + // A freshly started validator answers RPC before it has fetched and + // committed its first block: zebra serves getblockchaininfo and an + // empty getrawmempool while getbestblockhash returns "No blocks in + // state" until genesis arrives, which can take minutes of peer + // discovery. Everything downstream assumes a servable tip + // (Mempool::spawn returns Critical on get_best_block_hash and the + // ChainIndex sync loop escalates to CriticalError), so wait here + // instead of failing spawn and exit-looping the whole process. + while let Err(e) = fetcher.get_best_blockhash().await { + info!(%e, "Waiting for validator to serve its first block"); + tokio::time::sleep(std::time::Duration::from_secs(3)).await; + } + Ok((ValidatorConnector::Fetch(fetcher), info)) } @@ -188,10 +201,19 @@ impl ValidatorConnector { .and_then(|service| service.call(ReadRequest::Tip)) .await .map_err(BlockchainSourceError::unrecoverable)?; - let (syncer_height, syncer_tip_hash) = expected_read_response!(syncer_response, Tip) - .ok_or_else(|| { - BlockchainSourceError::Unrecoverable("no blocks in chain".to_string()) - })?; + // A freshly started validator answers RPC before it has committed + // its first block (getblockchaininfo reports a genesis-hash + // placeholder on an empty state), and the syncer has no tip until + // genesis arrives, which can take minutes of peer discovery. + // Everything below assumes a servable tip, so wait here instead + // of failing spawn and exit-looping the whole process. + let Some((syncer_height, syncer_tip_hash)) = + expected_read_response!(syncer_response, Tip) + else { + info!("Waiting for validator to serve its first block"); + tokio::time::sleep(std::time::Duration::from_secs(3)).await; + continue; + }; if server_height == syncer_height && server_tip_hash == syncer_tip_hash { info!( From 86f9303288979889cd6afff7760cf44473ff3e0d Mon Sep 17 00:00:00 2001 From: zancas Date: Thu, 9 Jul 2026 12:16:18 -0700 Subject: [PATCH 118/134] build: bump zcash_local_net to the tip of infrastructure PR #278 The new rev (85502cd0) picks up the twelve commits on the bump_to_NU6.3 branch since the previous pin, including deterministic listener-port allocation with an Indexer-convergence barrier, the generic Wallet abstraction, and the front proxies published as the canonical endpoints. Co-Authored-By: Claude Fable 5 --- Cargo.lock | 6 +++--- Cargo.toml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3d0124b57..08f1ca280 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6184,7 +6184,7 @@ dependencies = [ [[package]] name = "zcash_local_net" version = "0.7.0" -source = "git+https://github.com/zingolabs/infrastructure.git?rev=78511f7304ba27895967be860016ede923acec08#78511f7304ba27895967be860016ede923acec08" +source = "git+https://github.com/zingolabs/infrastructure.git?rev=85502cd0b5faf9308e590865dae51bf31fdd342e#85502cd0b5faf9308e590865dae51bf31fdd342e" dependencies = [ "hex", "reqwest 0.12.28", @@ -6735,12 +6735,12 @@ dependencies = [ [[package]] name = "zingo-consensus" version = "0.1.0" -source = "git+https://github.com/zingolabs/infrastructure.git?rev=78511f7304ba27895967be860016ede923acec08#78511f7304ba27895967be860016ede923acec08" +source = "git+https://github.com/zingolabs/infrastructure.git?rev=85502cd0b5faf9308e590865dae51bf31fdd342e#85502cd0b5faf9308e590865dae51bf31fdd342e" [[package]] name = "zingo_test_vectors" version = "0.0.1" -source = "git+https://github.com/zingolabs/infrastructure.git?rev=78511f7304ba27895967be860016ede923acec08#78511f7304ba27895967be860016ede923acec08" +source = "git+https://github.com/zingolabs/infrastructure.git?rev=85502cd0b5faf9308e590865dae51bf31fdd342e#85502cd0b5faf9308e590865dae51bf31fdd342e" [[package]] name = "zip32" diff --git a/Cargo.toml b/Cargo.toml index bacac4ad1..8578ba5ff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -143,7 +143,7 @@ zaino-testutils = { path = "live-tests/zaino-testutils" } # truth (infrastructure ADR 0003, PR #278): wallet clients derive their # heights via `WalletNetwork::from_validator`, and `ZainodConfig` carries a # payload-free network kind. Restore a release tag once one is cut upstream. -zcash_local_net = { git = "https://github.com/zingolabs/infrastructure.git", rev = "78511f7304ba27895967be860016ede923acec08" } +zcash_local_net = { git = "https://github.com/zingolabs/infrastructure.git", rev = "85502cd0b5faf9308e590865dae51bf31fdd342e" } wire_serialized_transaction_test_data = "1.0.0" config = { version = "0.15", default-features = false, features = ["toml"] } From 4285738fa7ee65150ab7d9952dbf8a9a932675c4 Mon Sep 17 00:00:00 2001 From: zancas Date: Thu, 9 Jul 2026 12:56:50 -0700 Subject: [PATCH 119/134] build: record zaino-common's rustls dependency in the lockfile The dev merge took the incoming Cargo.lock, which predates this branch keeping rustls in zaino-common (ADR-0006); cargo regenerates the entry on the first build. Co-Authored-By: Claude Fable 5 --- Cargo.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.lock b/Cargo.lock index fe728fec8..8398f82f9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5941,6 +5941,7 @@ dependencies = [ name = "zaino-common" version = "0.2.0" dependencies = [ + "rustls", "serde", "time", "tracing", From 5bc718057de458853721f02cd26e7aa52fb06ded Mon Sep 17 00:00:00 2001 From: zancas Date: Thu, 9 Jul 2026 13:03:08 -0700 Subject: [PATCH 120/134] refactor: derive the adoption plumbing from the mirror macro MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The NetworkUpgrade-variant ↔ ActivationHeights-field correspondence is recorded exactly once, in zaino-common's activation_heights_mirror! macro, but activation_heights_from_upgrades restated it as a hand-written thirteen-arm match and both it and its tests restated the all-None schedule as field-by-field literals. The macro now generates the two missing derived items — the NEVER_ACTIVATED schedule and the per-upgrade slot_mut accessor — and the adoption function collapses onto them. The generated match stays exhaustive, so a new zebra upgrade variant still fails the build until its pair is added to the mirror. Also remove the dead ChainIndexConfig::new: it had no callers and was the one constructor shape that invited supplying a network from somewhere other than validator adoption; from_backend_config is now the sole named production constructor (zaino#1076). Co-Authored-By: Claude Fable 5 --- packages/zaino-common/src/config/network.rs | 27 ++++++++++- .../chain_index/source/validator_connector.rs | 48 ++----------------- packages/zaino-state/src/config.rs | 16 ------- 3 files changed, 31 insertions(+), 60 deletions(-) diff --git a/packages/zaino-common/src/config/network.rs b/packages/zaino-common/src/config/network.rs index 5983e5580..a50cb31e0 100644 --- a/packages/zaino-common/src/config/network.rs +++ b/packages/zaino-common/src/config/network.rs @@ -98,7 +98,9 @@ impl Default for ActivationHeights { /// Records the `NetworkUpgrade`-variant ↔ `ActivationHeights`-field correspondence /// exactly once, generating everything derived from it: the two field-by-field /// `From` conversions between [`ActivationHeights`] and zebra's -/// [`ConfiguredActivationHeights`] (the structs share field names). +/// [`ConfiguredActivationHeights`] (the structs share field names), the +/// all-`None` [`ActivationHeights::NEVER_ACTIVATED`] schedule, and the +/// per-upgrade [`ActivationHeights::slot_mut`] accessor. /// /// A declarative macro rather than functions because plain `fn`s cannot abstract /// over struct fields, and the variant/field spellings (`Nu5`/`nu5`) differ only @@ -124,6 +126,29 @@ macro_rules! activation_heights_mirror { Self { $($field),* } } } + + impl ActivationHeights { + /// The all-`None` schedule: every upgrade never-activated. The + /// starting point for building heights from an external report, + /// where an upgrade absent from the report must stay unset. + pub const NEVER_ACTIVATED: Self = Self { $($field: None),* }; + + /// Mutable slot for `upgrade`'s activation height, or `None` for + /// `Genesis` (height 0 by definition; it has no slot). + pub fn slot_mut( + &mut self, + upgrade: zebra_chain::parameters::NetworkUpgrade, + ) -> Option<&mut Option> { + match upgrade { + zebra_chain::parameters::NetworkUpgrade::Genesis => None, + $( + zebra_chain::parameters::NetworkUpgrade::$variant => { + Some(&mut self.$field) + } + )* + } + } + } }; } diff --git a/packages/zaino-state/src/chain_index/source/validator_connector.rs b/packages/zaino-state/src/chain_index/source/validator_connector.rs index 919dc9c6f..1aff0432f 100644 --- a/packages/zaino-state/src/chain_index/source/validator_connector.rs +++ b/packages/zaino-state/src/chain_index/source/validator_connector.rs @@ -313,37 +313,12 @@ async fn adopt_network( fn activation_heights_from_upgrades( upgrades: &IndexMap, ) -> Result { - let mut heights = zaino_common::config::network::ActivationHeights { - before_overwinter: None, - overwinter: None, - sapling: None, - blossom: None, - heartwood: None, - canopy: None, - nu5: None, - nu6: None, - nu6_1: None, - nu6_2: None, - nu6_3: None, - nu7: None, - }; + let mut heights = zaino_common::config::network::ActivationHeights::NEVER_ACTIVATED; for upgrade_info in upgrades.values() { let (upgrade, height, _status) = upgrade_info.into_parts(); - let slot = match upgrade { - // Genesis is height 0 by definition; it has no configuration slot. - NetworkUpgrade::Genesis => continue, - NetworkUpgrade::BeforeOverwinter => &mut heights.before_overwinter, - NetworkUpgrade::Overwinter => &mut heights.overwinter, - NetworkUpgrade::Sapling => &mut heights.sapling, - NetworkUpgrade::Blossom => &mut heights.blossom, - NetworkUpgrade::Heartwood => &mut heights.heartwood, - NetworkUpgrade::Canopy => &mut heights.canopy, - NetworkUpgrade::Nu5 => &mut heights.nu5, - NetworkUpgrade::Nu6 => &mut heights.nu6, - NetworkUpgrade::Nu6_1 => &mut heights.nu6_1, - NetworkUpgrade::Nu6_2 => &mut heights.nu6_2, - NetworkUpgrade::Nu6_3 => &mut heights.nu6_3, - NetworkUpgrade::Nu7 => &mut heights.nu7, + // Genesis is height 0 by definition; it has no configuration slot. + let Some(slot) = heights.slot_mut(upgrade) else { + continue; }; if slot.replace(height.0).is_some() { return Err(format!("validator reported {upgrade:?} twice")); @@ -2792,20 +2767,7 @@ mod activation_heights_from_upgrades { /// All-`None` heights: the starting point adoption fills from the /// validator's map, and the expected value for every absent upgrade. - const NEVER_ACTIVATED: ActivationHeights = ActivationHeights { - before_overwinter: None, - overwinter: None, - sapling: None, - blossom: None, - heartwood: None, - canopy: None, - nu5: None, - nu6: None, - nu6_1: None, - nu6_2: None, - nu6_3: None, - nu7: None, - }; + const NEVER_ACTIVATED: ActivationHeights = ActivationHeights::NEVER_ACTIVATED; fn upgrades_map( json: &str, diff --git a/packages/zaino-state/src/config.rs b/packages/zaino-state/src/config.rs index 668f0dd82..3cfff0d76 100644 --- a/packages/zaino-state/src/config.rs +++ b/packages/zaino-state/src/config.rs @@ -251,22 +251,6 @@ pub struct ChainIndexConfig { } impl ChainIndexConfig { - /// Returns a new instance of [`ChainIndexConfig`]. - #[allow(dead_code)] - pub fn new( - storage: StorageConfig, - db_version: u32, - network: zebra_chain::parameters::Network, - ephemeral: bool, - ) -> Self { - ChainIndexConfig { - storage, - db_version, - network, - ephemeral, - } - } - /// Builds the chain-index config from the backend config plus the /// runtime network. The backend config carries only a network kind, so /// the adopted runtime network arrives as its own argument — there is From ef3ed37a0a95780c4a76248a5b4ac77bdb5944ff Mon Sep 17 00:00:00 2001 From: zancas Date: Thu, 9 Jul 2026 13:46:53 -0700 Subject: [PATCH 121/134] feat: verify the public-network schedule against the validator at spawn MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Public-network activation heights come only from checked-in, ecosystem-shared consensus code; regtest parameters are configured on the validator at scenario launch and every other component adopts them from it. adopt_network previously enforced the regtest half but took the compiled Mainnet/Testnet parameters on faith: a validator running a *configured* testnet (zebra supports custom testnet activation heights) would silently drift from the index — the exact bug class this branch exists to kill, on a different network kind. adopt_network now fetches getblockchaininfo on every network kind and, on Mainnet and Testnet, verifies each reported (upgrade, height) pair against the compiled parameters before trusting them, failing loud on the first disagreement. Only reported entries are checked, not report completeness: the upgrades map is branch-ID-keyed, so unconfigured upgrades are simply absent and an older validator may not know the newest ones. Also document two acceptance decisions the spec made but the code did not state: adoption happens exactly once at spawn (a validator restarted mid-session with a different schedule is out of scope), and before_overwinter is always None when adopted (BeforeOverwinter has no consensus branch ID, so it can never appear in the report; zebra's new_regtest supplies its handling). Co-Authored-By: Claude Fable 5 --- .../chain_index/source/validator_connector.rs | 148 ++++++++++++++++-- 1 file changed, 133 insertions(+), 15 deletions(-) diff --git a/packages/zaino-state/src/chain_index/source/validator_connector.rs b/packages/zaino-state/src/chain_index/source/validator_connector.rs index 1aff0432f..6ad37eb65 100644 --- a/packages/zaino-state/src/chain_index/source/validator_connector.rs +++ b/packages/zaino-state/src/chain_index/source/validator_connector.rs @@ -270,24 +270,47 @@ impl ValidatorConnector { /// /// The validator is the single source of truth for activation heights /// (zaino#1076): the config carries only a network kind, and the runtime -/// network is constructed here before anything consumes one — from zebra's -/// compiled parameters for the public networks, from the validator's -/// reported schedule for regtest. There is no fallback: a silently wrong -/// schedule is the failure mode this removes. +/// network is constructed here before anything consumes one. On regtest the +/// schedule is adopted wholesale from the validator's report. On Mainnet and +/// Testnet the compiled zebra parameters are used, but the validator's +/// reported schedule is verified against them first, so a validator running +/// a *configured* testnet (custom activation heights) fails loud here +/// instead of silently drifting from the index. There is no fallback: a +/// silently wrong schedule is the failure mode this removes. +/// +/// Adoption happens exactly once, at spawn. A validator restarted +/// mid-session with a different schedule is out of scope (the test harness +/// restarts validator and indexer together); it would invalidate the index +/// without being re-detected here. async fn adopt_network( common: &CommonBackendConfig, rpc_client: &JsonRpSeeConnector, ) -> Result { - Ok(match common.network { - zaino_common::Network::Mainnet => zebra_chain::parameters::Network::Mainnet, - zaino_common::Network::Testnet => zebra_chain::parameters::Network::new_default_testnet(), + let blockchain_info = rpc_client.get_blockchain_info().await.map_err(|error| { + BlockchainSourceError::Unrecoverable(format!( + "cannot fetch activation heights from the validator at {}: {error}", + common.validator_rpc_address + )) + })?; + + // Shared by the Mainnet/Testnet arms: the compiled network is only + // trusted after the validator's report agrees with it. + let verified = |network: zebra_chain::parameters::Network| { + verify_reported_upgrades(&network, &blockchain_info.upgrades).map_err(|reason| { + BlockchainSourceError::Unrecoverable(format!( + "the validator at {} disagrees with the compiled {network} parameters: {reason}", + common.validator_rpc_address + )) + })?; + Ok(network) + }; + + match common.network { + zaino_common::Network::Mainnet => verified(zebra_chain::parameters::Network::Mainnet), + zaino_common::Network::Testnet => { + verified(zebra_chain::parameters::Network::new_default_testnet()) + } zaino_common::Network::Regtest => { - let blockchain_info = rpc_client.get_blockchain_info().await.map_err(|error| { - BlockchainSourceError::Unrecoverable(format!( - "cannot fetch activation heights from the validator at {}: {error}", - common.validator_rpc_address - )) - })?; let heights = activation_heights_from_upgrades(&blockchain_info.upgrades).map_err(|reason| { BlockchainSourceError::Unrecoverable(format!( @@ -296,9 +319,35 @@ async fn adopt_network( )) })?; info!(?heights, "Adopted activation heights from the validator"); - heights.to_regtest_network() + Ok(heights.to_regtest_network()) } - }) + } +} + +/// Checks every `(upgrade, height)` the validator reports against `network`'s +/// compiled activation schedule, rejecting the first disagreement. +/// +/// Only reported entries are checked — the report's *completeness* is not: +/// the map is keyed by consensus branch ID, so upgrades zebra considers +/// unconfigured are simply absent, and an older validator may not know the +/// newest upgrades yet. +fn verify_reported_upgrades( + network: &zebra_chain::parameters::Network, + upgrades: &IndexMap, +) -> Result<(), String> { + for upgrade_info in upgrades.values() { + let (upgrade, height, _status) = upgrade_info.into_parts(); + let compiled = upgrade.activation_height(network); + if compiled != Some(height) { + return Err(format!( + "the validator reports {upgrade:?} activating at height {}, \ + but the compiled parameters say {:?}", + height.0, + compiled.map(|compiled_height| compiled_height.0) + )); + } + } + Ok(()) } /// Builds the regtest activation heights from the validator's reported @@ -310,6 +359,12 @@ async fn adopt_network( /// `Network` (zaino#1076). An upgrade absent from the validator's map is /// never-activated — nothing is backfilled from defaults. Mainnet and /// Testnet use zebra's compiled parameters and never take this path. +/// +/// `before_overwinter` is always `None` here: the map is keyed by consensus +/// branch ID, which `BeforeOverwinter` does not have, so it can never appear +/// in the report. Correctness relies on zebra's `new_regtest` supplying its +/// own `BeforeOverwinter` handling when `to_regtest_network` builds the +/// runtime network from these heights. fn activation_heights_from_upgrades( upgrades: &IndexMap, ) -> Result { @@ -2865,3 +2920,66 @@ mod activation_heights_from_upgrades { ); } } + +#[cfg(test)] +mod verify_reported_upgrades { + fn upgrades_map( + json: &str, + ) -> indexmap::IndexMap< + zebra_rpc::methods::ConsensusBranchIdHex, + zebra_rpc::methods::NetworkUpgradeInfo, + > { + serde_json::from_str(json).expect("upgrades fixture parses") + } + + /// A report agreeing with the compiled schedule passes. + #[test] + fn accepts_a_report_matching_the_compiled_schedule() { + let upgrades = upgrades_map( + r#"{ + "76b809bb": { "name": "Sapling", "activationheight": 419200, "status": "active" } + }"#, + ); + + super::verify_reported_upgrades(&zebra_chain::parameters::Network::Mainnet, &upgrades) + .expect("mainnet Sapling at 419200 matches the compiled parameters"); + } + + /// A reported height disagreeing with the compiled schedule fails loud — + /// the configured-testnet / wrong-network drift class. + #[test] + fn rejects_a_mismatched_height() { + let upgrades = upgrades_map( + r#"{ + "76b809bb": { "name": "Sapling", "activationheight": 419201, "status": "active" } + }"#, + ); + + let reason = + super::verify_reported_upgrades(&zebra_chain::parameters::Network::Mainnet, &upgrades) + .expect_err("a wrong Sapling height must be rejected"); + assert!( + reason.contains("419201"), + "error should name the reported height, got: {reason}" + ); + } + + /// An upgrade the compiled schedule never activates must also be + /// rejected when the validator reports a height for it. + #[test] + fn rejects_an_upgrade_the_compiled_schedule_lacks() { + let upgrades = upgrades_map( + r#"{ + "77190ad8": { "name": "NU7", "activationheight": 123, "status": "pending" } + }"#, + ); + + let reason = + super::verify_reported_upgrades(&zebra_chain::parameters::Network::Mainnet, &upgrades) + .expect_err("an unscheduled upgrade with a reported height must be rejected"); + assert!( + reason.contains("None"), + "error should show the compiled schedule has no height, got: {reason}" + ); + } +} From 0713cfb1f8c11dccdb1da53c8f443b1048158520 Mon Sep 17 00:00:00 2001 From: zancas Date: Thu, 9 Jul 2026 13:59:10 -0700 Subject: [PATCH 122/134] refactor!: name The Public Testnet exactly, everywhere it is meant The public shared-consensus test network and a hermetic regtest chain are categorically different things, and the bare word "testnet" has named both. Where the intention is the public network, the canonical vocabulary is now: "The Public Testnet" in prose, `the_pub_testnet` in identifiers, and `PubTestnet` in types. The CONTEXT.md glossary entry records the rule. The config kind renames to `Network::PubTestnet`. Its canonical config spelling becomes `"PubTestnet"`, with `#[serde(alias = "Testnet")]` keeping existing config files parsing; a zainod config test pins the legacy spelling. The Public-Testnet chain-cache static renames to ZEBRAD_THE_PUB_TESTNET_CACHE_DIR, and the boundary-walk suite renames to the_pub_testnet_ironwood_boundary (the Makefile ironwood filter follows). References to zebra's own types and config format (parameters::Network::Testnet, NetworkType::Testnet, zebrad.toml) are foreign vocabulary and keep their spelling. Co-Authored-By: Claude Fable 5 --- CONTEXT.md | 11 +++-- Makefile.toml | 2 +- live-tests/clientless/tests/state_service.rs | 10 ++-- ...s => the_pub_testnet_ironwood_boundary.rs} | 48 +++++++++--------- live-tests/e2e/tests/ironwood_activation.rs | 4 +- live-tests/zaino-testutils/src/lib.rs | 8 +-- packages/zaino-common/src/config/network.rs | 23 ++++++--- .../chain_index/source/validator_connector.rs | 16 +++--- packages/zainod/src/config.rs | 49 +++++++++++++++---- zainod-heights-from-validator-spec.md | 6 +-- 10 files changed, 109 insertions(+), 68 deletions(-) rename live-tests/clientless/tests/{testnet_ironwood_boundary.rs => the_pub_testnet_ironwood_boundary.rs} (73%) diff --git a/CONTEXT.md b/CONTEXT.md index 4b4e11ba8..6246105d7 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -32,13 +32,14 @@ _Avoid_: stale version, forgotten bump ### Chains and networks -**Testnet**: -The public Zcash test network, and only that. Testnet regimes are +**The Public Testnet**: +The public Zcash test network, and only that. Its regimes are non-hermetic — state is shared with other participants, and an epoch the public chain has left (e.g. pre-NU6.3 once NU6.3 activates there) -cannot be re-entered. -_Avoid_: "testnet" for any locally-launched chain, even one launched -under a testnet network kind +cannot be re-entered. Always this exact phrase in prose; identifiers +use `the_pub_testnet` and types use `PubTestnet`. +_Avoid_: bare "Testnet"/"testnet", and especially "testnet" for any +locally-launched chain, even one launched under a testnet network kind **Regtest net**: A hermetic, locally-launched chain whose activation heights the diff --git a/Makefile.toml b/Makefile.toml index a2a9b9bd9..83e1c47ea 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -197,7 +197,7 @@ case "$set" in # invoked directly rather than through the `live` summary runner, which # forwards no nextest args. Run every set even if an earlier one fails, # as in `all`. - ironwood_filter='binary(ironwood_activation) | binary(testnet_ironwood_boundary) | test(ironwood)' + ironwood_filter='binary(ironwood_activation) | binary(the_pub_testnet_ironwood_boundary) | test(ironwood)' rc=0 makers container-test -E "$ironwood_filter" "$@" || rc=1 makers live-clientless -E "$ironwood_filter" "$@" || rc=1 diff --git a/live-tests/clientless/tests/state_service.rs b/live-tests/clientless/tests/state_service.rs index c1ccbd6c5..59b607d40 100644 --- a/live-tests/clientless/tests/state_service.rs +++ b/live-tests/clientless/tests/state_service.rs @@ -2,7 +2,7 @@ use zaino_fetch::jsonrpsee::response::address_deltas::GetAddressDeltasParams; use zaino_state::{LightWalletIndexer, NodeBackedIndexerServiceSubscriber, ZcashIndexer}; use zaino_testutils::{StateAndFetchServices, ValidatorExt}; -use zaino_testutils::{ValidatorKind, ZEBRAD_TESTNET_CACHE_DIR}; +use zaino_testutils::{ValidatorKind, ZEBRAD_THE_PUB_TESTNET_CACHE_DIR}; use zcash_local_net::validator::zebrad::Zebrad; use zebra_chain::parameters::NetworkKind; use zebra_rpc::methods::{GetAddressBalanceRequest, GetAddressTxIdsRequest}; @@ -23,7 +23,7 @@ async fn launch_regtest(enable_zaino: bool) -> StateAndFetchServices { async fn launch_testnet_cached() -> StateAndFetchServices { zaino_testutils::launch_state_and_fetch_services::( &ValidatorKind::Zebrad, - ZEBRAD_TESTNET_CACHE_DIR.clone(), + ZEBRAD_THE_PUB_TESTNET_CACHE_DIR.clone(), false, Some(NetworkKind::Testnet), ) @@ -427,7 +427,7 @@ mod zebra { async fn testnet() { state_service_check_info::( &ValidatorKind::Zebrad, - ZEBRAD_TESTNET_CACHE_DIR.clone(), + ZEBRAD_THE_PUB_TESTNET_CACHE_DIR.clone(), NetworkKind::Testnet, ) .await; @@ -612,7 +612,7 @@ mod zebra { async fn block_object_testnet() { state_service_get_block_object( &ValidatorKind::Zebrad, - ZEBRAD_TESTNET_CACHE_DIR.clone(), + ZEBRAD_THE_PUB_TESTNET_CACHE_DIR.clone(), NetworkKind::Testnet, ) .await; @@ -628,7 +628,7 @@ mod zebra { async fn block_raw_testnet() { state_service_get_block_raw( &ValidatorKind::Zebrad, - ZEBRAD_TESTNET_CACHE_DIR.clone(), + ZEBRAD_THE_PUB_TESTNET_CACHE_DIR.clone(), NetworkKind::Testnet, ) .await; diff --git a/live-tests/clientless/tests/testnet_ironwood_boundary.rs b/live-tests/clientless/tests/the_pub_testnet_ironwood_boundary.rs similarity index 73% rename from live-tests/clientless/tests/testnet_ironwood_boundary.rs rename to live-tests/clientless/tests/the_pub_testnet_ironwood_boundary.rs index 08b5a7f7f..41833480d 100644 --- a/live-tests/clientless/tests/testnet_ironwood_boundary.rs +++ b/live-tests/clientless/tests/the_pub_testnet_ironwood_boundary.rs @@ -1,7 +1,7 @@ -//! Observational walk of the real NU6.3 activation boundary on the public -//! testnet (glossary: "testnet" means the public test network and nothing -//! else). Historical blocks are permanent, so these invariants are checkable -//! forever, long after the epoch that produced them closed. +//! Observational walk of the real NU6.3 activation boundary on The Public +//! Testnet (glossary: The Public Testnet is the public test network and +//! nothing else). Historical blocks are permanent, so these invariants are +//! checkable forever, long after the epoch that produced them closed. //! //! The invariants, enforced one block below the boundary and at it: //! @@ -13,33 +13,33 @@ //! //! The activation height itself is read from the running validator's //! `getblockchaininfo.upgrades` — the single source of truth for activation -//! heights — and cross-checked against the observed public-testnet -//! activation at height 4,134,000 (~2026-07-04). +//! heights — and cross-checked against the observed activation on The +//! Public Testnet at height 4,134,000 (~2026-07-04). //! //! Non-hermetic and therefore env-gated: the test needs a zebrad chain -//! cache at `~/.cache/zebra` (`ZEBRAD_TESTNET_CACHE_DIR`) synced past the +//! cache at `~/.cache/zebra` (`ZEBRAD_THE_PUB_TESTNET_CACHE_DIR`) synced past the //! activation height, and skips with a message when the cache is absent or //! short. Run it manually, or from a job that maintains the cache. //! //! The validator is launched through `ZebradConfig` directly rather than //! `TestManager`: the manager's launch path always writes regtest test -//! parameters into the config, while this test needs the public testnet +//! parameters into the config, while this test needs The Public Testnet //! (`network_type: NetworkType::Testnet`, which makes zebrad ignore //! `activation_heights` and `miner_address`). use zaino_fetch::jsonrpsee::connector::{test_node_and_return_url, JsonRpSeeConnector}; use zaino_fetch::jsonrpsee::response::GetBlockResponse; -use zaino_testutils::ZEBRAD_TESTNET_CACHE_DIR; +use zaino_testutils::ZEBRAD_THE_PUB_TESTNET_CACHE_DIR; use zcash_local_net::process::Process as _; use zcash_local_net::protocol::NetworkType; use zcash_local_net::validator::zebrad::{Zebrad, ZebradConfig}; use zcash_local_net::validator::Validator as _; use zebra_chain::parameters::NetworkUpgrade; -/// The height the public testnet activated NU6.3 at, recorded from the real -/// activation. A mismatch means either a testnet reset or a wrong validator -/// pin — both worth failing loudly over. -const OBSERVED_TESTNET_NU6_3_ACTIVATION: u32 = 4_134_000; +/// The height The Public Testnet activated NU6.3 at, recorded from the real +/// activation. A mismatch means either a reset of The Public Testnet or a +/// wrong validator pin — both worth failing loudly over. +const OBSERVED_NU6_3_ACTIVATION_ON_THE_PUB_TESTNET: u32 = 4_134_000; /// The chain value of `pool_id` as of `height`, from the validator's /// verbosity-2 block object. @@ -65,14 +65,14 @@ async fn pool_zats(connector: &JsonRpSeeConnector, height: u32, pool_id: &str) - /// multi_thread required: the test launches the validator process and polls /// it over RPC. #[tokio::test(flavor = "multi_thread")] -async fn value_pools_respect_the_boundary_on_testnet() { - let Some(cache_dir) = ZEBRAD_TESTNET_CACHE_DIR.clone() else { - eprintln!("skipping: no testnet cache dir configured"); +async fn value_pools_respect_the_boundary_on_the_pub_testnet() { + let Some(cache_dir) = ZEBRAD_THE_PUB_TESTNET_CACHE_DIR.clone() else { + eprintln!("skipping: no cache dir configured for The Public Testnet"); return; }; if !cache_dir.exists() { eprintln!( - "skipping: no zebrad testnet chain cache at {}", + "skipping: no zebrad chain cache of The Public Testnet at {}", cache_dir.display() ); return; @@ -83,7 +83,9 @@ async fn value_pools_respect_the_boundary_on_testnet() { chain_cache: Some(cache_dir), ..ZebradConfig::default() }; - let mut zebrad = Zebrad::launch(config).await.expect("launch testnet zebrad"); + let mut zebrad = Zebrad::launch(config) + .await + .expect("launch a zebrad on The Public Testnet"); let rpc_address = format!("127.0.0.1:{}", zebrad.get_port()); let connector = JsonRpSeeConnector::new_with_basic_auth( @@ -106,7 +108,7 @@ async fn value_pools_respect_the_boundary_on_testnet() { .expect("getblockchaininfo"); // The validator's reported schedule is the source of truth for the - // boundary; the recorded constant pins the real public-testnet history. + // boundary; the recorded constant pins the real history of The Public Testnet. let boundary = blockchain_info .upgrades .values() @@ -114,16 +116,16 @@ async fn value_pools_respect_the_boundary_on_testnet() { let (upgrade, height, _status) = upgrade_info.into_parts(); (upgrade == NetworkUpgrade::Nu6_3).then_some(height.0) }) - .expect("the testnet validator must report an NU6.3 activation height"); + .expect("the validator on The Public Testnet must report an NU6.3 activation height"); assert_eq!( - boundary, OBSERVED_TESTNET_NU6_3_ACTIVATION, - "the validator's NU6.3 height must match the observed testnet activation" + boundary, OBSERVED_NU6_3_ACTIVATION_ON_THE_PUB_TESTNET, + "the validator's NU6.3 height must match the observed activation on The Public Testnet" ); let tip = blockchain_info.blocks.0; if tip <= boundary { eprintln!( - "skipping: testnet cache tip {tip} has not crossed the NU6.3 boundary {boundary}" + "skipping: cache tip {tip} of The Public Testnet has not crossed the NU6.3 boundary {boundary}" ); zebrad.stop(); return; diff --git a/live-tests/e2e/tests/ironwood_activation.rs b/live-tests/e2e/tests/ironwood_activation.rs index 6d2fbc6b1..a5db94ea1 100644 --- a/live-tests/e2e/tests/ironwood_activation.rs +++ b/live-tests/e2e/tests/ironwood_activation.rs @@ -2,7 +2,7 @@ //! //! Every test here runs a devtool wallet on //! [`ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS`] — the hermetic replay of what -//! the public testnet did once at height 4,134,000: heights 2 through 5 are +//! The Public Testnet did once at height 4,134,000: heights 2 through 5 are //! Orchard era, [`NU6_3_TRANSITION_BOUNDARY`] (6) onward is Ironwood era. //! The wallets derive their activation schedule from the running validator //! (`WalletNetwork::from_validator`, infrastructure ADR 0003), so the @@ -29,7 +29,7 @@ //! wire in `compact_block_wire.rs`; this file owns the cells that need a //! wallet on both sides of the boundary. //! -//! The public testnet cannot host the migration cell for us: its pre-NU6.3 +//! The Public Testnet cannot host the migration cell for us: its pre-NU6.3 //! epoch closed at height 4,134,000, no new value may enter Orchard from //! there (post-activation Orchard actions permit only same-receiver change //! or withdrawal — the cross-address restriction, diff --git a/live-tests/zaino-testutils/src/lib.rs b/live-tests/zaino-testutils/src/lib.rs index ce33203fc..78ecf274c 100644 --- a/live-tests/zaino-testutils/src/lib.rs +++ b/live-tests/zaino-testutils/src/lib.rs @@ -473,8 +473,8 @@ pub static ZEBRAD_CHAIN_CACHE_DIR: Lazy> = Lazy::new(|| { Some(workspace_root_path.join("live-tests/chain_cache/client_rpc_tests_large")) }); -/// Path for the Zebra chain cache in the user's home directory. -pub static ZEBRAD_TESTNET_CACHE_DIR: Lazy> = Lazy::new(|| { +/// Path for the Zebra chain cache of The Public Testnet in the user's home directory. +pub static ZEBRAD_THE_PUB_TESTNET_CACHE_DIR: Lazy> = Lazy::new(|| { let home_path = PathBuf::from(std::env::var("HOME").unwrap()); Some(home_path.join(".cache/zebra")) }); @@ -736,7 +736,7 @@ where // that adoption. let zaino_network_kind = match network_kind { NetworkKind::Mainnet => Network::Mainnet, - NetworkKind::Testnet => Network::Testnet, + NetworkKind::Testnet => Network::PubTestnet, NetworkKind::Regtest => Network::Regtest, }; @@ -1159,7 +1159,7 @@ pub async fn launch_state_and_fetch_services_mining_to( Some(NetworkKind::Testnet) => { println!("Waiting for validator to spawn.."); tokio::time::sleep(std::time::Duration::from_millis(5000)).await; - Network::Testnet + Network::PubTestnet } // The kind suffices: zainod adopts the schedule from the validator // at spawn (zaino#1076). diff --git a/packages/zaino-common/src/config/network.rs b/packages/zaino-common/src/config/network.rs index a50cb31e0..61a3dd294 100644 --- a/packages/zaino-common/src/config/network.rs +++ b/packages/zaino-common/src/config/network.rs @@ -16,8 +16,12 @@ use zebra_chain::parameters::testnet::ConfiguredActivationHeights; pub enum Network { /// Mainnet network Mainnet, - /// Testnet network - Testnet, + /// The Public Testnet: the shared-consensus test network, whose + /// activation heights come only from checked-in zebra parameters — + /// never from configuration. Accepts the legacy config spelling + /// `"Testnet"`. + #[serde(alias = "Testnet")] + PubTestnet, /// Regtest network (for local testing) Regtest, } @@ -26,13 +30,13 @@ impl fmt::Display for Network { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Network::Mainnet => write!(f, "Mainnet"), - Network::Testnet => write!(f, "Testnet"), + Network::PubTestnet => write!(f, "PubTestnet"), Network::Regtest => write!(f, "Regtest"), } } } -/// Configurable activation heights for Regtest and configured Testnets. +/// Configurable activation heights for a Regtest validator launch. /// /// We use our own type instead of the zebra type /// as the zebra type is missing a number of useful @@ -170,12 +174,15 @@ activation_heights_mirror!( impl Network { /// Determines if we should wait for the server to fully sync. Used for testing /// - /// - Mainnet/Testnet: Skip sync (false) because we don't want to sync real chains in tests + /// - Mainnet / The Public Testnet: Skip sync (false) because we don't want + /// to sync real chains in tests /// - Regtest: Enable sync (true) because regtest is local and fast to sync pub fn wait_on_server_sync(&self) -> bool { match self { - Network::Mainnet | Network::Testnet => false, // Real networks - don't try to sync the whole chain - Network::Regtest => true, // Local network - safe and fast to sync + // Real networks - don't try to sync the whole chain + Network::Mainnet | Network::PubTestnet => false, + // Local network - safe and fast to sync + Network::Regtest => true, } } } @@ -188,7 +195,7 @@ impl From for Network { if parameters.is_regtest() { Network::Regtest } else { - Network::Testnet + Network::PubTestnet } } } diff --git a/packages/zaino-state/src/chain_index/source/validator_connector.rs b/packages/zaino-state/src/chain_index/source/validator_connector.rs index 6ad37eb65..e2130c218 100644 --- a/packages/zaino-state/src/chain_index/source/validator_connector.rs +++ b/packages/zaino-state/src/chain_index/source/validator_connector.rs @@ -272,9 +272,10 @@ impl ValidatorConnector { /// (zaino#1076): the config carries only a network kind, and the runtime /// network is constructed here before anything consumes one. On regtest the /// schedule is adopted wholesale from the validator's report. On Mainnet and -/// Testnet the compiled zebra parameters are used, but the validator's -/// reported schedule is verified against them first, so a validator running -/// a *configured* testnet (custom activation heights) fails loud here +/// The Public Testnet the compiled zebra parameters are used, but the +/// validator's reported schedule is verified against them first, so a +/// validator reporting custom activation heights under the `PubTestnet` +/// kind (a regtest net in configured-testnet clothing) fails loud here /// instead of silently drifting from the index. There is no fallback: a /// silently wrong schedule is the failure mode this removes. /// @@ -293,7 +294,7 @@ async fn adopt_network( )) })?; - // Shared by the Mainnet/Testnet arms: the compiled network is only + // Shared by the Mainnet / The Public Testnet arms: the compiled network is only // trusted after the validator's report agrees with it. let verified = |network: zebra_chain::parameters::Network| { verify_reported_upgrades(&network, &blockchain_info.upgrades).map_err(|reason| { @@ -307,7 +308,7 @@ async fn adopt_network( match common.network { zaino_common::Network::Mainnet => verified(zebra_chain::parameters::Network::Mainnet), - zaino_common::Network::Testnet => { + zaino_common::Network::PubTestnet => { verified(zebra_chain::parameters::Network::new_default_testnet()) } zaino_common::Network::Regtest => { @@ -358,7 +359,8 @@ fn verify_reported_upgrades( /// runtime network here at first contact, before anything consumes a /// `Network` (zaino#1076). An upgrade absent from the validator's map is /// never-activated — nothing is backfilled from defaults. Mainnet and -/// Testnet use zebra's compiled parameters and never take this path. +/// The Public Testnet use zebra's compiled parameters and never take this +/// path. /// /// `before_overwinter` is always `None` here: the map is keyed by consensus /// branch ID, which `BeforeOverwinter` does not have, so it can never appear @@ -2946,7 +2948,7 @@ mod verify_reported_upgrades { } /// A reported height disagreeing with the compiled schedule fails loud — - /// the configured-testnet / wrong-network drift class. + /// the wrong-schedule / wrong-network drift class. #[test] fn rejects_a_mismatched_height() { let upgrades = upgrades_map( diff --git a/packages/zainod/src/config.rs b/packages/zainod/src/config.rs index 315eaf804..889359c65 100644 --- a/packages/zainod/src/config.rs +++ b/packages/zainod/src/config.rs @@ -103,7 +103,8 @@ pub struct ZainodConfig { /// When enabled, Zaino does not use a persistent on-disk finalised-state database. Finalised /// state reads are served from the configured validator/source instead. pub ephemeral_finalised_state: bool, - /// Network to connect to (Mainnet, Testnet, or Regtest). + /// Network to connect to (Mainnet, PubTestnet — The Public Testnet — or Regtest; + /// `"Testnet"` is accepted as a legacy spelling of PubTestnet). pub network: Network, /// Prometheus metrics endpoint listen address. /// @@ -266,7 +267,7 @@ impl Default for ZainodConfig { storage: StorageConfig::default(), ephemeral_finalised_state: false, zebra_db_path: default_zebra_db_path(), - network: Network::Testnet, + network: Network::PubTestnet, donation_address: None, } } @@ -602,7 +603,7 @@ key_path = "{}" let toml_content = r#" backend = "state" -network = "Testnet" +network = "PubTestnet" zebra_db_path = "/opt/zebra/data" [storage.database] @@ -621,6 +622,7 @@ listen_address = "127.0.0.1:8137" // legacy `backend = "state"` still parses via the serde alias assert_eq!(config.backend, BackendType::Direct); + assert_eq!(config.network, Network::PubTestnet); assert!(config.json_server_settings.is_none()); assert_eq!( config.validator_settings.validator_user, @@ -632,6 +634,33 @@ listen_address = "127.0.0.1:8137" ); } + /// The pre-rename config spelling of The Public Testnet still parses, + /// via the `#[serde(alias = "Testnet")]` on `Network::PubTestnet`. + #[test] + fn legacy_testnet_spelling_parses_as_the_pub_testnet() { + let _guard = EnvGuard::new(); + let temp_dir = TempDir::new().unwrap(); + + let toml_content = r#" +backend = "state" +network = "Testnet" +zebra_db_path = "/opt/zebra/data" + +[storage.database] +path = "/opt/zaino/data" + +[validator_settings] +validator_jsonrpc_listen_address = "127.0.0.1:18232" + +[grpc_settings] +listen_address = "127.0.0.1:8137" +"#; + + let config_path = create_test_config_file(&temp_dir, toml_content, "legacy_testnet.toml"); + let config = load_config(&config_path).expect("load_config failed"); + assert_eq!(config.network, Network::PubTestnet); + } + #[test] fn test_cookie_dir_logic() { let _guard = EnvGuard::new(); @@ -640,7 +669,7 @@ listen_address = "127.0.0.1:8137" // Scenario 1: auth enabled, cookie_dir empty (should use default ephemeral path) let toml_content = r#" backend = "fetch" -network = "Testnet" +network = "PubTestnet" zebra_db_path = "/zebra/db" [storage.database] @@ -670,7 +699,7 @@ listen_address = "127.0.0.1:8137" // Scenario 2: auth enabled, cookie_dir specified let toml_content2 = r#" backend = "fetch" -network = "Testnet" +network = "PubTestnet" zebra_db_path = "/zebra/db" [storage.database] @@ -697,7 +726,7 @@ listen_address = "127.0.0.1:8137" // Scenario 3: cookie_dir not specified (should be None) let toml_content3 = r#" backend = "fetch" -network = "Testnet" +network = "PubTestnet" zebra_db_path = "/zebra/db" [storage.database] @@ -808,7 +837,7 @@ listen_address = "127.0.0.1:8137" let temp_dir = TempDir::new().unwrap(); let toml_content = r#" -network = "Testnet" +network = "PubTestnet" [validator_settings] validator_jsonrpc_listen_address = "127.0.0.1:18232" @@ -899,7 +928,7 @@ listen_address = "127.0.0.1:8137" let toml_content = r#" backend = "fetch" -network = "Testnet" +network = "PubTestnet" [validator_settings] validator_jsonrpc_listen_address = "192.168.1.10:18232" @@ -930,7 +959,7 @@ listen_address = "127.0.0.1:8137" let toml_content = r#" backend = "fetch" -network = "Testnet" +network = "PubTestnet" [validator_settings] validator_jsonrpc_listen_address = "8.8.8.8:18232" @@ -1291,7 +1320,7 @@ listen_address = "127.0.0.1:8137" let toml_content = r#" backend = "fetch" -network = "Testnet" +network = "PubTestnet" ephemeral_finalised_state = true [validator_settings] diff --git a/zainod-heights-from-validator-spec.md b/zainod-heights-from-validator-spec.md index 66725a501..d78f94b0b 100644 --- a/zainod-heights-from-validator-spec.md +++ b/zainod-heights-from-validator-spec.md @@ -34,7 +34,7 @@ zebrad launch config. The wallet derives them via `WalletNetwork::from_validator`; zainod adopts them via this spec; no test-side constant needs to mirror anything. Expect the zcash_local_net pin bump to be loud: `ZainodConfig.network` narrows to a payload-free -kind (Mainnet/Testnet/Regtest), `ZcashDevtoolConfig::faucet()/recipient()` +kind (Mainnet / PubTestnet / Regtest), `ZcashDevtoolConfig::faucet()/recipient()` take a `WalletNetwork` parameter, and any test that assigned heights into a wallet config stops compiling — the fix is always to launch the validator first and derive. @@ -88,7 +88,7 @@ from it. (`state.rs:190-196`), so the ordering is achievable without restructuring. 2. **Learned heights are the only source.** `ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS` ceases to exist as a truth source. Recommended shape: the *config-level* - network type degrades to a kind (Mainnet / Testnet / Regtest, no payload), + network type degrades to a kind (Mainnet / PubTestnet / Regtest, no payload), and only the *runtime* type constructed after the handshake carries heights — making pre-handshake height reads unrepresentable. A minimal variant (populate the existing payload from the handshake before first @@ -157,7 +157,7 @@ nu5 = 2, nu6 = 2, nu6_1 = 2, nu6_2 = 2, nu6_3 = 6 ## Out of scope -- Mainnet / Testnet: compiled zebra network parameters are correct there; no +- Mainnet / The Public Testnet: compiled zebra network parameters are correct there; no handshake is needed and none should be added. - The infras-side client guard lift (Provider heights) — lands on `infrastructure#278` independently. From 5ecfe7113e16867984886d003f9d901c6a891c37 Mon Sep 17 00:00:00 2001 From: zancas Date: Thu, 9 Jul 2026 14:02:15 -0700 Subject: [PATCH 123/134] style: clear the two outstanding clippy warnings ChainWork::to_u256 takes self by value (the type is Copy, and clippy's wrong_self_convention expects to_* methods on Copy types to consume self), and the address-deltas test fixture uses i.is_multiple_of(2) in place of the manual modulo test. Co-Authored-By: Claude Fable 5 --- packages/zaino-fetch/src/jsonrpsee/response/address_deltas.rs | 2 +- packages/zaino-state/src/chain_index/types/db/block.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/zaino-fetch/src/jsonrpsee/response/address_deltas.rs b/packages/zaino-fetch/src/jsonrpsee/response/address_deltas.rs index ef422bb39..f509ec857 100644 --- a/packages/zaino-fetch/src/jsonrpsee/response/address_deltas.rs +++ b/packages/zaino-fetch/src/jsonrpsee/response/address_deltas.rs @@ -278,7 +278,7 @@ mod tests { fn sample_delta_with_block_index(i: u32, bi: Option) -> AddressDelta { AddressDelta { - satoshis: if i % 2 == 0 { 1_000 } else { -500 }, + satoshis: if i.is_multiple_of(2) { 1_000 } else { -500 }, txid: format!("deadbeef{:02x}", i), index: i, height: 123_456 + i, diff --git a/packages/zaino-state/src/chain_index/types/db/block.rs b/packages/zaino-state/src/chain_index/types/db/block.rs index 1a224b5f2..cb55b4ba1 100644 --- a/packages/zaino-state/src/chain_index/types/db/block.rs +++ b/packages/zaino-state/src/chain_index/types/db/block.rs @@ -321,7 +321,7 @@ mod tests { impl ChainWork { /// Returns ChainWork as a U256. - pub(super) fn to_u256(&self) -> primitive_types::U256 { + pub(super) fn to_u256(self) -> primitive_types::U256 { primitive_types::U256::from_big_endian(&self.0) } From 46c5e93afe2d95e08ff076af7daf14ff502e8691 Mon Sep 17 00:00:00 2001 From: zancas Date: Mon, 13 Jul 2026 08:44:50 -0700 Subject: [PATCH 124/134] bump deps --- Cargo.lock | 130 ++++++++++++++++++----------------------------------- Cargo.toml | 17 ++++--- 2 files changed, 52 insertions(+), 95 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8398f82f9..e6ed72c67 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -306,26 +306,6 @@ dependencies = [ "serde", ] -[[package]] -name = "bindgen" -version = "0.69.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" -dependencies = [ - "bitflags 2.13.0", - "cexpr", - "clang-sys", - "itertools 0.12.1", - "lazy_static", - "lazycell", - "proc-macro2", - "quote", - "regex", - "rustc-hash 1.1.0", - "shlex 1.3.0", - "syn 2.0.118", -] - [[package]] name = "bindgen" version = "0.72.1" @@ -339,7 +319,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "rustc-hash 2.1.2", + "rustc-hash", "shlex 1.3.0", "syn 2.0.118", ] @@ -2247,15 +2227,6 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" -[[package]] -name = "itertools" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.13.0" @@ -2374,7 +2345,7 @@ dependencies = [ "jsonrpsee-types", "parking_lot", "rand 0.8.6", - "rustc-hash 2.1.2", + "rustc-hash", "serde", "serde_json", "thiserror 1.0.69", @@ -2481,12 +2452,6 @@ dependencies = [ "spin", ] -[[package]] -name = "lazycell" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" - [[package]] name = "left-right" version = "0.11.7" @@ -2531,14 +2496,13 @@ dependencies = [ [[package]] name = "librocksdb-sys" -version = "0.16.0+8.10.0" +version = "0.17.3+10.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce3d60bc059831dc1c83903fb45c103f75db65c5a7bf22272764d9cc683e348c" +checksum = "cef2a00ee60fe526157c9023edab23943fae1ce2ab6f4abb2a807c1746835de9" dependencies = [ - "bindgen 0.69.5", + "bindgen", "bzip2-sys", "cc", - "glob", "libc", "libz-sys", "lz4-sys", @@ -2561,7 +2525,7 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f8ce05b56f3cbc65ec7d0908adb308ed91281e022f61c8c3a0c9388b5380b17" dependencies = [ - "bindgen 0.72.1", + "bindgen", "cc", "thiserror 2.0.18", "tracing", @@ -2953,9 +2917,9 @@ checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" [[package]] name = "orchard" -version = "0.15.0-pre.1" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8e277dd4b46f5d06deae3ffb8af1a951e8622368f028c2a4d6fe59339566403" +checksum = "cca2ede6a4a77bd729eed3be9303d98a08b217edc85190dcf4dbe004086d5a65" dependencies = [ "aes", "bitvec", @@ -3473,7 +3437,7 @@ dependencies = [ "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash 2.1.2", + "rustc-hash", "rustls", "socket2", "thiserror 2.0.18", @@ -3493,7 +3457,7 @@ dependencies = [ "lru-slab", "rand 0.9.4", "ring", - "rustc-hash 2.1.2", + "rustc-hash", "rustls", "rustls-pki-types", "slab", @@ -3936,9 +3900,9 @@ dependencies = [ [[package]] name = "rocksdb" -version = "0.22.0" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bd13e55d6d7b8cd0ea569161127567cd587676c99f4472f779a0279aa60a7a7" +checksum = "ddb7af00d2b17dbd07d82c0063e25411959748ff03e8d4f96134c2ff41fce34f" dependencies = [ "libc", "librocksdb-sys", @@ -3956,12 +3920,6 @@ version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" -[[package]] -name = "rustc-hash" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" - [[package]] name = "rustc-hash" version = "2.1.2" @@ -5016,9 +4974,9 @@ dependencies = [ [[package]] name = "tower-batch-control" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e6cf52578f98b4da47335c26c4f883f7993b1a9b9d2f5420eb8dbfd5dd19a28" +checksum = "0a49cdaf2b00c1150c93a8dbbb63cae193573903a3c9a07b5d4d61251b9e32f0" dependencies = [ "futures", "futures-core", @@ -5033,9 +4991,9 @@ dependencies = [ [[package]] name = "tower-fallback" -version = "0.2.41" +version = "0.2.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4434e19ee996ee5c6aa42f11463a355138452592e5c5b5b73b6f0f19534556af" +checksum = "700cd6732c9c9e2c77501cc663708b2229abc9e8fa27d3deae15126330a1ed87" dependencies = [ "futures-core", "pin-project", @@ -6116,9 +6074,9 @@ dependencies = [ [[package]] name = "zcash_address" -version = "0.13.0-pre.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cf2918e73eff76388cda87695a6e7398f96a2e3383a0de7b77729a204ec00a0" +checksum = "5a854b28c07dba372f4410ea8ad62b4bf7d5c2bf8be32fc4b31bc0db6521a975" dependencies = [ "bech32", "bs58", @@ -6141,9 +6099,9 @@ dependencies = [ [[package]] name = "zcash_history" -version = "0.5.0-pre.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82e8634d011026cb181cb67b2a412e601dc344a2827b9c576fe3a727fdebf441" +checksum = "69d2ed9a9fa7b466a7adedab1ad5a21f8330e4943756912fc776cf7f3beae32b" dependencies = [ "blake2b_simd", "byteorder", @@ -6152,9 +6110,9 @@ dependencies = [ [[package]] name = "zcash_keys" -version = "0.15.0-pre.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a0bac3a9e5b0d954684ba1f07386d55b5ae4588b3ba2d93cad05ee09f3e0947" +checksum = "36391ebfce7df2510c6564f79e608ed93f1c0692c6464e2397b248e06dae3912" dependencies = [ "bech32", "blake2b_simd", @@ -6211,9 +6169,9 @@ dependencies = [ [[package]] name = "zcash_primitives" -version = "0.29.0-pre.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba7bfe66975658f44dba87d535f9ebb9fb4c9c0c4fecca3f5c40cbe1583dece6" +checksum = "bdbeccc05bfe63b6dee9e989c6ff5027421741562a4853c36504dd621f6a81b1" dependencies = [ "blake2b_simd", "block-buffer 0.11.0-rc.3", @@ -6242,9 +6200,9 @@ dependencies = [ [[package]] name = "zcash_proofs" -version = "0.29.0-pre.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39b90964ffe6bdc314368c7b849aaa6dcc2b495249a3bf1b9bfbdc92ba4e4f6" +checksum = "b91fb0b420fb66a765f1fa92ab7a6b631aa54c08415415ef6185256826e74fbd" dependencies = [ "bellman", "blake2b_simd", @@ -6265,9 +6223,9 @@ dependencies = [ [[package]] name = "zcash_protocol" -version = "0.10.0-pre.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97f339a9801c7f70295732a5cf822346f194202cea6425fc2fc14a5af679b004" +checksum = "2973d210e74ebd81adc50828fbbb284ab6aaa099308097c42c47dc63cf3420fc" dependencies = [ "corez", "document-features", @@ -6304,9 +6262,9 @@ dependencies = [ [[package]] name = "zcash_transparent" -version = "0.9.0-pre.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e941230f67056aad41c8fa9b926f9cc4d9d1a321f32e95c39c0b0e38e85f79b" +checksum = "72e0f8c142bed366ed5886dcfa8772ec90b5420cc127e6cdb76b6744c53a287b" dependencies = [ "bip32", "bs58", @@ -6329,9 +6287,9 @@ dependencies = [ [[package]] name = "zebra-chain" -version = "11.0.0" +version = "11.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "600a4c35ee81350384226f8178fab2b088e38d5e0a9f6cb0b1051caba30702d7" +checksum = "014adde138ea00b1cf6c873aaf56aacb5aa0d55b8e5451a3e3b1b06c461bd107" dependencies = [ "bech32", "bitflags 2.13.0", @@ -6398,9 +6356,9 @@ dependencies = [ [[package]] name = "zebra-consensus" -version = "10.0.0" +version = "11.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0aa2b2678c4ce6d18172bad73a9b72b24a4b3a65af4c2a5e9120a1470c3fc402" +checksum = "4481ced3223226db18e33b9fb2e8eab7f7cefeae1681f4e8d8d7e3c2285a5c4d" dependencies = [ "bellman", "blake2b_simd", @@ -6441,9 +6399,9 @@ dependencies = [ [[package]] name = "zebra-network" -version = "10.0.0" +version = "10.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36b1e6ca4687bc8d128553ca04bb82cc38701c4dac7107f3e56717b39d4b18da" +checksum = "9a59dc9e9ac723c4c01d985a5ecf3ddd85c0f1681574457d52173c43263905f2" dependencies = [ "bitflags 2.13.0", "byteorder", @@ -6479,9 +6437,9 @@ dependencies = [ [[package]] name = "zebra-node-services" -version = "9.0.0" +version = "9.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4446db9b576c0de14ff91c7fd7b38a59d340c1969a1604787185a52b8b7f53a6" +checksum = "006992c62b9c4742a13a306f2e7443646fa0b5b884aac63a6fba5b0be273f864" dependencies = [ "color-eyre", "jsonrpsee-types", @@ -6495,9 +6453,9 @@ dependencies = [ [[package]] name = "zebra-rpc" -version = "11.0.0" +version = "11.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb180542f1ffc0f66c20e1cd66c7741662053ccda588e3847a8ed2df8c83dae7" +checksum = "4d071b177740cdc1107f90dd50fac67e1b440a84c8509ae1529b3609fb6f801d" dependencies = [ "base64", "chrono", @@ -6556,9 +6514,9 @@ dependencies = [ [[package]] name = "zebra-script" -version = "10.0.0" +version = "10.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8648c201e3dadb1c95022eb7605d528de4bf94b6c0cebf76537805849aee76c5" +checksum = "f7c967badb1bb99573dba430948c98fbca940daf23915f2e9736e1a1d2a166f3" dependencies = [ "libzcash_script", "rand 0.8.6", @@ -6570,9 +6528,9 @@ dependencies = [ [[package]] name = "zebra-state" -version = "10.0.0" +version = "10.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b95b81181253a46885e3254cc5a20e4d82b5258116c9e43db5c8e9660c6b973" +checksum = "39295d344c1cc56ed725646cfc7bb7e16f8edae7518ea5a9e599665f979e9cf6" dependencies = [ "bincode", "chrono", diff --git a/Cargo.toml b/Cargo.toml index 8578ba5ff..0b6ece458 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,17 +40,16 @@ license = "Apache-2.0" # Librustzcash incrementalmerkletree = "0.8" -zcash_address = "0.13.0-pre.0" -zcash_keys = "0.15.0-pre.0" -zcash_protocol = "0.10.0-pre.0" -zcash_primitives = "0.29.0-pre.0" -zcash_transparent = "0.9.0-pre.0" -zcash_client_backend = "0.24.0-pre.0" +zcash_address = "0.13" +zcash_keys = "0.15" +zcash_protocol = "0.10" +zcash_primitives = "0.29" +zcash_transparent = "0.9" # Zebra -zebra-chain = "11.0" -zebra-state = "10.0" -zebra-rpc = "11.0" +zebra-chain = "11.1" +zebra-state = "10.1" +zebra-rpc = "11.1" # Runtime tokio = { version = "1.38", features = ["full"] } From 1a795db91daee239df4562f6653ec70bf33edd60 Mon Sep 17 00:00:00 2001 From: zancas Date: Mon, 13 Jul 2026 08:58:40 -0700 Subject: [PATCH 125/134] build(zainod): drop the unused zebra-chain dependency An audit of every zebra* and zcash* dependency edge (cargo machete, the compiler's unused_crate_dependencies lint, and inspection of each remaining single-reference hit) found exactly one dead edge: zainod declared zebra-chain but never referenced it. Remove it. Every other declared edge is genuinely used; zebra-chain remains the type backbone of the workspace, with some 350 references in zaino-state alone. Co-Authored-By: Claude Fable 5 --- Cargo.lock | 1 - packages/zainod/Cargo.toml | 1 - 2 files changed, 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e6ed72c67..d919c5245 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6068,7 +6068,6 @@ dependencies = [ "zaino-state", "zcash_address", "zcash_protocol", - "zebra-chain", "zebra-state", ] diff --git a/packages/zainod/Cargo.toml b/packages/zainod/Cargo.toml index 4fc9a786d..bc9d0b938 100644 --- a/packages/zainod/Cargo.toml +++ b/packages/zainod/Cargo.toml @@ -70,7 +70,6 @@ zaino-state = { workspace = true } zaino-serve = { workspace = true } # Zebra -zebra-chain = { workspace = true } zebra-state = { workspace = true } # Runtime From 1e255d083b2ab92974a98485d1ff457261ded587 Mon Sep 17 00:00:00 2001 From: zancas Date: Mon, 13 Jul 2026 08:58:40 -0700 Subject: [PATCH 126/134] build: advance ZEBRA_VERSION to the 6.0.0 final release Advance ZEBRA_VERSION in .env.testing-artifacts from 6.0.0-rc.0 to the final 6.0.0 release. Zebrad 6.0.0 depends on zebra-chain ^11.1.0, the same crate generation the workspace now pins, so the zebrad binary the live tests run against and the zebra libraries zaino links are once again from a single release. The makers check-matching-zebras guard passes. Co-Authored-By: Claude Fable 5 --- .env.testing-artifacts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.testing-artifacts b/.env.testing-artifacts index 92b2fbae2..e1e14d228 100644 --- a/.env.testing-artifacts +++ b/.env.testing-artifacts @@ -5,7 +5,7 @@ # zcashd Git tag (https://github.com/zcash/zcash/releases) ZCASH_VERSION=6.20.0 -ZEBRA_VERSION=6.0.0-rc.0 +ZEBRA_VERSION=6.0.0 # zcash-devtool git ref (https://github.com/zingolabs/zcash-devtool) — # the wallet client driven by wallet-tests via zcash_local_net, built From 483a6ca17612638b0c5bf6906921b4d3b41dfb0b Mon Sep 17 00:00:00 2001 From: zancas Date: Mon, 13 Jul 2026 09:12:38 -0700 Subject: [PATCH 127/134] build: remove unused dependency edges across the workspace A workspace-wide unused-dependency audit (the compiler's unused_crate_dependencies lint analyzed per target, cross-checked with cargo machete and a text search of every candidate's package, including feature-gated code) found twelve dead edges. Remove them: prost, bs58, nonempty, derive_more, and once_cell from zaino-state; tracing-subscriber from zainod; and tracing, corez, serde_json, tempfile, tower, and zip32 from clientless. The bs58, nonempty, and zip32 entries in [workspace.dependencies] lost their last inheritors and are removed with them. One edge that looks dead is kept deliberately: clientless does not import zainod in any test source, but its transparent_address_history_experimental feature forwards to zainod/transparent_address_history_experimental, and cargo rejects a feature forward to a non-dependency. A comment in the manifest now records that. Co-Authored-By: Claude Fable 5 --- Cargo.lock | 28 +++++----------------------- Cargo.toml | 3 --- live-tests/clientless/Cargo.toml | 9 +++------ packages/zaino-state/Cargo.toml | 5 ----- packages/zainod/Cargo.toml | 1 - 5 files changed, 8 insertions(+), 38 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d919c5245..798fd33aa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -669,14 +669,9 @@ name = "clientless" version = "0.2.0" dependencies = [ "anyhow", - "corez", "futures", "hex", - "serde_json", - "tempfile", "tokio", - "tower 0.4.13", - "tracing", "wire_serialized_transaction_test_data", "zaino-common", "zaino-fetch", @@ -688,7 +683,6 @@ dependencies = [ "zebra-chain", "zebra-rpc", "zebra-state", - "zip32", ] [[package]] @@ -2773,12 +2767,6 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "549e471b99ccaf2f89101bec68f4d244457d5a95a9c3d0672e9564124397741d" -[[package]] -name = "nonempty" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9737e026353e5cd0736f98eddae28665118eb6f6600902a7f50db585621fecb6" - [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -2936,7 +2924,7 @@ dependencies = [ "incrementalmerkletree", "lazy_static", "memuse", - "nonempty 0.11.0", + "nonempty", "pasta_curves", "rand 0.8.6", "rand_core 0.6.4", @@ -5976,11 +5964,9 @@ dependencies = [ "arc-swap", "bitflags 2.13.0", "blake2", - "bs58", "chrono", "corez", "dashmap", - "derive_more", "futures", "hex", "incrementalmerkletree", @@ -5988,11 +5974,8 @@ dependencies = [ "lmdb", "lmdb-sys", "metrics", - "nonempty 0.12.0", - "once_cell", "primitive-types 0.14.0", "proptest", - "prost", "rand 0.10.1", "reqwest 0.13.1", "sapling-crypto", @@ -6061,7 +6044,6 @@ dependencies = [ "tokio", "toml", "tracing", - "tracing-subscriber", "zaino-common", "zaino-fetch", "zaino-serve", @@ -6093,7 +6075,7 @@ checksum = "1440921903cdb86133fb9e2fe800be488015db2939a30bedb413078a1acb0306" dependencies = [ "corez", "hex", - "nonempty 0.11.0", + "nonempty", ] [[package]] @@ -6121,7 +6103,7 @@ dependencies = [ "document-features", "group", "memuse", - "nonempty 0.11.0", + "nonempty", "orchard", "rand_core 0.6.4", "sapling-crypto", @@ -6183,7 +6165,7 @@ dependencies = [ "incrementalmerkletree", "jubjub", "memuse", - "nonempty 0.11.0", + "nonempty", "orchard", "rand_core 0.6.4", "redjubjub", @@ -6271,7 +6253,7 @@ dependencies = [ "document-features", "getset", "hex", - "nonempty 0.11.0", + "nonempty", "ripemd 0.1.3", "secp256k1", "sha2 0.10.9", diff --git a/Cargo.toml b/Cargo.toml index 0b6ece458..e13a29ac6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -114,7 +114,6 @@ blake2 = "0.10" hex = "0.4" toml = "1.1" primitive-types = "0.14" -bs58 = "0.5" bitflags = "2.9" # Test @@ -146,9 +145,7 @@ zcash_local_net = { git = "https://github.com/zingolabs/infrastructure.git", rev wire_serialized_transaction_test_data = "1.0.0" config = { version = "0.15", default-features = false, features = ["toml"] } -nonempty = "0.12.0" proptest = "~1.11" -zip32 = "0.2.1" anyhow = "1.0" arc-swap = "1.7.1" derive_more = "2.0.1" diff --git a/live-tests/clientless/Cargo.toml b/live-tests/clientless/Cargo.toml index ba912e690..c8945a81a 100644 --- a/live-tests/clientless/Cargo.toml +++ b/live-tests/clientless/Cargo.toml @@ -36,7 +36,6 @@ zaino-fetch = { workspace = true } zaino-testutils = { workspace = true } # The lib's z_validate helper calls `ZcashIndexer::z_validate_address`. zaino-state = { workspace = true, features = ["test_dependencies"] } -tracing.workspace = true [dev-dependencies] anyhow = { workspace = true } @@ -46,6 +45,9 @@ anyhow = { workspace = true } zaino-state = { workspace = true, features = ["fast-test-seam"] } # Test utility +# Not imported by any test source; present so the +# transparent_address_history_experimental feature can forward to +# zainod/transparent_address_history_experimental in the spawned daemon. zainod = { workspace = true } zaino-testutils = { workspace = true } zaino-fetch = { workspace = true } @@ -55,13 +57,8 @@ wire_serialized_transaction_test_data = { workspace = true } zebra-chain = { workspace = true } zebra-state = { workspace = true } zebra-rpc = { workspace = true } -zip32 = { workspace = true } -corez = { workspace = true } -serde_json = { workspace = true } futures = { workspace = true } -tempfile = { workspace = true } -tower = { workspace = true } hex = { workspace = true } # Runtime diff --git a/packages/zaino-state/Cargo.toml b/packages/zaino-state/Cargo.toml index 4f96dbe9f..6b341c1c9 100644 --- a/packages/zaino-state/Cargo.toml +++ b/packages/zaino-state/Cargo.toml @@ -83,22 +83,17 @@ lmdb = { workspace = true } lmdb-sys = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true, features = ["preserve_order"] } -prost = { workspace = true } primitive-types = { workspace = true } blake2 = { workspace = true } sha2 = { workspace = true } corez = { workspace = true } -bs58 = { workspace = true } -nonempty = { workspace = true } arc-swap = { workspace = true } reqwest.workspace = true bitflags = { workspace = true } -derive_more = { workspace = true, features = ["from"] } [dev-dependencies] tempfile = { workspace = true } tracing-subscriber = { workspace = true } -once_cell = { workspace = true } zebra-chain = { workspace = true, features = ["proptest-impl"] } proptest.workspace = true incrementalmerkletree = "0.8.2" diff --git a/packages/zainod/Cargo.toml b/packages/zainod/Cargo.toml index bc9d0b938..c3a54016e 100644 --- a/packages/zainod/Cargo.toml +++ b/packages/zainod/Cargo.toml @@ -80,7 +80,6 @@ clap = { workspace = true, features = ["derive"] } # Tracing tracing = { workspace = true } -tracing-subscriber = { workspace = true, features = ["fmt", "env-filter", "time"] } # Network / RPC http = { workspace = true } From 46d18c623b993e4394436017be2c37a89f7c00b0 Mon Sep 17 00:00:00 2001 From: zancas Date: Mon, 13 Jul 2026 09:13:11 -0700 Subject: [PATCH 128/134] build: demote test-only dependencies to dev-dependencies The same audit that removed the dead edges found three dependencies that are real but used exclusively under #[cfg(test)]: primitive-types in zaino-state (the legacy_chainwork_reference round-trip tests), tempfile in zainod (the config tests), and tonic in zaino-testutils (the cfg(test)-gated build_client helper). Move each from [dependencies] to [dev-dependencies] so shipped builds stop linking them. The lockfile is unchanged: dependency kind does not affect resolution. Co-Authored-By: Claude Fable 5 --- live-tests/zaino-testutils/Cargo.toml | 4 +++- packages/zaino-state/Cargo.toml | 2 +- packages/zainod/Cargo.toml | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/live-tests/zaino-testutils/Cargo.toml b/live-tests/zaino-testutils/Cargo.toml index 38f2ef883..09b132e9f 100644 --- a/live-tests/zaino-testutils/Cargo.toml +++ b/live-tests/zaino-testutils/Cargo.toml @@ -65,5 +65,7 @@ futures = { workspace = true } http = { workspace = true } once_cell = { workspace = true } tokio = { workspace = true } -tonic.workspace = true tracing.workspace = true + +[dev-dependencies] +tonic.workspace = true diff --git a/packages/zaino-state/Cargo.toml b/packages/zaino-state/Cargo.toml index 6b341c1c9..0f60eab31 100644 --- a/packages/zaino-state/Cargo.toml +++ b/packages/zaino-state/Cargo.toml @@ -83,7 +83,6 @@ lmdb = { workspace = true } lmdb-sys = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true, features = ["preserve_order"] } -primitive-types = { workspace = true } blake2 = { workspace = true } sha2 = { workspace = true } corez = { workspace = true } @@ -94,6 +93,7 @@ bitflags = { workspace = true } [dev-dependencies] tempfile = { workspace = true } tracing-subscriber = { workspace = true } +primitive-types = { workspace = true } zebra-chain = { workspace = true, features = ["proptest-impl"] } proptest.workspace = true incrementalmerkletree = "0.8.2" diff --git a/packages/zainod/Cargo.toml b/packages/zainod/Cargo.toml index c3a54016e..beba27ad9 100644 --- a/packages/zainod/Cargo.toml +++ b/packages/zainod/Cargo.toml @@ -95,8 +95,8 @@ thiserror = { workspace = true } # Formats toml = { workspace = true } config = { workspace = true } -tempfile = { workspace = true } [dev-dependencies] +tempfile = { workspace = true } zcash_address = { workspace = true } zcash_protocol = { workspace = true } From 4141a5d4dceb4e1705a196348b3b774e5114bb1b Mon Sep 17 00:00:00 2001 From: zancas Date: Mon, 13 Jul 2026 09:19:18 -0700 Subject: [PATCH 129/134] feat(test-runner): add a packages row and per-suite timing to the summary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `makers test all` previously ran the packages set outside the summary runner, so the closing table covered only the two live partitions. The `all` branch now delegates to `live-summary --all`, which runs `makers container-test` first and prints its tallies as a `packages:` row above `clientless:`, `e2e:`, and `TOTAL:`, under a "test summary" header. `makers test live` is unchanged apart from the new column. The old contract is preserved: every set runs even when an earlier one fails, and the runner exits non-zero if any set did. Each row also gains a wall-clock time column, parsed from the nextest summary line's `[ 510.718s]` stamp. Durations are held as integer milliseconds in a u64 — whole seconds and a right-padded three-digit fraction parsed digit-by-digit, no floating point — and the TOTAL row sums them, which reads as true wall time because the suites run sequentially. Six new unit tests cover the parser (padding, short and absent fractions, and propagation through parse_summary), and the pre-existing `filter(..).last()` on a double-ended iterator is tightened to `rfind(..)` per clippy. Co-Authored-By: Claude Fable 5 --- Makefile.toml | 10 +- tools/test-runner/src/bin/live-summary.rs | 127 +++++++++++++++++++--- 2 files changed, 118 insertions(+), 19 deletions(-) diff --git a/Makefile.toml b/Makefile.toml index 83e1c47ea..2bcaec624 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -182,12 +182,10 @@ case "$set" in clientless) exec makers live-clientless "$@" ;; live) exec cargo run -q --manifest-path tools/test-runner/Cargo.toml --bin live-summary -- "$@" ;; all) - # Run both even if the first fails, then fail if either did, so the - # omnibus reflects the whole suite. - rc=0 - makers container-test "$@" || rc=1 - cargo run -q --manifest-path tools/test-runner/Cargo.toml --bin live-summary -- "$@" || rc=1 - exit "$rc" ;; + # The summary runner's --all mode runs the packages set and both live + # partitions (each set runs even when an earlier one fails, and the exit + # code reflects any failure), so the combined table covers everything. + exec cargo run -q --manifest-path tools/test-runner/Cargo.toml --bin live-summary -- --all "$@" ;; ironwood) # The one definition of the ironwood selection: the two dedicated test # binaries, plus every test whose name carries the pool's name (the diff --git a/tools/test-runner/src/bin/live-summary.rs b/tools/test-runner/src/bin/live-summary.rs index 47d3ff3bd..3af1d56ed 100644 --- a/tools/test-runner/src/bin/live-summary.rs +++ b/tools/test-runner/src/bin/live-summary.rs @@ -1,10 +1,12 @@ -//! `live-summary` — run both live-test partitions and print a combined summary. +//! `live-summary` — run the test partitions and print a combined summary. //! -//! Invoked from `makers test live` (and `makers test all`) as -//! `cargo run --bin live-summary -- `. Runs the `clientless` then `e2e` -//! partition (each in its own CI container via its own `makers` task), streams -//! each run's output while capturing it, parses the nextest summary line, and -//! aggregates the totals. +//! Invoked from `makers test live` as `cargo run --bin live-summary -- ` +//! and from `makers test all` with the extra `--all` flag. The live set runs +//! the `clientless` then `e2e` partition (each in its own CI container via its +//! own `makers` task); `--all` first runs the `packages` set (`makers +//! container-test`) and adds its row to the table. The runner streams each +//! run's output while capturing it, parses the nextest summary line (tallies +//! and wall-clock duration), and aggregates the totals. //! //! Unlike a cargo-make `dependencies` list (which is fail-fast), this runs BOTH //! partitions even when the first fails, so the summary reflects the whole @@ -19,6 +21,14 @@ use std::error::Error; use std::io::{BufRead, BufReader}; use std::process::{Command, Stdio}; +/// Which partitions this invocation runs and summarizes. +enum TestSet { + /// The two live partitions (`makers test live`). + Live, + /// The packages set plus the two live partitions (`makers test all`). + All, +} + /// One nextest run's tallies, zero where the summary line was absent. #[derive(Default)] struct Summary { @@ -26,6 +36,8 @@ struct Summary { passed: u64, failed: u64, skipped: u64, + /// Wall-clock milliseconds from the summary line's `[ 510.718s]` stamp. + millis: u64, } impl Summary { @@ -35,6 +47,7 @@ impl Summary { passed: self.passed + other.passed, failed: self.failed + other.failed, skipped: self.skipped + other.skipped, + millis: self.millis + other.millis, } } } @@ -104,6 +117,25 @@ fn count_before(line: &str, marker: &str) -> u64 { head[head.len() - digit_count..].parse().unwrap_or(0) } +/// The wall-clock milliseconds from the summary line's `[ 510.718s]` stamp, +/// or 0 if absent. Parsed as integers (whole seconds and up to three +/// fractional digits) — no floating point. +fn duration_millis(line: &str) -> u64 { + let Some(idx) = line.find("s]") else { + return 0; + }; + let head = &line[..idx]; + let start = head + .rfind(|c: char| !(c.is_ascii_digit() || c == '.')) + .map_or(0, |i| i + 1); + let stamp = &head[start..]; + let (whole, frac) = stamp.split_once('.').unwrap_or((stamp, "")); + let whole: u64 = whole.parse().unwrap_or(0); + // Right-pad the fraction to exactly three digits ("7" -> 700ms). + let frac: u64 = format!("{frac:0<3.3}").parse().unwrap_or(0); + whole * 1000 + frac +} + /// Parse the last nextest summary line out of a captured run. /// /// nextest prints e.g.: @@ -115,8 +147,7 @@ fn parse_summary(log: &str) -> Summary { let line = log .lines() .map(strip_ansi) - .filter(|l| l.contains("run:") && l.contains("test")) - .last() + .rfind(|l| l.contains("run:") && l.contains("test")) .unwrap_or_default(); Summary { @@ -125,18 +156,34 @@ fn parse_summary(log: &str) -> Summary { passed: count_before(&line, "passed"), failed: count_before(&line, "failed"), skipped: count_before(&line, "skipped"), + millis: duration_millis(&line), } } fn print_row(label: &str, s: &Summary) { + let secs = format!("{}.{:03}", s.millis / 1000, s.millis % 1000); println!( - " {label:<18} {:>4} run, {:>4} passed, {:>4} failed, {:>4} skipped", + " {label:<18} {:>4} run, {:>4} passed, {:>4} failed, {:>4} skipped, {secs:>10}s", s.run, s.passed, s.failed, s.skipped ); } fn main() -> Result<(), Box> { let with_zcashd = std::env::args().any(|a| a == "--with-zcashd"); + let set = if std::env::args().any(|a| a == "--all") { + TestSet::All + } else { + TestSet::Live + }; + + let packages = match set { + TestSet::All => { + println!(">>> all: running packages set"); + let (rc, log) = run_partition("container-test", with_zcashd)?; + Some((rc, parse_summary(&log))) + } + TestSet::Live => None, + }; println!(">>> live: running clientless partition"); let (cl_rc, cl_log) = run_partition("live-clientless", with_zcashd)?; @@ -147,15 +194,32 @@ fn main() -> Result<(), Box> { let cl = parse_summary(&cl_log); let e2e = parse_summary(&e2e_log); + let header = match packages { + Some(_) => "test summary", + None => "live summary", + }; + let mut total = cl.add(&e2e); + if let Some((_, p)) = &packages { + total = total.add(p); + } + println!(); - println!("====================== live summary =========================="); + println!("====================== {header} =========================="); + if let Some((_, p)) = &packages { + print_row("packages:", p); + } print_row("clientless:", &cl); print_row("e2e:", &e2e); - print_row("TOTAL:", &cl.add(&e2e)); + print_row("TOTAL:", &total); println!("=============================================================="); // A partition that errored without producing a summary line likely failed // to build; call it out so the zeros above aren't read as "all clear". + if let Some((pkg_rc, p)) = &packages { + if *pkg_rc != 0 && p.run == 0 { + println!(" warning: packages produced no nextest summary (build failure?) — see output above."); + } + } if cl_rc != 0 && cl.run == 0 { println!(" warning: clientless produced no nextest summary (build failure?) — see output above."); } @@ -163,8 +227,9 @@ fn main() -> Result<(), Box> { println!(" warning: e2e produced no nextest summary (build failure?) — see output above."); } - // Fail the front door if either partition failed. - if cl_rc != 0 || e2e_rc != 0 { + // Fail the front door if any set failed. + let pkg_rc = packages.map_or(0, |(rc, _)| rc); + if pkg_rc != 0 || cl_rc != 0 || e2e_rc != 0 { std::process::exit(1); } Ok(()) @@ -215,3 +280,39 @@ mod parse_summary { check(log, 9, 7, 1, 1); } } + +#[cfg(test)] +mod duration_millis { + use super::*; + + #[test] + fn fractional_seconds_with_padding() { + assert_eq!(duration_millis("Summary [ 73.207s] 8 tests run: 8 passed"), 73_207); + } + + #[test] + fn no_padding() { + assert_eq!(duration_millis("Summary [510.718s] 29 tests run: 23 passed"), 510_718); + } + + #[test] + fn short_fraction_is_right_padded() { + assert_eq!(duration_millis("Summary [2.7s] 1 test run: 1 passed"), 2_700); + } + + #[test] + fn whole_seconds_without_fraction() { + assert_eq!(duration_millis("Summary [2s] 1 test run: 1 passed"), 2_000); + } + + #[test] + fn missing_stamp_is_zero() { + assert_eq!(duration_millis("no summary line here"), 0); + } + + #[test] + fn survives_through_parse_summary() { + let s = parse_summary("Summary [ 1.795s] 1 test run: 0 passed, 1 failed, 114 skipped"); + assert_eq!(s.millis, 1_795); + } +} From 2f3541a4a85352bfed928453ed71b38354051373 Mon Sep 17 00:00:00 2001 From: zancas Date: Mon, 13 Jul 2026 09:37:25 -0700 Subject: [PATCH 130/134] fix(ci): refresh the trixie apt pins the point release superseded The test-environment image rebuild (first since the ZEBRA_VERSION bump changed the image tag) failed because Debian trixie point updates replaced two pinned package versions, and the archive does not keep superseded versions: protobuf-compiler 3.21.12-11 -> 3.21.12-11+deb13u1 (all three build stages) and curl 8.14.1-2+deb13u3 -> 8.14.1-2+deb13u4 (both stages). Every other pin in the Containerfile still matches the archive; verified against qa.debian.org/madison and a live trixie-slim container. Co-Authored-By: Claude Fable 5 --- live-tests/test_environment/Containerfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/live-tests/test_environment/Containerfile b/live-tests/test_environment/Containerfile index dabb4dea2..a6a19c7fd 100644 --- a/live-tests/test_environment/Containerfile +++ b/live-tests/test_environment/Containerfile @@ -46,7 +46,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ clang=1:19.0-63 \ cmake=3.31.6-2 \ pkg-config=1.8.1-4 \ - protobuf-compiler=3.21.12-11 \ + protobuf-compiler=3.21.12-11+deb13u1 \ libstdc++6=14.2.0-19 WORKDIR /zebra # Single RUN (DL3059): clone, checkout, build, and stage the binary. @@ -71,7 +71,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ autoconf=2.72-3.1 \ libtool=2.5.4-4 \ bsdmainutils=12.1.8 \ - curl=8.14.1-2+deb13u3 \ + curl=8.14.1-2+deb13u4 \ ca-certificates=20250419 \ netbase=6.5 @@ -99,7 +99,7 @@ ARG DEVTOOL_REV RUN apt-get update && apt-get install -y --no-install-recommends \ git=1:2.47.3-0+deb13u1 \ pkg-config=1.8.1-4 \ - protobuf-compiler=3.21.12-11 \ + protobuf-compiler=3.21.12-11+deb13u1 \ libsqlite3-dev=3.46.1-7+deb13u1 WORKDIR /devtool # Single RUN (DL3059): clone, checkout, build, and stage the binary. The @@ -140,12 +140,12 @@ LABEL org.zaino.test-artifacts.versions.rust="${RUST_VERSION}" # Versions pinned (DL3008) to the candidates in rust:1.95.0-trixie; bump # together with the base image (query with `apt-cache policy `). RUN apt-get update && apt-get install -y --no-install-recommends \ - curl=8.14.1-2+deb13u3 \ + curl=8.14.1-2+deb13u4 \ libclang-dev=1:19.0-63 \ build-essential=12.12 \ cmake=3.31.6-2 \ librocksdb-dev=9.10.0-1+b1 \ - protobuf-compiler=3.21.12-11 \ + protobuf-compiler=3.21.12-11+deb13u1 \ && rm -rf /var/lib/apt/lists/* # Create container_user. The GID may already belong to a base-image group From 45ed68d16a0f76776009e296d76847a10f47dd0e Mon Sep 17 00:00:00 2001 From: nachog00 Date: Mon, 13 Jul 2026 14:11:13 -0300 Subject: [PATCH 131/134] refactor(test-runner): measure wall-clock time with Instant, not output parsing Replace the duration_millis parser (which scraped nextest's human-readable summary line) with Instant::now()/elapsed() around each run_partition call. The nextest display format is not a stable contract; the runner already owns the process lifecycle so it can time it directly. This also captures build time, which the nextest stamp did not include. --- tools/test-runner/src/bin/live-summary.rs | 88 ++++++----------------- 1 file changed, 23 insertions(+), 65 deletions(-) diff --git a/tools/test-runner/src/bin/live-summary.rs b/tools/test-runner/src/bin/live-summary.rs index 3af1d56ed..2ca93c47e 100644 --- a/tools/test-runner/src/bin/live-summary.rs +++ b/tools/test-runner/src/bin/live-summary.rs @@ -5,8 +5,8 @@ //! the `clientless` then `e2e` partition (each in its own CI container via its //! own `makers` task); `--all` first runs the `packages` set (`makers //! container-test`) and adds its row to the table. The runner streams each -//! run's output while capturing it, parses the nextest summary line (tallies -//! and wall-clock duration), and aggregates the totals. +//! run's output while capturing it, parses the nextest summary line for +//! tallies, and measures wall-clock duration via `Instant`. //! //! Unlike a cargo-make `dependencies` list (which is fail-fast), this runs BOTH //! partitions even when the first fails, so the summary reflects the whole @@ -20,6 +20,7 @@ use std::error::Error; use std::io::{BufRead, BufReader}; use std::process::{Command, Stdio}; +use std::time::{Duration, Instant}; /// Which partitions this invocation runs and summarizes. enum TestSet { @@ -36,8 +37,8 @@ struct Summary { passed: u64, failed: u64, skipped: u64, - /// Wall-clock milliseconds from the summary line's `[ 510.718s]` stamp. - millis: u64, + /// Wall-clock duration measured by the runner (includes build + test time). + elapsed: Duration, } impl Summary { @@ -47,7 +48,7 @@ impl Summary { passed: self.passed + other.passed, failed: self.failed + other.failed, skipped: self.skipped + other.skipped, - millis: self.millis + other.millis, + elapsed: self.elapsed + other.elapsed, } } } @@ -117,25 +118,6 @@ fn count_before(line: &str, marker: &str) -> u64 { head[head.len() - digit_count..].parse().unwrap_or(0) } -/// The wall-clock milliseconds from the summary line's `[ 510.718s]` stamp, -/// or 0 if absent. Parsed as integers (whole seconds and up to three -/// fractional digits) — no floating point. -fn duration_millis(line: &str) -> u64 { - let Some(idx) = line.find("s]") else { - return 0; - }; - let head = &line[..idx]; - let start = head - .rfind(|c: char| !(c.is_ascii_digit() || c == '.')) - .map_or(0, |i| i + 1); - let stamp = &head[start..]; - let (whole, frac) = stamp.split_once('.').unwrap_or((stamp, "")); - let whole: u64 = whole.parse().unwrap_or(0); - // Right-pad the fraction to exactly three digits ("7" -> 700ms). - let frac: u64 = format!("{frac:0<3.3}").parse().unwrap_or(0); - whole * 1000 + frac -} - /// Parse the last nextest summary line out of a captured run. /// /// nextest prints e.g.: @@ -156,12 +138,14 @@ fn parse_summary(log: &str) -> Summary { passed: count_before(&line, "passed"), failed: count_before(&line, "failed"), skipped: count_before(&line, "skipped"), - millis: duration_millis(&line), + ..Summary::default() } } fn print_row(label: &str, s: &Summary) { - let secs = format!("{}.{:03}", s.millis / 1000, s.millis % 1000); + let total_secs = s.elapsed.as_secs(); + let millis = s.elapsed.subsec_millis(); + let secs = format!("{total_secs}.{millis:03}"); println!( " {label:<18} {:>4} run, {:>4} passed, {:>4} failed, {:>4} skipped, {secs:>10}s", s.run, s.passed, s.failed, s.skipped @@ -179,20 +163,30 @@ fn main() -> Result<(), Box> { let packages = match set { TestSet::All => { println!(">>> all: running packages set"); + let start = Instant::now(); let (rc, log) = run_partition("container-test", with_zcashd)?; - Some((rc, parse_summary(&log))) + let elapsed = start.elapsed(); + let mut s = parse_summary(&log); + s.elapsed = elapsed; + Some((rc, s)) } TestSet::Live => None, }; println!(">>> live: running clientless partition"); + let cl_start = Instant::now(); let (cl_rc, cl_log) = run_partition("live-clientless", with_zcashd)?; + let cl_elapsed = cl_start.elapsed(); println!(">>> live: running e2e partition"); + let e2e_start = Instant::now(); let (e2e_rc, e2e_log) = run_partition("live-e2e", with_zcashd)?; + let e2e_elapsed = e2e_start.elapsed(); - let cl = parse_summary(&cl_log); - let e2e = parse_summary(&e2e_log); + let mut cl = parse_summary(&cl_log); + cl.elapsed = cl_elapsed; + let mut e2e = parse_summary(&e2e_log); + e2e.elapsed = e2e_elapsed; let header = match packages { Some(_) => "test summary", @@ -280,39 +274,3 @@ mod parse_summary { check(log, 9, 7, 1, 1); } } - -#[cfg(test)] -mod duration_millis { - use super::*; - - #[test] - fn fractional_seconds_with_padding() { - assert_eq!(duration_millis("Summary [ 73.207s] 8 tests run: 8 passed"), 73_207); - } - - #[test] - fn no_padding() { - assert_eq!(duration_millis("Summary [510.718s] 29 tests run: 23 passed"), 510_718); - } - - #[test] - fn short_fraction_is_right_padded() { - assert_eq!(duration_millis("Summary [2.7s] 1 test run: 1 passed"), 2_700); - } - - #[test] - fn whole_seconds_without_fraction() { - assert_eq!(duration_millis("Summary [2s] 1 test run: 1 passed"), 2_000); - } - - #[test] - fn missing_stamp_is_zero() { - assert_eq!(duration_millis("no summary line here"), 0); - } - - #[test] - fn survives_through_parse_summary() { - let s = parse_summary("Summary [ 1.795s] 1 test run: 0 passed, 1 failed, 114 skipped"); - assert_eq!(s.millis, 1_795); - } -} From dda8497c280ac7c2488f11c1e510131f41363ed5 Mon Sep 17 00:00:00 2001 From: nachog00 Date: Mon, 13 Jul 2026 15:00:33 -0300 Subject: [PATCH 132/134] fix(ci): make publish dry-run advisory for PRs targeting rc/stable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PRs that target rc/** or stable carry unbumped dev code by design — version bumps happen on the RC branch after merge. The blocking mode was failing these PRs for an expected, non-actionable condition. Only direct pushes to rc/** or stable are blocking now; PRs targeting them are advisory. --- .github/workflows/publish-dry-run.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/publish-dry-run.yml b/.github/workflows/publish-dry-run.yml index 8add6d556..f2fb2e948 100644 --- a/.github/workflows/publish-dry-run.yml +++ b/.github/workflows/publish-dry-run.yml @@ -13,13 +13,11 @@ jobs: publish-dry-run: name: cargo publish --dry-run runs-on: ubuntu-latest - # Blocking context (pushes to rc/** or stable, and PRs targeting them): - # this job MUST pass. Advisory context (everything else): failures exit 0 - # after emitting a warning annotation, so they do not paint a red X on - # PRs (a continue-on-error'd failing job still shows as failed in the PR - # checks list even though the run succeeds — hence no job-level - # continue-on-error here). The context is computed once in the - # "Compute check mode" step below. + # Blocking context: direct pushes to rc/** or stable — the branch is in + # release-prep state and must publish cleanly. Advisory context: everything + # else, including PRs that *target* rc/** or stable. Those PRs carry + # unbumped dev code; version bumps happen on the RC branch after merge, so + # a publish failure at PR time is expected, not actionable. steps: - name: Checkout repository uses: actions/checkout@v7 @@ -30,10 +28,12 @@ jobs: - name: Compute check mode id: mode run: | - if [[ "${{ github.event_name }}" == "pull_request" ]]; then - target="${{ github.base_ref }}" - else + if [[ "${{ github.event_name }}" == "push" ]]; then target="${{ github.ref_name }}" + else + # PRs (including those targeting rc/** or stable) are always + # advisory — they carry pre-bump code by design. + target="" fi if [[ "$target" == rc/* || "$target" == stable ]]; then echo "mode=blocking" >> "$GITHUB_OUTPUT" From 87ae07f5d288a21c9cfe3bc5413d0bfb40605fd9 Mon Sep 17 00:00:00 2001 From: nachog00 Date: Mon, 13 Jul 2026 16:53:30 -0300 Subject: [PATCH 133/134] build: bump crate versions for 0.6.0-rc.3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit zaino-common 0.2.0 → 0.4.0-rc.3 zaino-fetch 0.2.1 → 0.4.0-rc.3 zaino-proto 0.1.3 → 0.2.0-rc.3 zaino-serve 0.3.1 → 0.5.0-rc.3 zaino-state 0.3.1 → 0.5.0-rc.3 zainod 0.4.3-ironwood.1 → 0.6.0-rc.3 Workspace dependency version pins updated to match. --- Cargo.lock | 12 ++++++------ Cargo.toml | 10 +++++----- packages/zaino-common/Cargo.toml | 2 +- packages/zaino-fetch/Cargo.toml | 2 +- packages/zaino-proto/Cargo.toml | 2 +- packages/zaino-serve/Cargo.toml | 2 +- packages/zaino-state/Cargo.toml | 2 +- packages/zainod/Cargo.toml | 2 +- 8 files changed, 17 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 798fd33aa..140328fc7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5885,7 +5885,7 @@ dependencies = [ [[package]] name = "zaino-common" -version = "0.2.0" +version = "0.4.0-rc.3" dependencies = [ "rustls", "serde", @@ -5898,7 +5898,7 @@ dependencies = [ [[package]] name = "zaino-fetch" -version = "0.2.1" +version = "0.4.0-rc.3" dependencies = [ "base64", "derive_more", @@ -5925,7 +5925,7 @@ dependencies = [ [[package]] name = "zaino-proto" -version = "0.1.3" +version = "0.2.0-rc.3" dependencies = [ "prost", "tonic", @@ -5938,7 +5938,7 @@ dependencies = [ [[package]] name = "zaino-serve" -version = "0.3.1" +version = "0.5.0-rc.3" dependencies = [ "futures", "jsonrpsee", @@ -5959,7 +5959,7 @@ dependencies = [ [[package]] name = "zaino-state" -version = "0.3.1" +version = "0.5.0-rc.3" dependencies = [ "arc-swap", "bitflags 2.13.0", @@ -6031,7 +6031,7 @@ dependencies = [ [[package]] name = "zainod" -version = "0.4.3-ironwood.1" +version = "0.6.0-rc.3" dependencies = [ "clap", "config", diff --git a/Cargo.toml b/Cargo.toml index e13a29ac6..cdd30c168 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -122,15 +122,15 @@ tempfile = "3.2" # no longer need `default-features = false` to drop it: their default feature # set is already empty. Enable zcashd end-to-end with `--features zcashd_support`. zainod = { path = "packages/zainod" } -zaino-common = { path = "packages/zaino-common", version = "0.2.0" } +zaino-common = { path = "packages/zaino-common", version = "0.4.0-rc.3" } # default-features = false so the default-on `zcashd_support` feature can be # dropped end-to-end via `--no-default-features`; each consumer re-enables it # by forwarding `zcashd_support` in its own `default`. See # docs/adr/0001-zcashd-support-feature-gate.md. -zaino-fetch = { path = "packages/zaino-fetch", version = "0.2.1", default-features = false } -zaino-proto = { path = "packages/zaino-proto", version = "0.1.3" } -zaino-state = { path = "packages/zaino-state", version = "0.3.1", default-features = false } -zaino-serve = { path = "packages/zaino-serve", version = "0.3.1", default-features = false } +zaino-fetch = { path = "packages/zaino-fetch", version = "0.4.0-rc.3", default-features = false } +zaino-proto = { path = "packages/zaino-proto", version = "0.2.0-rc.3" } +zaino-state = { path = "packages/zaino-state", version = "0.5.0-rc.3", default-features = false } +zaino-serve = { path = "packages/zaino-serve", version = "0.5.0-rc.3", default-features = false } zaino-testutils = { path = "live-tests/zaino-testutils" } # Zingo-infra (validator launcher driving the live tests; not a prod dep). diff --git a/packages/zaino-common/Cargo.toml b/packages/zaino-common/Cargo.toml index b3f12bad7..708c5bb04 100644 --- a/packages/zaino-common/Cargo.toml +++ b/packages/zaino-common/Cargo.toml @@ -6,7 +6,7 @@ repository = { workspace = true } homepage = { workspace = true } edition = { workspace = true } license = { workspace = true } -version = "0.2.0" +version = "0.4.0-rc.3" [dependencies] # Zebra diff --git a/packages/zaino-fetch/Cargo.toml b/packages/zaino-fetch/Cargo.toml index 6d912aa54..94fe46925 100644 --- a/packages/zaino-fetch/Cargo.toml +++ b/packages/zaino-fetch/Cargo.toml @@ -6,7 +6,7 @@ repository = { workspace = true } homepage = { workspace = true } edition = { workspace = true } license = { workspace = true } -version = "0.2.1" +version = "0.4.0-rc.3" [features] default = [] diff --git a/packages/zaino-proto/Cargo.toml b/packages/zaino-proto/Cargo.toml index 92676dc62..6f8e7b6b9 100644 --- a/packages/zaino-proto/Cargo.toml +++ b/packages/zaino-proto/Cargo.toml @@ -6,7 +6,7 @@ repository = { workspace = true } homepage = { workspace = true } edition = { workspace = true } license = { workspace = true } -version = "0.1.3" +version = "0.2.0-rc.3" [features] default = ["heavy"] diff --git a/packages/zaino-serve/Cargo.toml b/packages/zaino-serve/Cargo.toml index c421c8a4e..7eea25c62 100644 --- a/packages/zaino-serve/Cargo.toml +++ b/packages/zaino-serve/Cargo.toml @@ -6,7 +6,7 @@ repository = { workspace = true } homepage = { workspace = true } edition = { workspace = true } license = { workspace = true } -version = "0.3.1" +version = "0.5.0-rc.3" [features] default = [] diff --git a/packages/zaino-state/Cargo.toml b/packages/zaino-state/Cargo.toml index 0f60eab31..e73c1204b 100644 --- a/packages/zaino-state/Cargo.toml +++ b/packages/zaino-state/Cargo.toml @@ -6,7 +6,7 @@ repository = { workspace = true } homepage = { workspace = true } edition = { workspace = true } license = { workspace = true } -version = "0.3.1" +version = "0.5.0-rc.3" [features] default = [] diff --git a/packages/zainod/Cargo.toml b/packages/zainod/Cargo.toml index beba27ad9..dd9de9117 100644 --- a/packages/zainod/Cargo.toml +++ b/packages/zainod/Cargo.toml @@ -6,7 +6,7 @@ repository = { workspace = true } homepage = { workspace = true } edition = { workspace = true } license = { workspace = true } -version = "0.4.3-ironwood.1" +version = "0.6.0-rc.3" [[bin]] name = "zainod" From 3d36756da0d941a644fb507d387a34fef8186fb3 Mon Sep 17 00:00:00 2001 From: nachog00 Date: Mon, 13 Jul 2026 17:06:05 -0300 Subject: [PATCH 134/134] build: merge rc/0.6.0 and bump crate versions for 0.6.0 release MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Merge rc/0.6.0 into stable and bump all crate versions to their final stable form: zaino-common 0.4.0-rc.3 → 0.4.0 zaino-fetch 0.4.0-rc.3 → 0.4.0 zaino-proto 0.2.0-rc.3 → 0.2.0 zaino-serve 0.5.0-rc.3 → 0.5.0 zaino-state 0.5.0-rc.3 → 0.5.0 zainod 0.6.0-rc.3 → 0.6.0 Resolves stable/rc divergence from the 0.5.1 hotfix commits that were never backported to dev (zebra git patch removal, version bumps). --- Cargo.lock | 272 +++++++----------- Cargo.toml | 10 +- packages/zaino-common/Cargo.toml | 2 +- packages/zaino-fetch/Cargo.toml | 2 +- packages/zaino-proto/Cargo.toml | 2 +- packages/zaino-serve/Cargo.toml | 2 +- packages/zaino-state/Cargo.toml | 2 +- .../chain_index/source/validator_connector.rs | 6 +- packages/zainod/Cargo.toml | 3 +- 9 files changed, 120 insertions(+), 181 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 99423747e..5b75d7196 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -491,9 +491,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.12.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "bzip2-sys" @@ -516,9 +516,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.65" +version = "1.2.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" +checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" dependencies = [ "find-msvc-tools", "jobserver", @@ -880,18 +880,18 @@ checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" [[package]] name = "crossbeam-channel" -version = "0.5.15" +version = "0.5.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-deque" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" dependencies = [ "crossbeam-epoch", "crossbeam-utils", @@ -899,18 +899,18 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.18" +version = "0.9.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.21" +version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" [[package]] name = "crunchy" @@ -1355,7 +1355,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "835c052cb0c08c1acf6ffd71c022172e18723949c8282f2b9f27efbc51e64534" dependencies = [ "byteorder", - "rand 0.8.6", + "rand 0.8.7", "rustc-hex", "static_assertions", ] @@ -1598,11 +1598,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", - "js-sys", "libc", "r-efi 5.3.0", "wasip2", - "wasm-bindgen", ] [[package]] @@ -1612,9 +1610,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 6.0.0", "rand_core 0.10.1", + "wasm-bindgen", ] [[package]] @@ -1685,7 +1685,7 @@ dependencies = [ "halo2_proofs", "lazy_static", "pasta_curves", - "rand 0.8.6", + "rand 0.8.7", "sinsemilla", "subtle", "uint 0.9.5", @@ -1850,9 +1850,9 @@ dependencies = [ [[package]] name = "http-body" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" dependencies = [ "bytes", "http", @@ -1860,9 +1860,9 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" dependencies = [ "bytes", "futures-core", @@ -2291,11 +2291,11 @@ dependencies = [ [[package]] name = "jobserver" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" dependencies = [ - "getrandom 0.3.4", + "getrandom 0.4.3", "libc", ] @@ -2338,7 +2338,7 @@ dependencies = [ "http-body-util", "jsonrpsee-types", "parking_lot", - "rand 0.8.6", + "rand 0.8.7", "rustc-hash", "serde", "serde_json", @@ -2637,9 +2637,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.8.2" +version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "memuse" @@ -2689,7 +2689,7 @@ dependencies = [ "hashbrown 0.16.1", "metrics", "quanta", - "rand 0.9.4", + "rand 0.9.5", "rand_xoshiro", "rapidhash", "sketches-ddsketch", @@ -2718,9 +2718,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" dependencies = [ "libc", "wasi 0.11.1+wasi-snapshot-preview1", @@ -2778,9 +2778,9 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c863e9ab5e7bf9c99ba75e1050f1e4d624ae87ed3532d6238ffbdc7b585dbbe6" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" dependencies = [ "num-integer", "num-traits", @@ -2926,7 +2926,7 @@ dependencies = [ "memuse", "nonempty", "pasta_curves", - "rand 0.8.6", + "rand 0.8.7", "rand_core 0.6.4", "reddsa", "serde", @@ -3025,7 +3025,7 @@ dependencies = [ "ff", "group", "lazy_static", - "rand 0.8.6", + "rand 0.8.7", "static_assertions", "subtle", ] @@ -3261,7 +3261,7 @@ dependencies = [ "bit-vec", "bitflags 2.13.0", "num-traits", - "rand 0.9.4", + "rand 0.9.5", "rand_chacha 0.9.0", "rand_xorshift", "regex-syntax", @@ -3436,14 +3436,15 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.15" +version = "0.11.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e" +checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" dependencies = [ "bytes", - "getrandom 0.3.4", + "getrandom 0.4.3", "lru-slab", - "rand 0.9.4", + "rand 0.10.2", + "rand_pcg", "ring", "rustc-hash", "rustls", @@ -3457,16 +3458,16 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.14" +version = "0.5.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" dependencies = [ "cfg_aliases", "libc", "once_cell", "socket2", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -3511,9 +3512,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" dependencies = [ "libc", "rand_chacha 0.3.1", @@ -3522,9 +3523,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.5", @@ -3613,6 +3614,15 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + [[package]] name = "rand_xorshift" version = "0.4.0" @@ -3633,9 +3643,9 @@ dependencies = [ [[package]] name = "rapidhash" -version = "4.5.0" +version = "4.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1a224897b65b7ce38bf7b0adb569e7d77e11460d0cc3577908b263d8b5a89aa" +checksum = "5da7e78a036ce858e8d55b7e7dc8ba3a88b78350fd2155d3591bbd966b58589e" dependencies = [ "rustversion", ] @@ -3743,9 +3753,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.12.4" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" +checksum = "2a0e75113e14dc5acb068cd0786884f214f1312650a3d36d269f5c4f3cdee8a2" dependencies = [ "aho-corasick", "memchr", @@ -3755,9 +3765,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.14" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +checksum = "1f388202e4b80542a0921078cc23b6333bcf1409c1e3f86404cae4766a6131db" dependencies = [ "aho-corasick", "memchr", @@ -3904,13 +3914,13 @@ checksum = "afab94fb28594581f62d981211a9a4d53cc8130bbcbbb89a0440d9b8e81a7746" [[package]] name = "rustc-demangle" -version = "0.1.27" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" +checksum = "b74b56ffa8bb2830709a538c2cbcae9aa062db0d2a42563bfb09bdaae44020eb" [[package]] name = "rustc-hash" -version = "2.1.2" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" @@ -3944,9 +3954,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.41" +version = "0.23.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" +checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138" dependencies = [ "aws-lc-rs", "log", @@ -4021,9 +4031,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" [[package]] name = "rusty-fork" @@ -4075,7 +4085,7 @@ dependencies = [ "jubjub", "lazy_static", "memuse", - "rand 0.8.6", + "rand 0.8.7", "rand_core 0.6.4", "redjubjub", "subtle", @@ -4329,9 +4339,9 @@ dependencies = [ [[package]] name = "sha1" -version = "0.10.6" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" dependencies = [ "cfg-if", "cpufeatures 0.2.17", @@ -4449,9 +4459,9 @@ checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" [[package]] name = "socket2" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ "libc", "windows-sys 0.61.2", @@ -4469,7 +4479,7 @@ dependencies = [ "http", "httparse", "log", - "rand 0.8.6", + "rand 0.8.7", "sha1", ] @@ -4497,9 +4507,9 @@ dependencies = [ [[package]] name = "spin" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" dependencies = [ "lock_api", ] @@ -4662,9 +4672,9 @@ dependencies = [ [[package]] name = "thread_local" -version = "1.1.9" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" dependencies = [ "cfg-if", ] @@ -4711,9 +4721,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" dependencies = [ "tinyvec_macros", ] @@ -5602,15 +5612,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "windows-sys" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" -dependencies = [ - "windows-targets 0.53.5", -] - [[package]] name = "windows-sys" version = "0.61.2" @@ -5644,30 +5645,13 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", + "windows_i686_gnullvm", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] -[[package]] -name = "windows-targets" -version = "0.53.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" -dependencies = [ - "windows-link", - "windows_aarch64_gnullvm 0.53.1", - "windows_aarch64_msvc 0.53.1", - "windows_i686_gnu 0.53.1", - "windows_i686_gnullvm 0.53.1", - "windows_i686_msvc 0.53.1", - "windows_x86_64_gnu 0.53.1", - "windows_x86_64_gnullvm 0.53.1", - "windows_x86_64_msvc 0.53.1", -] - [[package]] name = "windows_aarch64_gnullvm" version = "0.42.2" @@ -5680,12 +5664,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" - [[package]] name = "windows_aarch64_msvc" version = "0.42.2" @@ -5698,12 +5676,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" - [[package]] name = "windows_i686_gnu" version = "0.42.2" @@ -5716,24 +5688,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" -[[package]] -name = "windows_i686_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" - [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" - [[package]] name = "windows_i686_msvc" version = "0.42.2" @@ -5746,12 +5706,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_i686_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" - [[package]] name = "windows_x86_64_gnu" version = "0.42.2" @@ -5764,12 +5718,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" - [[package]] name = "windows_x86_64_gnullvm" version = "0.42.2" @@ -5782,12 +5730,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" - [[package]] name = "windows_x86_64_msvc" version = "0.42.2" @@ -5800,17 +5742,11 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" - [[package]] name = "winnow" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" dependencies = [ "memchr", ] @@ -5885,7 +5821,7 @@ dependencies = [ [[package]] name = "zaino-common" -version = "0.4.0-rc.3" +version = "0.4.0" dependencies = [ "rustls", "serde", @@ -5898,7 +5834,7 @@ dependencies = [ [[package]] name = "zaino-fetch" -version = "0.4.0-rc.3" +version = "0.4.0" dependencies = [ "base64", "derive_more", @@ -5925,7 +5861,7 @@ dependencies = [ [[package]] name = "zaino-proto" -version = "0.2.0-rc.3" +version = "0.2.0" dependencies = [ "prost", "tonic", @@ -5938,7 +5874,7 @@ dependencies = [ [[package]] name = "zaino-serve" -version = "0.5.0-rc.3" +version = "0.5.0" dependencies = [ "futures", "jsonrpsee", @@ -5959,7 +5895,7 @@ dependencies = [ [[package]] name = "zaino-state" -version = "0.5.0-rc.3" +version = "0.5.0" dependencies = [ "arc-swap", "bitflags 2.13.0", @@ -5976,7 +5912,7 @@ dependencies = [ "metrics", "primitive-types 0.14.0", "proptest", - "rand 0.10.1", + "rand 0.10.2", "reqwest 0.13.1", "sapling-crypto", "serde", @@ -6031,7 +5967,7 @@ dependencies = [ [[package]] name = "zainod" -version = "0.6.0-rc.3" +version = "0.6.0" dependencies = [ "clap", "config", @@ -6137,9 +6073,9 @@ dependencies = [ [[package]] name = "zcash_note_encryption" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77efec759c3798b6e4d829fcc762070d9b229b0f13338c40bf993b7b609c2272" +checksum = "e1cb1b9170c94370e3d66c5cc0877661db743337588b64de7711239eed462198" dependencies = [ "chacha20 0.9.1", "chacha20poly1305", @@ -6300,7 +6236,7 @@ dependencies = [ "primitive-types 0.12.2", "proptest", "proptest-derive", - "rand 0.8.6", + "rand 0.8.7", "rand_chacha 0.3.1", "rand_core 0.6.4", "rayon", @@ -6356,7 +6292,7 @@ dependencies = [ "mset", "once_cell", "orchard", - "rand 0.8.6", + "rand 0.8.7", "rayon", "sapling-crypto", "serde", @@ -6399,7 +6335,7 @@ dependencies = [ "num-integer", "ordered-map", "pin-project", - "rand 0.8.6", + "rand 0.8.7", "rayon", "regex", "schemars 1.2.1", @@ -6458,7 +6394,7 @@ dependencies = [ "orchard", "phf", "prost", - "rand 0.8.6", + "rand 0.8.7", "sapling-crypto", "schemars 1.2.1", "semver", @@ -6500,7 +6436,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7c967badb1bb99573dba430948c98fbca940daf23915f2e9736e1a1d2a166f3" dependencies = [ "libzcash_script", - "rand 0.8.6", + "rand 0.8.7", "thiserror 2.0.18", "zcash_primitives", "zcash_script", @@ -6563,7 +6499,7 @@ dependencies = [ "once_cell", "owo-colors", "proptest", - "rand 0.8.6", + "rand 0.8.7", "regex", "spandoc", "thiserror 2.0.18", @@ -6576,18 +6512,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.52" +version = "0.8.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" +checksum = "b7cbbc0a705a0fd05cc3676525980d2bf5a9bc4adac6d6475209a7887cf59d19" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.52" +version = "0.8.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" +checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5" dependencies = [ "proc-macro2", "quote", @@ -6693,6 +6629,6 @@ dependencies = [ [[package]] name = "zmij" -version = "1.0.21" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/Cargo.toml b/Cargo.toml index cdd30c168..e49ef17fb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -122,15 +122,15 @@ tempfile = "3.2" # no longer need `default-features = false` to drop it: their default feature # set is already empty. Enable zcashd end-to-end with `--features zcashd_support`. zainod = { path = "packages/zainod" } -zaino-common = { path = "packages/zaino-common", version = "0.4.0-rc.3" } +zaino-common = { path = "packages/zaino-common", version = "0.4.0" } # default-features = false so the default-on `zcashd_support` feature can be # dropped end-to-end via `--no-default-features`; each consumer re-enables it # by forwarding `zcashd_support` in its own `default`. See # docs/adr/0001-zcashd-support-feature-gate.md. -zaino-fetch = { path = "packages/zaino-fetch", version = "0.4.0-rc.3", default-features = false } -zaino-proto = { path = "packages/zaino-proto", version = "0.2.0-rc.3" } -zaino-state = { path = "packages/zaino-state", version = "0.5.0-rc.3", default-features = false } -zaino-serve = { path = "packages/zaino-serve", version = "0.5.0-rc.3", default-features = false } +zaino-fetch = { path = "packages/zaino-fetch", version = "0.4.0", default-features = false } +zaino-proto = { path = "packages/zaino-proto", version = "0.2.0" } +zaino-state = { path = "packages/zaino-state", version = "0.5.0", default-features = false } +zaino-serve = { path = "packages/zaino-serve", version = "0.5.0", default-features = false } zaino-testutils = { path = "live-tests/zaino-testutils" } # Zingo-infra (validator launcher driving the live tests; not a prod dep). diff --git a/packages/zaino-common/Cargo.toml b/packages/zaino-common/Cargo.toml index 708c5bb04..2ac216842 100644 --- a/packages/zaino-common/Cargo.toml +++ b/packages/zaino-common/Cargo.toml @@ -6,7 +6,7 @@ repository = { workspace = true } homepage = { workspace = true } edition = { workspace = true } license = { workspace = true } -version = "0.4.0-rc.3" +version = "0.4.0" [dependencies] # Zebra diff --git a/packages/zaino-fetch/Cargo.toml b/packages/zaino-fetch/Cargo.toml index 94fe46925..e835aab94 100644 --- a/packages/zaino-fetch/Cargo.toml +++ b/packages/zaino-fetch/Cargo.toml @@ -6,7 +6,7 @@ repository = { workspace = true } homepage = { workspace = true } edition = { workspace = true } license = { workspace = true } -version = "0.4.0-rc.3" +version = "0.4.0" [features] default = [] diff --git a/packages/zaino-proto/Cargo.toml b/packages/zaino-proto/Cargo.toml index 6f8e7b6b9..27b6df8c6 100644 --- a/packages/zaino-proto/Cargo.toml +++ b/packages/zaino-proto/Cargo.toml @@ -6,7 +6,7 @@ repository = { workspace = true } homepage = { workspace = true } edition = { workspace = true } license = { workspace = true } -version = "0.2.0-rc.3" +version = "0.2.0" [features] default = ["heavy"] diff --git a/packages/zaino-serve/Cargo.toml b/packages/zaino-serve/Cargo.toml index 7eea25c62..d33624cd1 100644 --- a/packages/zaino-serve/Cargo.toml +++ b/packages/zaino-serve/Cargo.toml @@ -6,7 +6,7 @@ repository = { workspace = true } homepage = { workspace = true } edition = { workspace = true } license = { workspace = true } -version = "0.5.0-rc.3" +version = "0.5.0" [features] default = [] diff --git a/packages/zaino-state/Cargo.toml b/packages/zaino-state/Cargo.toml index e73c1204b..058519476 100644 --- a/packages/zaino-state/Cargo.toml +++ b/packages/zaino-state/Cargo.toml @@ -6,7 +6,7 @@ repository = { workspace = true } homepage = { workspace = true } edition = { workspace = true } license = { workspace = true } -version = "0.5.0-rc.3" +version = "0.5.0" [features] default = [] diff --git a/packages/zaino-state/src/chain_index/source/validator_connector.rs b/packages/zaino-state/src/chain_index/source/validator_connector.rs index b616aca0f..e2130c218 100644 --- a/packages/zaino-state/src/chain_index/source/validator_connector.rs +++ b/packages/zaino-state/src/chain_index/source/validator_connector.rs @@ -1711,7 +1711,11 @@ impl BlockchainSource for ValidatorConnector { }) => { match read_state_service .clone() - .call(zebra_state::ReadRequest::NonFinalizedBlocksListener) + // Empty `known_chain_tips` requests every block currently in the + // non-finalized state (the prior unit-variant behaviour). + .call(zebra_state::ReadRequest::NonFinalizedBlocksListener { + known_chain_tips: Default::default(), + }) .await { Ok(ReadResponse::NonFinalizedBlocksListener(listener)) => { diff --git a/packages/zainod/Cargo.toml b/packages/zainod/Cargo.toml index 5ad6d00d6..4e2c7d501 100644 --- a/packages/zainod/Cargo.toml +++ b/packages/zainod/Cargo.toml @@ -1,13 +1,12 @@ [package] name = "zainod" -version = "0.5.1" +version = "0.6.0" description = "Crate containing the Zaino Indexer binary." authors = { workspace = true } repository = { workspace = true } homepage = { workspace = true } edition = { workspace = true } license = { workspace = true } -version = "0.6.0-rc.3" [[bin]] name = "zainod"