From 8780e94aec38a3de8ea0249ab7fe5afcc301c1ac Mon Sep 17 00:00:00 2001 From: zancas Date: Wed, 1 Jul 2026 21:13:33 -0700 Subject: [PATCH 01/58] Rename `makers test` set `package` -> `packages` The front-door test set is named for the `packages/` directory it runs, so the plural is the faithful name. Hard-cut: `makers test package` now errors with `unknown set` rather than aliasing. Updates every reference site: Makefile.toml (default, case matcher, dispatch, descriptions, comment block), help.sh, docs/testing.md, README.md, and the live-tests/CONTEXT.md glossary term (now records the retired singular under _Avoid_). ADR-0004 gets a one-line supersession footer; its historical text is preserved. Co-Authored-By: Claude Opus 4.8 (1M context) --- Makefile.toml | 32 +++++++++---------- README.md | 2 +- ...ame-integration-partition-to-clientless.md | 3 ++ docs/testing.md | 6 ++-- live-tests/CONTEXT.md | 12 ++++--- tools/scripts/help.sh | 8 ++--- 6 files changed, 34 insertions(+), 29 deletions(-) diff --git a/Makefile.toml b/Makefile.toml index f38a60907..3595ca15a 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -93,7 +93,7 @@ script.main = 'source "./tools/scripts/push-image.sh"' [tasks.container-test] clear = true -description = "Engine: run nextest in the local CI container, forwarding args (zcashd_support OFF by default; --with-zcashd adds --features zcashd_support). Devs use `makers test [package|e2e|clientless|live|all]`; live-clientless / live-e2e and the `test` front door call this." +description = "Engine: run nextest in the local CI container, forwarding args (zcashd_support OFF by default; --with-zcashd adds --features zcashd_support). Devs use `makers test [packages|e2e|clientless|live|all]`; live-clientless / live-e2e and the `test` front door call this." # This task runs tests inside the container built from live-tests/test_environment. # The entrypoint.sh script in the container sets up test binaries (zcashd, zebrad, zcash-cli) # by creating symlinks from /home/container_user/artifacts to the expected live-tests/test_binaries/bins location. @@ -132,16 +132,16 @@ script.main = 'source "./tools/scripts/container-nextest-workspace.sh"' # =================================================================== # Developer-facing test front door: a single `makers test [SET]` task. # -# makers test packaged (the default) -# makers test package packages/* production-crate tests (no validator) +# makers test packages (the default) +# makers test packages packages/* production-crate tests (no validator) # makers test e2e the e2e live partition # makers test clientless the clientless live partition # makers test live both live partitions + combined summary -# makers test all package then live (everything) +# makers test all packages then live (everything) # -# SET names mirror the crate/dir names (`e2e`, `clientless`); `package`, -# `live`, and `all` are the groupings. `live` = all non-package tests; `all` -# = `package` + `live` (see live-tests/CONTEXT.md). Each set delegates to an +# SET names mirror the crate/dir names (`e2e`, `clientless`); `packages`, +# `live`, and `all` are the groupings. `live` = all non-packages tests; `all` +# = `packages` + `live` (see live-tests/CONTEXT.md). Each set delegates to an # existing engine rather than re-implementing a run. `--with-zcashd` (handled # by the engines) adds the zcashd-backed tests to any set. # =================================================================== @@ -150,33 +150,33 @@ script.main = 'source "./tools/scripts/container-nextest-workspace.sh"' # clear = true fully replaces cargo-make's built-in `test` task (which runs # `cargo test`); without it our `script` would merge with the builtin `command`. clear = true -description = "Run a test set: `makers test [package|e2e|clientless|live|all]` (default: package). Pass --with-zcashd for the zcashd-backed tests." +description = "Run a test set: `makers test [packages|e2e|clientless|live|all]` (default: packages). Pass --with-zcashd for the zcashd-backed tests." script_runner = "bash" script = ''' set -uo pipefail usage() { - echo "usage: makers test [package|e2e|clientless|live|all] [-- nextest args]" >&2 - echo " package (default) packages/* tests, no live validator" >&2 + echo "usage: makers test [packages|e2e|clientless|live|all] [-- nextest args]" >&2 + echo " packages (default) packages/* tests, no live validator" >&2 echo " e2e | clientless that single live partition" >&2 echo " live both live partitions + combined summary" >&2 - echo " all package then live (everything)" >&2 + echo " all packages then live (everything)" >&2 } # A non-flag first arg selects the set and must be a known name; a flag first -# arg (or none) keeps the default `package` and is forwarded to nextest. So -# `makers test --with-zcashd` runs package with the flag, while a mistyped set +# arg (or none) keeps the default `packages` and is forwarded to nextest. So +# `makers test --with-zcashd` runs packages with the flag, while a mistyped set # name errors instead of silently running the default. -set="package" +set="packages" if [ "$#" -gt 0 ] && [ "${1#-}" = "$1" ]; then case "$1" in - package|e2e|clientless|live|all) set="$1"; shift ;; + packages|e2e|clientless|live|all) set="$1"; shift ;; *) echo "makers test: unknown set '$1'" >&2; usage; exit 2 ;; esac fi case "$set" in - package) exec makers container-test "$@" ;; + packages) exec makers container-test "$@" ;; e2e) exec makers live-e2e "$@" ;; clientless) exec makers live-clientless "$@" ;; live) exec cargo run -q --manifest-path tools/test-runner/Cargo.toml --bin live-summary -- "$@" ;; diff --git a/README.md b/README.md index a72a9a495..0b8ca92f4 100644 --- a/README.md +++ b/README.md @@ -106,7 +106,7 @@ The test suites run inside a **podman** container via `makers` (cargo-make): ```sh makers test # packages/* tests that need no live validator (default) makers test live # both live partitions (clientless + e2e) + combined summary -makers test all # everything: package then live +makers test all # everything: packages then live ``` zcashd-backed tests are **off by default**; add `--with-zcashd` to include them diff --git a/docs/adr/0004-rename-integration-partition-to-clientless.md b/docs/adr/0004-rename-integration-partition-to-clientless.md index c98de87d4..cb80fbfa7 100644 --- a/docs/adr/0004-rename-integration-partition-to-clientless.md +++ b/docs/adr/0004-rename-integration-partition-to-clientless.md @@ -50,3 +50,6 @@ unchanged. "clientless" is no longer avoided. - ADR-0003's naming bullet and its three-front-door task surface are superseded; its two-crate split and the `live` umbrella decision still stand. + +> **Later note:** the `package` set was renamed `packages` (plural, matching the +> `packages/` dir); `makers test package` now errors. See `live-tests/CONTEXT.md`. diff --git a/docs/testing.md b/docs/testing.md index f5c6d8f10..9bfd24575 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -26,14 +26,14 @@ The `makers` tasks below build and run the test suites inside a **podman** container, so you don't need the validator binaries on your host `$PATH`. The container image is built or pulled automatically on first run. -The single front door is `makers test [SET]`, where `SET` defaults to `package`: +The single front door is `makers test [SET]`, where `SET` defaults to `packages`: -- `makers test` (or `makers test package`) — the **package** set: the +- `makers test` (or `makers test packages`) — the **packages** set: the `packages/*` production-crate tests that need no live validator. - `makers test e2e` / `makers test clientless` — one live partition. - `makers test live` — both live partitions (`clientless` then `e2e`) against a live validator, with a combined pass/fail summary. -- `makers test all` — the whole suite: package then live. +- `makers test all` — the whole suite: packages then live. (`container-test`, `live-clientless`, and `live-e2e` are the internal engines the `test` front door delegates to; invoke them directly only when you need to diff --git a/live-tests/CONTEXT.md b/live-tests/CONTEXT.md index d00e0f064..1bef4076e 100644 --- a/live-tests/CONTEXT.md +++ b/live-tests/CONTEXT.md @@ -40,15 +40,17 @@ owns shared fixtures and the feature-forwarding surface (`zcashd_support`, `transparent_address_history_experimental`). _Avoid_: test helpers, testlib. -**package (test set)**: +**packages (test set)**: The tests of the `packages/*` production crates — the workspace `default-members`, exercised by bare `cargo nextest run` and by `makers test` -(the default with no argument, equivalently `makers test package`). Named for +(the default with no argument, equivalently `makers test packages`). Named for where the crates live (`packages/`); the functional reason they run apart from the live partitions is that they need **no** live validator. They are *not* network-free, though: e.g. zaino-serve's gRPC `spawn` regression test binds a loopback socket and stands up a tonic server. The absent ingredient is a validator, not the network. -_Avoid_: offline (overclaims "no network" — these tests bind loopback sockets); -unit test (the set includes crate-level integration tests); container test -(names the run mechanism, which the live suite shares). +_Avoid_: package (singular — the set is plural, matching the `packages/` dir; +`makers test package` now errors with `unknown set`); offline (overclaims "no +network" — these tests bind loopback sockets); unit test (the set includes +crate-level integration tests); container test (names the run mechanism, which +the live suite shares). diff --git a/tools/scripts/help.sh b/tools/scripts/help.sh index 524c80b6c..7f962451c 100755 --- a/tools/scripts/help.sh +++ b/tools/scripts/help.sh @@ -12,7 +12,7 @@ echo "" echo "Common usage:" echo " makers test # packages/* tests, no live validator (default)" echo " makers test live # both live partitions + combined summary" -echo " makers test all # everything: package then live" +echo " makers test all # everything: packages then live" echo "" echo "If you modify '.env.testing-artifacts', the test command will \ automatically:" @@ -21,15 +21,15 @@ echo " - Build a new local container image if needed" echo "" echo "Available commands:" echo "" -echo " test [SET] Front door. SET = package (default) | e2e | \ +echo " test [SET] Front door. SET = packages (default) | e2e | \ clientless | live | all" -echo " package packages/* tests, no live \ +echo " packages packages/* tests, no live \ validator" echo " e2e the e2e live partition" echo " clientless the clientless live partition" echo " live both live partitions + \ combined summary" -echo " all package then live (everything)" +echo " all packages then live (everything)" echo " container-test Engine: run nextest in the container \ (used by the front door; invoke directly to forward engine flags)" echo " live-clientless Engine: run the clientless live-test \ From 9923aaa0680b3c1ef72c9ccaabcc43df79cd627a Mon Sep 17 00:00:00 2001 From: idky137 Date: Thu, 2 Jul 2026 15:34:12 +0100 Subject: [PATCH 02/58] added ChainIndexRpcExt trait to split functionality not required by zallet --- packages/zaino-state/src/backends/fetch.rs | 2 +- packages/zaino-state/src/backends/state.rs | 2 +- packages/zaino-state/src/chain_index.rs | 1569 +++++++++-------- .../src/chain_index/tests/mockchain_tests.rs | 2 +- packages/zaino-state/src/lib.rs | 4 +- 5 files changed, 804 insertions(+), 775 deletions(-) diff --git a/packages/zaino-state/src/backends/fetch.rs b/packages/zaino-state/src/backends/fetch.rs index 058004802..74f63dfcc 100644 --- a/packages/zaino-state/src/backends/fetch.rs +++ b/packages/zaino-state/src/backends/fetch.rs @@ -76,7 +76,7 @@ use crate::{ }; use crate::{ chain_index::{non_finalised_state::ChainIndexSnapshot, NonFinalizedSnapshot}, - ChainIndex, NodeBackedChainIndex, NodeBackedChainIndexSubscriber, + ChainIndex, ChainIndexRpcExt, NodeBackedChainIndex, NodeBackedChainIndexSubscriber, }; /// Chain fetch service backed by Zcashd's JsonRPC engine. diff --git a/packages/zaino-state/src/backends/state.rs b/packages/zaino-state/src/backends/state.rs index 62945c1df..5617ce9e2 100644 --- a/packages/zaino-state/src/backends/state.rs +++ b/packages/zaino-state/src/backends/state.rs @@ -5,7 +5,7 @@ use crate::{ chain_index::{ mempool::{Mempool, MempoolSubscriber}, source::ValidatorConnector, - types as chain_types, ChainIndex, + types as chain_types, ChainIndex, ChainIndexRpcExt, }, config::{DonationAddress, StateServiceConfig}, error::{BlockCacheError, StateServiceError}, diff --git a/packages/zaino-state/src/chain_index.rs b/packages/zaino-state/src/chain_index.rs index 182fd6107..f5a005b76 100644 --- a/packages/zaino-state/src/chain_index.rs +++ b/packages/zaino-state/src/chain_index.rs @@ -315,6 +315,14 @@ fn branch_len_to_active_chain( /// /// When a call asks for info (e.g. a block), Zaino selects sources in this order: #[doc = simple_mermaid::mermaid!("chain_index_passthrough.mmd")] +/// +/// This trait holds the core methods required by the embedded wallet consumer +/// (zallet). RPC-server-only methods live on the [`ChainIndexRpcExt`] extension +/// trait. +/// +/// TODO: The core/extension split is a provisional first pass. It should be refined +/// into finer capability-based traits (zallet / lwd / block-explorer) in a follow-up +/// PR. pub trait ChainIndex { /// A snapshot of the nonfinalized state, needed for atomic access type Snapshot: NonFinalizedSnapshot; @@ -387,50 +395,6 @@ pub trait ChainIndex { end: Option, ) -> Option, Self::Error>>>; - /// Returns the *compact* block for the given height. - /// - /// Returns `None` if the specified `height` is greater than the snapshot's tip. - /// - /// ## Pool filtering - /// - /// - `pool_types` controls which per-transaction components are populated. - /// - Transactions that contain no elements in any requested pool are omitted from `vtx`. - /// The original transaction index is preserved in `CompactTx.index`. - /// - `PoolTypeFilter::default()` preserves the legacy behaviour (only Sapling and Orchard - /// components are populated). - #[allow(clippy::type_complexity)] - fn get_compact_block( - &self, - nonfinalized_snapshot: &Self::Snapshot, - height: types::Height, - pool_types: PoolTypeFilter, - ) -> impl std::future::Future< - Output = Result, Self::Error>, - >; - - /// Streams *compact* blocks for an inclusive height range. - /// - /// Returns `None` if the requested range is entirely above the snapshot's tip. - /// - /// - The stream covers `[start_height, end_height]` (inclusive). - /// - If `start_height <= end_height` the stream is ascending; otherwise it is descending. - /// - /// ## Pool filtering - /// - /// - `pool_types` controls which per-transaction components are populated. - /// - Transactions that contain no elements in any requested pool are omitted from `vtx`. - /// The original transaction index is preserved in `CompactTx.index`. - /// - `PoolTypeFilter::default()` preserves the legacy behaviour (only Sapling and Orchard - /// components are populated). - #[allow(clippy::type_complexity)] - fn get_compact_block_stream( - &self, - nonfinalized_snapshot: &Self::Snapshot, - start_height: types::Height, - end_height: types::Height, - pool_types: PoolTypeFilter, - ) -> impl std::future::Future, Self::Error>>; - // ********** Transaction methods ********** /// given a transaction id, returns the transaction, along with @@ -515,12 +479,6 @@ pub trait ChainIndex { // ********** Transparent address history methods ********** - /// Returns all changes for the given transparent addresses. - fn get_address_deltas( - &self, - params: GetAddressDeltasParams, - ) -> impl std::future::Future>; - /// Returns the total transparent balance for the given addresses. fn get_address_balance( &self, @@ -556,6 +514,73 @@ pub trait ChainIndex { outpoints: Vec, scope: types::ChainScope, ) -> impl std::future::Future>, Self::Error>>; +} + +/// RPC-server extension methods layered on top of [`ChainIndex`]. +/// +/// The core [`ChainIndex`] trait holds the subset required by the embedded wallet +/// consumer (zallet). This extension holds the additional functionality required by +/// the gRPC (lightwalletd) and JSON-RPC servers: compact-block serving, mempool +/// metadata, address deltas, and the block-explorer / node-passthrough RPCs. +/// +/// TODO: This two-way core/extension split is a provisional first pass. It should be +/// refined into finer capability-based traits (zallet / lwd / block-explorer) in a +/// follow-up PR, at which point methods will be redistributed to their narrowest +/// capability. +pub trait ChainIndexRpcExt: ChainIndex { + // ********** Block methods ********** + + /// Returns the *compact* block for the given height. + /// + /// Returns `None` if the specified `height` is greater than the snapshot's tip. + /// + /// ## Pool filtering + /// + /// - `pool_types` controls which per-transaction components are populated. + /// - Transactions that contain no elements in any requested pool are omitted from `vtx`. + /// The original transaction index is preserved in `CompactTx.index`. + /// - `PoolTypeFilter::default()` preserves the legacy behaviour (only Sapling and Orchard + /// components are populated). + #[allow(clippy::type_complexity)] + fn get_compact_block( + &self, + nonfinalized_snapshot: &Self::Snapshot, + height: types::Height, + pool_types: PoolTypeFilter, + ) -> impl std::future::Future< + Output = Result, Self::Error>, + >; + + /// Streams *compact* blocks for an inclusive height range. + /// + /// Returns `None` if the requested range is entirely above the snapshot's tip. + /// + /// - The stream covers `[start_height, end_height]` (inclusive). + /// - If `start_height <= end_height` the stream is ascending; otherwise it is descending. + /// + /// ## Pool filtering + /// + /// - `pool_types` controls which per-transaction components are populated. + /// - Transactions that contain no elements in any requested pool are omitted from `vtx`. + /// The original transaction index is preserved in `CompactTx.index`. + /// - `PoolTypeFilter::default()` preserves the legacy behaviour (only Sapling and Orchard + /// components are populated). + #[allow(clippy::type_complexity)] + fn get_compact_block_stream( + &self, + nonfinalized_snapshot: &Self::Snapshot, + start_height: types::Height, + end_height: types::Height, + pool_types: PoolTypeFilter, + ) -> impl std::future::Future, Self::Error>>; + + // ********** Transparent address history methods ********** + + /// Returns all changes for the given transparent addresses. + fn get_address_deltas( + &self, + params: GetAddressDeltasParams, + ) -> impl std::future::Future>; // ********** Metadata methods ********** @@ -1684,378 +1709,82 @@ impl ChainIndex for NodeBackedChainIndexSubscriber Result, Self::Error> { - match snapshot { - ChainIndexSnapshot::NonFinalizedStateExists { - non_finalized_snapshot, - } => { - if height <= non_finalized_snapshot.best_tip.height { - Ok(Some(match snapshot.get_chainblock_by_height(&height) { - Some(block) => compact_block_with_pool_types( - block.to_compact_block(), - &pool_types.to_pool_types_vector(), - ), - None => { - match self - .finalized_state - .get_compact_block(height, pool_types.clone()) - .await - { - Ok(block) => block, - Err(_) => self - .get_compact_block_from_node(height, &pool_types) - .await? - .ok_or(ChainIndexError::database_hole(height, None))?, - } - } - })) - } else { - Ok(None) + txid: &types::TransactionHash, + ) -> Result, Option)>, Self::Error> { + // ChainIndex step 1 + if let Some(mempool_tx) = self + .mempool + .get_transaction(&mempool::MempoolKey { + txid: txid.to_rpc_hex(), + }) + .await + { + let bytes = mempool_tx.serialized_tx.as_ref().as_ref().to_vec(); + let mempool_branch_id = self.mempool_branch_id(snapshot); + + return Ok(Some((bytes, mempool_branch_id))); + } + + let Some((transaction, location)) = self + .source() + .get_transaction(*txid) + .await + .map_err(ChainIndexError::backing_validator)? + else { + return Ok(None); + }; + // as the reorg process cannot modify a transaction + // it's safe to serve nonfinalized state directly here + let height = match location { + GetTransactionLocation::BestChain(height) => height, + GetTransactionLocation::NonbestChain => { + // if the tranasction isn't on the best chain + // check our indexes. We need to find out the height from our index + // to determine the consensus branch ID + let Some(non_finalized_snapshot) = snapshot.get_nfs_snapshot() else { + // If we don't have a block containing the transaction + // locally and the transaction's not on the validator's + // best chain, we can't determine its consensus branch ID + return Ok(None); + }; + + match self + .blocks_containing_transaction(non_finalized_snapshot, txid.0) + .await? + .next() + { + Some(block) => block.context.index.height.into(), + // As above Ok(None) + None => return Ok(None), } } + // We've already checked the mempool. Should be unreachable? + // todo: error here? + GetTransactionLocation::Mempool => return Ok(None), + }; - ChainIndexSnapshot::StillSyncingFinalizedState { - validator_finalized_height: _, - //TODO: Once we make chainwork an option field we should be able to - // support passthrougth for this - } => Ok(None), - } + Ok(Some(( + zebra_chain::transaction::SerializedTransaction::from(transaction) + .as_ref() + .to_vec(), + ConsensusBranchId::current(&self.network, height).map(u32::from), + ))) } - /// Streams *compact* blocks for an inclusive height range. - /// - /// Returns `Ok(None)` if the request is descending and `start_height` exceeds the chain tip. - /// For ascending requests that exceed the tip, returns a stream that ends with an - /// `out_of_range` error after all available blocks have been sent. - /// - /// - The stream covers `[start_height, end_height]` (inclusive). - /// - If `start_height <= end_height` the stream is ascending; otherwise it is descending. - /// - /// ## Pool filtering - /// - /// - `pool_types` controls which per-transaction components are populated. - /// - Transactions that contain no elements in any requested pool are omitted from `vtx`. - /// The original transaction index is preserved in `CompactTx.index`. - /// - `PoolTypeFilter::default()` preserves the legacy behaviour (only Sapling and Orchard - /// components are populated). + /// Given a transaction ID, returns all known blocks containing this transaction /// - /// **NOTE: This Method is currently not "passthrough aware", this should be added by - /// fetching block data from the backing validator when not locally available.** - #[allow(clippy::type_complexity)] - async fn get_compact_block_stream( - &self, - nonfinalized_snapshot: &Self::Snapshot, - start_height: types::Height, - end_height: types::Height, - pool_types: PoolTypeFilter, - ) -> Result, Self::Error> { - let chain_tip_height = self.best_chaintip(nonfinalized_snapshot).await?.height; - - // The nonfinalized cache holds the tip block plus the previous 99 blocks (100 total), - // so the lowest possible cached height is `tip - 99` (saturating at 0). - let lowest_nonfinalized_height = types::Height(chain_tip_height.0.saturating_sub(99)); - - let is_ascending = start_height <= end_height; - - // Descending: the first block we'd try to return is already above the tip → error immediately. - if !is_ascending && start_height > chain_tip_height { - return Ok(None); - } - - let pool_types_vector = pool_types.to_pool_types_vector(); - - // For ascending requests that extend past the tip: cap the streaming range at the tip, - // then append a trailing out_of_range error after all valid blocks have been sent. - let needs_out_of_range = is_ascending && end_height > chain_tip_height; - let capped_end_height = if needs_out_of_range { - chain_tip_height - } else { - end_height - }; - - // Pre-create any finalized-state stream(s) we will need so that errors are returned - // from this method (not deferred into the spawned task). - let finalized_stream: Option = if is_ascending { - if start_height < lowest_nonfinalized_height { - let finalized_end_height = types::Height(std::cmp::min( - capped_end_height.0, - lowest_nonfinalized_height.0.saturating_sub(1), - )); - - if start_height <= finalized_end_height { - Some( - self.finalized_state - .get_compact_block_stream( - start_height, - finalized_end_height, - pool_types.clone(), - ) - .await - .map_err(ChainIndexError::from)?, - ) - } else { - None - } - } else { - None - } - // Serve in reverse order. - } else if end_height < lowest_nonfinalized_height { - let finalized_start_height = if start_height < lowest_nonfinalized_height { - start_height - } else { - types::Height(lowest_nonfinalized_height.0.saturating_sub(1)) - }; - - Some( - self.finalized_state - .get_compact_block_stream( - finalized_start_height, - end_height, - pool_types.clone(), - ) - .await - .map_err(ChainIndexError::from)?, - ) - } else { - None - }; - - let nonfinalized_snapshot = nonfinalized_snapshot.clone(); - let source = self.source.clone(); - let network = self.network.clone(); - let pool_types_for_node = pool_types.clone(); - // TODO: Investigate whether channel size should be changed, added to config, or set dynamically based on resources. - let (channel_sender, channel_receiver) = tokio::sync::mpsc::channel(128); - - tokio::spawn(async move { - if is_ascending { - // 1) Finalized segment (if any), ascending. - if let Some(mut finalized_stream) = finalized_stream { - while let Some(stream_item) = finalized_stream.next().await { - if channel_sender.send(stream_item).await.is_err() { - return; - } - } - } - - // 2) Nonfinalized segment, ascending. - let nonfinalized_start_height = - types::Height(std::cmp::max(start_height.0, lowest_nonfinalized_height.0)); - - for height_value in nonfinalized_start_height.0..=capped_end_height.0 { - let Some(indexed_block) = nonfinalized_snapshot - .get_chainblock_by_height(&types::Height(height_value)) - else { - match compact_block_from_source( - &source, - network.clone(), - types::Height(height_value), - &pool_types_for_node, - ) - .await - { - Ok(Some(compact_block)) => { - if channel_sender.send(Ok(compact_block)).await.is_err() { - return; - } - continue; - } - Ok(None) => { - let _ = channel_sender - .send(Err(tonic::Status::internal(format!( - "Internal error, missing nonfinalized block at height [{height_value}].", - )))) - .await; - return; - } - Err(error) => { - let _ = channel_sender - .send(Err(tonic::Status::internal(error.to_string()))) - .await; - return; - } - } - }; - let compact_block = compact_block_with_pool_types( - indexed_block.to_compact_block(), - &pool_types_vector, - ); - if channel_sender.send(Ok(compact_block)).await.is_err() { - return; - } - } - // If the original end_height was above the tip, signal out_of_range after all valid blocks. - if needs_out_of_range { - let _ = channel_sender - .send(Err(tonic::Status::out_of_range(format!( - "Error: Height out of range [{}]. Height requested is greater than the best chain tip [{}].", - end_height.0, chain_tip_height.0, - )))) - .await; - } - } else { - // 1) Nonfinalized segment, descending. - if start_height >= lowest_nonfinalized_height { - let nonfinalized_end_height = - types::Height(std::cmp::max(end_height.0, lowest_nonfinalized_height.0)); - - for height_value in (nonfinalized_end_height.0..=start_height.0).rev() { - let Some(indexed_block) = nonfinalized_snapshot - .get_chainblock_by_height(&types::Height(height_value)) - else { - match compact_block_from_source( - &source, - network.clone(), - types::Height(height_value), - &pool_types_for_node, - ) - .await - { - Ok(Some(compact_block)) => { - if channel_sender.send(Ok(compact_block)).await.is_err() { - return; - } - continue; - } - Ok(None) => { - let _ = channel_sender - .send(Err(tonic::Status::internal(format!( - "Internal error, missing nonfinalized block at height [{height_value}].", - )))) - .await; - return; - } - Err(error) => { - let _ = channel_sender - .send(Err(tonic::Status::internal(error.to_string()))) - .await; - return; - } - } - }; - let compact_block = compact_block_with_pool_types( - indexed_block.to_compact_block(), - &pool_types_vector, - ); - if channel_sender.send(Ok(compact_block)).await.is_err() { - return; - } - } - } - - // 2) Finalized segment (if any), descending. - if let Some(mut finalized_stream) = finalized_stream { - while let Some(stream_item) = finalized_stream.next().await { - if channel_sender.send(stream_item).await.is_err() { - return; - } - } - } - } - }); - - Ok(Some(CompactBlockStream::new(channel_receiver))) - } - - // ********** Transaction methods ********** - - /// given a transaction id, returns the transaction - /// and the consensus branch ID for the block the transaction - /// is in - async fn get_raw_transaction( - &self, - snapshot: &Self::Snapshot, - txid: &types::TransactionHash, - ) -> Result, Option)>, Self::Error> { - // ChainIndex step 1 - if let Some(mempool_tx) = self - .mempool - .get_transaction(&mempool::MempoolKey { - txid: txid.to_rpc_hex(), - }) - .await - { - let bytes = mempool_tx.serialized_tx.as_ref().as_ref().to_vec(); - let mempool_branch_id = self.mempool_branch_id(snapshot); - - return Ok(Some((bytes, mempool_branch_id))); - } - - let Some((transaction, location)) = self - .source() - .get_transaction(*txid) - .await - .map_err(ChainIndexError::backing_validator)? - else { - return Ok(None); - }; - // as the reorg process cannot modify a transaction - // it's safe to serve nonfinalized state directly here - let height = match location { - GetTransactionLocation::BestChain(height) => height, - GetTransactionLocation::NonbestChain => { - // if the tranasction isn't on the best chain - // check our indexes. We need to find out the height from our index - // to determine the consensus branch ID - let Some(non_finalized_snapshot) = snapshot.get_nfs_snapshot() else { - // If we don't have a block containing the transaction - // locally and the transaction's not on the validator's - // best chain, we can't determine its consensus branch ID - return Ok(None); - }; - - match self - .blocks_containing_transaction(non_finalized_snapshot, txid.0) - .await? - .next() - { - Some(block) => block.context.index.height.into(), - // As above Ok(None) - None => return Ok(None), - } - } - // We've already checked the mempool. Should be unreachable? - // todo: error here? - GetTransactionLocation::Mempool => return Ok(None), - }; - - Ok(Some(( - zebra_chain::transaction::SerializedTransaction::from(transaction) - .as_ref() - .to_vec(), - ConsensusBranchId::current(&self.network, height).map(u32::from), - ))) - } - - /// Given a transaction ID, returns all known blocks containing this transaction - /// - /// If the transaction is in the mempool, it will be in the `BestChainLocation` - /// if the mempool and snapshot are up-to-date, and the `NonBestChainLocation` set - /// if the snapshot is out-of-date compared to the mempool - async fn get_transaction_status( + /// If the transaction is in the mempool, it will be in the `BestChainLocation` + /// if the mempool and snapshot are up-to-date, and the `NonBestChainLocation` set + /// if the snapshot is out-of-date compared to the mempool + async fn get_transaction_status( &self, snapshot: &Self::Snapshot, txid: &types::TransactionHash, @@ -2169,260 +1898,704 @@ impl ChainIndex for NodeBackedChainIndexSubscriber Result, Self::Error> { - self.mempool - .get_mempool() + /// Returns all txids currently in the mempool. + async fn get_mempool_txids(&self) -> Result, Self::Error> { + self.mempool + .get_mempool() + .await + .into_iter() + .map(|(txid_key, _)| { + TransactionHash::from_hex(&txid_key.txid) + .map_err(ChainIndexError::backing_validator) + }) + .collect::>() + } + + /// Returns all transactions currently in the mempool, filtered by `exclude_list`. + /// + /// The `exclude_list` may contain shortened transaction ID hex prefixes (client-endian). + /// The transaction IDs in the Exclude list can be shortened to any number of bytes to make the request + /// more bandwidth-efficient; if two or more transactions in the mempool + /// match a shortened txid, they are all sent (none is excluded). Transactions + /// in the exclude list that don't exist in the mempool are ignored. + async fn get_mempool_transactions( + &self, + exclude_list: Vec, + ) -> Result>, Self::Error> { + // Use the mempool's own filtering (it already handles client-endian shortened prefixes). + let pairs: Vec<(mempool::MempoolKey, mempool::MempoolValue)> = + self.mempool.get_filtered_mempool(exclude_list).await; + + // Transform to the Vec> that the trait requires. + let bytes: Vec> = pairs + .into_iter() + .map(|(_, v)| v.serialized_tx.as_ref().as_ref().to_vec()) + .collect(); + + Ok(bytes) + } + + /// Returns a stream of mempool transactions, ending the stream when the chain tip block hash + /// changes (a new block is mined or a reorg occurs). + /// + /// If a snapshot is given and the chain tip has changed from the given spanshot, returns None. + fn get_mempool_stream( + &self, + snapshot: Option<&Self::Snapshot>, + ) -> Option, Self::Error>>> { + let non_finalized_snapshot = match snapshot { + Some(s) => match s { + ChainIndexSnapshot::NonFinalizedStateExists { + non_finalized_snapshot, + } => Some(non_finalized_snapshot), + // If we're still syncing the finalized state, the chain tip + // is newer than the snapshot's tip. Return None. + ChainIndexSnapshot::StillSyncingFinalizedState { .. } => return None, + }, + None => None, + }; + let expected_chain_tip = non_finalized_snapshot.map(|snapshot| snapshot.best_tip.hash); + let mut subscriber = self.mempool.clone(); + + match subscriber + .get_mempool_stream(expected_chain_tip) + .now_or_never() + { + Some(Ok((in_rx, _handle))) => { + let (out_tx, out_rx) = + tokio::sync::mpsc::channel::, ChainIndexError>>(32); + + tokio::spawn(async move { + let mut in_stream = tokio_stream::wrappers::ReceiverStream::new(in_rx); + while let Some(item) = in_stream.next().await { + match item { + Ok((_key, value)) => { + let _ = out_tx + .send(Ok(value.serialized_tx.as_ref().as_ref().to_vec())) + .await; + } + Err(e) => { + let _ = out_tx + .send(Err(ChainIndexError::child_process_status_error( + "mempool", e, + ))) + .await; + break; + } + } + } + }); + + Some(tokio_stream::wrappers::ReceiverStream::new(out_rx)) + } + Some(Err(crate::error::MempoolError::IncorrectChainTip { .. })) => None, + Some(Err(e)) => { + let (out_tx, out_rx) = + tokio::sync::mpsc::channel::, ChainIndexError>>(1); + let _ = out_tx.try_send(Err(e.into())); + Some(tokio_stream::wrappers::ReceiverStream::new(out_rx)) + } + None => { + // Should not happen because the inner tip check is synchronous, but fail safe. + let (out_tx, out_rx) = + tokio::sync::mpsc::channel::, ChainIndexError>>(1); + let _ = out_tx.try_send(Err(ChainIndexError::child_process_status_error( + "mempool", + crate::error::StatusError { + server_status: crate::StatusType::RecoverableError, + }, + ))); + Some(tokio_stream::wrappers::ReceiverStream::new(out_rx)) + } + } + } + + // ********** Chain methods ********** + + /// For a given block, + /// find its newest main-chain ancestor, + /// or the block itself if it is on the main-chain. + /// Returns Ok(None) if no fork point found. This is not an error, + /// as zaino does not guarentee knowledge of all sidechain data. + async fn find_fork_point( + &self, + snapshot: &Self::Snapshot, + hash: &types::BlockHash, + ) -> Result, Self::Error> { + // ChainIndex step 1: Skip + // mempool blocks have no canon height, guaranteed to return None + // todo: possible efficiency boost by checking mempool for a negative? + + match snapshot { + ChainIndexSnapshot::NonFinalizedStateExists { + non_finalized_snapshot, + } => { + match non_finalized_snapshot.get_chainblock_by_hash(hash) { + Some(block) => { + // At this point, we know that + // The block is non-FINALIZED in the INDEXER + // ChainIndex step 3: + if non_finalized_snapshot + .heights_to_hashes + .get(&block.height()) + == Some(block.hash()) + { + // The block is in the best chain. + Ok(Some((*block.hash(), block.height()))) + } else { + // Otherwise, it's non-best chain! Grab its parent, and recurse + Box::pin(self.find_fork_point(snapshot, &block.context.parent_hash)) + .await + // gotta pin recursive async functions to prevent infinite-sized + // Future-implementing types + } + } + None => { + // At this point, we know that + // the block is NOT non-FINALIZED in the INDEXER. + // as the non finalzed state is known to be populated, + // we now check the finalized state + match self.finalized_state.get_block_height(*hash).await { + Ok(Some(height)) => { + // the block is FINALIZED in the INDEXER + Ok(Some((*hash, height))) + } + Err(e) => Err(ChainIndexError::database_hole(hash, Some(Box::new(e)))), + Ok(None) => Ok(None), + } + } + } + } + ChainIndexSnapshot::StillSyncingFinalizedState { + validator_finalized_height, + } => { + // We're not fully synced, so we pass through. + // Now, we ask the VALIDATOR. + // ChainIndex step 5 + match self + .source() + .get_block(HashOrHeight::Hash(zebra_chain::block::Hash::from(*hash))) + .await + { + Ok(Some(block)) => { + // At this point, we know that + // the block is in the VALIDATOR. + match block.coinbase_height() { + None => { + // the block is in the VALIDATOR. but doesnt have a height. That would imply a bug. + Err(ChainIndexError::validator_data_error_block_coinbase_height_missing()) + } + Some(height) => { + // The VALIDATOR returned a block with a height. + // However, there is as of yet no guaranteed the Block is FINALIZED + if height <= *validator_finalized_height { + Ok(Some(( + types::BlockHash::from(block.hash()), + types::Height::from(height), + ))) + } else { + // non-finalized block + // no passthrough + Ok(None) + } + } + } + } + + Ok(None) => { + // At this point, we know that + // the block is NOT FINALIZED in the VALIDATOR. + // Return Ok(None) = no block found. + Ok(None) + } + Err(e) => Err(ChainIndexError::backing_validator(e)), + } + } + } + } + + /// Returns the block commitment tree data by hash. + async fn get_treestate( + &self, + hash: &types::BlockHash, + ) -> Result<(Option>, Option>), Self::Error> { + let snapshot = self.snapshot_nonfinalized_state().await?; + if !self.block_hash_known_for_treestate(&snapshot, hash).await? { + return Err(ChainIndexError::internal(format!( + "block hash {hash} not found in local chain index" + ))); + } + + match self.source().get_treestate(*hash).await { + Ok(resp) => Ok(resp), + Err(e) => Err(ChainIndexError { + kind: ChainIndexErrorKind::InternalServerError, + message: "failed to fetch treestate from validator".to_string(), + source: Some(Box::new(e)), + }), + } + } + + /// Gets the subtree roots of a given pool and the end heights of each root, + /// starting at the provided index, up to an optional maximum number of roots. + async fn get_subtree_roots( + &self, + pool: ShieldedPool, + start_index: u16, + max_entries: Option, + ) -> Result, Self::Error> { + self.source() + .get_subtree_roots(pool, start_index, max_entries) + .await + .map_err(ChainIndexError::backing_validator) + } + + // ********** Transparent address history methods ********** + + /// Returns the total transparent balance for the given addresses. + async fn get_address_balance( + &self, + address_strings: GetAddressBalanceRequest, + ) -> Result { + self.source() + .get_address_balance(address_strings) .await - .into_iter() - .map(|(txid_key, _)| { - TransactionHash::from_hex(&txid_key.txid) - .map_err(ChainIndexError::backing_validator) - }) - .collect::>() + .map_err(ChainIndexError::backing_validator) } - /// Returns all transactions currently in the mempool, filtered by `exclude_list`. - /// - /// The `exclude_list` may contain shortened transaction ID hex prefixes (client-endian). - /// The transaction IDs in the Exclude list can be shortened to any number of bytes to make the request - /// more bandwidth-efficient; if two or more transactions in the mempool - /// match a shortened txid, they are all sent (none is excluded). Transactions - /// in the exclude list that don't exist in the mempool are ignored. - async fn get_mempool_transactions( + /// Returns the transaction ids made by the given transparent addresses. + async fn get_address_txids( &self, - exclude_list: Vec, - ) -> Result>, Self::Error> { - // Use the mempool's own filtering (it already handles client-endian shortened prefixes). - let pairs: Vec<(mempool::MempoolKey, mempool::MempoolValue)> = - self.mempool.get_filtered_mempool(exclude_list).await; - - // Transform to the Vec> that the trait requires. - let bytes: Vec> = pairs - .into_iter() - .map(|(_, v)| v.serialized_tx.as_ref().as_ref().to_vec()) - .collect(); + request: GetAddressTxIdsRequest, + ) -> Result, Self::Error> { + self.source() + .get_address_txids(request) + .await + .map_err(ChainIndexError::backing_validator) + } - Ok(bytes) + /// Returns all unspent transparent outputs for the given addresses. + async fn get_address_utxos( + &self, + address_strings: GetAddressBalanceRequest, + ) -> Result, Self::Error> { + self.source() + .get_address_utxos(address_strings) + .await + .map_err(ChainIndexError::backing_validator) } - /// Returns a stream of mempool transactions, ending the stream when the chain tip block hash - /// changes (a new block is mined or a reorg occurs). - /// - /// If a snapshot is given and the chain tip has changed from the given spanshot, returns None. - fn get_mempool_stream( + async fn get_outpoint_spenders( &self, - snapshot: Option<&Self::Snapshot>, - ) -> Option, Self::Error>>> { - let non_finalized_snapshot = match snapshot { - Some(s) => match s { - ChainIndexSnapshot::NonFinalizedStateExists { - non_finalized_snapshot, - } => Some(non_finalized_snapshot), - // If we're still syncing the finalized state, the chain tip - // is newer than the snapshot's tip. Return None. - ChainIndexSnapshot::StillSyncingFinalizedState { .. } => return None, - }, - None => None, - }; - let expected_chain_tip = non_finalized_snapshot.map(|snapshot| snapshot.best_tip.hash); - let mut subscriber = self.mempool.clone(); + snapshot: &Self::Snapshot, + outpoints: Vec, + scope: types::ChainScope, + ) -> Result>, Self::Error> { + use std::collections::HashMap; - match subscriber - .get_mempool_stream(expected_chain_tip) - .now_or_never() - { - Some(Ok((in_rx, _handle))) => { - let (out_tx, out_rx) = - tokio::sync::mpsc::channel::, ChainIndexError>>(32); + let mut result: Vec> = vec![None; outpoints.len()]; - tokio::spawn(async move { - let mut in_stream = tokio_stream::wrappers::ReceiverStream::new(in_rx); - while let Some(item) = in_stream.next().await { - match item { - Ok((_key, value)) => { - let _ = out_tx - .send(Ok(value.serialized_tx.as_ref().as_ref().to_vec())) - .await; - } - Err(e) => { - let _ = out_tx - .send(Err(ChainIndexError::child_process_status_error( - "mempool", e, - ))) - .await; - break; - } - } + // 1) Non-finalised best chain (FullChain scope only). Scan only the blocks reachable + // via `heights_to_hashes` (the `blocks` map also holds reorged-away blocks, which + // must not count). One pass builds an outpoint -> spending-txid map regardless of + // how many outpoints we look up. Under `Finalised` scope this is skipped so results + // are reorg-stable. + if let ( + types::ChainScope::FullChain, + ChainIndexSnapshot::NonFinalizedStateExists { + non_finalized_snapshot, + }, + ) = (scope, snapshot) + { + let mut nfs_spenders: HashMap = HashMap::new(); + for hash in non_finalized_snapshot.heights_to_hashes.values() { + let Some(block) = non_finalized_snapshot.blocks.get(hash) else { + continue; + }; + for tx in block.transactions() { + let txid = *tx.txid(); + // `spent_outpoints` already skips coinbase null prevouts and builds each + // `Outpoint`, keeping the construction in one place (see #1332). + for outpoint in tx.transparent().spent_outpoints() { + nfs_spenders.insert(outpoint, txid); } - }); - - Some(tokio_stream::wrappers::ReceiverStream::new(out_rx)) + } } - Some(Err(crate::error::MempoolError::IncorrectChainTip { .. })) => None, - Some(Err(e)) => { - let (out_tx, out_rx) = - tokio::sync::mpsc::channel::, ChainIndexError>>(1); - let _ = out_tx.try_send(Err(e.into())); - Some(tokio_stream::wrappers::ReceiverStream::new(out_rx)) + for (i, outpoint) in outpoints.iter().enumerate() { + if let Some(txid) = nfs_spenders.get(outpoint) { + result[i] = Some(*txid); + } } - None => { - // Should not happen because the inner tip check is synchronous, but fail safe. - let (out_tx, out_rx) = - tokio::sync::mpsc::channel::, ChainIndexError>>(1); - let _ = out_tx.try_send(Err(ChainIndexError::child_process_status_error( - "mempool", - crate::error::StatusError { - server_status: crate::StatusType::RecoverableError, - }, - ))); - Some(tokio_stream::wrappers::ReceiverStream::new(out_rx)) + } + + // 2) Finalised lookup for the still-unresolved outpoints, batched into one DB call. + let unresolved_indices: Vec = result + .iter() + .enumerate() + .filter_map(|(i, r)| r.is_none().then_some(i)) + .collect(); + if unresolved_indices.is_empty() { + return Ok(result); + } + let unresolved_outpoints: Vec = + unresolved_indices.iter().map(|&i| outpoints[i]).collect(); + let locations = self + .finalized_state + .get_outpoint_spenders(unresolved_outpoints) + .await?; + + // 3) Resolve each finalised `TxLocation` to a txid. Dedup identical locations so a + // block spending several queried outpoints is only fetched once. `get_txid` is a + // single keyed lookup, far cheaper than reconstructing the whole block. + let mut slots_by_location: HashMap> = HashMap::new(); + for (slot, location) in unresolved_indices.into_iter().zip(locations) { + if let Some(location) = location { + slots_by_location.entry(location).or_default().push(slot); } } + for (location, slots) in slots_by_location { + let txid = self.finalized_state.get_txid(location).await?; + for slot in slots { + result[slot] = Some(txid); + } + } + + Ok(result) } - // ********** Chain methods ********** + // ********** Metadata methods ********** - /// For a given block, - /// find its newest main-chain ancestor, - /// or the block itself if it is on the main-chain. - /// Returns Ok(None) if no fork point found. This is not an error, - /// as zaino does not guarentee knowledge of all sidechain data. - async fn find_fork_point( + async fn best_chaintip(&self, snapshot: &Self::Snapshot) -> Result { + Ok(match snapshot { + ChainIndexSnapshot::NonFinalizedStateExists { + non_finalized_snapshot, + } => non_finalized_snapshot.best_tip, + + ChainIndexSnapshot::StillSyncingFinalizedState { + validator_finalized_height, + } => { + BlockIndex { + height: *validator_finalized_height, + hash: self + .source() + // TODO: do something more efficient than getting the whole block + .get_block(HashOrHeight::Height((*validator_finalized_height).into())) + .await + .map_err(|e| { + ChainIndexError::database_hole( + validator_finalized_height, + Some(Box::new(e)), + ) + })? + .ok_or(ChainIndexError::database_hole( + validator_finalized_height, + None, + ))? + .hash() + .into(), + } + } + }) + } +} + +impl ChainIndexRpcExt for NodeBackedChainIndexSubscriber { + /// Returns the *compact* block for the given height. + /// + /// Returns `None` if the specified `height` is greater than the snapshot's tip. + /// + /// ## Pool filtering + /// + /// - `pool_types` controls which per-transaction components are populated. + /// - Transactions that contain no elements in any requested pool are omitted from `vtx`. + /// The original transaction index is preserved in `CompactTx.index`. + /// - `PoolTypeFilter::default()` preserves the legacy behaviour (only Sapling and Orchard + /// components are populated). + /// + /// Returns None if the specified height + /// is greater than the snapshot's tip + /// + /// **NOTE: This Method is currently not "passthrough aware", this should be added by + /// fetching block data from the backing validator when not locally available.** + async fn get_compact_block( &self, snapshot: &Self::Snapshot, - hash: &types::BlockHash, - ) -> Result, Self::Error> { - // ChainIndex step 1: Skip - // mempool blocks have no canon height, guaranteed to return None - // todo: possible efficiency boost by checking mempool for a negative? - + height: types::Height, + pool_types: PoolTypeFilter, + ) -> Result, Self::Error> { match snapshot { ChainIndexSnapshot::NonFinalizedStateExists { non_finalized_snapshot, } => { - match non_finalized_snapshot.get_chainblock_by_hash(hash) { - Some(block) => { - // At this point, we know that - // The block is non-FINALIZED in the INDEXER - // ChainIndex step 3: - if non_finalized_snapshot - .heights_to_hashes - .get(&block.height()) - == Some(block.hash()) - { - // The block is in the best chain. - Ok(Some((*block.hash(), block.height()))) - } else { - // Otherwise, it's non-best chain! Grab its parent, and recurse - Box::pin(self.find_fork_point(snapshot, &block.context.parent_hash)) + if height <= non_finalized_snapshot.best_tip.height { + Ok(Some(match snapshot.get_chainblock_by_height(&height) { + Some(block) => compact_block_with_pool_types( + block.to_compact_block(), + &pool_types.to_pool_types_vector(), + ), + None => { + match self + .finalized_state + .get_compact_block(height, pool_types.clone()) .await - // gotta pin recursive async functions to prevent infinite-sized - // Future-implementing types - } - } - None => { - // At this point, we know that - // the block is NOT non-FINALIZED in the INDEXER. - // as the non finalzed state is known to be populated, - // we now check the finalized state - match self.finalized_state.get_block_height(*hash).await { - Ok(Some(height)) => { - // the block is FINALIZED in the INDEXER - Ok(Some((*hash, height))) + { + Ok(block) => block, + Err(_) => self + .get_compact_block_from_node(height, &pool_types) + .await? + .ok_or(ChainIndexError::database_hole(height, None))?, } - Err(e) => Err(ChainIndexError::database_hole(hash, Some(Box::new(e)))), - Ok(None) => Ok(None), } - } + })) + } else { + Ok(None) } } + ChainIndexSnapshot::StillSyncingFinalizedState { - validator_finalized_height, - } => { - // We're not fully synced, so we pass through. - // Now, we ask the VALIDATOR. - // ChainIndex step 5 - match self - .source() - .get_block(HashOrHeight::Hash(zebra_chain::block::Hash::from(*hash))) + validator_finalized_height: _, + //TODO: Once we make chainwork an option field we should be able to + // support passthrougth for this + } => Ok(None), + } + } + + /// Streams *compact* blocks for an inclusive height range. + /// + /// Returns `Ok(None)` if the request is descending and `start_height` exceeds the chain tip. + /// For ascending requests that exceed the tip, returns a stream that ends with an + /// `out_of_range` error after all available blocks have been sent. + /// + /// - The stream covers `[start_height, end_height]` (inclusive). + /// - If `start_height <= end_height` the stream is ascending; otherwise it is descending. + /// + /// ## Pool filtering + /// + /// - `pool_types` controls which per-transaction components are populated. + /// - Transactions that contain no elements in any requested pool are omitted from `vtx`. + /// The original transaction index is preserved in `CompactTx.index`. + /// - `PoolTypeFilter::default()` preserves the legacy behaviour (only Sapling and Orchard + /// components are populated). + /// + /// **NOTE: This Method is currently not "passthrough aware", this should be added by + /// fetching block data from the backing validator when not locally available.** + #[allow(clippy::type_complexity)] + async fn get_compact_block_stream( + &self, + nonfinalized_snapshot: &Self::Snapshot, + start_height: types::Height, + end_height: types::Height, + pool_types: PoolTypeFilter, + ) -> Result, Self::Error> { + let chain_tip_height = self.best_chaintip(nonfinalized_snapshot).await?.height; + + // The nonfinalized cache holds the tip block plus the previous 99 blocks (100 total), + // so the lowest possible cached height is `tip - 99` (saturating at 0). + let lowest_nonfinalized_height = types::Height(chain_tip_height.0.saturating_sub(99)); + + let is_ascending = start_height <= end_height; + + // Descending: the first block we'd try to return is already above the tip → error immediately. + if !is_ascending && start_height > chain_tip_height { + return Ok(None); + } + + let pool_types_vector = pool_types.to_pool_types_vector(); + + // For ascending requests that extend past the tip: cap the streaming range at the tip, + // then append a trailing out_of_range error after all valid blocks have been sent. + let needs_out_of_range = is_ascending && end_height > chain_tip_height; + let capped_end_height = if needs_out_of_range { + chain_tip_height + } else { + end_height + }; + + // Pre-create any finalized-state stream(s) we will need so that errors are returned + // from this method (not deferred into the spawned task). + let finalized_stream: Option = if is_ascending { + if start_height < lowest_nonfinalized_height { + let finalized_end_height = types::Height(std::cmp::min( + capped_end_height.0, + lowest_nonfinalized_height.0.saturating_sub(1), + )); + + if start_height <= finalized_end_height { + Some( + self.finalized_state + .get_compact_block_stream( + start_height, + finalized_end_height, + pool_types.clone(), + ) + .await + .map_err(ChainIndexError::from)?, + ) + } else { + None + } + } else { + None + } + // Serve in reverse order. + } else if end_height < lowest_nonfinalized_height { + let finalized_start_height = if start_height < lowest_nonfinalized_height { + start_height + } else { + types::Height(lowest_nonfinalized_height.0.saturating_sub(1)) + }; + + Some( + self.finalized_state + .get_compact_block_stream( + finalized_start_height, + end_height, + pool_types.clone(), + ) .await - { - Ok(Some(block)) => { - // At this point, we know that - // the block is in the VALIDATOR. - match block.coinbase_height() { - None => { - // the block is in the VALIDATOR. but doesnt have a height. That would imply a bug. - Err(ChainIndexError::validator_data_error_block_coinbase_height_missing()) + .map_err(ChainIndexError::from)?, + ) + } else { + None + }; + + let nonfinalized_snapshot = nonfinalized_snapshot.clone(); + let source = self.source.clone(); + let network = self.network.clone(); + let pool_types_for_node = pool_types.clone(); + // TODO: Investigate whether channel size should be changed, added to config, or set dynamically based on resources. + let (channel_sender, channel_receiver) = tokio::sync::mpsc::channel(128); + + tokio::spawn(async move { + if is_ascending { + // 1) Finalized segment (if any), ascending. + if let Some(mut finalized_stream) = finalized_stream { + while let Some(stream_item) = finalized_stream.next().await { + if channel_sender.send(stream_item).await.is_err() { + return; + } + } + } + + // 2) Nonfinalized segment, ascending. + let nonfinalized_start_height = + types::Height(std::cmp::max(start_height.0, lowest_nonfinalized_height.0)); + + for height_value in nonfinalized_start_height.0..=capped_end_height.0 { + let Some(indexed_block) = nonfinalized_snapshot + .get_chainblock_by_height(&types::Height(height_value)) + else { + match compact_block_from_source( + &source, + network.clone(), + types::Height(height_value), + &pool_types_for_node, + ) + .await + { + Ok(Some(compact_block)) => { + if channel_sender.send(Ok(compact_block)).await.is_err() { + return; + } + continue; } - Some(height) => { - // The VALIDATOR returned a block with a height. - // However, there is as of yet no guaranteed the Block is FINALIZED - if height <= *validator_finalized_height { - Ok(Some(( - types::BlockHash::from(block.hash()), - types::Height::from(height), - ))) - } else { - // non-finalized block - // no passthrough - Ok(None) + Ok(None) => { + let _ = channel_sender + .send(Err(tonic::Status::internal(format!( + "Internal error, missing nonfinalized block at height [{height_value}].", + )))) + .await; + return; + } + Err(error) => { + let _ = channel_sender + .send(Err(tonic::Status::internal(error.to_string()))) + .await; + return; + } + } + }; + let compact_block = compact_block_with_pool_types( + indexed_block.to_compact_block(), + &pool_types_vector, + ); + if channel_sender.send(Ok(compact_block)).await.is_err() { + return; + } + } + // If the original end_height was above the tip, signal out_of_range after all valid blocks. + if needs_out_of_range { + let _ = channel_sender + .send(Err(tonic::Status::out_of_range(format!( + "Error: Height out of range [{}]. Height requested is greater than the best chain tip [{}].", + end_height.0, chain_tip_height.0, + )))) + .await; + } + } else { + // 1) Nonfinalized segment, descending. + if start_height >= lowest_nonfinalized_height { + let nonfinalized_end_height = + types::Height(std::cmp::max(end_height.0, lowest_nonfinalized_height.0)); + + for height_value in (nonfinalized_end_height.0..=start_height.0).rev() { + let Some(indexed_block) = nonfinalized_snapshot + .get_chainblock_by_height(&types::Height(height_value)) + else { + match compact_block_from_source( + &source, + network.clone(), + types::Height(height_value), + &pool_types_for_node, + ) + .await + { + Ok(Some(compact_block)) => { + if channel_sender.send(Ok(compact_block)).await.is_err() { + return; + } + continue; + } + Ok(None) => { + let _ = channel_sender + .send(Err(tonic::Status::internal(format!( + "Internal error, missing nonfinalized block at height [{height_value}].", + )))) + .await; + return; + } + Err(error) => { + let _ = channel_sender + .send(Err(tonic::Status::internal(error.to_string()))) + .await; + return; } } + }; + let compact_block = compact_block_with_pool_types( + indexed_block.to_compact_block(), + &pool_types_vector, + ); + if channel_sender.send(Ok(compact_block)).await.is_err() { + return; } } + } - Ok(None) => { - // At this point, we know that - // the block is NOT FINALIZED in the VALIDATOR. - // Return Ok(None) = no block found. - Ok(None) + // 2) Finalized segment (if any), descending. + if let Some(mut finalized_stream) = finalized_stream { + while let Some(stream_item) = finalized_stream.next().await { + if channel_sender.send(stream_item).await.is_err() { + return; + } } - Err(e) => Err(ChainIndexError::backing_validator(e)), } } - } - } - - /// Returns the block commitment tree data by hash. - async fn get_treestate( - &self, - hash: &types::BlockHash, - ) -> Result<(Option>, Option>), Self::Error> { - let snapshot = self.snapshot_nonfinalized_state().await?; - if !self.block_hash_known_for_treestate(&snapshot, hash).await? { - return Err(ChainIndexError::internal(format!( - "block hash {hash} not found in local chain index" - ))); - } - - match self.source().get_treestate(*hash).await { - Ok(resp) => Ok(resp), - Err(e) => Err(ChainIndexError { - kind: ChainIndexErrorKind::InternalServerError, - message: "failed to fetch treestate from validator".to_string(), - source: Some(Box::new(e)), - }), - } - } + }); - /// Gets the subtree roots of a given pool and the end heights of each root, - /// starting at the provided index, up to an optional maximum number of roots. - async fn get_subtree_roots( - &self, - pool: ShieldedPool, - start_index: u16, - max_entries: Option, - ) -> Result, Self::Error> { - self.source() - .get_subtree_roots(pool, start_index, max_entries) - .await - .map_err(ChainIndexError::backing_validator) + Ok(Some(CompactBlockStream::new(channel_receiver))) } - // ********** Transparent address history methods ********** - /// Returns all changes for the given transparent addresses. async fn get_address_deltas( &self, @@ -2434,119 +2607,6 @@ impl ChainIndex for NodeBackedChainIndexSubscriber Result { - self.source() - .get_address_balance(address_strings) - .await - .map_err(ChainIndexError::backing_validator) - } - - /// Returns the transaction ids made by the given transparent addresses. - async fn get_address_txids( - &self, - request: GetAddressTxIdsRequest, - ) -> Result, Self::Error> { - self.source() - .get_address_txids(request) - .await - .map_err(ChainIndexError::backing_validator) - } - - /// Returns all unspent transparent outputs for the given addresses. - async fn get_address_utxos( - &self, - address_strings: GetAddressBalanceRequest, - ) -> Result, Self::Error> { - self.source() - .get_address_utxos(address_strings) - .await - .map_err(ChainIndexError::backing_validator) - } - - async fn get_outpoint_spenders( - &self, - snapshot: &Self::Snapshot, - outpoints: Vec, - scope: types::ChainScope, - ) -> Result>, Self::Error> { - use std::collections::HashMap; - - let mut result: Vec> = vec![None; outpoints.len()]; - - // 1) Non-finalised best chain (FullChain scope only). Scan only the blocks reachable - // via `heights_to_hashes` (the `blocks` map also holds reorged-away blocks, which - // must not count). One pass builds an outpoint -> spending-txid map regardless of - // how many outpoints we look up. Under `Finalised` scope this is skipped so results - // are reorg-stable. - if let ( - types::ChainScope::FullChain, - ChainIndexSnapshot::NonFinalizedStateExists { - non_finalized_snapshot, - }, - ) = (scope, snapshot) - { - let mut nfs_spenders: HashMap = HashMap::new(); - for hash in non_finalized_snapshot.heights_to_hashes.values() { - let Some(block) = non_finalized_snapshot.blocks.get(hash) else { - continue; - }; - for tx in block.transactions() { - let txid = *tx.txid(); - // `spent_outpoints` already skips coinbase null prevouts and builds each - // `Outpoint`, keeping the construction in one place (see #1332). - for outpoint in tx.transparent().spent_outpoints() { - nfs_spenders.insert(outpoint, txid); - } - } - } - for (i, outpoint) in outpoints.iter().enumerate() { - if let Some(txid) = nfs_spenders.get(outpoint) { - result[i] = Some(*txid); - } - } - } - - // 2) Finalised lookup for the still-unresolved outpoints, batched into one DB call. - let unresolved_indices: Vec = result - .iter() - .enumerate() - .filter_map(|(i, r)| r.is_none().then_some(i)) - .collect(); - if unresolved_indices.is_empty() { - return Ok(result); - } - let unresolved_outpoints: Vec = - unresolved_indices.iter().map(|&i| outpoints[i]).collect(); - let locations = self - .finalized_state - .get_outpoint_spenders(unresolved_outpoints) - .await?; - - // 3) Resolve each finalised `TxLocation` to a txid. Dedup identical locations so a - // block spending several queried outpoints is only fetched once. `get_txid` is a - // single keyed lookup, far cheaper than reconstructing the whole block. - let mut slots_by_location: HashMap> = HashMap::new(); - for (slot, location) in unresolved_indices.into_iter().zip(locations) { - if let Some(location) = location { - slots_by_location.entry(location).or_default().push(slot); - } - } - for (location, slots) in slots_by_location { - let txid = self.finalized_state.get_txid(location).await?; - for slot in slots { - result[slot] = Some(txid); - } - } - - Ok(result) - } - - // ********** Metadata methods ********** - /// Returns Information about the mempool state: /// - size: Current tx count /// - bytes: Sum of all tx sizes @@ -2555,39 +2615,6 @@ impl ChainIndex for NodeBackedChainIndexSubscriber Result { - Ok(match snapshot { - ChainIndexSnapshot::NonFinalizedStateExists { - non_finalized_snapshot, - } => non_finalized_snapshot.best_tip, - - ChainIndexSnapshot::StillSyncingFinalizedState { - validator_finalized_height, - } => { - BlockIndex { - height: *validator_finalized_height, - hash: self - .source() - // TODO: do something more efficient than getting the whole block - .get_block(HashOrHeight::Height((*validator_finalized_height).into())) - .await - .map_err(|e| { - ChainIndexError::database_hole( - validator_finalized_height, - Some(Box::new(e)), - ) - })? - .ok_or(ChainIndexError::database_hole( - validator_finalized_height, - None, - ))? - .hash() - .into(), - } - } - }) - } - async fn get_tx_out_set_info(&self) -> Result { use crate::chain_index::types::db::metadata::{ is_unspendable_tx_out, ZAINO_TXOUTSET_ENTRY_LEN, diff --git a/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs b/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs index 31518411e..b7552f8d5 100644 --- a/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs +++ b/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs @@ -7,7 +7,7 @@ use crate::{ vectors::{indexed_block_chain, load_test_vectors, TestVectorBlockData}, }, types::{BestChainLocation, ChainScope, TransactionHash}, - ChainIndex, NodeBackedChainIndexSubscriber, + ChainIndex, ChainIndexRpcExt, NodeBackedChainIndexSubscriber, }, BlockchainSource as _, Outpoint, }; diff --git a/packages/zaino-state/src/lib.rs b/packages/zaino-state/src/lib.rs index 548c1da16..22cb0121a 100644 --- a/packages/zaino-state/src/lib.rs +++ b/packages/zaino-state/src/lib.rs @@ -67,7 +67,9 @@ pub use backends::{ pub mod chain_index; // Core ChainIndex trait and implementations -pub use chain_index::{ChainIndex, NodeBackedChainIndex, NodeBackedChainIndexSubscriber}; +pub use chain_index::{ + ChainIndex, ChainIndexRpcExt, NodeBackedChainIndex, NodeBackedChainIndexSubscriber, +}; // Source types for ChainIndex backends pub use chain_index::source::{BlockchainSource, State, ValidatorConnector}; // Supporting types From abbeb86aa40445c4702ba46234f447fffaa352fa Mon Sep 17 00:00:00 2001 From: idky137 Date: Thu, 2 Jul 2026 16:25:47 +0100 Subject: [PATCH 03/58] move methods into chain_index / blockchainsource pt 1 --- packages/zaino-state/src/backends/fetch.rs | 12 +++++--- packages/zaino-state/src/backends/state.rs | 32 ++++++---------------- 2 files changed, 16 insertions(+), 28 deletions(-) diff --git a/packages/zaino-state/src/backends/fetch.rs b/packages/zaino-state/src/backends/fetch.rs index 74f63dfcc..237204479 100644 --- a/packages/zaino-state/src/backends/fetch.rs +++ b/packages/zaino-state/src/backends/fetch.rs @@ -485,7 +485,9 @@ impl ZcashIndexer for FetchServiceSubscriber { /// [The function in rpc/blockchain.cpp](https://github.com/zcash/zcash/blob/654a8be2274aa98144c80c1ac459400eaf0eacbe/src/rpc/blockchain.cpp#L325) /// where `return chainActive.Tip()->GetBlockHash().GetHex();` is the [return expression](https://github.com/zcash/zcash/blob/654a8be2274aa98144c80c1ac459400eaf0eacbe/src/rpc/blockchain.cpp#L339)returning a `std::string` async fn get_best_blockhash(&self) -> Result { - Ok(self.fetcher.get_best_blockhash().await?.into()) + let snapshot = self.indexer.snapshot_nonfinalized_state().await?; + let tip = self.indexer.best_chaintip(&snapshot).await?; + Ok(GetBlockHashResponse::new(tip.hash.into())) } /// Returns the current block count in the best valid block chain. @@ -494,15 +496,17 @@ impl ZcashIndexer for FetchServiceSubscriber { /// method: post /// tags: blockchain async fn get_block_count(&self) -> Result { - Ok(self.fetcher.get_block_count().await?.into()) + let snapshot = self.indexer.snapshot_nonfinalized_state().await?; + let tip = self.indexer.best_chaintip(&snapshot).await?; + Ok(tip.height.into()) } + #[allow(deprecated)] async fn get_chain_tips(&self) -> Result { let snapshot = self.indexer.snapshot_nonfinalized_state().await?; let Some(non_finalized_snapshot) = snapshot.get_nfs_snapshot() else { - return Ok(self.fetcher.get_chain_tips().await?); + return Err(FetchServiceError::UnavailableNotSyncedEnough); }; - Ok(chain_tips_from_nonfinalized_snapshot( non_finalized_snapshot, )) diff --git a/packages/zaino-state/src/backends/state.rs b/packages/zaino-state/src/backends/state.rs index 5617ce9e2..e2fe4f08b 100644 --- a/packages/zaino-state/src/backends/state.rs +++ b/packages/zaino-state/src/backends/state.rs @@ -3,6 +3,7 @@ #[allow(deprecated)] use crate::{ chain_index::{ + chain_tips_from_nonfinalized_snapshot, mempool::{Mempool, MempoolSubscriber}, source::ValidatorConnector, types as chain_types, ChainIndex, ChainIndexRpcExt, @@ -1545,19 +1546,9 @@ impl ZcashIndexer for StateServiceSubscriber { /// [The function in rpc/blockchain.cpp](https://github.com/zcash/zcash/blob/654a8be2274aa98144c80c1ac459400eaf0eacbe/src/rpc/blockchain.cpp#L325) /// where `return chainActive.Tip()->GetBlockHash().GetHex();` is the [return expression](https://github.com/zcash/zcash/blob/654a8be2274aa98144c80c1ac459400eaf0eacbe/src/rpc/blockchain.cpp#L339)returning a `std::string` async fn get_best_blockhash(&self) -> Result { - // return should be valid hex encoded. - // Hash from zebra says: - // Return the hash bytes in big-endian byte-order suitable for printing out byte by byte. - // - // Zebra displays transaction and block hashes in big-endian byte-order, - // following the u256 convention set by Bitcoin and zcashd. - match self.read_state_service.best_tip() { - Some(x) => return Ok(GetBlockHash::new(x.1)), - None => { - // try RPC if state read fails: - Ok(self.rpc_client.get_best_blockhash().await?.into()) - } - } + let snapshot = self.indexer.snapshot_nonfinalized_state().await?; + let tip = self.indexer.best_chaintip(&snapshot).await?; + Ok(GetBlockHash::new(tip.hash.into())) } /// Returns the current block count in the best valid block chain. @@ -1567,23 +1558,16 @@ impl ZcashIndexer for StateServiceSubscriber { /// tags: blockchain async fn get_block_count(&self) -> Result { let snapshot = self.indexer.snapshot_nonfinalized_state().await?; - let Some(non_finalized_snapshot) = snapshot.get_nfs_snapshot() else { - // TODO: This probably shouldn't be an error. - // this is an improvement over previous behaviour of - // acting as if we are only synced to the genesis block - return Err(StateServiceError::UnavailableNotSyncedEnough); - }; - let h = non_finalized_snapshot.best_tip.height; - Ok(h.into()) + let tip = self.indexer.best_chaintip(&snapshot).await?; + Ok(tip.height.into()) } async fn get_chain_tips(&self) -> Result { let snapshot = self.indexer.snapshot_nonfinalized_state().await?; let Some(non_finalized_snapshot) = snapshot.get_nfs_snapshot() else { - return Ok(self.rpc_client.get_chain_tips().await?); + return Err(StateServiceError::UnavailableNotSyncedEnough); }; - - Ok(crate::chain_index::chain_tips_from_nonfinalized_snapshot( + Ok(chain_tips_from_nonfinalized_snapshot( non_finalized_snapshot, )) } From c7671b8f1418cf8c558fe0bb34c1257787942c99 Mon Sep 17 00:00:00 2001 From: idky137 Date: Thu, 2 Jul 2026 17:19:25 +0100 Subject: [PATCH 04/58] move methods into chain_index / blockchainsource pt 2 - get_block / get_block_header --- packages/zaino-state/src/backends/fetch.rs | 8 +- packages/zaino-state/src/backends/state.rs | 387 +------------- packages/zaino-state/src/chain_index.rs | 48 +- .../zaino-state/src/chain_index/source.rs | 23 +- .../chain_index/source/mockchain_source.rs | 151 +++++- .../chain_index/source/validator_connector.rs | 470 +++++++++++++++++- .../src/chain_index/tests/mockchain_tests.rs | 94 +++- .../chain_index/tests/proptest_blockgen.rs | 19 + 8 files changed, 809 insertions(+), 391 deletions(-) diff --git a/packages/zaino-state/src/backends/fetch.rs b/packages/zaino-state/src/backends/fetch.rs index 237204479..98f1b7b67 100644 --- a/packages/zaino-state/src/backends/fetch.rs +++ b/packages/zaino-state/src/backends/fetch.rs @@ -426,11 +426,7 @@ impl ZcashIndexer for FetchServiceSubscriber { hash_or_height: String, verbosity: Option, ) -> Result { - Ok(self - .fetcher - .get_block(hash_or_height, verbosity) - .await? - .try_into()?) + Ok(self.indexer.z_get_block(hash_or_height, verbosity).await?) } /// Returns information about the given block and its transactions. @@ -449,7 +445,7 @@ impl ZcashIndexer for FetchServiceSubscriber { hash: String, verbose: bool, ) -> Result { - Ok(self.fetcher.get_block_header(hash, verbose).await?) + Ok(self.indexer.get_block_header(hash, verbose).await?) } async fn get_mining_info(&self) -> Result { diff --git a/packages/zaino-state/src/backends/state.rs b/packages/zaino-state/src/backends/state.rs index e2fe4f08b..59b6e748e 100644 --- a/packages/zaino-state/src/backends/state.rs +++ b/packages/zaino-state/src/backends/state.rs @@ -66,22 +66,20 @@ use zcash_protocol::consensus::NetworkType; use zcash_transparent::address::TransparentAddress; use zebra_chain::{ amount::{Amount, NonNegative}, - block::{Header, Height, SerializedBlock}, + block::Height, chain_tip::NetworkChainTipHeightEstimator, - parameters::{ConsensusBranchId, Network, NetworkKind, NetworkUpgrade}, - serialization::{BytesInDisplayOrder as _, ZcashDeserialize as _, ZcashSerialize}, + parameters::{ConsensusBranchId, NetworkKind, NetworkUpgrade}, + serialization::ZcashDeserialize as _, subtree::NoteCommitmentSubtreeIndex, }; use zebra_rpc::{ client::{ - GetAddressBalanceRequest, GetBlockchainInfoBalance, GetSubtreesByIndexResponse, - GetTreestateResponse, HexData, Input, SubtreeRpcData, TransactionObject, - ValidateAddressResponse, + GetAddressBalanceRequest, GetSubtreesByIndexResponse, GetTreestateResponse, Input, + SubtreeRpcData, TransactionObject, ValidateAddressResponse, }, methods::{ chain_tip_difficulty, AddressBalance, ConsensusBranchIdHex, GetAddressTxIdsRequest, - GetAddressUtxos, GetBlock, GetBlockHash, GetBlockHeader as GetBlockHeaderZebra, - GetBlockHeaderObject, GetBlockTransaction, GetBlockTrees, GetBlockchainInfoResponse, + GetAddressUtxos, GetBlock, GetBlockHash, GetBlockTransaction, GetBlockchainInfoResponse, GetInfo, GetRawTransaction, NetworkUpgradeInfo, NetworkUpgradeStatus, SentTransactionHash, TipConsensusBranch, }, @@ -90,7 +88,7 @@ use zebra_rpc::{ }; use zebra_state::{HashOrHeight, ReadRequest, ReadResponse, ReadStateService}; -use chrono::{DateTime, Utc}; +use chrono::Utc; use futures::TryFutureExt as _; use hex::{FromHex as _, ToHex}; use indexmap::IndexMap; @@ -405,149 +403,6 @@ impl StateServiceSubscriber { monitor: self.chain_tip_change.clone(), } } - /// Returns the requested block header by hash or height, as a [`GetBlockHeader`] JSON string. - /// If the block is not in Zebra's state, - /// returns [error code `-8`.](https://github.com/zcash/zcash/issues/5758) - /// if a height was passed or -5 if a hash was passed. - /// - /// zcashd reference: [`getblockheader`](https://zcash.github.io/rpc/getblockheader.html) - /// method: post - /// tags: blockchain - /// - /// # Parameters - /// - /// - `hash_or_height`: (string, required, example="1") The hash or height - /// for the block to be returned. - /// - `verbose`: (bool, optional, default=false, example=true) false for hex encoded data, - /// true for a json object - /// - /// # Notes - /// - /// The undocumented `chainwork` field is not returned. - /// - /// This rpc is used by get_block(verbose), there is currently no - /// plan to offer this RPC publicly. - async fn get_block_header_inner( - state: &ReadStateService, - network: &Network, - hash_or_height: HashOrHeight, - verbose: Option, - ) -> Result { - let mut state = state.clone(); - let verbose = verbose.unwrap_or(true); - let network = network.clone(); - - let zebra_state::ReadResponse::BlockHeader { - header, - hash, - height, - next_block_hash, - } = state - .ready() - .and_then(|service| service.call(zebra_state::ReadRequest::BlockHeader(hash_or_height))) - .await - .map_err(|_| { - StateServiceError::RpcError(RpcError { - // Compatibility with zcashd. Note that since this function - // is reused by getblock(), we return the errors expected - // by it (they differ whether a hash or a height was passed) - code: LegacyCode::InvalidParameter as i64, - message: "block height not in best chain".to_string(), - data: None, - }) - })? - else { - return Err(StateServiceError::Custom( - "Unexpected response to BlockHeader request".to_string(), - )); - }; - - let response = if !verbose { - GetBlockHeaderZebra::Raw(HexData(header.zcash_serialize_to_vec()?)) - } else { - let zebra_state::ReadResponse::SaplingTree(sapling_tree) = state - .ready() - .and_then(|service| { - service.call(zebra_state::ReadRequest::SaplingTree(hash_or_height)) - }) - .await? - else { - return Err(StateServiceError::Custom( - "Unexpected response to SaplingTree request".to_string(), - )); - }; - // This could be `None` if there's a chain reorg between state queries. - let sapling_tree = sapling_tree.ok_or_else(|| { - StateServiceError::RpcError(zaino_fetch::jsonrpsee::connector::RpcError { - code: LegacyCode::InvalidParameter as i64, - message: "missing sapling tree for block".to_string(), - data: None, - }) - })?; - - let zebra_state::ReadResponse::Depth(depth) = state - .ready() - .and_then(|service| service.call(zebra_state::ReadRequest::Depth(hash))) - .await? - else { - return Err(StateServiceError::Custom( - "Unexpected response to Depth request".to_string(), - )); - }; - - // From - // TODO: Deduplicate const definition, consider - // refactoring this to avoid duplicate logic - const NOT_IN_BEST_CHAIN_CONFIRMATIONS: i64 = -1; - - // Confirmations are one more than the depth. - // Depth is limited by height, so it will never overflow an i64. - let confirmations = depth - .map(|depth| i64::from(depth) + 1) - .unwrap_or(NOT_IN_BEST_CHAIN_CONFIRMATIONS); - - let mut nonce = *header.nonce; - nonce.reverse(); - - let sapling_activation = NetworkUpgrade::Sapling.activation_height(&network); - let sapling_tree_size = sapling_tree.count(); - let final_sapling_root: [u8; 32] = - if sapling_activation.is_some() && height >= sapling_activation.unwrap() { - let mut root: [u8; 32] = sapling_tree.root().into(); - root.reverse(); - root - } else { - [0; 32] - }; - - let difficulty = header.difficulty_threshold.relative_to_network(&network); - let block_commitments = - header_to_block_commitments(&header, &network, height, final_sapling_root)?; - - let block_header = GetBlockHeaderObject::new( - hash, - confirmations, - height, - header.version, - header.merkle_root, - block_commitments, - final_sapling_root, - sapling_tree_size, - header.time.timestamp(), - nonce, - header.solution, - header.difficulty_threshold, - difficulty, - header.previous_block_hash, - next_block_hash, - ); - - GetBlockHeaderZebra::Object(Box::new(block_header)) - }; - - Ok(response) - } - /// Return a list of consecutive compact blocks. #[allow(dead_code, deprecated)] async fn get_block_range_inner( @@ -704,185 +559,6 @@ impl StateServiceSubscriber { }) } - pub(crate) async fn get_block_inner( - state: &ReadStateService, - network: &Network, - hash_or_height: HashOrHeight, - verbosity: Option, - ) -> Result { - let mut state_1 = state.clone(); - - let verbosity = verbosity.unwrap_or(1); - match verbosity { - 0 => { - let request = ReadRequest::Block(hash_or_height); - let response = state_1 - .ready() - .and_then(|service| service.call(request)) - .await?; - let block = expected_read_response!(response, Block); - block.map(SerializedBlock::from).map(GetBlock::Raw).ok_or( - StateServiceError::RpcError(RpcError::new_from_legacycode( - LegacyCode::InvalidParameter, - "block not found", - )), - ) - } - 1 | 2 => { - let state_2 = state.clone(); - let state_3 = state.clone(); - let state_4 = state.clone(); - - let blockandsize_future = { - let req = ReadRequest::BlockAndSize(hash_or_height); - async move { state_1.ready().and_then(|service| service.call(req)).await } - }; - let orchard_future = { - let req = ReadRequest::OrchardTree(hash_or_height); - async move { - state_2 - .clone() - .ready() - .and_then(|service| service.call(req)) - .await - } - }; - - let block_info_future = { - let req = ReadRequest::BlockInfo(hash_or_height); - async move { - state_4 - .clone() - .ready() - .and_then(|service| service.call(req)) - .await - } - }; - let (fullblock, orchard_tree_response, header, block_info) = futures::join!( - blockandsize_future, - orchard_future, - StateServiceSubscriber::get_block_header_inner( - &state_3, - network, - hash_or_height, - Some(true) - ), - block_info_future - ); - - let header_obj = match header? { - GetBlockHeaderZebra::Raw(_hex_data) => unreachable!( - "`true` was passed to get_block_header, an object should be returned" - ), - GetBlockHeaderZebra::Object(get_block_header_object) => get_block_header_object, - }; - - let (transactions_response, size, block_info): (Vec, _, _) = - match (fullblock, block_info) { - ( - Ok(ReadResponse::BlockAndSize(Some((block, size)))), - Ok(ReadResponse::BlockInfo(Some(block_info))), - ) => Ok(( - block - .transactions - .iter() - .map(|transaction| { - match verbosity { - 1 => GetBlockTransaction::Hash(transaction.hash()), - 2 => GetBlockTransaction::Object(Box::new( - TransactionObject::from_transaction( - transaction.clone(), - Some(header_obj.height()), - Some(header_obj.confirmations() as u32), - network, - DateTime::::from_timestamp( - header_obj.time(), - 0, - ), - Some(header_obj.hash()), - // block header has a non-optional height, which indicates - // a mainchain block. It is implied this method cannot return sidechain - // data, at least for now. This is subject to change: TODO - // return Some(true/false) after this assumption is resolved - None, - transaction.hash(), - ), - )), - _ => unreachable!("verbosity known to be 1 or 2"), - } - }) - .collect(), - size, - block_info, - )), - (Ok(ReadResponse::Block(None)), Ok(ReadResponse::BlockInfo(None))) => { - Err(StateServiceError::RpcError(RpcError::new_from_legacycode( - LegacyCode::InvalidParameter, - "block not found", - ))) - } - (Ok(unexpected), Ok(unexpected2)) => { - unreachable!("Unexpected responses from state service: {unexpected:?} {unexpected2:?}") - } - (Err(e), _) | (_, Err(e)) => Err(e.into()), - }?; - - let orchard_tree_response = orchard_tree_response?; - let orchard_tree = expected_read_response!(orchard_tree_response, OrchardTree) - .ok_or(StateServiceError::RpcError(RpcError::new_from_legacycode( - LegacyCode::Misc, - "missing orchard tree", - )))?; - - let final_orchard_root = match NetworkUpgrade::Nu5.activation_height(network) { - Some(activation_height) if header_obj.height() >= activation_height => { - Some(orchard_tree.root().into()) - } - _otherwise => None, - }; - - let trees = - GetBlockTrees::new(header_obj.sapling_tree_size(), orchard_tree.count()); - - let (chain_supply, value_pools) = ( - GetBlockchainInfoBalance::chain_supply(*block_info.value_pools()), - GetBlockchainInfoBalance::value_pools(*block_info.value_pools(), None), - ); - let transaction_count = transactions_response.len(); - - Ok(GetBlock::Object(Box::new( - zebra_rpc::client::BlockObject::new( - header_obj.hash(), - header_obj.confirmations(), - Some(size as i64), - Some(header_obj.height()), - Some(header_obj.version()), - Some(header_obj.merkle_root()), - Some(header_obj.block_commitments()), - Some(header_obj.final_sapling_root()), - final_orchard_root, - transaction_count, - transactions_response, - Some(header_obj.time()), - Some(header_obj.nonce()), - Some(header_obj.solution()), - Some(header_obj.bits()), - Some(header_obj.difficulty()), - Some(chain_supply), - Some(value_pools), - trees, - Some(header_obj.previous_block_hash()), - header_obj.next_block_hash(), - ), - ))) - } - more_than_two => Err(StateServiceError::RpcError(RpcError::new_from_legacycode( - LegacyCode::InvalidParameter, - format!("invalid verbosity of {more_than_two}"), - ))), - } - } - /// Returns the network type running. #[allow(deprecated)] pub fn network(&self) -> zaino_common::Network { @@ -1217,10 +893,7 @@ impl ZcashIndexer for StateServiceSubscriber { hash: String, verbose: bool, ) -> Result { - self.rpc_client - .get_block_header(hash, verbose) - .await - .map_err(|e| StateServiceError::Custom(e.to_string())) + Ok(self.indexer.get_block_header(hash, verbose).await?) } async fn z_get_block( @@ -1228,15 +901,10 @@ impl ZcashIndexer for StateServiceSubscriber { hash_or_height_string: String, verbosity: Option, ) -> Result { - let hash_or_height = HashOrHeight::from_str(&hash_or_height_string); - - StateServiceSubscriber::get_block_inner( - &self.read_state_service.clone(), - &self.data.network(), - hash_or_height?, - verbosity, - ) - .await + Ok(self + .indexer + .z_get_block(hash_or_height_string, verbosity) + .await?) } async fn get_block_deltas(&self, hash: String) -> Result { @@ -2718,37 +2386,6 @@ impl LightWalletIndexer for StateServiceSubscriber { } } -#[allow(clippy::result_large_err, deprecated)] -fn header_to_block_commitments( - header: &Header, - network: &Network, - height: Height, - final_sapling_root: [u8; 32], -) -> Result<[u8; 32], StateServiceError> { - let hash = match header.commitment(network, height).map_err(|e| { - StateServiceError::SerializationError( - zebra_chain::serialization::SerializationError::Parse( - // For some reason this error type takes a - // &'static str, and the the only way to create one - // dynamically is to leak a String. This shouldn't - // be a concern, as this error case should - // never occur when communing with a zebrad, which - // validates this field before serializing it - e.to_string().leak(), - ), - ) - })? { - zebra_chain::block::Commitment::PreSaplingReserved(bytes) => bytes, - zebra_chain::block::Commitment::FinalSaplingRoot(_root) => final_sapling_root, - zebra_chain::block::Commitment::ChainHistoryActivationReserved => [0; 32], - zebra_chain::block::Commitment::ChainHistoryRoot(root) => root.bytes_in_display_order(), - zebra_chain::block::Commitment::ChainHistoryBlockTxAuthCommitment(hash) => { - hash.bytes_in_display_order() - } - }; - Ok(hash) -} - /// An error type for median time past calculation errors #[derive(Debug, Clone)] pub enum MedianTimePast { diff --git a/packages/zaino-state/src/chain_index.rs b/packages/zaino-state/src/chain_index.rs index f5a005b76..66ce9dcd9 100644 --- a/packages/zaino-state/src/chain_index.rs +++ b/packages/zaino-state/src/chain_index.rs @@ -39,6 +39,7 @@ use tokio_util::sync::CancellationToken; use tracing::{info, instrument}; use zaino_fetch::jsonrpsee::response::{ address_deltas::{GetAddressDeltasParams, GetAddressDeltasResponse}, + block_header::GetBlockHeader, chain_tips::{ChainTip, ChainTipStatus, GetChainTipsResponse}, EmptyTxOutSetInfo, GetTxOutSetInfo, GetTxOutSetInfoResponse, }; @@ -48,7 +49,7 @@ pub use zebra_chain::parameters::Network as ZebraNetwork; use zebra_chain::serialization::ZcashSerialize; use zebra_rpc::{ client::{GetAddressBalanceRequest, GetAddressTxIdsRequest}, - methods::{AddressBalance, GetAddressUtxos}, + methods::{AddressBalance, GetAddressUtxos, GetBlock}, }; use zebra_state::HashOrHeight; @@ -574,6 +575,27 @@ pub trait ChainIndexRpcExt: ChainIndex { pool_types: PoolTypeFilter, ) -> impl std::future::Future, Self::Error>>; + /// Returns the `getblock`-shaped block for the given hash-or-height string. + /// + /// `verbosity` follows the zcashd `getblock` convention (0 = raw, 1 = object with + /// txids, 2 = object with full transaction data). + /// + /// zcashd reference: [`getblock`](https://zcash.github.io/rpc/getblock.html) + fn z_get_block( + &self, + hash_or_height: String, + verbosity: Option, + ) -> impl std::future::Future>; + + /// Returns the `getblockheader`-shaped header for the given block hash. + /// + /// zcashd reference: [`getblockheader`](https://zcash.github.io/rpc/getblockheader.html) + fn get_block_header( + &self, + hash: String, + verbose: bool, + ) -> impl std::future::Future>; + // ********** Transparent address history methods ********** /// Returns all changes for the given transparent addresses. @@ -2596,6 +2618,30 @@ impl ChainIndexRpcExt for NodeBackedChainIndexSubscrib Ok(Some(CompactBlockStream::new(channel_receiver))) } + async fn z_get_block( + &self, + hash_or_height: String, + verbosity: Option, + ) -> Result { + let id = HashOrHeight::from_str(&hash_or_height) + .map_err(|error| ChainIndexError::internal(error.to_string()))?; + self.source() + .get_block_verbose(id, verbosity) + .await + .map_err(ChainIndexError::backing_validator) + } + + async fn get_block_header( + &self, + hash: String, + verbose: bool, + ) -> Result { + self.source() + .get_block_header(hash, verbose) + .await + .map_err(ChainIndexError::backing_validator) + } + /// Returns all changes for the given transparent addresses. async fn get_address_deltas( &self, diff --git a/packages/zaino-state/src/chain_index/source.rs b/packages/zaino-state/src/chain_index/source.rs index 73e3e13de..b2352e3a7 100644 --- a/packages/zaino-state/src/chain_index/source.rs +++ b/packages/zaino-state/src/chain_index/source.rs @@ -1,6 +1,6 @@ //! Traits and types for the blockchain source thats serves zaino, commonly a validator connection. -use std::{error::Error, str::FromStr as _, sync::Arc}; +use std::{error::Error, sync::Arc}; use crate::chain_index::{ types::{BlockHash, TransactionHash}, @@ -15,6 +15,7 @@ use zaino_fetch::jsonrpsee::{ connector::{JsonRpSeeConnector, RpcRequestError}, response::{ address_deltas::{GetAddressDeltasParams, GetAddressDeltasResponse}, + block_header::GetBlockHeader, GetBlockError, GetBlockResponse, GetTransactionResponse, GetTreestateResponse, }, }; @@ -61,6 +62,26 @@ pub trait BlockchainSource: Clone + Send + Sync + 'static { id: HashOrHeight, ) -> impl SendFut>>>; + /// Returns the `getblock`-shaped verbose block for the given hash or height. + /// + /// `verbosity` follows the zcashd `getblock` convention (0 = raw, 1 = object with + /// txids, 2 = object with full transaction data). + fn get_block_verbose( + &self, + hash_or_height: HashOrHeight, + verbosity: Option, + ) -> impl SendFut>; + + /// Returns the `getblockheader`-shaped block header for the given block hash. + /// + /// When `verbose` is false the header is returned in raw hex form; when true it is + /// returned as a structured object. + fn get_block_header( + &self, + hash: String, + verbose: bool, + ) -> impl SendFut>; + // ********** Transaction methods ********** /// Returns the transaction by txid diff --git a/packages/zaino-state/src/chain_index/source/mockchain_source.rs b/packages/zaino-state/src/chain_index/source/mockchain_source.rs index 5f262527b..dde2e2552 100644 --- a/packages/zaino-state/src/chain_index/source/mockchain_source.rs +++ b/packages/zaino-state/src/chain_index/source/mockchain_source.rs @@ -1,21 +1,29 @@ //! Mock BlockchainSourceResult implementation. +use super::validator_connector::{ + build_block_header_object, build_verbose_block, confirmations_from_depth, final_orchard_root, + final_sapling_root, zebra_block_header_to_wire, +}; use super::*; use std::collections::{HashMap, HashSet}; +use std::str::FromStr as _; use std::sync::{ atomic::{AtomicU32, Ordering}, Arc, Mutex, }; use zaino_common::network::ActivationHeights; -use zaino_fetch::jsonrpsee::response::address_deltas::BlockInfo; +use zaino_fetch::jsonrpsee::response::{address_deltas::BlockInfo, block_header::GetBlockHeader}; use zebra_chain::{block::Block, orchard::tree as orchard, sapling::tree as sapling}; use zebra_chain::{ - block::Height, + block::{Height, SerializedBlock}, parameters::NetworkKind, - serialization::BytesInDisplayOrder as _, + serialization::{BytesInDisplayOrder as _, ZcashSerialize as _}, transparent::{Address, OutPoint, Output, OutputIndex}, }; -use zebra_rpc::{client::TransactionObject, methods::ValidateAddresses as _}; +use zebra_rpc::{ + client::{HexData, TransactionObject}, + methods::{GetBlock, GetBlockHeaderResponse, ValidateAddresses as _}, +}; use zebra_state::HashOrHeight; /// Build the txid → (height, tx) lookup map used by @@ -332,6 +340,65 @@ impl MockchainSource { self.active_height() as usize } + /// Resolves a hash-or-height request to a block index within the active chain. + fn resolve_index(&self, id: &HashOrHeight) -> Option { + match id { + HashOrHeight::Height(height) => self.valid_height(height.0), + HashOrHeight::Hash(hash) => self.valid_hash(hash), + } + } + + /// The zebra hash of the block after `height_index`, if it is within the active chain. + fn next_block_hash(&self, height_index: usize) -> Option { + let next = height_index + 1; + (next <= self.active_chain_height_as_usize()).then(|| self.blocks[next].hash()) + } + + /// Builds the zebra `getblockheader` response for the block at `height_index` from the + /// test-vector block and tree-root data, using the same builders as the validator path. + fn block_header_response_at( + &self, + height_index: usize, + verbose: bool, + ) -> BlockchainSourceResult { + let block = &self.blocks[height_index]; + let header = &block.header; + if !verbose { + return Ok(GetBlockHeaderResponse::Raw(HexData( + header + .zcash_serialize_to_vec() + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?, + ))); + } + + let network = mockchain_network(); + let hash = block.hash(); + let height = block.coinbase_height().ok_or_else(|| { + BlockchainSourceError::Unrecoverable("block missing coinbase height".to_string()) + })?; + let depth = self.active_height().checked_sub(height.0); + let confirmations = confirmations_from_depth(depth); + + let (sapling_root_bytes, sapling_tree_size) = match &self.roots[height_index].0 { + Some((root, size)) => (<[u8; 32]>::from(*root), *size), + None => ([0u8; 32], 0), + }; + let final_sapling_root = final_sapling_root(sapling_root_bytes, height, &network); + let next_block_hash = self.next_block_hash(height_index); + + let header_obj = build_block_header_object( + header, + hash, + height, + confirmations, + final_sapling_root, + sapling_tree_size, + next_block_hash, + &network, + )?; + Ok(GetBlockHeaderResponse::Object(Box::new(header_obj))) + } + fn block_height_at_index(&self, block_index: usize) -> Height { self.blocks[block_index] .coinbase_height() @@ -450,6 +517,82 @@ impl BlockchainSource for MockchainSource { } } + async fn get_block_verbose( + &self, + hash_or_height: HashOrHeight, + verbosity: Option, + ) -> BlockchainSourceResult { + let verbosity = verbosity.unwrap_or(1); + let height_index = self + .resolve_index(&hash_or_height) + .ok_or_else(|| BlockchainSourceError::Unrecoverable("block not found".to_string()))?; + + match verbosity { + 0 => Ok(GetBlock::Raw(SerializedBlock::from(Arc::clone( + &self.blocks[height_index], + )))), + 1 | 2 => { + let block = &self.blocks[height_index]; + let network = mockchain_network(); + + let GetBlockHeaderResponse::Object(header_obj) = + self.block_header_response_at(height_index, true)? + else { + unreachable!("`true` yields an object") + }; + + let height = block.coinbase_height().ok_or_else(|| { + BlockchainSourceError::Unrecoverable( + "block missing coinbase height".to_string(), + ) + })?; + let (orchard_root_bytes, orchard_tree_size) = match &self.roots[height_index].1 { + Some((root, size)) => (<[u8; 32]>::from(*root), *size), + None => ([0u8; 32], 0), + }; + let final_orchard_root = final_orchard_root(orchard_root_bytes, height, &network); + + // The verbose block reports the block's serialized byte size; the mock has the + // full block, so serialize it to measure. + let size = block + .zcash_serialize_to_vec() + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))? + .len() as i64; + + // `chain_supply` / `value_pools` are cumulative pool balances that the test + // vectors do not carry, so they are `None` for the mock. + Ok(build_verbose_block( + &header_obj, + block, + verbosity, + size, + final_orchard_root, + orchard_tree_size, + None, + None, + &network, + )) + } + more_than_two => Err(BlockchainSourceError::Unrecoverable(format!( + "invalid verbosity of {more_than_two}" + ))), + } + } + + async fn get_block_header( + &self, + hash: String, + verbose: bool, + ) -> BlockchainSourceResult { + let hash_or_height = HashOrHeight::from_str(&hash) + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + let height_index = self.resolve_index(&hash_or_height).ok_or_else(|| { + BlockchainSourceError::Unrecoverable("block height not in best chain".to_string()) + })?; + let header = self.block_header_response_at(height_index, verbose)?; + zebra_block_header_to_wire(header) + } + // ********** Transaction methods ********** async fn get_transaction( diff --git a/packages/zaino-state/src/chain_index/source/validator_connector.rs b/packages/zaino-state/src/chain_index/source/validator_connector.rs index 8560853e1..c9095ebd8 100644 --- a/packages/zaino-state/src/chain_index/source/validator_connector.rs +++ b/packages/zaino-state/src/chain_index/source/validator_connector.rs @@ -1,9 +1,22 @@ //! validator connected blockchain source. +use std::str::FromStr as _; + +use chrono::{DateTime, Utc}; use hex::FromHex as _; -use zaino_fetch::jsonrpsee::response::address_deltas::BlockInfo; -use zebra_chain::serialization::BytesInDisplayOrder as _; -use zebra_rpc::methods::ValidateAddresses as _; +use zaino_fetch::jsonrpsee::response::{address_deltas::BlockInfo, block_header::GetBlockHeader}; +use zebra_chain::{ + block::{Header, SerializedBlock}, + parameters::NetworkUpgrade, + serialization::{BytesInDisplayOrder as _, ZcashSerialize as _}, +}; +use zebra_rpc::{ + client::{BlockObject, GetBlockchainInfoBalance, HexData, TransactionObject}, + methods::{ + GetBlock, GetBlockHeaderObject, GetBlockHeaderResponse, GetBlockTransaction, GetBlockTrees, + ValidateAddresses as _, + }, +}; use crate::Height; @@ -112,6 +125,47 @@ impl BlockchainSource for ValidatorConnector { } } + async fn get_block_verbose( + &self, + hash_or_height: HashOrHeight, + verbosity: Option, + ) -> BlockchainSourceResult { + match self { + ValidatorConnector::State(state) => { + state.get_block_inner(hash_or_height, verbosity).await + } + ValidatorConnector::Fetch(fetch) => fetch + .get_block(hash_or_height.to_string(), verbosity) + .await + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))? + .try_into() + .map_err(|error: zebra_chain::serialization::SerializationError| { + BlockchainSourceError::Unrecoverable(error.to_string()) + }), + } + } + + async fn get_block_header( + &self, + hash: String, + verbose: bool, + ) -> BlockchainSourceResult { + match self { + ValidatorConnector::State(state) => { + let hash_or_height = HashOrHeight::from_str(&hash) + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + let header = state + .get_block_header_inner(hash_or_height, Some(verbose)) + .await?; + zebra_block_header_to_wire(header) + } + ValidatorConnector::Fetch(fetch) => fetch + .get_block_header(hash, verbose) + .await + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string())), + } + } + // ********** Transaction methods ********** // Returns the transaction, and the height of the block that transaction is in if on the best chain @@ -1077,3 +1131,413 @@ impl BlockchainSource for ValidatorConnector { } } } + +impl State { + /// Builds the `getblockheader`-shaped block header from the `ReadStateService`. + /// + /// Returns the zebra `GetBlockHeaderResponse` shape. This is the shared State-path + /// builder used by both [`ValidatorConnector::get_block_header`] (which converts it + /// to the JSON-RPC wire form) and [`State::get_block_inner`] (which reuses the + /// object fields to assemble a verbose block). + /// + /// Moved from the former `StateServiceSubscriber::get_block_header_inner`; the error + /// type is now [`BlockchainSourceError`] rather than the backend error, so the + /// zcashd-compatible `LegacyCode` on the not-in-best-chain case survives only in the + /// message string. + pub(super) async fn get_block_header_inner( + &self, + hash_or_height: HashOrHeight, + verbose: Option, + ) -> BlockchainSourceResult { + let mut state = self.read_state_service.clone(); + let verbose = verbose.unwrap_or(true); + let network = self.network.to_zebra_network(); + + let ReadResponse::BlockHeader { + header, + hash, + height, + next_block_hash, + } = state + .ready() + .and_then(|service| service.call(ReadRequest::BlockHeader(hash_or_height))) + .await + .map_err(|_| { + BlockchainSourceError::Unrecoverable("block height not in best chain".to_string()) + })? + else { + return Err(BlockchainSourceError::Unrecoverable( + "Unexpected response to BlockHeader request".to_string(), + )); + }; + + let response = if !verbose { + GetBlockHeaderResponse::Raw(HexData( + header + .zcash_serialize_to_vec() + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?, + )) + } else { + let ReadResponse::SaplingTree(sapling_tree) = state + .ready() + .and_then(|service| service.call(ReadRequest::SaplingTree(hash_or_height))) + .await + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))? + else { + return Err(BlockchainSourceError::Unrecoverable( + "Unexpected response to SaplingTree request".to_string(), + )); + }; + // This could be `None` if there's a chain reorg between state queries. + let sapling_tree = sapling_tree.ok_or_else(|| { + BlockchainSourceError::Unrecoverable("missing sapling tree for block".to_string()) + })?; + + let ReadResponse::Depth(depth) = state + .ready() + .and_then(|service| service.call(ReadRequest::Depth(hash))) + .await + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))? + else { + return Err(BlockchainSourceError::Unrecoverable( + "Unexpected response to Depth request".to_string(), + )); + }; + + // Confirmations are one more than the depth. Depth is limited by height, so + // it will never overflow an i64. + let confirmations = confirmations_from_depth(depth); + + let sapling_tree_size = sapling_tree.count(); + let final_sapling_root = + final_sapling_root(sapling_tree.root().into(), height, &network); + + let block_header = build_block_header_object( + &header, + hash, + height, + confirmations, + final_sapling_root, + sapling_tree_size, + next_block_hash, + &network, + )?; + + GetBlockHeaderResponse::Object(Box::new(block_header)) + }; + + Ok(response) + } + + /// Builds the `getblock`-shaped verbose block from the `ReadStateService`. + /// + /// Moved from the former `StateServiceSubscriber::get_block_inner`; the error type is + /// now [`BlockchainSourceError`]. + pub(super) async fn get_block_inner( + &self, + hash_or_height: HashOrHeight, + verbosity: Option, + ) -> BlockchainSourceResult { + let mut state_1 = self.read_state_service.clone(); + let verbosity = verbosity.unwrap_or(1); + match verbosity { + 0 => { + let request = ReadRequest::Block(hash_or_height); + let response = state_1 + .ready() + .and_then(|service| service.call(request)) + .await + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + let block = expected_read_response!(response, Block); + block + .map(SerializedBlock::from) + .map(GetBlock::Raw) + .ok_or_else(|| { + BlockchainSourceError::Unrecoverable("block not found".to_string()) + }) + } + 1 | 2 => { + let network = self.network.to_zebra_network(); + let state_2 = self.read_state_service.clone(); + let state_4 = self.read_state_service.clone(); + + let blockandsize_future = { + let req = ReadRequest::BlockAndSize(hash_or_height); + async move { state_1.ready().and_then(|service| service.call(req)).await } + }; + let orchard_future = { + let req = ReadRequest::OrchardTree(hash_or_height); + async move { + state_2 + .clone() + .ready() + .and_then(|service| service.call(req)) + .await + } + }; + let block_info_future = { + let req = ReadRequest::BlockInfo(hash_or_height); + async move { + state_4 + .clone() + .ready() + .and_then(|service| service.call(req)) + .await + } + }; + let (fullblock, orchard_tree_response, header, block_info) = futures::join!( + blockandsize_future, + orchard_future, + self.get_block_header_inner(hash_or_height, Some(true)), + block_info_future + ); + + let header_obj = match header? { + GetBlockHeaderResponse::Raw(_hex_data) => unreachable!( + "`true` was passed to get_block_header, an object should be returned" + ), + GetBlockHeaderResponse::Object(get_block_header_object) => { + get_block_header_object + } + }; + + let (block, size, block_info) = match (fullblock, block_info) { + ( + Ok(ReadResponse::BlockAndSize(Some((block, size)))), + Ok(ReadResponse::BlockInfo(Some(block_info))), + ) => (block, size, block_info), + (Ok(ReadResponse::Block(None)), Ok(ReadResponse::BlockInfo(None))) => { + return Err(BlockchainSourceError::Unrecoverable( + "block not found".to_string(), + )); + } + (Ok(unexpected), Ok(unexpected2)) => { + unreachable!("Unexpected responses from state service: {unexpected:?} {unexpected2:?}") + } + (Err(e), _) | (_, Err(e)) => { + return Err(BlockchainSourceError::Unrecoverable(e.to_string())); + } + }; + + let orchard_tree_response = orchard_tree_response + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + let orchard_tree = expected_read_response!(orchard_tree_response, OrchardTree) + .ok_or_else(|| { + BlockchainSourceError::Unrecoverable("missing orchard tree".to_string()) + })?; + + let final_orchard_root = + final_orchard_root(orchard_tree.root().into(), header_obj.height(), &network); + + let (chain_supply, value_pools) = ( + Some(GetBlockchainInfoBalance::chain_supply( + *block_info.value_pools(), + )), + Some(GetBlockchainInfoBalance::value_pools( + *block_info.value_pools(), + None, + )), + ); + + Ok(build_verbose_block( + &header_obj, + &block, + verbosity, + size as i64, + final_orchard_root, + orchard_tree.count(), + chain_supply, + value_pools, + &network, + )) + } + more_than_two => Err(BlockchainSourceError::Unrecoverable(format!( + "invalid verbosity of {more_than_two}" + ))), + } + } +} + +/// Confirmations are one more than the depth, or -1 when the block is not on the best +/// chain. Depth is limited by height, so it never overflows an `i64`. +pub(crate) fn confirmations_from_depth(depth: Option) -> i64 { + const NOT_IN_BEST_CHAIN_CONFIRMATIONS: i64 = -1; + depth + .map(|depth| i64::from(depth) + 1) + .unwrap_or(NOT_IN_BEST_CHAIN_CONFIRMATIONS) +} + +/// The `finalsaplingroot` field: the sapling note-commitment tree root in display +/// (big-endian) byte order once Sapling has activated, else all-zero. +pub(crate) fn final_sapling_root( + root: [u8; 32], + height: zebra_chain::block::Height, + network: &zebra_chain::parameters::Network, +) -> [u8; 32] { + match NetworkUpgrade::Sapling.activation_height(network) { + Some(activation) if height >= activation => { + let mut root = root; + root.reverse(); + root + } + _ => [0; 32], + } +} + +/// The `finalorchardroot` field: `Some(root)` once NU5 has activated, else `None`. +pub(crate) fn final_orchard_root( + root: [u8; 32], + height: zebra_chain::block::Height, + network: &zebra_chain::parameters::Network, +) -> Option<[u8; 32]> { + match NetworkUpgrade::Nu5.activation_height(network) { + Some(activation) if height >= activation => Some(root), + _ => None, + } +} + +/// Assembles a `getblockheader` object from its already-resolved primitive pieces. +/// +/// Shared by the [`ValidatorConnector`] State path (which reads the pieces from the +/// `ReadStateService`) and by `MockchainSource` (which reads them from test vectors), +/// so the two never drift. +#[allow(clippy::too_many_arguments)] +pub(crate) fn build_block_header_object( + header: &Header, + hash: zebra_chain::block::Hash, + height: zebra_chain::block::Height, + confirmations: i64, + final_sapling_root: [u8; 32], + sapling_tree_size: u64, + next_block_hash: Option, + network: &zebra_chain::parameters::Network, +) -> BlockchainSourceResult { + let mut nonce = *header.nonce; + nonce.reverse(); + let difficulty = header.difficulty_threshold.relative_to_network(network); + let block_commitments = + header_to_block_commitments(header, network, height, final_sapling_root)?; + + Ok(GetBlockHeaderObject::new( + hash, + confirmations, + height, + header.version, + header.merkle_root, + block_commitments, + final_sapling_root, + sapling_tree_size, + header.time.timestamp(), + nonce, + header.solution, + header.difficulty_threshold, + difficulty, + header.previous_block_hash, + next_block_hash, + )) +} + +/// Assembles a verbose (`verbosity` 1 or 2) `getblock` object from a resolved header +/// object plus the block's transactions and pool aggregates. +/// +/// Shared by the [`ValidatorConnector`] State path and `MockchainSource`. `chain_supply` +/// / `value_pools` are the cumulative pool balances (present for the validator, `None` +/// for the mock, whose vectors don't carry them). +#[allow(clippy::too_many_arguments)] +pub(crate) fn build_verbose_block( + header_obj: &GetBlockHeaderObject, + block: &zebra_chain::block::Block, + verbosity: u8, + size: i64, + final_orchard_root: Option<[u8; 32]>, + orchard_tree_size: u64, + chain_supply: Option, + value_pools: Option<[GetBlockchainInfoBalance; 5]>, + network: &zebra_chain::parameters::Network, +) -> GetBlock { + let transactions_response: Vec = block + .transactions + .iter() + .map(|transaction| match verbosity { + 1 => GetBlockTransaction::Hash(transaction.hash()), + 2 => GetBlockTransaction::Object(Box::new(TransactionObject::from_transaction( + transaction.clone(), + Some(header_obj.height()), + Some(header_obj.confirmations() as u32), + network, + DateTime::::from_timestamp(header_obj.time(), 0), + Some(header_obj.hash()), + // A non-optional header height indicates a mainchain block; sidechain + // data is out of scope for now. TODO: return Some(true/false) once resolved. + None, + transaction.hash(), + ))), + _ => unreachable!("verbosity known to be 1 or 2"), + }) + .collect(); + + let trees = GetBlockTrees::new(header_obj.sapling_tree_size(), orchard_tree_size); + let transaction_count = transactions_response.len(); + + GetBlock::Object(Box::new(BlockObject::new( + header_obj.hash(), + header_obj.confirmations(), + Some(size), + Some(header_obj.height()), + Some(header_obj.version()), + Some(header_obj.merkle_root()), + Some(header_obj.block_commitments()), + Some(header_obj.final_sapling_root()), + final_orchard_root, + transaction_count, + transactions_response, + Some(header_obj.time()), + Some(header_obj.nonce()), + Some(header_obj.solution()), + Some(header_obj.bits()), + Some(header_obj.difficulty()), + chain_supply, + value_pools, + trees, + Some(header_obj.previous_block_hash()), + header_obj.next_block_hash(), + ))) +} + +/// Computes the `blockcommitments` field for a block header. +/// +/// Moved verbatim (bar the error type) from the former `state.rs` +/// `header_to_block_commitments`. +pub(crate) fn header_to_block_commitments( + header: &Header, + network: &zebra_chain::parameters::Network, + height: zebra_chain::block::Height, + final_sapling_root: [u8; 32], +) -> BlockchainSourceResult<[u8; 32]> { + let hash = match header.commitment(network, height).map_err(|error| { + BlockchainSourceError::Unrecoverable(format!("invalid block commitment: {error}")) + })? { + zebra_chain::block::Commitment::PreSaplingReserved(bytes) => bytes, + zebra_chain::block::Commitment::FinalSaplingRoot(_root) => final_sapling_root, + zebra_chain::block::Commitment::ChainHistoryActivationReserved => [0; 32], + zebra_chain::block::Commitment::ChainHistoryRoot(root) => root.bytes_in_display_order(), + zebra_chain::block::Commitment::ChainHistoryBlockTxAuthCommitment(hash) => { + hash.bytes_in_display_order() + } + }; + Ok(hash) +} + +/// Converts the zebra `GetBlockHeaderResponse` (Raw/Object) into the JSON-RPC wire +/// `getblockheader` response (Compact/Verbose) expected by the backends. +/// +/// The two shapes serialize to the same JSON, so a serde round-trip is the least-error- +/// prone conversion. +pub(crate) fn zebra_block_header_to_wire( + header: GetBlockHeaderResponse, +) -> BlockchainSourceResult { + let value = serde_json::to_value(&header) + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + serde_json::from_value(value) + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string())) +} diff --git a/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs b/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs index b7552f8d5..8fef9fa05 100644 --- a/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs +++ b/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs @@ -16,8 +16,11 @@ use tokio_stream::StreamExt as _; use zaino_fetch::jsonrpsee::response::address_deltas::{ GetAddressDeltasParams, GetAddressDeltasResponse, }; -use zebra_chain::serialization::ZcashDeserializeInto; +use zaino_fetch::jsonrpsee::response::block_header::GetBlockHeader; +use zebra_chain::serialization::{ZcashDeserializeInto, ZcashSerialize as _}; use zebra_rpc::client::{GetAddressBalanceRequest, GetAddressTxIdsRequest}; +use zebra_rpc::methods::GetBlock; +use zebra_state::HashOrHeight; /// Polls the indexer's nonfinalized-state snapshot until its best-tip height /// equals `expected`, or panics after a 10 s budget. @@ -904,3 +907,92 @@ async fn get_outpoint_spenders_empty_and_single() { vec![None], ); } + +/// `z_get_block` served through the ChainIndex from the mock vectors: verbosity 0 +/// round-trips to the stored block, and verbosity 1 matches the source and reports the +/// block's hash / height / txids. +#[tokio::test(flavor = "multi_thread")] +async fn z_get_block() { + let (_blocks, _indexer, index_reader, mockchain) = + load_test_vectors_and_sync_chain_index(MockchainMode::Static).await; + let active_height = mockchain.active_height(); + + for height in [1u32, active_height / 2, active_height] { + let id = HashOrHeight::Height(zebra_chain::block::Height(height)); + let expected_block = mockchain.get_block(id).await.unwrap().unwrap(); + + // Verbosity 0: the raw serialized block round-trips to the stored block. + let GetBlock::Raw(serialized) = index_reader + .z_get_block(height.to_string(), Some(0)) + .await + .unwrap() + else { + panic!("expected a raw block at verbosity 0"); + }; + let decoded: zebra_chain::block::Block = + serialized.as_ref().zcash_deserialize_into().unwrap(); + assert_eq!(decoded, *expected_block); + + // Verbosity 1: the ChainIndex delegates to the source and the object reports the + // block's hash, height, and every txid. + let via_index = index_reader + .z_get_block(height.to_string(), Some(1)) + .await + .unwrap(); + let via_source = mockchain.get_block_verbose(id, Some(1)).await.unwrap(); + let value = serde_json::to_value(&via_index).unwrap(); + assert_eq!(value, serde_json::to_value(&via_source).unwrap()); + assert_eq!( + value["hash"].as_str().unwrap(), + expected_block.hash().to_string() + ); + assert_eq!(value["height"].as_u64().unwrap(), u64::from(height)); + assert_eq!( + value["tx"].as_array().unwrap().len(), + expected_block.transactions.len() + ); + } +} + +/// `get_block_header` served through the ChainIndex from the mock vectors: the compact +/// (non-verbose) form round-trips to the stored header bytes, and the verbose form +/// matches the source and reports the right hash / height. +#[tokio::test(flavor = "multi_thread")] +async fn get_block_header() { + let (_blocks, _indexer, index_reader, mockchain) = + load_test_vectors_and_sync_chain_index(MockchainMode::Static).await; + let active_height = mockchain.active_height(); + + for height in [1u32, active_height / 2, active_height] { + let id = HashOrHeight::Height(zebra_chain::block::Height(height)); + let block = mockchain.get_block(id).await.unwrap().unwrap(); + let hash = block.hash().to_string(); + + // Non-verbose: the compact hex decodes to the block header's serialization. + let GetBlockHeader::Compact(compact) = index_reader + .get_block_header(hash.clone(), false) + .await + .unwrap() + else { + panic!("expected a compact header when verbose = false"); + }; + assert_eq!( + hex::decode(compact).unwrap(), + block.header.zcash_serialize_to_vec().unwrap() + ); + + // Verbose: the ChainIndex delegates to the source and reports hash / height. + let via_index = index_reader + .get_block_header(hash.clone(), true) + .await + .unwrap(); + let via_source = mockchain + .get_block_header(hash.clone(), true) + .await + .unwrap(); + let value = serde_json::to_value(&via_index).unwrap(); + assert_eq!(value, serde_json::to_value(&via_source).unwrap()); + assert_eq!(value["hash"].as_str().unwrap(), hash); + assert_eq!(value["height"].as_u64().unwrap(), u64::from(height)); + } +} diff --git a/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs b/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs index 8dde1281c..c3a767a19 100644 --- a/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs +++ b/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs @@ -616,6 +616,25 @@ impl BlockchainSource for ProptestMockchain { } } + async fn get_block_verbose( + &self, + _hash_or_height: HashOrHeight, + _verbosity: Option, + ) -> BlockchainSourceResult { + // ProptestMockchain exercises sync/reorg, not the verbose getblock RPC. + unimplemented!() + } + + async fn get_block_header( + &self, + _hash: String, + _verbose: bool, + ) -> BlockchainSourceResult + { + // ProptestMockchain exercises sync/reorg, not the getblockheader RPC. + unimplemented!() + } + /// Returns the block commitment tree data by hash async fn get_commitment_tree_roots( &self, From 2a4df3fc1a5efd2d9d4136b7d59c63496c56ac93 Mon Sep 17 00:00:00 2001 From: idky137 Date: Fri, 3 Jul 2026 08:44:54 +0100 Subject: [PATCH 05/58] move methods into chain_index / blockchainsource pt 2 - get_block_deltas --- packages/zaino-state/src/backends/fetch.rs | 2 +- packages/zaino-state/src/backends/state.rs | 283 +----------------- packages/zaino-state/src/chain_index.rs | 21 ++ .../zaino-state/src/chain_index/source.rs | 7 + .../chain_index/source/mockchain_source.rs | 87 +++++- .../chain_index/source/validator_connector.rs | 272 ++++++++++++++++- .../src/chain_index/tests/mockchain_tests.rs | 36 +++ .../chain_index/tests/proptest_blockgen.rs | 8 + 8 files changed, 431 insertions(+), 285 deletions(-) diff --git a/packages/zaino-state/src/backends/fetch.rs b/packages/zaino-state/src/backends/fetch.rs index 98f1b7b67..876d8afd1 100644 --- a/packages/zaino-state/src/backends/fetch.rs +++ b/packages/zaino-state/src/backends/fetch.rs @@ -437,7 +437,7 @@ impl ZcashIndexer for FetchServiceSubscriber { /// /// Note: This method has only been implemented in `zcashd`. Zebra has no intention of supporting it. async fn get_block_deltas(&self, hash: String) -> Result { - Ok(self.fetcher.get_block_deltas(hash).await?) + Ok(self.indexer.get_block_deltas(hash).await?) } async fn get_block_header( diff --git a/packages/zaino-state/src/backends/state.rs b/packages/zaino-state/src/backends/state.rs index 59b6e748e..babbd7f37 100644 --- a/packages/zaino-state/src/backends/state.rs +++ b/packages/zaino-state/src/backends/state.rs @@ -33,7 +33,7 @@ use zaino_fetch::{ raw_transaction::validate_raw_transaction_hex, response::{ address_deltas::{GetAddressDeltasParams, GetAddressDeltasResponse}, - block_deltas::{BlockDelta, BlockDeltas, InputDelta, OutputDelta}, + block_deltas::BlockDeltas, block_header::GetBlockHeader, block_subsidy::GetBlockSubsidy, chain_tips::GetChainTipsResponse, @@ -65,7 +65,6 @@ use zcash_keys::{address::Address, encoding::AddressCodec}; use zcash_protocol::consensus::NetworkType; use zcash_transparent::address::TransparentAddress; use zebra_chain::{ - amount::{Amount, NonNegative}, block::Height, chain_tip::NetworkChainTipHeightEstimator, parameters::{ConsensusBranchId, NetworkKind, NetworkUpgrade}, @@ -74,13 +73,13 @@ use zebra_chain::{ }; use zebra_rpc::{ client::{ - GetAddressBalanceRequest, GetSubtreesByIndexResponse, GetTreestateResponse, Input, - SubtreeRpcData, TransactionObject, ValidateAddressResponse, + GetAddressBalanceRequest, GetSubtreesByIndexResponse, GetTreestateResponse, SubtreeRpcData, + TransactionObject, ValidateAddressResponse, }, methods::{ chain_tip_difficulty, AddressBalance, ConsensusBranchIdHex, GetAddressTxIdsRequest, - GetAddressUtxos, GetBlock, GetBlockHash, GetBlockTransaction, GetBlockchainInfoResponse, - GetInfo, GetRawTransaction, NetworkUpgradeInfo, NetworkUpgradeStatus, SentTransactionHash, + GetAddressUtxos, GetBlock, GetBlockHash, GetBlockchainInfoResponse, GetInfo, + GetRawTransaction, NetworkUpgradeInfo, NetworkUpgradeStatus, SentTransactionHash, TipConsensusBranch, }, server::error::LegacyCode, @@ -92,7 +91,7 @@ use chrono::Utc; use futures::TryFutureExt as _; use hex::{FromHex as _, ToHex}; use indexmap::IndexMap; -use std::{collections::HashMap, error::Error, fmt, str::FromStr, sync::Arc}; +use std::{str::FromStr, sync::Arc}; use tokio::{ sync::mpsc, time::{self, timeout}, @@ -564,56 +563,6 @@ impl StateServiceSubscriber { pub fn network(&self) -> zaino_common::Network { self.config.common.network } - - /// Returns the median time of the last 11 blocks. - async fn median_time_past( - &self, - start: &zebra_rpc::client::BlockObject, - ) -> Result { - const MEDIAN_TIME_PAST_WINDOW: usize = 11; - - let mut times = Vec::with_capacity(MEDIAN_TIME_PAST_WINDOW); - - let start_hash = start.hash().to_string(); - let time_0 = start - .time() - .ok_or_else(|| MedianTimePast::StartMissingTime { - hash: start_hash.clone(), - })?; - times.push(time_0); - - let mut prev = start.previous_block_hash(); - - for _ in 0..(MEDIAN_TIME_PAST_WINDOW - 1) { - let hash = match prev { - Some(h) => h.to_string(), - None => break, // genesis - }; - - match self.z_get_block(hash.clone(), Some(1)).await { - Ok(GetBlock::Object(obj)) => { - if let Some(t) = obj.time() { - times.push(t); - } - prev = obj.previous_block_hash(); - } - Ok(GetBlock::Raw(_)) => { - return Err(MedianTimePast::UnexpectedRaw { hash }); - } - Err(_e) => { - // Use values up to this point - break; - } - } - } - - if times.is_empty() { - return Err(MedianTimePast::EmptyWindow); - } - - times.sort_unstable(); - Ok(times[times.len() / 2]) - } } /// Extracts the diversifier and pk_d bytes from a validated Sapling @@ -908,194 +857,7 @@ impl ZcashIndexer for StateServiceSubscriber { } async fn get_block_deltas(&self, hash: String) -> Result { - // Get the block WITH the transaction data - let zblock = self.z_get_block(hash, Some(2)).await?; - - match zblock { - GetBlock::Object(boxed_block) => { - // Resolve each transparent spend's prevout ourselves: the - // verbosity-2 object from Zebra's stateless - // `TransactionObject::from_transaction` leaves input - // value/address `None`, so we look the spent output up via the - // best-chain `ReadRequest::Transaction` (finalized + - // non-finalized, so same-block prevouts resolve too). - let network = self.data.network(); - let mut state = self.read_state_service.clone(); - // Per-call cache: many inputs may reference the same prevtxid, - // so each previous transaction is fetched at most once. - let mut prevtx_cache: HashMap< - zebra_chain::transaction::Hash, - Arc, - > = HashMap::new(); - - let mut deltas = Vec::with_capacity(boxed_block.tx().len()); - for (tx_index, tx) in boxed_block.tx().iter().enumerate() { - let txo = match tx { - GetBlockTransaction::Object(txo) => txo, - GetBlockTransaction::Hash(_) => { - return Err(StateServiceError::Custom( - "Unexpected hash when expecting object".to_string(), - )) - } - }; - - let txid = txo.txid().to_string(); - - let mut inputs: Vec = Vec::new(); - for (i, vin) in txo.inputs().iter().enumerate() { - let (prevtxid, prevout) = match vin { - Input::Coinbase { .. } => continue, - Input::NonCoinbase { - txid: prevtxid, - vout: prevout, - .. - } => (prevtxid, *prevout), - }; - - let prev_hash = zebra_chain::transaction::Hash::from_str(prevtxid) - .map_err(|e| { - StateServiceError::Custom(format!( - "getblockdeltas: invalid prevout txid {prevtxid}: {e}" - )) - })?; - - let prev_tx = match prevtx_cache.get(&prev_hash) { - Some(prev_tx) => prev_tx.clone(), - None => { - let response = state - .ready() - .and_then(|service| { - service.call(ReadRequest::Transaction(prev_hash)) - }) - .await?; - let mined_tx = expected_read_response!(response, Transaction) - .ok_or_else(|| { - StateServiceError::Custom(format!( - "getblockdeltas: prevout tx {prevtxid} not in best chain" - )) - })?; - prevtx_cache.insert(prev_hash, mined_tx.tx.clone()); - mined_tx.tx - } - }; - - let output = prev_tx.outputs().get(prevout as usize).ok_or_else(|| { - StateServiceError::Custom(format!( - "getblockdeltas: prevout index {prevout} out of range for {prevtxid}" - )) - })?; - - // Nonstandard script ⇒ no derivable address ⇒ skip - // (matches the outputs branch). This is the only - // legitimate skip; it is not loss of a resolvable spend. - let address = match output.address(&network) { - Some(a) => a.to_string(), - None => continue, - }; - - // Inputs are debits, so the amount leaves the address. - let satoshis: Amount = - (-output.value().zatoshis()).try_into().map_err(|e| { - StateServiceError::Custom(format!( - "getblockdeltas: input amount out of range for {prevtxid}:{prevout}: {e}" - )) - })?; - - inputs.push(InputDelta { - address, - satoshis, - index: i as u32, - prevtxid: prevtxid.clone(), - prevout, - }); - } - - let outputs: Vec = txo - .outputs() - .iter() - .filter_map(|vout| { - let addr_opt = vout - .script_pub_key() - .addresses() - .as_ref() - .and_then(|v| if v.len() == 1 { v.first() } else { None }); - - let addr = addr_opt?.clone(); - - let output_amt: Amount = match vout.value_zat().try_into() - { - Ok(a) => a, - Err(_) => return None, - }; - - Some(OutputDelta { - address: addr, - satoshis: output_amt, - index: vout.n(), - }) - }) - .collect::>(); - - deltas.push(BlockDelta { - txid, - index: tx_index as u32, - inputs, - outputs, - }); - } - - Ok(BlockDeltas { - hash: boxed_block.hash().to_string(), - confirmations: boxed_block.confirmations(), - size: boxed_block.size().ok_or_else(|| { - StateServiceError::Custom("getblockdeltas: block size missing".into()) - })?, - height: boxed_block - .height() - .ok_or_else(|| { - StateServiceError::Custom("getblockdeltas: block height missing".into()) - })? - .0, - version: boxed_block.version().ok_or_else(|| { - StateServiceError::Custom("getblockdeltas: block version missing".into()) - })?, - merkle_root: boxed_block - .merkle_root() - .ok_or_else(|| { - StateServiceError::Custom( - "getblockdeltas: block merkle root missing".into(), - ) - })? - .encode_hex::(), - deltas, - time: boxed_block.time().ok_or_else(|| { - StateServiceError::Custom("getblockdeltas: block time missing".into()) - })?, - median_time: self.median_time_past(&boxed_block).await.map_err(|e| { - StateServiceError::Custom(format!("getblockdeltas: median_time_past: {e}")) - })?, - nonce: hex::encode(boxed_block.nonce().ok_or_else(|| { - StateServiceError::Custom("getblockdeltas: block nonce missing".into()) - })?), - bits: boxed_block - .bits() - .ok_or_else(|| { - StateServiceError::Custom("getblockdeltas: block bits missing".into()) - })? - .to_string(), - difficulty: boxed_block.difficulty().ok_or_else(|| { - StateServiceError::Custom("getblockdeltas: block difficulty missing".into()) - })?, - previous_block_hash: boxed_block - .previous_block_hash() - .map(|hash| hash.to_string()), - next_block_hash: boxed_block.next_block_hash().map(|h| h.to_string()), - }) - } - GetBlock::Raw(_serialized_block) => Err(StateServiceError::Custom( - "Unexpected raw block".to_string(), - )), - } + Ok(self.indexer.get_block_deltas(hash).await?) } async fn get_raw_mempool(&self) -> Result, Self::Error> { @@ -2386,37 +2148,6 @@ impl LightWalletIndexer for StateServiceSubscriber { } } -/// An error type for median time past calculation errors -#[derive(Debug, Clone)] -pub enum MedianTimePast { - /// The start block has no `time`. - StartMissingTime { hash: String }, - - /// Ignored verbosity. - UnexpectedRaw { hash: String }, - - /// No timestamps collected at all. - EmptyWindow, -} - -impl fmt::Display for MedianTimePast { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - MedianTimePast::StartMissingTime { hash } => { - write!(f, "start block {hash} is missing `time`") - } - MedianTimePast::UnexpectedRaw { hash } => { - write!(f, "unexpected raw payload for block {hash}") - } - MedianTimePast::EmptyWindow => { - write!(f, "no timestamps collected (empty MTP window)") - } - } - } -} - -impl Error for MedianTimePast {} - #[cfg(test)] mod tests { /// Classifies the byte-level relationship between two slices. diff --git a/packages/zaino-state/src/chain_index.rs b/packages/zaino-state/src/chain_index.rs index 66ce9dcd9..139e42636 100644 --- a/packages/zaino-state/src/chain_index.rs +++ b/packages/zaino-state/src/chain_index.rs @@ -39,6 +39,7 @@ use tokio_util::sync::CancellationToken; use tracing::{info, instrument}; use zaino_fetch::jsonrpsee::response::{ address_deltas::{GetAddressDeltasParams, GetAddressDeltasResponse}, + block_deltas::BlockDeltas, block_header::GetBlockHeader, chain_tips::{ChainTip, ChainTipStatus, GetChainTipsResponse}, EmptyTxOutSetInfo, GetTxOutSetInfo, GetTxOutSetInfoResponse, @@ -596,6 +597,15 @@ pub trait ChainIndexRpcExt: ChainIndex { verbose: bool, ) -> impl std::future::Future>; + /// Returns the `getblockdeltas`-shaped transparent input/output deltas for the block + /// with the given hash. + /// + /// zcashd reference: [`getblockdeltas`](https://zcash.github.io/rpc/getblockdeltas.html) + fn get_block_deltas( + &self, + hash: String, + ) -> impl std::future::Future>; + // ********** Transparent address history methods ********** /// Returns all changes for the given transparent addresses. @@ -2642,6 +2652,17 @@ impl ChainIndexRpcExt for NodeBackedChainIndexSubscrib .map_err(ChainIndexError::backing_validator) } + // TODO(internal-first): `getblockdeltas` is buildable from the indexed chainblocks + // + finalised/non-finalised prevout resolution. Build it internally by default once + // an internal prevout resolver (spanning non-finalised + finalised, reconstructing + // addresses from `TxOutCompact`) exists, keeping this source call as the fallback. + async fn get_block_deltas(&self, hash: String) -> Result { + self.source() + .get_block_deltas(hash) + .await + .map_err(ChainIndexError::backing_validator) + } + /// Returns all changes for the given transparent addresses. async fn get_address_deltas( &self, diff --git a/packages/zaino-state/src/chain_index/source.rs b/packages/zaino-state/src/chain_index/source.rs index b2352e3a7..9315f6548 100644 --- a/packages/zaino-state/src/chain_index/source.rs +++ b/packages/zaino-state/src/chain_index/source.rs @@ -82,6 +82,13 @@ pub trait BlockchainSource: Clone + Send + Sync + 'static { verbose: bool, ) -> impl SendFut>; + /// Returns the `getblockdeltas`-shaped transparent input/output deltas for the block + /// with the given hash. + fn get_block_deltas( + &self, + hash: String, + ) -> impl SendFut>; + // ********** Transaction methods ********** /// Returns the transaction by txid diff --git a/packages/zaino-state/src/chain_index/source/mockchain_source.rs b/packages/zaino-state/src/chain_index/source/mockchain_source.rs index dde2e2552..f7842ceb8 100644 --- a/packages/zaino-state/src/chain_index/source/mockchain_source.rs +++ b/packages/zaino-state/src/chain_index/source/mockchain_source.rs @@ -1,8 +1,9 @@ //! Mock BlockchainSourceResult implementation. use super::validator_connector::{ - build_block_header_object, build_verbose_block, confirmations_from_depth, final_orchard_root, - final_sapling_root, zebra_block_header_to_wire, + assemble_block_deltas, build_block_header_object, build_verbose_block, + confirmations_from_depth, final_orchard_root, final_sapling_root, median_of_block_times, + zebra_block_header_to_wire, }; use super::*; use std::collections::{HashMap, HashSet}; @@ -12,7 +13,9 @@ use std::sync::{ Arc, Mutex, }; use zaino_common::network::ActivationHeights; -use zaino_fetch::jsonrpsee::response::{address_deltas::BlockInfo, block_header::GetBlockHeader}; +use zaino_fetch::jsonrpsee::response::{ + address_deltas::BlockInfo, block_deltas::BlockDeltas, block_header::GetBlockHeader, +}; use zebra_chain::{block::Block, orchard::tree as orchard, sapling::tree as sapling}; use zebra_chain::{ block::{Height, SerializedBlock}, @@ -21,8 +24,8 @@ use zebra_chain::{ transparent::{Address, OutPoint, Output, OutputIndex}, }; use zebra_rpc::{ - client::{HexData, TransactionObject}, - methods::{GetBlock, GetBlockHeaderResponse, ValidateAddresses as _}, + client::{BlockObject, HexData, Input, TransactionObject}, + methods::{GetBlock, GetBlockHeaderResponse, GetBlockTransaction, ValidateAddresses as _}, }; use zebra_state::HashOrHeight; @@ -399,6 +402,39 @@ impl MockchainSource { Ok(GetBlockHeaderResponse::Object(Box::new(header_obj))) } + /// Median time past over the 11-block window ending at `start`, walking backwards via + /// verbosity-1 `getblock` lookups against the mock vectors. + async fn median_time_past(&self, start: &BlockObject) -> BlockchainSourceResult { + const MEDIAN_TIME_PAST_WINDOW: usize = 11; + let mut times = Vec::with_capacity(MEDIAN_TIME_PAST_WINDOW); + let start_time = start.time().ok_or_else(|| { + BlockchainSourceError::Unrecoverable("getblockdeltas: start block missing time".into()) + })?; + times.push(start_time); + + let mut prev = start.previous_block_hash(); + for _ in 0..(MEDIAN_TIME_PAST_WINDOW - 1) { + let Some(hash) = prev else { + break; // genesis + }; + match self + .get_block_verbose(HashOrHeight::Hash(hash), Some(1)) + .await + { + Ok(GetBlock::Object(object)) => { + if let Some(time) = object.time() { + times.push(time); + } + prev = object.previous_block_hash(); + } + Ok(GetBlock::Raw(_)) => break, + Err(_) => break, + } + } + + median_of_block_times(times) + } + fn block_height_at_index(&self, block_index: usize) -> Height { self.blocks[block_index] .coinbase_height() @@ -593,6 +629,47 @@ impl BlockchainSource for MockchainSource { zebra_block_header_to_wire(header) } + async fn get_block_deltas(&self, hash: String) -> BlockchainSourceResult { + let hash_or_height = HashOrHeight::from_str(&hash) + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + let GetBlock::Object(object) = self.get_block_verbose(hash_or_height, Some(2)).await? + else { + return Err(BlockchainSourceError::Unrecoverable( + "getblockdeltas: unexpected raw block".to_string(), + )); + }; + + // The mock holds every transaction, so prevout resolution is a direct index lookup. + let mut prevtx_cache: HashMap< + zebra_chain::transaction::Hash, + Arc, + > = HashMap::new(); + for tx in object.tx() { + let GetBlockTransaction::Object(txo) = tx else { + continue; + }; + for input in txo.inputs() { + let Input::NonCoinbase { txid: prevtxid, .. } = input else { + continue; + }; + let prev_hash = zebra_chain::transaction::Hash::from_str(prevtxid) + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + if prevtx_cache.contains_key(&prev_hash) { + continue; + } + let (_height, prev_tx) = self.txid_index.get(&prev_hash).ok_or_else(|| { + BlockchainSourceError::Unrecoverable(format!( + "getblockdeltas: prevout tx {prevtxid} not found in mock chain" + )) + })?; + prevtx_cache.insert(prev_hash, Arc::clone(prev_tx)); + } + } + + let median_time = self.median_time_past(&object).await?; + assemble_block_deltas(&object, &prevtx_cache, median_time, &mockchain_network()) + } + // ********** Transaction methods ********** async fn get_transaction( diff --git a/packages/zaino-state/src/chain_index/source/validator_connector.rs b/packages/zaino-state/src/chain_index/source/validator_connector.rs index c9095ebd8..a720b9496 100644 --- a/packages/zaino-state/src/chain_index/source/validator_connector.rs +++ b/packages/zaino-state/src/chain_index/source/validator_connector.rs @@ -1,17 +1,23 @@ //! validator connected blockchain source. +use std::collections::HashMap; use std::str::FromStr as _; use chrono::{DateTime, Utc}; -use hex::FromHex as _; -use zaino_fetch::jsonrpsee::response::{address_deltas::BlockInfo, block_header::GetBlockHeader}; +use hex::{FromHex as _, ToHex as _}; +use zaino_fetch::jsonrpsee::response::{ + address_deltas::BlockInfo, + block_deltas::{BlockDelta, BlockDeltas, InputDelta, OutputDelta}, + block_header::GetBlockHeader, +}; use zebra_chain::{ + amount::{Amount, NonNegative}, block::{Header, SerializedBlock}, parameters::NetworkUpgrade, serialization::{BytesInDisplayOrder as _, ZcashSerialize as _}, }; use zebra_rpc::{ - client::{BlockObject, GetBlockchainInfoBalance, HexData, TransactionObject}, + client::{BlockObject, GetBlockchainInfoBalance, HexData, Input, TransactionObject}, methods::{ GetBlock, GetBlockHeaderObject, GetBlockHeaderResponse, GetBlockTransaction, GetBlockTrees, ValidateAddresses as _, @@ -166,6 +172,16 @@ impl BlockchainSource for ValidatorConnector { } } + async fn get_block_deltas(&self, hash: String) -> BlockchainSourceResult { + match self { + ValidatorConnector::State(state) => state.get_block_deltas(hash).await, + ValidatorConnector::Fetch(fetch) => fetch + .get_block_deltas(hash) + .await + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string())), + } + } + // ********** Transaction methods ********** // Returns the transaction, and the height of the block that transaction is in if on the best chain @@ -1356,6 +1372,94 @@ impl State { ))), } } + + /// Builds the `getblockdeltas` response from the `ReadStateService`. + /// + /// Moved from the former `StateServiceSubscriber::get_block_deltas`; resolves each + /// spent prevout via a best-chain `ReadRequest::Transaction` (finalised + + /// non-finalised), then hands off to the shared [`assemble_block_deltas`]. + async fn get_block_deltas(&self, hash: String) -> BlockchainSourceResult { + let hash_or_height = HashOrHeight::from_str(&hash) + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + let GetBlock::Object(object) = self.get_block_inner(hash_or_height, Some(2)).await? else { + return Err(BlockchainSourceError::Unrecoverable( + "getblockdeltas: unexpected raw block".to_string(), + )); + }; + + // Per-call cache: many inputs may reference the same prevtxid, so each previous + // transaction is fetched at most once. + let mut prevtx_cache: HashMap< + zebra_chain::transaction::Hash, + Arc, + > = HashMap::new(); + for tx in object.tx() { + let GetBlockTransaction::Object(txo) = tx else { + continue; + }; + for input in txo.inputs() { + let Input::NonCoinbase { txid: prevtxid, .. } = input else { + continue; + }; + let prev_hash = zebra_chain::transaction::Hash::from_str(prevtxid) + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + if prevtx_cache.contains_key(&prev_hash) { + continue; + } + let mut state = self.read_state_service.clone(); + let response = state + .ready() + .and_then(|service| service.call(ReadRequest::Transaction(prev_hash))) + .await + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + let mined_tx = expected_read_response!(response, Transaction).ok_or_else(|| { + BlockchainSourceError::Unrecoverable(format!( + "getblockdeltas: prevout tx {prevtxid} not in best chain" + )) + })?; + prevtx_cache.insert(prev_hash, mined_tx.tx); + } + } + + let median_time = self.median_time_past(&object).await?; + let network = self.network.to_zebra_network(); + assemble_block_deltas(&object, &prevtx_cache, median_time, &network) + } + + /// Median time past over the 11-block window ending at `start`, walking backwards via + /// verbosity-1 `getblock` lookups against the `ReadStateService`. + // TODO(DRY): MockchainSource duplicates this walk; the only difference is the + // per-block fetch. A shared helper generic over an async block fetcher would unify them. + async fn median_time_past(&self, start: &BlockObject) -> BlockchainSourceResult { + const MEDIAN_TIME_PAST_WINDOW: usize = 11; + let mut times = Vec::with_capacity(MEDIAN_TIME_PAST_WINDOW); + let start_time = start.time().ok_or_else(|| { + BlockchainSourceError::Unrecoverable("getblockdeltas: start block missing time".into()) + })?; + times.push(start_time); + + let mut prev = start.previous_block_hash(); + for _ in 0..(MEDIAN_TIME_PAST_WINDOW - 1) { + let Some(hash) = prev else { + break; // genesis + }; + match self + .get_block_inner(HashOrHeight::Hash(hash), Some(1)) + .await + { + Ok(GetBlock::Object(object)) => { + if let Some(time) = object.time() { + times.push(time); + } + prev = object.previous_block_hash(); + } + Ok(GetBlock::Raw(_)) => break, + Err(_) => break, // use values collected so far + } + } + + median_of_block_times(times) + } } /// Confirmations are one more than the depth, or -1 when the block is not on the best @@ -1541,3 +1645,165 @@ pub(crate) fn zebra_block_header_to_wire( serde_json::from_value(value) .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string())) } + +/// Returns the median of a non-empty set of block times. +pub(crate) fn median_of_block_times(mut times: Vec) -> BlockchainSourceResult { + if times.is_empty() { + return Err(BlockchainSourceError::Unrecoverable( + "getblockdeltas: no block times collected for median".to_string(), + )); + } + times.sort_unstable(); + Ok(times[times.len() / 2]) +} + +/// Assembles a `getblockdeltas` response from a verbosity-2 `getblock` object plus a +/// cache of previous transactions (keyed by txid) needed to resolve each spend's address +/// and value, its median time, and the running network. +/// +/// Shared by the [`ValidatorConnector`] State path and `MockchainSource`: both resolve +/// the prevout transactions their own way, then hand the assembled inputs here so the +/// delta-shaping logic lives in one place. +pub(crate) fn assemble_block_deltas( + object: &BlockObject, + prevtx_cache: &HashMap< + zebra_chain::transaction::Hash, + Arc, + >, + median_time: i64, + network: &zebra_chain::parameters::Network, +) -> BlockchainSourceResult { + let mut deltas = Vec::with_capacity(object.tx().len()); + for (tx_index, tx) in object.tx().iter().enumerate() { + let GetBlockTransaction::Object(txo) = tx else { + return Err(BlockchainSourceError::Unrecoverable( + "getblockdeltas: unexpected hash when expecting object".to_string(), + )); + }; + let txid = txo.txid().to_string(); + + let mut inputs: Vec = Vec::new(); + for (i, vin) in txo.inputs().iter().enumerate() { + let (prevtxid, prevout) = match vin { + Input::Coinbase { .. } => continue, + Input::NonCoinbase { + txid: prevtxid, + vout: prevout, + .. + } => (prevtxid, *prevout), + }; + + let prev_hash = zebra_chain::transaction::Hash::from_str(prevtxid) + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + let prev_tx = prevtx_cache.get(&prev_hash).ok_or_else(|| { + BlockchainSourceError::Unrecoverable(format!( + "getblockdeltas: prevout tx {prevtxid} not resolved" + )) + })?; + + let output = prev_tx.outputs().get(prevout as usize).ok_or_else(|| { + BlockchainSourceError::Unrecoverable(format!( + "getblockdeltas: prevout index {prevout} out of range for {prevtxid}" + )) + })?; + + // Nonstandard script ⇒ no derivable address ⇒ skip (matches the outputs branch). + let address = match output.address(network) { + Some(address) => address.to_string(), + None => continue, + }; + + // Inputs are debits, so the amount leaves the address. + let satoshis: Amount = (-output.value().zatoshis()).try_into().map_err(|error| { + BlockchainSourceError::Unrecoverable(format!( + "getblockdeltas: input amount out of range for {prevtxid}:{prevout}: {error}" + )) + })?; + + inputs.push(InputDelta { + address, + satoshis, + index: i as u32, + prevtxid: prevtxid.clone(), + prevout, + }); + } + + let outputs: Vec = txo + .outputs() + .iter() + .filter_map(|vout| { + let address = vout + .script_pub_key() + .addresses() + .as_ref() + .and_then(|addresses| addresses.first().cloned())?; + let satoshis: Amount = vout.value_zat().try_into().ok()?; + Some(OutputDelta { + address, + satoshis, + index: vout.n(), + }) + }) + .collect(); + + deltas.push(BlockDelta { + txid, + index: tx_index as u32, + inputs, + outputs, + }); + } + + Ok(BlockDeltas { + hash: object.hash().to_string(), + confirmations: object.confirmations(), + size: object.size().ok_or_else(|| { + BlockchainSourceError::Unrecoverable("getblockdeltas: block size missing".to_string()) + })?, + height: object + .height() + .ok_or_else(|| { + BlockchainSourceError::Unrecoverable( + "getblockdeltas: block height missing".to_string(), + ) + })? + .0, + version: object.version().ok_or_else(|| { + BlockchainSourceError::Unrecoverable( + "getblockdeltas: block version missing".to_string(), + ) + })?, + merkle_root: object + .merkle_root() + .ok_or_else(|| { + BlockchainSourceError::Unrecoverable( + "getblockdeltas: block merkle root missing".to_string(), + ) + })? + .encode_hex::(), + deltas, + time: object.time().ok_or_else(|| { + BlockchainSourceError::Unrecoverable("getblockdeltas: block time missing".to_string()) + })?, + median_time, + nonce: hex::encode(object.nonce().ok_or_else(|| { + BlockchainSourceError::Unrecoverable("getblockdeltas: block nonce missing".to_string()) + })?), + bits: object + .bits() + .ok_or_else(|| { + BlockchainSourceError::Unrecoverable( + "getblockdeltas: block bits missing".to_string(), + ) + })? + .to_string(), + difficulty: object.difficulty().ok_or_else(|| { + BlockchainSourceError::Unrecoverable( + "getblockdeltas: block difficulty missing".to_string(), + ) + })?, + previous_block_hash: object.previous_block_hash().map(|hash| hash.to_string()), + next_block_hash: object.next_block_hash().map(|hash| hash.to_string()), + }) +} diff --git a/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs b/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs index 8fef9fa05..2ab98a4e4 100644 --- a/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs +++ b/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs @@ -996,3 +996,39 @@ async fn get_block_header() { assert_eq!(value["height"].as_u64().unwrap(), u64::from(height)); } } + +/// `get_block_deltas` served through the ChainIndex from the mock vectors: it matches the +/// source, reports the right block hash / height, and surfaces transparent deltas. +#[tokio::test(flavor = "multi_thread")] +async fn get_block_deltas() { + let (_blocks, _indexer, index_reader, mockchain) = + load_test_vectors_and_sync_chain_index(MockchainMode::Static).await; + let active_height = mockchain.active_height(); + + let mut saw_delta_entries = false; + for height in [1u32, active_height / 2, active_height] { + let id = HashOrHeight::Height(zebra_chain::block::Height(height)); + let block = mockchain.get_block(id).await.unwrap().unwrap(); + let hash = block.hash().to_string(); + + let via_index = index_reader.get_block_deltas(hash.clone()).await.unwrap(); + let via_source = mockchain.get_block_deltas(hash.clone()).await.unwrap(); + assert_eq!( + serde_json::to_value(&via_index).unwrap(), + serde_json::to_value(&via_source).unwrap() + ); + assert_eq!(via_index.hash, hash); + assert_eq!(via_index.height, height); + if via_index + .deltas + .iter() + .any(|delta| !delta.inputs.is_empty() || !delta.outputs.is_empty()) + { + saw_delta_entries = true; + } + } + assert!( + saw_delta_entries, + "expected transparent deltas in at least one sampled block" + ); +} diff --git a/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs b/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs index c3a767a19..bd1450d4e 100644 --- a/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs +++ b/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs @@ -635,6 +635,14 @@ impl BlockchainSource for ProptestMockchain { unimplemented!() } + async fn get_block_deltas( + &self, + _hash: String, + ) -> BlockchainSourceResult { + // ProptestMockchain exercises sync/reorg, not the getblockdeltas RPC. + unimplemented!() + } + /// Returns the block commitment tree data by hash async fn get_commitment_tree_roots( &self, From 4b2c2315433005a4bda905f8c9f7f229ca76e2a9 Mon Sep 17 00:00:00 2001 From: idky137 Date: Fri, 3 Jul 2026 08:55:44 +0100 Subject: [PATCH 06/58] move methods into chain_index / blockchainsource - get_difficulty --- packages/zaino-state/src/backends/fetch.rs | 2 +- packages/zaino-state/src/backends/state.rs | 13 +---------- packages/zaino-state/src/chain_index.rs | 16 +++++++++++++ .../zaino-state/src/chain_index/source.rs | 4 ++++ .../chain_index/source/mockchain_source.rs | 13 +++++++++++ .../chain_index/source/validator_connector.rs | 23 +++++++++++++++++-- .../src/chain_index/tests/mockchain_tests.rs | 16 +++++++++++++ .../chain_index/tests/proptest_blockgen.rs | 5 ++++ 8 files changed, 77 insertions(+), 15 deletions(-) diff --git a/packages/zaino-state/src/backends/fetch.rs b/packages/zaino-state/src/backends/fetch.rs index 876d8afd1..f08c4c6f0 100644 --- a/packages/zaino-state/src/backends/fetch.rs +++ b/packages/zaino-state/src/backends/fetch.rs @@ -334,7 +334,7 @@ impl ZcashIndexer for FetchServiceSubscriber { /// method: post /// tags: blockchain async fn get_difficulty(&self) -> Result { - Ok(self.fetcher.get_difficulty().await?.0) + Ok(self.indexer.get_difficulty().await?) } async fn get_block_subsidy(&self, height: u32) -> Result { diff --git a/packages/zaino-state/src/backends/state.rs b/packages/zaino-state/src/backends/state.rs index babbd7f37..2e879c341 100644 --- a/packages/zaino-state/src/backends/state.rs +++ b/packages/zaino-state/src/backends/state.rs @@ -633,18 +633,7 @@ impl ZcashIndexer for StateServiceSubscriber { } async fn get_difficulty(&self) -> Result { - chain_tip_difficulty( - self.config.common.network.to_zebra_network(), - self.read_state_service.clone(), - false, - ) - .await - .map_err(|e| { - StateServiceError::RpcError(RpcError::new_from_errorobject( - e, - "failed to get difficulty", - )) - }) + Ok(self.indexer.get_difficulty().await?) } async fn get_block_subsidy(&self, height: u32) -> Result { diff --git a/packages/zaino-state/src/chain_index.rs b/packages/zaino-state/src/chain_index.rs index 139e42636..9fe87a3c0 100644 --- a/packages/zaino-state/src/chain_index.rs +++ b/packages/zaino-state/src/chain_index.rs @@ -606,6 +606,12 @@ pub trait ChainIndexRpcExt: ChainIndex { hash: String, ) -> impl std::future::Future>; + /// Returns the proof-of-work difficulty of the best chain as a multiple of the + /// minimum difficulty. + /// + /// zcashd reference: [`getdifficulty`](https://zcash.github.io/rpc/getdifficulty.html) + fn get_difficulty(&self) -> impl std::future::Future>; + // ********** Transparent address history methods ********** /// Returns all changes for the given transparent addresses. @@ -2663,6 +2669,16 @@ impl ChainIndexRpcExt for NodeBackedChainIndexSubscrib .map_err(ChainIndexError::backing_validator) } + // `getdifficulty` is the difficulty-adjusted expected difficulty (over a block + // window), not the tip block's stored bits, so it cannot be built from indexed data: + // always delegate to the backing validator. + async fn get_difficulty(&self) -> Result { + self.source() + .get_difficulty() + .await + .map_err(ChainIndexError::backing_validator) + } + /// Returns all changes for the given transparent addresses. async fn get_address_deltas( &self, diff --git a/packages/zaino-state/src/chain_index/source.rs b/packages/zaino-state/src/chain_index/source.rs index 9315f6548..7776d9667 100644 --- a/packages/zaino-state/src/chain_index/source.rs +++ b/packages/zaino-state/src/chain_index/source.rs @@ -121,6 +121,10 @@ pub trait BlockchainSource: Clone + Send + Sync + 'static { &self, ) -> impl SendFut>>; + /// Returns the proof-of-work difficulty of the best chain as a multiple of the + /// minimum difficulty (the `getdifficulty` RPC value). + fn get_difficulty(&self) -> impl SendFut>; + /// Returns the sapling and orchard treestate by hash fn get_treestate(&self, id: BlockHash) -> impl SendFut>; diff --git a/packages/zaino-state/src/chain_index/source/mockchain_source.rs b/packages/zaino-state/src/chain_index/source/mockchain_source.rs index f7842ceb8..0f0ca4478 100644 --- a/packages/zaino-state/src/chain_index/source/mockchain_source.rs +++ b/packages/zaino-state/src/chain_index/source/mockchain_source.rs @@ -670,6 +670,19 @@ impl BlockchainSource for MockchainSource { assemble_block_deltas(&object, &prevtx_cache, median_time, &mockchain_network()) } + // ********** Chain methods ********** + + async fn get_difficulty(&self) -> BlockchainSourceResult { + let tip_index = self.active_chain_height_as_usize(); + let tip_block = self.blocks.get(tip_index).ok_or_else(|| { + BlockchainSourceError::Unrecoverable("mock chain has no tip block".to_string()) + })?; + Ok(tip_block + .header + .difficulty_threshold + .relative_to_network(&mockchain_network())) + } + // ********** Transaction methods ********** async fn get_transaction( diff --git a/packages/zaino-state/src/chain_index/source/validator_connector.rs b/packages/zaino-state/src/chain_index/source/validator_connector.rs index a720b9496..941aec4a5 100644 --- a/packages/zaino-state/src/chain_index/source/validator_connector.rs +++ b/packages/zaino-state/src/chain_index/source/validator_connector.rs @@ -19,8 +19,8 @@ use zebra_chain::{ use zebra_rpc::{ client::{BlockObject, GetBlockchainInfoBalance, HexData, Input, TransactionObject}, methods::{ - GetBlock, GetBlockHeaderObject, GetBlockHeaderResponse, GetBlockTransaction, GetBlockTrees, - ValidateAddresses as _, + chain_tip_difficulty, GetBlock, GetBlockHeaderObject, GetBlockHeaderResponse, + GetBlockTransaction, GetBlockTrees, ValidateAddresses as _, }, }; @@ -182,6 +182,25 @@ impl BlockchainSource for ValidatorConnector { } } + // ********** Chain methods ********** + + async fn get_difficulty(&self) -> BlockchainSourceResult { + match self { + ValidatorConnector::State(state) => chain_tip_difficulty( + state.network.to_zebra_network(), + state.read_state_service.clone(), + false, + ) + .await + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string())), + ValidatorConnector::Fetch(fetch) => Ok(fetch + .get_difficulty() + .await + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))? + .0), + } + } + // ********** Transaction methods ********** // Returns the transaction, and the height of the block that transaction is in if on the best chain diff --git a/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs b/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs index 2ab98a4e4..c287b22d3 100644 --- a/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs +++ b/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs @@ -1032,3 +1032,19 @@ async fn get_block_deltas() { "expected transparent deltas in at least one sampled block" ); } + +/// `get_difficulty` served through the ChainIndex from the mock vectors matches the +/// source and is a positive difficulty value. +#[tokio::test(flavor = "multi_thread")] +async fn get_difficulty() { + let (_blocks, _indexer, index_reader, mockchain) = + load_test_vectors_and_sync_chain_index(MockchainMode::Static).await; + + let via_index = index_reader.get_difficulty().await.unwrap(); + let via_source = mockchain.get_difficulty().await.unwrap(); + assert_eq!(via_index, via_source); + assert!( + via_index > 0.0, + "difficulty should be positive, got {via_index}" + ); +} diff --git a/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs b/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs index bd1450d4e..ff6229369 100644 --- a/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs +++ b/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs @@ -643,6 +643,11 @@ impl BlockchainSource for ProptestMockchain { unimplemented!() } + async fn get_difficulty(&self) -> BlockchainSourceResult { + // ProptestMockchain exercises sync/reorg, not the getdifficulty RPC. + unimplemented!() + } + /// Returns the block commitment tree data by hash async fn get_commitment_tree_roots( &self, From 91c3772e95578b1e0f7f8e3f3d3c9923394b1957 Mon Sep 17 00:00:00 2001 From: idky137 Date: Fri, 3 Jul 2026 09:17:10 +0100 Subject: [PATCH 07/58] move methods into chain_index / blockchainsource - proxy methods --- packages/zaino-state/src/backends/fetch.rs | 14 +-- packages/zaino-state/src/backends/state.rs | 26 ++-- packages/zaino-state/src/chain_index.rs | 111 +++++++++++++++++- .../zaino-state/src/chain_index/source.rs | 49 +++++++- .../chain_index/source/mockchain_source.rs | 63 ++++++++++ .../chain_index/source/validator_connector.rs | 83 ++++++++++++- .../chain_index/tests/proptest_blockgen.rs | 49 ++++++++ 7 files changed, 364 insertions(+), 31 deletions(-) diff --git a/packages/zaino-state/src/backends/fetch.rs b/packages/zaino-state/src/backends/fetch.rs index f08c4c6f0..54e37a1ee 100644 --- a/packages/zaino-state/src/backends/fetch.rs +++ b/packages/zaino-state/src/backends/fetch.rs @@ -282,7 +282,7 @@ impl ZcashIndexer for FetchServiceSubscriber { /// in Zebra's [`GetInfo`]. Zebra uses the field names and formats from the /// [zcashd code](https://github.com/zcash/zcash/blob/v4.6.0-1/src/rpc/misc.cpp#L86-L87). async fn get_info(&self) -> Result { - Ok(self.fetcher.get_info().await?.into()) + Ok(self.indexer.get_info().await?) } /// Returns blockchain state information, as a [`GetBlockchainInfoResponse`] JSON struct. @@ -325,7 +325,7 @@ impl ZcashIndexer for FetchServiceSubscriber { } async fn get_peer_info(&self) -> Result { - Ok(self.fetcher.get_peer_info().await?) + Ok(self.indexer.get_peer_info().await?) } /// Returns the proof-of-work difficulty as a multiple of the minimum difficulty. @@ -338,7 +338,7 @@ impl ZcashIndexer for FetchServiceSubscriber { } async fn get_block_subsidy(&self, height: u32) -> Result { - Ok(self.fetcher.get_block_subsidy(height).await?) + Ok(self.indexer.get_block_subsidy(height).await?) } /// Returns the total balance of a provided `addresses` in an [`AddressBalance`] instance. @@ -449,7 +449,7 @@ impl ZcashIndexer for FetchServiceSubscriber { } async fn get_mining_info(&self) -> Result { - Ok(self.fetcher.get_mining_info().await?) + Ok(self.indexer.get_mining_info().await?) } /// Returns statistics about the unspent transaction output set. @@ -800,14 +800,14 @@ impl ZcashIndexer for FetchServiceSubscriber { n: u32, include_mempool: Option, ) -> Result { - Ok(self.fetcher.get_tx_out(txid, n, include_mempool).await?) + Ok(self.indexer.get_tx_out(txid, n, include_mempool).await?) } async fn get_spent_info( &self, request: GetSpentInfoRequest, ) -> Result { - Ok(self.fetcher.get_spent_info(request).await?) + Ok(self.indexer.get_spent_info(request).await?) } async fn chain_height(&self) -> Result { @@ -888,7 +888,7 @@ impl ZcashIndexer for FetchServiceSubscriber { blocks: Option, height: Option, ) -> Result { - Ok(self.fetcher.get_network_sol_ps(blocks, height).await?) + Ok(self.indexer.get_network_sol_ps(blocks, height).await?) } } diff --git a/packages/zaino-state/src/backends/state.rs b/packages/zaino-state/src/backends/state.rs index 2e879c341..337a24d2e 100644 --- a/packages/zaino-state/src/backends/state.rs +++ b/packages/zaino-state/src/backends/state.rs @@ -601,13 +601,7 @@ impl ZcashIndexer for StateServiceSubscriber { type Error = StateServiceError; async fn get_info(&self) -> Result { - // A number of these fields are difficult to access from the state service - // TODO: Fix this - self.rpc_client - .get_info() - .await - .map(GetInfo::from) - .map_err(|e| StateServiceError::Custom(e.to_string())) + Ok(self.indexer.get_info().await?) } /// Returns all changes for an address. @@ -637,10 +631,7 @@ impl ZcashIndexer for StateServiceSubscriber { } async fn get_block_subsidy(&self, height: u32) -> Result { - self.rpc_client - .get_block_subsidy(height) - .await - .map_err(|e| StateServiceError::Custom(e.to_string())) + Ok(self.indexer.get_block_subsidy(height).await?) } async fn get_blockchain_info(&self) -> Result { @@ -802,7 +793,7 @@ impl ZcashIndexer for StateServiceSubscriber { } async fn get_peer_info(&self) -> Result { - Ok(self.rpc_client.get_peer_info().await?) + Ok(self.indexer.get_peer_info().await?) } async fn z_get_address_balance( @@ -946,7 +937,7 @@ impl ZcashIndexer for StateServiceSubscriber { } async fn get_mining_info(&self) -> Result { - Ok(self.rpc_client.get_mining_info().await?) + Ok(self.indexer.get_mining_info().await?) } /// Returns statistics about the unspent transaction output set. @@ -1235,14 +1226,14 @@ impl ZcashIndexer for StateServiceSubscriber { n: u32, include_mempool: Option, ) -> Result { - Ok(self.rpc_client.get_tx_out(txid, n, include_mempool).await?) + Ok(self.indexer.get_tx_out(txid, n, include_mempool).await?) } async fn get_spent_info( &self, request: GetSpentInfoRequest, ) -> Result { - Ok(self.rpc_client.get_spent_info(request).await?) + Ok(self.indexer.get_spent_info(request).await?) } async fn get_address_tx_ids( @@ -1283,10 +1274,7 @@ impl ZcashIndexer for StateServiceSubscriber { blocks: Option, height: Option, ) -> Result { - self.rpc_client - .get_network_sol_ps(blocks, height) - .await - .map_err(|e| StateServiceError::Custom(e.to_string())) + Ok(self.indexer.get_network_sol_ps(blocks, height).await?) } // Helper function, to get the chain height in rpc implementations diff --git a/packages/zaino-state/src/chain_index.rs b/packages/zaino-state/src/chain_index.rs index 9fe87a3c0..0fcf90ffc 100644 --- a/packages/zaino-state/src/chain_index.rs +++ b/packages/zaino-state/src/chain_index.rs @@ -41,8 +41,12 @@ use zaino_fetch::jsonrpsee::response::{ address_deltas::{GetAddressDeltasParams, GetAddressDeltasResponse}, block_deltas::BlockDeltas, block_header::GetBlockHeader, + block_subsidy::GetBlockSubsidy, chain_tips::{ChainTip, ChainTipStatus, GetChainTipsResponse}, - EmptyTxOutSetInfo, GetTxOutSetInfo, GetTxOutSetInfoResponse, + mining_info::GetMiningInfoWire, + peer_info::GetPeerInfo, + EmptyTxOutSetInfo, GetNetworkSolPsResponse, GetSpentInfoRequest, GetSpentInfoResponse, + GetTxOutResponse, GetTxOutSetInfo, GetTxOutSetInfoResponse, }; use zaino_proto::proto::utils::{compact_block_with_pool_types, PoolTypeFilter}; use zebra_chain::parameters::ConsensusBranchId; @@ -50,7 +54,7 @@ pub use zebra_chain::parameters::Network as ZebraNetwork; use zebra_chain::serialization::ZcashSerialize; use zebra_rpc::{ client::{GetAddressBalanceRequest, GetAddressTxIdsRequest}, - methods::{AddressBalance, GetAddressUtxos, GetBlock}, + methods::{AddressBalance, GetAddressUtxos, GetBlock, GetInfo}, }; use zebra_state::HashOrHeight; @@ -612,6 +616,48 @@ pub trait ChainIndexRpcExt: ChainIndex { /// zcashd reference: [`getdifficulty`](https://zcash.github.io/rpc/getdifficulty.html) fn get_difficulty(&self) -> impl std::future::Future>; + // ********** Node-passthrough methods ********** + // + // No local-index equivalent; always delegate to the backing validator. + + /// Returns the `getinfo` response. + fn get_info(&self) -> impl std::future::Future>; + + /// Returns the `getpeerinfo` response. + fn get_peer_info(&self) -> impl std::future::Future>; + + /// Returns the `getblocksubsidy` response at the given height. + fn get_block_subsidy( + &self, + height: u32, + ) -> impl std::future::Future>; + + /// Returns the `getmininginfo` response. + fn get_mining_info( + &self, + ) -> impl std::future::Future>; + + /// Returns the `gettxout` response for the given outpoint. + fn get_tx_out( + &self, + txid: String, + n: u32, + include_mempool: Option, + ) -> impl std::future::Future>; + + /// Returns the `getspentinfo` response for the given request. + fn get_spent_info( + &self, + request: GetSpentInfoRequest, + ) -> impl std::future::Future>; + + /// Returns the `getnetworksolps` response. + fn get_network_sol_ps( + &self, + blocks: Option, + height: Option, + ) -> impl std::future::Future>; + // ********** Transparent address history methods ********** /// Returns all changes for the given transparent addresses. @@ -2679,6 +2725,67 @@ impl ChainIndexRpcExt for NodeBackedChainIndexSubscrib .map_err(ChainIndexError::backing_validator) } + async fn get_info(&self) -> Result { + self.source() + .get_info() + .await + .map_err(ChainIndexError::backing_validator) + } + + async fn get_peer_info(&self) -> Result { + self.source() + .get_peer_info() + .await + .map_err(ChainIndexError::backing_validator) + } + + async fn get_block_subsidy(&self, height: u32) -> Result { + self.source() + .get_block_subsidy(height) + .await + .map_err(ChainIndexError::backing_validator) + } + + async fn get_mining_info(&self) -> Result { + self.source() + .get_mining_info() + .await + .map_err(ChainIndexError::backing_validator) + } + + async fn get_tx_out( + &self, + txid: String, + n: u32, + include_mempool: Option, + ) -> Result { + self.source() + .get_tx_out(txid, n, include_mempool) + .await + .map_err(ChainIndexError::backing_validator) + } + + async fn get_spent_info( + &self, + request: GetSpentInfoRequest, + ) -> Result { + self.source() + .get_spent_info(request) + .await + .map_err(ChainIndexError::backing_validator) + } + + async fn get_network_sol_ps( + &self, + blocks: Option, + height: Option, + ) -> Result { + self.source() + .get_network_sol_ps(blocks, height) + .await + .map_err(ChainIndexError::backing_validator) + } + /// Returns all changes for the given transparent addresses. async fn get_address_deltas( &self, diff --git a/packages/zaino-state/src/chain_index/source.rs b/packages/zaino-state/src/chain_index/source.rs index 7776d9667..8d50b06ec 100644 --- a/packages/zaino-state/src/chain_index/source.rs +++ b/packages/zaino-state/src/chain_index/source.rs @@ -16,7 +16,11 @@ use zaino_fetch::jsonrpsee::{ response::{ address_deltas::{GetAddressDeltasParams, GetAddressDeltasResponse}, block_header::GetBlockHeader, - GetBlockError, GetBlockResponse, GetTransactionResponse, GetTreestateResponse, + block_subsidy::GetBlockSubsidy, + mining_info::GetMiningInfoWire, + peer_info::GetPeerInfo, + GetBlockError, GetBlockResponse, GetNetworkSolPsResponse, GetSpentInfoRequest, + GetSpentInfoResponse, GetTransactionResponse, GetTreestateResponse, GetTxOutResponse, }, }; use zcash_primitives::merkle_tree::{read_commitment_tree, write_commitment_tree}; @@ -25,7 +29,7 @@ use zebra_chain::{ }; use zebra_rpc::{ client::{GetAddressBalanceRequest, GetAddressTxIdsRequest}, - methods::{AddressBalance, GetAddressUtxos}, + methods::{AddressBalance, GetAddressUtxos, GetInfo}, }; use zebra_state::{HashOrHeight, ReadRequest, ReadResponse, ReadStateService}; @@ -125,6 +129,47 @@ pub trait BlockchainSource: Clone + Send + Sync + 'static { /// minimum difficulty (the `getdifficulty` RPC value). fn get_difficulty(&self) -> impl SendFut>; + // ********** Node-passthrough methods ********** + // + // These have no local-index equivalent and always proxy to the backing validator's + // JSON-RPC interface. + + /// Returns the `getinfo` response. + fn get_info(&self) -> impl SendFut>; + + /// Returns the `getpeerinfo` response. + fn get_peer_info(&self) -> impl SendFut>; + + /// Returns the `getblocksubsidy` response at the given height. + fn get_block_subsidy( + &self, + height: u32, + ) -> impl SendFut>; + + /// Returns the `getmininginfo` response. + fn get_mining_info(&self) -> impl SendFut>; + + /// Returns the `gettxout` response for the given outpoint. + fn get_tx_out( + &self, + txid: String, + n: u32, + include_mempool: Option, + ) -> impl SendFut>; + + /// Returns the `getspentinfo` response for the given request. + fn get_spent_info( + &self, + request: GetSpentInfoRequest, + ) -> impl SendFut>; + + /// Returns the `getnetworksolps` response. + fn get_network_sol_ps( + &self, + blocks: Option, + height: Option, + ) -> impl SendFut>; + /// Returns the sapling and orchard treestate by hash fn get_treestate(&self, id: BlockHash) -> impl SendFut>; diff --git a/packages/zaino-state/src/chain_index/source/mockchain_source.rs b/packages/zaino-state/src/chain_index/source/mockchain_source.rs index 0f0ca4478..83e066dc4 100644 --- a/packages/zaino-state/src/chain_index/source/mockchain_source.rs +++ b/packages/zaino-state/src/chain_index/source/mockchain_source.rs @@ -683,6 +683,69 @@ impl BlockchainSource for MockchainSource { .relative_to_network(&mockchain_network())) } + // ********** Node-passthrough methods ********** + // + // These are node-only RPCs with no chain data in the vectors. Test vectors must be + // extended to let MockchainSource serve them; tracked by the update-test-vectors + // follow-up (see "Future work"). + + async fn get_info(&self) -> BlockchainSourceResult { + unimplemented!("MockchainSource cannot serve get_info until test vectors are extended") + } + + async fn get_peer_info( + &self, + ) -> BlockchainSourceResult { + unimplemented!("MockchainSource cannot serve get_peer_info until test vectors are extended") + } + + async fn get_block_subsidy( + &self, + _height: u32, + ) -> BlockchainSourceResult + { + unimplemented!( + "MockchainSource cannot serve get_block_subsidy until test vectors are extended" + ) + } + + async fn get_mining_info( + &self, + ) -> BlockchainSourceResult + { + unimplemented!( + "MockchainSource cannot serve get_mining_info until test vectors are extended" + ) + } + + async fn get_tx_out( + &self, + _txid: String, + _n: u32, + _include_mempool: Option, + ) -> BlockchainSourceResult { + unimplemented!("MockchainSource cannot serve get_tx_out until test vectors are extended") + } + + async fn get_spent_info( + &self, + _request: zaino_fetch::jsonrpsee::response::GetSpentInfoRequest, + ) -> BlockchainSourceResult { + unimplemented!( + "MockchainSource cannot serve get_spent_info until test vectors are extended" + ) + } + + async fn get_network_sol_ps( + &self, + _blocks: Option, + _height: Option, + ) -> BlockchainSourceResult { + unimplemented!( + "MockchainSource cannot serve get_network_sol_ps until test vectors are extended" + ) + } + // ********** Transaction methods ********** async fn get_transaction( diff --git a/packages/zaino-state/src/chain_index/source/validator_connector.rs b/packages/zaino-state/src/chain_index/source/validator_connector.rs index 941aec4a5..450a72744 100644 --- a/packages/zaino-state/src/chain_index/source/validator_connector.rs +++ b/packages/zaino-state/src/chain_index/source/validator_connector.rs @@ -9,6 +9,10 @@ use zaino_fetch::jsonrpsee::response::{ address_deltas::BlockInfo, block_deltas::{BlockDelta, BlockDeltas, InputDelta, OutputDelta}, block_header::GetBlockHeader, + block_subsidy::GetBlockSubsidy, + mining_info::GetMiningInfoWire, + peer_info::GetPeerInfo, + GetNetworkSolPsResponse, GetSpentInfoRequest, GetSpentInfoResponse, GetTxOutResponse, }; use zebra_chain::{ amount::{Amount, NonNegative}, @@ -20,7 +24,7 @@ use zebra_rpc::{ client::{BlockObject, GetBlockchainInfoBalance, HexData, Input, TransactionObject}, methods::{ chain_tip_difficulty, GetBlock, GetBlockHeaderObject, GetBlockHeaderResponse, - GetBlockTransaction, GetBlockTrees, ValidateAddresses as _, + GetBlockTransaction, GetBlockTrees, GetInfo, ValidateAddresses as _, }, }; @@ -69,6 +73,18 @@ pub enum ValidatorConnector { Fetch(JsonRpSeeConnector), } +impl ValidatorConnector { + /// The JSON-RPC connector for this validator, used by the node-passthrough RPCs that + /// have no local-index equivalent. The `State` variant proxies these through its + /// `mempool_fetcher` until the `ReadStateService` can serve them. + fn json_rpc_connector(&self) -> &JsonRpSeeConnector { + match self { + ValidatorConnector::State(state) => &state.mempool_fetcher, + ValidatorConnector::Fetch(fetch) => fetch, + } + } +} + impl BlockchainSource for ValidatorConnector { // ********** Block methods ********** @@ -201,6 +217,71 @@ impl BlockchainSource for ValidatorConnector { } } + // ********** Node-passthrough methods ********** + + async fn get_info(&self) -> BlockchainSourceResult { + Ok(self + .json_rpc_connector() + .get_info() + .await + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))? + .into()) + } + + async fn get_peer_info(&self) -> BlockchainSourceResult { + self.json_rpc_connector() + .get_peer_info() + .await + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string())) + } + + async fn get_block_subsidy(&self, height: u32) -> BlockchainSourceResult { + self.json_rpc_connector() + .get_block_subsidy(height) + .await + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string())) + } + + async fn get_mining_info(&self) -> BlockchainSourceResult { + self.json_rpc_connector() + .get_mining_info() + .await + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string())) + } + + async fn get_tx_out( + &self, + txid: String, + n: u32, + include_mempool: Option, + ) -> BlockchainSourceResult { + self.json_rpc_connector() + .get_tx_out(txid, n, include_mempool) + .await + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string())) + } + + async fn get_spent_info( + &self, + request: GetSpentInfoRequest, + ) -> BlockchainSourceResult { + self.json_rpc_connector() + .get_spent_info(request) + .await + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string())) + } + + async fn get_network_sol_ps( + &self, + blocks: Option, + height: Option, + ) -> BlockchainSourceResult { + self.json_rpc_connector() + .get_network_sol_ps(blocks, height) + .await + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string())) + } + // ********** Transaction methods ********** // Returns the transaction, and the height of the block that transaction is in if on the best chain diff --git a/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs b/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs index ff6229369..c24b6e499 100644 --- a/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs +++ b/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs @@ -648,6 +648,55 @@ impl BlockchainSource for ProptestMockchain { unimplemented!() } + async fn get_info(&self) -> BlockchainSourceResult { + unimplemented!() + } + + async fn get_peer_info( + &self, + ) -> BlockchainSourceResult { + unimplemented!() + } + + async fn get_block_subsidy( + &self, + _height: u32, + ) -> BlockchainSourceResult + { + unimplemented!() + } + + async fn get_mining_info( + &self, + ) -> BlockchainSourceResult + { + unimplemented!() + } + + async fn get_tx_out( + &self, + _txid: String, + _n: u32, + _include_mempool: Option, + ) -> BlockchainSourceResult { + unimplemented!() + } + + async fn get_spent_info( + &self, + _request: zaino_fetch::jsonrpsee::response::GetSpentInfoRequest, + ) -> BlockchainSourceResult { + unimplemented!() + } + + async fn get_network_sol_ps( + &self, + _blocks: Option, + _height: Option, + ) -> BlockchainSourceResult { + unimplemented!() + } + /// Returns the block commitment tree data by hash async fn get_commitment_tree_roots( &self, From 68cb4fb1fd54dde217348014bc7ef2cc20b84582 Mon Sep 17 00:00:00 2001 From: idky137 Date: Fri, 3 Jul 2026 09:35:53 +0100 Subject: [PATCH 08/58] move methods into chain_index / blockchainsource - validate address --- packages/zaino-state/src/backends/fetch.rs | 22 ++-- packages/zaino-state/src/backends/state.rs | 134 ++------------------ packages/zaino-state/src/indexer.rs | 136 ++++++++++++++++++++- 3 files changed, 153 insertions(+), 139 deletions(-) diff --git a/packages/zaino-state/src/backends/fetch.rs b/packages/zaino-state/src/backends/fetch.rs index 54e37a1ee..8e4fb5447 100644 --- a/packages/zaino-state/src/backends/fetch.rs +++ b/packages/zaino-state/src/backends/fetch.rs @@ -34,9 +34,7 @@ use zaino_fetch::{ chain_tips::GetChainTipsResponse, mining_info::GetMiningInfoWire, peer_info::GetPeerInfo, - z_validate_address::{ - ZValidateAddressResponse, DEPRECATION_NOTICE as Z_VALIDATE_DEPRECATION, - }, + z_validate_address::ZValidateAddressResponse, GetMempoolInfoResponse, GetNetworkSolPsResponse, GetSpentInfoRequest, GetSpentInfoResponse, GetTxOutResponse, GetTxOutSetInfoResponse, }, @@ -520,7 +518,9 @@ impl ZcashIndexer for FetchServiceSubscriber { &self, address: String, ) -> Result { - Ok(self.fetcher.validate_address(address).await?) + #[allow(deprecated)] + let network = self.config.common.network.to_zebra_network(); + Ok(crate::indexer::validate_address(address, &network)) } #[allow(deprecated)] @@ -528,8 +528,9 @@ impl ZcashIndexer for FetchServiceSubscriber { &self, address: String, ) -> Result { - tracing::warn!("{}", Z_VALIDATE_DEPRECATION); - Ok(self.fetcher.z_validate_address(address).await?) + #[allow(deprecated)] + let network = self.config.common.network.to_zebra_network(); + Ok(crate::indexer::z_validate_address(address, &network)) } /// Returns all transaction ids in the memory pool, as a JSON array. @@ -811,13 +812,8 @@ impl ZcashIndexer for FetchServiceSubscriber { } async fn chain_height(&self) -> Result { - Ok(Height( - self.indexer - .snapshot_nonfinalized_state() - .await? - .max_serviceable_height() - .0, - )) + let snapshot = self.indexer.snapshot_nonfinalized_state().await?; + Ok(self.indexer.best_chaintip(&snapshot).await?.height.into()) } /// Returns the transaction ids made by the provided transparent addresses. /// diff --git a/packages/zaino-state/src/backends/state.rs b/packages/zaino-state/src/backends/state.rs index 337a24d2e..2310e6d53 100644 --- a/packages/zaino-state/src/backends/state.rs +++ b/packages/zaino-state/src/backends/state.rs @@ -39,10 +39,7 @@ use zaino_fetch::{ chain_tips::GetChainTipsResponse, mining_info::GetMiningInfoWire, peer_info::GetPeerInfo, - z_validate_address::{ - InvalidZValidateAddress, KnownZValidateAddress, ZValidateAddressResponse, - DEPRECATION_NOTICE as Z_VALIDATE_DEPRECATION, - }, + z_validate_address::ZValidateAddressResponse, GetMempoolInfoResponse, GetNetworkSolPsResponse, GetSpentInfoRequest, GetSpentInfoResponse, GetSubtreesResponse, GetTxOutResponse, GetTxOutSetInfoResponse, }, @@ -60,14 +57,10 @@ use zaino_proto::proto::{ SendResponse, TransparentAddressBlockFilter, TreeState, TxFilter, }, }; -use zcash_keys::{address::Address, encoding::AddressCodec}; - -use zcash_protocol::consensus::NetworkType; -use zcash_transparent::address::TransparentAddress; use zebra_chain::{ block::Height, chain_tip::NetworkChainTipHeightEstimator, - parameters::{ConsensusBranchId, NetworkKind, NetworkUpgrade}, + parameters::{ConsensusBranchId, NetworkUpgrade}, serialization::ZcashDeserialize as _, subtree::NoteCommitmentSubtreeIndex, }; @@ -565,37 +558,6 @@ impl StateServiceSubscriber { } } -/// Extracts the diversifier and pk_d bytes from a validated Sapling -/// [`PaymentAddress`], returning pk_d in zcashd's big-endian byte order. -/// -/// # Deprecation -/// -/// See [`DEPRECATION_NOTICE`]. This function exists to support the -/// `z_validateaddress` RPC endpoint, which itself exists solely for zcashd -/// compatibility. The pk_d bytes are -/// reversed from `sapling-crypto`'s native little-endian representation to -/// match zcashd's big-endian hex output. -/// -/// # Precondition -/// -/// The caller must have obtained `s` through [`PaymentAddress::from_bytes`] or -/// equivalent (e.g. `ZcashAddress::convert_if_network`), which guarantees the -/// diversifier has a valid `g_d()` and pk_d is a non-identity Jubjub subgroup -/// point. No additional validation is performed here. -/// -/// # Layout -/// -/// `PaymentAddress::to_bytes()` returns 43 bytes: `diversifier (11) || pk_d (32)`. -/// `DiversifiedTransmissionKey::to_bytes()` is `pub(crate)` in `sapling-crypto`, -/// so we extract pk_d from the serialized form. -fn sapling_key_bytes(s: &sapling_crypto::PaymentAddress) -> ([u8; 11], [u8; 32]) { - let bytes = s.to_bytes(); - let diversifier: [u8; 11] = bytes[..11].try_into().unwrap(); - let mut pk_d: [u8; 32] = bytes[11..].try_into().unwrap(); - pk_d.reverse(); - (diversifier, pk_d) -} - // #[allow(deprecated)] impl ZcashIndexer for StateServiceSubscriber { type Error = StateServiceError; @@ -986,34 +948,8 @@ impl ZcashIndexer for StateServiceSubscriber { &self, raw_address: String, ) -> Result { - use zcash_transparent::address::TransparentAddress; - - let Ok(address) = raw_address.parse::() else { - return Ok(ValidateAddressResponse::invalid()); - }; - - let address = match address.convert_if_network::
( - match self.config.common.network.to_zebra_network().kind() { - NetworkKind::Mainnet => NetworkType::Main, - NetworkKind::Testnet => NetworkType::Test, - NetworkKind::Regtest => NetworkType::Regtest, - }, - ) { - Ok(address) => address, - Err(err) => { - tracing::debug!(?err, "conversion error"); - return Ok(ValidateAddressResponse::invalid()); - } - }; - - Ok(match address { - Address::Transparent(taddr) => ValidateAddressResponse::new( - true, - Some(raw_address), - Some(matches!(taddr, TransparentAddress::ScriptHash(_))), - ), - _ => ValidateAddressResponse::invalid(), - }) + let network = self.config.common.network.to_zebra_network(); + Ok(crate::indexer::validate_address(raw_address, &network)) } #[allow(deprecated)] @@ -1021,53 +957,8 @@ impl ZcashIndexer for StateServiceSubscriber { &self, address: String, ) -> Result { - tracing::warn!("{}", Z_VALIDATE_DEPRECATION); - - let Ok(parsed_address) = address.parse::() else { - return Ok(ZValidateAddressResponse::Known( - KnownZValidateAddress::Invalid(InvalidZValidateAddress::new()), - )); - }; - - let converted_address = match parsed_address.convert_if_network::
( - match self.config.common.network.to_zebra_network().kind() { - NetworkKind::Mainnet => NetworkType::Main, - NetworkKind::Testnet => NetworkType::Test, - NetworkKind::Regtest => NetworkType::Regtest, - }, - ) { - Ok(address) => address, - Err(err) => { - tracing::debug!(?err, "conversion error"); - return Ok(ZValidateAddressResponse::Known( - KnownZValidateAddress::Invalid(InvalidZValidateAddress::new()), - )); - } - }; - - // Note: It could be the case that Zaino needs to support Sprout. For now, it's been disabled. - match converted_address { - Address::Transparent(t) => match t { - TransparentAddress::PublicKeyHash(_) => { - Ok(ZValidateAddressResponse::p2pkh(address)) - } - TransparentAddress::ScriptHash(_) => Ok(ZValidateAddressResponse::p2sh(address)), - }, - Address::Unified(u) => Ok(ZValidateAddressResponse::unified( - u.encode(&self.network().to_zebra_network()), - )), - Address::Sapling(s) => { - let (diversifier, pk_d) = sapling_key_bytes(&s); - Ok(ZValidateAddressResponse::sapling( - s.encode(&self.network().to_zebra_network()), - Some(hex::encode(diversifier)), - Some(hex::encode(pk_d)), - )) - } - _ => Ok(ZValidateAddressResponse::Known( - KnownZValidateAddress::Invalid(InvalidZValidateAddress::new()), - )), - } + let network = self.config.common.network.to_zebra_network(); + Ok(crate::indexer::z_validate_address(address, &network)) } async fn z_get_subtrees_by_index( @@ -1279,15 +1170,8 @@ impl ZcashIndexer for StateServiceSubscriber { // Helper function, to get the chain height in rpc implementations async fn chain_height(&self) -> Result { - let mut state = self.read_state_service.clone(); - let response = state - .ready() - .and_then(|service| service.call(ReadRequest::Tip)) - .await?; - let (chain_height, _chain_hash) = expected_read_response!(response, Tip).ok_or( - RpcError::new_from_legacycode(LegacyCode::Misc, "no blocks in chain"), - )?; - Ok(chain_height) + let snapshot = self.indexer.snapshot_nonfinalized_state().await?; + Ok(self.indexer.best_chaintip(&snapshot).await?.height.into()) } } @@ -2225,7 +2109,7 @@ mod tests { /// addresses on other networks (mainnet, testnet). #[test] fn sapling_pk_d_byte_order_matches_test_vector() { - use super::sapling_key_bytes; + use crate::indexer::sapling_key_bytes; use zcash_keys::address::Address; use zcash_protocol::consensus::NetworkType; diff --git a/packages/zaino-state/src/indexer.rs b/packages/zaino-state/src/indexer.rs index acdf57a43..ed1d0ba18 100644 --- a/packages/zaino-state/src/indexer.rs +++ b/packages/zaino-state/src/indexer.rs @@ -12,7 +12,10 @@ use zaino_fetch::jsonrpsee::response::{ chain_tips::GetChainTipsResponse, mining_info::GetMiningInfoWire, peer_info::GetPeerInfo, - z_validate_address::ZValidateAddressResponse, + z_validate_address::{ + InvalidZValidateAddress, KnownZValidateAddress, ZValidateAddressResponse, + DEPRECATION_NOTICE as Z_VALIDATE_DEPRECATION, + }, GetMempoolInfoResponse, GetNetworkSolPsResponse, GetSpentInfoRequest, GetSpentInfoResponse, GetTxOutSetInfoResponse, }; @@ -983,3 +986,134 @@ pub(crate) async fn handle_raw_transaction( } } } + +/// Maps a Zebra network to the `zcash_protocol` network type used for address decoding. +fn address_network_type( + network: &zebra_chain::parameters::Network, +) -> zcash_protocol::consensus::NetworkType { + use zcash_protocol::consensus::NetworkType; + use zebra_chain::parameters::NetworkKind; + match network.kind() { + NetworkKind::Mainnet => NetworkType::Main, + NetworkKind::Testnet => NetworkType::Test, + NetworkKind::Regtest => NetworkType::Regtest, + } +} + +/// Validates a Zcash address for the `validateaddress` RPC. +/// +/// Pure address parsing over `network`; no chain data required, so both backends share +/// this implementation. +pub(crate) fn validate_address( + raw_address: String, + network: &zebra_chain::parameters::Network, +) -> ValidateAddressResponse { + use zcash_keys::address::Address; + use zcash_transparent::address::TransparentAddress; + + let Ok(address) = raw_address.parse::() else { + return ValidateAddressResponse::invalid(); + }; + + let address = match address.convert_if_network::
(address_network_type(network)) { + Ok(address) => address, + Err(err) => { + tracing::debug!(?err, "conversion error"); + return ValidateAddressResponse::invalid(); + } + }; + + match address { + Address::Transparent(taddr) => ValidateAddressResponse::new( + true, + Some(raw_address), + Some(matches!(taddr, TransparentAddress::ScriptHash(_))), + ), + _ => ValidateAddressResponse::invalid(), + } +} + +/// Validates a Zcash address for the deprecated `z_validateaddress` RPC. +/// +/// Pure address parsing over `network`; shared by both backends. +pub(crate) fn z_validate_address( + address: String, + network: &zebra_chain::parameters::Network, +) -> ZValidateAddressResponse { + use zcash_keys::address::Address; + use zcash_keys::encoding::AddressCodec as _; + use zcash_transparent::address::TransparentAddress; + + tracing::warn!("{}", Z_VALIDATE_DEPRECATION); + + let invalid = || { + ZValidateAddressResponse::Known(KnownZValidateAddress::Invalid( + InvalidZValidateAddress::new(), + )) + }; + + let Ok(parsed_address) = address.parse::() else { + return invalid(); + }; + + let converted_address = + match parsed_address.convert_if_network::
(address_network_type(network)) { + Ok(address) => address, + Err(err) => { + tracing::debug!(?err, "conversion error"); + return invalid(); + } + }; + + // Note: It could be the case that Zaino needs to support Sprout. For now, it's been disabled. + match converted_address { + Address::Transparent(TransparentAddress::PublicKeyHash(_)) => { + ZValidateAddressResponse::p2pkh(address) + } + Address::Transparent(TransparentAddress::ScriptHash(_)) => { + ZValidateAddressResponse::p2sh(address) + } + Address::Unified(u) => ZValidateAddressResponse::unified(u.encode(network)), + Address::Sapling(s) => { + let (diversifier, pk_d) = sapling_key_bytes(&s); + ZValidateAddressResponse::sapling( + s.encode(network), + Some(hex::encode(diversifier)), + Some(hex::encode(pk_d)), + ) + } + _ => invalid(), + } +} + +/// Extracts the diversifier and pk_d bytes from a validated Sapling +/// [`sapling_crypto::PaymentAddress`], returning pk_d in zcashd's big-endian byte order. +/// +/// # Deprecation +/// +/// See [`Z_VALIDATE_DEPRECATION`]. This function exists to support the `z_validateaddress` +/// RPC endpoint, which itself exists solely for zcashd compatibility. The pk_d bytes are +/// reversed from `sapling-crypto`'s native little-endian representation to match zcashd's +/// big-endian hex output. +/// +/// # Precondition +/// +/// The caller must have obtained `s` through `PaymentAddress::from_bytes` or equivalent +/// (e.g. `ZcashAddress::convert_if_network`), which guarantees the diversifier has a valid +/// `g_d()` and pk_d is a non-identity Jubjub subgroup point. No additional validation is +/// performed here. +/// +/// # Layout +/// +/// `PaymentAddress::to_bytes()` returns 43 bytes: `diversifier (11) || pk_d (32)`. +pub(crate) fn sapling_key_bytes(s: &sapling_crypto::PaymentAddress) -> ([u8; 11], [u8; 32]) { + let bytes = s.to_bytes(); + let diversifier: [u8; 11] = bytes[..11] + .try_into() + .expect("PaymentAddress::to_bytes always returns 43 bytes: diversifier is the first 11"); + let mut pk_d: [u8; 32] = bytes[11..] + .try_into() + .expect("PaymentAddress::to_bytes always returns 43 bytes: pk_d is the last 32"); + pk_d.reverse(); + (diversifier, pk_d) +} From 179feabcf15ad9a5715df91a342b95315c3bf316 Mon Sep 17 00:00:00 2001 From: idky137 Date: Fri, 3 Jul 2026 10:03:13 +0100 Subject: [PATCH 09/58] move methods into chain_index / blockchainsource - blockchain info --- packages/zaino-state/src/backends/fetch.rs | 14 +- packages/zaino-state/src/backends/state.rs | 153 +----------------- packages/zaino-state/src/chain_index.rs | 17 +- .../zaino-state/src/chain_index/source.rs | 7 +- .../chain_index/source/mockchain_source.rs | 11 ++ .../chain_index/source/validator_connector.rs | 149 ++++++++++++++++- .../chain_index/tests/proptest_blockgen.rs | 7 + 7 files changed, 191 insertions(+), 167 deletions(-) diff --git a/packages/zaino-state/src/backends/fetch.rs b/packages/zaino-state/src/backends/fetch.rs index 8e4fb5447..a26c7f6a8 100644 --- a/packages/zaino-state/src/backends/fetch.rs +++ b/packages/zaino-state/src/backends/fetch.rs @@ -294,19 +294,7 @@ impl ZcashIndexer for FetchServiceSubscriber { /// Some fields from the zcashd reference are missing from Zebra's [`GetBlockchainInfoResponse`]. It only contains the fields /// [required for lightwalletd support.](https://github.com/zcash/lightwalletd/blob/v0.4.9/common/common.go#L72-L89) async fn get_blockchain_info(&self) -> Result { - Ok(self - .fetcher - .get_blockchain_info() - .await? - .try_into() - .map_err(|_e| { - #[allow(deprecated)] - FetchServiceError::SerializationError( - zebra_chain::serialization::SerializationError::Parse( - "chainwork not hex-encoded integer", - ), - ) - })?) + Ok(self.indexer.get_blockchain_info().await?) } /// Returns details on the active state of the TX memory pool. diff --git a/packages/zaino-state/src/backends/state.rs b/packages/zaino-state/src/backends/state.rs index 2310e6d53..67e6efe55 100644 --- a/packages/zaino-state/src/backends/state.rs +++ b/packages/zaino-state/src/backends/state.rs @@ -58,11 +58,7 @@ use zaino_proto::proto::{ }, }; use zebra_chain::{ - block::Height, - chain_tip::NetworkChainTipHeightEstimator, - parameters::{ConsensusBranchId, NetworkUpgrade}, - serialization::ZcashDeserialize as _, - subtree::NoteCommitmentSubtreeIndex, + block::Height, serialization::ZcashDeserialize as _, subtree::NoteCommitmentSubtreeIndex, }; use zebra_rpc::{ client::{ @@ -70,20 +66,16 @@ use zebra_rpc::{ TransactionObject, ValidateAddressResponse, }, methods::{ - chain_tip_difficulty, AddressBalance, ConsensusBranchIdHex, GetAddressTxIdsRequest, - GetAddressUtxos, GetBlock, GetBlockHash, GetBlockchainInfoResponse, GetInfo, - GetRawTransaction, NetworkUpgradeInfo, NetworkUpgradeStatus, SentTransactionHash, - TipConsensusBranch, + AddressBalance, GetAddressTxIdsRequest, GetAddressUtxos, GetBlock, GetBlockHash, + GetBlockchainInfoResponse, GetInfo, GetRawTransaction, SentTransactionHash, }, server::error::LegacyCode, sync::init_read_state_with_syncer, }; use zebra_state::{HashOrHeight, ReadRequest, ReadResponse, ReadStateService}; -use chrono::Utc; use futures::TryFutureExt as _; use hex::{FromHex as _, ToHex}; -use indexmap::IndexMap; use std::{str::FromStr, sync::Arc}; use tokio::{ sync::mpsc, @@ -597,144 +589,7 @@ impl ZcashIndexer for StateServiceSubscriber { } async fn get_blockchain_info(&self) -> Result { - let mut state = self.read_state_service.clone(); - - let response = state - .ready() - .and_then(|service| service.call(ReadRequest::TipPoolValues)) - .await?; - let (height, hash, balance) = match response { - ReadResponse::TipPoolValues { - tip_height, - tip_hash, - value_balance, - } => (tip_height, tip_hash, value_balance), - unexpected => { - unreachable!("Unexpected response from state service: {unexpected:?}") - } - }; - - let usage_response = state - .ready() - .and_then(|service| service.call(ReadRequest::UsageInfo)) - .await?; - let size_on_disk = expected_read_response!(usage_response, UsageInfo); - - let request = zebra_state::ReadRequest::BlockHeader(hash.into()); - let response = state - .ready() - .and_then(|service| service.call(request)) - .await?; - let header = match response { - ReadResponse::BlockHeader { header, .. } => header, - unexpected => { - unreachable!("Unexpected response from state service: {unexpected:?}") - } - }; - - let now = Utc::now(); - let zebra_estimated_height = NetworkChainTipHeightEstimator::new( - header.time, - height, - &self.config.common.network.into(), - ) - .estimate_height_at(now); - let estimated_height = if header.time > now || zebra_estimated_height < height { - height - } else { - zebra_estimated_height - }; - - let upgrades = IndexMap::from_iter( - self.config - .common - .network - .to_zebra_network() - .full_activation_list() - .into_iter() - .filter_map(|(activation_height, network_upgrade)| { - // Zebra defines network upgrades based on incompatible consensus rule changes, - // but zcashd defines them based on ZIPs. - // - // All the network upgrades with a consensus branch ID - // are the same in Zebra and zcashd. - network_upgrade.branch_id().map(|branch_id| { - // zcashd's RPC seems to ignore Disabled network upgrades, - // so Zebra does too. - let status = if height >= activation_height { - NetworkUpgradeStatus::Active - } else { - NetworkUpgradeStatus::Pending - }; - - ( - ConsensusBranchIdHex::new(branch_id.into()), - NetworkUpgradeInfo::from_parts( - network_upgrade, - activation_height, - status, - ), - ) - }) - }), - ); - - let next_block_height = - (height + 1).expect("valid chain tips are a lot less than Height::MAX"); - let consensus = TipConsensusBranch::from_parts( - ConsensusBranchIdHex::new( - NetworkUpgrade::current(&self.config.common.network.into(), height) - .branch_id() - .unwrap_or(ConsensusBranchId::RPC_MISSING_ID) - .into(), - ) - .inner(), - ConsensusBranchIdHex::new( - NetworkUpgrade::current(&self.config.common.network.into(), next_block_height) - .branch_id() - .unwrap_or(ConsensusBranchId::RPC_MISSING_ID) - .into(), - ) - .inner(), - ); - - // TODO: Remove unwrap() - let difficulty = chain_tip_difficulty( - self.config.common.network.to_zebra_network(), - self.read_state_service.clone(), - false, - ) - .await - .unwrap(); - - let verification_progress = f64::from(height.0) / f64::from(zebra_estimated_height.0); - - Ok(GetBlockchainInfoResponse::new( - self.config - .common - .network - .to_zebra_network() - .bip70_network_name(), - height, - hash, - estimated_height, - zebra_rpc::client::GetBlockchainInfoBalance::chain_supply(balance), - // TODO: account for new delta_pools arg? - zebra_rpc::client::GetBlockchainInfoBalance::value_pools(balance, None), - upgrades, - consensus, - height, - difficulty, - verification_progress, - // TODO: store work in the finalized state for each height - // see https://github.com/ZcashFoundation/zebra/issues/7109 - 0, - false, - size_on_disk, - // TODO (copied from zebra): Investigate whether this needs to - // be implemented (it's sprout-only in zcashd) - 0, - )) + Ok(self.indexer.get_blockchain_info().await?) } /// Returns details on the active state of the TX memory pool. diff --git a/packages/zaino-state/src/chain_index.rs b/packages/zaino-state/src/chain_index.rs index 0fcf90ffc..cd1e8f028 100644 --- a/packages/zaino-state/src/chain_index.rs +++ b/packages/zaino-state/src/chain_index.rs @@ -54,7 +54,7 @@ pub use zebra_chain::parameters::Network as ZebraNetwork; use zebra_chain::serialization::ZcashSerialize; use zebra_rpc::{ client::{GetAddressBalanceRequest, GetAddressTxIdsRequest}, - methods::{AddressBalance, GetAddressUtxos, GetBlock, GetInfo}, + methods::{AddressBalance, GetAddressUtxos, GetBlock, GetBlockchainInfoResponse, GetInfo}, }; use zebra_state::HashOrHeight; @@ -623,6 +623,11 @@ pub trait ChainIndexRpcExt: ChainIndex { /// Returns the `getinfo` response. fn get_info(&self) -> impl std::future::Future>; + /// Returns the `getblockchaininfo` response. + fn get_blockchain_info( + &self, + ) -> impl std::future::Future>; + /// Returns the `getpeerinfo` response. fn get_peer_info(&self) -> impl std::future::Future>; @@ -2732,6 +2737,16 @@ impl ChainIndexRpcExt for NodeBackedChainIndexSubscrib .map_err(ChainIndexError::backing_validator) } + // `getblockchaininfo` needs cumulative pool value balances (TipPoolValues) and on-disk + // size, which are not in the ChainIndex's indexed data, so it cannot be built + // internally: always delegate to the backing validator. + async fn get_blockchain_info(&self) -> Result { + self.source() + .get_blockchain_info() + .await + .map_err(ChainIndexError::backing_validator) + } + async fn get_peer_info(&self) -> Result { self.source() .get_peer_info() diff --git a/packages/zaino-state/src/chain_index/source.rs b/packages/zaino-state/src/chain_index/source.rs index 8d50b06ec..4515e9b77 100644 --- a/packages/zaino-state/src/chain_index/source.rs +++ b/packages/zaino-state/src/chain_index/source.rs @@ -29,7 +29,7 @@ use zebra_chain::{ }; use zebra_rpc::{ client::{GetAddressBalanceRequest, GetAddressTxIdsRequest}, - methods::{AddressBalance, GetAddressUtxos, GetInfo}, + methods::{AddressBalance, GetAddressUtxos, GetBlockchainInfoResponse, GetInfo}, }; use zebra_state::{HashOrHeight, ReadRequest, ReadResponse, ReadStateService}; @@ -129,6 +129,11 @@ pub trait BlockchainSource: Clone + Send + Sync + 'static { /// minimum difficulty (the `getdifficulty` RPC value). fn get_difficulty(&self) -> impl SendFut>; + /// Returns the `getblockchaininfo` response. + fn get_blockchain_info( + &self, + ) -> impl SendFut>; + // ********** Node-passthrough methods ********** // // These have no local-index equivalent and always proxy to the backing validator's diff --git a/packages/zaino-state/src/chain_index/source/mockchain_source.rs b/packages/zaino-state/src/chain_index/source/mockchain_source.rs index 83e066dc4..420ca24a8 100644 --- a/packages/zaino-state/src/chain_index/source/mockchain_source.rs +++ b/packages/zaino-state/src/chain_index/source/mockchain_source.rs @@ -683,6 +683,17 @@ impl BlockchainSource for MockchainSource { .relative_to_network(&mockchain_network())) } + async fn get_blockchain_info( + &self, + ) -> BlockchainSourceResult { + // Needs cumulative pool value balances (TipPoolValues) and on-disk size, which the + // vectors don't carry. Test vectors must be extended to serve this method; tracked + // by the update-test-vectors follow-up (see "Future work"). + unimplemented!( + "MockchainSource cannot serve get_blockchain_info until test vectors are extended" + ) + } + // ********** Node-passthrough methods ********** // // These are node-only RPCs with no chain data in the vectors. Test vectors must be diff --git a/packages/zaino-state/src/chain_index/source/validator_connector.rs b/packages/zaino-state/src/chain_index/source/validator_connector.rs index 450a72744..1bd6ac48a 100644 --- a/packages/zaino-state/src/chain_index/source/validator_connector.rs +++ b/packages/zaino-state/src/chain_index/source/validator_connector.rs @@ -5,6 +5,7 @@ use std::str::FromStr as _; use chrono::{DateTime, Utc}; use hex::{FromHex as _, ToHex as _}; +use indexmap::IndexMap; use zaino_fetch::jsonrpsee::response::{ address_deltas::BlockInfo, block_deltas::{BlockDelta, BlockDeltas, InputDelta, OutputDelta}, @@ -17,14 +18,17 @@ use zaino_fetch::jsonrpsee::response::{ use zebra_chain::{ amount::{Amount, NonNegative}, block::{Header, SerializedBlock}, - parameters::NetworkUpgrade, + chain_tip::NetworkChainTipHeightEstimator, + parameters::{ConsensusBranchId, NetworkUpgrade}, serialization::{BytesInDisplayOrder as _, ZcashSerialize as _}, }; use zebra_rpc::{ client::{BlockObject, GetBlockchainInfoBalance, HexData, Input, TransactionObject}, methods::{ - chain_tip_difficulty, GetBlock, GetBlockHeaderObject, GetBlockHeaderResponse, - GetBlockTransaction, GetBlockTrees, GetInfo, ValidateAddresses as _, + chain_tip_difficulty, ConsensusBranchIdHex, GetBlock, GetBlockHeaderObject, + GetBlockHeaderResponse, GetBlockTransaction, GetBlockTrees, GetBlockchainInfoResponse, + GetInfo, NetworkUpgradeInfo, NetworkUpgradeStatus, TipConsensusBranch, + ValidateAddresses as _, }, }; @@ -217,6 +221,22 @@ impl BlockchainSource for ValidatorConnector { } } + async fn get_blockchain_info(&self) -> BlockchainSourceResult { + match self { + ValidatorConnector::State(state) => state.get_blockchain_info().await, + ValidatorConnector::Fetch(fetch) => fetch + .get_blockchain_info() + .await + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))? + .try_into() + .map_err(|_error| { + BlockchainSourceError::Unrecoverable( + "getblockchaininfo: chainwork not hex-encoded integer".to_string(), + ) + }), + } + } + // ********** Node-passthrough methods ********** async fn get_info(&self) -> BlockchainSourceResult { @@ -1560,6 +1580,129 @@ impl State { median_of_block_times(times) } + + /// Builds the `getblockchaininfo` response from the `ReadStateService`. + /// + /// Moved from the former `StateServiceSubscriber::get_blockchain_info`; error type is + /// now [`BlockchainSourceError`], and the difficulty is propagated rather than + /// unwrapped. + async fn get_blockchain_info(&self) -> BlockchainSourceResult { + let mut state = self.read_state_service.clone(); + let network = self.network.to_zebra_network(); + + let response = state + .ready() + .and_then(|service| service.call(ReadRequest::TipPoolValues)) + .await + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + let (height, hash, balance) = match response { + ReadResponse::TipPoolValues { + tip_height, + tip_hash, + value_balance, + } => (tip_height, tip_hash, value_balance), + unexpected => { + unreachable!("Unexpected response from state service: {unexpected:?}") + } + }; + + let usage_response = state + .ready() + .and_then(|service| service.call(ReadRequest::UsageInfo)) + .await + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + let size_on_disk = expected_read_response!(usage_response, UsageInfo); + + let response = state + .ready() + .and_then(|service| service.call(ReadRequest::BlockHeader(hash.into()))) + .await + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + let header = match response { + ReadResponse::BlockHeader { header, .. } => header, + unexpected => { + unreachable!("Unexpected response from state service: {unexpected:?}") + } + }; + + let now = Utc::now(); + let zebra_estimated_height = + NetworkChainTipHeightEstimator::new(header.time, height, &network) + .estimate_height_at(now); + let estimated_height = if header.time > now || zebra_estimated_height < height { + height + } else { + zebra_estimated_height + }; + + let upgrades = IndexMap::from_iter(network.full_activation_list().into_iter().filter_map( + |(activation_height, network_upgrade)| { + // Zebra defines network upgrades by consensus rule changes, zcashd by ZIPs; + // upgrades with a consensus branch ID are the same in both. + network_upgrade.branch_id().map(|branch_id| { + // zcashd's RPC ignores Disabled network upgrades, so Zebra does too. + let status = if height >= activation_height { + NetworkUpgradeStatus::Active + } else { + NetworkUpgradeStatus::Pending + }; + ( + ConsensusBranchIdHex::new(branch_id.into()), + NetworkUpgradeInfo::from_parts(network_upgrade, activation_height, status), + ) + }) + }, + )); + + let next_block_height = + (height + 1).expect("valid chain tips are a lot less than Height::MAX"); + let consensus = TipConsensusBranch::from_parts( + ConsensusBranchIdHex::new( + NetworkUpgrade::current(&network, height) + .branch_id() + .unwrap_or(ConsensusBranchId::RPC_MISSING_ID) + .into(), + ) + .inner(), + ConsensusBranchIdHex::new( + NetworkUpgrade::current(&network, next_block_height) + .branch_id() + .unwrap_or(ConsensusBranchId::RPC_MISSING_ID) + .into(), + ) + .inner(), + ); + + let difficulty = + chain_tip_difficulty(network.clone(), self.read_state_service.clone(), false) + .await + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + + let verification_progress = f64::from(height.0) / f64::from(zebra_estimated_height.0); + + Ok(GetBlockchainInfoResponse::new( + network.bip70_network_name(), + height, + hash, + estimated_height, + GetBlockchainInfoBalance::chain_supply(balance), + // TODO: account for new delta_pools arg? + GetBlockchainInfoBalance::value_pools(balance, None), + upgrades, + consensus, + height, + difficulty, + verification_progress, + // TODO: store work in the finalized state for each height + // (see https://github.com/ZcashFoundation/zebra/issues/7109) + 0, + false, + size_on_disk, + // TODO (copied from zebra): investigate whether this needs implementing + // (it's sprout-only in zcashd) + 0, + )) + } } /// Confirmations are one more than the depth, or -1 when the block is not on the best diff --git a/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs b/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs index c24b6e499..a5f694c4e 100644 --- a/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs +++ b/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs @@ -648,6 +648,13 @@ impl BlockchainSource for ProptestMockchain { unimplemented!() } + async fn get_blockchain_info( + &self, + ) -> BlockchainSourceResult { + // ProptestMockchain exercises sync/reorg, not the getblockchaininfo RPC. + unimplemented!() + } + async fn get_info(&self) -> BlockchainSourceResult { unimplemented!() } From 60a8dc2676a92a23e196aebbba6c39d19f5b8e2e Mon Sep 17 00:00:00 2001 From: idky137 Date: Fri, 3 Jul 2026 10:30:57 +0100 Subject: [PATCH 10/58] move methods into chain_index / blockchainsource - subtree roots --- packages/zaino-state/src/backends/fetch.rs | 27 ++++++-- packages/zaino-state/src/backends/state.rs | 81 ++++++---------------- packages/zaino-state/src/indexer.rs | 65 ++++++++++++++++- 3 files changed, 107 insertions(+), 66 deletions(-) diff --git a/packages/zaino-state/src/backends/fetch.rs b/packages/zaino-state/src/backends/fetch.rs index a26c7f6a8..6aa8a19fe 100644 --- a/packages/zaino-state/src/backends/fetch.rs +++ b/packages/zaino-state/src/backends/fetch.rs @@ -667,17 +667,34 @@ impl ZcashIndexer for FetchServiceSubscriber { /// starting at the chain tip. This RPC will return an empty list if the `start_index` subtree /// exists, but has not been rebuilt yet. This matches `zcashd`'s behaviour when subtrees aren't /// available yet. (But `zcashd` does its rebuild before syncing any blocks.) + #[allow(deprecated)] async fn z_get_subtrees_by_index( &self, pool: String, start_index: NoteCommitmentSubtreeIndex, limit: Option, ) -> Result { - Ok(self - .fetcher - .get_subtrees_by_index(pool, start_index.0, limit.map(|limit_index| limit_index.0)) - .await? - .into()) + let shielded_pool = match pool.as_str() { + "sapling" => crate::chain_index::ShieldedPool::Sapling, + "orchard" => crate::chain_index::ShieldedPool::Orchard, + otherwise => { + return Err(FetchServiceError::RpcError(RpcError::new_from_legacycode( + zebra_rpc::server::error::LegacyCode::Misc, + format!( + "invalid pool name \"{otherwise}\", must be \"sapling\" or \"orchard\"" + ), + ))) + } + }; + let roots = self + .indexer + .get_subtree_roots(shielded_pool, start_index.0, limit.map(|index| index.0)) + .await?; + Ok(crate::indexer::build_subtrees_by_index_response( + pool, + start_index, + roots, + )) } /// Returns the raw transaction data, as a [`GetRawTransaction`] JSON string or structure. diff --git a/packages/zaino-state/src/backends/state.rs b/packages/zaino-state/src/backends/state.rs index 67e6efe55..b17ce6906 100644 --- a/packages/zaino-state/src/backends/state.rs +++ b/packages/zaino-state/src/backends/state.rs @@ -41,7 +41,7 @@ use zaino_fetch::{ peer_info::GetPeerInfo, z_validate_address::ZValidateAddressResponse, GetMempoolInfoResponse, GetNetworkSolPsResponse, GetSpentInfoRequest, - GetSpentInfoResponse, GetSubtreesResponse, GetTxOutResponse, GetTxOutSetInfoResponse, + GetSpentInfoResponse, GetTxOutResponse, GetTxOutSetInfoResponse, }, }, }; @@ -62,7 +62,7 @@ use zebra_chain::{ }; use zebra_rpc::{ client::{ - GetAddressBalanceRequest, GetSubtreesByIndexResponse, GetTreestateResponse, SubtreeRpcData, + GetAddressBalanceRequest, GetSubtreesByIndexResponse, GetTreestateResponse, TransactionObject, ValidateAddressResponse, }, methods::{ @@ -822,64 +822,27 @@ impl ZcashIndexer for StateServiceSubscriber { start_index: NoteCommitmentSubtreeIndex, limit: Option, ) -> Result { - let mut state = self.read_state_service.clone(); - - match pool.as_str() { - "sapling" => { - let request = zebra_state::ReadRequest::SaplingSubtrees { start_index, limit }; - let response = state - .ready() - .and_then(|service| service.call(request)) - .await?; - let sapling_subtrees = expected_read_response!(response, SaplingSubtrees); - let subtrees = sapling_subtrees - .values() - .map(|subtree| { - SubtreeRpcData { - root: subtree.root.to_bytes().encode_hex(), - end_height: subtree.end_height, - } - .into() - }) - .collect(); - - Ok(GetSubtreesResponse { - pool, - start_index, - subtrees, - } - .into()) - } - "orchard" => { - let request = zebra_state::ReadRequest::OrchardSubtrees { start_index, limit }; - let response = state - .ready() - .and_then(|service| service.call(request)) - .await?; - let orchard_subtrees = expected_read_response!(response, OrchardSubtrees); - let subtrees = orchard_subtrees - .values() - .map(|subtree| { - SubtreeRpcData { - root: subtree.root.encode_hex(), - end_height: subtree.end_height, - } - .into() - }) - .collect(); - - Ok(GetSubtreesResponse { - pool, - start_index, - subtrees, - } - .into()) + let shielded_pool = match pool.as_str() { + "sapling" => crate::chain_index::ShieldedPool::Sapling, + "orchard" => crate::chain_index::ShieldedPool::Orchard, + otherwise => { + return Err(StateServiceError::RpcError(RpcError::new_from_legacycode( + LegacyCode::Misc, + format!( + "invalid pool name \"{otherwise}\", must be \"sapling\" or \"orchard\"" + ), + ))) } - otherwise => Err(StateServiceError::RpcError(RpcError::new_from_legacycode( - LegacyCode::Misc, - format!("invalid pool name \"{otherwise}\", must be \"sapling\" or \"orchard\""), - ))), - } + }; + let roots = self + .indexer + .get_subtree_roots(shielded_pool, start_index.0, limit.map(|index| index.0)) + .await?; + Ok(crate::indexer::build_subtrees_by_index_response( + pool, + start_index, + roots, + )) } async fn get_raw_transaction( diff --git a/packages/zaino-state/src/indexer.rs b/packages/zaino-state/src/indexer.rs index ed1d0ba18..d2ffca07b 100644 --- a/packages/zaino-state/src/indexer.rs +++ b/packages/zaino-state/src/indexer.rs @@ -17,7 +17,7 @@ use zaino_fetch::jsonrpsee::response::{ DEPRECATION_NOTICE as Z_VALIDATE_DEPRECATION, }, GetMempoolInfoResponse, GetNetworkSolPsResponse, GetSpentInfoRequest, GetSpentInfoResponse, - GetTxOutSetInfoResponse, + GetSubtreesResponse, GetTxOutSetInfoResponse, }; use zaino_proto::proto::{ compact_formats::CompactBlock, @@ -32,7 +32,9 @@ use zebra_chain::{ block::Height, serialization::BytesInDisplayOrder as _, subtree::NoteCommitmentSubtreeIndex, }; use zebra_rpc::{ - client::{GetSubtreesByIndexResponse, GetTreestateResponse, ValidateAddressResponse}, + client::{ + GetSubtreesByIndexResponse, GetTreestateResponse, SubtreeRpcData, ValidateAddressResponse, + }, methods::{ AddressBalance, GetAddressBalanceRequest, GetAddressTxIdsRequest, GetAddressUtxos, GetBlock, GetBlockHash, GetBlockchainInfoResponse, GetInfo, GetRawTransaction, @@ -1117,3 +1119,62 @@ pub(crate) fn sapling_key_bytes(s: &sapling_crypto::PaymentAddress) -> ([u8; 11] pk_d.reverse(); (diversifier, pk_d) } + +/// Shapes a `z_getsubtreesbyindex` JSON-RPC response from raw subtree roots. +/// +/// The `(root, end_height)` pairs from [`ChainIndex::get_subtree_roots`] are already in +/// the byte order the JSON-RPC uses (sapling `to_bytes`, orchard `to_repr` — zcashd's +/// `z_getsubtreesbyindex` does not reverse orchard subtree roots), so they are hex-encoded +/// as-is. Shared by both backends so the shaping lives in one place. +/// +/// [`ChainIndex::get_subtree_roots`]: crate::ChainIndex::get_subtree_roots +pub(crate) fn build_subtrees_by_index_response( + pool: String, + start_index: NoteCommitmentSubtreeIndex, + roots: Vec<([u8; 32], u32)>, +) -> GetSubtreesByIndexResponse { + use hex::ToHex as _; + + let subtrees = roots + .into_iter() + .map(|(root, end_height)| { + SubtreeRpcData { + root: root.encode_hex(), + end_height: Height(end_height), + } + .into() + }) + .collect(); + + GetSubtreesResponse { + pool, + start_index, + subtrees, + } + .into() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn build_subtrees_by_index_response_hex_encodes_roots() { + let roots = vec![([0xabu8; 32], 100u32), ([0xcdu8; 32], 200u32)]; + let response = build_subtrees_by_index_response( + "orchard".to_string(), + NoteCommitmentSubtreeIndex(5), + roots, + ); + + assert_eq!(response.pool().as_str(), "orchard"); + assert_eq!(response.start_index(), NoteCommitmentSubtreeIndex(5)); + + let subtrees = response.subtrees(); + assert_eq!(subtrees.len(), 2); + assert_eq!(subtrees[0].root, hex::encode([0xabu8; 32])); + assert_eq!(subtrees[0].end_height, Height(100)); + assert_eq!(subtrees[1].root, hex::encode([0xcdu8; 32])); + assert_eq!(subtrees[1].end_height, Height(200)); + } +} From 511fe58d26f7e7b23a154312d9c38501730f1a77 Mon Sep 17 00:00:00 2001 From: idky137 Date: Fri, 3 Jul 2026 11:21:50 +0100 Subject: [PATCH 11/58] move methods into chain_index / blockchainsource - mempool --- packages/zaino-state/src/backends/fetch.rs | 38 ++--- packages/zaino-state/src/backends/state.rs | 152 +++++++++--------- packages/zaino-state/src/chain_index.rs | 41 ++++- .../zaino-state/src/chain_index/source.rs | 19 ++- .../chain_index/source/mockchain_source.rs | 17 ++ .../chain_index/source/validator_connector.rs | 29 +++- .../chain_index/tests/proptest_blockgen.rs | 14 ++ 7 files changed, 201 insertions(+), 109 deletions(-) diff --git a/packages/zaino-state/src/backends/fetch.rs b/packages/zaino-state/src/backends/fetch.rs index 6aa8a19fe..2c9a49d7a 100644 --- a/packages/zaino-state/src/backends/fetch.rs +++ b/packages/zaino-state/src/backends/fetch.rs @@ -25,7 +25,6 @@ use zaino_fetch::{ chain::{transaction::FullTransaction, utils::ParseFromSlice}, jsonrpsee::{ connector::{JsonRpSeeConnector, RpcError}, - raw_transaction::validate_raw_transaction_hex, response::{ address_deltas::{GetAddressDeltasParams, GetAddressDeltasResponse}, block_deltas::BlockDeltas, @@ -90,7 +89,9 @@ use crate::{ pub struct FetchService { /// JsonRPC Client. /// - /// NOTE: DEPRCATED, USE INDEXER OR VALIDATOR_CONNECTOR. + /// NOTE: DEPRECATED — no longer read now that every fetch goes through the indexer. + /// TODO(task 3): remove this field (services should hold only `indexer`, `data`, `config`). + #[allow(dead_code)] fetcher: JsonRpSeeConnector, /// Core indexer. indexer: NodeBackedChainIndex, @@ -207,7 +208,9 @@ impl Drop for FetchService { pub struct FetchServiceSubscriber { /// JsonRPC Client. /// - /// NOTE: DEPRCATED, USE INDEXER OR VALIDATOR_CONNECTOR. + /// NOTE: DEPRECATED — no longer read now that every fetch goes through the indexer. + /// TODO(task 3): remove this field (services should hold only `indexer`, `data`, `config`). + #[allow(dead_code)] fetcher: JsonRpSeeConnector, /// Core indexer. pub indexer: NodeBackedChainIndexSubscriber, @@ -375,12 +378,10 @@ impl ZcashIndexer for FetchServiceSubscriber { &self, raw_transaction_hex: String, ) -> Result { - validate_raw_transaction_hex(&raw_transaction_hex)?; Ok(self - .fetcher + .indexer .send_raw_transaction(raw_transaction_hex) - .await? - .into()) + .await?) } /// Returns the requested block by hash or height, as a [`GetBlock`] JSON string. @@ -628,25 +629,10 @@ impl ZcashIndexer for FetchServiceSubscriber { return local_result; } - self.fetcher - .get_treestate(fallback_hash_or_height) - .await - .map_err(|_error| { - #[allow(deprecated)] - FetchServiceError::RpcError(RpcError::new_from_legacycode( - zebra_rpc::server::error::LegacyCode::InvalidParameter, - "Failed to fetch treestate.", - )) - }) - .and_then(|treestate| { - treestate.try_into().map_err(|_error| { - #[allow(deprecated)] - FetchServiceError::RpcError(RpcError::new_from_legacycode( - zebra_rpc::server::error::LegacyCode::InvalidParameter, - "Failed to parse treestate.", - )) - }) - }) + Ok(self + .indexer + .get_treestate_by_id(fallback_hash_or_height) + .await?) } /// Returns information about a range of Sapling or Orchard subtrees. diff --git a/packages/zaino-state/src/backends/state.rs b/packages/zaino-state/src/backends/state.rs index b17ce6906..af317cac1 100644 --- a/packages/zaino-state/src/backends/state.rs +++ b/packages/zaino-state/src/backends/state.rs @@ -30,7 +30,6 @@ use zaino_fetch::{ chain::{transaction::FullTransaction, utils::ParseFromSlice}, jsonrpsee::{ connector::{JsonRpSeeConnector, RpcError}, - raw_transaction::validate_raw_transaction_hex, response::{ address_deltas::{GetAddressDeltasParams, GetAddressDeltasResponse}, block_deltas::BlockDeltas, @@ -624,14 +623,10 @@ impl ZcashIndexer for StateServiceSubscriber { &self, raw_transaction_hex: String, ) -> Result { - validate_raw_transaction_hex(&raw_transaction_hex).map_err(StateServiceError::RpcError)?; - // Offload to the json rpc connector, as ReadStateService - // doesn't yet interface with the mempool - self.rpc_client + Ok(self + .indexer .send_raw_transaction(raw_transaction_hex) - .await - .map(SentTransactionHash::from) - .map_err(Into::into) + .await?) } async fn get_block_header( @@ -734,23 +729,10 @@ impl ZcashIndexer for StateServiceSubscriber { return local_result; } - self.rpc_client - .get_treestate(fallback_hash_or_height) - .await - .map_err(|_error| { - StateServiceError::RpcError(RpcError::new_from_legacycode( - zebra_rpc::server::error::LegacyCode::InvalidParameter, - "Failed to fetch treestate.", - )) - }) - .and_then(|treestate| { - treestate.try_into().map_err(|_error| { - StateServiceError::RpcError(RpcError::new_from_legacycode( - zebra_rpc::server::error::LegacyCode::InvalidParameter, - "Failed to parse treestate.", - )) - }) - }) + Ok(self + .indexer + .get_treestate_by_id(fallback_hash_or_height) + .await?) } async fn get_mining_info(&self) -> Result { @@ -1397,7 +1379,7 @@ impl LightWalletIndexer for StateServiceSubscriber { } }; - let mempool = self.mempool.clone(); + let indexer = self.indexer.clone(); let service_timeout = self.config.common.service.timeout; let (channel_tx, channel_rx) = mpsc::channel(self.config.common.service.channel_size as usize); @@ -1405,11 +1387,21 @@ impl LightWalletIndexer for StateServiceSubscriber { let timeout = timeout( time::Duration::from_secs((service_timeout * 4) as u64), async { - for (mempool_key, mempool_value) in - mempool.get_filtered_mempool(exclude_txids).await - { - let txid_bytes = match hex::decode(mempool_key.txid) { - Ok(bytes) => bytes, + let transactions = match indexer.get_mempool_transactions(exclude_txids).await { + Ok(transactions) => transactions, + Err(e) => { + channel_tx + .send(Err(tonic::Status::unknown(e.to_string()))) + .await + .ok(); + return; + } + }; + for serialized_transaction_bytes in transactions { + let txid = match zebra_chain::transaction::Transaction::zcash_deserialize( + &mut std::io::Cursor::new(&serialized_transaction_bytes), + ) { + Ok(transaction) => transaction.hash().0.to_vec(), Err(error) => { if channel_tx .send(Err(tonic::Status::unknown(error.to_string()))) @@ -1423,8 +1415,8 @@ impl LightWalletIndexer for StateServiceSubscriber { } }; match ::parse_from_slice( - mempool_value.serialized_tx.as_ref().as_ref(), - Some(vec![txid_bytes]), + &serialized_transaction_bytes, + Some(vec![txid]), None, ) { Ok(transaction) => { @@ -1488,65 +1480,66 @@ impl LightWalletIndexer for StateServiceSubscriber { /// Return a stream of current Mempool transactions. This will keep the output stream open while /// there are mempool transactions. It will close the returned stream when a new block is mined. async fn get_mempool_stream(&self) -> Result { - let mut mempool = self.mempool.clone(); + let indexer = self.indexer.clone(); let service_timeout = self.config.common.service.timeout; let (channel_tx, channel_rx) = mpsc::channel(self.config.common.service.channel_size as usize); - let snapshot = self.indexer.snapshot_nonfinalized_state().await?; - let Some(non_finalized_snapshot) = snapshot.get_nfs_snapshot() else { - // TODO: This probably shouldn't be an error. - // this is an improvement over previous behaviour of - // acting as if we are only synced to the genesis block - return Err(StateServiceError::UnavailableNotSyncedEnough); - }; - let mempool_height = non_finalized_snapshot.best_tip.height.0; + let snapshot = indexer.snapshot_nonfinalized_state().await?; tokio::spawn(async move { let timeout = timeout( time::Duration::from_secs((service_timeout * 6) as u64), async { - let (mut mempool_stream, _mempool_handle) = match mempool - .get_mempool_stream(None) - .await - { - Ok(stream) => stream, - Err(e) => { - warn!(?e, "error fetching mempool stream"); + let Some(non_finalized_snapshot) = snapshot.get_nfs_snapshot() else { + // TODO: This probably shouldn't be an error. + // this is an improvement over previous behaviour of + // acting as if we are only synced to the genesis block + if let Err(e) = channel_tx + .send(Err(tonic::Status::failed_precondition( + "zaino not yet synced".to_string(), + ))) + .await + { + warn!(%e, "GetMempoolStream channel closed unexpectedly"); + }; + return; + }; + let mempool_height = non_finalized_snapshot.best_tip.height.0; + match indexer.get_mempool_stream(None) { + Some(mut mempool_stream) => { + while let Some(result) = mempool_stream.next().await { + match result { + Ok(transaction_bytes) => { + if channel_tx + .send(Ok(RawTransaction { + data: transaction_bytes, + height: mempool_height as u64, + })) + .await + .is_err() + { + break; + } + } + Err(e) => { + channel_tx + .send(Err(tonic::Status::internal(format!( + "Error in mempool stream: {e:?}" + )))) + .await + .ok(); + break; + } + } + } + } + None => { + warn!("Error fetching stream from mempool, Incorrect chain tip!"); channel_tx .send(Err(tonic::Status::internal("Error getting mempool stream"))) .await .ok(); - return; } }; - while let Some(result) = mempool_stream.recv().await { - match result { - Ok((_mempool_key, mempool_value)) => { - if channel_tx - .send(Ok(RawTransaction { - data: mempool_value - .serialized_tx - .as_ref() - .as_ref() - .to_vec(), - height: mempool_height as u64, - })) - .await - .is_err() - { - break; - } - } - Err(e) => { - channel_tx - .send(Err(tonic::Status::internal(format!( - "Error in mempool stream: {e:?}" - )))) - .await - .ok(); - break; - } - } - } }, ) .await; @@ -1562,7 +1555,6 @@ impl LightWalletIndexer for StateServiceSubscriber { } } }); - Ok(RawTransactionStream::new(channel_rx)) } diff --git a/packages/zaino-state/src/chain_index.rs b/packages/zaino-state/src/chain_index.rs index cd1e8f028..d796d8941 100644 --- a/packages/zaino-state/src/chain_index.rs +++ b/packages/zaino-state/src/chain_index.rs @@ -37,6 +37,7 @@ use source::{BlockchainSource, ValidatorConnector}; use tokio_stream::StreamExt; use tokio_util::sync::CancellationToken; use tracing::{info, instrument}; +use zaino_fetch::jsonrpsee::raw_transaction::validate_raw_transaction_hex; use zaino_fetch::jsonrpsee::response::{ address_deltas::{GetAddressDeltasParams, GetAddressDeltasResponse}, block_deltas::BlockDeltas, @@ -54,7 +55,10 @@ pub use zebra_chain::parameters::Network as ZebraNetwork; use zebra_chain::serialization::ZcashSerialize; use zebra_rpc::{ client::{GetAddressBalanceRequest, GetAddressTxIdsRequest}, - methods::{AddressBalance, GetAddressUtxos, GetBlock, GetBlockchainInfoResponse, GetInfo}, + methods::{ + AddressBalance, GetAddressUtxos, GetBlock, GetBlockchainInfoResponse, GetInfo, + SentTransactionHash, + }, }; use zebra_state::HashOrHeight; @@ -663,6 +667,19 @@ pub trait ChainIndexRpcExt: ChainIndex { height: Option, ) -> impl std::future::Future>; + /// Submits a raw transaction to the network (`sendrawtransaction`). + fn send_raw_transaction( + &self, + raw_transaction_hex: String, + ) -> impl std::future::Future>; + + /// Returns the full `z_gettreestate` response for the given hash-or-height, via the + /// backing validator (node-passthrough fallback for treestates not locally serviceable). + fn get_treestate_by_id( + &self, + hash_or_height: String, + ) -> impl std::future::Future>; + // ********** Transparent address history methods ********** /// Returns all changes for the given transparent addresses. @@ -2801,6 +2818,28 @@ impl ChainIndexRpcExt for NodeBackedChainIndexSubscrib .map_err(ChainIndexError::backing_validator) } + async fn send_raw_transaction( + &self, + raw_transaction_hex: String, + ) -> Result { + validate_raw_transaction_hex(&raw_transaction_hex) + .map_err(|error| ChainIndexError::internal(error.to_string()))?; + self.source() + .send_raw_transaction(raw_transaction_hex) + .await + .map_err(ChainIndexError::backing_validator) + } + + async fn get_treestate_by_id( + &self, + hash_or_height: String, + ) -> Result { + self.source() + .get_treestate_by_id(hash_or_height) + .await + .map_err(ChainIndexError::backing_validator) + } + /// Returns all changes for the given transparent addresses. async fn get_address_deltas( &self, diff --git a/packages/zaino-state/src/chain_index/source.rs b/packages/zaino-state/src/chain_index/source.rs index 4515e9b77..1ae1afe9d 100644 --- a/packages/zaino-state/src/chain_index/source.rs +++ b/packages/zaino-state/src/chain_index/source.rs @@ -29,7 +29,9 @@ use zebra_chain::{ }; use zebra_rpc::{ client::{GetAddressBalanceRequest, GetAddressTxIdsRequest}, - methods::{AddressBalance, GetAddressUtxos, GetBlockchainInfoResponse, GetInfo}, + methods::{ + AddressBalance, GetAddressUtxos, GetBlockchainInfoResponse, GetInfo, SentTransactionHash, + }, }; use zebra_state::{HashOrHeight, ReadRequest, ReadResponse, ReadStateService}; @@ -175,6 +177,21 @@ pub trait BlockchainSource: Clone + Send + Sync + 'static { height: Option, ) -> impl SendFut>; + /// Submits a raw transaction to the network via the validator's mempool + /// (`sendrawtransaction`). + fn send_raw_transaction( + &self, + raw_transaction_hex: String, + ) -> impl SendFut>; + + /// Returns the full `z_gettreestate` response for the given hash-or-height string. + /// + /// Node-passthrough fallback for treestates not locally serviceable by the index. + fn get_treestate_by_id( + &self, + hash_or_height: String, + ) -> impl SendFut>; + /// Returns the sapling and orchard treestate by hash fn get_treestate(&self, id: BlockHash) -> impl SendFut>; diff --git a/packages/zaino-state/src/chain_index/source/mockchain_source.rs b/packages/zaino-state/src/chain_index/source/mockchain_source.rs index 420ca24a8..0d867ded4 100644 --- a/packages/zaino-state/src/chain_index/source/mockchain_source.rs +++ b/packages/zaino-state/src/chain_index/source/mockchain_source.rs @@ -757,6 +757,23 @@ impl BlockchainSource for MockchainSource { ) } + async fn send_raw_transaction( + &self, + _raw_transaction_hex: String, + ) -> BlockchainSourceResult { + // The mock chain has no mempool to accept submissions. + unimplemented!("MockchainSource cannot serve send_raw_transaction") + } + + async fn get_treestate_by_id( + &self, + _hash_or_height: String, + ) -> BlockchainSourceResult { + // The `z_get_treestate` local path serves the mock; the node-passthrough fallback + // is never reached, so this is left unimplemented. + unimplemented!("MockchainSource cannot serve the get_treestate_by_id passthrough") + } + // ********** Transaction methods ********** async fn get_transaction( diff --git a/packages/zaino-state/src/chain_index/source/validator_connector.rs b/packages/zaino-state/src/chain_index/source/validator_connector.rs index 1bd6ac48a..f46d07bcc 100644 --- a/packages/zaino-state/src/chain_index/source/validator_connector.rs +++ b/packages/zaino-state/src/chain_index/source/validator_connector.rs @@ -27,7 +27,7 @@ use zebra_rpc::{ methods::{ chain_tip_difficulty, ConsensusBranchIdHex, GetBlock, GetBlockHeaderObject, GetBlockHeaderResponse, GetBlockTransaction, GetBlockTrees, GetBlockchainInfoResponse, - GetInfo, NetworkUpgradeInfo, NetworkUpgradeStatus, TipConsensusBranch, + GetInfo, NetworkUpgradeInfo, NetworkUpgradeStatus, SentTransactionHash, TipConsensusBranch, ValidateAddresses as _, }, }; @@ -302,6 +302,33 @@ impl BlockchainSource for ValidatorConnector { .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string())) } + async fn send_raw_transaction( + &self, + raw_transaction_hex: String, + ) -> BlockchainSourceResult { + // ReadStateService does not yet interface with the mempool, so both variants + // submit via the JSON-RPC connector. + self.json_rpc_connector() + .send_raw_transaction(raw_transaction_hex) + .await + .map(SentTransactionHash::from) + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string())) + } + + async fn get_treestate_by_id( + &self, + hash_or_height: String, + ) -> BlockchainSourceResult { + self.json_rpc_connector() + .get_treestate(hash_or_height) + .await + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))? + .try_into() + .map_err(|_error| { + BlockchainSourceError::Unrecoverable("failed to parse treestate".to_string()) + }) + } + // ********** Transaction methods ********** // Returns the transaction, and the height of the block that transaction is in if on the best chain diff --git a/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs b/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs index a5f694c4e..53e1d7751 100644 --- a/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs +++ b/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs @@ -704,6 +704,20 @@ impl BlockchainSource for ProptestMockchain { unimplemented!() } + async fn send_raw_transaction( + &self, + _raw_transaction_hex: String, + ) -> BlockchainSourceResult { + unimplemented!() + } + + async fn get_treestate_by_id( + &self, + _hash_or_height: String, + ) -> BlockchainSourceResult { + unimplemented!() + } + /// Returns the block commitment tree data by hash async fn get_commitment_tree_roots( &self, From 9c697eeeb1997efa4d8e751ae441ffdbb1708e6e Mon Sep 17 00:00:00 2001 From: idky137 Date: Fri, 3 Jul 2026 15:37:52 +0100 Subject: [PATCH 12/58] moved last split functionality out of indexerservices --- CHANGELOG.md | 16 ++ live-tests/clientless/tests/chain_cache.rs | 5 +- live-tests/e2e/tests/devtool.rs | 2 +- live-tests/e2e/tests/test_vectors.rs | 2 +- packages/zaino-state/src/backends/fetch.rs | 30 +-- packages/zaino-state/src/backends/state.rs | 233 +++++------------- packages/zaino-state/src/chain_index.rs | 15 +- .../zaino-state/src/chain_index/source.rs | 9 + .../chain_index/source/validator_connector.rs | 181 +++++++++++++- 9 files changed, 285 insertions(+), 208 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 621a927ce..c74c79e32 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,22 @@ and this library adheres to Rust's notion of ## Unreleased ### Changed +- `zaino-state`: the `ChainIndex` trait is split into `ChainIndex` (the + wallet-essential core: chain/tx/address/mempool access) and a + `ChainIndexRpcExt: ChainIndex` extension (compact-block serving, subtree + roots, and the block-explorer / mining / node-passthrough RPCs). The split is + a provisional first pass to be refined into finer capability traits later. +- `zaino-state`: all remaining backend-split RPC functionality has moved out of + the `FetchService` (`JsonRpSeeConnector`) and `StateService` + (`ReadStateService`) backends and into `BlockchainSource` / + `ChainIndex`. Both backends now resolve every fetch through their `ChainIndex` + indexer — building responses from Zaino's own indexed state where possible and + delegating to the `ValidatorConnector` (`BlockchainSource`) only for + validator-only or passthrough data. Validator connection/syncer spawning also + moves into `ValidatorConnector::spawn_fetch` / `spawn_state`, so each + service/subscriber now holds only `{ indexer, data, config }`. This readies + the two backends for their eventual merge into a single + `ValidatorBackedIndexerService`. No behaviour change. - **Breaking** — config: `storage.database.sync_write_batch_bytes` (bytes) is renamed to `sync_write_batch_size` and given in **GiB** (default raised from 4 GiB to 32 GiB); this budget now also bounds the txout-set accumulator diff --git a/live-tests/clientless/tests/chain_cache.rs b/live-tests/clientless/tests/chain_cache.rs index 8f724bd3c..2bc08ea2e 100644 --- a/live-tests/clientless/tests/chain_cache.rs +++ b/live-tests/clientless/tests/chain_cache.rs @@ -46,7 +46,10 @@ mod chain_query_interface { source::ValidatorConnector, NodeBackedChainIndex, NodeBackedChainIndexSubscriber, ShieldedPool, }, - test_dependencies::{chain_index::ChainIndex, ChainIndexConfig}, + test_dependencies::{ + chain_index::{ChainIndex, ChainIndexRpcExt}, + ChainIndexConfig, + }, FetchService, Height, StateService, StateServiceConfig, ZcashService, }; #[cfg(feature = "zcashd_support")] diff --git a/live-tests/e2e/tests/devtool.rs b/live-tests/e2e/tests/devtool.rs index 5e14d4443..6252148ab 100644 --- a/live-tests/e2e/tests/devtool.rs +++ b/live-tests/e2e/tests/devtool.rs @@ -2263,7 +2263,7 @@ async fn get_mempool_info_state() { let mut svc = fund_and_fill_mempool_dual().await; let info = svc.state_subscriber.get_mempool_info().await.unwrap(); - let entries = svc.state_subscriber.mempool.get_mempool().await; + let entries = svc.state_subscriber.mempool().get_mempool().await; assert_eq!(entries.len() as u64, info.size); assert!(info.size >= 1); diff --git a/live-tests/e2e/tests/test_vectors.rs b/live-tests/e2e/tests/test_vectors.rs index 83dc4ac47..36128df39 100644 --- a/live-tests/e2e/tests/test_vectors.rs +++ b/live-tests/e2e/tests/test_vectors.rs @@ -282,7 +282,7 @@ async fn create_200_block_regtest_chain_vectors() { }) .unwrap(); - let mut state = state_service_subscriber.read_state_service.clone(); + let mut state = state_service_subscriber.read_state_service(); let (sapling_root, orchard_root) = { let (sapling_tree_response, orchard_tree_response) = futures::future::join( state.clone().call(zebra_state::ReadRequest::SaplingTree( diff --git a/packages/zaino-state/src/backends/fetch.rs b/packages/zaino-state/src/backends/fetch.rs index 2c9a49d7a..273851768 100644 --- a/packages/zaino-state/src/backends/fetch.rs +++ b/packages/zaino-state/src/backends/fetch.rs @@ -24,7 +24,7 @@ use zebra_rpc::{ use zaino_fetch::{ chain::{transaction::FullTransaction, utils::ParseFromSlice}, jsonrpsee::{ - connector::{JsonRpSeeConnector, RpcError}, + connector::RpcError, response::{ address_deltas::{GetAddressDeltasParams, GetAddressDeltasResponse}, block_deltas::BlockDeltas, @@ -87,12 +87,6 @@ use crate::{ #[derive(Debug)] #[deprecated = "Will be eventually replaced by `BlockchainSource`"] pub struct FetchService { - /// JsonRPC Client. - /// - /// NOTE: DEPRECATED — no longer read now that every fetch goes through the indexer. - /// TODO(task 3): remove this field (services should hold only `indexer`, `data`, `config`). - #[allow(dead_code)] - fetcher: JsonRpSeeConnector, /// Core indexer. indexer: NodeBackedChainIndex, /// Service metadata. @@ -126,15 +120,10 @@ impl ZcashService for FetchService { "Launching Fetch Service" ); - let fetcher = JsonRpSeeConnector::new_from_config_parts( - &config.common.validator_rpc_address, - config.common.validator_rpc_user.clone(), - config.common.validator_rpc_password.clone(), - config.common.validator_cookie_path.clone(), - ) - .await?; + let (source, zebra_build_data) = ValidatorConnector::spawn_fetch(&config.common) + .await + .map_err(|error| FetchServiceError::Critical(error.to_string()))?; - let zebra_build_data = fetcher.get_info().await?; let data = ServiceMetadata::new( get_build_info(config.common.indexer_version.clone()), config.common.network.to_zebra_network(), @@ -143,13 +132,11 @@ impl ZcashService for FetchService { ); info!(build = %data.zebra_build(), subversion = %data.zebra_subversion(), "Connected to Zcash node"); - let source = ValidatorConnector::Fetch(fetcher.clone()); let indexer = NodeBackedChainIndex::new(source, config.clone().into()) .await - .unwrap(); + .map_err(|error| FetchServiceError::Critical(error.to_string()))?; let fetch_service = Self { - fetcher, indexer, data, config, @@ -176,7 +163,6 @@ impl ZcashService for FetchService { /// Returns a [`FetchServiceSubscriber`]. fn get_subscriber(&self) -> IndexerSubscriber { IndexerSubscriber::new(FetchServiceSubscriber { - fetcher: self.fetcher.clone(), indexer: self.indexer.subscriber(), data: self.data.clone(), config: self.config.clone(), @@ -206,12 +192,6 @@ impl Drop for FetchService { #[derive(Debug, Clone)] #[allow(deprecated)] pub struct FetchServiceSubscriber { - /// JsonRPC Client. - /// - /// NOTE: DEPRECATED — no longer read now that every fetch goes through the indexer. - /// TODO(task 3): remove this field (services should hold only `indexer`, `data`, `config`). - #[allow(dead_code)] - fetcher: JsonRpSeeConnector, /// Core indexer. pub indexer: NodeBackedChainIndexSubscriber, /// Service metadata. diff --git a/packages/zaino-state/src/backends/state.rs b/packages/zaino-state/src/backends/state.rs index af317cac1..cc6f4b609 100644 --- a/packages/zaino-state/src/backends/state.rs +++ b/packages/zaino-state/src/backends/state.rs @@ -3,23 +3,21 @@ #[allow(deprecated)] use crate::{ chain_index::{ - chain_tips_from_nonfinalized_snapshot, - mempool::{Mempool, MempoolSubscriber}, - source::ValidatorConnector, - types as chain_types, ChainIndex, ChainIndexRpcExt, + chain_tips_from_nonfinalized_snapshot, source::ValidatorConnector, types as chain_types, + ChainIndex, ChainIndexRpcExt, }, config::{DonationAddress, StateServiceConfig}, error::{BlockCacheError, StateServiceError}, indexer::{ handle_raw_transaction, IndexerSubscriber, LightWalletIndexer, ZcashIndexer, ZcashService, }, - status::{NamedAtomicStatus, Status, StatusType}, + status::{Status, StatusType}, stream::{ AddressStream, CompactBlockStream, CompactTransactionStream, RawTransactionStream, UtxoReplyStream, }, utils::{get_build_info, ServiceMetadata}, - BackendType, NodeBackedChainIndex, NodeBackedChainIndexSubscriber, State, + BackendType, NodeBackedChainIndex, NodeBackedChainIndexSubscriber, }; use crate::{ chain_index::{types::BestChainLocation, NonFinalizedSnapshot}, @@ -29,7 +27,7 @@ use tokio_stream::StreamExt as _; use zaino_fetch::{ chain::{transaction::FullTransaction, utils::ParseFromSlice}, jsonrpsee::{ - connector::{JsonRpSeeConnector, RpcError}, + connector::RpcError, response::{ address_deltas::{GetAddressDeltasParams, GetAddressDeltasResponse}, block_deltas::BlockDeltas, @@ -69,31 +67,17 @@ use zebra_rpc::{ GetBlockchainInfoResponse, GetInfo, GetRawTransaction, SentTransactionHash, }, server::error::LegacyCode, - sync::init_read_state_with_syncer, }; -use zebra_state::{HashOrHeight, ReadRequest, ReadResponse, ReadStateService}; +use zebra_state::HashOrHeight; -use futures::TryFutureExt as _; use hex::{FromHex as _, ToHex}; -use std::{str::FromStr, sync::Arc}; +use std::str::FromStr; use tokio::{ sync::mpsc, time::{self, timeout}, }; -use tower::{Service, ServiceExt}; use tracing::{info, instrument, warn}; -macro_rules! expected_read_response { - ($response:ident, $expected_variant:ident) => { - match $response { - ReadResponse::$expected_variant(inner) => inner, - unexpected => { - unreachable!("Unexpected response from state service: {unexpected:?}") - } - } - }; -} - /// Chain fetch service backed by Zebra's `ReadStateService` and `TrustedChainSync`. /// /// NOTE: We currently dop not implement clone for chain fetch services @@ -105,51 +89,20 @@ macro_rules! expected_read_response { #[derive(Debug)] // #[deprecated = "Will be eventually replaced by `BlockchainSource"] pub struct StateService { - /// `ReadeStateService` from Zebra-State. - read_state_service: ReadStateService, - - /// Internal mempool. - mempool: Mempool, - - /// StateService config data. - #[allow(deprecated)] - config: StateServiceConfig, - - /// Listener for when the chain tip changes - chain_tip_change: zebra_state::ChainTipChange, - - /// Sync task handle. - sync_task_handle: Option>>, - - /// JsonRPC Client. - rpc_client: JsonRpSeeConnector, - /// Core indexer. indexer: NodeBackedChainIndex, /// Service metadata. data: ServiceMetadata, - /// Thread-safe status indicator. - status: NamedAtomicStatus, -} - -impl StateService { - #[cfg(feature = "test_dependencies")] - /// Helper for tests - pub fn read_state_service(&self) -> &ReadStateService { - &self.read_state_service - } + /// StateService config data. + #[allow(deprecated)] + config: StateServiceConfig, } impl Status for StateService { fn status(&self) -> StatusType { - let current_status = self.status.load(); - if current_status == StatusType::Closing { - current_status - } else { - self.indexer.status() - } + self.indexer.status() } } @@ -169,15 +122,9 @@ impl ZcashService for StateService { "Spawning State Service" ); - let rpc_client = JsonRpSeeConnector::new_from_config_parts( - &config.common.validator_rpc_address, - config.common.validator_rpc_user.clone(), - config.common.validator_rpc_password.clone(), - config.common.validator_cookie_path.clone(), - ) - .await?; - - let zebra_build_data = rpc_client.get_info().await?; + let (source, zebra_build_data) = ValidatorConnector::spawn_state(&config) + .await + .map_err(|error| StateServiceError::Critical(error.to_string()))?; let data = ServiceMetadata::new( get_build_info(config.common.indexer_version.clone()), @@ -187,101 +134,25 @@ impl ZcashService for StateService { ); info!(build = %data.zebra_build(), subversion = %data.zebra_subversion(), "Connected to Zcash node"); - info!( - grpc_address = %config.validator_grpc_address, - "Launching Chain Syncer" - ); - let (mut read_state_service, _latest_chain_tip, chain_tip_change, sync_task_handle) = - init_read_state_with_syncer( - config.validator_state_config.clone(), - &config.common.network.to_zebra_network(), - config.validator_grpc_address, - ) - .await??; - - info!("Chain syncer launched"); - - // Wait for ReadStateService to catch up to the validator's best chain tip. - // Height alone is insufficient during reorgs: the same height can refer to - // different blocks until JSON-RPC and ReadStateService agree on tip hash. - loop { - let blockchain_info = rpc_client.get_blockchain_info().await?; - let server_height = blockchain_info.blocks; - let server_tip_hash = blockchain_info.best_block_hash; - - let syncer_response = read_state_service - .ready() - .and_then(|service| service.call(ReadRequest::Tip)) - .await?; - let (syncer_height, syncer_tip_hash) = - expected_read_response!(syncer_response, Tip).ok_or( - RpcError::new_from_legacycode(LegacyCode::Misc, "no blocks in chain"), - )?; - - if server_height == syncer_height && server_tip_hash == syncer_tip_hash { - info!( - height = syncer_height.0, - tip_hash = %syncer_tip_hash, - "ReadStateService synced with Zebra" - ); - break; - } else { - info!( - syncer_height = syncer_height.0, - validator_height = server_height.0, - syncer_tip_hash = %syncer_tip_hash, - validator_tip_hash = %server_tip_hash, - "ReadStateService syncing with Zebra" - ); - tokio::time::sleep(std::time::Duration::from_millis(1000)).await; - continue; - } - } - - let mempool_source = ValidatorConnector::State(crate::chain_index::source::State { - read_state_service: read_state_service.clone(), - mempool_fetcher: rpc_client.clone(), - network: config.common.network, - }); - - let mempool = Mempool::spawn(mempool_source, None).await?; - - let chain_index = NodeBackedChainIndex::new( - ValidatorConnector::State(State { - read_state_service: read_state_service.clone(), - mempool_fetcher: rpc_client.clone(), - network: config.common.network, - }), - config.clone().into(), - ) - .await - .unwrap(); + let indexer = NodeBackedChainIndex::new(source, config.clone().into()) + .await + .map_err(|error| StateServiceError::Critical(error.to_string()))?; let state_service = Self { - chain_tip_change, - read_state_service, - sync_task_handle: Some(Arc::new(sync_task_handle)), - rpc_client: rpc_client.clone(), - mempool, - indexer: chain_index, + indexer, data, config, - status: NamedAtomicStatus::new("StateService", StatusType::Spawning), }; // wait for sync to complete, return error on sync fail. loop { match state_service.status() { - StatusType::Ready => { - state_service.status.store(StatusType::Ready); - break; - } + StatusType::Ready | StatusType::Closing => break, StatusType::CriticalError => { return Err(StateServiceError::Critical( "Chain index sync failed".to_string(), )); } - StatusType::Closing => break, _ => { tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; } @@ -293,23 +164,23 @@ impl ZcashService for StateService { fn get_subscriber(&self) -> IndexerSubscriber { IndexerSubscriber::new(StateServiceSubscriber { - read_state_service: self.read_state_service.clone(), - rpc_client: self.rpc_client.clone(), - mempool: self.mempool.subscriber(), indexer: self.indexer.subscriber(), data: self.data.clone(), config: self.config.clone(), - chain_tip_change: self.chain_tip_change.clone(), }) } /// Shuts down the StateService. + /// + /// Delegates to the indexer, which cancels its sync loop, tears down the + /// finalised DB and mempool, and aborts the source-owned Zebra chain-syncer + /// task via [`crate::chain_index::source::BlockchainSource::shutdown`]. fn close(&mut self) { - if self.sync_task_handle.is_some() { - if let Some(handle) = self.sync_task_handle.take() { - handle.abort(); - } - } + tokio::task::block_in_place(|| { + tokio::runtime::Handle::current().block_on(async { + let _ = self.indexer.shutdown().await; + }); + }); } } @@ -326,27 +197,41 @@ impl Drop for StateService { #[derive(Debug, Clone)] // #[deprecated] pub struct StateServiceSubscriber { - /// Remote wrappper functionality for zebra's [`ReadStateService`]. - pub read_state_service: ReadStateService, + /// Core indexer. + pub indexer: NodeBackedChainIndexSubscriber, - /// Internal mempool. - pub mempool: MempoolSubscriber, + /// Service metadata. + pub data: ServiceMetadata, /// StateService config data. #[allow(deprecated)] config: StateServiceConfig, +} - /// Listener for when the chain tip changes - chain_tip_change: zebra_state::ChainTipChange, - - /// JsonRPC Client. - pub rpc_client: JsonRpSeeConnector, - - /// Core indexer. - pub indexer: NodeBackedChainIndexSubscriber, +impl StateServiceSubscriber { + /// The backing Zebra [`ReadStateService`]. + /// + /// Test-only escape hatch: live tests recompute expected chain data (e.g. + /// treestate roots) directly off the `ReadStateService`. Production code goes + /// through the `ChainIndex` API. + #[cfg(feature = "test_dependencies")] + pub fn read_state_service(&self) -> zebra_state::ReadStateService { + self.indexer + .source() + .read_state_service() + .expect("StateServiceSubscriber is always State-backed") + .clone() + } - /// Service metadata. - pub data: ServiceMetadata, + /// The indexer's mempool subscriber. + /// + /// Test-only escape hatch: live tests recompute expected `getmempoolinfo` + /// values directly off the mempool's entries. Production code goes through the + /// `ChainIndex` mempool API. + #[cfg(feature = "test_dependencies")] + pub fn mempool(&self) -> &crate::chain_index::mempool::MempoolSubscriber { + self.indexer.mempool_subscriber() + } } impl Status for StateServiceSubscriber { @@ -383,7 +268,11 @@ impl StateServiceSubscriber { /// Gets a Subscriber to any updates to the latest chain tip pub fn chaintip_update_subscriber(&self) -> ChainTipSubscriber { ChainTipSubscriber { - monitor: self.chain_tip_change.clone(), + monitor: self + .indexer + .source() + .chain_tip_change() + .expect("StateServiceSubscriber is always State-backed"), } } /// Return a list of consecutive compact blocks. diff --git a/packages/zaino-state/src/chain_index.rs b/packages/zaino-state/src/chain_index.rs index d796d8941..66a4d5873 100644 --- a/packages/zaino-state/src/chain_index.rs +++ b/packages/zaino-state/src/chain_index.rs @@ -973,6 +973,9 @@ impl NodeBackedChainIndex { self.status.store(StatusType::Closing); self.finalized_db.shutdown().await?; self.mempool.close(); + // Release any source-owned background work (e.g. the Zebra chain-syncer + // task behind the `State` connector). No-op for poll-only sources. + self.source.shutdown(); Ok(()) } @@ -1291,10 +1294,20 @@ async fn compact_block_from_source( } impl NodeBackedChainIndexSubscriber { - fn source(&self) -> &Source { + pub(crate) fn source(&self) -> &Source { &self.source } + /// The indexer's mempool subscriber. + /// + /// Test-only escape hatch: live tests recompute expected `getmempoolinfo` + /// values directly off the mempool's entries. Production code goes through + /// the `ChainIndex` mempool API. + #[cfg(feature = "test_dependencies")] + pub(crate) fn mempool_subscriber(&self) -> &mempool::MempoolSubscriber { + &self.mempool + } + /// Returns the combined status of all chain index components. pub fn combined_status(&self) -> StatusType { let finalized_status = self.finalized_state.status(); diff --git a/packages/zaino-state/src/chain_index/source.rs b/packages/zaino-state/src/chain_index/source.rs index 1ae1afe9d..acc8a1915 100644 --- a/packages/zaino-state/src/chain_index/source.rs +++ b/packages/zaino-state/src/chain_index/source.rs @@ -328,6 +328,15 @@ pub trait BlockchainSource: Clone + Send + Sync + 'static { fn subscribe_to_blocks_received(&self) -> Option> { None } + + /// Release any long-lived resources the source owns (e.g. a background + /// syncer task feeding a `ReadStateService`). + /// + /// Default is a no-op — poll-only sources (`JsonRpSeeConnector`) and test + /// mockchains own nothing to tear down. Sources that spawn their own + /// validator plumbing (the `State` arm of `ValidatorConnector`, which owns + /// the Zebra syncer task) override this to abort that task on shutdown. + fn shutdown(&self) {} } /// Sleep up to `duration`, but return early if `change_rx` resolves first. diff --git a/packages/zaino-state/src/chain_index/source/validator_connector.rs b/packages/zaino-state/src/chain_index/source/validator_connector.rs index f46d07bcc..d8b3bb50e 100644 --- a/packages/zaino-state/src/chain_index/source/validator_connector.rs +++ b/packages/zaino-state/src/chain_index/source/validator_connector.rs @@ -6,6 +6,7 @@ use std::str::FromStr as _; use chrono::{DateTime, Utc}; use hex::{FromHex as _, ToHex as _}; use indexmap::IndexMap; +use tracing::info; use zaino_fetch::jsonrpsee::response::{ address_deltas::BlockInfo, block_deltas::{BlockDelta, BlockDeltas, InputDelta, OutputDelta}, @@ -13,8 +14,12 @@ use zaino_fetch::jsonrpsee::response::{ block_subsidy::GetBlockSubsidy, mining_info::GetMiningInfoWire, peer_info::GetPeerInfo, - GetNetworkSolPsResponse, GetSpentInfoRequest, GetSpentInfoResponse, GetTxOutResponse, + GetInfoResponse, GetNetworkSolPsResponse, GetSpentInfoRequest, GetSpentInfoResponse, + GetTxOutResponse, }; +use zebra_rpc::sync::init_read_state_with_syncer; + +use crate::config::{CommonBackendConfig, StateServiceConfig}; use zebra_chain::{ amount::{Amount, NonNegative}, block::{Header, SerializedBlock}, @@ -62,6 +67,12 @@ pub struct State { pub mempool_fetcher: JsonRpSeeConnector, /// Current network type being run. pub network: Network, + /// Watches the Zebra syncer's chain-tip changes; served to consumers via + /// [`ValidatorConnector::chain_tip_change`]. + pub chain_tip_change: zebra_state::ChainTipChange, + /// Handle to the Zebra `ReadStateService` sync task, kept alive for the lifetime of + /// the connector and aborted on [`ValidatorConnector::shutdown`]. Shared across clones. + pub sync_task_handle: Option>>, } /// A connection to a validator. @@ -87,6 +98,151 @@ impl ValidatorConnector { ValidatorConnector::Fetch(fetch) => fetch, } } + + /// Spawns a JSON-RPC-backed [`ValidatorConnector::Fetch`] from the common backend + /// config, returning the connector plus the validator's `getinfo` response (used by + /// the backend to build its `ServiceMetadata`). + /// + /// Owns the `JsonRpSeeConnector` setup that previously lived in `FetchService::spawn`. + pub(crate) async fn spawn_fetch( + common: &CommonBackendConfig, + ) -> Result<(Self, GetInfoResponse), BlockchainSourceError> { + let fetcher = JsonRpSeeConnector::new_from_config_parts( + &common.validator_rpc_address, + common.validator_rpc_user.clone(), + common.validator_rpc_password.clone(), + common.validator_cookie_path.clone(), + ) + .await + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + + let info = fetcher + .get_info() + .await + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + + Ok((ValidatorConnector::Fetch(fetcher), info)) + } + + /// Spawns a `ReadStateService`-backed [`ValidatorConnector::State`] from the state + /// backend config, returning the connector plus the validator's `getinfo` response. + /// + /// Owns the JSON-RPC + Zebra chain-syncer setup that previously lived in + /// `StateService::spawn`: builds the mempool JSON-RPC fetcher, launches the syncer + /// and `ReadStateService`, then blocks until the syncer has caught up to the + /// validator's best chain tip (comparing tip *hash* as well as height so a reorg + /// mid-sync cannot report a false match). The syncer task handle is retained on the + /// `State` so [`ValidatorConnector::shutdown`] can abort it. + pub(crate) async fn spawn_state( + config: &StateServiceConfig, + ) -> Result<(Self, GetInfoResponse), BlockchainSourceError> { + let map_err = + |error: &dyn std::fmt::Display| BlockchainSourceError::Unrecoverable(error.to_string()); + + let rpc_client = JsonRpSeeConnector::new_from_config_parts( + &config.common.validator_rpc_address, + config.common.validator_rpc_user.clone(), + config.common.validator_rpc_password.clone(), + config.common.validator_cookie_path.clone(), + ) + .await + .map_err(|error| map_err(&error))?; + + let info = rpc_client + .get_info() + .await + .map_err(|error| map_err(&error))?; + + info!( + grpc_address = %config.validator_grpc_address, + "Launching Chain Syncer" + ); + let (mut read_state_service, _latest_chain_tip, chain_tip_change, sync_task_handle) = + init_read_state_with_syncer( + config.validator_state_config.clone(), + &config.common.network.to_zebra_network(), + config.validator_grpc_address, + ) + .await + .map_err(|error| map_err(&error))? + .map_err(|error| map_err(&error))?; + + info!("Chain syncer launched"); + + // Wait for ReadStateService to catch up to the validator's best chain tip. + // Height alone is insufficient during reorgs: the same height can refer to + // different blocks until JSON-RPC and ReadStateService agree on tip hash. + loop { + let blockchain_info = rpc_client + .get_blockchain_info() + .await + .map_err(|error| map_err(&error))?; + let server_height = blockchain_info.blocks; + let server_tip_hash = blockchain_info.best_block_hash; + + let syncer_response = read_state_service + .ready() + .and_then(|service| service.call(ReadRequest::Tip)) + .await + .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + let (syncer_height, syncer_tip_hash) = expected_read_response!(syncer_response, Tip) + .ok_or_else(|| { + BlockchainSourceError::Unrecoverable("no blocks in chain".to_string()) + })?; + + if server_height == syncer_height && server_tip_hash == syncer_tip_hash { + info!( + height = syncer_height.0, + tip_hash = %syncer_tip_hash, + "ReadStateService synced with Zebra" + ); + break; + } else { + info!( + syncer_height = syncer_height.0, + validator_height = server_height.0, + syncer_tip_hash = %syncer_tip_hash, + validator_tip_hash = %server_tip_hash, + "ReadStateService syncing with Zebra" + ); + tokio::time::sleep(std::time::Duration::from_millis(1000)).await; + continue; + } + } + + let source = ValidatorConnector::State(State { + read_state_service, + mempool_fetcher: rpc_client, + network: config.common.network, + chain_tip_change, + sync_task_handle: Some(Arc::new(sync_task_handle)), + }); + + Ok((source, info)) + } + + /// Watches the Zebra syncer's chain-tip changes, when this connector owns one. + /// + /// `Some` for the `State` variant (which drives its own syncer); `None` for the + /// JSON-RPC `Fetch` variant, which has no local tip-change stream. + pub(crate) fn chain_tip_change(&self) -> Option { + match self { + ValidatorConnector::State(state) => Some(state.chain_tip_change.clone()), + ValidatorConnector::Fetch(_) => None, + } + } + + /// The backing [`ReadStateService`], when this connector is `State`-backed. + /// + /// Test-only escape hatch: live tests recompute expected chain data directly off + /// the `ReadStateService`. Production code goes through the `ChainIndex` API. + #[cfg(feature = "test_dependencies")] + pub(crate) fn read_state_service(&self) -> Option<&ReadStateService> { + match self { + ValidatorConnector::State(state) => Some(&state.read_state_service), + ValidatorConnector::Fetch(_) => None, + } + } } impl BlockchainSource for ValidatorConnector { @@ -345,7 +501,7 @@ impl BlockchainSource for ValidatorConnector { ValidatorConnector::State(State { read_state_service, mempool_fetcher, - network: _, + .. }) => { // Check state for transaction let mut read_state_service = read_state_service.clone(); @@ -502,7 +658,7 @@ impl BlockchainSource for ValidatorConnector { ValidatorConnector::State(State { read_state_service, mempool_fetcher, - network: _, + .. }) => { match read_state_service.best_tip() { Some((_height, hash)) => Ok(Some(hash)), @@ -544,7 +700,7 @@ impl BlockchainSource for ValidatorConnector { ValidatorConnector::State(State { read_state_service, mempool_fetcher, - network: _, + .. }) => { match read_state_service.best_tip() { Some((height, _hash)) => Ok(Some(height)), @@ -1269,9 +1425,7 @@ impl BlockchainSource for ValidatorConnector { > { match self { ValidatorConnector::State(State { - read_state_service, - mempool_fetcher: _, - network: _, + read_state_service, .. }) => { match read_state_service .clone() @@ -1293,6 +1447,19 @@ impl BlockchainSource for ValidatorConnector { ValidatorConnector::Fetch(_fetch) => Ok(None), } } + + fn shutdown(&self) { + // Only the `State` arm owns a long-lived resource: the Zebra chain-syncer + // task feeding the `ReadStateService`. Abort it so the backend drops cleanly. + // The `Fetch` arm holds only a stateless JSON-RPC client — nothing to tear down. + if let ValidatorConnector::State(State { + sync_task_handle: Some(handle), + .. + }) = self + { + handle.abort(); + } + } } impl State { From 6d838c05e7f1e4d69bd02167bbf3d39a2d59fe49 Mon Sep 17 00:00:00 2001 From: idky137 Date: Sun, 5 Jul 2026 15:37:34 +0100 Subject: [PATCH 13/58] merged fetchservice and stateservice into unified NodeBackedIndexerService --- CHANGELOG.md | 13 + live-tests/clientless/tests/chain_cache.rs | 171 +- live-tests/clientless/tests/fetch_service.rs | 9 +- live-tests/clientless/tests/json_server.rs | 19 +- live-tests/clientless/tests/state_service.rs | 9 +- live-tests/e2e/tests/devtool.rs | 309 ++- live-tests/e2e/tests/devtool_zcashd.rs | 12 +- live-tests/e2e/tests/test_vectors.rs | 12 +- live-tests/zaino-testutils/src/lib.rs | 308 ++- packages/zaino-state/src/backends.rs | 71 - packages/zaino-state/src/backends/state.rs | 1896 ----------------- .../chain_index/source/validator_connector.rs | 40 +- .../src/chain_index/tests/mockchain_tests.rs | 24 + packages/zaino-state/src/config.rs | 176 +- packages/zaino-state/src/error.rs | 166 +- packages/zaino-state/src/indexer.rs | 266 ++- .../node_backed_indexer.rs} | 527 +++-- packages/zaino-state/src/lib.rs | 16 +- packages/zainod/src/config.rs | 209 +- packages/zainod/src/error.rs | 27 +- packages/zainod/src/indexer.rs | 27 +- 21 files changed, 1250 insertions(+), 3057 deletions(-) delete mode 100644 packages/zaino-state/src/backends.rs delete mode 100644 packages/zaino-state/src/backends/state.rs rename packages/zaino-state/src/{backends/fetch.rs => indexer/node_backed_indexer.rs} (82%) diff --git a/CHANGELOG.md b/CHANGELOG.md index c74c79e32..ee246f9b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,19 @@ and this library adheres to Rust's notion of ## Unreleased ### Changed +- `zaino-state`: `FetchService` and `StateService` are merged into a single + generic `NodeBackedIndexerService` (module + `zaino_state::indexer::node_backed_indexer`; the former `backends` module is + gone). The validator connection is now selected at runtime rather than by type: + `NodeBackedIndexerServiceConfig { common, connection }` carries a + `ValidatorConnectionType` of either `Rpc` (JSON-RPC, formerly `Fetch`) or + `Direct(DirectConnectionConfig)` (Zebra `ReadStateService`, formerly `State`). + The per-backend `Fetch/StateServiceConfig`, `Fetch/StateServiceError`, and + `BackendConfig` types are replaced by `NodeBackedIndexerServiceConfig`, + `NodeBackedIndexerServiceError`, and `ValidatorConnectionType`. +- **Breaking** — config: `zainod.toml`'s `backend` selector is renamed + `state` → `direct` and `fetch` → `rpc`. The legacy `"state"` / `"fetch"` + values are still accepted as aliases, so existing config files keep working. - `zaino-state`: the `ChainIndex` trait is split into `ChainIndex` (the wallet-essential core: chain/tx/address/mempool access) and a `ChainIndexRpcExt: ChainIndex` extension (compact-block serving, subtree diff --git a/live-tests/clientless/tests/chain_cache.rs b/live-tests/clientless/tests/chain_cache.rs index 2bc08ea2e..cad4f6816 100644 --- a/live-tests/clientless/tests/chain_cache.rs +++ b/live-tests/clientless/tests/chain_cache.rs @@ -1,24 +1,20 @@ use zaino_common::network::ActivationHeights; use zaino_fetch::jsonrpsee::connector::JsonRpSeeConnector; -use zaino_state::{ZcashIndexer, ZcashService}; -use zaino_testutils::{TestManager, ValidatorExt, ValidatorKind}; -use zainodlib::error::IndexerError; +use zaino_testutils::{Direct, Rpc, TestManager, ValidatorExt, ValidatorKind}; #[allow(deprecated)] -async fn create_test_manager_and_connector( +async fn create_test_manager_and_connector( validator: &ValidatorKind, activation_heights: Option, chain_cache: Option, enable_zaino: bool, enable_clients: bool, -) -> (TestManager, JsonRpSeeConnector) +) -> (TestManager, JsonRpSeeConnector) where T: ValidatorExt, - Service: zaino_testutils::TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: zaino_testutils::PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { - let test_manager = TestManager::::launch( + let test_manager = TestManager::::launch( validator, None, activation_heights, @@ -50,7 +46,7 @@ mod chain_query_interface { chain_index::{ChainIndex, ChainIndexRpcExt}, ChainIndexConfig, }, - FetchService, Height, StateService, StateServiceConfig, ZcashService, + Height, NodeBackedIndexerService, NodeBackedIndexerServiceConfig, ZcashService, }; #[cfg(feature = "zcashd_support")] use zcash_local_net::validator::zcashd::Zcashd; @@ -66,26 +62,24 @@ mod chain_query_interface { use super::*; #[allow(deprecated)] - async fn create_test_manager_and_chain_index( + async fn create_test_manager_and_chain_index( validator: &ValidatorKind, chain_cache: Option, enable_zaino: bool, enable_clients: bool, ephemeral: bool, ) -> ( - TestManager, + TestManager, JsonRpSeeConnector, - Option, + Option, NodeBackedChainIndex, NodeBackedChainIndexSubscriber, ) where C: ValidatorExt, - Service: zaino_testutils::TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: zaino_testutils::PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { - let (test_manager, json_service) = create_test_manager_and_connector::( + let (test_manager, json_service) = create_test_manager_and_connector::( validator, None, chain_cache.clone(), @@ -125,42 +119,43 @@ mod chain_query_interface { NetworkKind::Testnet => zebra_chain::parameters::Network::new_default_testnet(), NetworkKind::Mainnet => zebra_chain::parameters::Network::Mainnet, }; - // FIXME: when state service is integrated into chain index this initialization must change - let state_service = StateService::spawn(StateServiceConfig::new( - zebra_state::Config { - cache_dir: state_chain_cache_dir, - ephemeral: false, - delete_old_database: true, - debug_stop_at_height: None, - debug_validity_check_interval: None, - // todo: does this matter? - should_backup_non_finalized_state: true, - debug_skip_non_finalized_state_backup_task: false, - }, - test_manager.full_node_rpc_listen_address.to_string(), - test_manager.full_node_grpc_listen_address, - false, - None, - None, - None, - ServiceConfig::default(), - StorageConfig { - cache: CacheConfig::default(), - database: DatabaseConfig { - path: test_manager - .data_dir - .as_path() - .to_path_buf() - .join("state-service-zaino"), - ..Default::default() + // FIXME: when the direct connection is integrated into chain index this initialization must change + let state_service = + NodeBackedIndexerService::spawn(NodeBackedIndexerServiceConfig::new_direct( + zebra_state::Config { + cache_dir: state_chain_cache_dir, + ephemeral: false, + delete_old_database: true, + debug_stop_at_height: None, + debug_validity_check_interval: None, + // todo: does this matter? + should_backup_non_finalized_state: true, + debug_skip_non_finalized_state_backup_task: false, }, - }, - false, - network.into(), - None, - )) - .await - .unwrap(); + test_manager.full_node_rpc_listen_address.to_string(), + test_manager.full_node_grpc_listen_address, + false, + None, + None, + None, + ServiceConfig::default(), + StorageConfig { + cache: CacheConfig::default(), + database: DatabaseConfig { + path: test_manager + .data_dir + .as_path() + .to_path_buf() + .join("state-service-zaino"), + ..Default::default() + }, + }, + false, + network.into(), + None, + )) + .await + .unwrap(); let config = ChainIndexConfig { storage: StorageConfig { database: DatabaseConfig { @@ -240,25 +235,23 @@ mod chain_query_interface { // #[ignore = "prone to timeouts and hangs, to be fixed in chain index integration"] #[tokio::test(flavor = "multi_thread")] async fn get_block_range_zebrad() { - get_block_range::(&ValidatorKind::Zebrad).await + get_block_range::(&ValidatorKind::Zebrad).await } #[cfg(feature = "zcashd_support")] #[ignore = "prone to timeouts and hangs, to be fixed in chain index integration"] #[tokio::test(flavor = "multi_thread")] async fn get_block_range_zcashd() { - get_block_range::(&ValidatorKind::Zcashd).await + get_block_range::(&ValidatorKind::Zcashd).await } - async fn get_block_range(validator: &ValidatorKind) + async fn get_block_range(validator: &ValidatorKind) where C: ValidatorExt, - Service: zaino_testutils::TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: zaino_testutils::PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { let (test_manager, _json_service, _option_state_service, _chain_index, indexer) = - create_test_manager_and_chain_index::(validator, None, false, false, false) + create_test_manager_and_chain_index::(validator, None, false, false, false) .await; test_manager @@ -280,7 +273,7 @@ mod chain_query_interface { #[tokio::test(flavor = "multi_thread")] async fn ephemeral_serves_finalised_blocks_zebrad() { - ephemeral_serves_finalised_blocks::(&ValidatorKind::Zebrad).await + ephemeral_serves_finalised_blocks::(&ValidatorKind::Zebrad).await } /// Ephemeral mode on regtest: the chain index opens no persistent @@ -296,17 +289,15 @@ mod chain_query_interface { /// its hash, and asserts the two are identical; /// - streams compact blocks across the finalised / non-finalised boundary; /// - asserts nothing was persisted to disk. - async fn ephemeral_serves_finalised_blocks(validator: &ValidatorKind) + async fn ephemeral_serves_finalised_blocks(validator: &ValidatorKind) where C: ValidatorExt, - Service: zaino_testutils::TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: zaino_testutils::PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { use zaino_proto::proto::utils::PoolTypeFilter; let (test_manager, json_service, _option_state_service, _chain_index, indexer) = - create_test_manager_and_chain_index::(validator, None, false, false, true) + create_test_manager_and_chain_index::(validator, None, false, false, true) .await; // Generate well past MAX_NFS_DEPTH (110) so low heights are evicted from @@ -375,25 +366,23 @@ mod chain_query_interface { #[ignore = "prone to timeouts and hangs, to be fixed in chain index integration"] #[tokio::test(flavor = "multi_thread")] async fn sync_large_chain_zebrad() { - sync_large_chain::(&ValidatorKind::Zebrad).await + sync_large_chain::(&ValidatorKind::Zebrad).await } #[cfg(feature = "zcashd_support")] #[ignore = "prone to timeouts and hangs, to be fixed in chain index integration"] #[tokio::test(flavor = "multi_thread")] async fn sync_large_chain_zcashd() { - sync_large_chain::(&ValidatorKind::Zcashd).await + sync_large_chain::(&ValidatorKind::Zcashd).await } - async fn sync_large_chain(validator: &ValidatorKind) + async fn sync_large_chain(validator: &ValidatorKind) where C: ValidatorExt, - Service: zaino_testutils::TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: zaino_testutils::PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { let (test_manager, json_service, option_state_service, _chain_index, indexer) = - create_test_manager_and_chain_index::(validator, None, false, false, false) + create_test_manager_and_chain_index::(validator, None, false, false, false) .await; test_manager @@ -451,25 +440,23 @@ mod chain_query_interface { // #[ignore = "prone to timeouts and hangs, to be fixed in chain index integration"] #[tokio::test(flavor = "multi_thread")] async fn get_subtree_roots_zebrad() { - get_subtree_roots::(&ValidatorKind::Zebrad).await + get_subtree_roots::(&ValidatorKind::Zebrad).await } #[cfg(feature = "zcashd_support")] #[ignore = "prone to timeouts and hangs, to be fixed in chain index integration"] #[tokio::test(flavor = "multi_thread")] async fn get_subtree_roots_zcashd() { - get_subtree_roots::(&ValidatorKind::Zcashd).await + get_subtree_roots::(&ValidatorKind::Zcashd).await } - async fn get_subtree_roots(validator: &ValidatorKind) + async fn get_subtree_roots(validator: &ValidatorKind) where C: ValidatorExt, - Service: zaino_testutils::TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: zaino_testutils::PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { let (test_manager, json_service, _option_state_service, _chain_index, indexer) = - create_test_manager_and_chain_index::(validator, None, false, false, false) + create_test_manager_and_chain_index::(validator, None, false, false, false) .await; test_manager @@ -556,30 +543,26 @@ mod chain_query_interface { // #[ignore = "prone to timeouts and hangs, to be fixed in chain index integration"] #[tokio::test(flavor = "multi_thread")] async fn get_mempool_stream_fresh_snapshot_repeated_zebrad() { - get_mempool_stream_fresh_snapshot_repeated::(&ValidatorKind::Zebrad) - .await + get_mempool_stream_fresh_snapshot_repeated::(&ValidatorKind::Zebrad).await } #[cfg(feature = "zcashd_support")] #[ignore = "prone to timeouts and hangs, to be fixed in chain index integration"] #[tokio::test(flavor = "multi_thread")] async fn get_mempool_stream_fresh_snapshot_repeated_zcashd() { - get_mempool_stream_fresh_snapshot_repeated::(&ValidatorKind::Zcashd) - .await + get_mempool_stream_fresh_snapshot_repeated::(&ValidatorKind::Zcashd).await } - async fn get_mempool_stream_fresh_snapshot_repeated(validator: &ValidatorKind) + async fn get_mempool_stream_fresh_snapshot_repeated(validator: &ValidatorKind) where C: ValidatorExt, - Service: zaino_testutils::TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: zaino_testutils::PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { use futures::StreamExt as _; use tokio::time::{timeout, Duration}; let (test_manager, _json_service, _option_state_service, _chain_index, indexer) = - create_test_manager_and_chain_index::(validator, None, false, false, false) + create_test_manager_and_chain_index::(validator, None, false, false, false) .await; test_manager @@ -615,28 +598,26 @@ mod chain_query_interface { // #[ignore = "prone to timeouts and hangs, to be fixed in chain index integration"] #[tokio::test(flavor = "multi_thread")] async fn zallet_like_steady_state_loop_zebrad() { - zallet_like_steady_state_loop::(&ValidatorKind::Zebrad).await + zallet_like_steady_state_loop::(&ValidatorKind::Zebrad).await } #[cfg(feature = "zcashd_support")] #[ignore = "prone to timeouts and hangs, to be fixed in chain index integration"] #[tokio::test(flavor = "multi_thread")] async fn zallet_like_steady_state_loop_zcashd() { - zallet_like_steady_state_loop::(&ValidatorKind::Zcashd).await + zallet_like_steady_state_loop::(&ValidatorKind::Zcashd).await } - async fn zallet_like_steady_state_loop(validator: &ValidatorKind) + async fn zallet_like_steady_state_loop(validator: &ValidatorKind) where C: ValidatorExt, - Service: zaino_testutils::TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: zaino_testutils::PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { use futures::{StreamExt as _, TryStreamExt as _}; use tokio::time::{timeout, Duration}; let (test_manager, _json_service, _option_state_service, _chain_index, indexer) = - create_test_manager_and_chain_index::(validator, None, false, false, false) + create_test_manager_and_chain_index::(validator, None, false, false, false) .await; test_manager diff --git a/live-tests/clientless/tests/fetch_service.rs b/live-tests/clientless/tests/fetch_service.rs index 53ea44785..f4ac94f37 100644 --- a/live-tests/clientless/tests/fetch_service.rs +++ b/live-tests/clientless/tests/fetch_service.rs @@ -5,11 +5,10 @@ use futures::StreamExt as _; use zaino_fetch::jsonrpsee::connector::JsonRpSeeConnector; use zaino_proto::proto::compact_formats::CompactBlock; use zaino_proto::proto::service::{BlockId, BlockRange, GetSubtreeRootsArg}; -#[allow(deprecated)] use zaino_state::{ - FetchService, FetchServiceSubscriber, LightWalletIndexer, Status, StatusType, ZcashIndexer, + LightWalletIndexer, NodeBackedIndexerServiceSubscriber, Status, StatusType, ZcashIndexer, }; -use zaino_testutils::{TestManager, ValidatorExt, ValidatorKind}; +use zaino_testutils::{Rpc, TestManager, ValidatorExt, ValidatorKind}; use zebra_chain::parameters::subsidy::ParameterSubsidy as _; use zebra_rpc::client::ValidateAddressResponse; use zebra_rpc::methods::{GetBlock, GetBlockHash}; @@ -95,7 +94,7 @@ async fn fetch_service_get_latest_block(validator: &ValidatorKi #[allow(deprecated)] async fn assert_subscriber_matches_rpc( validator: &ValidatorKind, - fetch_query: impl FnOnce(FetchServiceSubscriber) -> FFut, + fetch_query: impl FnOnce(NodeBackedIndexerServiceSubscriber) -> FFut, rpc_query: impl FnOnce(JsonRpSeeConnector) -> RFut, ) where V: ValidatorExt, @@ -321,7 +320,7 @@ async fn fetch_service_get_block_header(validator: &ValidatorKi #[allow(deprecated)] async fn launch_and_mine_five_blocks( validator: &ValidatorKind, -) -> (TestManager, FetchServiceSubscriber) { +) -> (TestManager, NodeBackedIndexerServiceSubscriber) { let (test_manager, fetch_service_subscriber) = zaino_testutils::launch_with_fetch_subscriber::(validator, None).await; diff --git a/live-tests/clientless/tests/json_server.rs b/live-tests/clientless/tests/json_server.rs index 199e0709f..72df376c5 100644 --- a/live-tests/clientless/tests/json_server.rs +++ b/live-tests/clientless/tests/json_server.rs @@ -5,9 +5,8 @@ //! docs/adr/0001-zcashd-support-feature-gate.md. #![cfg(feature = "zcashd_support")] -#[allow(deprecated)] -use zaino_state::{FetchService, FetchServiceSubscriber, ZcashIndexer}; -use zaino_testutils::TestManager; +use zaino_state::{NodeBackedIndexerServiceSubscriber, ZcashIndexer}; +use zaino_testutils::{Rpc, TestManager}; use zcash_local_net::validator::zcashd::Zcashd; /// Assert that `query` returns the same value from the zcashd-backed and @@ -16,11 +15,11 @@ use zcash_local_net::validator::zcashd::Zcashd; /// problem. The pure-compare building block of the json_server tests. #[allow(deprecated)] async fn assert_subscribers_agree( - zcashd_subscriber: &FetchServiceSubscriber, - zaino_subscriber: &FetchServiceSubscriber, + zcashd_subscriber: &NodeBackedIndexerServiceSubscriber, + zaino_subscriber: &NodeBackedIndexerServiceSubscriber, query: Q, ) where - Q: Fn(FetchServiceSubscriber) -> Fut, + Q: Fn(NodeBackedIndexerServiceSubscriber) -> Fut, Fut: std::future::Future, T: std::fmt::Debug + PartialEq, { @@ -34,13 +33,13 @@ async fn assert_subscribers_agree( /// tests that check an invariant holds across the chain. #[allow(deprecated)] async fn compare_over_blocks( - test_manager: &TestManager, - zcashd_subscriber: &FetchServiceSubscriber, - zaino_subscriber: &FetchServiceSubscriber, + test_manager: &TestManager, + zcashd_subscriber: &NodeBackedIndexerServiceSubscriber, + zaino_subscriber: &NodeBackedIndexerServiceSubscriber, blocks: u32, query: Q, ) where - Q: Fn(FetchServiceSubscriber) -> Fut, + Q: Fn(NodeBackedIndexerServiceSubscriber) -> Fut, Fut: std::future::Future, T: std::fmt::Debug + PartialEq, { diff --git a/live-tests/clientless/tests/state_service.rs b/live-tests/clientless/tests/state_service.rs index 5655dced3..9a4190a12 100644 --- a/live-tests/clientless/tests/state_service.rs +++ b/live-tests/clientless/tests/state_service.rs @@ -1,9 +1,6 @@ use zaino_fetch::jsonrpsee::response::address_deltas::GetAddressDeltasParams; -#[allow(deprecated)] -use zaino_state::{ - FetchServiceSubscriber, LightWalletIndexer, StateServiceSubscriber, ZcashIndexer, -}; +use zaino_state::{LightWalletIndexer, NodeBackedIndexerServiceSubscriber, ZcashIndexer}; use zaino_testutils::{StateAndFetchServices, ValidatorExt}; use zaino_testutils::{ValidatorKind, ZEBRAD_TESTNET_CACHE_DIR}; use zcash_local_net::validator::zebrad::Zebrad; @@ -42,8 +39,8 @@ async fn launch_testnet_cached() -> StateAndFetchServices { #[allow(deprecated)] async fn assert_subscribers_agree( services: &StateAndFetchServices, - fetch_query: impl FnOnce(FetchServiceSubscriber) -> FFut, - state_query: impl FnOnce(StateServiceSubscriber) -> SFut, + fetch_query: impl FnOnce(NodeBackedIndexerServiceSubscriber) -> FFut, + state_query: impl FnOnce(NodeBackedIndexerServiceSubscriber) -> SFut, ) -> T where T: std::fmt::Debug + PartialEq, diff --git a/live-tests/e2e/tests/devtool.rs b/live-tests/e2e/tests/devtool.rs index 6252148ab..0fa75c8e0 100644 --- a/live-tests/e2e/tests/devtool.rs +++ b/live-tests/e2e/tests/devtool.rs @@ -44,9 +44,8 @@ use e2e::devtool::DevtoolClients; use zaino_proto::proto::service::{ AddressList, BlockId, BlockRange, GetAddressUtxosArg, TransparentAddressBlockFilter, TxFilter, }; -use zaino_state::{LightWalletIndexer, ZcashIndexer, ZcashService}; -use zaino_testutils::{PollableTip, TestManager, TestService, ValidatorKind}; -use zainodlib::error::IndexerError; +use zaino_state::{LightWalletIndexer, ZcashIndexer}; +use zaino_testutils::{PollableTip, Rpc, TestManager, ValidatorKind}; use zcash_local_net::validator::zebrad::Zebrad; use zebra_chain::subtree::NoteCommitmentSubtreeIndex; use zebra_rpc::methods::{GetAddressBalanceRequest, GetAddressTxIdsRequest}; @@ -59,15 +58,13 @@ use zebra_rpc::methods::{GetAddressBalanceRequest, GetAddressTxIdsRequest}; /// spendable orchard notes the faucet ends with — one per send a test makes /// before mining (devtool will not chain unconfirmed change). The shared /// launch+fund preamble of the ports below. -async fn launch_and_fund_faucet( +async fn launch_and_fund_faucet( coinbase_blocks: u32, -) -> (TestManager, DevtoolClients) +) -> (TestManager, DevtoolClients) where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { - let test_manager = TestManager::::launch_mining_to( + let test_manager = TestManager::::launch_mining_to( zaino_testutils::SHIELDED_FUNDING_POOL, &ValidatorKind::Zebrad, None, @@ -99,13 +96,11 @@ where /// Launch an orchard-mining zebrad + Zaino on the `Service` backend and build /// devtool faucet/recipient wallets against it, without mining or syncing — the /// minimal preamble for tests that only exercise wallet↔server connectivity. -async fn launch_and_build_clients() -> (TestManager, DevtoolClients) +async fn launch_and_build_clients() -> (TestManager, DevtoolClients) where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { - let test_manager = TestManager::::launch_mining_to( + let test_manager = TestManager::::launch_mining_to( zaino_testutils::SHIELDED_FUNDING_POOL, &ValidatorKind::Zebrad, None, @@ -133,13 +128,11 @@ where /// and recipient wallets can report node/server info without erroring. Smoke /// test — the original discards the result. (`chain_name` is "test" for regtest /// and `server_uri` has no trailing slash, so this asserts neither.) -async fn connect_to_node_get_info() +async fn connect_to_node_get_info() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { - let (mut test_manager, clients) = launch_and_build_clients::().await; + let (mut test_manager, clients) = launch_and_build_clients::().await; clients.get_info_faucet().await; clients.get_info_recipient().await; @@ -149,13 +142,11 @@ where /// Port of `wallet_to_validator::check_received_mining_reward` (zebrad): the /// faucet's synced wallet sees the orchard coinbase note. -async fn receives_mining_reward() +async fn receives_mining_reward() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { - let (mut test_manager, clients) = launch_and_fund_faucet::(1).await; + let (mut test_manager, clients) = launch_and_fund_faucet::(1).await; let faucet_balance = dbg!(clients.faucet_balance().await); assert!( @@ -175,13 +166,11 @@ where /// `send_to_transparent` additionally mines across the finalization boundary /// under transparent mining; that variant is deferred, as it exercises the /// transparent-coinbase detection path devtool has not yet been run against.) -async fn send_to_pool(pool: e2e::Pool) +async fn send_to_pool(pool: e2e::Pool) where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { - let (mut test_manager, mut clients) = launch_and_fund_faucet::(1).await; + let (mut test_manager, mut clients) = launch_and_fund_faucet::(1).await; let recipient = clients.get_recipient_address(pool.address_kind()).await; let txid = clients.send_from_faucet(&recipient, 250_000).await; @@ -210,14 +199,12 @@ where /// balance assertions — the sends confirm in one block, and received funds are /// regular (non-coinbase) outputs with no maturity rule — and 100 orchard /// blocks would cost 100 halo2 proofs against the net-speedup criterion. -async fn send_to_all() +async fn send_to_all() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { // Three orchard notes — one per send (devtool will not chain unconfirmed change). - let (mut test_manager, mut clients) = launch_and_fund_faucet::(3).await; + let (mut test_manager, mut clients) = launch_and_fund_faucet::(3).await; let recipient_ua = clients.get_recipient_address("unified").await; let recipient_zaddr = clients.get_recipient_address("sapling").await; @@ -245,13 +232,11 @@ where /// orchard balance net of the ZIP-317 fee (250_000 − 15_000 = 235_000 — the /// first devtool exercise of `shield`, and the constant flagged as needing /// re-verification against devtool's note selection). -async fn shield_for_validator() +async fn shield_for_validator() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { - let (mut test_manager, mut clients) = launch_and_fund_faucet::(1).await; + let (mut test_manager, mut clients) = launch_and_fund_faucet::(1).await; let recipient_taddr = clients.get_recipient_address("transparent").await; clients.send_from_faucet(&recipient_taddr, 250_000).await; @@ -283,14 +268,12 @@ where /// mining) one transparent and one unified send into the mempool. Returns the /// manager and the two broadcast txids (transparent, then unified). The /// devtool analogue of `fund_and_fill_mempool`. -async fn fund_and_fill_mempool() -> (TestManager, String, String) +async fn fund_and_fill_mempool() -> (TestManager, String, String) where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { // Two orchard notes — one per unmined send. - let (test_manager, mut clients) = launch_and_fund_faucet::(2).await; + let (test_manager, mut clients) = launch_and_fund_faucet::(2).await; let recipient_taddr = clients.get_recipient_address("transparent").await; let recipient_ua = clients.get_recipient_address("unified").await; @@ -308,15 +291,13 @@ where /// txid hex. The txid is in display (reversed) order — devtool prints it that /// way, which matches the txid strings zaino's address queries return, so no /// conversion is needed for those comparisons. -async fn fund_and_send_to( +async fn fund_and_send_to( pool: e2e::Pool, -) -> (TestManager, DevtoolClients, String) +) -> (TestManager, DevtoolClients, String) where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { - let (test_manager, mut clients) = launch_and_fund_faucet::(1).await; + let (test_manager, mut clients) = launch_and_fund_faucet::(1).await; let recipient = clients.get_recipient_address(pool.address_kind()).await; let txid_hex = clients.send_from_faucet(&recipient, 250_000).await; @@ -331,14 +312,12 @@ where /// Port of `fetch_service_get_address_tx_ids` (zebrad): after a transparent /// send to the recipient, `get_address_tx_ids` over the recipient's taddr /// returns the send's txid. -async fn get_address_tx_ids() +async fn get_address_tx_ids() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip + LightWalletIndexer, + Conn: zaino_testutils::ValidatorConnectionMarker, { let (mut test_manager, clients, txid_hex) = - fund_and_send_to::(e2e::Pool::Transparent).await; + fund_and_send_to::(e2e::Pool::Transparent).await; let recipient_taddr = clients.get_recipient_address("transparent").await; // A range start a couple of blocks below the tip, covering the send block. @@ -362,14 +341,12 @@ where /// Port of `fetch_service_get_address_utxos` (zebrad): after a transparent /// send to the recipient, `z_get_address_utxos` over the recipient's taddr /// returns a utxo whose txid is the send's. -async fn get_address_utxos() +async fn get_address_utxos() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip + LightWalletIndexer, + Conn: zaino_testutils::ValidatorConnectionMarker, { let (mut test_manager, clients, txid_hex) = - fund_and_send_to::(e2e::Pool::Transparent).await; + fund_and_send_to::(e2e::Pool::Transparent).await; let recipient_taddr = clients.get_recipient_address("transparent").await; let utxos = test_manager @@ -387,14 +364,12 @@ where /// Port of `fetch_service_z_get_treestate` (zebrad, smoke): `z_get_treestate` /// at the tip height succeeds. -async fn z_get_treestate() +async fn z_get_treestate() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip + LightWalletIndexer, + Conn: zaino_testutils::ValidatorConnectionMarker, { let (mut test_manager, _clients, _txid_hex) = - fund_and_send_to::(e2e::Pool::Orchard).await; + fund_and_send_to::(e2e::Pool::Orchard).await; let tip = test_manager.subscriber().tip_height().await; dbg!(test_manager @@ -408,14 +383,12 @@ where /// Port of `fetch_service_z_get_subtrees_by_index` (zebrad, smoke): /// `z_get_subtrees_by_index` for orchard succeeds. -async fn z_get_subtrees_by_index() +async fn z_get_subtrees_by_index() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip + LightWalletIndexer, + Conn: zaino_testutils::ValidatorConnectionMarker, { let (mut test_manager, _clients, _txid_hex) = - fund_and_send_to::(e2e::Pool::Orchard).await; + fund_and_send_to::(e2e::Pool::Orchard).await; dbg!(test_manager .subscriber() @@ -428,14 +401,11 @@ where /// Port of `fetch_service_get_raw_transaction` (zebrad, smoke): /// `get_raw_transaction` for the orchard send's txid succeeds. -async fn get_raw_transaction() +async fn get_raw_transaction() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip + LightWalletIndexer, + Conn: zaino_testutils::ValidatorConnectionMarker, { - let (mut test_manager, _clients, txid_hex) = - fund_and_send_to::(e2e::Pool::Orchard).await; + let (mut test_manager, _clients, txid_hex) = fund_and_send_to::(e2e::Pool::Orchard).await; dbg!(test_manager .subscriber() @@ -449,16 +419,14 @@ where /// Port of `fetch_service_get_taddress_txids` (zebrad, smoke): /// `get_taddress_txids` over the recipient's taddr and a height range around /// the send succeeds. -async fn get_taddress_txids() +async fn get_taddress_txids() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip + LightWalletIndexer, + Conn: zaino_testutils::ValidatorConnectionMarker, { use futures::StreamExt as _; let (mut test_manager, clients, _txid_hex) = - fund_and_send_to::(e2e::Pool::Transparent).await; + fund_and_send_to::(e2e::Pool::Transparent).await; let recipient_taddr = clients.get_recipient_address("transparent").await; let tip = test_manager.subscriber().tip_height().await; @@ -492,14 +460,12 @@ where /// Port of `fetch_service_get_taddress_utxos` (zebrad, smoke): /// `get_address_utxos` over the recipient's taddr succeeds. -async fn get_taddress_utxos() +async fn get_taddress_utxos() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip + LightWalletIndexer, + Conn: zaino_testutils::ValidatorConnectionMarker, { let (mut test_manager, clients, _txid_hex) = - fund_and_send_to::(e2e::Pool::Transparent).await; + fund_and_send_to::(e2e::Pool::Transparent).await; let recipient_taddr = clients.get_recipient_address("transparent").await; let utxos_arg = GetAddressUtxosArg { @@ -518,16 +484,14 @@ where /// Port of `fetch_service_get_taddress_utxos_stream` (zebrad, smoke): /// `get_address_utxos_stream` over the recipient's taddr succeeds. -async fn get_taddress_utxos_stream() +async fn get_taddress_utxos_stream() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip + LightWalletIndexer, + Conn: zaino_testutils::ValidatorConnectionMarker, { use futures::StreamExt as _; let (mut test_manager, clients, _txid_hex) = - fund_and_send_to::(e2e::Pool::Transparent).await; + fund_and_send_to::(e2e::Pool::Transparent).await; let recipient_taddr = clients.get_recipient_address("transparent").await; let utxos_arg = GetAddressUtxosArg { @@ -551,14 +515,12 @@ where /// Port of `fetch_service_get_raw_mempool` (zebrad): with two transactions /// broadcast but unmined, the indexer's `get_raw_mempool` matches the /// validator's own JSON-RPC `getrawmempool`. -async fn get_raw_mempool() +async fn get_raw_mempool() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip + LightWalletIndexer, + Conn: zaino_testutils::ValidatorConnectionMarker, { let (mut test_manager, _transparent_txid, _unified_txid) = - fund_and_fill_mempool::().await; + fund_and_fill_mempool::().await; let json_service = test_manager.full_node_jsonrpc_connector().await; @@ -577,17 +539,14 @@ where /// Port of `fetch_service_get_mempool_tx` (zebrad): the `get_mempool_tx` /// stream returns the two unmined transactions keyed by txid (internal byte /// order), and the exclude-by-txid-suffix filter drops the named one. -async fn get_mempool_tx() +async fn get_mempool_tx() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip + LightWalletIndexer, + Conn: zaino_testutils::ValidatorConnectionMarker, { use futures::StreamExt as _; use zaino_proto::proto::service::GetMempoolTxRequest; - let (mut test_manager, transparent_txid, unified_txid) = - fund_and_fill_mempool::().await; + let (mut test_manager, transparent_txid, unified_txid) = fund_and_fill_mempool::().await; let to_bytes = |hex: &str| -> [u8; 32] { e2e::devtool::txid_internal_bytes(hex) @@ -638,16 +597,14 @@ where /// Port of `fetch_service_get_mempool_stream` (zebrad): a `get_mempool_stream` /// subscription opened before two sends observes those unmined transactions. /// Smoke test — the original only `dbg!`s the collected stream. -async fn get_mempool_stream() +async fn get_mempool_stream() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip + LightWalletIndexer, + Conn: zaino_testutils::ValidatorConnectionMarker, { use futures::StreamExt as _; // Two orchard notes — one per unmined send. - let (mut test_manager, mut clients) = launch_and_fund_faucet::(2).await; + let (mut test_manager, mut clients) = launch_and_fund_faucet::(2).await; // Subscribe before the sends so the stream observes them entering the mempool. let subscriber = test_manager.subscriber().clone(); @@ -687,13 +644,11 @@ where /// order. The devtool analogue of `fund_and_send(Pool::Orchard)`; the wallet /// clients are dropped once the transaction is mined (the queries below hit /// zaino, not the wallet). -async fn fund_and_send_orchard() -> (TestManager, Vec) +async fn fund_and_send_orchard() -> (TestManager, Vec) where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { - let (test_manager, mut clients) = launch_and_fund_faucet::(1).await; + let (test_manager, mut clients) = launch_and_fund_faucet::(1).await; let recipient_ua = clients.get_recipient_address("unified").await; let txid_hex = clients.send_from_faucet(&recipient_ua, 250_000).await; @@ -708,13 +663,11 @@ where /// Port of `fetch_service_get_transaction_mined` (zebrad): the indexer serves /// `get_transaction` for the mined orchard send, keyed by its txid. Also /// confirms `txid_internal_bytes` yields the order the indexer matches on. -async fn get_transaction_mined() +async fn get_transaction_mined() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip + LightWalletIndexer, + Conn: zaino_testutils::ValidatorConnectionMarker, { - let (mut test_manager, txid_bytes) = fund_and_send_orchard::().await; + let (mut test_manager, txid_bytes) = fund_and_send_orchard::().await; let tx_filter = TxFilter { block: None, @@ -734,13 +687,11 @@ where /// Port of `fetch_service_get_transaction_mempool` (zebrad): the indexer /// serves `get_transaction` for an orchard send that is broadcast but not /// mined, keyed by its txid — i.e. from the mempool. -async fn get_transaction_mempool() +async fn get_transaction_mempool() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip + LightWalletIndexer, + Conn: zaino_testutils::ValidatorConnectionMarker, { - let (mut test_manager, mut clients) = launch_and_fund_faucet::(1).await; + let (mut test_manager, mut clients) = launch_and_fund_faucet::(1).await; let recipient_ua = clients.get_recipient_address("unified").await; let txid_hex = clients.send_from_faucet(&recipient_ua, 250_000).await; @@ -944,14 +895,12 @@ async fn block_range_returns_all_pools() { /// Port of `fetch_service_get_address_balance` (zebrad): after a transparent /// send of 250_000 to the recipient, `z_get_address_balance` over that taddr /// reports exactly 250_000. -async fn get_address_balance() +async fn get_address_balance() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip + LightWalletIndexer, + Conn: zaino_testutils::ValidatorConnectionMarker, { let (mut test_manager, clients, _txid_hex) = - fund_and_send_to::(e2e::Pool::Transparent).await; + fund_and_send_to::(e2e::Pool::Transparent).await; let recipient_taddr = clients.get_recipient_address("transparent").await; let balance = test_manager @@ -970,14 +919,12 @@ where /// Port of `fetch_service_get_taddress_balance` (zebrad): after a transparent /// send of 250_000 to the recipient, `get_taddress_balance` over that taddr /// reports value_zat 250_000. -async fn get_taddress_balance() +async fn get_taddress_balance() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip + LightWalletIndexer, + Conn: zaino_testutils::ValidatorConnectionMarker, { let (mut test_manager, clients, _txid_hex) = - fund_and_send_to::(e2e::Pool::Transparent).await; + fund_and_send_to::(e2e::Pool::Transparent).await; let recipient_taddr = clients.get_recipient_address("transparent").await; let address_list = AddressList { @@ -1590,13 +1537,11 @@ async fn get_block_range_out_of_range_lower_bound() { /// per-call override, so the advance can't be cheap transparent filler until /// per-call filler mining (round-3 P2) lands; un-ignore and mine the advance to /// a transparent throwaway then. -async fn send_to_transparent_finalization() +async fn send_to_transparent_finalization() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { - let (mut test_manager, mut clients) = launch_and_fund_faucet::(1).await; + let (mut test_manager, mut clients) = launch_and_fund_faucet::(1).await; let recipient_taddr = clients.get_recipient_address("transparent").await; clients.send_from_faucet(&recipient_taddr, 250_000).await; @@ -2148,13 +2093,11 @@ async fn get_outpoint_spenders_fetch_vs_state() { /// sync is block-based so it never scans the mempool). Restore the assertions /// (using `Pool::spendable_balance` for the confirmed ones) and un-ignore when /// devtool surfaces unconfirmed balances. -async fn monitor_unverified_mempool() +async fn monitor_unverified_mempool() where - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip, + Conn: zaino_testutils::ValidatorConnectionMarker, { - let (mut test_manager, mut clients) = launch_and_fund_faucet::(2).await; + let (mut test_manager, mut clients) = launch_and_fund_faucet::(2).await; let recipient_ua = clients.get_recipient_address("unified").await; let txid_1 = clients.send_from_faucet(&recipient_ua, 250_000).await; @@ -2221,13 +2164,11 @@ where /// Port of `test_get_mempool_info` (fetch_service, zebrad): `get_mempool_info` /// matches values recomputed from the fetch subscriber's mempool internals. /// FetchService-only — the recompute reads `FetchServiceSubscriber.indexer`. -#[allow(deprecated)] // FetchService is a deprecated re-export. async fn get_mempool_info_fetch() { use hex::ToHex as _; use zaino_state::ChainIndex as _; - let (mut test_manager, _transparent_txid, _unified_txid) = - fund_and_fill_mempool::().await; + let (mut test_manager, _transparent_txid, _unified_txid) = fund_and_fill_mempool::().await; let subscriber = test_manager.subscriber(); let info = subscriber.get_mempool_info().await.unwrap(); @@ -2284,45 +2225,42 @@ async fn get_mempool_info_state() { } mod zebrad { - // FetchService is a deprecated re-export; the deprecation fires at the - // turbofish use sites below, so the allow covers the whole module. - #[allow(deprecated)] mod fetch_service { - use zaino_state::FetchService; + use zaino_testutils::Rpc; #[tokio::test(flavor = "multi_thread")] async fn receives_mining_reward() { - crate::receives_mining_reward::().await; + crate::receives_mining_reward::().await; } #[tokio::test(flavor = "multi_thread")] async fn connect_to_node_get_info() { - crate::connect_to_node_get_info::().await; + crate::connect_to_node_get_info::().await; } #[tokio::test(flavor = "multi_thread")] async fn send_to_orchard() { - crate::send_to_pool::(e2e::Pool::Orchard).await; + crate::send_to_pool::(e2e::Pool::Orchard).await; } #[tokio::test(flavor = "multi_thread")] async fn send_to_sapling() { - crate::send_to_pool::(e2e::Pool::Sapling).await; + crate::send_to_pool::(e2e::Pool::Sapling).await; } #[tokio::test(flavor = "multi_thread")] async fn send_to_transparent() { - crate::send_to_pool::(e2e::Pool::Transparent).await; + crate::send_to_pool::(e2e::Pool::Transparent).await; } #[tokio::test(flavor = "multi_thread")] async fn send_to_all() { - crate::send_to_all::().await; + crate::send_to_all::().await; } #[tokio::test(flavor = "multi_thread")] async fn shield_for_validator() { - crate::shield_for_validator::().await; + crate::shield_for_validator::().await; } #[tokio::test(flavor = "multi_thread")] @@ -2331,27 +2269,27 @@ mod zebrad { ignore = "heavy: 99-block orchard advance (~99 halo2 proofs); un-ignore + transparent filler when round-3 P2 lands" )] async fn send_to_transparent_finalization() { - crate::send_to_transparent_finalization::().await; + crate::send_to_transparent_finalization::().await; } #[tokio::test(flavor = "multi_thread")] async fn get_transaction_mined() { - crate::get_transaction_mined::().await; + crate::get_transaction_mined::().await; } #[tokio::test(flavor = "multi_thread")] async fn get_raw_mempool() { - crate::get_raw_mempool::().await; + crate::get_raw_mempool::().await; } #[tokio::test(flavor = "multi_thread")] async fn get_mempool_tx() { - crate::get_mempool_tx::().await; + crate::get_mempool_tx::().await; } #[tokio::test(flavor = "multi_thread")] async fn get_mempool_stream() { - crate::get_mempool_stream::().await; + crate::get_mempool_stream::().await; } #[tokio::test(flavor = "multi_thread")] @@ -2365,62 +2303,62 @@ mod zebrad { ignore = "devtool WalletBalance has no unconfirmed_*/confirmed_* fields; balance asserts are commented out — restore + un-ignore when devtool surfaces unconfirmed balances" )] async fn monitor_unverified_mempool() { - crate::monitor_unverified_mempool::().await; + crate::monitor_unverified_mempool::().await; } #[tokio::test(flavor = "multi_thread")] async fn get_address_tx_ids() { - crate::get_address_tx_ids::().await; + crate::get_address_tx_ids::().await; } #[tokio::test(flavor = "multi_thread")] async fn get_address_utxos() { - crate::get_address_utxos::().await; + crate::get_address_utxos::().await; } #[tokio::test(flavor = "multi_thread")] async fn z_get_treestate() { - crate::z_get_treestate::().await; + crate::z_get_treestate::().await; } #[tokio::test(flavor = "multi_thread")] async fn z_get_subtrees_by_index() { - crate::z_get_subtrees_by_index::().await; + crate::z_get_subtrees_by_index::().await; } #[tokio::test(flavor = "multi_thread")] async fn get_raw_transaction() { - crate::get_raw_transaction::().await; + crate::get_raw_transaction::().await; } #[tokio::test(flavor = "multi_thread")] async fn get_taddress_txids() { - crate::get_taddress_txids::().await; + crate::get_taddress_txids::().await; } #[tokio::test(flavor = "multi_thread")] async fn get_taddress_utxos() { - crate::get_taddress_utxos::().await; + crate::get_taddress_utxos::().await; } #[tokio::test(flavor = "multi_thread")] async fn get_taddress_utxos_stream() { - crate::get_taddress_utxos_stream::().await; + crate::get_taddress_utxos_stream::().await; } #[tokio::test(flavor = "multi_thread")] async fn get_transaction_mempool() { - crate::get_transaction_mempool::().await; + crate::get_transaction_mempool::().await; } #[tokio::test(flavor = "multi_thread")] async fn get_address_balance() { - crate::get_address_balance::().await; + crate::get_address_balance::().await; } #[tokio::test(flavor = "multi_thread")] async fn get_taddress_balance() { - crate::get_taddress_balance::().await; + crate::get_taddress_balance::().await; } } @@ -2540,42 +2478,41 @@ mod zebrad { } mod state_service { - #[allow(deprecated)] - use zaino_state::StateService; + use zaino_testutils::Direct; #[tokio::test(flavor = "multi_thread")] async fn receives_mining_reward() { - crate::receives_mining_reward::().await; + crate::receives_mining_reward::().await; } #[tokio::test(flavor = "multi_thread")] async fn connect_to_node_get_info() { - crate::connect_to_node_get_info::().await; + crate::connect_to_node_get_info::().await; } #[tokio::test(flavor = "multi_thread")] async fn send_to_orchard() { - crate::send_to_pool::(e2e::Pool::Orchard).await; + crate::send_to_pool::(e2e::Pool::Orchard).await; } #[tokio::test(flavor = "multi_thread")] async fn send_to_sapling() { - crate::send_to_pool::(e2e::Pool::Sapling).await; + crate::send_to_pool::(e2e::Pool::Sapling).await; } #[tokio::test(flavor = "multi_thread")] async fn send_to_transparent() { - crate::send_to_pool::(e2e::Pool::Transparent).await; + crate::send_to_pool::(e2e::Pool::Transparent).await; } #[tokio::test(flavor = "multi_thread")] async fn send_to_all() { - crate::send_to_all::().await; + crate::send_to_all::().await; } #[tokio::test(flavor = "multi_thread")] async fn shield_for_validator() { - crate::shield_for_validator::().await; + crate::shield_for_validator::().await; } #[tokio::test(flavor = "multi_thread")] @@ -2584,17 +2521,17 @@ mod zebrad { ignore = "heavy: 99-block orchard advance (~99 halo2 proofs); un-ignore + transparent filler when round-3 P2 lands" )] async fn send_to_transparent_finalization() { - crate::send_to_transparent_finalization::().await; + crate::send_to_transparent_finalization::().await; } #[tokio::test(flavor = "multi_thread")] async fn get_transaction_mined() { - crate::get_transaction_mined::().await; + crate::get_transaction_mined::().await; } #[tokio::test(flavor = "multi_thread")] async fn get_raw_mempool() { - crate::get_raw_mempool::().await; + crate::get_raw_mempool::().await; } #[tokio::test(flavor = "multi_thread")] @@ -2608,7 +2545,7 @@ mod zebrad { ignore = "devtool WalletBalance has no unconfirmed_*/confirmed_* fields; balance asserts are commented out — restore + un-ignore when devtool surfaces unconfirmed balances" )] async fn monitor_unverified_mempool() { - crate::monitor_unverified_mempool::().await; + crate::monitor_unverified_mempool::().await; } // No get_mempool_tx here: the state backend returns mempool-tx txids diff --git a/live-tests/e2e/tests/devtool_zcashd.rs b/live-tests/e2e/tests/devtool_zcashd.rs index bcc002bd1..cb2edbb6e 100644 --- a/live-tests/e2e/tests/devtool_zcashd.rs +++ b/live-tests/e2e/tests/devtool_zcashd.rs @@ -22,11 +22,9 @@ // launchers, all gated behind `zcashd_support`. Gate the whole binary so it // compiles out under `--no-default-features` (mirrors the clientless partition's json_server.rs). #![cfg(feature = "zcashd_support")] -#![allow(deprecated)] // FetchService is a deprecated re-export. - use e2e::devtool::DevtoolClients; -use zaino_state::{ChainIndex, FetchService, ZcashIndexer}; -use zaino_testutils::{TestManager, ValidatorKind, ZcashdDualFetchServices}; +use zaino_state::{ChainIndex, ZcashIndexer}; +use zaino_testutils::{Rpc, TestManager, ValidatorKind, ZcashdDualFetchServices}; use zcash_local_net::validator::zcashd::Zcashd; use zebra_chain::subtree::NoteCommitmentSubtreeIndex; use zebra_rpc::client::GetAddressBalanceRequest; @@ -38,8 +36,8 @@ use zebra_rpc::methods::GetAddressTxIdsRequest; /// faucet/recipient wallets against the resulting Zaino, without mining or /// syncing. The zcashd analogue of devtool.rs's `launch_and_build_clients`, /// concrete on zcashd (which has no StateService backend). -async fn launch_zcashd_and_build_clients() -> (TestManager, DevtoolClients) { - let test_manager = TestManager::::launch_mining_to( +async fn launch_zcashd_and_build_clients() -> (TestManager, DevtoolClients) { + let test_manager = TestManager::::launch_mining_to( zaino_testutils::SHIELDED_FUNDING_POOL, // ORCHARD &ValidatorKind::Zcashd, None, // network -> Regtest @@ -70,7 +68,7 @@ async fn launch_zcashd_and_build_clients() -> (TestManager /// height 2). The send/shield analogue of devtool.rs's `launch_and_fund_faucet`. async fn launch_and_fund_zcashd_faucet( orchard_notes: u32, -) -> (TestManager, DevtoolClients) { +) -> (TestManager, DevtoolClients) { let (test_manager, mut clients) = launch_zcashd_and_build_clients().await; test_manager .generate_blocks_and_wait_for_tip(orchard_notes + 1, test_manager.subscriber()) diff --git a/live-tests/e2e/tests/test_vectors.rs b/live-tests/e2e/tests/test_vectors.rs index 36128df39..11cfa336c 100644 --- a/live-tests/e2e/tests/test_vectors.rs +++ b/live-tests/e2e/tests/test_vectors.rs @@ -14,11 +14,9 @@ use zaino_state::read_u64_le; use zaino_state::write_u32_le; use zaino_state::write_u64_le; use zaino_state::CompactSize; -#[allow(deprecated)] -use zaino_state::StateService; use zaino_state::ZcashIndexer; use zaino_state::{ChainWork, IndexedBlock}; -use zaino_testutils::{TestManager, ValidatorKind}; +use zaino_testutils::{Direct, TestManager, ValidatorKind}; use zcash_local_net::validator::zebrad::Zebrad; use zebra_chain::serialization::{ZcashDeserialize, ZcashSerialize}; use zebra_rpc::methods::GetAddressUtxos; @@ -47,7 +45,7 @@ async fn create_200_block_regtest_chain_vectors() { // The committed unit-test vectors encode a mixed-pool chain built by // repeatedly shielding transparent coinbase — mining must stay transparent // so regenerated vectors keep that shape. - let mut test_manager = TestManager::::launch_mining_to( + let mut test_manager = TestManager::::launch_mining_to( zaino_testutils::PoolType::Transparent, &ValidatorKind::Zebrad, None, @@ -242,7 +240,7 @@ async fn create_200_block_regtest_chain_vectors() { .await .and_then(|response| match response { zebra_rpc::methods::GetBlock::Raw(_) => { - Err(zaino_state::StateServiceError::Custom( + Err(zaino_state::NodeBackedIndexerServiceError::Custom( "Found transaction of `Raw` type, expected only `Object` types." .to_string(), )) @@ -254,7 +252,7 @@ async fn create_200_block_regtest_chain_vectors() { match item { GetBlockTransaction::Hash(h) => Ok(h.0.to_vec()), GetBlockTransaction::Object(_) => Err( - zaino_state::StateServiceError::Custom( + zaino_state::NodeBackedIndexerServiceError::Custom( "Found transaction of `Object` type, expected only `Hash` types." .to_string(), ), @@ -273,7 +271,7 @@ async fn create_200_block_regtest_chain_vectors() { .await .and_then(|response| match response { zebra_rpc::methods::GetBlock::Object { .. } => { - Err(zaino_state::StateServiceError::Custom( + Err(zaino_state::NodeBackedIndexerServiceError::Custom( "Found transaction of `Object` type, expected only `Raw` types." .to_string(), )) diff --git a/live-tests/zaino-testutils/src/lib.rs b/live-tests/zaino-testutils/src/lib.rs index 6ebb6298f..44969008f 100644 --- a/live-tests/zaino-testutils/src/lib.rs +++ b/live-tests/zaino-testutils/src/lib.rs @@ -7,6 +7,7 @@ use futures::StreamExt as _; use once_cell::sync::Lazy; use std::{ future::Future, + marker::PhantomData, net::{IpAddr, Ipv4Addr, SocketAddr}, path::PathBuf, }; @@ -24,15 +25,12 @@ use zaino_fetch::jsonrpsee::connector::{test_node_and_return_url, JsonRpSeeConne use zaino_proto::proto::compact_formats::CompactBlock; use zaino_proto::proto::service::{BlockId, BlockRange}; use zaino_serve::server::config::{GrpcServerConfig, JsonRpcServerConfig}; -#[cfg(feature = "zcashd_support")] -use zaino_state::BackendType; use zaino_state::{ - BlockchainSource, ChainIndex, FetchServiceSubscriber, LightWalletIndexer, LightWalletService, - NodeBackedChainIndexSubscriber, StateServiceSubscriber, ZcashIndexer, ZcashService, + BlockchainSource, ChainIndex, LightWalletIndexer, NodeBackedChainIndexSubscriber, + NodeBackedIndexerService, NodeBackedIndexerServiceConfig, NodeBackedIndexerServiceSubscriber, + ZcashService, }; -#[allow(deprecated)] -use zaino_state::{FetchService, FetchServiceConfig, StateService, StateServiceConfig}; -use zainodlib::{config::ZainodConfig, error::IndexerError, indexer::Indexer}; +use zainodlib::{config::BackendType, error::IndexerError, indexer::Indexer}; pub use zcash_local_net as services; use zcash_local_net::error::LaunchError; #[cfg(feature = "zcashd_support")] @@ -221,20 +219,11 @@ pub trait PollableTip: Status + Sync { fn tip_height(&self) -> impl std::future::Future; } -impl PollableTip for FetchServiceSubscriber { +impl PollableTip for NodeBackedIndexerServiceSubscriber { async fn tip_height(&self) -> u64 { self.get_latest_block() .await - .expect("PollableTip: FetchServiceSubscriber::get_latest_block failed") - .height - } -} - -impl PollableTip for StateServiceSubscriber { - async fn tip_height(&self) -> u64 { - self.get_latest_block() - .await - .expect("PollableTip: StateServiceSubscriber::get_latest_block failed") + .expect("PollableTip: NodeBackedIndexerServiceSubscriber::get_latest_block failed") .height } } @@ -254,39 +243,32 @@ impl PollableTip for NodeBackedChainIndexSubscriber` -/// - `Service::Subscriber: PollableTip` -/// -/// **Not** bundled: the reverse bound -/// `IndexerError: From`. Rust does not -/// propagate non-`Self` bounds declared in a trait's `where`-clause -/// through a `T: TestService` constraint, so call sites that touch -/// `TestManager::launch` (or anything else exercising -/// `Indexer::launch_inner`'s `?` propagation) must still restate that -/// one bound explicitly. Everything else collapses to `TestService`. -pub trait TestService: - LightWalletService, Subscriber: PollableTip> - + Send - + Sync - + 'static -{ +/// Replaces the former `FetchService` / `StateService` `Service` type parameters: +/// there is now one service, parameterized by connection. Implementors are the +/// zero-sized markers [`Rpc`] and [`Direct`]. +pub trait ValidatorConnectionMarker: Send + Sync + 'static { + /// The on-disk backend selector this connection maps to. + const BACKEND: BackendType; } -impl TestService for T where - T: LightWalletService< - Config: TryFrom, - Subscriber: PollableTip, - > + Send - + Sync - + 'static -{ +/// JSON-RPC validator connection (formerly `FetchService`). +#[derive(Debug, Clone, Copy)] +pub struct Rpc; + +impl ValidatorConnectionMarker for Rpc { + const BACKEND: BackendType = BackendType::Rpc; +} + +/// Direct Zebra `ReadStateService` validator connection (formerly `StateService`). +#[derive(Debug, Clone, Copy)] +pub struct Direct; + +impl ValidatorConnectionMarker for Direct { + const BACKEND: BackendType = BackendType::Direct; } // temporary until activation heights are unified to zebra-chain type. @@ -399,7 +381,11 @@ pub enum ValidatorTestConfig { } /// Configuration data for Zaino Tests. -pub struct TestManager { +/// +/// Generic over the validator control plane `C` and the [`ValidatorConnectionMarker`] +/// `Conn` — the validator connection ([`Rpc`] or [`Direct`]) the single +/// [`NodeBackedIndexerService`] uses. +pub struct TestManager { /// Control plane for a validator pub local_net: C, /// Data directory for the validator. @@ -417,9 +403,11 @@ pub struct TestManager, /// Service subscriber. - pub service_subscriber: Option, + pub service_subscriber: Option, /// JsonRPC server cookie dir. pub json_server_cookie_dir: Option, + /// Selected validator connection marker. + _connection: PhantomData, } /// Needed validator functionality that is not implemented in infrastructure @@ -503,19 +491,17 @@ pub fn default_mining_pool(validator: &ValidatorKind) -> PoolType { } } -impl TestManager +impl TestManager where C: ValidatorExt, - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip, + Conn: ValidatorConnectionMarker, { /// Returns the service subscriber, panicking if zaino wasn't enabled at launch. /// /// Convenience for tests that always pass `enable_zaino: true` and want to /// hand the subscriber to [`Self::generate_blocks_and_wait_for_tip`] without /// repeating `.service_subscriber.as_ref().unwrap()` at every call site. - pub fn subscriber(&self) -> &Service::Subscriber { + pub fn subscriber(&self) -> &NodeBackedIndexerServiceSubscriber { self.service_subscriber .as_ref() .expect("TestManager::subscriber called but service_subscriber is None (zaino disabled at launch?)") @@ -599,9 +585,9 @@ where enable_clients: bool, ) -> Result { #[cfg(feature = "zcashd_support")] - if (validator == &ValidatorKind::Zcashd) && (Service::BACKEND_TYPE == BackendType::State) { + if (validator == &ValidatorKind::Zcashd) && (Conn::BACKEND == BackendType::Direct) { return Err(std::io::Error::other( - "Cannot use state backend with zcashd.", + "Cannot use the direct (ReadStateService) connection with zcashd.", )); } zaino_common::logging::try_init(); @@ -677,8 +663,7 @@ where "[TEST] Launching Zaino indexer" ); let indexer_config = zainodlib::config::ZainodConfig { - // TODO: Make configurable. - backend: Service::BACKEND_TYPE, + backend: Conn::BACKEND, json_server_settings: if enable_zaino_jsonrpc_server { Some(JsonRpcServerConfig { json_rpc_listen_address: zaino_json_listen_address, @@ -707,15 +692,16 @@ where metrics_endpoint: None, }; - let (handle, service_subscriber) = Indexer::::launch_inner_with_listeners( - Service::Config::try_from(indexer_config.clone()) - .expect("Failed to convert ZainodConfig to service config"), - indexer_config, - zaino_grpc_listener, - zaino_json_listener, - ) - .await - .unwrap(); + let (handle, service_subscriber) = + Indexer::::launch_inner_with_listeners( + NodeBackedIndexerServiceConfig::try_from(indexer_config.clone()) + .expect("Failed to convert ZainodConfig to service config"), + indexer_config, + zaino_grpc_listener, + zaino_json_listener, + ) + .await + .unwrap(); ( Some(handle), @@ -752,6 +738,7 @@ where zaino_grpc_listen_address, service_subscriber: zaino_service_subscriber, json_server_cookie_dir: zaino_json_server_cookie_dir, + _connection: PhantomData, }; test_manager.activate_nu5_nu6(enable_zaino).await; @@ -938,25 +925,27 @@ where } } -/// Handles from [`launch_state_and_fetch_services`]: a state-backend -/// [`TestManager`] plus standalone fetch and state services pointed at the -/// same validator. The owned services exist only to keep their subscribers -/// alive — tests interact through the manager and the subscribers. -#[allow(deprecated)] +/// Handles from [`launch_state_and_fetch_services`]: a direct-connection +/// [`TestManager`] plus standalone `Rpc`- and `Direct`-connection +/// [`NodeBackedIndexerService`]s pointed at the same validator. The owned services +/// exist only to keep their subscribers alive — tests interact through the manager and +/// the subscribers. +/// +/// The `fetch_*` fields carry the `Rpc` (JSON-RPC) connection; the `state_*` fields carry +/// the `Direct` (`ReadStateService`) connection. pub struct StateAndFetchServices { - /// The launched validator + Zaino test manager. - pub test_manager: TestManager, - /// Owned fetch service; keeps `fetch_subscriber` alive. - pub fetch_service: FetchService, - /// Subscriber to the standalone fetch service's chain index. - pub fetch_subscriber: FetchServiceSubscriber, - /// Owned state service; keeps `state_subscriber` alive. - pub state_service: StateService, - /// Subscriber to the standalone state service's chain index. - pub state_subscriber: StateServiceSubscriber, + /// The launched validator + Zaino test manager (direct connection). + pub test_manager: TestManager, + /// Owned `Rpc`-connection service; keeps `fetch_subscriber` alive. + pub fetch_service: NodeBackedIndexerService, + /// Subscriber to the standalone `Rpc`-connection service's chain index. + pub fetch_subscriber: NodeBackedIndexerServiceSubscriber, + /// Owned `Direct`-connection service; keeps `state_subscriber` alive. + pub state_service: NodeBackedIndexerService, + /// Subscriber to the standalone `Direct`-connection service's chain index. + pub state_subscriber: NodeBackedIndexerServiceSubscriber, } -#[allow(deprecated)] impl StateAndFetchServices { /// Mine `n` blocks and wait for both the fetch and state subscribers to /// observe the new tip. @@ -975,7 +964,6 @@ impl StateAndFetchServices { /// harness used by both the clientless and e2e live-test partitions. /// Wallet callers wrap this and additionally build lightclients from the /// returned manager's gRPC address. -#[allow(deprecated)] pub async fn launch_state_and_fetch_services( validator: &ValidatorKind, chain_cache: Option, @@ -996,7 +984,6 @@ pub async fn launch_state_and_fetch_services( /// caller instead of [`default_mining_pool`]: [`SHIELDED_FUNDING_POOL`] for /// sessions funding wallets from coinbase, or a pinned pool for tests whose /// subject is the miner's coinbase footprint. -#[allow(deprecated)] pub async fn launch_state_and_fetch_services_mining_to( mine_to_pool: PoolType, validator: &ValidatorKind, @@ -1004,7 +991,7 @@ pub async fn launch_state_and_fetch_services_mining_to( enable_zaino: bool, network: Option, ) -> StateAndFetchServices { - let test_manager = TestManager::::launch_mining_to( + let test_manager = TestManager::::launch_mining_to( mine_to_pool, validator, network, @@ -1067,41 +1054,42 @@ pub async fn launch_state_and_fetch_services_mining_to( None => test_manager.data_dir.clone(), }; - let state_service = StateService::spawn(StateServiceConfig::new( - zebra_state::Config { - cache_dir: state_chain_cache_dir, - ephemeral: false, - delete_old_database: true, - debug_stop_at_height: None, - debug_validity_check_interval: None, - should_backup_non_finalized_state: false, - debug_skip_non_finalized_state_backup_task: false, - }, - test_manager.full_node_rpc_listen_address.to_string(), - test_manager.full_node_grpc_listen_address, - false, - None, - None, - None, - ServiceConfig::default(), - StorageConfig { - database: DatabaseConfig { - path: test_manager - .local_net - .data_dir() - .path() - .to_path_buf() - .join("state-srvice-zaino"), + let state_service = + NodeBackedIndexerService::spawn(NodeBackedIndexerServiceConfig::new_direct( + zebra_state::Config { + cache_dir: state_chain_cache_dir, + ephemeral: false, + delete_old_database: true, + debug_stop_at_height: None, + debug_validity_check_interval: None, + should_backup_non_finalized_state: false, + debug_skip_non_finalized_state_backup_task: false, + }, + test_manager.full_node_rpc_listen_address.to_string(), + test_manager.full_node_grpc_listen_address, + false, + None, + None, + None, + ServiceConfig::default(), + StorageConfig { + database: DatabaseConfig { + path: test_manager + .local_net + .data_dir() + .path() + .to_path_buf() + .join("state-srvice-zaino"), + ..Default::default() + }, ..Default::default() }, - ..Default::default() - }, - false, // ephemeral_finalised_state: tests use a persistent finalised DB - network_type, - None, - )) - .await - .unwrap(); + false, // ephemeral_finalised_state: tests use a persistent finalised DB + network_type, + None, + )) + .await + .unwrap(); let state_subscriber = state_service.get_subscriber().inner(); @@ -1116,16 +1104,15 @@ pub async fn launch_state_and_fetch_services_mining_to( } } -/// Spawn a standalone [`FetchService`] pointed at `rpc_url`, storing its index -/// under `db_path`. Shared by the launch helpers below. -#[allow(deprecated)] +/// Spawn a standalone `Rpc`-connection [`NodeBackedIndexerService`] pointed at `rpc_url`, +/// storing its index under `db_path`. Shared by the launch helpers below. async fn spawn_fetch_service( rpc_url: String, cookie_dir: Option, db_path: PathBuf, network: Network, -) -> FetchService { - FetchService::spawn(FetchServiceConfig::new( +) -> NodeBackedIndexerService { + NodeBackedIndexerService::spawn(NodeBackedIndexerServiceConfig::new_rpc( rpc_url, cookie_dir, None, @@ -1152,18 +1139,17 @@ async fn spawn_fetch_service( /// server — for tests that compare the two. The owned services exist only to /// keep their subscribers alive. #[cfg(feature = "zcashd_support")] -#[allow(deprecated)] pub struct ZcashdDualFetchServices { /// The launched zcashd + Zaino test manager. - pub test_manager: TestManager, - /// Owned zcashd-direct fetch service; keeps `zcashd_subscriber` alive. - pub zcashd_fetch_service: FetchService, + pub test_manager: TestManager, + /// Owned zcashd-direct `Rpc` service; keeps `zcashd_subscriber` alive. + pub zcashd_fetch_service: NodeBackedIndexerService, /// Subscriber whose answers come from zcashd directly. - pub zcashd_subscriber: FetchServiceSubscriber, - /// Owned Zaino-pointed fetch service; keeps `zaino_subscriber` alive. - pub zaino_fetch_service: FetchService, + pub zcashd_subscriber: NodeBackedIndexerServiceSubscriber, + /// Owned Zaino-pointed `Rpc` service; keeps `zaino_subscriber` alive. + pub zaino_fetch_service: NodeBackedIndexerService, /// Subscriber whose answers come through Zaino's JSON-RPC server. - pub zaino_subscriber: FetchServiceSubscriber, + pub zaino_subscriber: NodeBackedIndexerServiceSubscriber, } #[cfg(feature = "zcashd_support")] @@ -1202,7 +1188,7 @@ pub async fn launch_zcashd_dual_fetch_services() -> ZcashdDualFetchServices { pub async fn launch_zcashd_dual_fetch_services_at( activation_heights: ActivationHeights, ) -> ZcashdDualFetchServices { - let test_manager = TestManager::::launch( + let test_manager = TestManager::::launch( &ValidatorKind::Zcashd, None, Some(activation_heights), @@ -1299,11 +1285,10 @@ pub fn get_info_with_zeroed_timestamp(info: GetInfo) -> GetInfo { /// harness in both live-test partitions (`clientless` and `e2e`). The `e2e` /// partition wraps this and builds lightclients from the returned manager's /// gRPC address. -#[allow(deprecated)] pub async fn launch_with_fetch_subscriber( validator: &ValidatorKind, chain_cache: Option, -) -> (TestManager, FetchServiceSubscriber) { +) -> (TestManager, NodeBackedIndexerServiceSubscriber) { launch_with_fetch_subscriber_mining_to::( default_mining_pool(validator), validator, @@ -1316,13 +1301,12 @@ pub async fn launch_with_fetch_subscriber( /// caller instead of [`default_mining_pool`]: [`SHIELDED_FUNDING_POOL`] for /// sessions funding wallets from coinbase, or a pinned pool for tests whose /// subject is the miner's coinbase footprint. -#[allow(deprecated)] pub async fn launch_with_fetch_subscriber_mining_to( mine_to_pool: PoolType, validator: &ValidatorKind, chain_cache: Option, -) -> (TestManager, FetchServiceSubscriber) { - let mut test_manager = TestManager::::launch_mining_to( +) -> (TestManager, NodeBackedIndexerServiceSubscriber) { + let mut test_manager = TestManager::::launch_mining_to( mine_to_pool, validator, None, @@ -1338,9 +1322,7 @@ pub async fn launch_with_fetch_subscriber_mining_to( (test_manager, fetch_service_subscriber) } -impl Drop - for TestManager -{ +impl Drop for TestManager { fn drop(&mut self) { debug!("[TEST] Shutting down test environment"); if let Some(handle) = &self.zaino_handle { @@ -1362,24 +1344,20 @@ async fn build_client( #[cfg(test)] mod launch_testmanager { use super::*; - #[allow(deprecated)] - use zaino_state::FetchService; /// Launch with no network/heights overrides and the optional servers off — /// the smoke tests' shared launch shape; only the chain cache and the /// zaino toggle vary. - async fn launch_minimal( + async fn launch_minimal( validator: &ValidatorKind, chain_cache: Option, enable_zaino: bool, - ) -> TestManager + ) -> TestManager where C: ValidatorExt, - Service: TestService, - IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, - ::Subscriber: PollableTip, + Conn: ValidatorConnectionMarker, { - TestManager::::launch( + TestManager::::launch( validator, None, None, @@ -1402,7 +1380,7 @@ mod launch_testmanager { #[allow(deprecated)] pub(crate) async fn basic() { let mut test_manager = - launch_minimal::(&ValidatorKind::Zcashd, None, false).await; + launch_minimal::(&ValidatorKind::Zcashd, None, false).await; assert_eq!(2, (test_manager.local_net.get_chain_height().await)); test_manager.close().await; } @@ -1411,7 +1389,7 @@ mod launch_testmanager { #[allow(deprecated)] pub(crate) async fn generate_blocks() { let mut test_manager = - launch_minimal::(&ValidatorKind::Zcashd, None, false).await; + launch_minimal::(&ValidatorKind::Zcashd, None, false).await; assert_eq!(2, (test_manager.local_net.get_chain_height().await)); test_manager.local_net.generate_blocks(1).await.unwrap(); assert_eq!(3, (test_manager.local_net.get_chain_height().await)); @@ -1422,7 +1400,7 @@ mod launch_testmanager { #[tokio::test(flavor = "multi_thread")] #[allow(deprecated)] pub(crate) async fn with_chain() { - let mut test_manager = launch_minimal::( + let mut test_manager = launch_minimal::( &ValidatorKind::Zcashd, ZCASHD_CHAIN_CACHE_DIR.clone(), false, @@ -1436,7 +1414,7 @@ mod launch_testmanager { #[allow(deprecated)] pub(crate) async fn zaino() { let mut test_manager = - launch_minimal::(&ValidatorKind::Zcashd, None, true).await; + launch_minimal::(&ValidatorKind::Zcashd, None, true).await; let _grpc_client = build_client(test_manager.grpc_socket_to_uri()) .await @@ -1458,8 +1436,7 @@ mod launch_testmanager { #[allow(deprecated)] pub(crate) async fn basic() { let mut test_manager = - launch_minimal::(&ValidatorKind::Zebrad, None, false) - .await; + launch_minimal::(&ValidatorKind::Zebrad, None, false).await; assert_eq!(2, (test_manager.local_net.get_chain_height().await)); test_manager.close().await; } @@ -1468,8 +1445,7 @@ mod launch_testmanager { #[allow(deprecated)] pub(crate) async fn generate_blocks() { let mut test_manager = - launch_minimal::(&ValidatorKind::Zebrad, None, false) - .await; + launch_minimal::(&ValidatorKind::Zebrad, None, false).await; assert_eq!(2, (test_manager.local_net.get_chain_height().await)); test_manager.local_net.generate_blocks(1).await.unwrap(); assert_eq!(3, (test_manager.local_net.get_chain_height().await)); @@ -1480,7 +1456,7 @@ mod launch_testmanager { #[tokio::test(flavor = "multi_thread")] #[allow(deprecated)] pub(crate) async fn with_chain() { - let mut test_manager = launch_minimal::( + let mut test_manager = launch_minimal::( &ValidatorKind::Zebrad, ZEBRAD_CHAIN_CACHE_DIR.clone(), false, @@ -1494,8 +1470,7 @@ mod launch_testmanager { #[allow(deprecated)] pub(crate) async fn zaino() { let mut test_manager = - launch_minimal::(&ValidatorKind::Zebrad, None, true) - .await; + launch_minimal::(&ValidatorKind::Zebrad, None, true).await; let _grpc_client = build_client(test_manager.grpc_socket_to_uri()) .await .unwrap(); @@ -1505,15 +1480,12 @@ mod launch_testmanager { mod state_service { use super::*; - #[allow(deprecated)] - use zaino_state::StateService; #[tokio::test(flavor = "multi_thread")] #[allow(deprecated)] pub(crate) async fn basic() { let mut test_manager = - launch_minimal::(&ValidatorKind::Zebrad, None, false) - .await; + launch_minimal::(&ValidatorKind::Zebrad, None, false).await; assert_eq!(2, (test_manager.local_net.get_chain_height().await)); test_manager.close().await; } @@ -1522,8 +1494,7 @@ mod launch_testmanager { #[allow(deprecated)] pub(crate) async fn generate_blocks() { let mut test_manager = - launch_minimal::(&ValidatorKind::Zebrad, None, false) - .await; + launch_minimal::(&ValidatorKind::Zebrad, None, false).await; assert_eq!(2, (test_manager.local_net.get_chain_height().await)); test_manager.local_net.generate_blocks(1).await.unwrap(); assert_eq!(3, (test_manager.local_net.get_chain_height().await)); @@ -1534,7 +1505,7 @@ mod launch_testmanager { #[tokio::test(flavor = "multi_thread")] #[allow(deprecated)] pub(crate) async fn with_chain() { - let mut test_manager = launch_minimal::( + let mut test_manager = launch_minimal::( &ValidatorKind::Zebrad, ZEBRAD_CHAIN_CACHE_DIR.clone(), false, @@ -1548,8 +1519,7 @@ mod launch_testmanager { #[allow(deprecated)] pub(crate) async fn zaino() { let mut test_manager = - launch_minimal::(&ValidatorKind::Zebrad, None, true) - .await; + launch_minimal::(&ValidatorKind::Zebrad, None, true).await; let _grpc_client = build_client(test_manager.grpc_socket_to_uri()) .await .unwrap(); diff --git a/packages/zaino-state/src/backends.rs b/packages/zaino-state/src/backends.rs deleted file mode 100644 index 019cfc5f1..000000000 --- a/packages/zaino-state/src/backends.rs +++ /dev/null @@ -1,71 +0,0 @@ -//! Zaino's chain fetch and tx submission backend services. - -pub mod fetch; - -pub mod state; - -fn latest_network_upgrade( - upgrades: &indexmap::IndexMap< - zebra_rpc::methods::ConsensusBranchIdHex, - zebra_rpc::methods::NetworkUpgradeInfo, - >, -) -> Result<&zebra_rpc::methods::NetworkUpgradeInfo, tonic::Status> { - upgrades.last().map(|(_, upgrade)| upgrade).ok_or_else(|| { - tonic::Status::failed_precondition("validator returned no network upgrade metadata") - }) -} - -/// Maximum number of addresses a single `get_address_utxos` / `get_address_utxos_stream` -/// request may carry. -/// -/// Both backends resolve the full backend UTXO set before applying `max_entries` / -/// `start_height` (issue #974). A complete pushdown fix needs upstream interface changes -/// the caller-supplied entry cap cannot reach today, so until then this bounds the one -/// input the service controls locally: the address fan-out. It stops an unauthenticated -/// caller forcing an unbounded number of backend address lookups in a single request, and -/// is set well above realistic wallet usage. -/// -/// TODO: make this deployment-configurable rather than a fixed constant. -const UTXO_MAX_ADDRESSES: usize = 1000; - -/// Reject a `get_address_utxos` request whose address list exceeds [`UTXO_MAX_ADDRESSES`]. -/// -/// `max_entries` bounds the response size, not the backend work; this guard bounds the -/// address fan-out, the part the service can cap without upstream changes. -fn validate_utxo_address_count(count: usize) -> Result<(), tonic::Status> { - if count > UTXO_MAX_ADDRESSES { - return Err(tonic::Status::invalid_argument(format!( - "Error: too many addresses in request: {count} exceeds the maximum of {UTXO_MAX_ADDRESSES}." - ))); - } - Ok(()) -} - -#[cfg(test)] -mod tests { - #[test] - fn latest_network_upgrade_rejects_empty_metadata() { - let upgrades = indexmap::IndexMap::new(); - let err = super::latest_network_upgrade(&upgrades).expect_err("empty upgrades must fail"); - - assert_eq!(err.code(), tonic::Code::FailedPrecondition); - assert_eq!( - err.message(), - "validator returned no network upgrade metadata" - ); - } - - #[test] - fn utxo_address_count_within_limit_is_accepted() { - assert!(super::validate_utxo_address_count(0).is_ok()); - assert!(super::validate_utxo_address_count(super::UTXO_MAX_ADDRESSES).is_ok()); - } - - #[test] - fn utxo_address_count_over_limit_is_rejected() { - let err = super::validate_utxo_address_count(super::UTXO_MAX_ADDRESSES + 1) - .expect_err("over-limit address count must fail"); - - assert_eq!(err.code(), tonic::Code::InvalidArgument); - } -} diff --git a/packages/zaino-state/src/backends/state.rs b/packages/zaino-state/src/backends/state.rs deleted file mode 100644 index cc6f4b609..000000000 --- a/packages/zaino-state/src/backends/state.rs +++ /dev/null @@ -1,1896 +0,0 @@ -//! Zcash chain fetch and tx submission service backed by Zebras [`ReadStateService`]. - -#[allow(deprecated)] -use crate::{ - chain_index::{ - chain_tips_from_nonfinalized_snapshot, source::ValidatorConnector, types as chain_types, - ChainIndex, ChainIndexRpcExt, - }, - config::{DonationAddress, StateServiceConfig}, - error::{BlockCacheError, StateServiceError}, - indexer::{ - handle_raw_transaction, IndexerSubscriber, LightWalletIndexer, ZcashIndexer, ZcashService, - }, - status::{Status, StatusType}, - stream::{ - AddressStream, CompactBlockStream, CompactTransactionStream, RawTransactionStream, - UtxoReplyStream, - }, - utils::{get_build_info, ServiceMetadata}, - BackendType, NodeBackedChainIndex, NodeBackedChainIndexSubscriber, -}; -use crate::{ - chain_index::{types::BestChainLocation, NonFinalizedSnapshot}, - TransactionHash, -}; -use tokio_stream::StreamExt as _; -use zaino_fetch::{ - chain::{transaction::FullTransaction, utils::ParseFromSlice}, - jsonrpsee::{ - connector::RpcError, - response::{ - address_deltas::{GetAddressDeltasParams, GetAddressDeltasResponse}, - block_deltas::BlockDeltas, - block_header::GetBlockHeader, - block_subsidy::GetBlockSubsidy, - chain_tips::GetChainTipsResponse, - mining_info::GetMiningInfoWire, - peer_info::GetPeerInfo, - z_validate_address::ZValidateAddressResponse, - GetMempoolInfoResponse, GetNetworkSolPsResponse, GetSpentInfoRequest, - GetSpentInfoResponse, GetTxOutResponse, GetTxOutSetInfoResponse, - }, - }, -}; -use zaino_proto::proto::utils::{ - blockid_to_hashorheight, compact_block_to_nullifiers, GetBlockRangeError, PoolTypeError, - PoolTypeFilter, ValidatedBlockRangeRequest, -}; -use zaino_proto::proto::{ - compact_formats::CompactBlock, - service::{ - AddressList, Balance, BlockId, BlockRange, GetAddressUtxosArg, GetAddressUtxosReply, - GetAddressUtxosReplyList, GetMempoolTxRequest, LightdInfo, PingResponse, RawTransaction, - SendResponse, TransparentAddressBlockFilter, TreeState, TxFilter, - }, -}; -use zebra_chain::{ - block::Height, serialization::ZcashDeserialize as _, subtree::NoteCommitmentSubtreeIndex, -}; -use zebra_rpc::{ - client::{ - GetAddressBalanceRequest, GetSubtreesByIndexResponse, GetTreestateResponse, - TransactionObject, ValidateAddressResponse, - }, - methods::{ - AddressBalance, GetAddressTxIdsRequest, GetAddressUtxos, GetBlock, GetBlockHash, - GetBlockchainInfoResponse, GetInfo, GetRawTransaction, SentTransactionHash, - }, - server::error::LegacyCode, -}; -use zebra_state::HashOrHeight; - -use hex::{FromHex as _, ToHex}; -use std::str::FromStr; -use tokio::{ - sync::mpsc, - time::{self, timeout}, -}; -use tracing::{info, instrument, warn}; - -/// Chain fetch service backed by Zebra's `ReadStateService` and `TrustedChainSync`. -/// -/// NOTE: We currently dop not implement clone for chain fetch services -/// as this service is responsible for maintaining and closing its child processes. -/// ServiceSubscribers are used to create separate chain fetch processes -/// while allowing central state processes to be managed in a single place. -/// If we want the ability to clone Service all JoinHandle's should be -/// converted to Arc\. -#[derive(Debug)] -// #[deprecated = "Will be eventually replaced by `BlockchainSource"] -pub struct StateService { - /// Core indexer. - indexer: NodeBackedChainIndex, - - /// Service metadata. - data: ServiceMetadata, - - /// StateService config data. - #[allow(deprecated)] - config: StateServiceConfig, -} - -impl Status for StateService { - fn status(&self) -> StatusType { - self.indexer.status() - } -} - -// #[allow(deprecated)] -impl ZcashService for StateService { - const BACKEND_TYPE: BackendType = BackendType::State; - - type Subscriber = StateServiceSubscriber; - type Config = StateServiceConfig; - - /// Initializes a new StateService instance and starts sync process. - #[instrument(name = "StateService::spawn", skip(config), fields(network = %config.common.network))] - async fn spawn(config: StateServiceConfig) -> Result { - info!( - rpc_address = %config.common.validator_rpc_address, - network = %config.common.network, - "Spawning State Service" - ); - - let (source, zebra_build_data) = ValidatorConnector::spawn_state(&config) - .await - .map_err(|error| StateServiceError::Critical(error.to_string()))?; - - let data = ServiceMetadata::new( - get_build_info(config.common.indexer_version.clone()), - config.common.network.to_zebra_network(), - zebra_build_data.build, - zebra_build_data.subversion, - ); - info!(build = %data.zebra_build(), subversion = %data.zebra_subversion(), "Connected to Zcash node"); - - let indexer = NodeBackedChainIndex::new(source, config.clone().into()) - .await - .map_err(|error| StateServiceError::Critical(error.to_string()))?; - - let state_service = Self { - indexer, - data, - config, - }; - - // wait for sync to complete, return error on sync fail. - loop { - match state_service.status() { - StatusType::Ready | StatusType::Closing => break, - StatusType::CriticalError => { - return Err(StateServiceError::Critical( - "Chain index sync failed".to_string(), - )); - } - _ => { - tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; - } - } - } - - Ok(state_service) - } - - fn get_subscriber(&self) -> IndexerSubscriber { - IndexerSubscriber::new(StateServiceSubscriber { - indexer: self.indexer.subscriber(), - data: self.data.clone(), - config: self.config.clone(), - }) - } - - /// Shuts down the StateService. - /// - /// Delegates to the indexer, which cancels its sync loop, tears down the - /// finalised DB and mempool, and aborts the source-owned Zebra chain-syncer - /// task via [`crate::chain_index::source::BlockchainSource::shutdown`]. - fn close(&mut self) { - tokio::task::block_in_place(|| { - tokio::runtime::Handle::current().block_on(async { - let _ = self.indexer.shutdown().await; - }); - }); - } -} - -#[allow(deprecated)] -impl Drop for StateService { - fn drop(&mut self) { - self.close() - } -} - -/// A fetch service subscriber. -/// -/// Subscribers should be -#[derive(Debug, Clone)] -// #[deprecated] -pub struct StateServiceSubscriber { - /// Core indexer. - pub indexer: NodeBackedChainIndexSubscriber, - - /// Service metadata. - pub data: ServiceMetadata, - - /// StateService config data. - #[allow(deprecated)] - config: StateServiceConfig, -} - -impl StateServiceSubscriber { - /// The backing Zebra [`ReadStateService`]. - /// - /// Test-only escape hatch: live tests recompute expected chain data (e.g. - /// treestate roots) directly off the `ReadStateService`. Production code goes - /// through the `ChainIndex` API. - #[cfg(feature = "test_dependencies")] - pub fn read_state_service(&self) -> zebra_state::ReadStateService { - self.indexer - .source() - .read_state_service() - .expect("StateServiceSubscriber is always State-backed") - .clone() - } - - /// The indexer's mempool subscriber. - /// - /// Test-only escape hatch: live tests recompute expected `getmempoolinfo` - /// values directly off the mempool's entries. Production code goes through the - /// `ChainIndex` mempool API. - #[cfg(feature = "test_dependencies")] - pub fn mempool(&self) -> &crate::chain_index::mempool::MempoolSubscriber { - self.indexer.mempool_subscriber() - } -} - -impl Status for StateServiceSubscriber { - fn status(&self) -> StatusType { - self.indexer.status() - } -} - -/// A subscriber to any chaintip updates -#[derive(Clone)] -pub struct ChainTipSubscriber { - monitor: zebra_state::ChainTipChange, -} - -impl ChainTipSubscriber { - /// Waits until the tip hash has changed (relative to the last time this method - /// was called), then returns the best tip's block hash. - pub async fn next_tip_hash( - &mut self, - ) -> Result { - self.monitor - .wait_for_tip_change() - .await - .map(|tip| tip.best_tip_hash()) - } -} - -/// Private RPC methods, which are used as helper methods by the public ones -/// -/// These would be simple to add to the public interface if -/// needed, there are currently no plans to do so. -// #[allow(deprecated)] -impl StateServiceSubscriber { - /// Gets a Subscriber to any updates to the latest chain tip - pub fn chaintip_update_subscriber(&self) -> ChainTipSubscriber { - ChainTipSubscriber { - monitor: self - .indexer - .source() - .chain_tip_change() - .expect("StateServiceSubscriber is always State-backed"), - } - } - /// Return a list of consecutive compact blocks. - #[allow(dead_code, deprecated)] - async fn get_block_range_inner( - &self, - request: BlockRange, - nullifiers_only: bool, - ) -> Result { - let validated_request = ValidatedBlockRangeRequest::new_from_block_range(&request) - .map_err(StateServiceError::from)?; - - let pool_type_filter = PoolTypeFilter::new_from_pool_types(&validated_request.pool_types()) - .map_err(GetBlockRangeError::PoolTypeArgumentError) - .map_err(StateServiceError::from)?; - - // Note conversion here is safe due to the use of [`ValidatedBlockRangeRequest::new_from_block_range`] - let start = validated_request.start() as u32; - let end = validated_request.end() as u32; - - let state_service_clone = self.clone(); - let service_timeout = self.config.common.service.timeout; - let (channel_tx, channel_rx) = - mpsc::channel(self.config.common.service.channel_size as usize); - let snapshot = state_service_clone - .indexer - .snapshot_nonfinalized_state() - .await?; - - tokio::spawn(async move { - let timeout_result = timeout( - time::Duration::from_secs((service_timeout * 4) as u64), - async { - // This method does not support passthrough. Just return. - let Some(non_finalized_snapshot) = snapshot.get_nfs_snapshot() else {return}; - let chain_height = non_finalized_snapshot.best_tip.height.0; - - match state_service_clone - .indexer - .get_compact_block_stream( - &snapshot, - chain_types::Height(start), - chain_types::Height(end), - pool_type_filter.clone(), - ) - .await - { - Ok(Some(mut compact_block_stream)) => { - if nullifiers_only { - while let Some(stream_item) = compact_block_stream.next().await { - match stream_item { - Ok(block) => { - if channel_tx - .send(Ok(compact_block_to_nullifiers(block))) - .await - .is_err() - { - break; - } - } - Err(status) => { - if channel_tx.send(Err(status)).await.is_err() { - break; - } - } - } - } - } else { - while let Some(stream_item) = compact_block_stream.next().await { - if channel_tx.send(stream_item).await.is_err() { - break; - } - } - } - } - Ok(None) => { - // Per `get_compact_block_stream` semantics: `None` means at least one bound is above the tip. - let offending_height = if start > chain_height { start } else { end }; - - match channel_tx - .send(Err(tonic::Status::out_of_range(format!( - "Error: Height out of range [{offending_height}]. Height requested is greater than the best chain tip [{chain_height}].", - )))) - .await - { - Ok(_) => {} - Err(e) => { - warn!(%e, "GetBlockRange channel closed unexpectedly"); - } - } - } - Err(e) => { - // Preserve previous behaviour: if the request is above tip, surface OutOfRange; - // otherwise return the error (currently exposed for dev). - if start > chain_height || end > chain_height { - let offending_height = if start > chain_height { start } else { end }; - - match channel_tx - .send(Err(tonic::Status::out_of_range(format!( - "Error: Height out of range [{offending_height}]. Height requested is greater than the best chain tip [{chain_height}].", - )))) - .await - { - Ok(_) => {} - Err(e) => { - warn!(%e, "GetBlockRange channel closed unexpectedly"); - } - } - } else { - // TODO: Hide server error from clients before release. Currently useful for dev purposes. - if channel_tx - .send(Err(tonic::Status::unknown(e.to_string()))) - .await - .is_err() - { - warn!(%e, "GetBlockRangeStream closed unexpectedly"); - } - } - } - } - }, - ) - .await; - - if timeout_result.is_err() { - channel_tx - .send(Err(tonic::Status::deadline_exceeded( - "Error: get_block_range gRPC request timed out.", - ))) - .await - .ok(); - } - }); - - Ok(CompactBlockStream::new(channel_rx)) - } - - async fn error_get_block( - &self, - e: BlockCacheError, - height: u32, - ) -> Result { - let snapshot = self.indexer.snapshot_nonfinalized_state().await?; - let chain_height = snapshot.max_serviceable_height().0; - Err(if height >= chain_height { - StateServiceError::TonicStatusError(tonic::Status::out_of_range(format!( - "Error: Height out of range [{height}]. Height requested \ - is greater than Zaino's best chain tip [{chain_height}].", - ))) - } else { - // TODO: Hide server error from clients before release. - // Currently useful for dev purposes. - StateServiceError::TonicStatusError(tonic::Status::unknown(format!( - "Error: Failed to retrieve block from node. Server Error: {e}", - ))) - }) - } - - /// Returns the network type running. - #[allow(deprecated)] - pub fn network(&self) -> zaino_common::Network { - self.config.common.network - } -} - -// #[allow(deprecated)] -impl ZcashIndexer for StateServiceSubscriber { - type Error = StateServiceError; - - async fn get_info(&self) -> Result { - Ok(self.indexer.get_info().await?) - } - - /// Returns all changes for an address. - /// - /// Returns information about all changes to the given transparent addresses within the given (inclusive) - /// - /// block height range, default is the full blockchain. - /// If start or end are not specified, they default to zero. - /// If start is greater than the latest block height, it's interpreted as that height. - /// - /// If end is zero, it's interpreted as the latest block height. - /// - /// [Original zcashd implementation](https://github.com/zcash/zcash/blob/18238d90cd0b810f5b07d5aaa1338126aa128c06/src/rpc/misc.cpp#L881) - /// - /// zcashd reference: [`getaddressdeltas`](https://zcash.github.io/rpc/getaddressdeltas.html) - /// method: post - /// tags: address - async fn get_address_deltas( - &self, - params: GetAddressDeltasParams, - ) -> Result { - Ok(self.indexer.get_address_deltas(params).await?) - } - - async fn get_difficulty(&self) -> Result { - Ok(self.indexer.get_difficulty().await?) - } - - async fn get_block_subsidy(&self, height: u32) -> Result { - Ok(self.indexer.get_block_subsidy(height).await?) - } - - async fn get_blockchain_info(&self) -> Result { - Ok(self.indexer.get_blockchain_info().await?) - } - - /// Returns details on the active state of the TX memory pool. - /// In Zaino, this RPC call information is gathered from the local Zaino state instead of directly reflecting the full node's mempool. This state is populated from a gRPC stream, sourced from the full node. - /// There are no request parameters. - /// The Zcash source code is considered canonical: - /// [from the rpc definition](), [this function is called to produce the return value](>). - /// There are no required or optional parameters. - /// the `size` field is called by [this line of code](), and returns an int64. - /// `size` represents the number of transactions currently in the mempool. - /// the `bytes` field is called by [this line of code](), and returns an int64 from [this variable](). - /// `bytes` is the sum memory size in bytes of all transactions in the mempool: the sum of all transaction byte sizes. - /// the `usage` field is called by [this line of code](), and returns an int64 derived from the return of this function(), which includes a number of elements. - /// `usage` is the total memory usage for the mempool, in bytes. - /// the [optional `fullyNotified` field](), is only utilized for zcashd regtests, is deprecated, and is not included. - async fn get_mempool_info(&self) -> Result { - Ok(self.indexer.get_mempool_info().await.into()) - } - - async fn get_peer_info(&self) -> Result { - Ok(self.indexer.get_peer_info().await?) - } - - async fn z_get_address_balance( - &self, - address_strings: GetAddressBalanceRequest, - ) -> Result { - Ok(self.indexer.get_address_balance(address_strings).await?) - } - - async fn send_raw_transaction( - &self, - raw_transaction_hex: String, - ) -> Result { - Ok(self - .indexer - .send_raw_transaction(raw_transaction_hex) - .await?) - } - - async fn get_block_header( - &self, - hash: String, - verbose: bool, - ) -> Result { - Ok(self.indexer.get_block_header(hash, verbose).await?) - } - - async fn z_get_block( - &self, - hash_or_height_string: String, - verbosity: Option, - ) -> Result { - Ok(self - .indexer - .z_get_block(hash_or_height_string, verbosity) - .await?) - } - - async fn get_block_deltas(&self, hash: String) -> Result { - Ok(self.indexer.get_block_deltas(hash).await?) - } - - async fn get_raw_mempool(&self) -> Result, Self::Error> { - Ok(self - .indexer - .get_mempool_txids() - .await? - .into_iter() - .map(|txid| txid.to_rpc_hex()) - .collect()) - } - - /// NOTE: This method currently has to fetch data from 2 places (get_treestate and get_indexed_block_by_*), - /// If `ValidatorConnector::GetTreeState` was updated to return the additional information - /// required, this second call could be removed, improving the performance of this method. - // Pre-existing lint: `StateServiceError` is a large error type; returning it by value here is - // flagged by `result_large_err`. Suppressed to satisfy `-D warnings` without an invasive - // boxing refactor of the shared error enum. - #[allow(clippy::result_large_err)] - async fn z_get_treestate( - &self, - hash_or_height: String, - ) -> Result { - let fallback_hash_or_height = hash_or_height.clone(); - let local_result: Result = async { - let hash_or_height_struct: HashOrHeight = HashOrHeight::from_str(&hash_or_height)?; - let snapshot = self.indexer.snapshot_nonfinalized_state().await?; - - let block_data = match hash_or_height_struct { - HashOrHeight::Hash(hash) => self - .indexer - .get_indexed_block_by_hash(&snapshot, &hash.into()) - .await? - .ok_or(StateServiceError::RpcError(RpcError::new_from_legacycode( - zebra_rpc::server::error::LegacyCode::InvalidParameter, - "Failed to fetch block data.", - )))?, - HashOrHeight::Height(height) => self - .indexer - .get_indexed_block_by_height(&snapshot, &height.into()) - .await? - .ok_or(StateServiceError::RpcError(RpcError::new_from_legacycode( - zebra_rpc::server::error::LegacyCode::InvalidParameter, - "Failed to fetch block data.", - )))?, - }; - - let (sapling, orchard) = self.indexer.get_treestate(block_data.hash()).await?; - let time: u32 = block_data.data().time().try_into().map_err(|_error| { - StateServiceError::RpcError(RpcError::new_from_legacycode( - zebra_rpc::server::error::LegacyCode::InvalidParameter, - "Block time is out of range for u32.", - )) - })?; - - #[allow(deprecated)] - Ok(GetTreestateResponse::from_parts( - (*block_data.hash()).into(), - block_data.height().into(), - time, - sapling, - orchard, - )) - } - .await; - - if let Ok(response) = local_result { - return Ok(response); - } - - let snapshot = self.indexer.snapshot_nonfinalized_state().await?; - if !self - .indexer - .hash_or_height_known_for_treestate(&snapshot, &fallback_hash_or_height) - .await? - { - return local_result; - } - - Ok(self - .indexer - .get_treestate_by_id(fallback_hash_or_height) - .await?) - } - - async fn get_mining_info(&self) -> Result { - Ok(self.indexer.get_mining_info().await?) - } - - /// Returns statistics about the unspent transaction output set. - /// - /// zcashd reference: [`gettxoutsetinfo`](https://zcash.github.io/rpc/gettxoutsetinfo.html) - /// method: post - /// tags: blockchain - async fn get_tx_out_set_info(&self) -> Result { - Ok(self.indexer.get_tx_out_set_info().await?) - } - - // No request parameters. - /// Return the hex encoded hash of the best (tip) block, in the longest block chain. - /// The Zcash source code is considered canonical: - /// [In the rpc definition](https://github.com/zcash/zcash/blob/654a8be2274aa98144c80c1ac459400eaf0eacbe/src/rpc/common.h#L48) there are no required params, or optional params. - /// [The function in rpc/blockchain.cpp](https://github.com/zcash/zcash/blob/654a8be2274aa98144c80c1ac459400eaf0eacbe/src/rpc/blockchain.cpp#L325) - /// where `return chainActive.Tip()->GetBlockHash().GetHex();` is the [return expression](https://github.com/zcash/zcash/blob/654a8be2274aa98144c80c1ac459400eaf0eacbe/src/rpc/blockchain.cpp#L339)returning a `std::string` - async fn get_best_blockhash(&self) -> Result { - let snapshot = self.indexer.snapshot_nonfinalized_state().await?; - let tip = self.indexer.best_chaintip(&snapshot).await?; - Ok(GetBlockHash::new(tip.hash.into())) - } - - /// Returns the current block count in the best valid block chain. - /// - /// zcashd reference: [`getblockcount`](https://zcash.github.io/rpc/getblockcount.html) - /// method: post - /// tags: blockchain - async fn get_block_count(&self) -> Result { - let snapshot = self.indexer.snapshot_nonfinalized_state().await?; - let tip = self.indexer.best_chaintip(&snapshot).await?; - Ok(tip.height.into()) - } - - async fn get_chain_tips(&self) -> Result { - let snapshot = self.indexer.snapshot_nonfinalized_state().await?; - let Some(non_finalized_snapshot) = snapshot.get_nfs_snapshot() else { - return Err(StateServiceError::UnavailableNotSyncedEnough); - }; - Ok(chain_tips_from_nonfinalized_snapshot( - non_finalized_snapshot, - )) - } - - async fn validate_address( - &self, - raw_address: String, - ) -> Result { - let network = self.config.common.network.to_zebra_network(); - Ok(crate::indexer::validate_address(raw_address, &network)) - } - - #[allow(deprecated)] - async fn z_validate_address( - &self, - address: String, - ) -> Result { - let network = self.config.common.network.to_zebra_network(); - Ok(crate::indexer::z_validate_address(address, &network)) - } - - async fn z_get_subtrees_by_index( - &self, - pool: String, - start_index: NoteCommitmentSubtreeIndex, - limit: Option, - ) -> Result { - let shielded_pool = match pool.as_str() { - "sapling" => crate::chain_index::ShieldedPool::Sapling, - "orchard" => crate::chain_index::ShieldedPool::Orchard, - otherwise => { - return Err(StateServiceError::RpcError(RpcError::new_from_legacycode( - LegacyCode::Misc, - format!( - "invalid pool name \"{otherwise}\", must be \"sapling\" or \"orchard\"" - ), - ))) - } - }; - let roots = self - .indexer - .get_subtree_roots(shielded_pool, start_index.0, limit.map(|index| index.0)) - .await?; - Ok(crate::indexer::build_subtrees_by_index_response( - pool, - start_index, - roots, - )) - } - - async fn get_raw_transaction( - &self, - txid_hex: String, - verbose: Option, - ) -> Result { - #[allow(deprecated)] - let txid = TransactionHash::from_hex(&txid_hex).map_err(|error| { - StateServiceError::RpcError(RpcError::new_from_legacycode( - zebra_rpc::server::error::LegacyCode::InvalidAddressOrKey, - error.to_string(), - )) - })?; - - #[allow(deprecated)] - let not_found_error = || { - StateServiceError::RpcError(RpcError::new_from_legacycode( - zebra_rpc::server::error::LegacyCode::InvalidAddressOrKey, - "No such mempool or main chain transaction", - )) - }; - - let snapshot = self.indexer.snapshot_nonfinalized_state().await?; - - let Some((serialized_transaction, _consensus_branch_id)) = - self.indexer.get_raw_transaction(&snapshot, &txid).await? - else { - return Err(not_found_error()); - }; - - if verbose.is_none() { - return Ok(GetRawTransaction::Raw( - zebra_chain::transaction::SerializedTransaction::from(serialized_transaction), - )); - } - - let transaction = zebra_chain::transaction::Transaction::zcash_deserialize( - serialized_transaction.as_slice(), - ) - .map_err(|_| not_found_error())?; - - let (best_chain_location, _non_best_chain_locations) = self - .indexer - .get_transaction_status(&snapshot, &txid) - .await?; - - let (height, confirmations, block_hash, in_best_chain) = match best_chain_location { - Some(BestChainLocation::Block(block_hash, height)) => { - let confirmations = snapshot - .max_serviceable_height() - .0 - .saturating_sub(height.0) - .saturating_add(1); - - ( - Some(zebra_chain::block::Height::from(height)), - Some(confirmations), - Some(zebra_chain::block::Hash::from(block_hash)), - Some(true), - ) - } - Some(BestChainLocation::Mempool(_height)) => (None, Some(0), None, Some(false)), - None => (None, None, None, Some(false)), - }; - - Ok(GetRawTransaction::Object(Box::new( - TransactionObject::from_transaction( - transaction.into(), - height, - confirmations, - #[allow(deprecated)] - &self.config.common.network.to_zebra_network(), - None, - block_hash, - in_best_chain, - zebra_chain::transaction::Hash::from(txid), - ), - ))) - } - - /// Returns details about an unspent transaction output. - /// - /// zcashd reference: [`gettxout`](https://zcash.github.io/rpc/gettxout.html) - /// method: post - /// tags: transaction - async fn get_tx_out( - &self, - txid: String, - n: u32, - include_mempool: Option, - ) -> Result { - Ok(self.indexer.get_tx_out(txid, n, include_mempool).await?) - } - - async fn get_spent_info( - &self, - request: GetSpentInfoRequest, - ) -> Result { - Ok(self.indexer.get_spent_info(request).await?) - } - - async fn get_address_tx_ids( - &self, - request: GetAddressTxIdsRequest, - ) -> Result, Self::Error> { - Ok(self - .indexer - .get_address_txids(request) - .await? - .into_iter() - .map(|transaction_hash| transaction_hash.to_rpc_hex()) - .collect()) - } - - async fn z_get_address_utxos( - &self, - address_strings: GetAddressBalanceRequest, - ) -> Result, Self::Error> { - Ok(self.indexer.get_address_utxos(address_strings).await?) - } - - /// Returns the estimated network solutions per second based on the last n blocks. - /// - /// zcashd reference: [`getnetworksolps`](https://zcash.github.io/rpc/getnetworksolps.html) - /// method: post - /// tags: blockchain - /// - /// This RPC is implemented in the [mining.cpp](https://github.com/zcash/zcash/blob/d00fc6f4365048339c83f463874e4d6c240b63af/src/rpc/mining.cpp#L104) - /// file of the Zcash repository. The Zebra implementation can be found [here](https://github.com/ZcashFoundation/zebra/blob/19bca3f1159f9cb9344c9944f7e1cb8d6a82a07f/zebra-rpc/src/methods.rs#L2687). - /// - /// # Parameters - /// - /// - `blocks`: (number, optional, default=120) Number of blocks, or -1 for blocks over difficulty averaging window. - /// - `height`: (number, optional, default=-1) To estimate network speed at the time of a specific block height. - async fn get_network_sol_ps( - &self, - blocks: Option, - height: Option, - ) -> Result { - Ok(self.indexer.get_network_sol_ps(blocks, height).await?) - } - - // Helper function, to get the chain height in rpc implementations - async fn chain_height(&self) -> Result { - let snapshot = self.indexer.snapshot_nonfinalized_state().await?; - Ok(self.indexer.best_chaintip(&snapshot).await?.height.into()) - } -} - -// #[allow(deprecated)] -impl LightWalletIndexer for StateServiceSubscriber { - /// Return the height of the tip of the best chain - async fn get_latest_block(&self) -> Result { - let snapshot = self.indexer.snapshot_nonfinalized_state().await?; - let Some(non_finalized_snapshot) = snapshot.get_nfs_snapshot() else { - // TODO: This probably shouldn't be an error. - // this is an improvement over previous behaviour of - // acting as if we are only synced to the genesis block - return Err(StateServiceError::UnavailableNotSyncedEnough); - }; - Ok(non_finalized_snapshot.best_tip.to_wire()) - } - - /// Return the compact block corresponding to the given block identifier - async fn get_block(&self, request: BlockId) -> Result { - let hash_or_height = blockid_to_hashorheight(request).ok_or( - StateServiceError::TonicStatusError(tonic::Status::invalid_argument( - "Error: Invalid hash and/or height out of range. Failed to convert to u32.", - )), - )?; - - let snapshot = self.indexer.snapshot_nonfinalized_state().await?; - - // Convert HashOrHeight to chain_types::Height - let block_height = match hash_or_height { - HashOrHeight::Height(h) => chain_types::Height(h.0), - HashOrHeight::Hash(h) => self - .indexer - .get_block_height(&snapshot, chain_types::BlockHash(h.0)) - .await - .map_err(StateServiceError::ChainIndexError)? - .ok_or_else(|| { - StateServiceError::TonicStatusError(tonic::Status::not_found( - "Error: Block not found for given hash.", - )) - })?, - }; - - match self - .indexer - .get_compact_block(&snapshot, block_height, PoolTypeFilter::includes_all()) - .await - { - Ok(Some(block)) => Ok(block), - Ok(None) => { - let Some(non_finalized_snapshot) = snapshot.get_nfs_snapshot() else { - // TODO: This probably shouldn't be an error. - // this is an improvement over previous behaviour of - // acting as if we are only synced to the genesis block - return Err(StateServiceError::UnavailableNotSyncedEnough); - }; - let chain_height = non_finalized_snapshot.best_tip.height.0; - match hash_or_height { - HashOrHeight::Height(Height(height)) if height >= chain_height => Err( - StateServiceError::TonicStatusError(tonic::Status::out_of_range(format!( - "Error: Height out of range [{hash_or_height}]. Height requested \ - is greater than the best chain tip [{chain_height}].", - ))), - ), - _otherwise => Err(StateServiceError::TonicStatusError(tonic::Status::unknown( - "Error: Failed to retrieve block from state.", - ))), - } - } - Err(e) => { - let Some(non_finalized_snapshot) = snapshot.get_nfs_snapshot() else { - // TODO: This probably shouldn't be an error. - // this is an improvement over previous behaviour of - // acting as if we are only synced to the genesis block - return Err(StateServiceError::UnavailableNotSyncedEnough); - }; - let chain_height = non_finalized_snapshot.best_tip.height.0; - match hash_or_height { - HashOrHeight::Height(Height(height)) if height >= chain_height => Err( - StateServiceError::TonicStatusError(tonic::Status::out_of_range(format!( - "Error: Height out of range [{hash_or_height}]. Height requested \ - is greater than the best chain tip [{chain_height}].", - ))), - ), - _otherwise => - // TODO: Hide server error from clients before release. Currently useful for dev purposes. - { - Err(StateServiceError::TonicStatusError(tonic::Status::unknown( - format!("Error: Failed to retrieve block from node. Server Error: {e}",), - ))) - } - } - } - } - } - - /// Same as GetBlock except actions contain only nullifiers, - /// and saling outputs are not returned (Sapling spends still are) - async fn get_block_nullifiers(&self, request: BlockId) -> Result { - let height: u32 = request.height.try_into().map_err(|_| { - StateServiceError::TonicStatusError(tonic::Status::invalid_argument( - "Error: Height out of range. Failed to convert to u32.", - )) - })?; - - let snapshot = self.indexer.snapshot_nonfinalized_state().await?; - let block_height = chain_types::Height(height); - - match self - .indexer - .get_compact_block(&snapshot, block_height, PoolTypeFilter::includes_all()) - .await - { - Ok(Some(block)) => Ok(compact_block_to_nullifiers(block)), - Ok(None) => { - self.error_get_block( - BlockCacheError::Custom("Block not found".to_string()), - height, - ) - .await - } - Err(e) => Err(StateServiceError::ChainIndexError(e)), - } - } - - /// Return a list of consecutive compact blocks - async fn get_block_range( - &self, - blockrange: BlockRange, - ) -> Result { - self.get_block_range_inner(blockrange, false).await - } - /// Same as GetBlockRange except actions contain only nullifiers - async fn get_block_range_nullifiers( - &self, - request: BlockRange, - ) -> Result { - self.get_block_range_inner(request, true).await - } - - /// Return the requested full (not compact) transaction (as from zcashd) - async fn get_transaction(&self, request: TxFilter) -> Result { - let hash = zebra_chain::transaction::Hash::from( - <[u8; 32]>::try_from(request.hash).map_err(|_| { - StateServiceError::TonicStatusError(tonic::Status::invalid_argument( - "Error: Transaction hash incorrect", - )) - })?, - ); - let hex = hash.encode_hex(); - - // explicit over method call syntax to make it clear where this method is coming from - #[allow(clippy::result_large_err)] - ::get_raw_transaction(self, hex, Some(1)) - .await - .and_then(|grt| match grt { - GetRawTransaction::Raw(_serialized_transaction) => Err(StateServiceError::Custom( - "unreachable, verbose transaction expected".to_string(), - )), - GetRawTransaction::Object(transaction_object) => Ok(RawTransaction { - data: transaction_object.hex().as_ref().to_vec(), - height: transaction_object.height().unwrap_or(0) as u64, - }), - }) - } - - /// Submit the given transaction to the Zcash network - async fn send_transaction(&self, request: RawTransaction) -> Result { - let hex_tx = hex::encode(request.data); - let tx_output = self.send_raw_transaction(hex_tx).await?; - - Ok(SendResponse { - error_code: 0, - error_message: tx_output.hash().to_string(), - }) - } - - /// Return the transactions corresponding to the given t-address within the given block range - async fn get_taddress_transactions( - &self, - request: TransparentAddressBlockFilter, - ) -> Result { - let chain_height = self.chain_height().await?; - let txids = self.get_taddress_txids_helper(request).await?; - let fetch_service_clone = self.clone(); - let service_timeout = self.config.common.service.timeout; - let (transmitter, receiver) = - mpsc::channel(self.config.common.service.channel_size as usize); - tokio::spawn(async move { - let timeout = timeout( - time::Duration::from_secs((service_timeout * 4) as u64), - async { - for txid in txids { - let transaction = - fetch_service_clone.get_raw_transaction(txid, Some(1)).await; - if handle_raw_transaction::( - chain_height.0 as u64, - transaction, - transmitter.clone(), - ) - .await - .is_err() - { - break; - } - } - }, - ) - .await; - match timeout { - Ok(_) => {} - Err(_) => { - transmitter - .send(Err(tonic::Status::internal( - "Error: get_taddress_txids gRPC request timed out", - ))) - .await - .ok(); - } - } - }); - Ok(RawTransactionStream::new(receiver)) - } - - /// Return the txids corresponding to the given t-address within the given block range - /// This function is deprecated. Use `get_taddress_transactions`. - async fn get_taddress_txids( - &self, - request: TransparentAddressBlockFilter, - ) -> Result { - self.get_taddress_transactions(request).await - } - - /// Returns the total balance for a list of taddrs - async fn get_taddress_balance( - &self, - request: AddressList, - ) -> Result { - let taddrs = GetAddressBalanceRequest::new(request.addresses); - let balance = self.z_get_address_balance(taddrs).await?; - let checked_balance: i64 = match i64::try_from(balance.balance()) { - Ok(balance) => balance, - Err(_) => { - return Err(StateServiceError::TonicStatusError(tonic::Status::unknown( - "Error: Error converting balance from u64 to i64.", - ))); - } - }; - Ok(Balance { - value_zat: checked_balance, - }) - } - - /// Returns the total balance for a list of taddrs - /// - /// TODO: This is taken from fetch.rs, we could / probably should reconfigure into a trait implementation. - async fn get_taddress_balance_stream( - &self, - mut request: AddressStream, - ) -> Result { - let fetch_service_clone = self.clone(); - let service_timeout = self.config.common.service.timeout; - let (channel_tx, mut channel_rx) = - mpsc::channel::(self.config.common.service.channel_size as usize); - let fetcher_task_handle = tokio::spawn(async move { - let fetcher_timeout = timeout( - time::Duration::from_secs((service_timeout * 4) as u64), - async { - let mut total_balance: u64 = 0; - loop { - match channel_rx.recv().await { - Some(taddr) => { - let taddrs = GetAddressBalanceRequest::new(vec![taddr]); - let balance = - fetch_service_clone.z_get_address_balance(taddrs).await?; - total_balance += balance.balance(); - } - None => { - return Ok(total_balance); - } - } - } - }, - ) - .await; - match fetcher_timeout { - Ok(result) => result, - Err(_) => Err(tonic::Status::deadline_exceeded( - "Error: get_taddress_balance_stream request timed out.", - )), - } - }); - // NOTE: This timeout is so slow due to the blockcache not - // being implemented. This should be reduced to 30s once functionality is in place. - // TODO: Make [rpc_timout] a configurable system variable - // with [default = 30s] and [mempool_rpc_timout = 4*rpc_timeout] - let addr_recv_timeout = timeout( - time::Duration::from_secs((service_timeout * 4) as u64), - async { - while let Some(address_result) = request.next().await { - // TODO: Hide server error from clients before release. - // Currently useful for dev purposes. - let address = address_result.map_err(|e| { - tonic::Status::unknown(format!("Failed to read from stream: {e}")) - })?; - if channel_tx.send(address.address).await.is_err() { - // TODO: Hide server error from clients before release. - // Currently useful for dev purposes. - return Err(tonic::Status::unknown( - "Error: Failed to send address to balance task.", - )); - } - } - drop(channel_tx); - Ok::<(), tonic::Status>(()) - }, - ) - .await; - match addr_recv_timeout { - Ok(Ok(())) => {} - Ok(Err(e)) => { - fetcher_task_handle.abort(); - return Err(StateServiceError::TonicStatusError(e)); - } - Err(_) => { - fetcher_task_handle.abort(); - return Err(StateServiceError::TonicStatusError( - tonic::Status::deadline_exceeded( - "Error: get_taddress_balance_stream request timed out in address loop.", - ), - )); - } - } - match fetcher_task_handle.await { - Ok(Ok(total_balance)) => { - let checked_balance: i64 = match i64::try_from(total_balance) { - Ok(balance) => balance, - Err(_) => { - // TODO: Hide server error from clients before release. - // Currently useful for dev purposes. - return Err(StateServiceError::TonicStatusError(tonic::Status::unknown( - "Error: Error converting balance from u64 to i64.", - ))); - } - }; - Ok(Balance { - value_zat: checked_balance, - }) - } - Ok(Err(e)) => Err(StateServiceError::TonicStatusError(e)), - // TODO: Hide server error from clients before release. - // Currently useful for dev purposes. - Err(e) => Err(StateServiceError::TonicStatusError(tonic::Status::unknown( - format!("Fetcher Task failed: {e}"), - ))), - } - } - - /// Return the compact transactions currently in the mempool; the results - /// can be a few seconds out of date. If the Exclude list is empty, return - /// all transactions; otherwise return all *except* those in the Exclude list - /// (if any); this allows the client to avoid receiving transactions that it - /// already has (from an earlier call to this rpc). The transaction IDs in the - /// Exclude list can be shortened to any number of bytes to make the request - /// more bandwidth-efficient; if two or more transactions in the mempool - /// match a shortened txid, they are all sent (none is excluded). Transactions - /// in the exclude list that don't exist in the mempool are ignored. - async fn get_mempool_tx( - &self, - request: GetMempoolTxRequest, - ) -> Result { - let mut exclude_txids: Vec = vec![]; - - for (i, excluded_id) in request.exclude_txid_suffixes.iter().enumerate() { - if excluded_id.len() > 32 { - return Err(StateServiceError::TonicStatusError( - tonic::Status::invalid_argument(format!( - "Error: excluded txid {} is larger than 32 bytes", - i - )), - )); - } - - // NOTE: the TransactionHash methods cannot be used for this hex encoding as exclusions could be truncated to less than 32 bytes - let reversed_txid_bytes: Vec = excluded_id.iter().cloned().rev().collect(); - let hex_string_txid: String = hex::encode(&reversed_txid_bytes); - exclude_txids.push(hex_string_txid); - } - - let pool_types = match PoolTypeFilter::new_from_slice(&request.pool_types) { - Ok(pool_type_filter) => pool_type_filter, - Err(PoolTypeError::InvalidPoolType) => { - return Err(StateServiceError::TonicStatusError( - tonic::Status::invalid_argument( - "Error: An invalid `PoolType' was found".to_string(), - ), - )) - } - Err(PoolTypeError::UnknownPoolType(unknown_pool_type)) => { - return Err(StateServiceError::TonicStatusError( - tonic::Status::invalid_argument(format!( - "Error: Unknown `PoolType' {} was found", - unknown_pool_type - )), - )) - } - }; - - let indexer = self.indexer.clone(); - let service_timeout = self.config.common.service.timeout; - let (channel_tx, channel_rx) = - mpsc::channel(self.config.common.service.channel_size as usize); - tokio::spawn(async move { - let timeout = timeout( - time::Duration::from_secs((service_timeout * 4) as u64), - async { - let transactions = match indexer.get_mempool_transactions(exclude_txids).await { - Ok(transactions) => transactions, - Err(e) => { - channel_tx - .send(Err(tonic::Status::unknown(e.to_string()))) - .await - .ok(); - return; - } - }; - for serialized_transaction_bytes in transactions { - let txid = match zebra_chain::transaction::Transaction::zcash_deserialize( - &mut std::io::Cursor::new(&serialized_transaction_bytes), - ) { - Ok(transaction) => transaction.hash().0.to_vec(), - Err(error) => { - if channel_tx - .send(Err(tonic::Status::unknown(error.to_string()))) - .await - .is_err() - { - break; - } else { - continue; - } - } - }; - match ::parse_from_slice( - &serialized_transaction_bytes, - Some(vec![txid]), - None, - ) { - Ok(transaction) => { - // ParseFromSlice returns any data left after the conversion to a - // FullTransaction, If the conversion has succeeded this should be empty. - if transaction.0.is_empty() { - if channel_tx - .send( - transaction - .1 - .to_compact_tx(None, &pool_types) - .map_err(|e| tonic::Status::unknown(e.to_string())), - ) - .await - .is_err() - { - break; - } - } else { - // TODO: Hide server error from clients before release. Currently useful for dev purposes. - if channel_tx - .send(Err(tonic::Status::unknown("Error: "))) - .await - .is_err() - { - break; - } - } - } - Err(e) => { - // TODO: Hide server error from clients before release. Currently useful for dev purposes. - if channel_tx - .send(Err(tonic::Status::unknown(e.to_string()))) - .await - .is_err() - { - break; - } - } - } - } - }, - ) - .await; - match timeout { - Ok(_) => {} - Err(_) => { - channel_tx - .send(Err(tonic::Status::internal( - "Error: get_mempool_tx gRPC request timed out", - ))) - .await - .ok(); - } - } - }); - - Ok(CompactTransactionStream::new(channel_rx)) - } - - /// Return a stream of current Mempool transactions. This will keep the output stream open while - /// there are mempool transactions. It will close the returned stream when a new block is mined. - async fn get_mempool_stream(&self) -> Result { - let indexer = self.indexer.clone(); - let service_timeout = self.config.common.service.timeout; - let (channel_tx, channel_rx) = - mpsc::channel(self.config.common.service.channel_size as usize); - let snapshot = indexer.snapshot_nonfinalized_state().await?; - tokio::spawn(async move { - let timeout = timeout( - time::Duration::from_secs((service_timeout * 6) as u64), - async { - let Some(non_finalized_snapshot) = snapshot.get_nfs_snapshot() else { - // TODO: This probably shouldn't be an error. - // this is an improvement over previous behaviour of - // acting as if we are only synced to the genesis block - if let Err(e) = channel_tx - .send(Err(tonic::Status::failed_precondition( - "zaino not yet synced".to_string(), - ))) - .await - { - warn!(%e, "GetMempoolStream channel closed unexpectedly"); - }; - return; - }; - let mempool_height = non_finalized_snapshot.best_tip.height.0; - match indexer.get_mempool_stream(None) { - Some(mut mempool_stream) => { - while let Some(result) = mempool_stream.next().await { - match result { - Ok(transaction_bytes) => { - if channel_tx - .send(Ok(RawTransaction { - data: transaction_bytes, - height: mempool_height as u64, - })) - .await - .is_err() - { - break; - } - } - Err(e) => { - channel_tx - .send(Err(tonic::Status::internal(format!( - "Error in mempool stream: {e:?}" - )))) - .await - .ok(); - break; - } - } - } - } - None => { - warn!("Error fetching stream from mempool, Incorrect chain tip!"); - channel_tx - .send(Err(tonic::Status::internal("Error getting mempool stream"))) - .await - .ok(); - } - }; - }, - ) - .await; - match timeout { - Ok(_) => {} - Err(_) => { - channel_tx - .send(Err(tonic::Status::internal( - "Error: get_mempool_stream gRPC request timed out", - ))) - .await - .ok(); - } - } - }); - Ok(RawTransactionStream::new(channel_rx)) - } - - /// GetTreeState returns the note commitment tree state corresponding to the given block. - /// See section 3.7 of the Zcash protocol specification. It returns several other useful - /// values also (even though they can be obtained using GetBlock). - /// The block can be specified by either height or hash. - async fn get_tree_state(&self, request: BlockId) -> Result { - let hash_or_height = blockid_to_hashorheight(request).ok_or( - crate::error::StateServiceError::TonicStatusError(tonic::Status::invalid_argument( - "Invalid hash or height", - )), - )?; - #[allow(deprecated)] - let (hash, height, time, sapling, orchard) = - ::z_get_treestate( - self, - hash_or_height.to_string(), - ) - .await? - .into_parts(); - Ok(TreeState { - network: self - .config - .common - .network - .to_zebra_network() - .bip70_network_name(), - height: height.0 as u64, - hash: hash.to_string(), - time, - sapling_tree: sapling.map(hex::encode).unwrap_or_default(), - orchard_tree: orchard.map(hex::encode).unwrap_or_default(), - }) - } - - /// GetLatestTreeState returns the note commitment tree state corresponding to the chain tip. - async fn get_latest_tree_state(&self) -> Result { - let latest_block = self.chain_height().await?; - self.get_tree_state(BlockId { - height: latest_block.0 as u64, - hash: vec![], - }) - .await - } - - fn timeout_channel_size(&self) -> (u32, u32) { - ( - self.config.common.service.timeout, - self.config.common.service.channel_size, - ) - } - - /// Returns all unspent outputs for a list of addresses. - /// - /// Ignores all utxos below block height [GetAddressUtxosArg.start_height]. - /// Returns max [GetAddressUtxosArg.max_entries] utxos, or unrestricted if - /// [GetAddressUtxosArg.max_entries] = 0. - /// Utxos are collected and returned as a single Vec. - async fn get_address_utxos( - &self, - request: GetAddressUtxosArg, - ) -> Result { - super::validate_utxo_address_count(request.addresses.len())?; - let taddrs = GetAddressBalanceRequest::new(request.addresses); - let utxos = self.z_get_address_utxos(taddrs).await?; - let mut address_utxos: Vec = Vec::new(); - let mut entries: u32 = 0; - for utxo in utxos { - let (address, tx_hash, output_index, script, satoshis, height) = utxo.into_parts(); - if (height.0 as u64) < request.start_height { - continue; - } - entries += 1; - if request.max_entries > 0 && entries > request.max_entries { - break; - } - let checked_index = match i32::try_from(output_index.index()) { - Ok(index) => index, - Err(_) => { - return Err(StateServiceError::TonicStatusError(tonic::Status::unknown( - "Error: Index out of range. Failed to convert to i32.", - ))); - } - }; - let checked_satoshis = match i64::try_from(satoshis) { - Ok(satoshis) => satoshis, - Err(_) => { - return Err(StateServiceError::TonicStatusError(tonic::Status::unknown( - "Error: Satoshis out of range. Failed to convert to i64.", - ))); - } - }; - let utxo_reply = GetAddressUtxosReply { - address: address.to_string(), - txid: tx_hash.0.to_vec(), - index: checked_index, - script: script.as_raw_bytes().to_vec(), - value_zat: checked_satoshis, - height: height.0 as u64, - }; - address_utxos.push(utxo_reply) - } - Ok(GetAddressUtxosReplyList { address_utxos }) - } - - /// Returns all unspent outputs for a list of addresses. - /// - /// Ignores all utxos below block height [GetAddressUtxosArg.start_height]. - /// Returns max [GetAddressUtxosArg.max_entries] utxos, or unrestricted if - /// [GetAddressUtxosArg.max_entries] = 0. - /// Utxos are returned in a stream. - async fn get_address_utxos_stream( - &self, - request: GetAddressUtxosArg, - ) -> Result { - super::validate_utxo_address_count(request.addresses.len())?; - let taddrs = GetAddressBalanceRequest::new(request.addresses); - let utxos = self.z_get_address_utxos(taddrs).await?; - let service_timeout = self.config.common.service.timeout; - let (channel_tx, channel_rx) = - mpsc::channel(self.config.common.service.channel_size as usize); - tokio::spawn(async move { - let timeout = timeout( - time::Duration::from_secs((service_timeout * 4) as u64), - async { - let mut entries: u32 = 0; - for utxo in utxos { - let (address, tx_hash, output_index, script, satoshis, height) = - utxo.into_parts(); - if (height.0 as u64) < request.start_height { - continue; - } - entries += 1; - if request.max_entries > 0 && entries > request.max_entries { - break; - } - let checked_index = match i32::try_from(output_index.index()) { - Ok(index) => index, - Err(_) => { - let _ = channel_tx - .send(Err(tonic::Status::unknown( - "Error: Index out of range. Failed to convert to i32.", - ))) - .await; - return; - } - }; - let checked_satoshis = match i64::try_from(satoshis) { - Ok(satoshis) => satoshis, - Err(_) => { - let _ = channel_tx - .send(Err(tonic::Status::unknown( - "Error: Satoshis out of range. Failed to convert to i64.", - ))) - .await; - return; - } - }; - let utxo_reply = GetAddressUtxosReply { - address: address.to_string(), - txid: tx_hash.0.to_vec(), - index: checked_index, - script: script.as_raw_bytes().to_vec(), - value_zat: checked_satoshis, - height: height.0 as u64, - }; - if channel_tx.send(Ok(utxo_reply)).await.is_err() { - return; - } - } - }, - ) - .await; - match timeout { - Ok(_) => {} - Err(_) => { - channel_tx - .send(Err(tonic::Status::deadline_exceeded( - "Error: get_mempool_stream gRPC request timed out", - ))) - .await - .ok(); - } - } - }); - Ok(UtxoReplyStream::new(channel_rx)) - } - - /// Return information about this lightwalletd instance and the blockchain - /// - /// TODO: This could be made more efficient by fetching data directly (not using self.get_blockchain_info()) - async fn get_lightd_info(&self) -> Result { - let blockchain_info = self.get_blockchain_info().await?; - let sapling_id = zebra_rpc::methods::ConsensusBranchIdHex::new( - zebra_chain::parameters::ConsensusBranchId::from_hex("76b809bb") - .map_err(|_e| { - tonic::Status::internal( - "Internal Error - Consesnsus Branch ID hex conversion failed", - ) - })? - .into(), - ); - let sapling_activation_height = blockchain_info - .upgrades() - .get(&sapling_id) - .map_or(Height(1), |sapling_json| sapling_json.into_parts().1); - - let consensus_branch_id = zebra_chain::parameters::ConsensusBranchId::from( - blockchain_info.consensus().into_parts().0, - ) - .to_string(); - - let latest_upgrade = super::latest_network_upgrade(blockchain_info.upgrades()) - .map_err(StateServiceError::TonicStatusError)? - .into_parts(); - - let nu_name = latest_upgrade.0; - let nu_height = latest_upgrade.1; - - Ok(LightdInfo { - version: self.data.build_info().version(), - vendor: "ZingoLabs ZainoD".to_string(), - taddr_support: true, - chain_name: blockchain_info.chain().clone(), - sapling_activation_height: sapling_activation_height.0 as u64, - consensus_branch_id, - block_height: blockchain_info.blocks().0 as u64, - git_commit: self.data.build_info().commit_hash(), - branch: self.data.build_info().branch(), - build_date: self.data.build_info().build_date(), - build_user: self.data.build_info().build_user(), - estimated_height: blockchain_info.estimated_height().0 as u64, - zcashd_build: self.data.zebra_build(), - zcashd_subversion: self.data.zebra_subversion(), - donation_address: self - .config - .common - .donation_address - .as_ref() - .map(DonationAddress::encode) - .unwrap_or_default(), - upgrade_name: nu_name.to_string(), - upgrade_height: nu_height.0 as u64, - lightwallet_protocol_version: "v0.4.0".to_string(), - }) - } - - /// Testing-only, requires lightwalletd --ping-very-insecure (do not enable in production) - /// - /// NOTE: Currently unimplemented in Zaino. - async fn ping( - &self, - _request: zaino_proto::proto::service::Duration, - ) -> Result { - Err(crate::error::StateServiceError::TonicStatusError( - tonic::Status::unimplemented( - "Ping not yet implemented. If you require this RPC please open an \ - issue or PR at the Zaino github (https://github.com/zingolabs/zaino.git).", - ), - )) - } -} - -#[cfg(test)] -mod tests { - /// Classifies the byte-level relationship between two slices. - #[derive(Debug, PartialEq)] - enum ByteRelation { - /// The slices are identical. - Equal, - /// `actual` fully byte-reversed equals `expected` (endian swap). - FullByteReversal, - /// Each byte's bits reversed maps `actual` to `expected`. - PerByteBitReversal, - /// Reversing bytes within 16-bit chunks maps `actual` to `expected`. - ChunkSwap16, - /// Reversing bytes within 32-bit chunks maps `actual` to `expected`. - ChunkSwap32, - /// Reversing bytes within 64-bit chunks maps `actual` to `expected`. - ChunkSwap64, - /// No recognized transformation. - Unrecognized, - } - - impl std::fmt::Display for ByteRelation { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::Equal => write!(f, "equal"), - Self::FullByteReversal => write!(f, "full byte-reversal (endian swap)"), - Self::PerByteBitReversal => write!(f, "per-byte bit-reversal"), - Self::ChunkSwap16 => write!(f, "16-bit pairwise byte-swap"), - Self::ChunkSwap32 => write!(f, "32-bit chunk byte-reversal"), - Self::ChunkSwap64 => write!(f, "64-bit chunk byte-reversal"), - Self::Unrecognized => write!(f, "unrecognized mismatch"), - } - } - } - - /// Applies each candidate byte transformation to `actual` and returns - /// the first that produces `expected`, or [`ByteRelation::Unrecognized`]. - // `u32::is_multiple_of` is only stable from Rust 1.87; keep `% n == 0` for our older MSRV. - #[allow(clippy::manual_is_multiple_of)] - fn classify_byte_relation(actual: &[u8], expected: &[u8]) -> ByteRelation { - if actual == expected { - return ByteRelation::Equal; - } - - let chunk_swap = |size: usize| -> Vec { - actual - .chunks(size) - .flat_map(|c| c.iter().rev()) - .copied() - .collect() - }; - - let mut reversed = actual.to_vec(); - reversed.reverse(); - if reversed == expected { - return ByteRelation::FullByteReversal; - } - - let bit_reversed: Vec = actual.iter().map(|b| b.reverse_bits()).collect(); - if bit_reversed == expected { - return ByteRelation::PerByteBitReversal; - } - - if actual.len() % 2 == 0 && chunk_swap(2) == expected { - return ByteRelation::ChunkSwap16; - } - if actual.len() % 4 == 0 && chunk_swap(4) == expected { - return ByteRelation::ChunkSwap32; - } - if actual.len() % 8 == 0 && chunk_swap(8) == expected { - return ByteRelation::ChunkSwap64; - } - - ByteRelation::Unrecognized - } - - /// Verifies that our Sapling address parsing logic produces the same - /// diversifier and diversified transmission key (pk_d) hex strings as - /// zcashd's `z_validateaddress` RPC. - /// - /// # Guarantees - /// - /// - Exercises the production `sapling_key_bytes` function directly. - /// - The 11-byte diversifier matches the zcashd-derived test vector. - /// - The 32-byte pk_d (after the endian reversal inside `sapling_key_bytes`) - /// matches the zcashd-derived test vector. - /// - If the upstream serialization changes, the failure message - /// classifies the mismatch (endian swap, bit-reversal, chunk swap, - /// or unrecognized) to aid diagnosis. - /// - /// # Non-guarantees - /// - /// - Does not prove the test vector constants themselves are correct; - /// they were captured from zcashd and are trusted as ground truth. - /// - Does not exercise the full `z_validate_address` RPC path through - /// `StateService` — only the `sapling_key_bytes` extraction function. - /// - Does not verify behavior for malformed Sapling addresses or - /// addresses on other networks (mainnet, testnet). - #[test] - fn sapling_pk_d_byte_order_matches_test_vector() { - use crate::indexer::sapling_key_bytes; - use zcash_keys::address::Address; - use zcash_protocol::consensus::NetworkType; - - // Canonical source: live-tests/clientless/src/lib.rs::rpc::json_rpc - // Tracked for DRY consolidation: https://github.com/zingolabs/zaino/issues/988 - const SAPLING_ADDRESS: &str = "zregtestsapling1jalqhycwumq3unfxlzyzcktq3n478n82k2wacvl8gwfxk6ahshkxmtp2034qj28n7gl92ka5wca"; - const EXPECTED_DIVERSIFIER: &str = "977e0b930ee6c11e4d26f8"; - const EXPECTED_PK_D: &str = - "553ef2f328096a7c2aac6dec85b76b6b9243e733dc9db2eacce3eb8c60592c88"; - - let parsed: zcash_address::ZcashAddress = SAPLING_ADDRESS.parse().unwrap(); - let converted = parsed - .convert_if_network::
(NetworkType::Regtest) - .unwrap(); - - let Address::Sapling(s) = converted else { - panic!("expected Sapling address"); - }; - - let (diversifier, pk_d) = sapling_key_bytes(&s); - - let expected_diversifier = hex::decode(EXPECTED_DIVERSIFIER).unwrap(); - let expected_pk_d = hex::decode(EXPECTED_PK_D).unwrap(); - - // Diversifier - match classify_byte_relation(&diversifier, &expected_diversifier) { - ByteRelation::Equal => {} - relation => panic!( - "diversifier mismatch.\n relation: {relation}\n actual: {}\n expected: {}", - hex::encode(diversifier), - hex::encode(expected_diversifier), - ), - } - - // pk_d (sapling_key_bytes already applies the endian reversal) - match classify_byte_relation(&pk_d, &expected_pk_d) { - ByteRelation::Equal => {} - relation => panic!( - "pk_d mismatch — upstream serialization may have changed.\ - \n relation: {relation}\n actual: {}\n expected: {}", - hex::encode(pk_d), - hex::encode(expected_pk_d), - ), - } - } - - #[test] - fn classify_byte_relation_detects_known_transforms() { - let original = [0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08]; - - assert_eq!( - classify_byte_relation(&original, &original), - ByteRelation::Equal, - ); - - let mut reversed = original.to_vec(); - reversed.reverse(); - assert_eq!( - classify_byte_relation(&original, &reversed), - ByteRelation::FullByteReversal, - ); - - let bit_rev: Vec = original.iter().map(|b| b.reverse_bits()).collect(); - assert_eq!( - classify_byte_relation(&original, &bit_rev), - ByteRelation::PerByteBitReversal, - ); - - let swapped_16: Vec = original - .chunks(2) - .flat_map(|c| c.iter().rev()) - .copied() - .collect(); - assert_eq!( - classify_byte_relation(&original, &swapped_16), - ByteRelation::ChunkSwap16, - ); - - let garbage = [0xFF; 8]; - assert_eq!( - classify_byte_relation(&original, &garbage), - ByteRelation::Unrecognized, - ); - } -} diff --git a/packages/zaino-state/src/chain_index/source/validator_connector.rs b/packages/zaino-state/src/chain_index/source/validator_connector.rs index d8b3bb50e..231b27e06 100644 --- a/packages/zaino-state/src/chain_index/source/validator_connector.rs +++ b/packages/zaino-state/src/chain_index/source/validator_connector.rs @@ -19,7 +19,7 @@ use zaino_fetch::jsonrpsee::response::{ }; use zebra_rpc::sync::init_read_state_with_syncer; -use crate::config::{CommonBackendConfig, StateServiceConfig}; +use crate::config::{CommonBackendConfig, DirectConnectionConfig}; use zebra_chain::{ amount::{Amount, NonNegative}, block::{Header, SerializedBlock}, @@ -124,26 +124,28 @@ impl ValidatorConnector { Ok((ValidatorConnector::Fetch(fetcher), info)) } - /// Spawns a `ReadStateService`-backed [`ValidatorConnector::State`] from the state - /// backend config, returning the connector plus the validator's `getinfo` response. + /// Spawns a `ReadStateService`-backed [`ValidatorConnector::State`] from the common + /// backend config plus the [`DirectConnectionConfig`], returning the connector plus + /// the validator's `getinfo` response. /// - /// Owns the JSON-RPC + Zebra chain-syncer setup that previously lived in - /// `StateService::spawn`: builds the mempool JSON-RPC fetcher, launches the syncer - /// and `ReadStateService`, then blocks until the syncer has caught up to the - /// validator's best chain tip (comparing tip *hash* as well as height so a reorg - /// mid-sync cannot report a false match). The syncer task handle is retained on the - /// `State` so [`ValidatorConnector::shutdown`] can abort it. + /// Owns the JSON-RPC + Zebra chain-syncer setup for the `Direct` connection: builds + /// the mempool JSON-RPC fetcher, launches the syncer and `ReadStateService`, then + /// blocks until the syncer has caught up to the validator's best chain tip (comparing + /// tip *hash* as well as height so a reorg mid-sync cannot report a false match). The + /// syncer task handle is retained on the `State` so [`ValidatorConnector::shutdown`] + /// can abort it. pub(crate) async fn spawn_state( - config: &StateServiceConfig, + common: &CommonBackendConfig, + direct: &DirectConnectionConfig, ) -> Result<(Self, GetInfoResponse), BlockchainSourceError> { let map_err = |error: &dyn std::fmt::Display| BlockchainSourceError::Unrecoverable(error.to_string()); let rpc_client = JsonRpSeeConnector::new_from_config_parts( - &config.common.validator_rpc_address, - config.common.validator_rpc_user.clone(), - config.common.validator_rpc_password.clone(), - config.common.validator_cookie_path.clone(), + &common.validator_rpc_address, + common.validator_rpc_user.clone(), + common.validator_rpc_password.clone(), + common.validator_cookie_path.clone(), ) .await .map_err(|error| map_err(&error))?; @@ -154,14 +156,14 @@ impl ValidatorConnector { .map_err(|error| map_err(&error))?; info!( - grpc_address = %config.validator_grpc_address, + grpc_address = %direct.validator_grpc_address, "Launching Chain Syncer" ); let (mut read_state_service, _latest_chain_tip, chain_tip_change, sync_task_handle) = init_read_state_with_syncer( - config.validator_state_config.clone(), - &config.common.network.to_zebra_network(), - config.validator_grpc_address, + direct.validator_state_config.clone(), + &common.network.to_zebra_network(), + direct.validator_grpc_address, ) .await .map_err(|error| map_err(&error))? @@ -213,7 +215,7 @@ impl ValidatorConnector { let source = ValidatorConnector::State(State { read_state_service, mempool_fetcher: rpc_client, - network: config.common.network, + network: common.network, chain_tip_change, sync_task_handle: Some(Arc::new(sync_task_handle)), }); diff --git a/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs b/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs index c287b22d3..60e3091d4 100644 --- a/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs +++ b/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs @@ -1048,3 +1048,27 @@ async fn get_difficulty() { "difficulty should be positive, got {via_index}" ); } + +/// Drives the merged [`NodeBackedIndexerServiceSubscriber`] RPC layer over a +/// `MockchainSource`, confirming the service delegates to its chain index: the +/// service's `get_latest_block` reports the same tip the mockchain was synced to. +#[tokio::test(flavor = "multi_thread")] +async fn node_backed_indexer_service_serves_latest_block() { + use crate::indexer::node_backed_indexer::NodeBackedIndexerServiceSubscriber; + use crate::LightWalletIndexer as _; + use zaino_common::{network::ActivationHeights, Network}; + + let (blocks, _indexer, index_reader, _mockchain) = + load_test_vectors_and_sync_chain_index(MockchainMode::Static).await; + + let expected_tip = (blocks.len() as u32) - 1; + wait_for_indexer_tip(&index_reader, expected_tip).await; + + let service = NodeBackedIndexerServiceSubscriber::new_for_test( + index_reader, + Network::Regtest(ActivationHeights::default()), + ); + + let latest = service.get_latest_block().await.unwrap(); + assert_eq!(latest.height, expected_tip as u64); +} diff --git a/packages/zaino-state/src/config.rs b/packages/zaino-state/src/config.rs index 0083dbcd4..522c2be13 100644 --- a/packages/zaino-state/src/config.rs +++ b/packages/zaino-state/src/config.rs @@ -42,31 +42,37 @@ impl std::fmt::Display for DonationAddress { } } -/// Type of backend to be used. +/// How the [`NodeBackedIndexerService`](crate::NodeBackedIndexerService) connects to its +/// validator to source blockchain data. /// -/// Determines how Zaino fetches blockchain data from the validator. -#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, serde::Deserialize, serde::Serialize)] -#[serde(rename_all = "lowercase")] -pub enum BackendType { - /// Uses Zebra's ReadStateService for direct state access. - /// - /// More efficient but requires running on the same machine as Zebra. - State, - /// Uses JSON-RPC client to fetch data. +/// Carries the connection-specific configuration inline: the `Direct` variant owns the +/// Zebra `ReadStateService` settings, while `Rpc` needs only the JSON-RPC connection bits +/// already held in [`CommonBackendConfig`]. +#[derive(Debug, Clone)] +pub enum ValidatorConnectionType { + /// JSON-RPC connection (formerly `Fetch`). /// /// Compatible with Zcashd, Zebra, or another Zaino instance. - #[default] - Fetch, + Rpc, + /// Direct Zebra `ReadStateService` connection (formerly `State`). + /// + /// More efficient but requires running alongside a Zebra whose state DB and gRPC + /// sync endpoint we own. + Direct(DirectConnectionConfig), } -/// Unified backend configuration enum. +/// Connection parameters for the [`ValidatorConnectionType::Direct`] backend. +/// +/// Bundles the Zebra `ReadStateService` DB config with the gRPC sync endpoint used to +/// drive it, so the whole "direct connection" description travels as one value. #[derive(Debug, Clone)] -#[allow(deprecated)] -pub enum BackendConfig { - /// StateService config. - State(StateServiceConfig), - /// Fetchservice config. - Fetch(FetchServiceConfig), +pub struct DirectConnectionConfig { + /// Zebra [`zebra_state::ReadStateService`] config data (DB cache dir etc.). + pub validator_state_config: zebra_state::Config, + /// Validator gRPC address (requires ip:port format for Zebra state sync). + pub validator_grpc_address: std::net::SocketAddr, + /// Whether validator cookie authentication is enabled. + pub validator_cookie_auth: bool, } /// Configuration shared by every backend variant. @@ -109,29 +115,55 @@ pub struct CommonBackendConfig { pub indexer_version: String, } -/// Holds config data for [crate::StateService]. +impl CommonBackendConfig { + /// Builds a [`CommonBackendConfig`], applying the default RPC user/password + /// placeholder (`"xxxxxx"`) and this crate's `CARGO_PKG_VERSION` as the + /// indexer version. Shared by the [`NodeBackedIndexerServiceConfig`] constructors. + #[allow(clippy::too_many_arguments)] + pub fn new( + validator_rpc_address: String, + validator_cookie_path: Option, + validator_rpc_user: Option, + validator_rpc_password: Option, + service: ServiceConfig, + storage: StorageConfig, + ephemeral_finalised_state: bool, + network: Network, + donation_address: Option, + ) -> Self { + CommonBackendConfig { + validator_rpc_address, + validator_cookie_path, + validator_rpc_user: validator_rpc_user.unwrap_or_else(|| "xxxxxx".to_string()), + validator_rpc_password: validator_rpc_password.unwrap_or_else(|| "xxxxxx".to_string()), + service, + storage, + ephemeral_finalised_state, + network, + donation_address, + indexer_version: env!("CARGO_PKG_VERSION").to_string(), + } + } +} + +/// Holds config data for [`crate::NodeBackedIndexerService`]. +/// +/// Replaces the former per-backend `FetchServiceConfig` / `StateServiceConfig`: the +/// shared bits live in [`CommonBackendConfig`] and the backend-specific bits (only the +/// `Direct` backend has any) live in [`ValidatorConnectionType`]. #[derive(Debug, Clone)] -// #[deprecated] -pub struct StateServiceConfig { - /// Settings shared with [`FetchServiceConfig`]. +pub struct NodeBackedIndexerServiceConfig { + /// Connection-independent settings (validator RPC, storage, network, ...). pub common: CommonBackendConfig, - /// Zebra [`zebra_state::ReadStateService`] config data - pub validator_state_config: zebra_state::Config, - /// Validator gRPC address (requires ip:port format for Zebra state sync). - pub validator_grpc_address: std::net::SocketAddr, - /// Validator cookie auth. - pub validator_cookie_auth: bool, + /// Which validator connection to use, and its connection-specific config. + pub connection: ValidatorConnectionType, } -#[allow(deprecated)] -impl StateServiceConfig { - /// Returns a new instance of [`StateServiceConfig`]. +impl NodeBackedIndexerServiceConfig { + /// Returns a JSON-RPC (`Rpc`) service config (formerly `FetchServiceConfig::new`). #[allow(clippy::too_many_arguments)] - pub fn new( - validator_state_config: zebra_state::Config, + pub fn new_rpc( validator_rpc_address: String, - validator_grpc_address: std::net::SocketAddr, - validator_cookie_auth: bool, validator_cookie_path: Option, validator_rpc_user: Option, validator_rpc_password: Option, @@ -141,44 +173,30 @@ impl StateServiceConfig { network: Network, donation_address: Option, ) -> Self { - tracing::trace!( - activations = ?network.to_zebra_network().full_activation_list(), - "state service expecting NU activations" - ); - StateServiceConfig { - common: CommonBackendConfig { + NodeBackedIndexerServiceConfig { + common: CommonBackendConfig::new( validator_rpc_address, validator_cookie_path, - validator_rpc_user: validator_rpc_user.unwrap_or("xxxxxx".to_string()), - validator_rpc_password: validator_rpc_password.unwrap_or("xxxxxx".to_string()), + validator_rpc_user, + validator_rpc_password, service, storage, ephemeral_finalised_state, network, donation_address, - indexer_version: env!("CARGO_PKG_VERSION").to_string(), - }, - validator_state_config, - validator_grpc_address, - validator_cookie_auth, + ), + connection: ValidatorConnectionType::Rpc, } } -} - -/// Holds config data for [crate::FetchService]. -#[derive(Debug, Clone)] -#[deprecated] -pub struct FetchServiceConfig { - /// Settings shared with [`StateServiceConfig`]. - pub common: CommonBackendConfig, -} -#[allow(deprecated)] -impl FetchServiceConfig { - /// Returns a new instance of [`FetchServiceConfig`]. + /// Returns a direct-`ReadStateService` (`Direct`) service config + /// (formerly `StateServiceConfig::new`). #[allow(clippy::too_many_arguments)] - pub fn new( + pub fn new_direct( + validator_state_config: zebra_state::Config, validator_rpc_address: String, + validator_grpc_address: std::net::SocketAddr, + validator_cookie_auth: bool, validator_cookie_path: Option, validator_rpc_user: Option, validator_rpc_password: Option, @@ -188,19 +206,27 @@ impl FetchServiceConfig { network: Network, donation_address: Option, ) -> Self { - FetchServiceConfig { - common: CommonBackendConfig { + tracing::trace!( + activations = ?network.to_zebra_network().full_activation_list(), + "direct-connection service expecting NU activations" + ); + NodeBackedIndexerServiceConfig { + common: CommonBackendConfig::new( validator_rpc_address, validator_cookie_path, - validator_rpc_user: validator_rpc_user.unwrap_or("xxxxxx".to_string()), - validator_rpc_password: validator_rpc_password.unwrap_or("xxxxxx".to_string()), + validator_rpc_user, + validator_rpc_password, service, storage, ephemeral_finalised_state, network, donation_address, - indexer_version: env!("CARGO_PKG_VERSION").to_string(), - }, + ), + connection: ValidatorConnectionType::Direct(DirectConnectionConfig { + validator_state_config, + validator_grpc_address, + validator_cookie_auth, + }), } } } @@ -249,17 +275,9 @@ impl From for ChainIndexConfig { } } -#[allow(deprecated)] -impl From for ChainIndexConfig { - fn from(value: StateServiceConfig) -> Self { - value.common.into() - } -} - -#[allow(deprecated)] -impl From for ChainIndexConfig { - fn from(value: FetchServiceConfig) -> Self { - value.common.into() +impl From<&NodeBackedIndexerServiceConfig> for ChainIndexConfig { + fn from(value: &NodeBackedIndexerServiceConfig) -> Self { + value.common.clone().into() } } diff --git a/packages/zaino-state/src/error.rs b/packages/zaino-state/src/error.rs index 839c17045..2093ca68b 100644 --- a/packages/zaino-state/src/error.rs +++ b/packages/zaino-state/src/error.rs @@ -10,11 +10,11 @@ use std::{any::type_name, fmt::Display}; use zaino_fetch::jsonrpsee::connector::RpcRequestError; use zaino_proto::proto::utils::GetBlockRangeError; -/// Errors related to the `StateService`. -// #[deprecated] +/// Errors returned by the [`NodeBackedIndexerService`](crate::NodeBackedIndexerService) +/// subscriber's `ZcashIndexer` / `LightWalletIndexer` methods. #[derive(Debug, thiserror::Error)] #[allow(clippy::result_large_err)] -pub enum StateServiceError { +pub enum NodeBackedIndexerServiceError { /// Critical Errors, Restart Zaino. #[error("Critical error: {0}")] Critical(String), @@ -88,7 +88,7 @@ pub enum StateServiceError { UnavailableNotSyncedEnough, } -impl From for StateServiceError { +impl From for NodeBackedIndexerServiceError { fn from(value: GetBlockRangeError) -> Self { match value { GetBlockRangeError::StartHeightOutOfRange => { @@ -115,55 +115,59 @@ impl From for StateServiceError { } #[allow(deprecated)] -impl From for tonic::Status { - fn from(error: StateServiceError) -> Self { +impl From for tonic::Status { + fn from(error: NodeBackedIndexerServiceError) -> Self { match error { - StateServiceError::Critical(message) => tonic::Status::internal(message), - StateServiceError::Custom(message) => tonic::Status::internal(message), - StateServiceError::JoinError(err) => { + NodeBackedIndexerServiceError::Critical(message) => tonic::Status::internal(message), + NodeBackedIndexerServiceError::Custom(message) => tonic::Status::internal(message), + NodeBackedIndexerServiceError::JoinError(err) => { tonic::Status::internal(format!("Join error: {err}")) } - StateServiceError::JsonRpcConnectorError(err) => { + NodeBackedIndexerServiceError::JsonRpcConnectorError(err) => { tonic::Status::internal(format!("JsonRpcConnector error: {err}")) } - StateServiceError::RpcError(err) => { + NodeBackedIndexerServiceError::RpcError(err) => { tonic::Status::internal(format!("RPC error: {err:?}")) } - StateServiceError::ChainIndexError(err) => match err.kind { + NodeBackedIndexerServiceError::ChainIndexError(err) => match err.kind { ChainIndexErrorKind::InternalServerError => tonic::Status::internal(err.message), ChainIndexErrorKind::InvalidSnapshot => { tonic::Status::failed_precondition(err.message) } }, - StateServiceError::BlockCacheError(err) => { + NodeBackedIndexerServiceError::BlockCacheError(err) => { tonic::Status::internal(format!("BlockCache error: {err:?}")) } - StateServiceError::MempoolError(err) => { + NodeBackedIndexerServiceError::MempoolError(err) => { tonic::Status::internal(format!("Mempool error: {err:?}")) } - StateServiceError::TonicStatusError(err) => err, - StateServiceError::SerializationError(err) => { + NodeBackedIndexerServiceError::TonicStatusError(err) => err, + NodeBackedIndexerServiceError::SerializationError(err) => { tonic::Status::internal(format!("Serialization error: {err}")) } - StateServiceError::TryFromIntError(err) => { + NodeBackedIndexerServiceError::TryFromIntError(err) => { tonic::Status::internal(format!("Integer conversion error: {err}")) } - StateServiceError::IoError(err) => tonic::Status::internal(format!("IO error: {err}")), - StateServiceError::Generic(err) => { + NodeBackedIndexerServiceError::IoError(err) => { + tonic::Status::internal(format!("IO error: {err}")) + } + NodeBackedIndexerServiceError::Generic(err) => { tonic::Status::internal(format!("Generic error: {err}")) } - ref err @ StateServiceError::ZebradVersionMismatch { .. } => { + ref err @ NodeBackedIndexerServiceError::ZebradVersionMismatch { .. } => { tonic::Status::internal(err.to_string()) } - StateServiceError::UnhandledRpcError(e) => tonic::Status::internal(e.to_string()), - StateServiceError::UnavailableNotSyncedEnough => { + NodeBackedIndexerServiceError::UnhandledRpcError(e) => { + tonic::Status::internal(e.to_string()) + } + NodeBackedIndexerServiceError::UnavailableNotSyncedEnough => { tonic::Status::failed_precondition("zaino not yet synced".to_string()) } } } } -impl From> for StateServiceError { +impl From> for NodeBackedIndexerServiceError { fn from(value: RpcRequestError) -> Self { match value { RpcRequestError::Transport(transport_error) => { @@ -184,122 +188,6 @@ impl From> for StateServiceError { } } -/// Errors related to the `FetchService`. -#[deprecated] -#[derive(Debug, thiserror::Error)] -pub enum FetchServiceError { - /// Critical Errors, Restart Zaino. - #[error("Critical error: {0}")] - Critical(String), - - /// Error from JsonRpcConnector. - #[error("JsonRpcConnector error: {0}")] - JsonRpcConnectorError(#[from] zaino_fetch::jsonrpsee::error::TransportError), - - /// Chain index error. - #[error("Chain index error: {0}")] - ChainIndexError(#[from] ChainIndexError), - - /// RPC error in compatibility with zcashd. - #[error("RPC error: {0:?}")] - RpcError(#[from] zaino_fetch::jsonrpsee::connector::RpcError), - - /// Tonic gRPC error. - #[error("Tonic status error: {0}")] - TonicStatusError(#[from] tonic::Status), - - /// Serialization error. - #[error("Serialization error: {0}")] - SerializationError(#[from] zebra_chain::serialization::SerializationError), - #[error("Zaino has not synced high enough to serve this data")] - /// Zaino has not yet synced. - UnavailableNotSyncedEnough, -} - -impl From for tonic::Status { - fn from(error: FetchServiceError) -> Self { - match error { - FetchServiceError::Critical(message) => tonic::Status::internal(message), - FetchServiceError::JsonRpcConnectorError(err) => { - tonic::Status::internal(format!("JsonRpcConnector error: {err}")) - } - FetchServiceError::ChainIndexError(err) => match err.kind { - ChainIndexErrorKind::InternalServerError => tonic::Status::internal(err.message), - ChainIndexErrorKind::InvalidSnapshot => { - tonic::Status::failed_precondition(err.message) - } - }, - FetchServiceError::RpcError(err) => { - tonic::Status::internal(format!("RPC error: {err:?}")) - } - FetchServiceError::TonicStatusError(err) => err, - FetchServiceError::SerializationError(err) => { - tonic::Status::internal(format!("Serialization error: {err}")) - } - FetchServiceError::UnavailableNotSyncedEnough => { - tonic::Status::failed_precondition("zaino not yet synced".to_string()) - } - } - } -} - -impl From> for FetchServiceError { - fn from(value: RpcRequestError) -> Self { - match value { - RpcRequestError::Transport(transport_error) => { - FetchServiceError::JsonRpcConnectorError(transport_error) - } - RpcRequestError::JsonRpc(error) => { - FetchServiceError::Critical(format!("argument failed to serialze: {error}")) - } - RpcRequestError::InternalUnrecoverable(e) => { - FetchServiceError::Critical(format!("Internal unrecoverable error: {e}")) - } - RpcRequestError::ServerWorkQueueFull => FetchServiceError::Critical( - "Server queue full. Handling for this not yet implemented".to_string(), - ), - RpcRequestError::Method(e) => FetchServiceError::Critical(format!( - "unhandled rpc-specific {} error: {}", - type_name::(), - e.to_string() - )), - RpcRequestError::UnexpectedErrorResponse(error) => { - FetchServiceError::Critical(format!( - "unhandled rpc-specific {} error: {}", - type_name::(), - error - )) - } - } - } -} - -impl From for FetchServiceError { - fn from(value: GetBlockRangeError) -> Self { - match value { - GetBlockRangeError::StartHeightOutOfRange => { - FetchServiceError::TonicStatusError(tonic::Status::out_of_range( - "Error: Start height out of range. Failed to convert to u32.", - )) - } - GetBlockRangeError::NoStartHeightProvided => FetchServiceError::TonicStatusError( - tonic::Status::out_of_range("Error: No start height given"), - ), - GetBlockRangeError::EndHeightOutOfRange => { - FetchServiceError::TonicStatusError(tonic::Status::out_of_range( - "Error: End height out of range. Failed to convert to u32.", - )) - } - GetBlockRangeError::NoEndHeightProvided => FetchServiceError::TonicStatusError( - tonic::Status::out_of_range("Error: No end height given."), - ), - GetBlockRangeError::PoolTypeArgumentError(_) => FetchServiceError::TonicStatusError( - tonic::Status::invalid_argument("Error: invalid pool type"), - ), - } - } -} - /// These aren't the best conversions, but the MempoolError should go away /// in favor of a new type with the new chain cache is complete impl From> for MempoolError { diff --git a/packages/zaino-state/src/indexer.rs b/packages/zaino-state/src/indexer.rs index d2ffca07b..a2ef9d61d 100644 --- a/packages/zaino-state/src/indexer.rs +++ b/packages/zaino-state/src/indexer.rs @@ -1,5 +1,10 @@ -//! Holds the Indexer trait containing the zcash RPC definitions served by zaino -//! and generic wrapper structs for the various backend options available. +//! Zaino's indexer frontend: the `ZcashIndexer` / `LightWalletIndexer` RPC trait +//! definitions served by zaino, the generic [`IndexerService`] / [`IndexerSubscriber`] +//! wrappers, and the concrete [`node_backed_indexer::NodeBackedIndexerService`] — the +//! single validator-backed service (JSON-RPC `Rpc` or direct `ReadStateService` +//! connection, selected at runtime). + +pub(crate) mod node_backed_indexer; use crate::SendFut; use tokio::{sync::mpsc, time::timeout}; @@ -48,10 +53,10 @@ use crate::{ AddressStream, CompactBlockStream, CompactTransactionStream, RawTransactionStream, SubtreeRootReplyStream, UtxoReplyStream, }, - BackendType, }; -/// Wrapper Struct for a ZainoState chain-fetch service (StateService, FetchService) +/// Wrapper struct for a ZainoState chain-fetch service (currently the single +/// [`node_backed_indexer::NodeBackedIndexerService`]). /// /// The future plan is to also add a TonicService and DarksideService to this to enable /// wallets to use a single unified chain fetch service. @@ -92,9 +97,6 @@ where /// Implementors automatically gain [`Liveness`](zaino_common::probing::Liveness) and /// [`Readiness`](zaino_common::probing::Readiness) via the [`Status`] supertrait. pub trait ZcashService: Sized + Status { - /// Backend type. Read state or fetch service. - const BACKEND_TYPE: BackendType; - /// A subscriber to the service, used to fetch chain data. type Subscriber: Clone + ZcashIndexer + LightWalletIndexer + Status; @@ -113,7 +115,8 @@ pub trait ZcashService: Sized + Status { fn close(&mut self); } -/// Wrapper Struct for a ZainoState chain-fetch service subscriber (StateServiceSubscriber, FetchServiceSubscriber) +/// Wrapper struct for a ZainoState chain-fetch service subscriber (currently the single +/// [`node_backed_indexer::NodeBackedIndexerServiceSubscriber`]). /// /// The future plan is to also add a TonicServiceSubscriber and DarksideServiceSubscriber to this to enable wallets to use a single unified chain fetch service. #[derive(Clone)] @@ -1154,10 +1157,73 @@ pub(crate) fn build_subtrees_by_index_response( .into() } +fn latest_network_upgrade( + upgrades: &indexmap::IndexMap< + zebra_rpc::methods::ConsensusBranchIdHex, + zebra_rpc::methods::NetworkUpgradeInfo, + >, +) -> Result<&zebra_rpc::methods::NetworkUpgradeInfo, tonic::Status> { + upgrades.last().map(|(_, upgrade)| upgrade).ok_or_else(|| { + tonic::Status::failed_precondition("validator returned no network upgrade metadata") + }) +} + +/// Maximum number of addresses a single `get_address_utxos` / `get_address_utxos_stream` +/// request may carry. +/// +/// The service resolves the full backend UTXO set before applying `max_entries` / +/// `start_height` (issue #974). A complete pushdown fix needs upstream interface changes +/// the caller-supplied entry cap cannot reach today, so until then this bounds the one +/// input the service controls locally: the address fan-out. It stops an unauthenticated +/// caller forcing an unbounded number of backend address lookups in a single request, and +/// is set well above realistic wallet usage. +/// +/// TODO: make this deployment-configurable rather than a fixed constant. +const UTXO_MAX_ADDRESSES: usize = 1000; + +/// Reject a `get_address_utxos` request whose address list exceeds [`UTXO_MAX_ADDRESSES`]. +/// +/// `max_entries` bounds the response size, not the backend work; this guard bounds the +/// address fan-out, the part the service can cap without upstream changes. +fn validate_utxo_address_count(count: usize) -> Result<(), tonic::Status> { + if count > UTXO_MAX_ADDRESSES { + return Err(tonic::Status::invalid_argument(format!( + "Error: too many addresses in request: {count} exceeds the maximum of {UTXO_MAX_ADDRESSES}." + ))); + } + Ok(()) +} + #[cfg(test)] mod tests { use super::*; + #[test] + fn latest_network_upgrade_rejects_empty_metadata() { + let upgrades = indexmap::IndexMap::new(); + let err = super::latest_network_upgrade(&upgrades).expect_err("empty upgrades must fail"); + + assert_eq!(err.code(), tonic::Code::FailedPrecondition); + assert_eq!( + err.message(), + "validator returned no network upgrade metadata" + ); + } + + #[test] + fn utxo_address_count_within_limit_is_accepted() { + assert!(super::validate_utxo_address_count(0).is_ok()); + assert!(super::validate_utxo_address_count(super::UTXO_MAX_ADDRESSES).is_ok()); + } + + #[test] + fn utxo_address_count_over_limit_is_rejected() { + let err = super::validate_utxo_address_count(super::UTXO_MAX_ADDRESSES + 1) + .expect_err("over-limit address count must fail"); + + assert_eq!(err.code(), tonic::Code::InvalidArgument); + } + #[test] fn build_subtrees_by_index_response_hex_encodes_roots() { let roots = vec![([0xabu8; 32], 100u32), ([0xcdu8; 32], 200u32)]; @@ -1177,4 +1243,188 @@ mod tests { assert_eq!(subtrees[1].root, hex::encode([0xcdu8; 32])); assert_eq!(subtrees[1].end_height, Height(200)); } + + /// Classifies the byte-level relationship between two slices. + #[derive(Debug, PartialEq)] + enum ByteRelation { + /// The slices are identical. + Equal, + /// `actual` fully byte-reversed equals `expected` (endian swap). + FullByteReversal, + /// Each byte's bits reversed maps `actual` to `expected`. + PerByteBitReversal, + /// Reversing bytes within 16-bit chunks maps `actual` to `expected`. + ChunkSwap16, + /// Reversing bytes within 32-bit chunks maps `actual` to `expected`. + ChunkSwap32, + /// Reversing bytes within 64-bit chunks maps `actual` to `expected`. + ChunkSwap64, + /// No recognized transformation. + Unrecognized, + } + + impl std::fmt::Display for ByteRelation { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Equal => write!(f, "equal"), + Self::FullByteReversal => write!(f, "full byte-reversal (endian swap)"), + Self::PerByteBitReversal => write!(f, "per-byte bit-reversal"), + Self::ChunkSwap16 => write!(f, "16-bit pairwise byte-swap"), + Self::ChunkSwap32 => write!(f, "32-bit chunk byte-reversal"), + Self::ChunkSwap64 => write!(f, "64-bit chunk byte-reversal"), + Self::Unrecognized => write!(f, "unrecognized mismatch"), + } + } + } + + /// Applies each candidate byte transformation to `actual` and returns + /// the first that produces `expected`, or [`ByteRelation::Unrecognized`]. + // `u32::is_multiple_of` is only stable from Rust 1.87; keep `% n == 0` for our older MSRV. + #[allow(clippy::manual_is_multiple_of)] + fn classify_byte_relation(actual: &[u8], expected: &[u8]) -> ByteRelation { + if actual == expected { + return ByteRelation::Equal; + } + + let chunk_swap = |size: usize| -> Vec { + actual + .chunks(size) + .flat_map(|c| c.iter().rev()) + .copied() + .collect() + }; + + let mut reversed = actual.to_vec(); + reversed.reverse(); + if reversed == expected { + return ByteRelation::FullByteReversal; + } + + let bit_reversed: Vec = actual.iter().map(|b| b.reverse_bits()).collect(); + if bit_reversed == expected { + return ByteRelation::PerByteBitReversal; + } + + if actual.len() % 2 == 0 && chunk_swap(2) == expected { + return ByteRelation::ChunkSwap16; + } + if actual.len() % 4 == 0 && chunk_swap(4) == expected { + return ByteRelation::ChunkSwap32; + } + if actual.len() % 8 == 0 && chunk_swap(8) == expected { + return ByteRelation::ChunkSwap64; + } + + ByteRelation::Unrecognized + } + + /// Verifies that our Sapling address parsing logic produces the same + /// diversifier and diversified transmission key (pk_d) hex strings as + /// zcashd's `z_validateaddress` RPC. + /// + /// # Guarantees + /// + /// - Exercises the production `sapling_key_bytes` function directly. + /// - The 11-byte diversifier matches the zcashd-derived test vector. + /// - The 32-byte pk_d (after the endian reversal inside `sapling_key_bytes`) + /// matches the zcashd-derived test vector. + /// - If the upstream serialization changes, the failure message + /// classifies the mismatch (endian swap, bit-reversal, chunk swap, + /// or unrecognized) to aid diagnosis. + /// + /// # Non-guarantees + /// + /// - Does not prove the test vector constants themselves are correct; + /// they were captured from zcashd and are trusted as ground truth. + /// - Does not exercise the full `z_validate_address` RPC path through + /// `StateService` — only the `sapling_key_bytes` extraction function. + /// - Does not verify behavior for malformed Sapling addresses or + /// addresses on other networks (mainnet, testnet). + #[test] + fn sapling_pk_d_byte_order_matches_test_vector() { + use crate::indexer::sapling_key_bytes; + use zcash_keys::address::Address; + use zcash_protocol::consensus::NetworkType; + + // Canonical source: live-tests/clientless/src/lib.rs::rpc::json_rpc + // Tracked for DRY consolidation: https://github.com/zingolabs/zaino/issues/988 + const SAPLING_ADDRESS: &str = "zregtestsapling1jalqhycwumq3unfxlzyzcktq3n478n82k2wacvl8gwfxk6ahshkxmtp2034qj28n7gl92ka5wca"; + const EXPECTED_DIVERSIFIER: &str = "977e0b930ee6c11e4d26f8"; + const EXPECTED_PK_D: &str = + "553ef2f328096a7c2aac6dec85b76b6b9243e733dc9db2eacce3eb8c60592c88"; + + let parsed: zcash_address::ZcashAddress = SAPLING_ADDRESS.parse().unwrap(); + let converted = parsed + .convert_if_network::
(NetworkType::Regtest) + .unwrap(); + + let Address::Sapling(s) = converted else { + panic!("expected Sapling address"); + }; + + let (diversifier, pk_d) = sapling_key_bytes(&s); + + let expected_diversifier = hex::decode(EXPECTED_DIVERSIFIER).unwrap(); + let expected_pk_d = hex::decode(EXPECTED_PK_D).unwrap(); + + // Diversifier + match classify_byte_relation(&diversifier, &expected_diversifier) { + ByteRelation::Equal => {} + relation => panic!( + "diversifier mismatch.\n relation: {relation}\n actual: {}\n expected: {}", + hex::encode(diversifier), + hex::encode(expected_diversifier), + ), + } + + // pk_d (sapling_key_bytes already applies the endian reversal) + match classify_byte_relation(&pk_d, &expected_pk_d) { + ByteRelation::Equal => {} + relation => panic!( + "pk_d mismatch — upstream serialization may have changed.\ + \n relation: {relation}\n actual: {}\n expected: {}", + hex::encode(pk_d), + hex::encode(expected_pk_d), + ), + } + } + + #[test] + fn classify_byte_relation_detects_known_transforms() { + let original = [0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08]; + + assert_eq!( + classify_byte_relation(&original, &original), + ByteRelation::Equal, + ); + + let mut reversed = original.to_vec(); + reversed.reverse(); + assert_eq!( + classify_byte_relation(&original, &reversed), + ByteRelation::FullByteReversal, + ); + + let bit_rev: Vec = original.iter().map(|b| b.reverse_bits()).collect(); + assert_eq!( + classify_byte_relation(&original, &bit_rev), + ByteRelation::PerByteBitReversal, + ); + + let swapped_16: Vec = original + .chunks(2) + .flat_map(|c| c.iter().rev()) + .copied() + .collect(); + assert_eq!( + classify_byte_relation(&original, &swapped_16), + ByteRelation::ChunkSwap16, + ); + + let garbage = [0xFF; 8]; + assert_eq!( + classify_byte_relation(&original, &garbage), + ByteRelation::Unrecognized, + ); + } } diff --git a/packages/zaino-state/src/backends/fetch.rs b/packages/zaino-state/src/indexer/node_backed_indexer.rs similarity index 82% rename from packages/zaino-state/src/backends/fetch.rs rename to packages/zaino-state/src/indexer/node_backed_indexer.rs index 273851768..1619a3693 100644 --- a/packages/zaino-state/src/backends/fetch.rs +++ b/packages/zaino-state/src/indexer/node_backed_indexer.rs @@ -57,9 +57,15 @@ use zaino_proto::proto::{ #[allow(deprecated)] use crate::{ chain_index::chain_tips_from_nonfinalized_snapshot, - chain_index::{source::ValidatorConnector, types}, - config::{DonationAddress, FetchServiceConfig}, - error::FetchServiceError, + chain_index::{ + source::{BlockchainSource, ValidatorConnector}, + types, + }, + config::{ + CommonBackendConfig, DonationAddress, NodeBackedIndexerServiceConfig, + ValidatorConnectionType, + }, + error::NodeBackedIndexerServiceError, indexer::{ handle_raw_transaction, IndexerSubscriber, LightWalletIndexer, ZcashIndexer, ZcashService, }, @@ -69,60 +75,80 @@ use crate::{ UtxoReplyStream, }, utils::{get_build_info, ServiceMetadata}, - BackendType, }; use crate::{ chain_index::{non_finalised_state::ChainIndexSnapshot, NonFinalizedSnapshot}, ChainIndex, ChainIndexRpcExt, NodeBackedChainIndex, NodeBackedChainIndexSubscriber, }; -/// Chain fetch service backed by Zcashd's JsonRPC engine. +/// A single node-backed chain-fetch + tx-submission service, generic over its +/// [`BlockchainSource`]. /// -/// This service is a central service, [`FetchServiceSubscriber`] should be created to fetch data. -/// This is done to enable large numbers of concurrent subscribers without significant slowdowns. +/// Replaces the former `FetchService` / `StateService` split: the JSON-RPC (`Rpc`) and +/// direct-`ReadStateService` (`Direct`) backends are now one type selected at runtime +/// via [`NodeBackedIndexerServiceConfig`]. Production instantiates the default +/// `Source = ValidatorConnector` (which itself carries the `Rpc`/`Direct` arm); tests +/// may instantiate over a mock source. /// -/// NOTE: We currently do not implement clone for chain fetch services as this service is responsible for maintaining and closing its child processes. -/// ServiceSubscribers are used to create separate chain fetch processes while allowing central state processes to be managed in a single place. -/// If we want the ability to clone Service all JoinHandle's should be converted to Arc\. +/// This service is a central service — create a [`NodeBackedIndexerServiceSubscriber`] +/// (via [`ZcashService::get_subscriber`]) to fetch data. This lets large numbers of +/// concurrent subscribers run without contending on the single central process. +/// +/// NOTE: We do not implement `Clone` for the central service: it owns and closes its +/// child processes. Subscribers are the clone-safe read handles. #[derive(Debug)] -#[deprecated = "Will be eventually replaced by `BlockchainSource`"] -pub struct FetchService { +pub struct NodeBackedIndexerService { /// Core indexer. - indexer: NodeBackedChainIndex, + indexer: NodeBackedChainIndex, /// Service metadata. data: ServiceMetadata, - - /// StateService config data. - #[allow(deprecated)] - config: FetchServiceConfig, + /// Connection-independent config (validator RPC, storage, network, ...). + config: CommonBackendConfig, } -#[allow(deprecated)] -impl Status for FetchService { +impl Status for NodeBackedIndexerService { fn status(&self) -> StatusType { self.indexer.status() } } -#[allow(deprecated)] -impl ZcashService for FetchService { - const BACKEND_TYPE: BackendType = BackendType::Fetch; +impl NodeBackedIndexerService { + /// Tears down the indexer (sync loop, finalised DB, mempool, and any source-owned + /// syncer task) from a synchronous context. Shared by [`ZcashService::close`] and + /// [`Drop`]. + fn shutdown_blocking(&self) { + tokio::task::block_in_place(|| { + tokio::runtime::Handle::current().block_on(async { + let _ = self.indexer.shutdown().await; + }); + }); + } +} - type Subscriber = FetchServiceSubscriber; - type Config = FetchServiceConfig; +impl ZcashService for NodeBackedIndexerService { + type Subscriber = NodeBackedIndexerServiceSubscriber; + type Config = NodeBackedIndexerServiceConfig; - /// Initializes a new FetchService instance and starts sync process. - #[instrument(name = "FetchService::spawn", skip(config), fields(network = %config.common.network))] - async fn spawn(config: FetchServiceConfig) -> Result { + /// Initializes a new [`NodeBackedIndexerService`] and starts its sync process. + #[instrument(name = "NodeBackedIndexerService::spawn", skip(config), fields(network = %config.common.network))] + async fn spawn( + config: NodeBackedIndexerServiceConfig, + ) -> Result { info!( rpc_address = %config.common.validator_rpc_address, network = %config.common.network, - "Launching Fetch Service" + "Launching NodeBackedIndexerService" ); - let (source, zebra_build_data) = ValidatorConnector::spawn_fetch(&config.common) - .await - .map_err(|error| FetchServiceError::Critical(error.to_string()))?; + // Select the validator connection from config; both arms return the built source + // plus the validator `getinfo` used for service metadata. + let (source, zebra_build_data) = match &config.connection { + ValidatorConnectionType::Rpc => ValidatorConnector::spawn_fetch(&config.common).await, + ValidatorConnectionType::Direct(direct) => { + ValidatorConnector::spawn_state(&config.common, direct).await + } + } + .map_err(|error| NodeBackedIndexerServiceError::Critical(error.to_string()))?; let data = ServiceMetadata::new( get_build_info(config.common.indexer_version.clone()), @@ -132,22 +158,22 @@ impl ZcashService for FetchService { ); info!(build = %data.zebra_build(), subversion = %data.zebra_subversion(), "Connected to Zcash node"); - let indexer = NodeBackedChainIndex::new(source, config.clone().into()) + let indexer = NodeBackedChainIndex::new(source, (&config).into()) .await - .map_err(|error| FetchServiceError::Critical(error.to_string()))?; + .map_err(|error| NodeBackedIndexerServiceError::Critical(error.to_string()))?; - let fetch_service = Self { + let service = Self { indexer, data, - config, + config: config.common, }; // wait for sync to complete, return error on sync fail. loop { - match fetch_service.status() { + match service.status() { StatusType::Ready | StatusType::Closing => break, StatusType::CriticalError => { - return Err(FetchServiceError::Critical( + return Err(NodeBackedIndexerServiceError::Critical( "ChainIndex initial sync failed, check full log for details.".to_string(), )); } @@ -157,57 +183,51 @@ impl ZcashService for FetchService { } } - Ok(fetch_service) + Ok(service) } - /// Returns a [`FetchServiceSubscriber`]. - fn get_subscriber(&self) -> IndexerSubscriber { - IndexerSubscriber::new(FetchServiceSubscriber { + /// Returns a [`NodeBackedIndexerServiceSubscriber`]. + fn get_subscriber( + &self, + ) -> IndexerSubscriber> { + IndexerSubscriber::new(NodeBackedIndexerServiceSubscriber { indexer: self.indexer.subscriber(), data: self.data.clone(), config: self.config.clone(), }) } - /// Shuts down the StateService. + /// Shuts down the service, tearing down its indexer (sync loop, finalised DB, + /// mempool, and any source-owned syncer task). fn close(&mut self) { - tokio::task::block_in_place(|| { - tokio::runtime::Handle::current().block_on(async { - let _ = self.indexer.shutdown().await; - }); - }); + self.shutdown_blocking(); } } -#[allow(deprecated)] -impl Drop for FetchService { +impl Drop for NodeBackedIndexerService { fn drop(&mut self) { - self.close() + self.shutdown_blocking(); } } -/// A fetch service subscriber. -/// -/// Subscribers should be +/// A clone-safe, read-only subscriber to a [`NodeBackedIndexerService`]. #[derive(Debug, Clone)] -#[allow(deprecated)] -pub struct FetchServiceSubscriber { +pub struct NodeBackedIndexerServiceSubscriber { /// Core indexer. - pub indexer: NodeBackedChainIndexSubscriber, + pub indexer: NodeBackedChainIndexSubscriber, /// Service metadata. pub data: ServiceMetadata, - /// StateService config data. - #[allow(deprecated)] - config: FetchServiceConfig, + /// Connection-independent config (validator RPC, storage, network, ...). + config: CommonBackendConfig, } -impl Status for FetchServiceSubscriber { +impl Status for NodeBackedIndexerServiceSubscriber { fn status(&self) -> StatusType { self.indexer.status() } } -impl FetchServiceSubscriber { +impl NodeBackedIndexerServiceSubscriber { /// Fetches the current status #[deprecated(note = "Use the Status trait method instead")] pub fn get_status(&self) -> StatusType { @@ -215,15 +235,106 @@ impl FetchServiceSubscriber { } /// Returns the network type running. - #[allow(deprecated)] pub fn network(&self) -> zaino_common::Network { - self.config.common.network + self.config.network } } -impl ZcashIndexer for FetchServiceSubscriber { - #[allow(deprecated)] - type Error = FetchServiceError; +#[cfg(test)] +impl NodeBackedIndexerServiceSubscriber { + /// Wraps a chain-index subscriber in a service subscriber for tests, with placeholder + /// metadata/config. Lets unit tests drive the service RPC layer over a mock source + /// (no real validator). Production builds go through [`ZcashService::get_subscriber`]. + pub(crate) fn new_for_test( + indexer: NodeBackedChainIndexSubscriber, + network: zaino_common::Network, + ) -> Self { + Self { + data: ServiceMetadata::new( + get_build_info("test".to_string()), + network.to_zebra_network(), + "test-build".to_string(), + "test-subversion".to_string(), + ), + config: CommonBackendConfig::new( + "127.0.0.1:0".to_string(), + None, + None, + None, + zaino_common::ServiceConfig::default(), + zaino_common::StorageConfig::default(), + true, + network, + None, + ), + indexer, + } + } +} + +/// A subscriber to any chaintip updates. +#[derive(Clone)] +pub struct ChainTipSubscriber { + monitor: zebra_state::ChainTipChange, +} + +impl ChainTipSubscriber { + /// Waits until the tip hash has changed (relative to the last time this method + /// was called), then returns the best tip's block hash. + pub async fn next_tip_hash( + &mut self, + ) -> Result { + self.monitor + .wait_for_tip_change() + .await + .map(|tip| tip.best_tip_hash()) + } +} + +/// Methods available only on the production (`ValidatorConnector`-backed) subscriber: +/// the `Direct` connection owns a Zebra chain-tip stream and `ReadStateService` that the +/// generic source abstraction does not expose. +impl NodeBackedIndexerServiceSubscriber { + /// Gets a subscriber to any updates to the latest chain tip. + /// + /// Only meaningful for the `Direct` connection; the `Rpc` connection has no local + /// tip-change stream. + pub fn chaintip_update_subscriber(&self) -> ChainTipSubscriber { + ChainTipSubscriber { + monitor: self + .indexer + .source() + .chain_tip_change() + .expect("chaintip_update_subscriber requires the Direct connection"), + } + } + + /// The backing Zebra [`zebra_state::ReadStateService`] (`Direct` connection only). + /// + /// Test-only escape hatch: live tests recompute expected chain data directly off the + /// `ReadStateService`. Production code goes through the `ChainIndex` API. + #[cfg(feature = "test_dependencies")] + pub fn read_state_service(&self) -> zebra_state::ReadStateService { + self.indexer + .source() + .read_state_service() + .expect("read_state_service requires the Direct connection") + .clone() + } + + /// The indexer's mempool subscriber. + /// + /// Test-only escape hatch: live tests recompute expected `getmempoolinfo` values + /// directly off the mempool's entries. Production code goes through the `ChainIndex` + /// mempool API. + #[cfg(feature = "test_dependencies")] + pub fn mempool(&self) -> &crate::chain_index::mempool::MempoolSubscriber { + self.indexer.mempool_subscriber() + } +} + +impl ZcashIndexer for NodeBackedIndexerServiceSubscriber { + type Error = NodeBackedIndexerServiceError; /// Returns information about all changes to the given transparent addresses within the given inclusive block-height range. /// @@ -468,7 +579,7 @@ impl ZcashIndexer for FetchServiceSubscriber { async fn get_chain_tips(&self) -> Result { let snapshot = self.indexer.snapshot_nonfinalized_state().await?; let Some(non_finalized_snapshot) = snapshot.get_nfs_snapshot() else { - return Err(FetchServiceError::UnavailableNotSyncedEnough); + return Err(NodeBackedIndexerServiceError::UnavailableNotSyncedEnough); }; Ok(chain_tips_from_nonfinalized_snapshot( non_finalized_snapshot, @@ -488,7 +599,7 @@ impl ZcashIndexer for FetchServiceSubscriber { address: String, ) -> Result { #[allow(deprecated)] - let network = self.config.common.network.to_zebra_network(); + let network = self.config.network.to_zebra_network(); Ok(crate::indexer::validate_address(address, &network)) } @@ -498,7 +609,7 @@ impl ZcashIndexer for FetchServiceSubscriber { address: String, ) -> Result { #[allow(deprecated)] - let network = self.config.common.network.to_zebra_network(); + let network = self.config.network.to_zebra_network(); Ok(crate::indexer::z_validate_address(address, &network)) } @@ -538,7 +649,7 @@ impl ZcashIndexer for FetchServiceSubscriber { /// NOTE: This method currently has to fetch data from 2 places (get_treestate and get_indexed_block_by_*), /// If `ValidatorConnector::GetTreeState` was updated to return the additional information /// required, this second call could be removed, improving the performance of this method. - // Pre-existing lint: `FetchServiceError` is a large error type; returning it by value here is + // Pre-existing lint: `NodeBackedIndexerServiceError` is a large error type; returning it by value here is // flagged by `result_large_err`. Suppressed to satisfy `-D warnings` without an invasive // boxing refactor of the shared error enum. #[allow(clippy::result_large_err)] @@ -558,7 +669,7 @@ impl ZcashIndexer for FetchServiceSubscriber { .await? .ok_or( #[allow(deprecated)] - FetchServiceError::RpcError(RpcError::new_from_legacycode( + NodeBackedIndexerServiceError::RpcError(RpcError::new_from_legacycode( zebra_rpc::server::error::LegacyCode::InvalidParameter, "Failed to fetch block data.", )), @@ -569,7 +680,7 @@ impl ZcashIndexer for FetchServiceSubscriber { .await? .ok_or( #[allow(deprecated)] - FetchServiceError::RpcError(RpcError::new_from_legacycode( + NodeBackedIndexerServiceError::RpcError(RpcError::new_from_legacycode( zebra_rpc::server::error::LegacyCode::InvalidParameter, "Failed to fetch block data.", )), @@ -579,7 +690,7 @@ impl ZcashIndexer for FetchServiceSubscriber { let (sapling, orchard) = self.indexer.get_treestate(block_data.hash()).await?; let time: u32 = block_data.data().time().try_into().map_err(|_error| { #[allow(deprecated)] - FetchServiceError::RpcError(RpcError::new_from_legacycode( + NodeBackedIndexerServiceError::RpcError(RpcError::new_from_legacycode( zebra_rpc::server::error::LegacyCode::InvalidParameter, "Block time is out of range for u32.", )) @@ -644,12 +755,14 @@ impl ZcashIndexer for FetchServiceSubscriber { "sapling" => crate::chain_index::ShieldedPool::Sapling, "orchard" => crate::chain_index::ShieldedPool::Orchard, otherwise => { - return Err(FetchServiceError::RpcError(RpcError::new_from_legacycode( - zebra_rpc::server::error::LegacyCode::Misc, - format!( - "invalid pool name \"{otherwise}\", must be \"sapling\" or \"orchard\"" + return Err(NodeBackedIndexerServiceError::RpcError( + RpcError::new_from_legacycode( + zebra_rpc::server::error::LegacyCode::Misc, + format!( + "invalid pool name \"{otherwise}\", must be \"sapling\" or \"orchard\"" + ), ), - ))) + )) } }; let roots = self @@ -689,7 +802,7 @@ impl ZcashIndexer for FetchServiceSubscriber { ) -> Result { #[allow(deprecated)] let txid = types::TransactionHash::from_hex(&txid_hex).map_err(|error| { - FetchServiceError::RpcError(RpcError::new_from_legacycode( + NodeBackedIndexerServiceError::RpcError(RpcError::new_from_legacycode( zebra_rpc::server::error::LegacyCode::InvalidAddressOrKey, error.to_string(), )) @@ -697,7 +810,7 @@ impl ZcashIndexer for FetchServiceSubscriber { #[allow(deprecated)] let not_found_error = || { - FetchServiceError::RpcError(RpcError::new_from_legacycode( + NodeBackedIndexerServiceError::RpcError(RpcError::new_from_legacycode( zebra_rpc::server::error::LegacyCode::InvalidAddressOrKey, "No such mempool or main chain transaction", )) @@ -752,7 +865,7 @@ impl ZcashIndexer for FetchServiceSubscriber { height, confirmations, #[allow(deprecated)] - &self.config.common.network.to_zebra_network(), + &self.config.network.to_zebra_network(), None, block_hash, in_best_chain, @@ -860,7 +973,7 @@ impl ZcashIndexer for FetchServiceSubscriber { } #[allow(deprecated)] -impl LightWalletIndexer for FetchServiceSubscriber { +impl LightWalletIndexer for NodeBackedIndexerServiceSubscriber { /// Return the height of the tip of the best chain async fn get_latest_block(&self) -> Result { match self.indexer.snapshot_nonfinalized_state().await? { @@ -871,7 +984,7 @@ impl LightWalletIndexer for FetchServiceSubscriber { // TODO: This probably shouldn't be an error. // this is an improvement over previous behaviour of reporting // the genesis block - Err(FetchServiceError::UnavailableNotSyncedEnough) + Err(NodeBackedIndexerServiceError::UnavailableNotSyncedEnough) } } // dbg!(&tip); @@ -880,7 +993,7 @@ impl LightWalletIndexer for FetchServiceSubscriber { /// Return the compact block corresponding to the given block identifier async fn get_block(&self, request: BlockId) -> Result { let hash_or_height = blockid_to_hashorheight(request).ok_or( - FetchServiceError::TonicStatusError(tonic::Status::invalid_argument( + NodeBackedIndexerServiceError::TonicStatusError(tonic::Status::invalid_argument( "Error: Invalid hash and/or height out of range. Failed to convert to u32.", )), )?; @@ -892,12 +1005,12 @@ impl LightWalletIndexer for FetchServiceSubscriber { match self.indexer.get_block_height(&snapshot, hash.into()).await { Ok(Some(height)) => height.0, Ok(None) => { - return Err(FetchServiceError::TonicStatusError(tonic::Status::invalid_argument( + return Err(NodeBackedIndexerServiceError::TonicStatusError(tonic::Status::invalid_argument( "Error: Invalid hash and/or height out of range. Hash not founf in chain", ))); } Err(_e) => { - return Err(FetchServiceError::TonicStatusError( + return Err(NodeBackedIndexerServiceError::TonicStatusError( tonic::Status::internal("Error: Internal db error."), )); } @@ -909,7 +1022,7 @@ impl LightWalletIndexer for FetchServiceSubscriber { // TODO: This probably shouldn't be an error. // this is an improvement over previous behaviour of // acting as if we are only synced to the genesis block - return Err(FetchServiceError::UnavailableNotSyncedEnough); + return Err(NodeBackedIndexerServiceError::UnavailableNotSyncedEnough); }; match self @@ -925,32 +1038,38 @@ impl LightWalletIndexer for FetchServiceSubscriber { Ok(None) => { let chain_height = non_finalized_snapshot.best_tip.height.0; match hash_or_height { - HashOrHeight::Height(Height(height)) if height >= chain_height => Err( - FetchServiceError::TonicStatusError(tonic::Status::out_of_range(format!( - "Error: Height out of range [{hash_or_height}]. Height requested \ + HashOrHeight::Height(Height(height)) if height >= chain_height => { + Err(NodeBackedIndexerServiceError::TonicStatusError( + tonic::Status::out_of_range(format!( + "Error: Height out of range [{hash_or_height}]. Height requested \ is greater than the best chain tip [{chain_height}].", - ))), - ), - _otherwise => Err(FetchServiceError::TonicStatusError(tonic::Status::unknown( - "Error: Failed to retrieve block from state.", - ))), + )), + )) + } + _otherwise => Err(NodeBackedIndexerServiceError::TonicStatusError( + tonic::Status::unknown("Error: Failed to retrieve block from state."), + )), } } Err(e) => { let chain_height = non_finalized_snapshot.best_tip.height.0; match hash_or_height { - HashOrHeight::Height(Height(height)) if height >= chain_height => Err( - FetchServiceError::TonicStatusError(tonic::Status::out_of_range(format!( - "Error: Height out of range [{hash_or_height}]. Height requested \ + HashOrHeight::Height(Height(height)) if height >= chain_height => { + Err(NodeBackedIndexerServiceError::TonicStatusError( + tonic::Status::out_of_range(format!( + "Error: Height out of range [{hash_or_height}]. Height requested \ is greater than the best chain tip [{chain_height}].", - ))), - ), + )), + )) + } _otherwise => // TODO: Hide server error from clients before release. Currently useful for dev purposes. { - Err(FetchServiceError::TonicStatusError(tonic::Status::unknown( - format!("Error: Failed to retrieve block from node. Server Error: {e}",), - ))) + Err(NodeBackedIndexerServiceError::TonicStatusError( + tonic::Status::unknown(format!( + "Error: Failed to retrieve block from node. Server Error: {e}", + )), + )) } } } @@ -962,7 +1081,7 @@ impl LightWalletIndexer for FetchServiceSubscriber { /// NOTE: Currently this only returns Orchard nullifiers to follow Lightwalletd functionality but Sapling could be added if required by wallets. async fn get_block_nullifiers(&self, request: BlockId) -> Result { let hash_or_height = blockid_to_hashorheight(request).ok_or( - FetchServiceError::TonicStatusError(tonic::Status::invalid_argument( + NodeBackedIndexerServiceError::TonicStatusError(tonic::Status::invalid_argument( "Error: Invalid hash and/or height out of range. Failed to convert to u32.", )), )?; @@ -973,12 +1092,12 @@ impl LightWalletIndexer for FetchServiceSubscriber { match self.indexer.get_block_height(&snapshot, hash.into()).await { Ok(Some(height)) => height.0, Ok(None) => { - return Err(FetchServiceError::TonicStatusError(tonic::Status::invalid_argument( + return Err(NodeBackedIndexerServiceError::TonicStatusError(tonic::Status::invalid_argument( "Error: Invalid hash and/or height out of range. Hash not founf in chain", ))); } Err(_e) => { - return Err(FetchServiceError::TonicStatusError( + return Err(NodeBackedIndexerServiceError::TonicStatusError( tonic::Status::internal("Error: Internal db error."), )); } @@ -989,7 +1108,7 @@ impl LightWalletIndexer for FetchServiceSubscriber { // TODO: This probably shouldn't be an error. // this is an improvement over previous behaviour of // acting as if we are only synced to the genesis block - return Err(FetchServiceError::UnavailableNotSyncedEnough); + return Err(NodeBackedIndexerServiceError::UnavailableNotSyncedEnough); }; match self .indexer @@ -1004,40 +1123,44 @@ impl LightWalletIndexer for FetchServiceSubscriber { Ok(None) => { let chain_height = non_finalized_snapshot.best_tip.height.0; match hash_or_height { - HashOrHeight::Height(Height(height)) if height >= chain_height => Err( - FetchServiceError::TonicStatusError(tonic::Status::out_of_range(format!( - "Error: Height out of range [{hash_or_height}]. Height requested \ + HashOrHeight::Height(Height(height)) if height >= chain_height => { + Err(NodeBackedIndexerServiceError::TonicStatusError( + tonic::Status::out_of_range(format!( + "Error: Height out of range [{hash_or_height}]. Height requested \ is greater than the best chain tip [{chain_height}].", - ))), - ), + )), + )) + } HashOrHeight::Height(height) if height > self.data.network().sapling_activation_height() => { - Err(FetchServiceError::TonicStatusError( + Err(NodeBackedIndexerServiceError::TonicStatusError( tonic::Status::out_of_range(format!( "Error: Height out of range [{hash_or_height}]. Height requested \ is below sapling activation height [{chain_height}].", )), )) } - _otherwise => Err(FetchServiceError::TonicStatusError(tonic::Status::unknown( - "Error: Failed to retrieve block from state.", - ))), + _otherwise => Err(NodeBackedIndexerServiceError::TonicStatusError( + tonic::Status::unknown("Error: Failed to retrieve block from state."), + )), } } Err(e) => { let chain_height = non_finalized_snapshot.best_tip.height.0; match hash_or_height { - HashOrHeight::Height(Height(height)) if height >= chain_height => Err( - FetchServiceError::TonicStatusError(tonic::Status::out_of_range(format!( - "Error: Height out of range [{hash_or_height}]. Height requested \ + HashOrHeight::Height(Height(height)) if height >= chain_height => { + Err(NodeBackedIndexerServiceError::TonicStatusError( + tonic::Status::out_of_range(format!( + "Error: Height out of range [{hash_or_height}]. Height requested \ is greater than the best chain tip [{chain_height}].", - ))), - ), + )), + )) + } HashOrHeight::Height(height) if height > self.data.network().sapling_activation_height() => { - Err(FetchServiceError::TonicStatusError( + Err(NodeBackedIndexerServiceError::TonicStatusError( tonic::Status::out_of_range(format!( "Error: Height out of range [{hash_or_height}]. Height requested \ is below sapling activation height [{chain_height}].", @@ -1047,9 +1170,11 @@ impl LightWalletIndexer for FetchServiceSubscriber { _otherwise => // TODO: Hide server error from clients before release. Currently useful for dev purposes. { - Err(FetchServiceError::TonicStatusError(tonic::Status::unknown( - format!("Error: Failed to retrieve block from node. Server Error: {e}",), - ))) + Err(NodeBackedIndexerServiceError::TonicStatusError( + tonic::Status::unknown(format!( + "Error: Failed to retrieve block from node. Server Error: {e}", + )), + )) } } } @@ -1063,24 +1188,20 @@ impl LightWalletIndexer for FetchServiceSubscriber { request: BlockRange, ) -> Result { let validated_request = ValidatedBlockRangeRequest::new_from_block_range(&request) - .map_err(FetchServiceError::from)?; + .map_err(NodeBackedIndexerServiceError::from)?; let pool_type_filter = PoolTypeFilter::new_from_pool_types(&validated_request.pool_types()) .map_err(GetBlockRangeError::PoolTypeArgumentError) - .map_err(FetchServiceError::from)?; + .map_err(NodeBackedIndexerServiceError::from)?; // Note conversion here is safe due to the use of [`ValidatedBlockRangeRequest::new_from_block_range`] let start = validated_request.start() as u32; let end = validated_request.end() as u32; - let fetch_service_clone = self.clone(); - let service_timeout = self.config.common.service.timeout; - let (channel_tx, channel_rx) = - mpsc::channel(self.config.common.service.channel_size as usize); - let snapshot = fetch_service_clone - .indexer - .snapshot_nonfinalized_state() - .await?; + let service_clone = self.clone(); + let service_timeout = self.config.service.timeout; + let (channel_tx, channel_rx) = mpsc::channel(self.config.service.channel_size as usize); + let snapshot = service_clone.indexer.snapshot_nonfinalized_state().await?; tokio::spawn(async move { let timeout_result = timeout( @@ -1103,7 +1224,7 @@ impl LightWalletIndexer for FetchServiceSubscriber { // Use the snapshot tip directly, as this function doesn't support passthrough let chain_height = non_finalized_snapshot.best_tip.height.0; - match fetch_service_clone + match service_clone .indexer .get_compact_block_stream( &snapshot, @@ -1196,24 +1317,20 @@ impl LightWalletIndexer for FetchServiceSubscriber { request: BlockRange, ) -> Result { let validated_request = ValidatedBlockRangeRequest::new_from_block_range(&request) - .map_err(FetchServiceError::from)?; + .map_err(NodeBackedIndexerServiceError::from)?; let pool_type_filter = PoolTypeFilter::new_from_pool_types(&validated_request.pool_types()) .map_err(GetBlockRangeError::PoolTypeArgumentError) - .map_err(FetchServiceError::from)?; + .map_err(NodeBackedIndexerServiceError::from)?; // Note conversion here is safe due to the use of [`ValidatedBlockRangeRequest::new_from_block_range`] let start = validated_request.start() as u32; let end = validated_request.end() as u32; - let fetch_service_clone = self.clone(); - let service_timeout = self.config.common.service.timeout; - let (channel_tx, channel_rx) = - mpsc::channel(self.config.common.service.channel_size as usize); - let snapshot = fetch_service_clone - .indexer - .snapshot_nonfinalized_state() - .await?; + let service_clone = self.clone(); + let service_timeout = self.config.service.timeout; + let (channel_tx, channel_rx) = mpsc::channel(self.config.service.channel_size as usize); + let snapshot = service_clone.indexer.snapshot_nonfinalized_state().await?; tokio::spawn(async move { let timeout_result = timeout( @@ -1237,7 +1354,7 @@ impl LightWalletIndexer for FetchServiceSubscriber { // Use the snapshot tip directly, as this function doesn't support passthrough let chain_height = non_finalized_snapshot.best_tip.height.0; - match fetch_service_clone + match service_clone .indexer .get_compact_block_stream( &snapshot, @@ -1347,7 +1464,7 @@ impl LightWalletIndexer for FetchServiceSubscriber { let (hex, height) = if let GetRawTransaction::Object(tx_object) = tx { (tx_object.hex().clone(), tx_object.height()) } else { - return Err(FetchServiceError::TonicStatusError( + return Err(NodeBackedIndexerServiceError::TonicStatusError( tonic::Status::not_found("Error: Transaction not received"), )); }; @@ -1360,7 +1477,7 @@ impl LightWalletIndexer for FetchServiceSubscriber { // TODO: This probably shouldn't be an error. // this is an improvement over previous behaviour of // acting as if we are only synced to the genesis block - return Err(FetchServiceError::UnavailableNotSyncedEnough); + return Err(NodeBackedIndexerServiceError::UnavailableNotSyncedEnough); }; non_finalized_snapshot.best_tip.height.0 as u64 } @@ -1371,7 +1488,7 @@ impl LightWalletIndexer for FetchServiceSubscriber { height, }) } else { - Err(FetchServiceError::TonicStatusError( + Err(NodeBackedIndexerServiceError::TonicStatusError( tonic::Status::invalid_argument("Error: Transaction hash incorrect"), )) } @@ -1395,17 +1512,15 @@ impl LightWalletIndexer for FetchServiceSubscriber { ) -> Result { let chain_height = self.chain_height().await?; let txids = self.get_taddress_txids_helper(request).await?; - let fetch_service_clone = self.clone(); - let service_timeout = self.config.common.service.timeout; - let (transmitter, receiver) = - mpsc::channel(self.config.common.service.channel_size as usize); + let service_clone = self.clone(); + let service_timeout = self.config.service.timeout; + let (transmitter, receiver) = mpsc::channel(self.config.service.channel_size as usize); tokio::spawn(async move { let timeout = timeout( time::Duration::from_secs((service_timeout * 4) as u64), async { for txid in txids { - let transaction = - fetch_service_clone.get_raw_transaction(txid, Some(1)).await; + let transaction = service_clone.get_raw_transaction(txid, Some(1)).await; if handle_raw_transaction::( chain_height.0 as u64, transaction, @@ -1452,9 +1567,9 @@ impl LightWalletIndexer for FetchServiceSubscriber { let checked_balance: i64 = match i64::try_from(balance.balance()) { Ok(balance) => balance, Err(_) => { - return Err(FetchServiceError::TonicStatusError(tonic::Status::unknown( - "Error: Error converting balance from u64 to i64.", - ))); + return Err(NodeBackedIndexerServiceError::TonicStatusError( + tonic::Status::unknown("Error: Error converting balance from u64 to i64."), + )); } }; Ok(Balance { @@ -1468,10 +1583,10 @@ impl LightWalletIndexer for FetchServiceSubscriber { &self, mut request: AddressStream, ) -> Result { - let fetch_service_clone = self.clone(); - let service_timeout = self.config.common.service.timeout; + let service_clone = self.clone(); + let service_timeout = self.config.service.timeout; let (channel_tx, mut channel_rx) = - mpsc::channel::(self.config.common.service.channel_size as usize); + mpsc::channel::(self.config.service.channel_size as usize); let fetcher_task_handle = tokio::spawn(async move { let fetcher_timeout = timeout( time::Duration::from_secs((service_timeout * 4) as u64), @@ -1481,8 +1596,7 @@ impl LightWalletIndexer for FetchServiceSubscriber { match channel_rx.recv().await { Some(taddr) => { let taddrs = GetAddressBalanceRequest::new(vec![taddr]); - let balance = - fetch_service_clone.z_get_address_balance(taddrs).await?; + let balance = service_clone.z_get_address_balance(taddrs).await?; total_balance += balance.balance(); } None => { @@ -1530,11 +1644,11 @@ impl LightWalletIndexer for FetchServiceSubscriber { Ok(Ok(())) => {} Ok(Err(e)) => { fetcher_task_handle.abort(); - return Err(FetchServiceError::TonicStatusError(e)); + return Err(NodeBackedIndexerServiceError::TonicStatusError(e)); } Err(_) => { fetcher_task_handle.abort(); - return Err(FetchServiceError::TonicStatusError( + return Err(NodeBackedIndexerServiceError::TonicStatusError( tonic::Status::deadline_exceeded( "Error: get_taddress_balance_stream request timed out in address loop.", ), @@ -1548,21 +1662,23 @@ impl LightWalletIndexer for FetchServiceSubscriber { Err(_) => { // TODO: Hide server error from clients before release. // Currently useful for dev purposes. - return Err(FetchServiceError::TonicStatusError(tonic::Status::unknown( - "Error: Error converting balance from u64 to i64.", - ))); + return Err(NodeBackedIndexerServiceError::TonicStatusError( + tonic::Status::unknown( + "Error: Error converting balance from u64 to i64.", + ), + )); } }; Ok(Balance { value_zat: checked_balance, }) } - Ok(Err(e)) => Err(FetchServiceError::TonicStatusError(e)), + Ok(Err(e)) => Err(NodeBackedIndexerServiceError::TonicStatusError(e)), // TODO: Hide server error from clients before release. // Currently useful for dev purposes. - Err(e) => Err(FetchServiceError::TonicStatusError(tonic::Status::unknown( - format!("Fetcher Task failed: {e}"), - ))), + Err(e) => Err(NodeBackedIndexerServiceError::TonicStatusError( + tonic::Status::unknown(format!("Fetcher Task failed: {e}")), + )), } } @@ -1587,7 +1703,7 @@ impl LightWalletIndexer for FetchServiceSubscriber { for (i, excluded_id) in request.exclude_txid_suffixes.iter().enumerate() { if excluded_id.len() > 32 { - return Err(FetchServiceError::TonicStatusError( + return Err(NodeBackedIndexerServiceError::TonicStatusError( tonic::Status::invalid_argument(format!( "Error: excluded txid {} is larger than 32 bytes", i @@ -1603,9 +1719,8 @@ impl LightWalletIndexer for FetchServiceSubscriber { } let mempool = self.indexer.clone(); - let service_timeout = self.config.common.service.timeout; - let (channel_tx, channel_rx) = - mpsc::channel(self.config.common.service.channel_size as usize); + let service_timeout = self.config.service.timeout; + let (channel_tx, channel_rx) = mpsc::channel(self.config.service.channel_size as usize); tokio::spawn(async move { let timeout = timeout( @@ -1704,9 +1819,8 @@ impl LightWalletIndexer for FetchServiceSubscriber { #[allow(deprecated)] async fn get_mempool_stream(&self) -> Result { let indexer = self.indexer.clone(); - let service_timeout = self.config.common.service.timeout; - let (channel_tx, channel_rx) = - mpsc::channel(self.config.common.service.channel_size as usize); + let service_timeout = self.config.service.timeout; + let (channel_tx, channel_rx) = mpsc::channel(self.config.service.channel_size as usize); let snapshot = indexer.snapshot_nonfinalized_state().await?; tokio::spawn(async move { let timeout = timeout( @@ -1787,26 +1901,17 @@ impl LightWalletIndexer for FetchServiceSubscriber { /// The block can be specified by either height or hash. async fn get_tree_state(&self, request: BlockId) -> Result { let hash_or_height = blockid_to_hashorheight(request).ok_or( - crate::error::FetchServiceError::TonicStatusError(tonic::Status::invalid_argument( - "Invalid hash or height", - )), + crate::error::NodeBackedIndexerServiceError::TonicStatusError( + tonic::Status::invalid_argument("Invalid hash or height"), + ), )?; - #[allow(deprecated)] let (hash, height, time, sapling, orchard) = - ::z_get_treestate( - self, - hash_or_height.to_string(), - ) - .await? - .into_parts(); + ::z_get_treestate(self, hash_or_height.to_string()) + .await? + .into_parts(); Ok(TreeState { - network: self - .config - .common - .network - .to_zebra_network() - .bip70_network_name(), + network: self.config.network.to_zebra_network().bip70_network_name(), height: height.0 as u64, hash: hash.to_string(), time, @@ -1828,8 +1933,8 @@ impl LightWalletIndexer for FetchServiceSubscriber { #[allow(deprecated)] fn timeout_channel_size(&self) -> (u32, u32) { ( - self.config.common.service.timeout, - self.config.common.service.channel_size, + self.config.service.timeout, + self.config.service.channel_size, ) } @@ -1860,17 +1965,21 @@ impl LightWalletIndexer for FetchServiceSubscriber { let checked_index = match i32::try_from(output_index.index()) { Ok(index) => index, Err(_) => { - return Err(FetchServiceError::TonicStatusError(tonic::Status::unknown( - "Error: Index out of range. Failed to convert to i32.", - ))); + return Err(NodeBackedIndexerServiceError::TonicStatusError( + tonic::Status::unknown( + "Error: Index out of range. Failed to convert to i32.", + ), + )); } }; let checked_satoshis = match i64::try_from(satoshis) { Ok(satoshis) => satoshis, Err(_) => { - return Err(FetchServiceError::TonicStatusError(tonic::Status::unknown( - "Error: Satoshis out of range. Failed to convert to i64.", - ))); + return Err(NodeBackedIndexerServiceError::TonicStatusError( + tonic::Status::unknown( + "Error: Satoshis out of range. Failed to convert to i64.", + ), + )); } }; let utxo_reply = GetAddressUtxosReply { @@ -1900,9 +2009,8 @@ impl LightWalletIndexer for FetchServiceSubscriber { super::validate_utxo_address_count(request.addresses.len())?; let taddrs = GetAddressBalanceRequest::new(request.addresses); let utxos = self.z_get_address_utxos(taddrs).await?; - let service_timeout = self.config.common.service.timeout; - let (channel_tx, channel_rx) = - mpsc::channel(self.config.common.service.channel_size as usize); + let service_timeout = self.config.service.timeout; + let (channel_tx, channel_rx) = mpsc::channel(self.config.service.channel_size as usize); tokio::spawn(async move { let timeout = timeout( time::Duration::from_secs((service_timeout * 4) as u64), @@ -1994,7 +2102,7 @@ impl LightWalletIndexer for FetchServiceSubscriber { .to_string(); let latest_upgrade = super::latest_network_upgrade(blockchain_info.upgrades()) - .map_err(FetchServiceError::TonicStatusError)? + .map_err(NodeBackedIndexerServiceError::TonicStatusError)? .into_parts(); let nu_name = latest_upgrade.0; @@ -2017,7 +2125,6 @@ impl LightWalletIndexer for FetchServiceSubscriber { zcashd_subversion: self.data.zebra_subversion(), donation_address: self .config - .common .donation_address .as_ref() .map(DonationAddress::encode) @@ -2032,7 +2139,7 @@ impl LightWalletIndexer for FetchServiceSubscriber { /// /// NOTE: Currently unimplemented in Zaino. async fn ping(&self, _request: Duration) -> Result { - Err(FetchServiceError::TonicStatusError( + Err(NodeBackedIndexerServiceError::TonicStatusError( tonic::Status::unimplemented( "Ping not yet implemented. If you require this RPC \ please open an issue or PR at the Zaino github \ diff --git a/packages/zaino-state/src/lib.rs b/packages/zaino-state/src/lib.rs index 22cb0121a..a2c0d67a6 100644 --- a/packages/zaino-state/src/lib.rs +++ b/packages/zaino-state/src/lib.rs @@ -56,12 +56,8 @@ pub use indexer::{ ZcashService, }; -pub(crate) mod backends; - -#[allow(deprecated)] -pub use backends::{ - fetch::{FetchService, FetchServiceSubscriber}, - state::{StateService, StateServiceSubscriber}, +pub use indexer::node_backed_indexer::{ + ChainTipSubscriber, NodeBackedIndexerService, NodeBackedIndexerServiceSubscriber, }; pub mod chain_index; @@ -104,16 +100,14 @@ pub mod test_dependencies { pub(crate) mod config; -#[allow(deprecated)] pub use config::{ - BackendConfig, BackendType, ChainIndexConfig, CommonBackendConfig, DonationAddress, - FetchServiceConfig, StateServiceConfig, + ChainIndexConfig, CommonBackendConfig, DirectConnectionConfig, DonationAddress, + NodeBackedIndexerServiceConfig, ValidatorConnectionType, }; pub(crate) mod error; -#[allow(deprecated)] -pub use error::{FetchServiceError, StateServiceError}; +pub use error::NodeBackedIndexerServiceError; pub(crate) mod status; diff --git a/packages/zainod/src/config.rs b/packages/zainod/src/config.rs index aeb97dbbe..a5bd0a720 100644 --- a/packages/zainod/src/config.rs +++ b/packages/zainod/src/config.rs @@ -21,11 +21,32 @@ use zaino_common::{ try_resolve_address, AddressResolution, Network, ServiceConfig, StorageConfig, ValidatorConfig, }; use zaino_serve::server::config::{GrpcServerConfig, JsonRpcServerConfig}; -#[allow(deprecated)] use zaino_state::{ - BackendType, CommonBackendConfig, DonationAddress, FetchServiceConfig, StateServiceConfig, + CommonBackendConfig, DirectConnectionConfig, DonationAddress, NodeBackedIndexerServiceConfig, + ValidatorConnectionType, }; +/// On-disk selector for the validator connection (`backend = "direct" | "rpc"` in the +/// config file), mapped to [`zaino_state::ValidatorConnectionType`] at spawn. +/// +/// The legacy values `"state"` / `"fetch"` remain accepted as aliases for backward +/// compatibility with existing `zainod.toml` files. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum BackendType { + /// Direct Zebra `ReadStateService` access (formerly `state`). + /// + /// More efficient but requires running on the same machine as Zebra. + #[serde(alias = "state")] + Direct, + /// JSON-RPC access (formerly `fetch`). + /// + /// Compatible with Zcashd, Zebra, or another Zaino instance. + #[default] + #[serde(alias = "fetch")] + Rpc, +} + /// Header for generated configuration files. pub const GENERATED_CONFIG_HEADER: &str = r#"# Zaino Configuration # @@ -363,59 +384,56 @@ pub fn load_config_with_env( Ok(parsed_config) } -#[allow(deprecated)] -impl TryFrom for StateServiceConfig { +impl TryFrom for NodeBackedIndexerServiceConfig { type Error = IndexerError; fn try_from(cfg: ZainodConfig) -> Result { - let grpc_listen_address = cfg - .validator_settings - .validator_grpc_listen_address - .as_ref() - .ok_or_else(|| { - IndexerError::ConfigError( - "Missing validator_grpc_listen_address in configuration".to_string(), - ) - })?; - - let validator_grpc_address = - fetch_socket_addr_from_hostname(grpc_listen_address).map_err(|e| { - let msg = match e { - IndexerError::ConfigError(msg) => msg, - other => other.to_string(), + let connection = match cfg.backend { + BackendType::Rpc => ValidatorConnectionType::Rpc, + BackendType::Direct => { + let grpc_listen_address = cfg + .validator_settings + .validator_grpc_listen_address + .as_ref() + .ok_or_else(|| { + IndexerError::ConfigError( + "Missing validator_grpc_listen_address in configuration".to_string(), + ) + })?; + + let validator_grpc_address = fetch_socket_addr_from_hostname(grpc_listen_address) + .map_err(|e| { + let msg = match e { + IndexerError::ConfigError(msg) => msg, + other => other.to_string(), + }; + IndexerError::ConfigError(format!( + "Invalid validator_grpc_listen_address '{grpc_listen_address}': {msg}" + )) + })?; + + let validator_state_config = zebra_state::Config { + cache_dir: cfg.zebra_db_path.clone(), + ephemeral: false, + delete_old_database: true, + debug_stop_at_height: None, + debug_validity_check_interval: None, + should_backup_non_finalized_state: true, + debug_skip_non_finalized_state_backup_task: false, }; - IndexerError::ConfigError(format!( - "Invalid validator_grpc_listen_address '{grpc_listen_address}': {msg}" - )) - })?; - - let validator_state_config = zebra_state::Config { - cache_dir: cfg.zebra_db_path.clone(), - ephemeral: false, - delete_old_database: true, - debug_stop_at_height: None, - debug_validity_check_interval: None, - should_backup_non_finalized_state: true, - debug_skip_non_finalized_state_backup_task: false, - }; - let validator_cookie_auth = cfg.validator_settings.validator_cookie_path.is_some(); - - Ok(StateServiceConfig { - common: build_common(cfg), - validator_state_config, - validator_grpc_address, - validator_cookie_auth, - }) - } -} + let validator_cookie_auth = cfg.validator_settings.validator_cookie_path.is_some(); -#[allow(deprecated)] -impl TryFrom for FetchServiceConfig { - type Error = IndexerError; + ValidatorConnectionType::Direct(DirectConnectionConfig { + validator_state_config, + validator_grpc_address, + validator_cookie_auth, + }) + } + }; - fn try_from(cfg: ZainodConfig) -> Result { - Ok(FetchServiceConfig { + Ok(NodeBackedIndexerServiceConfig { common: build_common(cfg), + connection, }) } } @@ -555,7 +573,8 @@ key_path = "{}" let config_path = create_test_config_file(&temp_dir, &toml_content, "full_config.toml"); let config = load_config(&config_path).expect("load_config failed"); - assert_eq!(config.backend, BackendType::Fetch); + // legacy `backend = "fetch"` still parses via the serde alias + assert_eq!(config.backend, BackendType::Rpc); assert!(config.json_server_settings.is_some()); assert_eq!( config @@ -605,7 +624,8 @@ listen_address = "127.0.0.1:8137" let config = load_config(&config_path).expect("load_config failed"); let default_values = ZainodConfig::default(); - assert_eq!(config.backend, BackendType::State); + // legacy `backend = "state"` still parses via the serde alias + assert_eq!(config.backend, BackendType::Direct); assert!(config.json_server_settings.is_none()); assert_eq!( config.validator_settings.validator_user, @@ -1138,66 +1158,56 @@ listen_address = "127.0.0.1:8137" /// `CARGO_PKG_VERSION` at the boundary so the wire reflects the /// deployed binary, not zaino-state's library version. #[test] - #[allow(deprecated)] fn indexer_version_is_zainod_pkg_version() { let _guard = EnvGuard::new(); - let cfg = ZainodConfig::default(); - - let state_cfg = StateServiceConfig::try_from(cfg.clone()) - .expect("StateServiceConfig conversion should succeed for default ZainodConfig"); - assert_eq!(state_cfg.common.indexer_version, env!("CARGO_PKG_VERSION")); - - let fetch_cfg = FetchServiceConfig::try_from(cfg) - .expect("FetchServiceConfig conversion should succeed for default ZainodConfig"); - assert_eq!(fetch_cfg.common.indexer_version, env!("CARGO_PKG_VERSION")); + let service_cfg = NodeBackedIndexerServiceConfig::try_from(ZainodConfig::default()) + .expect("service config conversion should succeed for default ZainodConfig"); + assert_eq!( + service_cfg.common.indexer_version, + env!("CARGO_PKG_VERSION") + ); } - /// `StateServiceConfig::try_from` and `FetchServiceConfig::try_from` - /// share a single `build_common` helper, so the two backends can - /// never quietly disagree on the common payload they hand to a - /// service. Locks that property in across every field: a future - /// hand-rolled divergence (e.g. one path stops applying the - /// missing-credentials sentinel, or a new common field gets - /// populated on only one side) makes this fail. Pretty-Debug - /// equality is used because not every constituent of - /// `CommonBackendConfig` derives `PartialEq`, and a single - /// stringified compare future-proofs the test against fields added - /// later. + /// The `Rpc` and `Direct` connections share a single `build_common` helper, so the + /// common payload handed to the service is connection-independent. Locks that in + /// across every field: a future divergence (e.g. one path stops applying the + /// missing-credentials sentinel, or a new common field gets populated on only one + /// side) makes this fail. Pretty-Debug equality is used because not every constituent + /// of `CommonBackendConfig` derives `PartialEq`, and a single stringified compare + /// future-proofs the test against fields added later. #[test] - #[allow(deprecated)] - fn state_and_fetch_common_payloads_agree() { + fn common_payload_is_connection_independent() { let _guard = EnvGuard::new(); - let cfg = ZainodConfig::default(); + let rpc_cfg = NodeBackedIndexerServiceConfig::try_from(ZainodConfig { + backend: BackendType::Rpc, + ..ZainodConfig::default() + }) + .expect("Rpc conversion should succeed for default ZainodConfig"); + let direct_cfg = NodeBackedIndexerServiceConfig::try_from(ZainodConfig { + backend: BackendType::Direct, + ..ZainodConfig::default() + }) + .expect("Direct conversion should succeed for default ZainodConfig"); - let state_config = StateServiceConfig::try_from(cfg.clone()) - .expect("StateServiceConfig conversion should succeed for default ZainodConfig"); - let fetch_config = FetchServiceConfig::try_from(cfg) - .expect("FetchServiceConfig conversion should succeed for default ZainodConfig"); + assert!(matches!(rpc_cfg.connection, ValidatorConnectionType::Rpc)); + assert!(matches!( + direct_cfg.connection, + ValidatorConnectionType::Direct(_) + )); assert_eq!( - format!("{:#?}", state_config.common), - format!("{:#?}", fetch_config.common), + format!("{:#?}", rpc_cfg.common), + format!("{:#?}", direct_cfg.common), ); - let cfg = ZainodConfig { + let ephemeral_cfg = NodeBackedIndexerServiceConfig::try_from(ZainodConfig { ephemeral_finalised_state: true, ..ZainodConfig::default() - }; - - let state_config = StateServiceConfig::try_from(cfg.clone()) - .expect("StateServiceConfig conversion should succeed for ephemeral finalised state"); - let fetch_config = FetchServiceConfig::try_from(cfg) - .expect("FetchServiceConfig conversion should succeed for ephemeral finalised state"); - - assert!(state_config.common.ephemeral_finalised_state); - assert!(fetch_config.common.ephemeral_finalised_state); - - assert_eq!( - format!("{:#?}", state_config.common), - format!("{:#?}", fetch_config.common), - ); + }) + .expect("conversion should succeed for ephemeral finalised state"); + assert!(ephemeral_cfg.common.ephemeral_finalised_state); } /// Builds a default config with the JSON-RPC server bound to `addr`. @@ -1305,12 +1315,9 @@ listen_address = "127.0.0.1:8137" assert!(config.ephemeral_finalised_state); - #[allow(deprecated)] - { - let fetch_config = FetchServiceConfig::try_from(config) - .expect("FetchServiceConfig conversion should succeed"); + let service_config = NodeBackedIndexerServiceConfig::try_from(config) + .expect("service config conversion should succeed"); - assert!(fetch_config.common.ephemeral_finalised_state); - } + assert!(service_config.common.ephemeral_finalised_state); } } diff --git a/packages/zainod/src/error.rs b/packages/zainod/src/error.rs index 38a2433e4..0c27cecac 100644 --- a/packages/zainod/src/error.rs +++ b/packages/zainod/src/error.rs @@ -3,12 +3,10 @@ use zaino_fetch::jsonrpsee::error::TransportError; use zaino_serve::server::error::ServerError; -#[allow(deprecated)] -use zaino_state::{FetchServiceError, StateServiceError}; +use zaino_state::NodeBackedIndexerServiceError; /// Zingo-Indexer errors. #[derive(Debug, thiserror::Error)] -#[allow(deprecated)] pub enum IndexerError { /// Server based errors. #[error("Server error: {0}")] @@ -19,12 +17,9 @@ pub enum IndexerError { /// JSON RPSee connector errors. #[error("JSON RPSee connector error: {0}")] TransportError(#[from] TransportError), - /// FetchService errors. - #[error("FetchService error: {0}")] - FetchServiceError(Box), - /// FetchService errors. - #[error("StateService error: {0}")] - StateServiceError(Box), + /// NodeBackedIndexerService errors. + #[error("NodeBackedIndexerService error: {0}")] + NodeBackedIndexerServiceError(Box), /// HTTP related errors due to invalid URI. #[error("HTTP error: Invalid URI {0}")] HttpError(#[from] http::Error), @@ -43,16 +38,8 @@ pub enum IndexerError { Restart, } -#[allow(deprecated)] -impl From for IndexerError { - fn from(value: StateServiceError) -> Self { - IndexerError::StateServiceError(Box::new(value)) - } -} - -#[allow(deprecated)] -impl From for IndexerError { - fn from(value: FetchServiceError) -> Self { - IndexerError::FetchServiceError(Box::new(value)) +impl From for IndexerError { + fn from(value: NodeBackedIndexerServiceError) -> Self { + IndexerError::NodeBackedIndexerServiceError(Box::new(value)) } } diff --git a/packages/zainod/src/indexer.rs b/packages/zainod/src/indexer.rs index 1b23120a8..cfcdc2064 100644 --- a/packages/zainod/src/indexer.rs +++ b/packages/zainod/src/indexer.rs @@ -8,10 +8,9 @@ use zaino_serve::{ rpc::grpc_routes, server::{config::GrpcServerConfig, grpc::TonicServer, jsonrpc::JsonRpcServer}, }; -#[allow(deprecated)] use zaino_state::{ - BackendType, FetchService, FetchServiceConfig, IndexerService, LightWalletService, - StateService, StateServiceConfig, StatusType, ZcashIndexer, ZcashService, + IndexerService, LightWalletService, NodeBackedIndexerService, NodeBackedIndexerServiceConfig, + StatusType, ZcashIndexer, ZcashService, }; use crate::{config::ZainodConfig, error::IndexerError}; @@ -61,21 +60,13 @@ pub async fn spawn_indexer( info!(uri = %zebrad_uri, "Connected to node via JsonRPSee"); - #[allow(deprecated)] - match config.backend { - BackendType::State => { - let state_config = StateServiceConfig::try_from(config.clone())?; - Indexer::::launch_inner(state_config, config) - .await - .map(|res| res.0) - } - BackendType::Fetch => { - let fetch_config = FetchServiceConfig::try_from(config.clone())?; - Indexer::::launch_inner(fetch_config, config) - .await - .map(|res| res.0) - } - } + // Both the JSON-RPC (`Rpc`) and direct-`ReadStateService` (`Direct`) connections are + // now served by the single `NodeBackedIndexerService`; the connection is selected + // inside the config conversion from `config.backend`. + let service_config = NodeBackedIndexerServiceConfig::try_from(config.clone())?; + Indexer::::launch_inner(service_config, config) + .await + .map(|res| res.0) } impl Indexer From ec44cff337654edd64a86d6ea5c4aeb0f344cd4a Mon Sep 17 00:00:00 2001 From: zancas Date: Sun, 5 Jul 2026 14:33:05 -0700 Subject: [PATCH 14/58] refactor(zaino-common): DRY logging and activation-height conversions Deduplicate zaino-common and delete its dead public API; net -247 lines of Rust, behavior preserved. logging.rs (386 -> 187 lines): - Collapse the six init_*/try_init_* bodies into one private try_install(LogConfig); init() panics if a global subscriber is already set, try_init() ignores that case. - Delete unused DisplayHash/DisplayHexStr and the unreachable programmatic config surface (LogConfig builder methods, show_span_events, the *_with_config entry points); LogConfig and LogFormat are now private. - The ZAINOLOG_FORMAT/ZAINOLOG_COLOR/RUST_LOG interface and all three output formats are unchanged. config/network.rs (371 -> 308 lines): - New activation_heights_mirror! macro records the NetworkUpgrade variant <-> ActivationHeights field correspondence once and expands to the two mirror From impls plus from_zebra_pairs (all-None init + upgrade-list walk). Exhaustive destructures and match keep full compile-time drift detection; a future network upgrade lands as a single (Variant, field) pair. - Delete Network::zaino_regtest_heights (no callers workspace-wide). CHANGELOG: breaking removals and internal refactors recorded under Unreleased; version bump deferred to release. Verified: cargo check --workspace, cargo clippy --all-targets, cargo fmt, cargo nextest run -p zaino-common (19/19 passed), doctests (8/8 passed). Co-Authored-By: Claude Fable 5 --- packages/zaino-common/CHANGELOG.md | 15 + packages/zaino-common/src/config/network.rs | 211 +++++-------- packages/zaino-common/src/logging.rs | 309 ++++---------------- 3 files changed, 144 insertions(+), 391 deletions(-) diff --git a/packages/zaino-common/CHANGELOG.md b/packages/zaino-common/CHANGELOG.md index e2356e551..d5f413d0e 100644 --- a/packages/zaino-common/CHANGELOG.md +++ b/packages/zaino-common/CHANGELOG.md @@ -31,8 +31,23 @@ and this library adheres to Rust's notion of unrecognized key under `[storage.database]` (e.g. a stale `sync_write_batch_bytes`) is a hard parse error instead of being silently ignored and falling back to the default budget — the silent fallback previously OOM-killed nodes. +- The `ActivationHeights` ↔ `ConfiguredActivationHeights` conversions and the + zebra-network → `ActivationHeights` extraction are now generated from a single + variant/field pair list (internal refactor; conversion behavior unchanged). +- `logging::init` / `logging::try_init` build their subscriber through one shared + installer (internal refactor). The `ZAINOLOG_FORMAT` / `ZAINOLOG_COLOR` / + `RUST_LOG` runtime interface and output formats are unchanged. ### Deprecated ### Removed +- **Breaking** — `Network::zaino_regtest_heights` (unused; regtest heights come + from `ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS` or an explicit `ActivationHeights`). +- **Breaking** — `logging::DisplayHash` and `logging::DisplayHexStr` (unused). +- **Breaking** — the unused programmatic logging-configuration surface: + `logging::init_with_config` / `logging::try_init_with_config` and the `LogConfig` + / `LogFormat` types (including the `LogConfig` builder methods and + `show_span_events`, which no caller could reach). Logging is configured via the + `ZAINOLOG_FORMAT` / `ZAINOLOG_COLOR` / `RUST_LOG` environment variables, whose + behavior is unchanged. ### Fixed ## [0.2.0] - 2026-06-17 diff --git a/packages/zaino-common/src/config/network.rs b/packages/zaino-common/src/config/network.rs index d518d895b..da2e85862 100644 --- a/packages/zaino-common/src/config/network.rs +++ b/packages/zaino-common/src/config/network.rs @@ -141,97 +141,87 @@ impl Default for ActivationHeights { } } -impl From for ActivationHeights { - fn from( - ConfiguredActivationHeights { - before_overwinter, - overwinter, - sapling, - blossom, - heartwood, - canopy, - nu5, - nu6, - nu6_1, - nu6_2, - nu6_3, - nu7, - }: ConfiguredActivationHeights, - ) -> Self { - Self { - before_overwinter, - overwinter, - sapling, - blossom, - heartwood, - canopy, - nu5, - nu6, - nu6_1, - nu6_2, - nu6_3, - nu7, +/// Records the `NetworkUpgrade`-variant ↔ `ActivationHeights`-field correspondence +/// exactly once, generating everything derived from it: the two field-by-field +/// `From` conversions between [`ActivationHeights`] and zebra's +/// [`ConfiguredActivationHeights`] (the structs share field names), and +/// `ActivationHeights::from_zebra_pairs`, which walks zebra's +/// `(height, upgrade)` activation list. +/// +/// A declarative macro rather than functions because plain `fn`s cannot abstract +/// over struct fields, and the variant/field spellings (`Nu5`/`nu5`) differ only +/// by casing, which `macro_rules!` cannot derive — hence explicit pairs. +/// +/// Zebra's side of these conversions is structurally stable; the recurring edit +/// here is a new network upgrade, which lands as a single `(Variant, field)` +/// entry in the invocation below (after adding the struct field). The exhaustive +/// destructures and match keep full compile-time drift detection: a new zebra +/// field or variant fails the build until its pair is added. +macro_rules! activation_heights_mirror { + ($(($variant:ident, $field:ident)),* $(,)?) => { + impl From for ActivationHeights { + fn from( + ConfiguredActivationHeights { $($field),* }: ConfiguredActivationHeights, + ) -> Self { + Self { $($field),* } + } } - } -} -impl From for ConfiguredActivationHeights { - fn from( - ActivationHeights { - before_overwinter, - overwinter, - sapling, - blossom, - heartwood, - canopy, - nu5, - nu6, - nu6_1, - nu6_2, - nu6_3, - nu7, - }: ActivationHeights, - ) -> Self { - Self { - before_overwinter, - overwinter, - sapling, - blossom, - heartwood, - canopy, - nu5, - nu6, - nu6_1, - nu6_2, - nu6_3, - nu7, + + impl From for ConfiguredActivationHeights { + fn from(ActivationHeights { $($field),* }: ActivationHeights) -> Self { + Self { $($field),* } + } } - } + + impl ActivationHeights { + /// Builds heights from zebra's `(height, upgrade)` activation list; + /// upgrades absent from the list stay `None`. + fn from_zebra_pairs<'a>( + pairs: impl IntoIterator< + Item = ( + &'a zebra_chain::block::Height, + &'a zebra_chain::parameters::NetworkUpgrade, + ), + >, + ) -> Self { + let mut heights = Self { $($field: None),* }; + for (height, upgrade) in pairs { + match upgrade { + zebra_chain::parameters::NetworkUpgrade::Genesis => (), + $( + zebra_chain::parameters::NetworkUpgrade::$variant => { + heights.$field = Some(height.0) + } + )* + } + } + heights + } + } + }; } +activation_heights_mirror!( + (BeforeOverwinter, before_overwinter), + (Overwinter, overwinter), + (Sapling, sapling), + (Blossom, blossom), + (Heartwood, heartwood), + (Canopy, canopy), + (Nu5, nu5), + (Nu6, nu6), + (Nu6_1, nu6_1), + (Nu6_2, nu6_2), + (Nu6_3, nu6_3), + (Nu7, nu7), +); + impl Network { /// Convert to Zebra's network type for internal use (alias for to_zebra_default). pub fn to_zebra_network(&self) -> zebra_chain::parameters::Network { self.into() } - /// Get the standard regtest activation heights used by Zaino. - pub fn zaino_regtest_heights() -> ConfiguredActivationHeights { - ConfiguredActivationHeights { - before_overwinter: Some(1), - overwinter: Some(1), - sapling: Some(1), - blossom: Some(1), - heartwood: Some(1), - canopy: Some(1), - nu5: Some(1), - nu6: Some(1), - nu6_1: None, - nu6_2: None, - nu6_3: None, - nu7: None, - } - } - /// Determines if we should wait for the server to fully sync. Used for testing /// /// - Mainnet/Testnet: Skip sync (false) because we don't want to sync real chains in tests @@ -261,62 +251,9 @@ impl From for Network { zebra_chain::parameters::Network::Mainnet => Network::Mainnet, zebra_chain::parameters::Network::Testnet(parameters) => { if parameters.is_regtest() { - let mut activation_heights = ActivationHeights { - before_overwinter: None, - overwinter: None, - sapling: None, - blossom: None, - heartwood: None, - canopy: None, - nu5: None, - nu6: None, - nu6_1: None, - nu6_2: None, - nu6_3: None, - nu7: None, - }; - for (height, upgrade) in parameters.activation_heights().iter() { - match upgrade { - zebra_chain::parameters::NetworkUpgrade::Genesis => (), - zebra_chain::parameters::NetworkUpgrade::BeforeOverwinter => { - activation_heights.before_overwinter = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Overwinter => { - activation_heights.overwinter = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Sapling => { - activation_heights.sapling = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Blossom => { - activation_heights.blossom = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Heartwood => { - activation_heights.heartwood = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Canopy => { - activation_heights.canopy = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Nu5 => { - activation_heights.nu5 = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Nu6 => { - activation_heights.nu6 = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Nu6_1 => { - activation_heights.nu6_1 = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Nu6_2 => { - activation_heights.nu6_2 = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Nu6_3 => { - activation_heights.nu6_3 = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Nu7 => { - activation_heights.nu7 = Some(height.0) - } - } - } - Network::Regtest(activation_heights) + Network::Regtest(ActivationHeights::from_zebra_pairs( + parameters.activation_heights().iter(), + )) } else { Network::Testnet } diff --git a/packages/zaino-common/src/logging.rs b/packages/zaino-common/src/logging.rs index e97703bd3..7bf812671 100644 --- a/packages/zaino-common/src/logging.rs +++ b/packages/zaino-common/src/logging.rs @@ -18,23 +18,19 @@ //! ```no_run //! use zaino_common::logging; //! -//! // Initialize with defaults (tree format, info level) +//! // Initialize logging, configured via the environment variables above. //! logging::init(); -//! -//! // Or with custom configuration -//! logging::init_with_config(logging::LogConfig::default().format(logging::LogFormat::Json)); //! ``` use std::env; -use std::fmt; use std::io::IsTerminal; use time::macros::format_description; use tracing::Level; use tracing_subscriber::{ - fmt::{format::FmtSpan, time::UtcTime}, + fmt::time::UtcTime, layer::SubscriberExt, - util::SubscriberInitExt, + util::{SubscriberInitExt, TryInitError}, EnvFilter, }; use tracing_tree::HierarchicalLayer; @@ -45,7 +41,7 @@ const TIME_FORMAT: &[time::format_description::FormatItem<'static>] = /// Log output format. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -pub enum LogFormat { +enum LogFormat { /// Hierarchical tree view showing span nesting. Tree, /// Flat chronological stream (default). @@ -57,7 +53,7 @@ pub enum LogFormat { impl LogFormat { /// Parse from string (case-insensitive). - pub fn parse_str(s: &str) -> Option { + fn parse_str(s: &str) -> Option { match s.to_lowercase().as_str() { "tree" => Some(LogFormat::Tree), "stream" => Some(LogFormat::Stream), @@ -67,7 +63,7 @@ impl LogFormat { } /// Get from ZAINOLOG_FORMAT environment variable. - pub fn from_env() -> Self { + fn from_env() -> Self { env::var("ZAINOLOG_FORMAT") .ok() .and_then(|s| Self::parse_str(&s)) @@ -75,27 +71,15 @@ impl LogFormat { } } -impl fmt::Display for LogFormat { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - LogFormat::Tree => write!(f, "tree"), - LogFormat::Stream => write!(f, "stream"), - LogFormat::Json => write!(f, "json"), - } - } -} - -/// Logging configuration. +/// Logging configuration, read from the environment. #[derive(Debug, Clone)] -pub struct LogConfig { +struct LogConfig { /// Output format (tree, stream, or json). - pub format: LogFormat, + format: LogFormat, /// Enable ANSI colors. - pub color: bool, + color: bool, /// Default log level. - pub level: Level, - /// Show span events (enter/exit). - pub show_span_events: bool, + level: Level, } impl Default for LogConfig { @@ -119,228 +103,72 @@ impl Default for LogConfig { format: LogFormat::from_env(), color, level: Level::INFO, - show_span_events: false, } } } -impl LogConfig { - /// Create a new config with defaults. - pub fn new() -> Self { - Self::default() - } - - /// Set the log format. - pub fn format(mut self, format: LogFormat) -> Self { - self.format = format; - self - } - - /// Enable or disable colors. - pub fn color(mut self, color: bool) -> Self { - self.color = color; - self - } - - /// Set the default log level. - pub fn level(mut self, level: Level) -> Self { - self.level = level; - self - } - - /// Show span enter/exit events. - pub fn show_span_events(mut self, show: bool) -> Self { - self.show_span_events = show; - self - } -} - -/// Initialize logging with default configuration. +/// Initialize logging, configured from the environment (see module docs). +/// +/// # Panics /// -/// Reads `ZAINOLOG_FORMAT` environment variable to determine format: -/// - "stream" (default): Flat chronological output with timestamps -/// - "tree": Hierarchical span-based output -/// - "json": Machine-parseable JSON +/// Panics if a global tracing subscriber has already been set. pub fn init() { - init_with_config(LogConfig::default()); -} - -/// Initialize logging with custom configuration. -pub fn init_with_config(config: LogConfig) { - // If RUST_LOG is set, use it directly. Otherwise, default to zaino crates only. - // Users can set RUST_LOG=info to see all crates including zebra. - let env_filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| { - EnvFilter::new(format!( - "zaino={level},zainod={level},zainodlib={level}", - level = config.level.as_str() - )) - }); - - match config.format { - LogFormat::Tree => init_tree(env_filter, config), - LogFormat::Stream => init_stream(env_filter, config), - LogFormat::Json => init_json(env_filter), - } + try_install(LogConfig::default()).expect("global tracing subscriber already set"); } /// Try to initialize logging (won't fail if already initialized). /// /// Useful for tests where multiple test functions may try to initialize logging. pub fn try_init() { - try_init_with_config(LogConfig::default()); + let _ = try_install(LogConfig::default()); } -/// Try to initialize logging with custom configuration. -pub fn try_init_with_config(config: LogConfig) { +/// Build the subscriber described by `config` and install it as the global +/// default, erroring if one is already set. +fn try_install(config: LogConfig) -> Result<(), TryInitError> { + // If RUST_LOG is set, use it directly. Otherwise, default to zaino crates only. + // Users can set RUST_LOG=info to see all crates including zebra. let env_filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| { EnvFilter::new(format!( "zaino={level},zainod={level},zainodlib={level}", level = config.level.as_str() )) }); + let registry = tracing_subscriber::registry().with(env_filter); match config.format { - LogFormat::Tree => { - let _ = try_init_tree(env_filter, config); - } - LogFormat::Stream => { - let _ = try_init_stream(env_filter, config); - } - LogFormat::Json => { - let _ = try_init_json(env_filter); - } - } -} - -fn init_tree(env_filter: EnvFilter, config: LogConfig) { - let layer = HierarchicalLayer::new(2) - .with_ansi(config.color) - .with_targets(true) - .with_bracketed_fields(true) - .with_indent_lines(true) - .with_thread_ids(false) - .with_thread_names(false) - .with_deferred_spans(true) // Only show spans when they have events - .with_verbose_entry(false) // Don't repeat span info on entry - .with_verbose_exit(false); // Don't repeat span info on exit - - tracing_subscriber::registry() - .with(env_filter) - .with(layer) - .init(); -} - -fn try_init_tree( - env_filter: EnvFilter, - config: LogConfig, -) -> Result<(), tracing_subscriber::util::TryInitError> { - let layer = HierarchicalLayer::new(2) - .with_ansi(config.color) - .with_targets(true) - .with_bracketed_fields(true) - .with_indent_lines(true) - .with_thread_ids(false) - .with_thread_names(false) - .with_deferred_spans(true) // Only show spans when they have events - .with_verbose_entry(false) // Don't repeat span info on entry - .with_verbose_exit(false); // Don't repeat span info on exit - - tracing_subscriber::registry() - .with(env_filter) - .with(layer) - .try_init() -} - -fn init_stream(env_filter: EnvFilter, config: LogConfig) { - let span_events = if config.show_span_events { - FmtSpan::ENTER | FmtSpan::EXIT - } else { - FmtSpan::NONE - }; - - tracing_subscriber::fmt() - .with_env_filter(env_filter) - .with_timer(UtcTime::new(TIME_FORMAT)) - .with_target(true) - .with_ansi(config.color) - .with_span_events(span_events) - .pretty() - .init(); -} - -fn try_init_stream( - env_filter: EnvFilter, - config: LogConfig, -) -> Result<(), tracing_subscriber::util::TryInitError> { - let span_events = if config.show_span_events { - FmtSpan::ENTER | FmtSpan::EXIT - } else { - FmtSpan::NONE - }; - - let fmt_layer = tracing_subscriber::fmt::layer() - .with_timer(UtcTime::new(TIME_FORMAT)) - .with_target(true) - .with_ansi(config.color) - .with_span_events(span_events) - .pretty(); - - tracing_subscriber::registry() - .with(env_filter) - .with(fmt_layer) - .try_init() -} - -fn init_json(env_filter: EnvFilter) { - // JSON format keeps full RFC3339 timestamps for machine parsing - tracing_subscriber::fmt() - .with_env_filter(env_filter) - .json() - .with_timer(UtcTime::rfc_3339()) - .with_target(true) - .init(); -} - -fn try_init_json(env_filter: EnvFilter) -> Result<(), tracing_subscriber::util::TryInitError> { - // JSON format keeps full RFC3339 timestamps for machine parsing - let fmt_layer = tracing_subscriber::fmt::layer() - .json() - .with_timer(UtcTime::rfc_3339()) - .with_target(true); - - tracing_subscriber::registry() - .with(env_filter) - .with(fmt_layer) - .try_init() -} - -/// Wrapper for displaying hashes in a compact format. -/// -/// Shows the first 4 bytes (8 hex chars) followed by "..". -pub struct DisplayHash<'a>(pub &'a [u8]); - -impl fmt::Display for DisplayHash<'_> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - if self.0.len() >= 4 { - write!(f, "{}..", hex::encode(&self.0[..4])) - } else { - write!(f, "{}", hex::encode(self.0)) - } - } -} - -/// Wrapper for displaying hex strings in a compact format. -/// -/// Shows the first 8 chars followed by "..". -pub struct DisplayHexStr<'a>(pub &'a str); - -impl fmt::Display for DisplayHexStr<'_> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - if self.0.len() > 8 { - write!(f, "{}..", &self.0[..8]) - } else { - write!(f, "{}", self.0) - } + LogFormat::Tree => registry + .with( + HierarchicalLayer::new(2) + .with_ansi(config.color) + .with_targets(true) + .with_bracketed_fields(true) + .with_indent_lines(true) + .with_thread_ids(false) + .with_thread_names(false) + .with_deferred_spans(true) // Only show spans when they have events + .with_verbose_entry(false) // Don't repeat span info on entry + .with_verbose_exit(false), // Don't repeat span info on exit + ) + .try_init(), + LogFormat::Stream => registry + .with( + tracing_subscriber::fmt::layer() + .with_timer(UtcTime::new(TIME_FORMAT)) + .with_target(true) + .with_ansi(config.color) + .pretty(), + ) + .try_init(), + LogFormat::Json => registry + .with( + // JSON format keeps full RFC3339 timestamps for machine parsing + tracing_subscriber::fmt::layer() + .json() + .with_timer(UtcTime::rfc_3339()) + .with_target(true), + ) + .try_init(), } } @@ -356,31 +184,4 @@ mod tests { assert_eq!(LogFormat::parse_str("json"), Some(LogFormat::Json)); assert_eq!(LogFormat::parse_str("unknown"), None); } - - #[test] - fn test_display_hash() { - let hash = [0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc, 0xde, 0xf0]; - assert_eq!(format!("{}", DisplayHash(&hash)), "12345678.."); - } - - #[test] - fn test_display_hex_str() { - let hex_str = "1234567890abcdef"; - assert_eq!(format!("{}", DisplayHexStr(hex_str)), "12345678.."); - - let short_hex = "1234"; - assert_eq!(format!("{}", DisplayHexStr(short_hex)), "1234"); - } - - #[test] - fn test_config_builder() { - let config = LogConfig::new() - .format(LogFormat::Json) - .color(false) - .level(Level::DEBUG); - - assert_eq!(config.format, LogFormat::Json); - assert!(!config.color); - assert_eq!(config.level, Level::DEBUG); - } } From 7d19e0e91b8a75b2b59504e2cccd99bf13ab1b2e Mon Sep 17 00:00:00 2001 From: zancas Date: Sun, 5 Jul 2026 14:45:44 -0700 Subject: [PATCH 15/58] build(zaino-common): drop unused deps thiserror, nu-ansi-term, hex MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Compiler-verified sweep: each dependency in zaino-common's manifest was deleted one at a time and the crate re-checked with `cargo check -p zaino-common --all-targets`; a passing check kept the deletion, a failing one restored the dependency. Dropped (zero references, check passes without them): - thiserror — never used by this crate - nu-ansi-term — never used directly; tracing-tree pulls its own copy transitively - hex — last consumers were DisplayHash/DisplayHexStr, removed in the preceding DRY commit Kept (check fails without them): zebra-chain, serde, tracing, tracing-subscriber, tracing-tree, time. The root [workspace.dependencies] nu-ansi-term entry is now unused by every member but left in place; removing it is a separate root-manifest decision. Verified: cargo check --workspace --all-targets clean. Co-Authored-By: Claude Fable 5 --- Cargo.lock | 3 --- packages/zaino-common/CHANGELOG.md | 4 ++++ packages/zaino-common/Cargo.toml | 5 ----- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f725865bb..05f34f4d3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5942,10 +5942,7 @@ dependencies = [ name = "zaino-common" version = "0.2.0" dependencies = [ - "hex", - "nu-ansi-term", "serde", - "thiserror 2.0.18", "time", "tracing", "tracing-subscriber", diff --git a/packages/zaino-common/CHANGELOG.md b/packages/zaino-common/CHANGELOG.md index d5f413d0e..efdf02ad7 100644 --- a/packages/zaino-common/CHANGELOG.md +++ b/packages/zaino-common/CHANGELOG.md @@ -39,6 +39,10 @@ and this library adheres to Rust's notion of `RUST_LOG` runtime interface and output formats are unchanged. ### Deprecated ### Removed +- Unused dependencies `thiserror`, `nu-ansi-term`, and `hex` (`hex`'s last + consumers were the display wrappers removed below). Verified empirically: + each dependency was deleted in turn and the crate re-checked with + `cargo check --all-targets`. - **Breaking** — `Network::zaino_regtest_heights` (unused; regtest heights come from `ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS` or an explicit `ActivationHeights`). - **Breaking** — `logging::DisplayHash` and `logging::DisplayHexStr` (unused). diff --git a/packages/zaino-common/Cargo.toml b/packages/zaino-common/Cargo.toml index 5eed93aba..578f58489 100644 --- a/packages/zaino-common/Cargo.toml +++ b/packages/zaino-common/Cargo.toml @@ -15,13 +15,8 @@ zebra-chain = { workspace = true } # Serialization serde = { workspace = true, features = ["derive"] } -# Error handling -thiserror = { workspace = true } - # Logging tracing = { workspace = true } tracing-subscriber = { workspace = true } tracing-tree = { workspace = true } -nu-ansi-term = { workspace = true } -hex = { workspace = true } time = { workspace = true } From 24c31aa5da9f98fa01eb3475fee8909ce82f37ad Mon Sep 17 00:00:00 2001 From: zancas Date: Fri, 3 Jul 2026 19:24:38 -0700 Subject: [PATCH 16/58] fix: install rustls CryptoProvider explicitly and pin provider to ring zainod 0.5.0 panicked at startup when gRPC TLS was enabled: the zebra bump added a second rustls provider path (zebra-rpc -> zebra-node-services/rpc-client -> reqwest/rustls-tls -> ring) alongside zaino's own reqwest (aws-lc-rs), and rustls 0.23 refuses to auto-select a provider when both features are enabled. Reproduced and confirmed by a two-arm experiment recorded on the issue. Fixes zingolabs/zaino#1360. - workspace reqwest: "rustls" -> "rustls-no-provider" (drops aws-lc-rs from the graph entirely) - zaino-serve: tonic gains "tls-ring" so zaino chooses ring itself - zaino-common: new crypto::ensure_default_crypto_provider(), a guarded first-install-wins ring install that respects an embedder's provider; this is the load-bearing fix (Arm B: dual-provider graph passes with it), the feature pinning is defense-in-depth and build hygiene - called at the two boundaries that build rustls configs: the outbound reqwest client factory in zaino-fetch (new build_rpc_client, DRYing three hand-rolled ClientBuilder chains) and the gRPC TLS acceptor in zaino-serve - regression test: TLS-enabled spawn + real handshake against a committed test-CA/leaf fixture (CI previously ran plaintext only); fails with the exact reported panic under the pre-fix feature graph Co-Authored-By: Claude Fable 5 --- Cargo.lock | 49 +---------- Cargo.toml | 2 +- packages/zaino-common/Cargo.toml | 1 + packages/zaino-common/src/crypto.rs | 27 ++++++ packages/zaino-common/src/lib.rs | 1 + .../zaino-fetch/src/jsonrpsee/connector.rs | 48 ++++++----- packages/zaino-serve/Cargo.toml | 3 +- packages/zaino-serve/src/server/grpc.rs | 3 + packages/zaino-serve/src/server/grpc/tests.rs | 1 + .../grpc/tests/fixtures/tls_test_ca_cert.pem | 12 +++ .../grpc/tests/fixtures/tls_test_cert.pem | 12 +++ .../grpc/tests/fixtures/tls_test_key.pem | 5 ++ .../zaino-serve/src/server/grpc/tests/tls.rs | 82 +++++++++++++++++++ 13 files changed, 178 insertions(+), 68 deletions(-) create mode 100644 packages/zaino-common/src/crypto.rs create mode 100644 packages/zaino-serve/src/server/grpc/tests/fixtures/tls_test_ca_cert.pem create mode 100644 packages/zaino-serve/src/server/grpc/tests/fixtures/tls_test_cert.pem create mode 100644 packages/zaino-serve/src/server/grpc/tests/fixtures/tls_test_key.pem create mode 100644 packages/zaino-serve/src/server/grpc/tests/tls.rs diff --git a/Cargo.lock b/Cargo.lock index f725865bb..77ba5a672 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -171,28 +171,6 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" -[[package]] -name = "aws-lc-rs" -version = "1.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" -dependencies = [ - "aws-lc-sys", - "zeroize", -] - -[[package]] -name = "aws-lc-sys" -version = "0.41.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" -dependencies = [ - "cc", - "cmake", - "dunce", - "fs_extra", -] - [[package]] name = "axum" version = "0.8.9" @@ -710,15 +688,6 @@ dependencies = [ "zip32", ] -[[package]] -name = "cmake" -version = "0.1.58" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" -dependencies = [ - "cc", -] - [[package]] name = "cobs" version = "0.3.0" @@ -1190,12 +1159,6 @@ dependencies = [ "syn 2.0.118", ] -[[package]] -name = "dunce" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" - [[package]] name = "dyn-clone" version = "1.0.20" @@ -1467,12 +1430,6 @@ dependencies = [ "rand_core 0.6.4", ] -[[package]] -name = "fs_extra" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" - [[package]] name = "funty" version = "2.0.0" @@ -3487,7 +3444,6 @@ version = "0.11.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e" dependencies = [ - "aws-lc-rs", "bytes", "getrandom 0.3.4", "lru-slab", @@ -3877,7 +3833,6 @@ dependencies = [ "log", "percent-encoding", "pin-project-lite", - "quinn", "rustls", "rustls-pki-types", "rustls-platform-verifier", @@ -4003,7 +3958,6 @@ version = "0.23.41" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" dependencies = [ - "aws-lc-rs", "log", "once_cell", "ring", @@ -4068,7 +4022,6 @@ version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ - "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -5944,6 +5897,7 @@ version = "0.2.0" dependencies = [ "hex", "nu-ansi-term", + "rustls", "serde", "thiserror 2.0.18", "time", @@ -6006,6 +5960,7 @@ dependencies = [ "tonic", "tower 0.4.13", "tracing", + "zaino-common", "zaino-fetch", "zaino-proto", "zaino-state", diff --git a/Cargo.toml b/Cargo.toml index 0bf07dc94..de1292cf9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -77,7 +77,7 @@ http = "1.1" url = "2.5" reqwest = { version = "0.13", default-features = false, features = [ "cookies", - "rustls", + "rustls-no-provider", "webpki-roots", ] } tower = { version = "0.4", features = ["buffer", "util"] } diff --git a/packages/zaino-common/Cargo.toml b/packages/zaino-common/Cargo.toml index 5eed93aba..3ce75c1d6 100644 --- a/packages/zaino-common/Cargo.toml +++ b/packages/zaino-common/Cargo.toml @@ -25,3 +25,4 @@ tracing-tree = { workspace = true } nu-ansi-term = { workspace = true } hex = { workspace = true } time = { workspace = true } +rustls = { version = "0.23", default-features = false, features = ["ring", "std"] } diff --git a/packages/zaino-common/src/crypto.rs b/packages/zaino-common/src/crypto.rs new file mode 100644 index 000000000..8791bbae2 --- /dev/null +++ b/packages/zaino-common/src/crypto.rs @@ -0,0 +1,27 @@ +//! Process-level rustls `CryptoProvider` management. + +/// Installs rustls's `ring` provider as the process-level default if no +/// provider is installed yet. +/// +/// Call this before constructing anything that builds a rustls config — +/// a `reqwest` client (the workspace enables reqwest's +/// `rustls-no-provider` feature, which never auto-selects a provider) or +/// a TLS-enabled tonic server. rustls's own auto-selection is also +/// unreliable here: it works only when exactly one of rustls's `ring` / +/// `aws-lc-rs` features is enabled, and dependency feature-unification +/// can silently enable both, which panicked at runtime on TLS-enabled +/// deployments (zingolabs/zaino#1360). Installing explicitly removes +/// that dependence on the feature graph. +/// +/// The process default is first-install-wins and is not constrained by +/// our crates' rustls features, so an embedder (e.g. zallet) that has +/// already installed a provider keeps it: zaino then handshakes through +/// that provider instead of ring, which is fine for any provider +/// implementing the standard TLS suites (ring and aws-lc-rs both do). +pub fn ensure_default_crypto_provider() { + if rustls::crypto::CryptoProvider::get_default().is_none() { + // A racing concurrent install is the only error case; either + // provider serves. + let _ = rustls::crypto::ring::default_provider().install_default(); + } +} diff --git a/packages/zaino-common/src/lib.rs b/packages/zaino-common/src/lib.rs index e3f62182c..d2c5fcce5 100644 --- a/packages/zaino-common/src/lib.rs +++ b/packages/zaino-common/src/lib.rs @@ -5,6 +5,7 @@ pub mod config; pub mod consensus; +pub mod crypto; pub mod logging; pub mod net; pub mod probing; diff --git a/packages/zaino-fetch/src/jsonrpsee/connector.rs b/packages/zaino-fetch/src/jsonrpsee/connector.rs index cc8090150..056821afa 100644 --- a/packages/zaino-fetch/src/jsonrpsee/connector.rs +++ b/packages/zaino-fetch/src/jsonrpsee/connector.rs @@ -210,6 +210,29 @@ fn record_outbound_rpc_metrics(method: &'static str, start: std::time::Instant, } } +/// Whether the outbound JSON-RPC client keeps a cookie store (required +/// for cookie-authenticated validators). +enum CookieSupport { + Enabled, + Disabled, +} + +/// Builds the outbound JSON-RPC HTTP client shared by every connector +/// entry point: 2s connect timeout, 5s request timeout, no redirects. +fn build_rpc_client(cookies: CookieSupport) -> Result { + // reqwest is built with `rustls-no-provider`, which never + // auto-selects a rustls CryptoProvider (zingolabs/zaino#1360). + zaino_common::crypto::ensure_default_crypto_provider(); + let mut builder = ClientBuilder::new() + .connect_timeout(Duration::from_secs(2)) + .timeout(Duration::from_secs(5)) + .redirect(reqwest::redirect::Policy::none()); + if matches!(cookies, CookieSupport::Enabled) { + builder = builder.cookie_store(true); + } + builder.build() +} + impl JsonRpSeeConnector { /// Creates a new JsonRpSeeConnector with Basic Authentication. pub fn new_with_basic_auth( @@ -217,12 +240,8 @@ impl JsonRpSeeConnector { username: String, password: String, ) -> Result { - let client = ClientBuilder::new() - .connect_timeout(Duration::from_secs(2)) - .timeout(Duration::from_secs(5)) - .redirect(reqwest::redirect::Policy::none()) - .build() - .map_err(TransportError::ReqwestError)?; + let client = + build_rpc_client(CookieSupport::Disabled).map_err(TransportError::ReqwestError)?; Ok(Self { url, @@ -236,13 +255,8 @@ impl JsonRpSeeConnector { pub fn new_with_cookie_auth(url: Url, cookie_path: &Path) -> Result { let cookie_password = read_and_parse_cookie_token(cookie_path)?; - let client = ClientBuilder::new() - .connect_timeout(Duration::from_secs(2)) - .timeout(Duration::from_secs(5)) - .redirect(reqwest::redirect::Policy::none()) - .cookie_store(true) - .build() - .map_err(TransportError::ReqwestError)?; + let client = + build_rpc_client(CookieSupport::Enabled).map_err(TransportError::ReqwestError)?; Ok(Self { url, @@ -979,12 +993,8 @@ async fn test_node_connection( url: Url, auth_method: AuthMethod, ) -> Result<(), TestNodeConnectionError> { - let client = Client::builder() - .connect_timeout(std::time::Duration::from_secs(2)) - .timeout(std::time::Duration::from_secs(5)) - .redirect(reqwest::redirect::Policy::none()) - .build() - .map_err(TestNodeConnectionError::ClientBuild)?; + let client = + build_rpc_client(CookieSupport::Disabled).map_err(TestNodeConnectionError::ClientBuild)?; let request_body = r#"{"jsonrpc":"2.0","method":"getinfo","params":[],"id":1}"#; let mut request_builder = client diff --git a/packages/zaino-serve/Cargo.toml b/packages/zaino-serve/Cargo.toml index b5cee4792..cef85e961 100644 --- a/packages/zaino-serve/Cargo.toml +++ b/packages/zaino-serve/Cargo.toml @@ -53,9 +53,10 @@ metrics = { workspace = true, optional = true } # Miscellaneous Workspace tokio = { workspace = true, features = ["full"] } -tonic = { workspace = true, features = ["tls-native-roots"] } +tonic = { workspace = true, features = ["tls-native-roots", "tls-ring"] } thiserror = { workspace = true } futures = { workspace = true } jsonrpsee = { workspace = true, features = ["server", "macros"] } serde = { workspace = true, features = ["derive"] } tower = { workspace = true } +zaino-common.workspace = true diff --git a/packages/zaino-serve/src/server/grpc.rs b/packages/zaino-serve/src/server/grpc.rs index 7d77b9ad5..e0854aea1 100644 --- a/packages/zaino-serve/src/server/grpc.rs +++ b/packages/zaino-serve/src/server/grpc.rs @@ -74,6 +74,9 @@ impl TonicServer { let mut server_builder = Server::builder(); if let Some(tls_config) = server_config.get_valid_tls().await? { + // Building the TLS acceptor requires a process-level rustls + // CryptoProvider (zingolabs/zaino#1360). + zaino_common::crypto::ensure_default_crypto_provider(); server_builder = server_builder.tls_config(tls_config).map_err(|e| { ServerError::ServerConfigError(format!("TLS configuration error: {e}")) })?; diff --git a/packages/zaino-serve/src/server/grpc/tests.rs b/packages/zaino-serve/src/server/grpc/tests.rs index 05eb1db71..946cbaa9a 100644 --- a/packages/zaino-serve/src/server/grpc/tests.rs +++ b/packages/zaino-serve/src/server/grpc/tests.rs @@ -1,3 +1,4 @@ //! Zaino-Serve gRPC server unit tests. mod spawn; +mod tls; diff --git a/packages/zaino-serve/src/server/grpc/tests/fixtures/tls_test_ca_cert.pem b/packages/zaino-serve/src/server/grpc/tests/fixtures/tls_test_ca_cert.pem new file mode 100644 index 000000000..8c6998f49 --- /dev/null +++ b/packages/zaino-serve/src/server/grpc/tests/fixtures/tls_test_ca_cert.pem @@ -0,0 +1,12 @@ +-----BEGIN CERTIFICATE----- +MIIBzzCCAXWgAwIBAgIUKx27V49MTw1PXcKunTwIjHzXWuswCgYIKoZIzj0EAwIw +PDE6MDgGA1UEAwwxemFpbm8tc2VydmUgVExTIHJlZ3Jlc3Npb24tdGVzdCBDQSAo +Tk9UIEEgU0VDUkVUKTAgFw0yNjA3MDQwMTQ0MTNaGA8yMTI2MDYxMDAxNDQxM1ow +PDE6MDgGA1UEAwwxemFpbm8tc2VydmUgVExTIHJlZ3Jlc3Npb24tdGVzdCBDQSAo +Tk9UIEEgU0VDUkVUKTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABGT74i0fh1Vj +gYb0rPW36IYlyvvO/SdT0Mf8dJiFnKmoyqcXtORcw8antcjqe0myghHayx6xEOte +6blNT72knn+jUzBRMB0GA1UdDgQWBBSJ4axoCJxNd1mPygmrBgTI8IosCDAfBgNV +HSMEGDAWgBSJ4axoCJxNd1mPygmrBgTI8IosCDAPBgNVHRMBAf8EBTADAQH/MAoG +CCqGSM49BAMCA0gAMEUCIGhNgBQ5G/pars7fOccRedyOegikzDlMbdM8g4NIDErg +AiEAjlmMXJTBDW5Y/E1rLZgcC/dEdLodnTet3vatf6Vq1lM= +-----END CERTIFICATE----- diff --git a/packages/zaino-serve/src/server/grpc/tests/fixtures/tls_test_cert.pem b/packages/zaino-serve/src/server/grpc/tests/fixtures/tls_test_cert.pem new file mode 100644 index 000000000..f78071a86 --- /dev/null +++ b/packages/zaino-serve/src/server/grpc/tests/fixtures/tls_test_cert.pem @@ -0,0 +1,12 @@ +-----BEGIN CERTIFICATE----- +MIIB1TCCAXqgAwIBAgIBATAKBggqhkjOPQQDAjA8MTowOAYDVQQDDDF6YWluby1z +ZXJ2ZSBUTFMgcmVncmVzc2lvbi10ZXN0IENBIChOT1QgQSBTRUNSRVQpMCAXDTI2 +MDcwNDAxNDQxM1oYDzIxMjYwNjEwMDE0NDEzWjAUMRIwEAYDVQQDDAlsb2NhbGhv +c3QwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAASvy0yCNcEp5I28O0EOd7bJOZ2f +tTYE3E1AHSaMFmiiN/Foxw5KZzF7tRDFZxl0CANx+o+igLjQaerNk/bhoZepo4GS +MIGPMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATAMBgNVHRMBAf8EAjAAMA4G +A1UdDwEB/wQEAwIHgDATBgNVHSUEDDAKBggrBgEFBQcDATAdBgNVHQ4EFgQUtHTz +QTMs8vfbh3m4IVJIah8jAt8wHwYDVR0jBBgwFoAUieGsaAicTXdZj8oJqwYEyPCK +LAgwCgYIKoZIzj0EAwIDSQAwRgIhAIIkUaCFwsFMyAsZOasP/fVQSiU8C5jU8gmt +1xmx/2XvAiEApJiQJ1Kxdd/vjpJ5H1JYktPwoqUpZDmDKJMF1O7zMDM= +-----END CERTIFICATE----- diff --git a/packages/zaino-serve/src/server/grpc/tests/fixtures/tls_test_key.pem b/packages/zaino-serve/src/server/grpc/tests/fixtures/tls_test_key.pem new file mode 100644 index 000000000..290fd34eb --- /dev/null +++ b/packages/zaino-serve/src/server/grpc/tests/fixtures/tls_test_key.pem @@ -0,0 +1,5 @@ +-----BEGIN PRIVATE KEY----- +MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQg6+O2+tTz29iMo5yr +SS8d+lb3nG8G7x3s4bHfvC52G3KhRANCAASvy0yCNcEp5I28O0EOd7bJOZ2ftTYE +3E1AHSaMFmiiN/Foxw5KZzF7tRDFZxl0CANx+o+igLjQaerNk/bhoZep +-----END PRIVATE KEY----- diff --git a/packages/zaino-serve/src/server/grpc/tests/tls.rs b/packages/zaino-serve/src/server/grpc/tests/tls.rs new file mode 100644 index 000000000..02b8287ad --- /dev/null +++ b/packages/zaino-serve/src/server/grpc/tests/tls.rs @@ -0,0 +1,82 @@ +//! Regression test for zingolabs/zaino#1360. +//! +//! Spawning the gRPC server with TLS enabled must succeed and serve a +//! working TLS handshake. In 0.5.0, dependency feature-unification +//! enabled both of rustls's `ring` and `aws-lc-rs` provider features, +//! which defeats rustls's automatic provider selection: building the +//! TLS acceptor panicked at runtime ("Could not automatically determine +//! the process-level CryptoProvider") — but only on TLS-enabled +//! deployments, a path no test exercised because every existing test +//! ran plaintext. The fix pins the feature graph to `ring` and installs +//! the provider explicitly (`ensure_crypto_provider`); this test covers +//! the full TLS startup + handshake path so a future provider-selection +//! or certificate-wiring regression fails in CI instead of production. +//! +//! The certificate fixtures were minted once with openssl (EC P-256, +//! ~100-year expiry): a self-signed test CA plus a `localhost` leaf +//! (SAN `DNS:localhost` + `IP:127.0.0.1`) signed by it — rustls-webpki +//! rejects a single self-signed CA cert presented as the server's +//! end-entity cert (`CaUsedAsEndEntity`). The committed leaf key is not +//! a secret; the CA key was discarded, so regeneration remints all +//! three files. The server reads the leaf pair from disk (the +//! production `GrpcTls` path); the client trusts the CA as its root. + +use std::net::{Ipv4Addr, SocketAddr}; +use std::path::PathBuf; + +use tonic::service::Routes; +use tonic::transport::{server::TcpIncoming, Certificate, ClientTlsConfig, Endpoint}; + +use crate::server::config::{GrpcServerConfig, GrpcTls}; +use crate::server::grpc::TonicServer; + +fn fixture_path(name: &str) -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("src/server/grpc/tests/fixtures") + .join(name) +} + +#[tokio::test] +async fn tls_spawn_serves_a_handshake() { + // Pre-bind so the OS-assigned port is known before spawning; hand the + // open socket to `spawn_inner` directly (this module is a child of + // `grpc`, so the private constructor is reachable without the + // `test_dependencies`-gated `spawn_from_listener` wrapper). + let listener = tokio::net::TcpListener::bind(SocketAddr::from((Ipv4Addr::LOCALHOST, 0))) + .await + .expect("bind on ephemeral port must succeed"); + let local_addr = listener + .local_addr() + .expect("local_addr on a bound listener is infallible"); + + let mut server = TonicServer::spawn_inner( + Routes::default(), + GrpcServerConfig { + listen_address: local_addr, + tls: Some(GrpcTls { + cert_path: fixture_path("tls_test_cert.pem"), + key_path: fixture_path("tls_test_key.pem"), + }), + }, + TcpIncoming::from(listener), + ) + .await + .expect("TLS-enabled spawn must succeed (zingolabs/zaino#1360)"); + + let channel = Endpoint::try_from(format!("https://127.0.0.1:{}", local_addr.port())) + .expect("loopback endpoint URI is valid") + .tls_config( + ClientTlsConfig::new() + .ca_certificate(Certificate::from_pem(include_str!( + "fixtures/tls_test_ca_cert.pem" + ))) + .domain_name("localhost"), + ) + .expect("client TLS config from the fixture cert must be valid") + .connect() + .await + .expect("TLS handshake against the spawned server must succeed"); + + drop(channel); + server.close().await; +} From 13163ea0e9d40d74f15648a755797d5295b14a3e Mon Sep 17 00:00:00 2001 From: zancas Date: Fri, 3 Jul 2026 19:53:08 -0700 Subject: [PATCH 17/58] build: drop unnecessary rustls "std" feature from zaino-common MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two functions zaino-common calls — CryptoProvider::install_default and get_default — have no-std implementations in rustls 0.23 (the process default falls back from std::sync::OnceLock to once_cell::race::OnceBox), so "std" is not required by our usage. It is also enabled in any full build regardless, because tokio-rustls hard-enables rustls/std. Minimum-necessary-scope: declare only "ring". Co-Authored-By: Claude Fable 5 --- packages/zaino-common/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/zaino-common/Cargo.toml b/packages/zaino-common/Cargo.toml index 3ce75c1d6..437927085 100644 --- a/packages/zaino-common/Cargo.toml +++ b/packages/zaino-common/Cargo.toml @@ -25,4 +25,4 @@ tracing-tree = { workspace = true } nu-ansi-term = { workspace = true } hex = { workspace = true } time = { workspace = true } -rustls = { version = "0.23", default-features = false, features = ["ring", "std"] } +rustls = { version = "0.23", default-features = false, features = ["ring"] } From 30681b93ed4882433f734ea67b63fe0014f3b495 Mon Sep 17 00:00:00 2001 From: zancas Date: Sat, 4 Jul 2026 10:33:23 -0700 Subject: [PATCH 18/58] ci: report advisory publish dry-run failures as warnings, not a red X continue-on-error makes the run succeed but still shows the failed job as a red X in the PR checks list, which reads as a blocking failure. Move the advisory decision into the step: on non-rc/non-stable contexts a failure emits a ::warning annotation and exits 0; rc/** and stable keep the hard failure. Also generalize the failure epilogue: same-name version divergence from crates.io (new cross-crate API or features, resolved by release-time bumps) is as common a cause as patch overrides. Co-Authored-By: Claude Fable 5 --- .github/workflows/publish-dry-run.yml | 37 ++++++++++++--------------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/.github/workflows/publish-dry-run.yml b/.github/workflows/publish-dry-run.yml index 403de3465..8add6d556 100644 --- a/.github/workflows/publish-dry-run.yml +++ b/.github/workflows/publish-dry-run.yml @@ -14,19 +14,12 @@ jobs: name: cargo publish --dry-run runs-on: ubuntu-latest # Blocking context (pushes to rc/** or stable, and PRs targeting them): - # this job MUST pass. Advisory context (everything else): findings inform - # but the run stays green. Keep this predicate the exact inverse of the - # "Compute check mode" step below — job-level continue-on-error cannot - # read step outputs, so the two encode the same rule twice. - continue-on-error: >- - ${{ - github.event_name == 'pull_request' - && !startsWith(github.base_ref, 'rc/') - && github.base_ref != 'stable' - || github.event_name != 'pull_request' - && !startsWith(github.ref_name, 'rc/') - && github.ref_name != 'stable' - }} + # this job MUST pass. Advisory context (everything else): failures exit 0 + # after emitting a warning annotation, so they do not paint a red X on + # PRs (a continue-on-error'd failing job still shows as failed in the PR + # checks list even though the run succeeds — hence no job-level + # continue-on-error here). The context is computed once in the + # "Compute check mode" step below. steps: - name: Checkout repository uses: actions/checkout@v7 @@ -37,7 +30,6 @@ jobs: - name: Compute check mode id: mode run: | - # Inverse of the job's continue-on-error predicate above. if [[ "${{ github.event_name }}" == "pull_request" ]]; then target="${{ github.base_ref }}" else @@ -96,13 +88,18 @@ jobs: echo "============================================" echo "" echo "The workspace does not publish cleanly as a set. Common causes:" - echo " - a [patch.crates-io] override pulling in unreleased upstream code" + echo " - a [patch.crates-io] override pulling in unreleased upstream" + echo " code (published crates cannot depend on unpublished revisions)" echo " - a dependency on a git revision or path outside the workspace" echo " - a crate that does not compile from its packaged form" + echo " - new cross-crate API or features that exist in the workspace but" + echo " not in the published dependency; resolved by the version bump +" + echo " publish of that dependency at the next release" + echo "" + if [[ "${{ steps.mode.outputs.mode }}" == "advisory" ]]; then + echo "::warning::publish dry-run failed — the workspace would not release cleanly. Advisory on this branch, but it must be resolved before the next release." + exit 0 + fi + echo "::error::publish dry-run failed — the workspace would not release cleanly." exit 1 fi - - - name: Annotate on failure (advisory context only) - if: failure() && steps.mode.outputs.mode == 'advisory' - run: | - echo "::warning::publish dry-run failed — the workspace would not release cleanly. This must be resolved before the next release." From 90522a2e4154e5481fc6ea4383b126b99b70cc47 Mon Sep 17 00:00:00 2001 From: zancas Date: Sun, 5 Jul 2026 15:43:04 -0700 Subject: [PATCH 19/58] =?UTF-8?q?docs:=20ADR-0006=20=E2=80=94=20prefer=20a?= =?UTF-8?q?ws-lc-rs=20CryptoProvider;=20start=20CONTEXT.md=20glossary?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Record the decision reversing zaino's rustls provider from ring to aws-lc-rs, per Daira Emma's post-quantum argument: - aws-lc-rs is the preferred (first-install-wins, not mandated) provider - rustls prefer-post-quantum enabled: X25519MLKEM768 leads on the client leg; rustls servers follow client group preference, so inbound hybrid uptake is client-driven - classical key exchange accepted but deprecating, gated on wallet stacks (zingolib is ring-pinned; ZODL rides mobile platform TLS) - ring tolerated in Cargo.lock via the zebra-node-services reqwest path, fenced by an exact-snapshot CI lint, until the upstream zebra fix References: - https://crates.io/crates/rustls-post-quantum - https://www.reddit.com/r/rust/comments/1de13y6/comment/l89pbmc/ CONTEXT.md begins the project glossary: Preferred CryptoProvider, Hybrid key exchange, Classical key exchange. Co-Authored-By: Claude Fable 5 --- CONTEXT.md | 24 +++++ ...006-aws-lc-rs-preferred-crypto-provider.md | 90 +++++++++++++++++++ 2 files changed, 114 insertions(+) create mode 100644 docs/adr/0006-aws-lc-rs-preferred-crypto-provider.md diff --git a/CONTEXT.md b/CONTEXT.md index 4d40dd72f..11abf3640 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -29,3 +29,27 @@ its packaged content differs. The tree cannot be released until that crate's version is bumped. An unchanged crate keeping its published version is not a violation. _Avoid_: stale version, forgotten bump + +### TLS and cryptography + +**Preferred CryptoProvider**: +The TLS cryptography provider zaino installs as the process-wide default +when no provider is installed yet. A preference, not a mandate: an +embedder (e.g. zallet) that installs a provider before zaino keeps its +choice, and zaino handshakes through it. +_Avoid_: "enforced provider" (implies zaino overrides an embedder's +already-installed provider; it never does) + +**Hybrid key exchange**: +A TLS 1.3 key-exchange group combining a classical curve with a +post-quantum KEM (key-encapsulation mechanism), e.g. X25519MLKEM768. +Secure if either component holds. Hybrids are the post-quantum +deployment vehicle, not a classical fallback. +_Avoid_: calling hybrids "classical" because they contain a classical +component + +**Classical key exchange**: +A key-exchange group with no post-quantum component (X25519, SECP256R1, +SECP384R1). Deprecated in zaino: still accepted for client +compatibility, slated for refusal once major wallet stacks negotiate +hybrids. diff --git a/docs/adr/0006-aws-lc-rs-preferred-crypto-provider.md b/docs/adr/0006-aws-lc-rs-preferred-crypto-provider.md new file mode 100644 index 000000000..13306503c --- /dev/null +++ b/docs/adr/0006-aws-lc-rs-preferred-crypto-provider.md @@ -0,0 +1,90 @@ +# aws-lc-rs is the preferred CryptoProvider; classical key exchange is deprecating + +## Status + +accepted (reverses the ring choice made on zingolabs/zaino#1366 before it +merged) + +## Context and decision + +Zaino installs a rustls `CryptoProvider` as the process-wide default at +its two TLS boundaries (`zaino-serve`'s gRPC server, `zaino-fetch`'s +jsonrpsee connector) via `zaino_common::crypto::ensure_default_crypto_provider` +(zingolabs/zaino#1360). The provider was initially pinned to **ring** for +two reasons: zebra used ring (shared attack surface across the deployed +stack), and ring avoided the `aws-lc-sys` + `cmake` build dependencies. + +Daira Emma argued for **aws-lc-rs** instead: post-quantum TLS key +exchange in the rustls ecosystem requires the aws-lc-rs provider — the +ring provider has no ML-KEM support at all — and the provider +architecture makes the swap cheap +(, +). + +We decide: + +1. **aws-lc-rs is zaino's preferred CryptoProvider** — a *preference, + not a mandate*: installation stays first-install-wins, so an embedder + (e.g. zallet) that installs a provider before zaino keeps its choice. + Enforcement lives in the feature graph (`rustls` `aws_lc_rs` feature, + tonic `tls-aws-lc`), not at runtime. +2. **rustls's `prefer-post-quantum` feature is enabled**, reordering the + provider's default key-exchange groups so the hybrid X25519MLKEM768 + leads (same membership; in rustls 0.23 the hybrid is in the default + set either way). Verified effect in rustls 0.23.41: on zaino's + *outbound* (client-role) TLS the hybrid share is offered upfront, and + rustls adds the hybrid's X25519 component as a free second key share, + so classical-only servers negotiate without a HelloRetryRequest — the + cost is ~1.2 KB of ClientHello, not a round trip. On the *server* + side rustls follows the client's group preference, so inbound + negotiation outcomes are unchanged by the feature; hybrid uptake on + inbound connections is entirely client-driven, which is why the + deprecation below is gated on client stacks. +3. **Classical key exchange (X25519, SECP256R1, SECP384R1) remains + accepted but is deprecated.** Refusing it today would break both + flagship wallets: zingolib's TLS stack is feature-pinned to ring (no + ML-KEM possible), and ZODL rides mobile platform TLS where hybrid + support is still rolling out. The refusal precondition — zingolib on + aws-lc-rs with hybrid preferred, and ZODL's Android/iOS platform + stacks negotiating X25519MLKEM768 — is tracked in a dedicated issue; + deprecation is documented in the changelog rather than signalled at + runtime. +4. **Ring stays in `Cargo.lock`, tolerated and fenced, until zebra drops + it.** `zebra-rpc` hard-enables `zebra-node-services/rpc-client`, + whose reqwest 0.12 `rustls-tls` feature compiles ring in — reqwest + 0.12 offers no aws-lc alternative, only `*-no-provider` variants. This + is dormant code, not a live handshake path: reqwest consults the + process-default provider first and falls back to ring only when none + is installed, so both zaino (which installs aws-lc-rs early) and + embedders (whose earlier install wins) preempt it. A CI lint pins + `cargo tree --invert ring` to exactly this one known path and fails on + drift in *either* direction: a new ring consumer is a regression to + investigate; the path vanishing means zebra dropped ring, the + allowlist gets deleted, and the guard tightens to "ring absent". + +## Considered options + +- **Stay on ring, aligned with zebra.** Rejected: forecloses post-quantum + key exchange entirely; the harvest-now-decrypt-later argument applies + to wallet↔zaino TLS traffic today. +- **Refuse classical key exchange (hybrid-only).** Rejected for now: + hard-locks out every wallet whose TLS stack lacks X25519MLKEM768, + which today is all of them. Revisit when the tracked precondition is + met. +- **Fork-pin zebra to purge ring from the lock immediately.** Rejected: + the fork tax (pin churn here and in the wallet-tests workspace) buys + only the removal of dormant code; downstream is already protected by + provider preemption. Instead we upstream the fix to zebra — + `zebra-node-services` switches reqwest to a `*-no-provider` feature + and zebrad installs its own default provider (the same pattern as + zaino's #1360 fix) — and pick it up at the next zebra bump. + +## Consequences + +- `aws-lc-sys` and `cmake` return to the build graph (both build images + already install cmake). +- Zaino diverges from zebra's provider until zebra moves; the CI lint + makes the eventual convergence loud instead of silent. +- TLS 1.2 remains available only as long as classical key exchange + does; the refusal that ends the classical deprecation also implies + TLS 1.3-only. From a62dcd3bbe9ce6730fd50bb4ae1fcf6f833b1c0c Mon Sep 17 00:00:00 2001 From: zancas Date: Sun, 5 Jul 2026 16:18:28 -0700 Subject: [PATCH 20/58] build: prefer aws-lc-rs CryptoProvider with post-quantum key exchange MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Switch zaino's rustls provider from ring to aws-lc-rs and enable rustls's prefer-post-quantum feature, so the X25519MLKEM768 hybrid key exchange leads zaino's outbound handshakes (rustls servers follow the client's group preference, so inbound hybrid uptake is client-driven). Installation stays first-install-wins: an embedder that installs a provider before zaino keeps its choice. Full rationale in ADR-0006. - zaino-common: rustls features ring → aws_lc_rs + prefer-post-quantum; ensure_default_crypto_provider installs aws-lc-rs - zaino-serve: tonic tls-ring → tls-aws-lc - ring remains in Cargo.lock solely via zebra-rpc → zebra-node-services rpc-client → reqwest 0.12 "rustls-tls" (reqwest 0.12 offers no aws-lc alternative); it is compiled but dormant — reqwest consults the process-default provider first. Removal tracks an upstream zebra fix. - new workbench guard check-crypto-provider (makers lint-crypto-provider) pins the ring-selecting feature edges to exactly that tolerated path, failing loud on new edges and on ring's disappearance alike - classical key exchange (X25519, SECP256R1, SECP384R1) is deprecated: still accepted for wallet compatibility, slated for refusal once major wallet TLS stacks negotiate hybrids References: - https://crates.io/crates/rustls-post-quantum - https://www.reddit.com/r/rust/comments/1de13y6/comment/l89pbmc/ Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 11 + Cargo.lock | 46 ++++ packages/zaino-common/CHANGELOG.md | 4 + packages/zaino-common/Cargo.toml | 5 +- packages/zaino-common/src/crypto.rs | 17 +- packages/zaino-serve/CHANGELOG.md | 2 + packages/zaino-serve/Cargo.toml | 2 +- .../zaino-serve/src/server/grpc/tests/tls.rs | 5 +- tools/makefiles/lints.toml | 6 + .../src/bin/check-crypto-provider.rs | 206 ++++++++++++++++++ 10 files changed, 296 insertions(+), 8 deletions(-) create mode 100644 tools/workbench/src/bin/check-crypto-provider.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 621a927ce..dba661008 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,17 @@ and this library adheres to Rust's notion of ## Unreleased ### Changed +- TLS: zaino now installs rustls's **aws-lc-rs** CryptoProvider as its + preferred process-level default (was ring) and enables rustls's + `prefer-post-quantum` feature, so the X25519MLKEM768 hybrid key exchange + leads zaino's outbound handshakes (ADR-0006). Installation remains + first-install-wins: an embedder that installs a provider before zaino + keeps its choice. + +### Deprecated +- Classical TLS key exchange (X25519, SECP256R1, SECP384R1) is deprecated: + still offered and accepted for wallet compatibility, slated for refusal + once major wallet stacks negotiate hybrid key exchange (ADR-0006). - **Breaking** — config: `storage.database.sync_write_batch_bytes` (bytes) is renamed to `sync_write_batch_size` and given in **GiB** (default raised from 4 GiB to 32 GiB); this budget now also bounds the txout-set accumulator diff --git a/Cargo.lock b/Cargo.lock index 77ba5a672..5b4174254 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -171,6 +171,29 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "aws-lc-rs" +version = "1.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4342d8937fc7e5dd9b1c60292261c0670c882a2cd1719cfc11b1af41731e32ad" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d9ceb1da931507a12f4fccea479dccd00da1943e1b4ae72d8e502d707361444" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", + "pkg-config", +] + [[package]] name = "axum" version = "0.8.9" @@ -688,6 +711,15 @@ dependencies = [ "zip32", ] +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + [[package]] name = "cobs" version = "0.3.0" @@ -1159,6 +1191,12 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "dyn-clone" version = "1.0.20" @@ -1430,6 +1468,12 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "funty" version = "2.0.0" @@ -3958,6 +4002,7 @@ version = "0.23.41" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" dependencies = [ + "aws-lc-rs", "log", "once_cell", "ring", @@ -4022,6 +4067,7 @@ version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", diff --git a/packages/zaino-common/CHANGELOG.md b/packages/zaino-common/CHANGELOG.md index e2356e551..42890004b 100644 --- a/packages/zaino-common/CHANGELOG.md +++ b/packages/zaino-common/CHANGELOG.md @@ -21,6 +21,10 @@ and this library adheres to Rust's notion of `sync_write_batch_size` so the two operations cannot inflate each other's peak memory. ### Changed +- `crypto::ensure_default_crypto_provider` now installs rustls's + **aws-lc-rs** provider (was ring) as the process-level default, and the + crate's rustls features become `aws_lc_rs` + `prefer-post-quantum` + (ADR-0006). First-install-wins semantics are unchanged. - **Breaking** — `StorageConfig::database.sync_write_batch_bytes` (raw bytes) is renamed to `sync_write_batch_size` and now expressed in **GiB** (new `SyncWriteBatchSize` newtype, mirroring `DatabaseSize`); the default is 8 GiB. diff --git a/packages/zaino-common/Cargo.toml b/packages/zaino-common/Cargo.toml index 437927085..6c368e5ff 100644 --- a/packages/zaino-common/Cargo.toml +++ b/packages/zaino-common/Cargo.toml @@ -25,4 +25,7 @@ tracing-tree = { workspace = true } nu-ansi-term = { workspace = true } hex = { workspace = true } time = { workspace = true } -rustls = { version = "0.23", default-features = false, features = ["ring"] } +rustls = { version = "0.23", default-features = false, features = [ + "aws_lc_rs", + "prefer-post-quantum", +] } diff --git a/packages/zaino-common/src/crypto.rs b/packages/zaino-common/src/crypto.rs index 8791bbae2..c24dc59d9 100644 --- a/packages/zaino-common/src/crypto.rs +++ b/packages/zaino-common/src/crypto.rs @@ -1,7 +1,7 @@ //! Process-level rustls `CryptoProvider` management. -/// Installs rustls's `ring` provider as the process-level default if no -/// provider is installed yet. +/// Installs rustls's `aws-lc-rs` provider as the process-level default +/// if no provider is installed yet. /// /// Call this before constructing anything that builds a rustls config — /// a `reqwest` client (the workspace enables reqwest's @@ -13,15 +13,24 @@ /// deployments (zingolabs/zaino#1360). Installing explicitly removes /// that dependence on the feature graph. /// +/// aws-lc-rs is the preferred provider (ADR-0006): it is the only rustls +/// provider with post-quantum key exchange, and the workspace enables +/// rustls's `prefer-post-quantum` feature so the X25519MLKEM768 hybrid +/// group leads this provider's defaults. That ordering governs zaino's +/// outbound (client-role) handshakes; rustls servers follow the client's +/// group preference, so inbound hybrid uptake is decided by the +/// connecting wallet's TLS stack. Classical key-exchange groups remain +/// offered and accepted, but are deprecated (see the ADR). +/// /// The process default is first-install-wins and is not constrained by /// our crates' rustls features, so an embedder (e.g. zallet) that has /// already installed a provider keeps it: zaino then handshakes through -/// that provider instead of ring, which is fine for any provider +/// that provider instead of aws-lc-rs, which is fine for any provider /// implementing the standard TLS suites (ring and aws-lc-rs both do). pub fn ensure_default_crypto_provider() { if rustls::crypto::CryptoProvider::get_default().is_none() { // A racing concurrent install is the only error case; either // provider serves. - let _ = rustls::crypto::ring::default_provider().install_default(); + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); } } diff --git a/packages/zaino-serve/CHANGELOG.md b/packages/zaino-serve/CHANGELOG.md index 0df7dc8c7..8a6296e20 100644 --- a/packages/zaino-serve/CHANGELOG.md +++ b/packages/zaino-serve/CHANGELOG.md @@ -9,6 +9,8 @@ and this library adheres to Rust's notion of ### Added ### Changed +- tonic's TLS provider feature switches from `tls-ring` to `tls-aws-lc`, + following the workspace's aws-lc-rs preferred CryptoProvider (ADR-0006). ### Deprecated ### Removed ### Fixed diff --git a/packages/zaino-serve/Cargo.toml b/packages/zaino-serve/Cargo.toml index cef85e961..c421c8a4e 100644 --- a/packages/zaino-serve/Cargo.toml +++ b/packages/zaino-serve/Cargo.toml @@ -53,7 +53,7 @@ metrics = { workspace = true, optional = true } # Miscellaneous Workspace tokio = { workspace = true, features = ["full"] } -tonic = { workspace = true, features = ["tls-native-roots", "tls-ring"] } +tonic = { workspace = true, features = ["tls-native-roots", "tls-aws-lc"] } thiserror = { workspace = true } futures = { workspace = true } jsonrpsee = { workspace = true, features = ["server", "macros"] } diff --git a/packages/zaino-serve/src/server/grpc/tests/tls.rs b/packages/zaino-serve/src/server/grpc/tests/tls.rs index 02b8287ad..9b60a7567 100644 --- a/packages/zaino-serve/src/server/grpc/tests/tls.rs +++ b/packages/zaino-serve/src/server/grpc/tests/tls.rs @@ -7,8 +7,9 @@ //! TLS acceptor panicked at runtime ("Could not automatically determine //! the process-level CryptoProvider") — but only on TLS-enabled //! deployments, a path no test exercised because every existing test -//! ran plaintext. The fix pins the feature graph to `ring` and installs -//! the provider explicitly (`ensure_crypto_provider`); this test covers +//! ran plaintext. The fix pins the feature graph to a single provider — +//! `aws-lc-rs`, the preferred provider per ADR-0006 — and installs it +//! explicitly (`ensure_default_crypto_provider`); this test covers //! the full TLS startup + handshake path so a future provider-selection //! or certificate-wiring regression fails in CI instead of production. //! diff --git a/tools/makefiles/lints.toml b/tools/makefiles/lints.toml index cac50b8d3..e6824f01f 100644 --- a/tools/makefiles/lints.toml +++ b/tools/makefiles/lints.toml @@ -82,6 +82,11 @@ description = "Assert Dockerfile.deterministic's stagex/pallet-rust tag matches command = "cargo" args = ["run", "-q", "--manifest-path", "tools/workbench/Cargo.toml", "--bin", "check-toolchain-pin"] +[tasks.lint-crypto-provider] +description = "Assert rustls resolves with aws-lc-rs + prefer-post-quantum and that the only ring-selecting feature edges are the tolerated zebra path (ADR-0006). Fails loud in both directions: new edges and vanished edges." +command = "cargo" +args = ["run", "-q", "--manifest-path", "tools/workbench/Cargo.toml", "--bin", "check-crypto-provider"] + [tasks.lint-no-openssl-apt] description = "Fail if any Dockerfile installs a libssl*/openssl* apt package. TLS is rustls; the system-package companion to deny.toml's openssl crate ban." command = "cargo" @@ -106,6 +111,7 @@ dependencies = [ "lint-boundary-conversions", "lint-workbench", "lint-toolchain-pin", + "lint-crypto-provider", "lint-no-openssl-apt", "deny", "lint-shell", diff --git a/tools/workbench/src/bin/check-crypto-provider.rs b/tools/workbench/src/bin/check-crypto-provider.rs new file mode 100644 index 000000000..13c62a881 --- /dev/null +++ b/tools/workbench/src/bin/check-crypto-provider.rs @@ -0,0 +1,206 @@ +//! Guard: one rustls CryptoProvider — aws-lc-rs — plus the single tolerated +//! ring path through zebra (ADR-0006). +//! +//! Two checks against the production workspace's resolved dependency graph: +//! +//! 1. rustls must resolve with `aws_lc_rs` and `prefer-post-quantum`. +//! 2. The set of feature edges that select ring anywhere in the graph must +//! equal [`RING_EDGE_ALLOWLIST`] — today, the single chain rooted in +//! zebra-node-services' `rpc-client` feature (its reqwest `rustls-tls`; +//! reqwest 0.12 offers no aws-lc alternative). That ring is compiled but +//! dormant: reqwest consults the process-default provider first and zaino +//! installs aws-lc-rs at both TLS boundaries. +//! +//! Loud in both directions. A NEW edge means a second provider path is +//! creeping back in via feature unification — the regression this guard +//! exists to catch (zingolabs/zaino#1360). A MISSING edge — in particular +//! ring vanishing from the graph entirely — means zebra dropped its ring +//! dependence: delete [`RING_EDGE_ALLOWLIST`], make this check assert ring's +//! absence, and close the classical-deprecation tracking issue's zebra item. + +use std::collections::BTreeSet; +use std::path::Path; +use std::process::Command; + +use workbench::{repo_root, run}; + +/// Feature names that select the ring provider when enabled on any crate. +const RING_SELECTING_FEATURES: [&str; 3] = ["ring", "__rustls-ring", "tls-ring"]; + +/// Every feature edge expected to select ring, all downstream of +/// zebra-node-services' reqwest `rustls-tls` (see module docs). +const RING_EDGE_ALLOWLIST: [&str; 5] = [ + "hyper-rustls feature \"ring\"", + "reqwest feature \"__rustls-ring\"", + "rustls feature \"ring\"", + "rustls-webpki feature \"ring\"", + "tokio-rustls feature \"ring\"", +]; + +fn main() { + run("check-crypto-provider", check, |()| { + println!( + "check-crypto-provider: ok — rustls resolves aws-lc-rs + prefer-post-quantum; \ + ring edges match the tolerated zebra path" + ); + }) +} + +fn check() -> Result<(), Vec> { + let root = repo_root()?; + + let rustls_features = cargo_tree(&root, &["--package", "rustls", "--depth", "0"], "{f}")?; + check_preferred_provider(&rustls_features)?; + + let ring_tree = match cargo_tree(&root, &["--invert", "ring", "--edges", "features"], "{p}") { + Ok(out) => out, + Err(diag) + if diag + .iter() + .any(|l| l.contains("did not match any packages")) => + { + return Err(vec![ + "ring is GONE from the dependency graph — zebra dropped it!".to_string(), + "Tighten this guard: delete RING_EDGE_ALLOWLIST and assert ring's absence," + .to_string(), + "and close the zebra item on the classical-deprecation tracking issue.".to_string(), + ]); + } + Err(diag) => return Err(diag), + }; + check_ring_edges(&ring_tree) +} + +/// Assert the resolved rustls feature set contains the preferred-provider pair. +fn check_preferred_provider(features_line: &str) -> Result<(), Vec> { + let features: BTreeSet<&str> = features_line.trim().split(',').collect(); + let missing: Vec<&str> = ["aws_lc_rs", "prefer-post-quantum"] + .into_iter() + .filter(|f| !features.contains(f)) + .collect(); + if missing.is_empty() { + Ok(()) + } else { + Err(vec![format!( + "rustls resolved without {} (got: {}) — the preferred provider per \ + ADR-0006 is aws-lc-rs with prefer-post-quantum; check \ + packages/zaino-common/Cargo.toml's rustls features", + missing.join(" + "), + features_line.trim(), + )]) + } +} + +/// Assert the ring-selecting feature edges equal the tolerated allowlist. +fn check_ring_edges(ring_tree: &str) -> Result<(), Vec> { + let found = ring_selecting_edges(ring_tree); + let expected: BTreeSet = RING_EDGE_ALLOWLIST + .iter() + .map(ToString::to_string) + .collect(); + + if found == expected { + return Ok(()); + } + + let mut msg = Vec::new(); + for new in found.difference(&expected) { + msg.push(format!( + "NEW ring-selecting feature edge: {new} — a second CryptoProvider path is \ + creeping back in (zingolabs/zaino#1360); remove the enabling feature", + )); + } + for gone in expected.difference(&found) { + msg.push(format!( + "tolerated ring edge disappeared: {gone} — if zebra dropped ring, tighten \ + this guard (see the module docs in check-crypto-provider.rs)", + )); + } + msg.push("see docs/adr/0006-aws-lc-rs-preferred-crypto-provider.md".to_string()); + Err(msg) +} + +/// The deduplicated ` feature ""` edges in a +/// `cargo tree --edges features` output whose feature name selects ring. +fn ring_selecting_edges(tree: &str) -> BTreeSet { + tree.lines() + .map(|line| { + line.trim_start_matches(|c| matches!(c, '│' | '├' | '└' | '─' | ' ')) + .trim_end_matches(" (*)") + }) + .filter(|node| { + node.split_once(" feature ").is_some_and(|(_, feature)| { + RING_SELECTING_FEATURES + .iter() + .any(|f| feature == format!("\"{f}\"")) + }) + }) + .map(ToString::to_string) + .collect() +} + +/// Run `cargo tree --format ` against the production workspace +/// at `root`, returning stdout; diagnostics carry stderr so callers can +/// distinguish "package not in graph" from other failures. +fn cargo_tree(root: &Path, args: &[&str], format: &str) -> Result> { + let output = Command::new("cargo") + .arg("tree") + .args(args) + .args(["--format", format]) + .current_dir(root) + .output() + .map_err(|e| vec![format!("failed to run cargo tree: {e}")])?; + if !output.status.success() { + return Err(vec![format!( + "`cargo tree {}` failed: {}", + args.join(" "), + String::from_utf8_lossy(&output.stderr).trim(), + )]); + } + String::from_utf8(output.stdout).map_err(|e| vec![format!("cargo output not utf-8: {e}")]) +} + +#[cfg(test)] +mod ring_selecting_edges { + use super::*; + + #[test] + fn extracts_and_dedups_ring_edges_from_a_feature_tree() { + let tree = "\ +ring v0.17.14 +├── rustls v0.23.41 +│ ├── rustls feature \"ring\" +│ │ └── reqwest feature \"__rustls-ring\" +│ └── rustls feature \"ring\" (*) +└── rustls-webpki v0.103.13 + └── rustls-webpki feature \"ring\" +"; + let edges = ring_selecting_edges(tree); + assert_eq!( + edges.into_iter().collect::>(), + [ + "reqwest feature \"__rustls-ring\"", + "rustls feature \"ring\"", + "rustls-webpki feature \"ring\"", + ] + ); + } + + #[test] + fn ignores_package_nodes_and_non_ring_features() { + let tree = "\ +rustls v0.23.41 +├── rustls feature \"aws_lc_rs\" +├── reqwest feature \"rustls-tls\" +└── tokio-rustls feature \"logging\" +"; + assert!(ring_selecting_edges(tree).is_empty()); + } + + #[test] + fn matches_feature_names_exactly_not_by_substring() { + // "tls-ring-extra" must not match the "tls-ring" selector. + let tree = "└── tonic feature \"tls-ring-extra\"\n"; + assert!(ring_selecting_edges(tree).is_empty()); + } +} From 94cccea9714fbc0d5e6c08b87b29b9a316b498fa Mon Sep 17 00:00:00 2001 From: zancas Date: Sun, 5 Jul 2026 16:58:44 -0700 Subject: [PATCH 21/58] fix(workbench): satisfy clippy manual_pattern_char_comparison Replace the matches! closure in check-crypto-provider's tree-prefix trimming with the char-array pattern clippy suggests. The lint escaped local verification because a chained `fmt --check && clippy` short- circuited on the fmt diff, so clippy never ran. Co-Authored-By: Claude Fable 5 --- tools/workbench/src/bin/check-crypto-provider.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/workbench/src/bin/check-crypto-provider.rs b/tools/workbench/src/bin/check-crypto-provider.rs index 13c62a881..b3926a1cf 100644 --- a/tools/workbench/src/bin/check-crypto-provider.rs +++ b/tools/workbench/src/bin/check-crypto-provider.rs @@ -125,7 +125,7 @@ fn check_ring_edges(ring_tree: &str) -> Result<(), Vec> { fn ring_selecting_edges(tree: &str) -> BTreeSet { tree.lines() .map(|line| { - line.trim_start_matches(|c| matches!(c, '│' | '├' | '└' | '─' | ' ')) + line.trim_start_matches(['│', '├', '└', '─', ' ']) .trim_end_matches(" (*)") }) .filter(|node| { From 5e4d45e7ddb87e913889cf2b70fdfb27e55da5b4 Mon Sep 17 00:00:00 2001 From: idky137 Date: Mon, 6 Jul 2026 15:58:16 +0100 Subject: [PATCH 22/58] make persistent db validation wait on migration --- .../src/chain_index/finalised_state.rs | 28 +++++++++++++--- .../finalised_state/finalised_source.rs | 12 +++++++ .../finalised_state/finalised_source/v1.rs | 33 ++++++++++++++----- 3 files changed, 60 insertions(+), 13 deletions(-) diff --git a/packages/zaino-state/src/chain_index/finalised_state.rs b/packages/zaino-state/src/chain_index/finalised_state.rs index 11c824ec1..2873e6e49 100644 --- a/packages/zaino-state/src/chain_index/finalised_state.rs +++ b/packages/zaino-state/src/chain_index/finalised_state.rs @@ -543,12 +543,25 @@ impl FinalisedState { source: migration_source, }; - if let Err(error) = migration_manager.migrate().await { - tracing::error!("FinalisedState migration failed: {error}"); + match migration_manager.migrate().await { + Ok(()) => { + // Start the background validator only now that every migration has + // finished: its initial scan reads tables a migration populates (e.g. + // `commitment_tree_data_1_3_0`), so starting it earlier would race the + // migration and fail on a not-yet-written row. + migration_router.primary_backend().start_validator(); + } + Err(error) => { + tracing::error!("FinalisedState migration failed: {error}"); - migration_router.store_primary_status(StatusType::CriticalError); + migration_router.store_primary_status(StatusType::CriticalError); + } } }); + } else { + // No migration to run, so the on-disk tables the validator scans are already at the + // current schema: start it immediately. + router.primary_backend().start_validator(); } Ok(Self { db: router, cfg }) @@ -1053,6 +1066,14 @@ impl FinalisedState { migration_manager.migrate().await?; } + // This test helper builds a fixture at an arbitrary (often intermediate) version for + // inspection, so it deliberately does NOT start the validator: the validator only validates + // against the current schema and would fail on an intermediate-version database. The + // foreground migration is already complete, so mark the primary `Ready` directly (as + // `spawn_v1_0_0` does) to give callers a settled backend. Validation is exercised through + // the production `FinalisedState::spawn` path, which always targets the current schema. + router.store_primary_status(StatusType::Ready); + let metadata = router.get_metadata().await?; if metadata.version() != target_version { return Err(FinalisedStateError::Custom(format!( @@ -1084,7 +1105,6 @@ impl FinalisedState { source: T, ) -> Result, FinalisedStateError> { let db = FinalisedSource::spawn_v1_0_0(cfg).await?; - db.wait_until_ready().await; let tip = source.get_best_block_height().await?.ok_or_else(|| { FinalisedStateError::BlockchainSourceError(BlockchainSourceError::Unrecoverable( diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source.rs index cae4c96e3..e2ce0793e 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source.rs @@ -283,6 +283,18 @@ impl FinalisedSource { } } + /// Start the background validator on the primary v1 backend (no-op for the ephemeral + /// passthrough). + /// + /// Called by the orchestrator only once all pending migrations have completed, so the + /// validator never races a migration that populates the tables its initial scan reads. + pub(crate) fn start_validator(&self) { + match self { + Self::V1(db) => db.start_validator(), + Self::Ephemeral(_) => {} + } + } + /// Stores a new runtime status in the concrete backend. /// /// This is used by router-level background orchestration, for example to report an asynchronous diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1.rs index 73ed8b717..f75eacf21 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1.rs @@ -413,8 +413,15 @@ impl DbV1 { /// - opens or creates all V1 named databases, /// - validates or initializes the `"metadata"` record (schema hash + version), and /// - spawns the background validator / maintenance task. + /// Opens (and heals) the v1 database **without** starting the background validator. + /// + /// The validator is started separately via [`DbV1::start_validator`]. This split exists so the + /// orchestrator can guarantee that any pending schema migration finishes *before* validation + /// runs: the validator's `initial_block_scan` reads tables (e.g. `commitment_tree_data_1_3_0`) + /// that a migration populates, so starting it concurrently with a migration races the migration + /// and can fail on a not-yet-written row. pub(crate) async fn spawn(config: &ChainIndexConfig) -> Result { - let mut zaino_db = Self::open_env_and_dbs(config).await?; + let zaino_db = Self::open_env_and_dbs(config).await?; // Validate (or initialise) the metadata entry before we touch any tables. zaino_db.check_schema_version().await?; @@ -424,9 +431,6 @@ impl DbV1 { // on a quiescent database. zaino_db.reconcile_alpha_txid_location_index().await?; - // Spawn handler task to perform background validation and trailing tx cleanup. - zaino_db.spawn_handler().await?; - Ok(zaino_db) } @@ -598,7 +602,12 @@ impl DbV1 { /// `initial_block_scan`). /// - **Steady state:** periodically attempts to validate the next height after `validated_tip`. /// Separately, it performs periodic trailing-reader cleanup via `clean_trailing()`. - async fn spawn_handler(&mut self) -> Result<(), FinalisedStateError> { + /// + /// Kept separate from [`DbV1::spawn`] so the orchestrator starts it only once all pending + /// migrations have finished (the validator reads tables a migration populates). Takes `&self` + /// (the join handle lives behind a `Mutex`) so it can be driven through the shared + /// `Arc` the router holds after spawn. + pub(crate) fn start_validator(&self) { // Clone everything the task needs so we can move it into the async block. let zaino_db = self.detached_handle(); @@ -700,7 +709,6 @@ impl DbV1 { }); *self.db_handler.lock().expect("db_handler mutex poisoned") = Some(handle); - Ok(()) } /// Validates every stored spent-outpoint entry (`Outpoint` -> `TxLocation`) by checksum. @@ -1016,9 +1024,16 @@ impl DbV1 { // (see the method doc) — that is the behavioural difference from `spawn`. zaino_db.write_v1_0_0_metadata()?; - // Spawn handler task to perform background validation and trailing tx cleanup. - let mut zaino_db = zaino_db; - zaino_db.spawn_handler().await?; + // Deliberately does NOT start the background validator. This builds a *pre-migration* + // v1.0.0 fixture; the validator validates against the current (v1.3.0) schema — it reads + // `commitment_tree_data_1_3_0` and the `ironwood` table — so it must run only after the + // database has been migrated to the newest schema. Callers build the fixture with direct + // v1.0.0 writes, shut it down, then reopen through `FinalisedState::spawn`, which migrates + // first and starts the validator afterwards. + // + // With no validator to advance it, mark the empty fixture `Ready` directly so callers see a + // settled backend. + zaino_db.status.store(StatusType::Ready); Ok(zaino_db) } From b15510121c248e5e4a9383cf229fcf56851c5eef Mon Sep 17 00:00:00 2001 From: idky137 Date: Mon, 6 Jul 2026 21:33:58 +0100 Subject: [PATCH 23/58] fixed ironwood migration --- .../finalised_state/finalised_source.rs | 6 + .../finalised_state/finalised_source/v1.rs | 15 +- .../finalised_source/v1/write_core.rs | 30 ++- .../chain_index/finalised_state/migrations.rs | 211 ++++++++++++------ .../tests/finalised_state/migrations.rs | 2 + .../migrations/v1_2_to_v1_3.rs | 99 ++++++++ 6 files changed, 281 insertions(+), 82 deletions(-) create mode 100644 packages/zaino-state/src/chain_index/tests/finalised_state/migrations/v1_2_to_v1_3.rs diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source.rs index e2ce0793e..70a9c7137 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source.rs @@ -380,6 +380,12 @@ impl FinalisedSource { .require_v1("v1 commitment_tree_data db")? .commitment_tree_data_db()) } + + /// Provides access to the (v1.3.0) `ironwood` table, required for Migration1_2_1To1_3_0 to + /// backfill ironwood rows from validator-fetched block data. + pub(crate) fn ironwood_db(&self) -> Result { + Ok(self.require_v1("v1 ironwood db")?.ironwood_db()) + } } impl From for FinalisedSource { diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1.rs index f75eacf21..835579784 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1.rs @@ -405,15 +405,14 @@ pub(crate) struct DbV1 { /// - validated read fetchers used by the capability trait implementations, and /// - internal validation / indexing helpers. impl DbV1 { - /// Spawns a new [`DbV1`] and opens (or creates) the LMDB environment for the configured network. + /// Opens (and heals) the v1 database for the configured network **without** starting the + /// background validator. /// /// This method: /// - chooses a versioned path suffix (`...//v1`), /// - configures LMDB map size and reader slots, - /// - opens or creates all V1 named databases, - /// - validates or initializes the `"metadata"` record (schema hash + version), and - /// - spawns the background validator / maintenance task. - /// Opens (and heals) the v1 database **without** starting the background validator. + /// - opens or creates all V1 named databases, and + /// - validates or initializes the `"metadata"` record (schema hash + version). /// /// The validator is started separately via [`DbV1::start_validator`]. This split exists so the /// orchestrator can guarantee that any pending schema migration finishes *before* validation @@ -851,6 +850,12 @@ impl DbV1 { self.commitment_tree_data } + /// Provides access to the (v1.3.0) `ironwood` table, required for Migration1_2_1To1_3_0 to + /// backfill ironwood rows for post-NU6.3 blocks from validator-fetched block data. + pub(crate) fn ironwood_db(&self) -> Database { + self.ironwood + } + /// Provudes access to the spent DB table, required for Migration1_1_0To1_2_0. pub(crate) fn spent_db(&self) -> Database { self.spent diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/write_core.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/write_core.rs index 212ac75d3..5c3cfc07e 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/write_core.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/write_core.rs @@ -188,13 +188,35 @@ fn build_block_row_entries( ), sapling_entry: StoredEntryVar::new(block_height_bytes, SaplingTxList::new(sapling)), orchard_entry: StoredEntryVar::new(block_height_bytes, OrchardTxList::new(orchard)), - ironwood_entry: ironwood - .iter() - .any(Option::is_some) - .then(|| StoredEntryVar::new(block_height_bytes, OrchardTxList::new(ironwood))), + ironwood_entry: ironwood_entry(ironwood, block_height_bytes), } } +/// Builds the sparse ironwood row entry for a block's per-transaction ironwood list: `Some` only +/// when at least one transaction carries ironwood data, so a block with none stores no ironwood row +/// (readers treat an absent row as "no ironwood data"). +fn ironwood_entry( + ironwood: Vec>, + block_height_bytes: &[u8], +) -> Option> { + ironwood + .iter() + .any(Option::is_some) + .then(|| StoredEntryVar::new(block_height_bytes, OrchardTxList::new(ironwood))) +} + +/// Builds the sparse ironwood row entry for `block` (the same value the write path stores). Used by +/// the v1.2.1 → v1.3.0 migration to backfill the ironwood table from validator-fetched blocks. +pub(crate) fn build_block_ironwood_entry( + block: &IndexedBlock, + block_height_bytes: &[u8], +) -> Result>, FinalisedStateError> { + Ok(ironwood_entry( + extract_block_pool_lists(block)?.ironwood, + block_height_bytes, + )) +} + impl DbWrite for DbV1 { async fn write_block(&self, block: IndexedBlock) -> Result<(), FinalisedStateError> { self.write_block(block).await diff --git a/packages/zaino-state/src/chain_index/finalised_state/migrations.rs b/packages/zaino-state/src/chain_index/finalised_state/migrations.rs index b88009194..ef414f9d1 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/migrations.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/migrations.rs @@ -1017,27 +1017,28 @@ impl Migration for Migration1_2_0To1_2_1 { /// Minor migration: v1.2.1 → v1.3.0. /// /// Introduces the Ironwood (NU6.3) shielded pool to the finalised state: -/// - a new per-height `ironwood` table (`ironwood_1_3_0`), created empty by `DbV1::spawn`; new -/// blocks populate it via the write path. No backfill is needed because every block already on -/// disk predates NU6.3 (see the guard below). +/// - a new per-height `ironwood` table (`ironwood_1_3_0`), created empty by `DbV1::spawn`. New +/// blocks populate it via the write path; here it is backfilled for any stored block at or above +/// NU6.3 activation (see below). /// - the `commitment_tree_data` table is rebuilt from the legacy fixed-length /// `StoredEntryFixed` (V1) rows into the variable-length /// `StoredEntryVar` (V2) layout, which carries the optional Ironwood root and -/// size. The rebuild reads the legacy `commitment_tree_data_1_0_0` table and writes the new -/// `commitment_tree_data_1_3_0` table (opened as the primary's `commitment_tree_data` handle), -/// then clears the legacy table. +/// size. The new rows are written to `commitment_tree_data_1_3_0` (the primary's +/// `commitment_tree_data` handle), then the legacy table is cleared. /// -/// This is a **rebuild from existing on-disk data** (no validator refetch), correct only while every -/// stored block predates Ironwood. The migration therefore asserts the database tip is below NU6.3 -/// activation before running; a database already synced past NU6.3 under the old schema would be -/// missing ironwood data that cannot be reconstructed from existing rows and must be re-indexed from -/// the validator instead. +/// Per-height rebuild strategy, branching on NU6.3 activation: +/// - **Below activation:** rebuilt in place from the legacy on-disk commitment row (Ironwood +/// root/size default to `None`/`0` via `CommitmentTreeData` V1 decode). No validator access. +/// - **At or above activation:** the legacy data predates Ironwood, so both the commitment row +/// (now carrying the Ironwood root/size) and the sparse ironwood row are rebuilt from +/// validator-fetched block data via [`build_indexed_block_from_source`]. This lets the migration +/// run on a database already synced past NU6.3, rather than forcing a full re-index. /// /// Safety and resumability: -/// - Deterministic: each rebuilt row is derived only from the matching legacy row (Ironwood -/// root/size default to `None`/`0` via `CommitmentTreeData` V1 decode). +/// - Deterministic: a below-activation row is derived only from its legacy row; an at/above-activation +/// row is refetched from immutable finalised history, so a resumed rebuild reproduces the same bytes. /// - Resumable: the next height to rebuild is stored in the metadata DB under a temporary key. -/// - Crash-safe: each height's rebuilt row and the progress update commit in the same transaction; +/// - Crash-safe: each height's rebuilt rows and the progress update commit in the same transaction; /// idempotent on resume (`NO_OVERWRITE` + verify-match). struct Migration1_2_1To1_3_0; @@ -1062,10 +1063,16 @@ impl Migration for Migration1_2_1To1_3_0 { &self, router: Arc>, cfg: ChainIndexConfig, - _source: T, + source: T, ) -> Result<(), FinalisedStateError> { use lmdb::DatabaseFlags; + use crate::chain_index::finalised_state::{ + build_indexed_block_from_source, + finalised_source::v1::write_core::build_block_ironwood_entry, + }; + use crate::chain_index::ShieldedPool; + // Temporary metadata entry recording the next height to rebuild, removed on completion. const MIGRATION_CTD_PROGRESS_KEY: &[u8] = b"_migration_commitment_tree_data_progress_1_3_0_next_height"; @@ -1077,8 +1084,10 @@ impl Migration for Migration1_2_1To1_3_0 { let backend = router.primary_backend(); let env = backend.env()?; let metadata_db = backend.metadata_db()?; - // The primary's `commitment_tree_data` handle is the new `commitment_tree_data_1_3_0` table. + // The primary's `commitment_tree_data` handle is the new `commitment_tree_data_1_3_0` table; + // `ironwood_db` is the new (v1.3.0) sparse ironwood table backfilled below. let new_ctd_db = backend.commitment_tree_data_db()?; + let ironwood_db = backend.ironwood_db()?; // Open the legacy fixed-length commitment table by name. On a pre-v1.3.0 database it already // exists; `open_or_create_db` creating it empty on an unexpected fresh DB is harmless (the @@ -1091,10 +1100,19 @@ impl Migration for Migration1_2_1To1_3_0 { ) .await?; - // Guard: Ironwood data is only expected from NU6.3 activation. A rebuild-from-existing-data - // migration cannot reconstruct ironwood roots/sizes for post-NU6.3 blocks. - let zebra_network = cfg.network.to_zebra_network(); - let nu6_3_activation_height = crate::chain_index::ShieldedPool::Ironwood + // Network-upgrade activation heights (mirrors `write_blocks_to_height`). Blocks at or above + // NU6.3 have their ironwood root/size and ironwood tx list rebuilt from the validator (the + // legacy on-disk data predates ironwood); blocks below it are rebuilt in place from the + // legacy commitment row, with no ironwood. + let network = cfg.network; + let zebra_network = network.to_zebra_network(); + let sapling_activation_height = ShieldedPool::Sapling + .activation_upgrade() + .activation_height(&zebra_network); + let nu5_activation_height = ShieldedPool::Orchard + .activation_upgrade() + .activation_height(&zebra_network); + let nu6_3_activation_height = ShieldedPool::Ironwood .activation_upgrade() .activation_height(&zebra_network); @@ -1133,17 +1151,6 @@ impl Migration for Migration1_2_1To1_3_0 { if let Some(db_tip) = backend.db_height().await? { let db_tip = db_tip.0; - if let Some(activation) = nu6_3_activation_height { - if db_tip >= activation.0 { - return Err(FinalisedStateError::Custom(format!( - "cannot migrate finalised state to v1.3.0: tip {db_tip} is at or above NU6.3 \ - activation {} but was built without Ironwood data; wipe and re-index the \ - finalised state from the validator", - activation.0 - ))); - } - } - let mut next_height = read_progress(MIGRATION_CTD_PROGRESS_KEY)?.unwrap_or(GENESIS_HEIGHT.0); @@ -1154,58 +1161,116 @@ impl Migration for Migration1_2_1To1_3_0 { ); let started = std::time::Instant::now(); + // Idempotent write: on resume an already-written row must match byte-for-byte (the + // legacy rebuild is deterministic and the validator refetch is over immutable history). + fn put_idempotent( + txn: &mut lmdb::RwTransaction<'_>, + db: lmdb::Database, + key: &[u8], + value: &[u8], + what: &str, + ) -> Result<(), FinalisedStateError> { + match txn.put(db, &key, &value, WriteFlags::NO_OVERWRITE) { + Ok(()) => Ok(()), + Err(lmdb::Error::KeyExist) => { + let existing = txn.get(db, &key).map_err(FinalisedStateError::LmdbError)?; + if existing != value { + return Err(FinalisedStateError::Custom(format!( + "conflicting rebuilt {what}" + ))); + } + Ok(()) + } + Err(error) => Err(FinalisedStateError::LmdbError(error)), + } + } + while next_height <= db_tip { let height = Height::try_from(next_height) .map_err(|error| FinalisedStateError::Custom(error.to_string()))?; let height_bytes = height.to_bytes()?; - // Read + verify the legacy fixed-length row. - let commitment_tree_data: CommitmentTreeData = { - let txn = env.begin_ro_txn()?; - let raw = txn - .get(legacy_ctd_db, &height_bytes) - .map_err(FinalisedStateError::LmdbError)?; - let entry = StoredEntryFixed::::from_bytes(raw).map_err( - |error| { - FinalisedStateError::Custom(format!( - "legacy commitment_tree_data corrupt data: {error}" - )) - }, - )?; - if !entry.verify(&height_bytes) { - return Err(FinalisedStateError::Custom( - "legacy commitment_tree_data checksum mismatch".to_string(), - )); - } - *entry.inner() - }; - - // Re-wrap in the V2 `StoredEntryVar` layout and advance progress atomically. - let new_entry_bytes = - StoredEntryVar::new(&height_bytes, commitment_tree_data).to_bytes()?; + let ironwood_active = + nu6_3_activation_height.is_some_and(|activation| next_height >= activation.0); + + // Prepare the rows to write. Reads / validator fetches happen before the write txn + // so the commit stays short. + let commitment_bytes: Vec; + let ironwood_bytes: Option>; + + if ironwood_active { + // Post-NU6.3: the legacy row carries no ironwood root/size and the ironwood + // table has no row, so rebuild both from validator-fetched block data. + let sapling_activation_height = sapling_activation_height.ok_or_else(|| { + FinalisedStateError::Custom( + "Sapling activation height must be set to backfill ironwood" + .to_string(), + ) + })?; + let block = build_indexed_block_from_source( + &source, + network, + sapling_activation_height, + nu5_activation_height, + nu6_3_activation_height, + next_height, + // Chainwork is irrelevant here: only the commitment-tree and ironwood rows + // are extracted, and neither depends on it. + None, + ) + .await?; + + commitment_bytes = + StoredEntryVar::new(&height_bytes, *block.commitment_tree_data()) + .to_bytes()?; + ironwood_bytes = match build_block_ironwood_entry(&block, &height_bytes)? { + Some(entry) => Some(entry.to_bytes()?), + None => None, + }; + } else { + // Pre-NU6.3: rebuild the commitment row in place from the legacy fixed-length + // row (ironwood defaults to none). + let commitment_tree_data: CommitmentTreeData = { + let txn = env.begin_ro_txn()?; + let raw = txn + .get(legacy_ctd_db, &height_bytes) + .map_err(FinalisedStateError::LmdbError)?; + let entry = StoredEntryFixed::::from_bytes(raw) + .map_err(|error| { + FinalisedStateError::Custom(format!( + "legacy commitment_tree_data corrupt data: {error}" + )) + })?; + if !entry.verify(&height_bytes) { + return Err(FinalisedStateError::Custom( + "legacy commitment_tree_data checksum mismatch".to_string(), + )); + } + *entry.inner() + }; + commitment_bytes = + StoredEntryVar::new(&height_bytes, commitment_tree_data).to_bytes()?; + ironwood_bytes = None; + } + // Write commitment (+ ironwood) and advance progress atomically. { let mut txn = env.begin_rw_txn()?; - match txn.put( + put_idempotent( + &mut txn, new_ctd_db, &height_bytes, - &new_entry_bytes, - WriteFlags::NO_OVERWRITE, - ) { - Ok(()) => {} - // Idempotent on resume: an existing rebuilt row must match exactly. - Err(lmdb::Error::KeyExist) => { - let existing = txn - .get(new_ctd_db, &height_bytes) - .map_err(FinalisedStateError::LmdbError)?; - if existing != new_entry_bytes.as_slice() { - return Err(FinalisedStateError::Custom(format!( - "conflicting rebuilt commitment_tree_data at height {}", - height.0 - ))); - } - } - Err(error) => return Err(FinalisedStateError::LmdbError(error)), + &commitment_bytes, + &format!("commitment_tree_data at height {}", height.0), + )?; + if let Some(bytes) = &ironwood_bytes { + put_idempotent( + &mut txn, + ironwood_db, + &height_bytes, + bytes, + &format!("ironwood at height {}", height.0), + )?; } let progress = StoredEntryFixed::new(MIGRATION_CTD_PROGRESS_KEY, height + 1); diff --git a/packages/zaino-state/src/chain_index/tests/finalised_state/migrations.rs b/packages/zaino-state/src/chain_index/tests/finalised_state/migrations.rs index e0058faf6..11b399a6e 100644 --- a/packages/zaino-state/src/chain_index/tests/finalised_state/migrations.rs +++ b/packages/zaino-state/src/chain_index/tests/finalised_state/migrations.rs @@ -10,3 +10,5 @@ mod v1_0_to_v1_1; #[cfg(not(feature = "transparent_address_history_experimental"))] mod v1_1_to_v1_2; +#[cfg(not(feature = "transparent_address_history_experimental"))] +mod v1_2_to_v1_3; diff --git a/packages/zaino-state/src/chain_index/tests/finalised_state/migrations/v1_2_to_v1_3.rs b/packages/zaino-state/src/chain_index/tests/finalised_state/migrations/v1_2_to_v1_3.rs new file mode 100644 index 000000000..5d215be9c --- /dev/null +++ b/packages/zaino-state/src/chain_index/tests/finalised_state/migrations/v1_2_to_v1_3.rs @@ -0,0 +1,99 @@ +//! V1.2.1 to V1.3.0 migration tests. +//! +//! Coverage note: these tests use `ActivationHeights::default()`, whose NU6.3 activation is `None`, +//! so the migration takes the below-activation branch (rebuild each commitment row in place from the +//! legacy fixed-length table, no ironwood). The at/above-activation *ironwood backfill* branch — +//! which refetches block data from the validator — cannot be exercised yet: `MockchainSource` +//! serves no ironwood commitment roots (see its `get_commitment_tree_roots` TODO) and the test +//! vectors carry no ironwood actions, so building a post-NU6.3 block would fail resolving the +//! (required) ironwood root. That path needs ironwood-capable test vectors before it can be tested. + +use std::path::PathBuf; +use tempfile::TempDir; +use zaino_common::network::ActivationHeights; +use zaino_common::{DatabaseConfig, Network, StorageConfig}; + +use crate::chain_index::finalised_state::capability::{DbVersion, MigrationStatus}; +use crate::chain_index::finalised_state::finalised_source::v1::DB_VERSION_V1; +use crate::chain_index::finalised_state::FinalisedState; +use crate::chain_index::tests::init_tracing; +use crate::chain_index::tests::vectors::{ + build_active_mockchain_source, load_test_vectors, TestVectorData, +}; +use crate::{ChainIndexConfig, Height, StatusType}; + +fn v1_2_1() -> DbVersion { + DbVersion { + major: 1, + minor: 2, + patch: 1, + } +} + +/// Regression test for the startup ordering bug: opening a pre-1.3.0 cache used to start the +/// background validator concurrently with the v1.2.1 → v1.3.0 migration. The validator's +/// `initial_block_scan` read the freshly-created-empty `commitment_tree_data_1_3_0` table before the +/// migration rebuilt it, failing with `MDB_NOTFOUND` ("block scan") and latching `CriticalError`. +/// +/// The fix defers the validator until every migration finishes. This test builds an on-disk v1.2.1 +/// database, then reopens it through the production `FinalisedState::spawn` path (which migrates to +/// the current schema and only then starts the validator) and asserts it reaches `Ready` — i.e. the +/// migration completed *before* validation, and validation then passed. +#[tokio::test(flavor = "multi_thread")] +async fn v1_2_1_cache_migrates_to_current_then_validates() { + init_tracing(); + + let TestVectorData { blocks, .. } = load_test_vectors().unwrap(); + let active_height = Height(150); + + let temporary_directory: TempDir = tempfile::tempdir().unwrap(); + let database_path: PathBuf = temporary_directory.path().to_path_buf(); + + let database_config = ChainIndexConfig { + storage: StorageConfig { + database: DatabaseConfig { + path: database_path, + ..Default::default() + }, + ..Default::default() + }, + ephemeral: false, + db_version: 1, + network: Network::Regtest(ActivationHeights::default()), + }; + + let source = build_active_mockchain_source(active_height.0, blocks.clone()); + + // Build an on-disk v1.2.1 database (the pre-1.3.0 cache shape), then release it. + let old_database = + FinalisedState::build_db_to_version(database_config.clone(), source.clone(), v1_2_1()) + .await + .unwrap(); + old_database.wait_until_synced().await; + assert_eq!(old_database.get_metadata().await.unwrap().version, v1_2_1()); + old_database.shutdown().await.unwrap(); + drop(old_database); + + // Reopen through the production spawn path: it runs the v1.2.1 → v1.3.0 migration and only then + // starts the validator. Before the ordering fix this raced the migration and latched + // `CriticalError`; now it must migrate first and validate cleanly. + let migrated_database = FinalisedState::spawn(database_config.clone(), source.clone()) + .await + .unwrap(); + migrated_database.wait_until_synced().await; + + assert_eq!( + migrated_database.status(), + StatusType::Ready, + "the validator must run only after the migration completes, and then reach Ready" + ); + + let migrated_metadata = migrated_database.get_metadata().await.unwrap(); + assert_eq!(migrated_metadata.version, DB_VERSION_V1); + assert_eq!(migrated_metadata.migration_status, MigrationStatus::Empty); + + let migrated_height = migrated_database.db_height().await.unwrap().unwrap(); + assert_eq!(migrated_height, active_height); + + migrated_database.shutdown().await.unwrap(); +} From 7180e3c6eb59f38338048244178560c4d21290dd Mon Sep 17 00:00:00 2001 From: zancas Date: Mon, 6 Jul 2026 22:01:02 -0700 Subject: [PATCH 24/58] feat!: enforce the Validator as the sole source of activation-height truth MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Validator's configured activation heights are now the only source of truth for every component, enforced at compile time on both sides of the zcash_local_net seam (zaino#1076; infrastructure ADR 0003 and PR #278; the inbound spec is committed as zainod-heights-from-validator-spec.md). On the indexer side, zaino_common::Network is a payload-free kind, so a configuration value cannot carry heights and a pre-handshake height read does not typecheck. Both backends construct the runtime zebra_chain::parameters::Network in exactly one place: the spawn handshake, which uses zebra's compiled parameters for the public networks and adopts the validator's reported schedule from getblockchaininfo.upgrades for regtest. An upgrade absent from the validator's map is never-activated, nothing is backfilled from defaults, and a failed or unparseable handshake aborts startup with an error naming the validator endpoint. Every downstream consumer — the chain-index config, the connectors, the finalised state, and the subscribers via ServiceMetadata — receives that runtime network and can no longer manufacture its own; the conversions that let a config value impersonate chain truth (to_zebra_network and the From impls into zebra's type) are deleted, along with the dead accessors that invited a bypass. Test fixtures that are their own chain construct runtime networks explicitly through ActivationHeights::to_regtest_network. On the wallet side, the zcash_local_net pin moves to 78511f7, where the devtool client's network is a WalletNetwork whose regtest variant only WalletNetwork::from_validator can construct. The e2e client builder becomes build_clients(port, &validator) and derives the wallet schedule from the running validator, so: *wallet/validator height drift is unrepresentable* and a fixture's heights are typed exactly once, in the zebrad launch config. The harness no longer mirrors fixture heights into zainod: every custom-heights launch doubles as a regression test of the adoption path. The canonical launch set moves to zaino-testutils as ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS, which is validator-launch vocabulary rather than anyone's schedule. The uniqueness regressions live in clientless/tests/validator_heights.rs: a kind-only-configured zainod must sync a NU6.3-at-6 schedule its configuration never saw, and a golden test pins the upgrades map a live zebrad reports. Three unit tests beside activation_heights_from_upgrades pin the mapping itself. The production workspace, all live-test crates, and the unit suites compile and pass clean. Co-Authored-By: Claude Fable 5 --- Cargo.lock | 6 +- Cargo.toml | 9 +- live-tests/clientless/tests/chain_cache.rs | 22 ++- live-tests/clientless/tests/fetch_service.rs | 1 - .../clientless/tests/validator_heights.rs | 156 +++++++++++++++ live-tests/e2e/src/devtool.rs | 22 ++- live-tests/e2e/tests/devtool.rs | 10 + live-tests/e2e/tests/devtool_zcashd.rs | 5 +- live-tests/e2e/tests/test_vectors.rs | 5 +- live-tests/zaino-testutils/src/lib.rs | 62 +++--- packages/zaino-common/src/config/network.rs | 183 +++--------------- packages/zaino-common/src/lib.rs | 2 +- packages/zaino-state/src/backends.rs | 171 ++++++++++++++++ packages/zaino-state/src/backends/fetch.rs | 59 ++++-- packages/zaino-state/src/backends/state.rs | 162 ++++++++-------- packages/zaino-state/src/chain_index.rs | 2 +- .../src/chain_index/finalised_state.rs | 16 +- .../finalised_state/finalised_source/v1.rs | 6 +- .../v1/tx_out_set_accumulator.rs | 6 +- .../finalised_source/v1/write_core.rs | 9 +- .../chain_index/finalised_state/migrations.rs | 4 +- .../zaino-state/src/chain_index/source.rs | 1 - .../chain_index/source/mockchain_source.rs | 2 +- .../chain_index/source/validator_connector.rs | 12 +- packages/zaino-state/src/chain_index/tests.rs | 6 +- .../tests/finalised_state/ephemeral.rs | 4 +- .../migrations/v1_0_to_v1_1.rs | 6 +- .../migrations/v1_1_to_v1_2.rs | 8 +- .../chain_index/tests/finalised_state/v1.rs | 10 +- .../chain_index/tests/proptest_blockgen.rs | 22 +-- packages/zaino-state/src/config.rs | 52 +++-- packages/zainod/src/config.rs | 5 - zainod-heights-from-validator-spec.md | 170 ++++++++++++++++ 33 files changed, 821 insertions(+), 395 deletions(-) create mode 100644 live-tests/clientless/tests/validator_heights.rs create mode 100644 zainod-heights-from-validator-spec.md diff --git a/Cargo.lock b/Cargo.lock index 5b4174254..3d0124b57 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6184,7 +6184,7 @@ dependencies = [ [[package]] name = "zcash_local_net" version = "0.7.0" -source = "git+https://github.com/zingolabs/infrastructure.git?rev=0dc4a51f7d9666a3df7eaf28ad4f95cacb86d646#0dc4a51f7d9666a3df7eaf28ad4f95cacb86d646" +source = "git+https://github.com/zingolabs/infrastructure.git?rev=78511f7304ba27895967be860016ede923acec08#78511f7304ba27895967be860016ede923acec08" dependencies = [ "hex", "reqwest 0.12.28", @@ -6735,12 +6735,12 @@ dependencies = [ [[package]] name = "zingo-consensus" version = "0.1.0" -source = "git+https://github.com/zingolabs/infrastructure.git?rev=0dc4a51f7d9666a3df7eaf28ad4f95cacb86d646#0dc4a51f7d9666a3df7eaf28ad4f95cacb86d646" +source = "git+https://github.com/zingolabs/infrastructure.git?rev=78511f7304ba27895967be860016ede923acec08#78511f7304ba27895967be860016ede923acec08" [[package]] name = "zingo_test_vectors" version = "0.0.1" -source = "git+https://github.com/zingolabs/infrastructure.git?rev=0dc4a51f7d9666a3df7eaf28ad4f95cacb86d646#0dc4a51f7d9666a3df7eaf28ad4f95cacb86d646" +source = "git+https://github.com/zingolabs/infrastructure.git?rev=78511f7304ba27895967be860016ede923acec08#78511f7304ba27895967be860016ede923acec08" [[package]] name = "zip32" diff --git a/Cargo.toml b/Cargo.toml index de1292cf9..bacac4ad1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -138,9 +138,12 @@ zaino-testutils = { path = "live-tests/zaino-testutils" } # Zingo-infra (validator launcher driving the live tests; not a prod dep). # Pinned to an exact commit past zcash_local_net_v0.7.0: the v0.7.0 launcher's # zebrad config writer silently drops NU6.3 activation heights -# (https://github.com/zingolabs/zaino/issues/1368); this rev carries the -# "NU6.3" TOML plumbing. Restore a release tag once one is cut upstream. -zcash_local_net = { git = "https://github.com/zingolabs/infrastructure.git", rev = "0dc4a51f7d9666a3df7eaf28ad4f95cacb86d646" } +# (https://github.com/zingolabs/zaino/issues/1368). This rev additionally +# makes the Validator the sole compile-time source of activation-height +# truth (infrastructure ADR 0003, PR #278): wallet clients derive their +# heights via `WalletNetwork::from_validator`, and `ZainodConfig` carries a +# payload-free network kind. Restore a release tag once one is cut upstream. +zcash_local_net = { git = "https://github.com/zingolabs/infrastructure.git", rev = "78511f7304ba27895967be860016ede923acec08" } wire_serialized_transaction_test_data = "1.0.0" config = { version = "0.15", default-features = false, features = ["toml"] } diff --git a/live-tests/clientless/tests/chain_cache.rs b/live-tests/clientless/tests/chain_cache.rs index a12cb7291..8d0b4b058 100644 --- a/live-tests/clientless/tests/chain_cache.rs +++ b/live-tests/clientless/tests/chain_cache.rs @@ -173,11 +173,12 @@ mod chain_query_interface { }, ephemeral, db_version: 1, - network: zaino_common::Network::Regtest( - zaino_testutils::from_local_net_activation_heights( - &test_manager.local_net.get_activation_heights().await, - ), - ), + // This fixture derives its runtime network from the + // heights the harness launched the validator with. + network: zaino_testutils::from_local_net_activation_heights( + &test_manager.local_net.get_activation_heights().await, + ) + .to_regtest_network(), }; // **NOTE** The "fetch" backend is currently the backend used in the wild, and @@ -219,11 +220,12 @@ mod chain_query_interface { }, ephemeral, db_version: 1, - network: zaino_common::Network::Regtest( - zaino_testutils::from_local_net_activation_heights( - &test_manager.local_net.get_activation_heights().await, - ), - ), + // This fixture derives its runtime network from the + // heights the harness launched the validator with. + network: zaino_testutils::from_local_net_activation_heights( + &test_manager.local_net.get_activation_heights().await, + ) + .to_regtest_network(), }; let chain_index = NodeBackedChainIndex::new( ValidatorConnector::Fetch(json_service.clone()), diff --git a/live-tests/clientless/tests/fetch_service.rs b/live-tests/clientless/tests/fetch_service.rs index 53ea44785..0a74f69d3 100644 --- a/live-tests/clientless/tests/fetch_service.rs +++ b/live-tests/clientless/tests/fetch_service.rs @@ -214,7 +214,6 @@ async fn fetch_service_get_block_subsidy(validator: &ValidatorK // first halving boundary plus a margin on both validators. let height_limit = fetch_service_subscriber .network() - .to_zebra_network() .height_for_first_halving() .0 + 10; diff --git a/live-tests/clientless/tests/validator_heights.rs b/live-tests/clientless/tests/validator_heights.rs new file mode 100644 index 000000000..a7bbc434a --- /dev/null +++ b/live-tests/clientless/tests/validator_heights.rs @@ -0,0 +1,156 @@ +//! Regression tests for the single source of truth for activation heights +//! (zaino#1076, `zainod-heights-from-validator-spec.md`). +//! +//! The invariant: the validator's configured activation heights are +//! authoritative. zainod's config carries only a network kind — its regtest +//! placeholder is the canonical `ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS` +//! regardless of the fixture — and both backends adopt the real schedule +//! from `getblockchaininfo.upgrades` at spawn. +//! +//! Two halves, matching the spec's acceptance criteria: +//! +//! 1. [`zainod_syncs_a_schedule_its_config_never_saw`] — boundary sync and +//! the no-recompile proof in one: the same zainod build and (kind-only) +//! configuration that every canonical-heights test runs is here pointed +//! at a validator on the NU6.3-at-6 transition schedule, and must sync +//! across the boundary and serve era-correct compact blocks. Before +//! adoption this exact misalignment killed the chain-index sync with +//! `InvalidData("Block commitment could not be computed")`. +//! 2. [`getblockchaininfo_reports_the_configured_schedule`] — the input +//! contract: what zebrad actually puts in the `upgrades` map for a +//! configured schedule, pinned against a live node rather than assumed. +//! The mapping from that shape to adopted heights is unit-tested next to +//! `activation_heights_from_upgrades` in zaino-state. + +#[allow(deprecated)] +use zaino_state::FetchService; +use zaino_state::ZcashIndexer as _; +use zaino_testutils::{ + all_pools_i32, collect_block_range, MinerPool, TestManager, ValidatorKind, + NU6_3_TRANSITION_BOUNDARY, ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS, + ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS, +}; +use zcash_local_net::validator::zebrad::Zebrad; + +/// Launches an orchard-receiver-mining zebrad on the transition schedule +/// with zainod enabled. The harness hands zainod only the canonical +/// placeholder (see `launch_mining_to`), so the launch itself is the +/// deliberate config/validator misalignment under test. +#[allow(deprecated)] +async fn launch_transition_validator() -> TestManager { + assert_ne!( + ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS, ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS, + "premise: zainod's config placeholder must differ from the fixture \ + schedule, or this proves nothing" + ); + + TestManager::::launch_mining_to( + MinerPool::Orchard, + &ValidatorKind::Zebrad, + None, + Some(ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS), + None, + true, + false, + false, + ) + .await + .expect("launch TestManager") +} + +/// Boundary sync + no-recompile proof: a kind-only-configured zainod adopts +/// the NU6.3-at-6 schedule from the validator, syncs across the boundary, +/// and serves era-correct compact blocks for both eras. +/// +/// multi_thread required: the test manager spawns the validator and indexer +/// services. +#[tokio::test(flavor = "multi_thread")] +async fn zainod_syncs_a_schedule_its_config_never_saw() { + let mut test_manager = launch_transition_validator().await; + let subscriber = test_manager.subscriber().clone(); + + // Two blocks past the boundary, so both eras carry more than one block. + // Reaching the tip at all is the core regression: pre-adoption, the + // chain-index sync died on the first block whose commitment scheme the + // misconfigured heights got wrong. + test_manager + .generate_blocks_and_wait_for_tip(NU6_3_TRANSITION_BOUNDARY + 1, &subscriber) + .await; + let tip = u64::from(subscriber.chain_height().await.expect("chain height").0); + assert!( + tip > u64::from(NU6_3_TRANSITION_BOUNDARY), + "sync must cross the boundary, tip is {tip}" + ); + + // Era composition of the served chain proves the adopted schedule is the + // validator's, not the placeholder: under the placeholder (NU6.3 at 2) + // the pre-boundary orchard coinbases would be misread as ironwood-era. + let blocks = collect_block_range(&subscriber, 2, tip, all_pools_i32()).await; + for block in &blocks { + let height = block.height; + let has_orchard = block.vtx.iter().any(|tx| !tx.actions.is_empty()); + let has_ironwood = block.vtx.iter().any(|tx| !tx.ironwood_actions.is_empty()); + if height >= u64::from(NU6_3_TRANSITION_BOUNDARY) { + assert!( + has_ironwood && !has_orchard, + "height {height} must be ironwood-era, got orchard={has_orchard} ironwood={has_ironwood}" + ); + } else { + assert!( + has_orchard && !has_ironwood, + "height {height} must be orchard-era, got orchard={has_orchard} ironwood={has_ironwood}" + ); + } + } + + test_manager.close().await; +} + +/// The input contract for adoption: the `upgrades` map a live zebrad reports +/// for the transition schedule, pinned exactly — upgrade set, order, and +/// heights. Establishes (from real output, not reasoning) that nothing +/// pre-Overwinter appears: the map is keyed by consensus branch ID, which +/// pre-Overwinter eras don't have. +/// +/// multi_thread required: the test manager spawns the validator and indexer +/// services. +#[tokio::test(flavor = "multi_thread")] +async fn getblockchaininfo_reports_the_configured_schedule() { + use zebra_chain::parameters::NetworkUpgrade; + + let mut test_manager = launch_transition_validator().await; + + let blockchain_info = test_manager + .full_node_jsonrpc_connector() + .await + .get_blockchain_info() + .await + .expect("getblockchaininfo"); + + let reported: Vec<(NetworkUpgrade, u32)> = blockchain_info + .upgrades + .values() + .map(|upgrade_info| { + let (upgrade, height, _status) = upgrade_info.into_parts(); + (upgrade, height.0) + }) + .collect(); + + assert_eq!( + reported, + vec![ + (NetworkUpgrade::Overwinter, 1), + (NetworkUpgrade::Sapling, 1), + (NetworkUpgrade::Blossom, 1), + (NetworkUpgrade::Heartwood, 1), + (NetworkUpgrade::Canopy, 1), + (NetworkUpgrade::Nu5, 2), + (NetworkUpgrade::Nu6, 2), + (NetworkUpgrade::Nu6_1, 2), + (NetworkUpgrade::Nu6_2, 2), + (NetworkUpgrade::Nu6_3, 6), + ], + ); + + test_manager.close().await; +} diff --git a/live-tests/e2e/src/devtool.rs b/live-tests/e2e/src/devtool.rs index 4fed529e6..e987a386a 100644 --- a/live-tests/e2e/src/devtool.rs +++ b/live-tests/e2e/src/devtool.rs @@ -39,17 +39,27 @@ pub struct DevtoolClients { } /// Launch faucet + recipient devtool wallets against a running Zaino gRPC -/// listener. The devtool analogue of [`crate::build_clients`]; Zaino must -/// already be serving (wallet initialization fetches the chain tip and -/// birthday tree state from it). -pub async fn build_clients(zaino_grpc_listen_port: u16) -> DevtoolClients { - let mut faucet_config = ZcashDevtoolConfig::faucet(); +/// listener, deriving the wallet network from the running `validator`. That +/// derivation is the only construction the client API offers +/// (infrastructure ADR 0003: the Validator is the single source of truth +/// for activation heights), so wallet/validator height drift is +/// unrepresentable — the wallets follow whatever schedule the validator was +/// launched with, fixture or canonical. The devtool analogue of +/// [`crate::build_clients`]; Zaino must already be serving (wallet +/// initialization fetches the chain tip and birthday tree state from it). +pub async fn build_clients( + zaino_grpc_listen_port: u16, + validator: &V, +) -> DevtoolClients { + let network = zcash_local_net::client::WalletNetwork::from_validator(validator).await; + + let mut faucet_config = ZcashDevtoolConfig::faucet(network); faucet_config.indexer_port = zaino_grpc_listen_port; let faucet = ZcashDevtool::launch(faucet_config) .await .expect("launch devtool faucet wallet"); - let mut recipient_config = ZcashDevtoolConfig::recipient(); + let mut recipient_config = ZcashDevtoolConfig::recipient(network); recipient_config.indexer_port = zaino_grpc_listen_port; let recipient = ZcashDevtool::launch(recipient_config) .await diff --git a/live-tests/e2e/tests/devtool.rs b/live-tests/e2e/tests/devtool.rs index e334ca441..9f8db292f 100644 --- a/live-tests/e2e/tests/devtool.rs +++ b/live-tests/e2e/tests/devtool.rs @@ -84,6 +84,7 @@ where .zaino_grpc_listen_address .expect("zaino enabled") .port(), + &test_manager.local_net, ) .await; @@ -122,6 +123,7 @@ where .zaino_grpc_listen_address .expect("zaino enabled") .port(), + &test_manager.local_net, ) .await; @@ -789,6 +791,7 @@ async fn block_range_returns_default_pools() { .zaino_grpc_listen_address .expect("zaino enabled") .port(), + &svc.test_manager.local_net, ) .await; @@ -880,6 +883,7 @@ async fn block_range_returns_all_pools() { .zaino_grpc_listen_address .expect("zaino enabled") .port(), + &svc.test_manager.local_net, ) .await; @@ -1021,6 +1025,7 @@ async fn fund_and_send_dual( .zaino_grpc_listen_address .expect("zaino enabled") .port(), + &svc.test_manager.local_net, ) .await; @@ -1172,6 +1177,7 @@ async fn fund_and_fill_mempool_dual() -> zaino_testutils::StateAndFetchServices< .zaino_grpc_listen_address .expect("zaino enabled") .port(), + &svc.test_manager.local_net, ) .await; @@ -1346,6 +1352,7 @@ async fn launch_transparent_and_faucet_taddr( .zaino_grpc_listen_address .expect("zaino enabled") .port(), + &svc.test_manager.local_net, ) .await; @@ -1679,6 +1686,7 @@ async fn address_deltas() { .zaino_grpc_listen_address .expect("zaino enabled") .port(), + &svc.test_manager.local_net, ) .await; @@ -1838,6 +1846,7 @@ async fn get_block_deltas_resolves_transparent_spend() { .zaino_grpc_listen_address .expect("zaino enabled") .port(), + &svc.test_manager.local_net, ) .await; @@ -2093,6 +2102,7 @@ async fn get_outpoint_spenders_fetch_vs_state() { .zaino_grpc_listen_address .expect("zaino enabled") .port(), + &svc.test_manager.local_net, ) .await; diff --git a/live-tests/e2e/tests/devtool_zcashd.rs b/live-tests/e2e/tests/devtool_zcashd.rs index 4405ff344..3713509fc 100644 --- a/live-tests/e2e/tests/devtool_zcashd.rs +++ b/live-tests/e2e/tests/devtool_zcashd.rs @@ -44,7 +44,7 @@ async fn launch_zcashd_and_build_clients() -> (TestManager &ValidatorKind::Zcashd, None, // network -> Regtest // The heights the devtool wallet accepts (same as the zebrad path). - Some(zaino_common::network::ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS), + Some(zaino_testutils::ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS), None, // no chain cache: build fresh at these heights true, // enable zaino false, // no json-rpc server @@ -58,6 +58,7 @@ async fn launch_zcashd_and_build_clients() -> (TestManager .zaino_grpc_listen_address .expect("zaino enabled") .port(), + &test_manager.local_net, ) .await; @@ -110,7 +111,7 @@ async fn faucet_receives_zcashd_orchard_reward() { /// of json_server's `create_zcashd_test_manager_and_fetch_services`. async fn create_zcashd_devtool_services() -> (ZcashdDualFetchServices, DevtoolClients) { let services = zaino_testutils::launch_zcashd_dual_fetch_services_at( - zaino_common::network::ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS, + zaino_testutils::ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS, ) .await; let clients = e2e::devtool::build_clients( diff --git a/live-tests/e2e/tests/test_vectors.rs b/live-tests/e2e/tests/test_vectors.rs index aeee240f4..a44dffa40 100644 --- a/live-tests/e2e/tests/test_vectors.rs +++ b/live-tests/e2e/tests/test_vectors.rs @@ -67,6 +67,7 @@ async fn create_200_block_regtest_chain_vectors() { .zaino_grpc_listen_address .expect("zaino enabled") .port(), + &test_manager.local_net, ) .await; @@ -319,7 +320,7 @@ async fn create_200_block_regtest_chain_vectors() { }; let sapling_treestate = match zebra_chain::parameters::NetworkUpgrade::Sapling - .activation_height(&state_service_subscriber.network().to_zebra_network()) + .activation_height(&state_service_subscriber.network()) { Some(activation_height) if height >= activation_height.0 => Some( state @@ -342,7 +343,7 @@ async fn create_200_block_regtest_chain_vectors() { }) .unwrap(); let orchard_treestate = match zebra_chain::parameters::NetworkUpgrade::Nu5 - .activation_height(&state_service_subscriber.network().to_zebra_network()) + .activation_height(&state_service_subscriber.network()) { Some(activation_height) if height >= activation_height.0 => Some( state diff --git a/live-tests/zaino-testutils/src/lib.rs b/live-tests/zaino-testutils/src/lib.rs index 9fcd5649b..6e8a342e1 100644 --- a/live-tests/zaino-testutils/src/lib.rs +++ b/live-tests/zaino-testutils/src/lib.rs @@ -14,7 +14,7 @@ use std::{ use tonic::transport::Channel; use tracing::{debug, info, instrument}; use zaino_common::{ - network::{ActivationHeights, ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS}, + network::ActivationHeights, probing::{Liveness, Readiness}, status::Status, validator::ValidatorConfig, @@ -67,6 +67,29 @@ macro_rules! validator_tests { }; } +/// The canonical regtest activation heights the harness *launches* validators +/// with when a test names no others (everything through NU6.3 active from +/// height 2). This is validator-launch vocabulary — the configuring side of +/// the truth source — and is never zainod's own schedule: zainod's config +/// carries only a network kind, and both backends adopt the runtime schedule +/// from the running validator's `getblockchaininfo.upgrades` (zaino#1076). +/// It matches zcash_local_net's `supported_regtest_activation_heights`, which +/// the devtool wallet client currently requires (zaino#1368). +pub const ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS: ActivationHeights = ActivationHeights { + before_overwinter: Some(1), + overwinter: Some(1), + sapling: Some(1), + blossom: Some(1), + heartwood: Some(1), + canopy: Some(1), + nu5: Some(2), + nu6: Some(2), + nu6_1: Some(2), + nu6_2: Some(2), + nu6_3: Some(2), + nu7: None, +}; + /// Orchard-era-only fixture: every upgrade through NU6.2 active from height 2 and /// NU6.3 never activating, so coinbases stay Orchard for the whole chain. For /// client-free launches only — the zcash-devtool wallet hardcodes the canonical @@ -719,8 +742,17 @@ where let activation_heights = activation_heights.unwrap_or_else(|| validator.default_activation_heights()); let network_kind = network.unwrap_or(NetworkKind::Regtest); - let zaino_network_kind = - Network::from_network_kind_and_activation_heights(&network_kind, &activation_heights); + // The validator is the single source of truth for activation heights + // (zaino#1076). zainod's config is a payload-free network kind: the + // fixture heights configure the validator alone, and zainod adopts + // the real schedule from the validator at spawn. Every + // custom-heights launch is therefore a standing regression test of + // that adoption. + let zaino_network_kind = match network_kind { + NetworkKind::Mainnet => Network::Mainnet, + NetworkKind::Testnet => Network::Testnet, + NetworkKind::Regtest => Network::Regtest, + }; if enable_clients && !enable_zaino { return Err(std::io::Error::other( @@ -1142,23 +1174,9 @@ pub async fn launch_state_and_fetch_services_mining_to( tokio::time::sleep(std::time::Duration::from_millis(5000)).await; Network::Testnet } - _ => Network::Regtest({ - let activation_heights = test_manager.local_net.get_activation_heights().await; - ActivationHeights { - before_overwinter: activation_heights.overwinter(), - overwinter: activation_heights.overwinter(), - sapling: activation_heights.sapling(), - blossom: activation_heights.blossom(), - heartwood: activation_heights.heartwood(), - canopy: activation_heights.canopy(), - nu5: activation_heights.nu5(), - nu6: activation_heights.nu6(), - nu6_1: activation_heights.nu6_1(), - nu6_2: activation_heights.nu6_2(), - nu6_3: activation_heights.nu6_3(), - nu7: activation_heights.nu7(), - } - }), + // The kind suffices: zainod adopts the schedule from the validator + // at spawn (zaino#1076). + _ => Network::Regtest, }; test_manager.local_net.print_stdout(); @@ -1339,7 +1357,7 @@ pub async fn launch_zcashd_dual_fetch_services_at( .data_dir() .path() .join("zcashd-fetch-service-zaino"), - Network::Regtest(activation_heights), + Network::Regtest, ) .await; let zcashd_subscriber = zcashd_fetch_service.get_subscriber().inner(); @@ -1357,7 +1375,7 @@ pub async fn launch_zcashd_dual_fetch_services_at( .data_dir() .path() .join("zaino-fetch-service-zaino"), - Network::Regtest(activation_heights), + Network::Regtest, ) .await; let zaino_subscriber = zaino_fetch_service.get_subscriber().inner(); diff --git a/packages/zaino-common/src/config/network.rs b/packages/zaino-common/src/config/network.rs index d518d895b..398908ca1 100644 --- a/packages/zaino-common/src/config/network.rs +++ b/packages/zaino-common/src/config/network.rs @@ -5,36 +5,21 @@ use std::fmt; use serde::{Deserialize, Serialize}; use zebra_chain::parameters::testnet::ConfiguredActivationHeights; -/// Must equal zcash_local_net's `supported_regtest_activation_heights`: the -/// zcash-devtool wallet client hardcodes that canonical set (NU6.3 at 2), and a -/// zebrad configured differently rejects wallet-built transactions with -/// "incorrect consensus branch id". See -/// . -pub const ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS: ActivationHeights = ActivationHeights { - overwinter: Some(1), - before_overwinter: Some(1), - sapling: Some(1), - blossom: Some(1), - heartwood: Some(1), - canopy: Some(1), - nu5: Some(2), - nu6: Some(2), - nu6_1: Some(2), - nu6_2: Some(2), - nu6_3: Some(2), - nu7: None, -}; - -/// Network type for Zaino configuration. +/// The network *kind* zaino is configured for. Deliberately payload-free: +/// activation heights are chain facts the validator owns, so a config value +/// cannot carry them — the backends adopt the runtime schedule from the +/// validator's `getblockchaininfo.upgrades` at spawn and hold it as a +/// `zebra_chain::parameters::Network` +/// (). A pre-adoption +/// height read is unrepresentable: this type has no heights to read. #[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Deserialize, serde::Serialize)] -#[serde(from = "NetworkSerde", into = "NetworkSerde")] pub enum Network { /// Mainnet network Mainnet, /// Testnet network Testnet, /// Regtest network (for local testing) - Regtest(ActivationHeights), + Regtest, } impl fmt::Display for Network { @@ -42,38 +27,7 @@ impl fmt::Display for Network { match self { Network::Mainnet => write!(f, "Mainnet"), Network::Testnet => write!(f, "Testnet"), - Network::Regtest(_) => write!(f, "Regtest"), - } - } -} - -/// Helper type for Network serialization/deserialization. -/// -/// This allows Network to serialize as simple strings ("Mainnet", "Testnet", "Regtest") -/// while the actual Network::Regtest variant carries activation heights internally. -#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Deserialize, serde::Serialize)] -enum NetworkSerde { - Mainnet, - Testnet, - Regtest, -} - -impl From for Network { - fn from(value: NetworkSerde) -> Self { - match value { - NetworkSerde::Mainnet => Network::Mainnet, - NetworkSerde::Testnet => Network::Testnet, - NetworkSerde::Regtest => Network::Regtest(ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS), - } - } -} - -impl From for NetworkSerde { - fn from(value: Network) -> Self { - match value { - Network::Mainnet => NetworkSerde::Mainnet, - Network::Testnet => NetworkSerde::Testnet, - Network::Regtest(_) => NetworkSerde::Regtest, + Network::Regtest => write!(f, "Regtest"), } } } @@ -209,29 +163,6 @@ impl From for ConfiguredActivationHeights { } impl Network { - /// Convert to Zebra's network type for internal use (alias for to_zebra_default). - pub fn to_zebra_network(&self) -> zebra_chain::parameters::Network { - self.into() - } - - /// Get the standard regtest activation heights used by Zaino. - pub fn zaino_regtest_heights() -> ConfiguredActivationHeights { - ConfiguredActivationHeights { - before_overwinter: Some(1), - overwinter: Some(1), - sapling: Some(1), - blossom: Some(1), - heartwood: Some(1), - canopy: Some(1), - nu5: Some(1), - nu6: Some(1), - nu6_1: None, - nu6_2: None, - nu6_3: None, - nu7: None, - } - } - /// Determines if we should wait for the server to fully sync. Used for testing /// /// - Mainnet/Testnet: Skip sync (false) because we don't want to sync real chains in tests @@ -239,18 +170,7 @@ impl Network { pub fn wait_on_server_sync(&self) -> bool { match self { Network::Mainnet | Network::Testnet => false, // Real networks - don't try to sync the whole chain - Network::Regtest(_) => true, // Local network - safe and fast to sync - } - } - - pub fn from_network_kind_and_activation_heights( - network: &zebra_chain::parameters::NetworkKind, - activation_heights: &ActivationHeights, - ) -> Self { - match network { - zebra_chain::parameters::NetworkKind::Mainnet => Network::Mainnet, - zebra_chain::parameters::NetworkKind::Testnet => Network::Testnet, - zebra_chain::parameters::NetworkKind::Regtest => Network::Regtest(*activation_heights), + Network::Regtest => true, // Local network - safe and fast to sync } } } @@ -261,62 +181,7 @@ impl From for Network { zebra_chain::parameters::Network::Mainnet => Network::Mainnet, zebra_chain::parameters::Network::Testnet(parameters) => { if parameters.is_regtest() { - let mut activation_heights = ActivationHeights { - before_overwinter: None, - overwinter: None, - sapling: None, - blossom: None, - heartwood: None, - canopy: None, - nu5: None, - nu6: None, - nu6_1: None, - nu6_2: None, - nu6_3: None, - nu7: None, - }; - for (height, upgrade) in parameters.activation_heights().iter() { - match upgrade { - zebra_chain::parameters::NetworkUpgrade::Genesis => (), - zebra_chain::parameters::NetworkUpgrade::BeforeOverwinter => { - activation_heights.before_overwinter = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Overwinter => { - activation_heights.overwinter = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Sapling => { - activation_heights.sapling = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Blossom => { - activation_heights.blossom = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Heartwood => { - activation_heights.heartwood = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Canopy => { - activation_heights.canopy = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Nu5 => { - activation_heights.nu5 = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Nu6 => { - activation_heights.nu6 = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Nu6_1 => { - activation_heights.nu6_1 = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Nu6_2 => { - activation_heights.nu6_2 = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Nu6_3 => { - activation_heights.nu6_3 = Some(height.0) - } - zebra_chain::parameters::NetworkUpgrade::Nu7 => { - activation_heights.nu7 = Some(height.0) - } - } - } - Network::Regtest(activation_heights) + Network::Regtest } else { Network::Testnet } @@ -325,21 +190,17 @@ impl From for Network { } } -impl From for zebra_chain::parameters::Network { - fn from(val: Network) -> Self { - match val { - Network::Regtest(activation_heights) => zebra_chain::parameters::Network::new_regtest( - Into::::into(activation_heights).into(), - ), - Network::Testnet => zebra_chain::parameters::Network::new_default_testnet(), - Network::Mainnet => zebra_chain::parameters::Network::Mainnet, - } - } -} - -impl From<&Network> for zebra_chain::parameters::Network { - fn from(val: &Network) -> Self { - (*val).into() +impl ActivationHeights { + /// Builds the runtime regtest network for a chain whose schedule is + /// known first-hand: a validator being *launched* with these heights, or + /// a test fixture that is its own chain (mockchain sources, proptest + /// block generators). Production indexer code never calls this with + /// configured values — it adopts the runtime network from the + /// validator's reported schedule instead (zaino#1076). + pub fn to_regtest_network(&self) -> zebra_chain::parameters::Network { + zebra_chain::parameters::Network::new_regtest( + Into::::into(*self).into(), + ) } } diff --git a/packages/zaino-common/src/lib.rs b/packages/zaino-common/src/lib.rs index d2c5fcce5..818f23935 100644 --- a/packages/zaino-common/src/lib.rs +++ b/packages/zaino-common/src/lib.rs @@ -17,7 +17,7 @@ pub use net::{resolve_socket_addr, try_resolve_address, AddressResolution}; // Re-export commonly used config types at crate root for backward compatibility. // This allows existing code using `use zaino_common::Network` to continue working. -pub use config::network::{ActivationHeights, Network, ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS}; +pub use config::network::{ActivationHeights, Network}; pub use config::service::ServiceConfig; pub use config::storage::{ AccumulatorRebuildMemorySize, CacheConfig, DatabaseConfig, DatabaseSize, StorageConfig, diff --git a/packages/zaino-state/src/backends.rs b/packages/zaino-state/src/backends.rs index 7af6467ef..c5461bb2d 100644 --- a/packages/zaino-state/src/backends.rs +++ b/packages/zaino-state/src/backends.rs @@ -89,6 +89,62 @@ fn build_treestate_response( ) } +/// Builds the regtest activation heights from the validator's reported +/// upgrade schedule (`getblockchaininfo.upgrades`). +/// +/// The validator's configured activation heights are authoritative: the +/// config type is a payload-free kind, so both backends construct the +/// runtime network here at first contact, before anything consumes a +/// `Network` (zaino#1076). An upgrade absent from the validator's map is +/// never-activated — nothing is backfilled from defaults. Mainnet and +/// Testnet use zebra's compiled parameters and never take this path. +fn activation_heights_from_upgrades( + upgrades: &indexmap::IndexMap< + zebra_rpc::methods::ConsensusBranchIdHex, + zebra_rpc::methods::NetworkUpgradeInfo, + >, +) -> Result { + use zebra_chain::parameters::NetworkUpgrade; + + let mut heights = zaino_common::config::network::ActivationHeights { + before_overwinter: None, + overwinter: None, + sapling: None, + blossom: None, + heartwood: None, + canopy: None, + nu5: None, + nu6: None, + nu6_1: None, + nu6_2: None, + nu6_3: None, + nu7: None, + }; + for upgrade_info in upgrades.values() { + let (upgrade, height, _status) = upgrade_info.into_parts(); + let slot = match upgrade { + // Genesis is height 0 by definition; it has no configuration slot. + NetworkUpgrade::Genesis => continue, + NetworkUpgrade::BeforeOverwinter => &mut heights.before_overwinter, + NetworkUpgrade::Overwinter => &mut heights.overwinter, + NetworkUpgrade::Sapling => &mut heights.sapling, + NetworkUpgrade::Blossom => &mut heights.blossom, + NetworkUpgrade::Heartwood => &mut heights.heartwood, + NetworkUpgrade::Canopy => &mut heights.canopy, + NetworkUpgrade::Nu5 => &mut heights.nu5, + NetworkUpgrade::Nu6 => &mut heights.nu6, + NetworkUpgrade::Nu6_1 => &mut heights.nu6_1, + NetworkUpgrade::Nu6_2 => &mut heights.nu6_2, + NetworkUpgrade::Nu6_3 => &mut heights.nu6_3, + NetworkUpgrade::Nu7 => &mut heights.nu7, + }; + if slot.replace(height.0).is_some() { + return Err(format!("validator reported {upgrade:?} twice")); + } + } + Ok(heights) +} + fn latest_network_upgrade( upgrades: &indexmap::IndexMap< zebra_rpc::methods::ConsensusBranchIdHex, @@ -128,6 +184,121 @@ fn validate_utxo_address_count(count: usize) -> Result<(), tonic::Status> { #[cfg(test)] mod tests { + use zaino_common::config::network::ActivationHeights; + + /// All-`None` heights: the starting point adoption fills from the + /// validator's map, and the expected value for every absent upgrade. + const NEVER_ACTIVATED: ActivationHeights = ActivationHeights { + before_overwinter: None, + overwinter: None, + sapling: None, + blossom: None, + heartwood: None, + canopy: None, + nu5: None, + nu6: None, + nu6_1: None, + nu6_2: None, + nu6_3: None, + nu7: None, + }; + + fn upgrades_map( + json: &str, + ) -> indexmap::IndexMap< + zebra_rpc::methods::ConsensusBranchIdHex, + zebra_rpc::methods::NetworkUpgradeInfo, + > { + serde_json::from_str(json).expect("upgrades fixture parses") + } + + fn adopted_heights( + upgrades: &indexmap::IndexMap< + zebra_rpc::methods::ConsensusBranchIdHex, + zebra_rpc::methods::NetworkUpgradeInfo, + >, + ) -> ActivationHeights { + super::activation_heights_from_upgrades(upgrades).expect("valid schedule") + } + + /// An upgrade absent from the validator's map is never-activated — + /// nothing is backfilled from any default schedule. + #[test] + fn regtest_network_from_upgrades_leaves_absent_upgrades_never_activated() { + let upgrades = upgrades_map( + r#"{ + "c2d6d0b4": { "name": "NU5", "activationheight": 2, "status": "active" }, + "c8e71055": { "name": "NU6", "activationheight": 2, "status": "active" } + }"#, + ); + + assert_eq!( + adopted_heights(&upgrades), + ActivationHeights { + nu5: Some(2), + nu6: Some(2), + ..NEVER_ACTIVATED + } + ); + } + + /// The ORCHARD_THEN_IRONWOOD transition shape: everything through NU6.2 + /// at 1–2, NU6.3 at 6 — the schedule the ironwood_activation fixtures + /// launch validators with. + #[test] + fn regtest_network_from_upgrades_reads_a_transition_schedule() { + let upgrades = upgrades_map( + r#"{ + "5ba81b19": { "name": "Overwinter", "activationheight": 1, "status": "active" }, + "76b809bb": { "name": "Sapling", "activationheight": 1, "status": "active" }, + "2bb40e60": { "name": "Blossom", "activationheight": 1, "status": "active" }, + "f5b9230b": { "name": "Heartwood", "activationheight": 1, "status": "active" }, + "e9ff75a6": { "name": "Canopy", "activationheight": 1, "status": "active" }, + "c2d6d0b4": { "name": "NU5", "activationheight": 2, "status": "active" }, + "c8e71055": { "name": "NU6", "activationheight": 2, "status": "active" }, + "4dec4df0": { "name": "NU6.1", "activationheight": 2, "status": "active" }, + "5437f330": { "name": "NU6.2", "activationheight": 2, "status": "active" }, + "37a5165b": { "name": "NU6.3", "activationheight": 6, "status": "pending" } + }"#, + ); + + assert_eq!( + adopted_heights(&upgrades), + ActivationHeights { + overwinter: Some(1), + sapling: Some(1), + blossom: Some(1), + heartwood: Some(1), + canopy: Some(1), + nu5: Some(2), + nu6: Some(2), + nu6_1: Some(2), + nu6_2: Some(2), + nu6_3: Some(6), + ..NEVER_ACTIVATED + } + ); + } + + /// A validator reporting the same upgrade twice is nonsense; adoption + /// must fail loudly rather than pick a height. + #[test] + fn regtest_network_from_upgrades_rejects_a_duplicate_upgrade() { + let upgrades = upgrades_map( + r#"{ + "c2d6d0b4": { "name": "NU5", "activationheight": 2, "status": "active" }, + "c8e71055": { "name": "NU5", "activationheight": 3, "status": "pending" } + }"#, + ); + + let reason = + super::activation_heights_from_upgrades(&upgrades).expect_err("duplicate must fail"); + assert!( + reason.contains("twice"), + "error should name the duplication, got: {reason}" + ); + } + #[test] fn latest_network_upgrade_rejects_empty_metadata() { let upgrades = indexmap::IndexMap::new(); diff --git a/packages/zaino-state/src/backends/fetch.rs b/packages/zaino-state/src/backends/fetch.rs index aefbb97fb..b452b03f2 100644 --- a/packages/zaino-state/src/backends/fetch.rs +++ b/packages/zaino-state/src/backends/fetch.rs @@ -61,7 +61,7 @@ use zaino_proto::proto::{ use crate::{ chain_index::chain_tips_from_nonfinalized_snapshot, chain_index::{source::ValidatorConnector, types}, - config::{DonationAddress, FetchServiceConfig}, + config::{ChainIndexConfig, DonationAddress, FetchServiceConfig}, error::FetchServiceError, indexer::{ handle_raw_transaction, IndexerSubscriber, LightWalletIndexer, ZcashIndexer, ZcashService, @@ -135,19 +135,52 @@ impl ZcashService for FetchService { ) .await?; + // The validator is the single source of truth for activation heights + // (zaino#1076): the config carries only a network kind, and the + // runtime network is constructed here before anything consumes one — + // from zebra's compiled parameters for the public networks, from the + // validator's reported schedule for regtest. There is no fallback: a + // silently wrong schedule is the failure mode this removes. + let network = match config.common.network { + zaino_common::Network::Mainnet => zebra_chain::parameters::Network::Mainnet, + zaino_common::Network::Testnet => { + zebra_chain::parameters::Network::new_default_testnet() + } + zaino_common::Network::Regtest => { + let blockchain_info = fetcher.get_blockchain_info().await.map_err(|error| { + FetchServiceError::Critical(format!( + "cannot fetch activation heights from the validator at {}: {error}", + config.common.validator_rpc_address + )) + })?; + let heights = super::activation_heights_from_upgrades(&blockchain_info.upgrades) + .map_err(|reason| { + FetchServiceError::Critical(format!( + "cannot adopt activation heights from the validator at {}: {reason}", + config.common.validator_rpc_address + )) + })?; + info!(?heights, "Adopted activation heights from the validator"); + heights.to_regtest_network() + } + }; + let zebra_build_data = fetcher.get_info().await?; let data = ServiceMetadata::new( get_build_info(config.common.indexer_version.clone()), - config.common.network.to_zebra_network(), + network.clone(), zebra_build_data.build, zebra_build_data.subversion, ); info!(build = %data.zebra_build(), subversion = %data.zebra_subversion(), "Connected to Zcash node"); let source = ValidatorConnector::Fetch(fetcher.clone()); - let indexer = NodeBackedChainIndex::new(source, config.clone().into()) - .await - .unwrap(); + let indexer = NodeBackedChainIndex::new( + source, + ChainIndexConfig::from_backend_config(&config.common, network), + ) + .await + .unwrap(); let fetch_service = Self { fetcher, @@ -233,10 +266,10 @@ impl FetchServiceSubscriber { self.indexer.status() } - /// Returns the network type running. - #[allow(deprecated)] - pub fn network(&self) -> zaino_common::Network { - self.config.common.network + /// Returns the runtime network, carrying the activation schedule + /// adopted from the validator (zaino#1076). + pub fn network(&self) -> zebra_chain::parameters::Network { + self.data.network() } } @@ -778,7 +811,7 @@ impl ZcashIndexer for FetchServiceSubscriber { height, confirmations, #[allow(deprecated)] - &self.config.common.network.to_zebra_network(), + &self.data.network(), None, block_hash, in_best_chain, @@ -1830,11 +1863,7 @@ impl LightWalletIndexer for FetchServiceSubscriber { .await?; Ok(super::tree_state_from_treestate_response( - self.config - .common - .network - .to_zebra_network() - .bip70_network_name(), + self.data.network().bip70_network_name(), treestate_response, )) } diff --git a/packages/zaino-state/src/backends/state.rs b/packages/zaino-state/src/backends/state.rs index 33c19c87c..321a1ddd0 100644 --- a/packages/zaino-state/src/backends/state.rs +++ b/packages/zaino-state/src/backends/state.rs @@ -7,7 +7,7 @@ use crate::{ source::ValidatorConnector, types as chain_types, ChainIndex, }, - config::{DonationAddress, StateServiceConfig}, + config::{ChainIndexConfig, DonationAddress, StateServiceConfig}, error::{BlockCacheError, StateServiceError}, indexer::{ handle_raw_transaction, IndexerSubscriber, LightWalletIndexer, ZcashIndexer, ZcashService, @@ -195,11 +195,41 @@ impl ZcashService for StateService { ) .await?; + // The validator is the single source of truth for activation heights + // (zaino#1076): the config carries only a network kind, and the + // runtime network is constructed here before anything consumes one — + // from zebra's compiled parameters for the public networks, from the + // validator's reported schedule for regtest. There is no fallback: a + // silently wrong schedule is the failure mode this removes. + let network = match config.common.network { + zaino_common::Network::Mainnet => zebra_chain::parameters::Network::Mainnet, + zaino_common::Network::Testnet => { + zebra_chain::parameters::Network::new_default_testnet() + } + zaino_common::Network::Regtest => { + let blockchain_info = rpc_client.get_blockchain_info().await.map_err(|error| { + StateServiceError::Critical(format!( + "cannot fetch activation heights from the validator at {}: {error}", + config.common.validator_rpc_address + )) + })?; + let heights = super::activation_heights_from_upgrades(&blockchain_info.upgrades) + .map_err(|reason| { + StateServiceError::Critical(format!( + "cannot adopt activation heights from the validator at {}: {reason}", + config.common.validator_rpc_address + )) + })?; + info!(?heights, "Adopted activation heights from the validator"); + heights.to_regtest_network() + } + }; + let zebra_build_data = rpc_client.get_info().await?; let data = ServiceMetadata::new( get_build_info(config.common.indexer_version.clone()), - config.common.network.to_zebra_network(), + network.clone(), zebra_build_data.build, zebra_build_data.subversion, ); @@ -212,7 +242,7 @@ impl ZcashService for StateService { let (mut read_state_service, _latest_chain_tip, chain_tip_change, sync_task_handle) = init_read_state_with_syncer( config.validator_state_config.clone(), - &config.common.network.to_zebra_network(), + &network, config.validator_grpc_address, ) .await??; @@ -259,7 +289,7 @@ impl ZcashService for StateService { let mempool_source = ValidatorConnector::State(crate::chain_index::source::State { read_state_service: read_state_service.clone(), mempool_fetcher: rpc_client.clone(), - network: config.common.network, + network: network.clone(), }); let mempool = Mempool::spawn(mempool_source, None).await?; @@ -268,9 +298,9 @@ impl ZcashService for StateService { ValidatorConnector::State(State { read_state_service: read_state_service.clone(), mempool_fetcher: rpc_client.clone(), - network: config.common.network, + network: network.clone(), }), - config.clone().into(), + ChainIndexConfig::from_backend_config(&config.common, network), ) .await .unwrap(); @@ -907,10 +937,10 @@ impl StateServiceSubscriber { } } - /// Returns the network type running. - #[allow(deprecated)] - pub fn network(&self) -> zaino_common::Network { - self.config.common.network + /// Returns the runtime network, carrying the activation schedule + /// adopted from the validator (zaino#1076). + pub fn network(&self) -> zebra_chain::parameters::Network { + self.data.network() } /// Returns the median time of the last 11 blocks. @@ -1032,18 +1062,14 @@ impl ZcashIndexer for StateServiceSubscriber { } async fn get_difficulty(&self) -> Result { - chain_tip_difficulty( - self.config.common.network.to_zebra_network(), - self.read_state_service.clone(), - false, - ) - .await - .map_err(|e| { - StateServiceError::RpcError(RpcError::new_from_errorobject( - e, - "failed to get difficulty", - )) - }) + chain_tip_difficulty(self.data.network(), self.read_state_service.clone(), false) + .await + .map_err(|e| { + StateServiceError::RpcError(RpcError::new_from_errorobject( + e, + "failed to get difficulty", + )) + }) } async fn get_block_subsidy(&self, height: u32) -> Result { @@ -1090,12 +1116,9 @@ impl ZcashIndexer for StateServiceSubscriber { }; let now = Utc::now(); - let zebra_estimated_height = NetworkChainTipHeightEstimator::new( - header.time, - height, - &self.config.common.network.into(), - ) - .estimate_height_at(now); + let zebra_estimated_height = + NetworkChainTipHeightEstimator::new(header.time, height, &self.data.network()) + .estimate_height_at(now); let estimated_height = if header.time > now || zebra_estimated_height < height { height } else { @@ -1103,10 +1126,8 @@ impl ZcashIndexer for StateServiceSubscriber { }; let upgrades = IndexMap::from_iter( - self.config - .common - .network - .to_zebra_network() + self.data + .network() .full_activation_list() .into_iter() .filter_map(|(activation_height, network_upgrade)| { @@ -1140,14 +1161,14 @@ impl ZcashIndexer for StateServiceSubscriber { (height + 1).expect("valid chain tips are a lot less than Height::MAX"); let consensus = TipConsensusBranch::from_parts( ConsensusBranchIdHex::new( - NetworkUpgrade::current(&self.config.common.network.into(), height) + NetworkUpgrade::current(&self.data.network(), height) .branch_id() .unwrap_or(ConsensusBranchId::RPC_MISSING_ID) .into(), ) .inner(), ConsensusBranchIdHex::new( - NetworkUpgrade::current(&self.config.common.network.into(), next_block_height) + NetworkUpgrade::current(&self.data.network(), next_block_height) .branch_id() .unwrap_or(ConsensusBranchId::RPC_MISSING_ID) .into(), @@ -1156,22 +1177,15 @@ impl ZcashIndexer for StateServiceSubscriber { ); // TODO: Remove unwrap() - let difficulty = chain_tip_difficulty( - self.config.common.network.to_zebra_network(), - self.read_state_service.clone(), - false, - ) - .await - .unwrap(); + let difficulty = + chain_tip_difficulty(self.data.network(), self.read_state_service.clone(), false) + .await + .unwrap(); let verification_progress = f64::from(height.0) / f64::from(zebra_estimated_height.0); Ok(GetBlockchainInfoResponse::new( - self.config - .common - .network - .to_zebra_network() - .bip70_network_name(), + self.data.network().bip70_network_name(), height, hash, estimated_height, @@ -1621,19 +1635,18 @@ impl ZcashIndexer for StateServiceSubscriber { return Ok(ValidateAddressResponse::invalid()); }; - let address = match address.convert_if_network::
( - match self.config.common.network.to_zebra_network().kind() { + let address = + match address.convert_if_network::
(match self.data.network().kind() { NetworkKind::Mainnet => NetworkType::Main, NetworkKind::Testnet => NetworkType::Test, NetworkKind::Regtest => NetworkType::Regtest, - }, - ) { - Ok(address) => address, - Err(err) => { - tracing::debug!(?err, "conversion error"); - return Ok(ValidateAddressResponse::invalid()); - } - }; + }) { + Ok(address) => address, + Err(err) => { + tracing::debug!(?err, "conversion error"); + return Ok(ValidateAddressResponse::invalid()); + } + }; Ok(match address { Address::Transparent(taddr) => ValidateAddressResponse::new( @@ -1658,21 +1671,20 @@ impl ZcashIndexer for StateServiceSubscriber { )); }; - let converted_address = match parsed_address.convert_if_network::
( - match self.config.common.network.to_zebra_network().kind() { + let converted_address = + match parsed_address.convert_if_network::
(match self.data.network().kind() { NetworkKind::Mainnet => NetworkType::Main, NetworkKind::Testnet => NetworkType::Test, NetworkKind::Regtest => NetworkType::Regtest, - }, - ) { - Ok(address) => address, - Err(err) => { - tracing::debug!(?err, "conversion error"); - return Ok(ZValidateAddressResponse::Known( - KnownZValidateAddress::Invalid(InvalidZValidateAddress::new()), - )); - } - }; + }) { + Ok(address) => address, + Err(err) => { + tracing::debug!(?err, "conversion error"); + return Ok(ZValidateAddressResponse::Known( + KnownZValidateAddress::Invalid(InvalidZValidateAddress::new()), + )); + } + }; // Note: It could be the case that Zaino needs to support Sprout. For now, it's been disabled. match converted_address { @@ -1682,13 +1694,11 @@ impl ZcashIndexer for StateServiceSubscriber { } TransparentAddress::ScriptHash(_) => Ok(ZValidateAddressResponse::p2sh(address)), }, - Address::Unified(u) => Ok(ZValidateAddressResponse::unified( - u.encode(&self.network().to_zebra_network()), - )), + Address::Unified(u) => Ok(ZValidateAddressResponse::unified(u.encode(&self.network()))), Address::Sapling(s) => { let (diversifier, pk_d) = sapling_key_bytes(&s); Ok(ZValidateAddressResponse::sapling( - s.encode(&self.network().to_zebra_network()), + s.encode(&self.network()), Some(hex::encode(diversifier)), Some(hex::encode(pk_d)), )) @@ -1836,7 +1846,7 @@ impl ZcashIndexer for StateServiceSubscriber { height, confirmations, #[allow(deprecated)] - &self.config.common.network.to_zebra_network(), + &self.data.network(), None, block_hash, in_best_chain, @@ -2515,11 +2525,7 @@ impl LightWalletIndexer for StateServiceSubscriber { .await?; Ok(super::tree_state_from_treestate_response( - self.config - .common - .network - .to_zebra_network() - .bip70_network_name(), + self.data.network().bip70_network_name(), treestate_response, )) } diff --git a/packages/zaino-state/src/chain_index.rs b/packages/zaino-state/src/chain_index.rs index 4609c0011..8d53d0d20 100644 --- a/packages/zaino-state/src/chain_index.rs +++ b/packages/zaino-state/src/chain_index.rs @@ -809,7 +809,7 @@ impl NodeBackedChainIndex { finalized_db, sync_loop_handle: None, status: NamedAtomicStatus::new("ChainIndex", StatusType::Spawning), - network: config.network.to_zebra_network(), + network: config.network.clone(), source, sync_timings, cancel_token: CancellationToken::new(), diff --git a/packages/zaino-state/src/chain_index/finalised_state.rs b/packages/zaino-state/src/chain_index/finalised_state.rs index 11c824ec1..e57875d8a 100644 --- a/packages/zaino-state/src/chain_index/finalised_state.rs +++ b/packages/zaino-state/src/chain_index/finalised_state.rs @@ -251,7 +251,7 @@ use tokio::time::{interval, MissedTickBehavior}; /// owns the loop. pub(crate) async fn build_indexed_block_from_source( source: &S, - network: zaino_common::Network, + network: zebra_chain::parameters::Network, sapling_activation_height: zebra_chain::block::Height, nu5_activation_height: Option, nu6_3_activation_height: Option, @@ -312,7 +312,7 @@ pub(crate) async fn build_indexed_block_from_source( orchard_size, ironwood, parent_chainwork, - network.to_zebra_network(), + network, ); let block_with_metadata = BlockWithMetadata::new(block.as_ref(), metadata); @@ -464,7 +464,7 @@ impl FinalisedState { return Ok(Self { db: Arc::new(Router::new(Arc::new(FinalisedSource::ephemeral( source, - cfg.network.into(), + cfg.network.clone(), None, )))), cfg, @@ -658,7 +658,7 @@ impl FinalisedState { /// - `Some(version)` if a compatible database directory is found, /// - `None` if no database is detected (fresh DB creation case). async fn try_find_current_db_version(cfg: &ChainIndexConfig) -> Option { - let legacy_dir = match cfg.network.to_zebra_network().kind() { + let legacy_dir = match cfg.network.kind() { NetworkKind::Mainnet => "live", NetworkKind::Testnet => "test", NetworkKind::Regtest => "local", @@ -668,7 +668,7 @@ impl FinalisedState { return Some(0); } - let net_dir = match cfg.network.to_zebra_network().kind() { + let net_dir = match cfg.network.kind() { NetworkKind::Mainnet => "mainnet", NetworkKind::Testnet => "testnet", NetworkKind::Regtest => "regtest", @@ -786,7 +786,7 @@ impl FinalisedState { let ephemeral_reference = router .init_or_take_ephemeral( source.clone(), - cfg.network.to_zebra_network(), + cfg.network.clone(), EphemeralMode::ReadOnly, db_height_opt, ) @@ -1098,7 +1098,7 @@ impl FinalisedState { // defaults — mirroring `build_indexed_block_from_source`. let nu6_3_activation_height = super::ShieldedPool::Ironwood .activation_upgrade() - .activation_height(&cfg.network.to_zebra_network()); + .activation_height(&cfg.network); let mut parent_chainwork: Option = None; @@ -1145,7 +1145,7 @@ impl FinalisedState { orchard_size, ironwood, parent_chainwork, - cfg.network.to_zebra_network(), + cfg.network.clone(), ); let block_with_metadata = BlockWithMetadata::new(block.as_ref(), metadata); let chain_block = IndexedBlock::try_from(block_with_metadata).map_err(|_| { diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1.rs index 73ed8b717..1d0ed091e 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1.rs @@ -453,7 +453,7 @@ impl DbV1 { // Prepare database details and path. let db_size_bytes = config.storage.database.size.to_byte_count(); - let db_path_dir = match config.network.to_zebra_network().kind() { + let db_path_dir = match config.network.kind() { NetworkKind::Mainnet => "mainnet", NetworkKind::Testnet => "testnet", NetworkKind::Regtest => "regtest", @@ -1108,7 +1108,7 @@ mod tests { async fn initial_spent_scan_reports_corrupt_value() { use lmdb::{Transaction as _, WriteFlags}; use zaino_common::network::ActivationHeights; - use zaino_common::{DatabaseConfig, Network, StorageConfig}; + use zaino_common::{DatabaseConfig, StorageConfig}; let temp_dir = tempfile::tempdir().expect("tempdir"); let config = ChainIndexConfig { @@ -1121,7 +1121,7 @@ mod tests { }, ephemeral: false, db_version: 1, - network: Network::Regtest(ActivationHeights::default()), + network: ActivationHeights::default().to_regtest_network(), }; let db = DbV1::spawn(&config).await.expect("spawn empty v1 db"); diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/tx_out_set_accumulator.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/tx_out_set_accumulator.rs index e254e70f0..df3aad9ca 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/tx_out_set_accumulator.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/tx_out_set_accumulator.rs @@ -1710,7 +1710,7 @@ mod tests { use std::sync::Arc; use tempfile::TempDir; use zaino_common::network::ActivationHeights; - use zaino_common::{DatabaseConfig, Network, StorageConfig, SyncWriteBatchSize}; + use zaino_common::{DatabaseConfig, StorageConfig, SyncWriteBatchSize}; fn p2pkh_out(value: u64) -> TxOutCompact { TxOutCompact::new(value, [0x11; 20], 0).expect("P2PKH script_type should be valid") @@ -2111,7 +2111,7 @@ mod tests { }, ephemeral: false, db_version: 1, - network: Network::Regtest(ActivationHeights::default()), + network: ActivationHeights::default().to_regtest_network(), }; let zaino_db = FinalisedState::spawn(config, source.clone()).await.unwrap(); @@ -2186,7 +2186,7 @@ mod tests { }, ephemeral: false, db_version: 1, - network: Network::Regtest(ActivationHeights::default()), + network: ActivationHeights::default().to_regtest_network(), }; let zaino_db = FinalisedState::spawn(config, source.clone()).await.unwrap(); diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/write_core.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/write_core.rs index 212ac75d3..347c0ba8a 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/write_core.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/write_core.rs @@ -211,8 +211,7 @@ impl DbWrite for DbV1 { ) -> Result<(), FinalisedStateError> { use crate::chain_index::finalised_state::build_indexed_block_from_source; - let network = self.config.network; - let zebra_network = network.to_zebra_network(); + let zebra_network = self.config.network.clone(); let sapling_activation_height = ShieldedPool::Sapling .activation_upgrade() .activation_height(&zebra_network) @@ -264,7 +263,7 @@ impl DbWrite for DbV1 { info!( start_height, target = height.0, - ?network, + ?zebra_network, "write_blocks_to_height: syncing finalised blocks" ); @@ -309,7 +308,7 @@ impl DbWrite for DbV1 { let build_start = std::time::Instant::now(); let block = build_indexed_block_from_source( source, - network, + zebra_network.clone(), sapling_activation_height, nu5_activation_height, nu6_3_activation_height, @@ -336,7 +335,7 @@ impl DbWrite for DbV1 { info!( current = next - 1, target = height.0, - ?network, + ?zebra_network, "write_blocks_to_height: syncing" ); last_progress_log = std::time::Instant::now(); diff --git a/packages/zaino-state/src/chain_index/finalised_state/migrations.rs b/packages/zaino-state/src/chain_index/finalised_state/migrations.rs index b88009194..efae33e16 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/migrations.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/migrations.rs @@ -343,7 +343,7 @@ impl MigrationManager { .router .init_or_take_ephemeral( self.source.clone(), - self.cfg.network.to_zebra_network(), + self.cfg.network.clone(), EphemeralMode::Full, db_height, ) @@ -1093,7 +1093,7 @@ impl Migration for Migration1_2_1To1_3_0 { // Guard: Ironwood data is only expected from NU6.3 activation. A rebuild-from-existing-data // migration cannot reconstruct ironwood roots/sizes for post-NU6.3 blocks. - let zebra_network = cfg.network.to_zebra_network(); + let zebra_network = cfg.network.clone(); let nu6_3_activation_height = crate::chain_index::ShieldedPool::Ironwood .activation_upgrade() .activation_height(&zebra_network); diff --git a/packages/zaino-state/src/chain_index/source.rs b/packages/zaino-state/src/chain_index/source.rs index 9b5ea63f0..3acf8f34b 100644 --- a/packages/zaino-state/src/chain_index/source.rs +++ b/packages/zaino-state/src/chain_index/source.rs @@ -10,7 +10,6 @@ use crate::SendFut; use futures::TryFutureExt as _; use incrementalmerkletree::frontier::CommitmentTree; use tower::{Service, ServiceExt as _}; -use zaino_common::Network; use zaino_fetch::jsonrpsee::{ connector::{JsonRpSeeConnector, RpcRequestError}, response::{ diff --git a/packages/zaino-state/src/chain_index/source/mockchain_source.rs b/packages/zaino-state/src/chain_index/source/mockchain_source.rs index 00cab444d..8379bf0f3 100644 --- a/packages/zaino-state/src/chain_index/source/mockchain_source.rs +++ b/packages/zaino-state/src/chain_index/source/mockchain_source.rs @@ -125,7 +125,7 @@ fn normalize_requested_addresses_for_network( /// transparent address prefixes, so output-derived transparent addresses use /// `NetworkKind::Testnet`. fn mockchain_network() -> zebra_chain::parameters::Network { - zaino_common::Network::Regtest(ActivationHeights::default()).to_zebra_network() + ActivationHeights::default().to_regtest_network() } /// A test-only mock implementation of BlockchainReader using ordered lists by height. diff --git a/packages/zaino-state/src/chain_index/source/validator_connector.rs b/packages/zaino-state/src/chain_index/source/validator_connector.rs index cf7908a94..51816b606 100644 --- a/packages/zaino-state/src/chain_index/source/validator_connector.rs +++ b/packages/zaino-state/src/chain_index/source/validator_connector.rs @@ -34,8 +34,8 @@ pub struct State { pub read_state_service: ReadStateService, /// Temporarily used to fetch mempool data. pub mempool_fetcher: JsonRpSeeConnector, - /// Current network type being run. - pub network: Network, + /// The runtime network (activation schedule adopted from the validator). + pub network: zebra_chain::parameters::Network, } /// A connection to a validator. @@ -436,7 +436,7 @@ impl BlockchainSource for ValidatorConnector { let sapling = match ShieldedPool::Sapling .activation_upgrade() - .activation_height(&state.network.to_zebra_network()) + .activation_height(&state.network) { Some(activation_height) if height >= activation_height => Some( state @@ -467,7 +467,7 @@ impl BlockchainSource for ValidatorConnector { let orchard = match ShieldedPool::Orchard .activation_upgrade() - .activation_height(&state.network.to_zebra_network()) + .activation_height(&state.network) { Some(activation_height) if height >= activation_height => Some( state @@ -498,7 +498,7 @@ impl BlockchainSource for ValidatorConnector { let ironwood = match ShieldedPool::Ironwood .activation_upgrade() - .activation_height(&state.network.to_zebra_network()) + .activation_height(&state.network) { Some(activation_height) if height >= activation_height => Some( state @@ -860,7 +860,7 @@ impl BlockchainSource for ValidatorConnector { transaction.clone(), height, None, - &state.network.to_zebra_network(), + &state.network, None, None, Some(matches!( diff --git a/packages/zaino-state/src/chain_index/tests.rs b/packages/zaino-state/src/chain_index/tests.rs index d2cd1d58e..47e954214 100644 --- a/packages/zaino-state/src/chain_index/tests.rs +++ b/packages/zaino-state/src/chain_index/tests.rs @@ -26,7 +26,7 @@ use std::path::{Path, PathBuf}; use tempfile::TempDir; use tokio::sync::OnceCell; use tokio::time::Duration; -use zaino_common::{network::ActivationHeights, DatabaseConfig, Network, StorageConfig}; +use zaino_common::{network::ActivationHeights, DatabaseConfig, StorageConfig}; use crate::{ chain_index::{ @@ -134,7 +134,7 @@ async fn load_with_settings( }, ephemeral: false, db_version: 1, - network: Network::Regtest(ActivationHeights::default()), + network: ActivationHeights::default().to_regtest_network(), }; let indexer = NodeBackedChainIndex::new_with_sync_timings(source.clone(), config, sync_timings) @@ -222,7 +222,7 @@ async fn v1_finalised_seed_dir(mode: MockchainMode) -> &'static Path { }, ephemeral: false, db_version: 1, - network: Network::Regtest(ActivationHeights::default()), + network: ActivationHeights::default().to_regtest_network(), }; let zaino_db = FinalisedState::spawn(config, source).await.unwrap(); diff --git a/packages/zaino-state/src/chain_index/tests/finalised_state/ephemeral.rs b/packages/zaino-state/src/chain_index/tests/finalised_state/ephemeral.rs index d2b455b22..34475c4ef 100644 --- a/packages/zaino-state/src/chain_index/tests/finalised_state/ephemeral.rs +++ b/packages/zaino-state/src/chain_index/tests/finalised_state/ephemeral.rs @@ -15,7 +15,7 @@ use std::sync::Arc; use tempfile::TempDir; use zaino_common::network::ActivationHeights; -use zaino_common::{DatabaseConfig, Network, StorageConfig}; +use zaino_common::{DatabaseConfig, StorageConfig}; use zaino_proto::proto::utils::{compact_block_with_pool_types, PoolTypeFilter}; use crate::chain_index::finalised_state::FinalisedState; @@ -45,7 +45,7 @@ pub(crate) async fn spawn_ephemeral_finalised_state( }, ephemeral: true, db_version: 1, - network: Network::Regtest(ActivationHeights::default()), + network: ActivationHeights::default().to_regtest_network(), }; let finalised_state = FinalisedState::spawn(config, source).await?; diff --git a/packages/zaino-state/src/chain_index/tests/finalised_state/migrations/v1_0_to_v1_1.rs b/packages/zaino-state/src/chain_index/tests/finalised_state/migrations/v1_0_to_v1_1.rs index e2db7587b..3f70ca3aa 100644 --- a/packages/zaino-state/src/chain_index/tests/finalised_state/migrations/v1_0_to_v1_1.rs +++ b/packages/zaino-state/src/chain_index/tests/finalised_state/migrations/v1_0_to_v1_1.rs @@ -3,7 +3,7 @@ use std::path::PathBuf; use tempfile::TempDir; use zaino_common::network::ActivationHeights; -use zaino_common::{DatabaseConfig, Network, StorageConfig}; +use zaino_common::{DatabaseConfig, StorageConfig}; use crate::chain_index::finalised_state::capability::{ DbCore as _, DbRead as _, DbVersion, MigrationStatus, @@ -34,7 +34,7 @@ async fn v1_0_to_v1_1_metadata_migration() { }, ephemeral: false, db_version: 1, - network: Network::Regtest(ActivationHeights::default()), + network: ActivationHeights::default().to_regtest_network(), }; let source = build_active_mockchain_source(150, blocks.clone()); @@ -100,7 +100,7 @@ async fn v1_0_to_v1_1_mixed_blockheaderdata_formats() { }, ephemeral: false, db_version: 1, - network: Network::Regtest(ActivationHeights::default()), + network: ActivationHeights::default().to_regtest_network(), }; let source = build_active_mockchain_source(initial_active_height.0, blocks.clone()); diff --git a/packages/zaino-state/src/chain_index/tests/finalised_state/migrations/v1_1_to_v1_2.rs b/packages/zaino-state/src/chain_index/tests/finalised_state/migrations/v1_1_to_v1_2.rs index 882763cdc..bfa0a32ef 100644 --- a/packages/zaino-state/src/chain_index/tests/finalised_state/migrations/v1_1_to_v1_2.rs +++ b/packages/zaino-state/src/chain_index/tests/finalised_state/migrations/v1_1_to_v1_2.rs @@ -4,7 +4,7 @@ use lmdb::{Cursor as _, Transaction as _, WriteFlags}; use std::path::PathBuf; use tempfile::TempDir; use zaino_common::network::ActivationHeights; -use zaino_common::{DatabaseConfig, Network, StorageConfig}; +use zaino_common::{DatabaseConfig, StorageConfig}; use crate::chain_index::finalised_state::capability::{ BlockCoreExt as _, CapabilityRequest, DbRead as _, DbVersion, MigrationStatus, @@ -426,7 +426,7 @@ async fn v1_1_to_v1_2_spent_index_backfill_from_old_version() { }, ephemeral: false, db_version: 1, - network: Network::Regtest(ActivationHeights::default()), + network: ActivationHeights::default().to_regtest_network(), }; let source = build_active_mockchain_source(initial_active_height.0, blocks.clone()); @@ -498,7 +498,7 @@ async fn v1_1_to_v1_2_spent_index_migration_resumes_after_crash() { }, ephemeral: false, db_version: 1, - network: Network::Regtest(ActivationHeights::default()), + network: ActivationHeights::default().to_regtest_network(), }; let source = build_active_mockchain_source(initial_active_height.0, blocks.clone()); @@ -597,7 +597,7 @@ async fn v1_2_0_cache_missing_txid_location_index_is_rebuilt() { }, ephemeral: false, db_version: 1, - network: Network::Regtest(ActivationHeights::default()), + network: ActivationHeights::default().to_regtest_network(), }; let source = build_active_mockchain_source(initial_active_height.0, blocks.clone()); diff --git a/packages/zaino-state/src/chain_index/tests/finalised_state/v1.rs b/packages/zaino-state/src/chain_index/tests/finalised_state/v1.rs index 1ef3c02ce..d3febab5b 100644 --- a/packages/zaino-state/src/chain_index/tests/finalised_state/v1.rs +++ b/packages/zaino-state/src/chain_index/tests/finalised_state/v1.rs @@ -4,7 +4,7 @@ use hex::ToHex; use std::path::PathBuf; use tempfile::TempDir; use zaino_common::network::ActivationHeights; -use zaino_common::{DatabaseConfig, Network, StorageConfig}; +use zaino_common::{DatabaseConfig, StorageConfig}; use zaino_proto::proto::utils::{compact_block_with_pool_types, PoolTypeFilter}; use crate::chain_index::finalised_state::finalised_source::FinalisedSource; @@ -44,7 +44,7 @@ pub(crate) async fn spawn_v1_zaino_db( }, ephemeral: false, db_version: 1, - network: Network::Regtest(ActivationHeights::default()), + network: ActivationHeights::default().to_regtest_network(), }; let zaino_db = FinalisedState::spawn(config, source).await.unwrap(); @@ -193,7 +193,7 @@ async fn save_db_to_file_and_reload() { }, ephemeral: false, db_version: 1, - network: Network::Regtest(ActivationHeights::default()), + network: ActivationHeights::default().to_regtest_network(), }; let source = build_mockchain_source(blocks.clone()); @@ -272,7 +272,7 @@ async fn load_db_backend_from_file() { }, ephemeral: false, db_version: 1, - network: Network::Regtest(ActivationHeights::default()), + network: ActivationHeights::default().to_regtest_network(), }; let finalized_state_backend: FinalisedSource = FinalisedSource::spawn_v1(&config).await.unwrap(); @@ -338,7 +338,7 @@ async fn try_write_invalid_block() { orchard_tree_size as u32, None, None, // no parent chainwork for this test - zaino_common::Network::Regtest(ActivationHeights::default()).to_zebra_network(), + ActivationHeights::default().to_regtest_network(), ); let mut chain_block = diff --git a/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs b/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs index ddb17407a..54f273917 100644 --- a/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs +++ b/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs @@ -7,7 +7,7 @@ use proptest::{ }; use rand::seq::IndexedRandom; use tokio_stream::StreamExt as _; -use zaino_common::{network::ActivationHeights, DatabaseConfig, Network, StorageConfig}; +use zaino_common::{network::ActivationHeights, DatabaseConfig, StorageConfig}; use zaino_fetch::jsonrpsee::response::address_deltas::{ GetAddressDeltasParams, GetAddressDeltasResponse, }; @@ -58,7 +58,7 @@ fn passthrough_test( ), ) { passthrough_test_on( - Network::Regtest(ActivationHeights::default()), + ActivationHeights::default().to_regtest_network(), // Slow the source enough to hold the indexer in passthrough while the // assertions run, without slowing passthrough more than necessary. Some(Duration::from_millis(100)), @@ -76,7 +76,7 @@ fn passthrough_test( /// header's merkle root is already arbitrary — the passthrough path tolerates that by /// construction. fn passthrough_test_on( - network: Network, + network: zebra_chain::parameters::Network, source_delay: Option, mutate_segment: impl Fn(&mut Vec>), test: impl AsyncFn( @@ -92,7 +92,7 @@ fn passthrough_test_on( // from this line to `runtime.block_on(async {` are all // copy-pasted. Could a macro get rid of some of this boilerplate? - proptest::proptest!(proptest::test_runner::Config::with_cases(1), |(segments in make_branching_chain(branch_count, segment_length, network))| { + proptest::proptest!(proptest::test_runner::Config::with_cases(1), |(segments in make_branching_chain(branch_count, segment_length, network.clone()))| { let runtime = tokio::runtime::Builder::new_multi_thread().worker_threads(2).enable_time().build().unwrap(); runtime.block_on(async { let (mut genesis_segment, mut branching_segments) = segments; @@ -120,7 +120,7 @@ fn passthrough_test_on( }, ephemeral: true, db_version: 1, - network, + network: network.clone(), }; @@ -602,7 +602,7 @@ fn metadata_consistency_for_era( }; passthrough_test_on( - Network::Regtest(heights), + heights.to_regtest_network(), // No artificial source delay: this test waits for the indexer to finish // syncing, because compact blocks are not served while the finalised state // is still syncing (get_compact_block's StillSyncingFinalizedState arm). @@ -791,14 +791,14 @@ fn metadata_consistency_for_era( #[test] fn make_chain() { init_tracing(); - let network = Network::Regtest(ActivationHeights::default()); + let network = ActivationHeights::default().to_regtest_network(); let segment_length = 12; let branch_count = 2; // default is 256. As each case takes multiple seconds, this seems too many. // TODO: this should be higher than 1. Currently set to 1 for ease of iteration - proptest::proptest!(proptest::test_runner::Config::with_cases(1), |(segments in make_branching_chain(branch_count, segment_length, network))| { + proptest::proptest!(proptest::test_runner::Config::with_cases(1), |(segments in make_branching_chain(branch_count, segment_length, network.clone()))| { let runtime = tokio::runtime::Builder::new_multi_thread().worker_threads(2).enable_time().build().unwrap(); runtime.block_on(async { let (genesis_segment, branching_segments) = segments; @@ -822,7 +822,7 @@ fn make_chain() { }, ephemeral: true, db_version: 1, - network, + network: network.clone(), }; @@ -1257,9 +1257,9 @@ fn make_branching_chain( // The length of the initial segment, and of the branches // TODO: it would be useful to allow branches of different lengths. chain_size: usize, - network_override: Network, + network_override: zebra_chain::parameters::Network, ) -> BoxedStrategy<(ChainSegment, Vec)> { - let network_override = Some(network_override.to_zebra_network()); + let network_override = Some(network_override); add_segment( SummaryDebug(Vec::new()), network_override.clone(), diff --git a/packages/zaino-state/src/config.rs b/packages/zaino-state/src/config.rs index 0083dbcd4..282af0a93 100644 --- a/packages/zaino-state/src/config.rs +++ b/packages/zaino-state/src/config.rs @@ -141,10 +141,8 @@ impl StateServiceConfig { network: Network, donation_address: Option, ) -> Self { - tracing::trace!( - activations = ?network.to_zebra_network().full_activation_list(), - "state service expecting NU activations" - ); + // The config carries only the network kind; the activation schedule + // is adopted from the validator at spawn and logged there (#1076). StateServiceConfig { common: CommonBackendConfig { validator_rpc_address, @@ -212,8 +210,10 @@ pub struct ChainIndexConfig { pub storage: StorageConfig, /// Database version selected to be run. pub db_version: u32, - /// Network type. - pub network: Network, + /// The runtime network, carrying the activation schedule adopted from + /// the validator (zaino#1076) — or, in fixtures that are their own + /// chain, the fixture's schedule. + pub network: zebra_chain::parameters::Network, /// Ephemeral finalised state: /// /// If true, FinalisedState does not write data to disk, @@ -225,9 +225,14 @@ pub struct ChainIndexConfig { } impl ChainIndexConfig { - /// Returns a new instance of [`BlockCacheConfig`]. + /// Returns a new instance of [`ChainIndexConfig`]. #[allow(dead_code)] - pub fn new(storage: StorageConfig, db_version: u32, network: Network, ephemeral: bool) -> Self { + pub fn new( + storage: StorageConfig, + db_version: u32, + network: zebra_chain::parameters::Network, + ephemeral: bool, + ) -> Self { ChainIndexConfig { storage, db_version, @@ -235,34 +240,25 @@ impl ChainIndexConfig { ephemeral, } } -} -impl From for ChainIndexConfig { - fn from(value: CommonBackendConfig) -> Self { + /// Builds the chain-index config from the backend config plus the + /// runtime network. The backend config carries only a network kind, so + /// the adopted runtime network arrives as its own argument — there is + /// no conversion from a service config alone. + pub fn from_backend_config( + common: &CommonBackendConfig, + network: zebra_chain::parameters::Network, + ) -> Self { Self { - storage: value.storage, + storage: common.storage.clone(), // TODO: update zaino configs to include db version. db_version: 1, - network: value.network, - ephemeral: value.ephemeral_finalised_state, + network, + ephemeral: common.ephemeral_finalised_state, } } } -#[allow(deprecated)] -impl From for ChainIndexConfig { - fn from(value: StateServiceConfig) -> Self { - value.common.into() - } -} - -#[allow(deprecated)] -impl From for ChainIndexConfig { - fn from(value: FetchServiceConfig) -> Self { - value.common.into() - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/packages/zainod/src/config.rs b/packages/zainod/src/config.rs index aeb97dbbe..1a1ff77fd 100644 --- a/packages/zainod/src/config.rs +++ b/packages/zainod/src/config.rs @@ -222,11 +222,6 @@ impl ZainodConfig { Ok(()) } - - /// Returns the network type currently being used by the server. - pub fn get_network(&self) -> Result { - Ok(self.network.to_zebra_network()) - } } impl Default for ZainodConfig { diff --git a/zainod-heights-from-validator-spec.md b/zainod-heights-from-validator-spec.md new file mode 100644 index 000000000..66725a501 --- /dev/null +++ b/zainod-heights-from-validator-spec.md @@ -0,0 +1,170 @@ +# Spec: zainod learns regtest activation heights from the Validator + +Solves zingolabs/zaino#1076. Requested by infras (`zingolabs/infrastructure#278`, +`bump_to_NU6.3`) in coordination with zaino's `ironwood_activation` e2e suite +(zingolabs/zaino#1368). Companion spec on the infras side: +`infras/dev/zaino-ironwood-activation-infra-spec.md` (devtool wallets on +arbitrary regtest heights — "Provider heights"). + +Governing invariant, decided 2026-07-06 (recorded as infras ADR 0003): + +> **The Validator's configured activation heights are authoritative. Every +> other component mirrors them by the strongest mechanism its protocol +> allows.** No component may treat compiled-in or config-file regtest +> heights as chain truth. + +Concretely, per component: + +- **Indexer (this spec)**: queries the Validator's `getblockchaininfo` at + startup and adopts the reported schedule. +- **Wallet client (infras side, same release as the guard lift)**: the + light-client protocol doesn't expose the schedule, so the wallet can't + ask the Validator itself — the harness queries `getblockchaininfo` on + the wallet's behalf and writes the *derived* heights into the wallet's + `activation-heights.toml`. The derivation is enforced at compile time: + the wallet config's regtest variant demands an opaque + `ValidatorHeights`, whose only public constructor is + `WalletNetwork::from_validator(&validator)`. There is no + caller-supplied-heights escape hatch; writing a height vector into a + wallet config is unrepresentable. + +Consequence for zaino's e2e suite: once both sides land, your +`ironwood_activation` fixture heights get typed in exactly one place — the +zebrad launch config. The wallet derives them via +`WalletNetwork::from_validator`; zainod adopts them via this spec; no +test-side constant needs to mirror anything. Expect the zcash_local_net +pin bump to be loud: `ZainodConfig.network` narrows to a payload-free +kind (Mainnet/Testnet/Regtest), `ZcashDevtoolConfig::faucet()/recipient()` +take a `WalletNetwork` parameter, and any test that assigned heights into +a wallet config stops compiling — the fix is always to launch the +validator first and derive. + +## Problem + +zainod's regtest activation heights are a compiled-in constant, silently +assumed to match whatever zebrad was launched with: + +- The zainod TOML carries only the string `network = "Regtest"`. + Deserialization inflates it to + `Network::Regtest(ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS)` + (`packages/zaino-common/src/config/network.rs:66`); the constant + (`network.rs:13-26`) documents itself as "must equal zcash_local_net's + `supported_regtest_activation_heights`" — a cross-repo mirror maintained + by hand across three repos (zebra config, zaino constant, infras canonical + heights). +- That `Network` flows into everything downstream as truth: + `StateService::spawn` passes `config.common.network.to_zebra_network()` + into `init_read_state_with_syncer` (`zaino-state/src/backends/state.rs:213-218`) + and into `NodeBackedChainIndex::new` (`state.rs:267-276`); + `FetchService::spawn` does the equivalent (`backends/fetch.rs:122+`). +- When the running zebrad's heights differ, block commitment verification + computes against the wrong upgrade schedule and the chain-index sync loop + dies with `InvalidData("Block commitment could not be computed")` + (`zaino-state/src/chain_index/types/helpers.rs:183`, + `block.commitment(network)`). + +The mismatch is not hypothetical: infras is lifting its client-side guard so +devtool wallets run on **arbitrary** regtest heights, and zaino's own +`ironwood_activation` fixture (`ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS`) +puts NU6.3 at height 6 with everything else at 1–2. No compiled constant can +ever match caller-chosen heights. The only correct source is the running +zebrad — and it already publishes its schedule: `getblockchaininfo` returns +an `upgrades` map with per-upgrade `activationheight`, which zaino already +parses (`GetBlockchainInfoResponse.upgrades`, +`zaino-fetch/src/jsonrpsee/response.rs:356-359`, round-trip tested at +`response.rs:530+`). Both backends already call the validator at startup +(`get_info` at `state.rs:198`, `get_blockchain_info` in the tip-wait loop at +`state.rs:226`). The handshake exists; the heights are simply never adopted +from it. + +## Requested change + +1. **Learn heights at first contact.** When the configured network kind is + Regtest, each backend fetches `getblockchaininfo` from the validator + **before** constructing anything that consumes a + `zebra_chain::parameters::Network` — the ReadStateService syncer, the + chain index, the mempool source. Build the runtime regtest `Network` from + the reported `upgrades` map. The RPC client is already constructed first + (`state.rs:190-196`), so the ordering is achievable without restructuring. +2. **Learned heights are the only source.** `ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS` + ceases to exist as a truth source. Recommended shape: the *config-level* + network type degrades to a kind (Mainnet / Testnet / Regtest, no payload), + and only the *runtime* type constructed after the handshake carries + heights — making pre-handshake height reads unrepresentable. A minimal + variant (populate the existing payload from the handshake before first + use) is acceptable if the type split is too invasive, but then nothing may + read the payload before adoption. Behavior is mandated; shape is + implementer's choice. +3. **An upgrade absent from the validator's map is never-activated.** This + matches zebra's own semantics and the devtool's absent-key encoding. + Do not backfill absent upgrades from any default. +4. **Failure is loud, never defaulted.** If the network kind is Regtest and + the upgrades map cannot be fetched or parsed, startup fails with an error + naming the validator endpoint. Under no circumstances fall back to a + compiled or configured height set — a silently wrong schedule is the + exact failure mode this spec removes. + +## Acceptance + +The fixture that must work end-to-end (as zebrad regtest config heights): + +``` +overwinter = 1, sapling = 1, blossom = 1, heartwood = 1, canopy = 1, +nu5 = 2, nu6 = 2, nu6_1 = 2, nu6_2 = 2, nu6_3 = 6 +``` + +1. **Boundary sync**: launch zebrad regtest with the fixture; launch zainod + against it with only `network = "Regtest"` in its TOML. Mine past height + 6. zainod's chain index syncs across the NU6.3 boundary with no + commitment error, and serves compact blocks for both eras (Orchard + coinbase at heights 2–5, Ironwood from 6). +2. **No-recompile proof**: the same zainod binary and config, pointed at a + zebrad running a *different* schedule (the canonical all-at-2 set), also + syncs — demonstrating heights come from the validator, not the build. +3. **Contract test on the upgrades map**: pin the `getblockchaininfo → + heights` mapping with a test against real zebrad output (live regtest or + a recorded golden response) — not hand-written assertions about what the + RPC is believed to return. In particular, establish from the real output + (not from reasoning) which upgrades zebrad includes in the map (e.g. + whether anything pre-Overwinter appears) and encode exactly that. +4. **No residual truth source**: no non-test code path consumes + `ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS` or any other compiled regtest + schedule as chain truth. Test-only fixtures that *are* their own chain + (mockchain sources, proptest block generators) legitimately keep explicit + heights — there, the test is the validator. + +## Known unknowns to check while implementing + +- **StateService init ordering**: `init_read_state_with_syncer` receives the + network at construction and the ReadStateService validates blocks with it. + Confirm nothing else needs a `Network` before the validator RPC is + reachable; if something does, that is a design conflict to surface, not + paper over. +- **zcashd as validator**: the legacy e2e path (`devtool_zcashd.rs`) drives + zcashd, whose `getblockchaininfo` also reports an `upgrades` map. Prefer + making the adoption path validator-generic so zcashd rides along; if its + map shape differs materially, scope this spec to zebrad and record the + divergence in the zcashd test docs. +- **zaino-testutils**: `live-tests/zaino-testutils/src/lib.rs:497` uses the + constant to *launch zebrad* — that is legitimate harness-side + configuration of the truth source, not adoption of it. Keep the height + set, but move/rename it so it can no longer be mistaken for (or imported + as) zainod's own schedule. +- **Reorg/restart**: heights are learned once at startup. A validator + restarted mid-session with different heights invalidates the indexer's + world; document that zainod must be restarted with its validator (regtest + harnesses already do this), rather than attempting live re-adoption. + +## Out of scope + +- Mainnet / Testnet: compiled zebra network parameters are correct there; no + handshake is needed and none should be added. +- The infras-side client guard lift (Provider heights) — lands on + `infrastructure#278` independently. + +## Delivery + +A rev on a zaino branch that `live-tests` can exercise. Downstream effects +when it lands: zaino#1368's `ironwood_activation` tests gain a zainod that +can actually serve their fixture, and infras' nu6_3-at-6 integration test +(currently `#[ignore]`d naming #1076) flips live. From d445446640bb2c02f3a2d248aa0fcfe50d09df45 Mon Sep 17 00:00:00 2001 From: zancas Date: Mon, 6 Jul 2026 22:44:55 -0700 Subject: [PATCH 25/58] test(e2e): wallet-tier Ironwood activation suite on the transition fixture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ironwood_activation suite exercises the wallet-observable predicates across the Orchard-to-Ironwood boundary on ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS, the hermetic replay of the public testnet's NU6.3 activation at height 4,134,000. The wallets derive their schedule from the running validator and zainod adopts the same schedule over getblockchaininfo, so the fixture heights are typed exactly once, in the zebrad launch config. unified_receipt_lands_in_orchard_before_boundary pins Orchard-era receipt semantics. orchard_note_spends_to_ironwood_across_boundary exercises the ZIP 318 migration shape: an Orchard note minted before the boundary, spent after activation to a freshly generated unified address, with the receipt asserted in the Ironwood pool. It also pins that the send actually exits the Orchard pool, which is sound under the cross-address restriction because change can return only to the spent note's own receiver. Both cells register on the fetch and state backends. The public testnet can never host the migration cell: its pre-NU6.3 epoch is closed and no pre-activation Orchard TAZ is obtainable, so this fixture is the only controlled venue. Renames apply the pool-naming convention (NU6_3_ACTIVE_* becomes IRONWOOD_ONLY_*: pool names pair with pool names, and NU6.3 names only the upgrade itself). CONTEXT.md gains the supporting glossary entries — testnet versus regtest nets, the era-naming convention, and the cross-address restriction with its normative constraint quoted verbatim. docs/notes/ironwood-activation-plan.md records the design decisions and their sources. Co-Authored-By: Claude Fable 5 --- CONTEXT.md | 42 +++ docs/notes/ironwood-activation-plan.md | 227 ++++++++++++++++ .../tests/compact_block_consistency.rs | 6 +- live-tests/e2e/tests/compact_block_wire.rs | 4 +- live-tests/e2e/tests/ironwood_activation.rs | 243 ++++++++++++++++++ live-tests/zaino-testutils/src/lib.rs | 2 +- .../chain_index/tests/proptest_blockgen.rs | 6 +- 7 files changed, 521 insertions(+), 9 deletions(-) create mode 100644 docs/notes/ironwood-activation-plan.md create mode 100644 live-tests/e2e/tests/ironwood_activation.rs diff --git a/CONTEXT.md b/CONTEXT.md index 11abf3640..4b4e11ba8 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -30,6 +30,48 @@ version is bumped. An unchanged crate keeping its published version is not a violation. _Avoid_: stale version, forgotten bump +### Chains and networks + +**Testnet**: +The public Zcash test network, and only that. Testnet regimes are +non-hermetic — state is shared with other participants, and an epoch +the public chain has left (e.g. pre-NU6.3 once NU6.3 activates there) +cannot be re-entered. +_Avoid_: "testnet" for any locally-launched chain, even one launched +under a testnet network kind + +**Regtest net**: +A hermetic, locally-launched chain whose activation heights the +launcher chooses. Every hermetic local net is a regtest net, whatever +network-kind flag it runs under. +_Avoid_: local testnet, custom testnet + +### Pools and upgrades + +**Ironwood / Orchard (era naming)**: +Eras, fixtures, and predicates that speak of shielded pools are named by +pool — Orchard, Ironwood — and a name that mentions one pool pairs with +the other pool's name, never with the upgrade's. **NU6.3** names only the +network upgrade itself: activation heights, consensus branch ID, +consensus rules. +_Avoid_: mixing vocabularies in one name or one sibling set (e.g. an +`ORCHARD_ONLY_*` fixture whose sibling is `NU6_3_ACTIVE_*` — the sibling +is `IRONWOOD_ONLY_*`) + +**Cross-address restriction**: +The post-NU6.3 rule the Orchard Action circuit enforces: "(g_d, pk_d) +of the output note must equal (g_d, pk_d) of the spent note" — the +output note must carry the same expanded receiver (diversified base +g_d, diversified transmission key pk_d) as its spent note, so each +Orchard action is either change to the spent note's own address or a +withdrawal (positive value balance). Orchard-to-Orchard transfers to +any other address — including another address of the same wallet — are +prohibited. A companion transaction-level rule forbids new value +entering the pool. Source: + +_Avoid_: "exit-only" (overclaims — same-receiver change still lands in +the pool and its commitment tree still grows) + ### TLS and cryptography **Preferred CryptoProvider**: diff --git a/docs/notes/ironwood-activation-plan.md b/docs/notes/ironwood-activation-plan.md new file mode 100644 index 000000000..abb2d3b97 --- /dev/null +++ b/docs/notes/ironwood-activation-plan.md @@ -0,0 +1,227 @@ +# Ironwood activation: test design and heights source-of-truth roadmap + +State capture, 2026-07-06. Design session artifacts live in this tree +(uncommitted at time of writing); coordination artifacts live in +`zingolabs/infrastructure` (PR #278 worktree). This note is the durable +record of the decisions and the facts they rest on. + +## Domain facts, with sources + +- **Ironwood is a new shielded pool** activated by NU6.3 + (). It shares the Action-circuit shape + with Orchard but is a distinct pool. +- **Public testnet activated NU6.3 at height 4,134,000, ~2026-07-04.** + Defined identically in zebra-chain 11.0.0 + (`parameters/constants.rs::activation_heights::testnet::NU6_3`) and + zcash_protocol 0.10.0-pre.0 (`consensus.rs`), consensus branch ID + `0x37a5165b` in both. Wallet stack and validator stack therefore flip eras + at the same height with no drift. **Mainnet has no NU6.3 height** in those + pins. (Gotcha: public explorers can mislabel testnet/mainnet heights — + verify against a synced node.) +- **ZIP 318 (zcash/zips PR #1317), "Orchard to Ironwood migration"**: wallet + best-practices for a two-phase scheduled migration. Normative anchor: "The + user MUST be able to migrate Orchard-pool funds to the Ironwood pool, and + MUST be informed that doing so is necessary to retain access to those + funds." The wallet-visible migration transaction is an Orchard spend with + Ironwood outputs. +- **Cross-address restriction** + (): + post-NU6.3 the Orchard Action circuit requires "(g_d, pk_d) of the output + note must equal (g_d, pk_d) of the spent note" — every Orchard action is + either change to the spent note's own address or a withdrawal (positive + value balance). Cross-address Orchard transfers are circuit-impossible, + including between two addresses of one wallet. A companion + transaction-level rule forbids new value entering the pool. + - Do **not** describe Orchard as "exit-only": same-receiver change still + lands in the pool, so the Orchard note-commitment tree keeps growing + after activation. There is **no frozen-finalRoot predicate**; the + correct chain-walk predicate is **Orchard pool value non-increasing from + the boundary** (observable via `valuePools`). +- **No Orchard TAZ is obtainable** (and the testnet pre-epoch is closed): + post-activation nothing new can enter Orchard, so pre-activation Orchard + notes exist only in wallets that held them before the flip. Zaino holds + none. Consequence: the migration cell is exercisable **only on hermetic + regtest transition chains** — permanently. + +## Language decisions (canonical in CONTEXT.md) + +- **Testnet** names only the public test network. Every hermetic local net + is a **regtest net**, whatever network-kind flag it runs under. +- **Pool names pair with pool names** (Orchard ↔ Ironwood); **NU6.3** names + only the upgrade itself (activation heights, branch ID, consensus rules). + Applied renames: `NU6_3_ACTIVE_ACTIVATION_HEIGHTS` → + `IRONWOOD_ONLY_ACTIVATION_HEIGHTS` (zaino-testutils), + `NU6_3_ACTIVE_HEIGHTS` → `IRONWOOD_ONLY_HEIGHTS` + (`proptest_blockgen.rs`). `NU6_3_TRANSITION_BOUNDARY` keeps the upgrade + vocabulary: an activation height is the upgrade's concept. + +## Design model: predicates × regimes + +A **regime** is a (network, era) pair; a **predicate** is a +wallet-observable claim whose truth is fixed per regime. The suite is the +truth table; tests instantiate predicates in every reachable regime. + +| predicate | Orchard era | Ironwood era | +|---|---|---| +| unified-address receipt lands in Orchard | true | false | +| unified-address receipt lands in Ironwood | false (pool inactive) | true | +| shielded-receiver coinbase pays the era pool | Orchard | Ironwood | +| Orchard note spends into an Ironwood receipt (ZIP 318 migration) | n/a | true | +| `shield` deposits into the era pool | Orchard | Ironwood | +| anything can land in Orchard from outside | true | false (no-new-value rule) | +| Orchard pool value non-increasing | n/a | true (cross-address restriction) | + +Regime venues: + +- **Regtest, Ironwood-only** (`IRONWOOD_ONLY_ACTIVATION_HEIGHTS`, canonical + NU6.3-at-2): the existing devtool wallet tests (`send_to_ironwood`, + `shield_for_validator`, `receives_mining_reward`, `send_to_all` in + `live-tests/e2e/tests/devtool.rs`). +- **Regtest, Orchard-only** (`ORCHARD_ONLY_ACTIVATION_HEIGHTS`): clientless + / wire mining fixtures only (devtool wallet cannot run there today). +- **Regtest, transition** (`ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS`, + NU6.3 at `NU6_3_TRANSITION_BOUNDARY` = 6): coinbase-routing and era + composition covered clientless + (`clientless/tests/compact_block_consistency.rs`) and over the wire + (`e2e/tests/compact_block_wire.rs`); **wallet cells live in + `e2e/tests/ironwood_activation.rs`** (see below). +- **Public testnet**: observational boundary walk across height 4,134,000 — + durable forever, historical blocks don't expire; third-party ZIP 318 + migrations supply the Orchard-spend traffic zaino must index but cannot + author. Active cells post-flip only, Orchard-free: faucet TAZ → + transparent receipt → `shield` → Ironwood send/receive. Env-gated manual + runs; record txids/heights in a dated evidence log + (`docs/notes/ironwood-activation-evidence.md`, created on first run). + `ZEBRAD_TESTNET_CACHE_DIR` (zaino-testutils) anticipates the cached sync. + +## Landed artifacts (this tree) + +- **`live-tests/e2e/tests/ironwood_activation.rs`** — the transition-regime + wallet cells, full real bodies, gated + `#[should_panic(expected = "UnsupportedActivationHeights")]`: + - `unified_receipt_lands_in_orchard_before_boundary` + - `orchard_note_spends_to_ironwood_across_boundary` — the migration cell; + recipient unified address generated *after* activation; asserts the + faucet's Orchard balance strictly shrinks, guarding against devtool note + selection dodging the migration by spending the boundary-height Ironwood + coinbase instead (sound under the cross-address restriction: change + returns only to the spent note's receiver, so a genuine Orchard spend + nets sent-plus-fee out of the pool). + - Verified: both pass as should_panic in ~5 s each + (`cargo nextest run -p e2e -E 'binary(ironwood_activation)'`). + - Registered on one backend (FetchService) while known-red: the panic + fires before the backend is exercised; mirror the fetch/state matrix + when the cells go live. +- **`e2e::devtool::build_clients_at(port, &heights)`** — devtool wallets on + caller-chosen regtest heights; `build_clients` delegates with the + canonical set. +- CONTEXT.md glossary entries; the renames above. + +## The blocker, and the infra handoff + +`zcash_local_net`'s devtool client rejects every regtest heights set except +its canonical NU6.3-at-2 one at wallet launch +(`ClientError::UnsupportedActivationHeights`; the guard exists because +heights drift between wallet and validator produces "incorrect consensus +branch id" — see zaino#1368). The guard predates the current devtool, which +reads heights at `init` from the `--activation-heights` TOML the client +already writes. + +Handoff: **`~/src/zingolabs/infras/dev/zaino-ironwood-activation-infra-spec.md`** +(for zingolabs/infrastructure PR #278, branch `bump_to_NU6.3`; zaino pins +rev `0dc4a51f...`). Asks: lift the equality guard (canonical set stays the +default), retire the error variant (zaino's known-red tests key on its name +so the pin bump flips them loudly), keep the caller-responsibility doc +contract, plus an infra-side integration test proving era-correct scanning +and branch-ID selection from file heights. Known unknown flagged: devtool +note-selection policy across pools. + +Scope addition (confirmed): infra also narrows `ZainodConfig.network` from +`NetworkType` to a payload-free `NetworkKind` in the same release — the +heights payload there is dead code (discarded at +`network_type_to_string()`; the Indexer never received heights that way). +Validator configs keep full `NetworkType`: they configure the source of +truth. + +## Roadmap: source of truth first + +Current directive (supersedes "unblock the suite next"): **first** make the +Validator the single source of truth for activation heights, with +regression tests that keep that source unique, and fix zaino#1076. + +Invariant: *the Indexer learns activation heights from the Validator at +startup and accepts them from nowhere else; configs carry network kind +only.* + +Audit facts grounding the implementation: + +- zainod's TOML is already kind-only: `NetworkSerde` serializes + "Mainnet"/"Testnet"/"Regtest", and deserializing "Regtest" injects the + compiled-in `ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS` + (`zaino-common/src/config/network.rs`). The real second channel is the + library seam: `Network::Regtest(ActivationHeights)` constructed + programmatically (e.g. by `TestManager`) and consumed via + `config.get_network()` → `to_zebra_network()`. +- The validator channel already exists in zaino's types: + `GetBlockchainInfoResponse.upgrades` is parsed (zaino-fetch + `jsonrpsee/response.rs`) and consumed (`backends/fetch.rs`, + `latest_network_upgrade`). The work is deriving the runtime + `zebra_chain::parameters::Network` from that response at startup and + narrowing the config type to kind-only. +- Regression tests that maintain uniqueness: launch the validator at + non-canonical heights (the transition fixture) with a kind-only-configured + zainod and assert era-correct serving (fails if anyone re-couples zainod + to a compiled-in set); the narrowed config type itself is the compile-time + regression test; a serde test pins the TOML shape. +- Payoff: the `InvalidData("Block commitment could not be computed")` + config-drift failure class becomes structurally impossible — the same + invariant the infra guard lift expresses on the wallet side (wallets get + heights from a file matched to the validator; the indexer gets them from + the validator directly). +- **#1076 residue** (issue text predates the all-at-2 canonical set, which + already replaced the NU6.1-at-1000 sentinel): wire + `lockbox_disbursements` into zebrad-launching fixtures (plumbing landed + at pinned rev, infra#243); add one NU6.1 boundary-crossing test (same + transition-fixture pattern as the Ironwood suite); watch for + block-commitment regressions. + +## Sequencing + +1. Zaino: heights source-of-truth implementation + uniqueness regression + tests (**landed in this tree**: `regtest_network_from_upgrades` + + spawn-time adoption in the zaino-state backends, de-mirrored testutils, + unit tests beside the mapping, live regressions in + `clientless/tests/validator_heights.rs`); remaining from this item: the + config-type split (kind-only config type, heights only post-handshake) + and the #1076 residue (lockbox fixtures, NU6.1 boundary test). +2. Infra: `NetworkKind` narrowing plus the strict form of the wallet-side + change on `bump_to_NU6.3`. The standing directive (2026-07-06) is that + the Validator is the *only* source of truth, enforced at compile time: + the harness derives the wallet's `activation-heights.toml` from + `Validator::get_activation_heights()`, and no client API accepts + caller-supplied heights (the spec's original "lift the guard, accept + caller heights" shape is superseded — a caller heights parameter is + itself a second source). +3. Zaino pin bump: adapt to the narrowed `ZainodConfig`; the two + `ironwood_activation` tests flip red (their expected + `UnsupportedActivationHeights` panic disappears along with the error + variant); replace `e2e::devtool::build_clients_at(port, &heights)` — + which exists only to express the caller-supplied shape — with a + derivation-based builder, so a fixture's heights are typed in exactly + one place: the zebrad launch config. First live runs settle devtool + note selection and the compact form of Orchard-spend data. +4. Deferred cells: testnet observational walk + post-flip Ironwood active + cells (env-gated, evidence log); chain-walk cells for the cross-address + restriction (Orchard pool-value monotonicity, same-receiver-change-only + commitments); the Orchard spend-window/freeze sub-regime waits on a + consensus source. + +## Unrelated finding from the same session + +The red `Integration tests / RPC tests shard-2/3` statuses on zaino commits +(zcash/integration-tests runs) are an upstream zallet break, not zaino's: +every run since 2026-07-03 ~23:25 UTC fails `zcashd_key_import{,_db}.py` in +zallet's `migrate-zcashd-wallet` with `UNIQUE constraint failed: +addresses.cached_transparent_receiver_address`, immediately after zallet +main's librustzcash/NU6.3 dependency bumps. No zallet issue existed for it +as of 2026-07-06. diff --git a/live-tests/clientless/tests/compact_block_consistency.rs b/live-tests/clientless/tests/compact_block_consistency.rs index 98d60daa8..753456026 100644 --- a/live-tests/clientless/tests/compact_block_consistency.rs +++ b/live-tests/clientless/tests/compact_block_consistency.rs @@ -14,7 +14,7 @@ use zaino_fetch::jsonrpsee::response::GetBlockResponse; use zaino_state::FetchService; use zaino_state::ZcashIndexer as _; use zaino_testutils::{ - MinerPool, TestManager, ValidatorKind, NU6_3_ACTIVE_ACTIVATION_HEIGHTS, + MinerPool, TestManager, ValidatorKind, IRONWOOD_ONLY_ACTIVATION_HEIGHTS, NU6_3_TRANSITION_BOUNDARY, ORCHARD_ONLY_ACTIVATION_HEIGHTS, ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS, }; @@ -33,7 +33,7 @@ async fn unfiltered_compact_blocks_match_chain_metadata_zebrad() { MinerPool::Orchard, &ValidatorKind::Zebrad, None, - Some(NU6_3_ACTIVE_ACTIVATION_HEIGHTS), + Some(IRONWOOD_ONLY_ACTIVATION_HEIGHTS), None, true, false, @@ -326,7 +326,7 @@ async fn orchard_only_coinbase_routing_zebrad() { /// multi_thread required: the test manager spawns the validator and indexer services. #[tokio::test(flavor = "multi_thread")] async fn ironwood_only_coinbase_routing_zebrad() { - assert_coinbase_routing(NU6_3_ACTIVE_ACTIVATION_HEIGHTS, 6, |height| { + assert_coinbase_routing(IRONWOOD_ONLY_ACTIVATION_HEIGHTS, 6, |height| { if height >= 2 { CoinbaseEra::Ironwood } else { diff --git a/live-tests/e2e/tests/compact_block_wire.rs b/live-tests/e2e/tests/compact_block_wire.rs index 495de0c2d..565869d3f 100644 --- a/live-tests/e2e/tests/compact_block_wire.rs +++ b/live-tests/e2e/tests/compact_block_wire.rs @@ -18,7 +18,7 @@ use zaino_proto::proto::service::{BlockId, BlockRange}; use zaino_state::FetchService; use zaino_state::ZcashIndexer as _; use zaino_testutils::{ - make_uri, MinerPool, TestManager, ValidatorKind, NU6_3_ACTIVE_ACTIVATION_HEIGHTS, + make_uri, MinerPool, TestManager, ValidatorKind, IRONWOOD_ONLY_ACTIVATION_HEIGHTS, NU6_3_TRANSITION_BOUNDARY, ORCHARD_ONLY_ACTIVATION_HEIGHTS, ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS, }; @@ -171,7 +171,7 @@ async fn orchard_only_wire_serving_zebrad() { /// multi_thread required: the test manager spawns the validator, indexer, and zainod. #[tokio::test(flavor = "multi_thread")] async fn ironwood_only_wire_serving_zebrad() { - assert_wire_served_eras(NU6_3_ACTIVE_ACTIVATION_HEIGHTS, 6, |height| { + assert_wire_served_eras(IRONWOOD_ONLY_ACTIVATION_HEIGHTS, 6, |height| { if height >= 2 { CoinbaseEra::Ironwood } else { diff --git a/live-tests/e2e/tests/ironwood_activation.rs b/live-tests/e2e/tests/ironwood_activation.rs new file mode 100644 index 000000000..6e04096de --- /dev/null +++ b/live-tests/e2e/tests/ironwood_activation.rs @@ -0,0 +1,243 @@ +//! Wallet-tier predicates across the Orchard→Ironwood activation boundary. +//! +//! Every test here runs a devtool wallet on +//! [`ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS`] — the hermetic replay of what +//! the public testnet did once at height 4,134,000: heights 2 through 5 are +//! Orchard era, [`NU6_3_TRANSITION_BOUNDARY`] (6) onward is Ironwood era. +//! The wallets derive their activation schedule from the running validator +//! (`WalletNetwork::from_validator`, infrastructure ADR 0003), so the +//! fixture heights are typed in exactly one place: the zebrad launch +//! config. Height drift between wallet, indexer, and validator is +//! unrepresentable — zainod adopts the same schedule over +//! `getblockchaininfo` (zaino#1076). +//! +//! # The predicates, and where each era's cell is covered +//! +//! | predicate (wallet-observable) | Orchard era | Ironwood era | +//! |----------------------------------------------|-------------|--------------| +//! | unified-address receipt lands in Orchard | here | false — `devtool.rs` `send_to_ironwood` asserts the Orchard pool stays empty | +//! | unified-address receipt lands in Ironwood | false (pool inactive) | `devtool.rs` `send_to_ironwood` | +//! | shielded-receiver coinbase pays the era pool | here (wallet view); wire tier in `compact_block_wire.rs` | `devtool.rs` `receives_mining_reward`; wire tier in `compact_block_wire.rs` | +//! | an Orchard note spends into an Ironwood receipt (ZIP 318 migration) | n/a (nothing to exit) | here | +//! +//! Era composition of the *served* chain (coinbase routing, compact-block +//! action fields) is covered clientless in +//! `clientless/tests/compact_block_consistency.rs` and over the real gRPC +//! wire in `compact_block_wire.rs`; this file owns the cells that need a +//! wallet on both sides of the boundary. +//! +//! The public testnet cannot host the migration cell for us: its pre-NU6.3 +//! epoch closed at height 4,134,000, no new value may enter Orchard from +//! there (post-activation Orchard actions permit only same-receiver change +//! or withdrawal — the cross-address restriction, +//! ), +//! and we hold no pre-activation Orchard TAZ — so this hermetic fixture is +//! the only controlled venue for it. +//! +//! Deferred cells the cross-address restriction implies (chain-walk tier, +//! not wallet tier): the Orchard pool value is non-increasing from the +//! boundary, and post-activation Orchard commitments exist only as +//! same-receiver change — note the Orchard note-commitment tree therefore +//! still grows after activation; do not encode a frozen-finalRoot predicate. +//! +//! Requires a `zcash-devtool` binary built with `--features regtest_support` +//! in `TEST_BINARIES_DIR`/`PATH`, alongside the usual validator binaries. + +use e2e::devtool::DevtoolClients; +use zaino_state::{ZcashIndexer, ZcashService}; +use zaino_testutils::{ + PollableTip, TestManager, TestService, ValidatorKind, NU6_3_TRANSITION_BOUNDARY, + ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS, +}; +use zainodlib::error::IndexerError; +use zcash_local_net::validator::zebrad::Zebrad; + +/// Launch an orchard-receiver-mining zebrad + Zaino on the transition +/// heights, build devtool faucet/recipient wallets (their schedule derived +/// from the launched validator), mine one block (height 2: the first +/// Orchard-era coinbase note), and sync the faucet. The transition-fixture +/// analogue of `devtool.rs::launch_and_fund_faucet`. +async fn launch_transition_chain_and_fund_faucet( +) -> (TestManager, DevtoolClients) +where + Service: TestService, + IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, + ::Subscriber: PollableTip, +{ + let test_manager = TestManager::::launch_mining_to( + zaino_testutils::SHIELDED_FUNDING_POOL, + &ValidatorKind::Zebrad, + None, + Some(ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS), + None, + true, + false, + false, + ) + .await + .expect("launch TestManager"); + + let mut clients = e2e::devtool::build_clients( + test_manager + .zaino_grpc_listen_address + .expect("zaino enabled") + .port(), + &test_manager.local_net, + ) + .await; + + test_manager + .generate_blocks_and_wait_for_tip(1, test_manager.subscriber()) + .await; + clients.sync_faucet().await; + + (test_manager, clients) +} + +/// Orchard-era receipt: with the tip still below the boundary, the faucet's +/// coinbase note is an Orchard note (not Ironwood), and a unified-address +/// send received before the boundary lands in the recipient's Orchard pool +/// with the Ironwood pool exactly empty — the era-mirror of +/// `devtool.rs::send_to_pool(Ironwood)`. +async fn unified_receipt_lands_in_orchard_before_boundary() +where + Service: TestService, + IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, + ::Subscriber: PollableTip, +{ + let (mut test_manager, mut clients) = + launch_transition_chain_and_fund_faucet::().await; + + // Tip is 2: inside the Orchard era, with room to confirm the send at + // height 3 while staying below the boundary at 6. + let faucet_balance = clients.faucet_balance().await; + assert!( + faucet_balance.orchard_spendable > 0, + "pre-boundary coinbase should be an orchard note, got {faucet_balance:?}" + ); + assert_eq!( + faucet_balance.ironwood_spendable, 0, + "no ironwood note can exist below the boundary, got {faucet_balance:?}" + ); + + let recipient = clients.get_recipient_address("unified").await; + let txid = clients.send_from_faucet(&recipient, 250_000).await; + dbg!(txid); + + test_manager + .generate_blocks_and_wait_for_tip(1, test_manager.subscriber()) + .await; + clients.sync_recipient().await; + + let balance = clients.recipient_balance().await; + assert_eq!(e2e::Pool::Orchard.spendable_balance(&balance), 250_000); + assert_eq!(e2e::Pool::Ironwood.spendable_balance(&balance), 0); + + test_manager.close().await; +} + +/// The ZIP 318 migration shape: an Orchard note minted before the boundary +/// is spent after it, to a unified address generated after activation, and +/// the receipt lands in the Ironwood pool with the recipient's Orchard pool +/// exactly empty. The faucet's Orchard balance must shrink: from the +/// boundary, the cross-address restriction limits each Orchard action to +/// same-receiver change or withdrawal +/// (), +/// so a genuine Orchard spend nets sent-amount-plus-fee out of the pool even +/// when change returns to the spent note's address. +async fn orchard_note_spends_to_ironwood_across_boundary() +where + Service: TestService, + IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, + ::Subscriber: PollableTip, +{ + let (mut test_manager, mut clients) = + launch_transition_chain_and_fund_faucet::().await; + + let pre_boundary_balance = clients.faucet_balance().await; + assert!( + pre_boundary_balance.orchard_spendable > 0, + "pre-boundary coinbase should be an orchard note, got {pre_boundary_balance:?}" + ); + + // Tip is 2; mine to the boundary itself. Heights 3–5 add more Orchard + // coinbase notes, height 6 is the first Ironwood-era block (its coinbase + // is the faucet's first Ironwood note). + test_manager + .generate_blocks_and_wait_for_tip(NU6_3_TRANSITION_BOUNDARY - 2, test_manager.subscriber()) + .await; + clients.sync_faucet().await; + let crossed_balance = clients.faucet_balance().await; + let orchard_before_send = crossed_balance.orchard_spendable; + assert!( + crossed_balance.ironwood_spendable > 0, + "the boundary coinbase should be an ironwood note, got {crossed_balance:?}" + ); + + // Generated only now — after activation — per the migration shape under + // test: old-pool note, new-era address. + let recipient = clients.get_recipient_address("unified").await; + let txid = clients.send_from_faucet(&recipient, 250_000).await; + dbg!(txid); + + test_manager + .generate_blocks_and_wait_for_tip(1, test_manager.subscriber()) + .await; + clients.sync_faucet().await; + clients.sync_recipient().await; + + let balance = clients.recipient_balance().await; + assert_eq!(e2e::Pool::Ironwood.spendable_balance(&balance), 250_000); + assert_eq!(e2e::Pool::Orchard.spendable_balance(&balance), 0); + + // Pins that the send actually exited the Orchard pool rather than + // spending the boundary-height Ironwood coinbase — the note-selection + // question the first live runs of this suite exist to answer. + assert!( + clients.faucet_balance().await.orchard_spendable < orchard_before_send, + "the migration send must spend an orchard note" + ); + + test_manager.close().await; +} + +mod zebrad { + // FetchService is a deprecated re-export; the deprecation fires at the + // turbofish use sites below, so the allow covers the whole module. + #[allow(deprecated)] + mod fetch_service { + use zaino_state::FetchService; + + /// multi_thread required: the test manager spawns the validator and + /// indexer services. + #[tokio::test(flavor = "multi_thread")] + async fn unified_receipt_lands_in_orchard_before_boundary() { + crate::unified_receipt_lands_in_orchard_before_boundary::().await; + } + + /// multi_thread required: the test manager spawns the validator and + /// indexer services. + #[tokio::test(flavor = "multi_thread")] + async fn orchard_note_spends_to_ironwood_across_boundary() { + crate::orchard_note_spends_to_ironwood_across_boundary::().await; + } + } + + mod state_service { + use zaino_state::StateService; + + /// multi_thread required: the test manager spawns the validator and + /// indexer services. + #[tokio::test(flavor = "multi_thread")] + async fn unified_receipt_lands_in_orchard_before_boundary() { + crate::unified_receipt_lands_in_orchard_before_boundary::().await; + } + + /// multi_thread required: the test manager spawns the validator and + /// indexer services. + #[tokio::test(flavor = "multi_thread")] + async fn orchard_note_spends_to_ironwood_across_boundary() { + crate::orchard_note_spends_to_ironwood_across_boundary::().await; + } + } +} diff --git a/live-tests/zaino-testutils/src/lib.rs b/live-tests/zaino-testutils/src/lib.rs index 6e8a342e1..f6c8ba692 100644 --- a/live-tests/zaino-testutils/src/lib.rs +++ b/live-tests/zaino-testutils/src/lib.rs @@ -112,7 +112,7 @@ pub const ORCHARD_ONLY_ACTIVATION_HEIGHTS: ActivationHeights = ActivationHeights /// Zebrad regtest heights with every upgrade through NU6.3 active from height 2, so /// generated blocks carry V6 coinbases from the first post-genesis era. -pub const NU6_3_ACTIVE_ACTIVATION_HEIGHTS: ActivationHeights = ActivationHeights { +pub const IRONWOOD_ONLY_ACTIVATION_HEIGHTS: ActivationHeights = ActivationHeights { before_overwinter: Some(1), overwinter: Some(1), sapling: Some(1), diff --git a/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs b/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs index 54f273917..93843c6db 100644 --- a/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs +++ b/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs @@ -451,7 +451,7 @@ fn zebra_arbitrary_generates_v6_transactions_for_nu6_3() { /// NU6.3 active from height 2, so post-activation generated blocks carry V6 /// transactions whose shielded data lands in the Ironwood pool. -const NU6_3_ACTIVE_HEIGHTS: ActivationHeights = ActivationHeights { +const IRONWOOD_ONLY_HEIGHTS: ActivationHeights = ActivationHeights { before_overwinter: Some(1), overwinter: Some(1), sapling: Some(1), @@ -478,7 +478,7 @@ const NU6_3_ACTIVE_HEIGHTS: ActivationHeights = ActivationHeights { /// source of truth. #[test] fn passthrough_metadata_consistency_ironwood_only() { - metadata_consistency_for_era(NU6_3_ACTIVE_HEIGHTS, Some(2), false) + metadata_consistency_for_era(IRONWOOD_ONLY_HEIGHTS, Some(2), false) } /// Orchard-only heights: every upgrade through NU6.2 at height 2, NU6.3 never @@ -516,7 +516,7 @@ fn passthrough_metadata_consistency_orchard_to_ironwood_transition() { metadata_consistency_for_era( ActivationHeights { nu6_3: Some(boundary), - ..NU6_3_ACTIVE_HEIGHTS + ..IRONWOOD_ONLY_HEIGHTS }, Some(boundary), true, From c6ec6796d1ada0d784247e81e4ce16556dc0eb4b Mon Sep 17 00:00:00 2001 From: zancas Date: Mon, 6 Jul 2026 22:45:16 -0700 Subject: [PATCH 26/58] test: enforce the Ironwood boundary invariants at its edges, and on testnet MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ironwood_activation suite gains the boundary-edge cells the plan deferred. receipts_flip_pools_exactly_at_the_boundary confirms one send in the last Orchard-era block and a second in the activation block itself: the receipt pool flips between adjacent heights, the pre-boundary receipt survives the flip unchanged, and the second send — built while the tip was still Orchard-era, spending an Orchard note — is a migration transaction inside the activation block, pinning the wallet's era anticipation at the edge. shield_deposits_to_orchard_before_boundary is the era-mirror of the Ironwood-era shield cell. The migration test additionally walks the served per-height value pools and enforces the chain-tier consequences of the cross-address restriction: the Ironwood pool holds no value below the boundary, the Orchard pool grows only below it, holds exactly steady across the 5-to-6 edge, and shrinks at the migration block. All cells register on both backends (eight tests). The same invariants are enforced against the real activation on the public testnet in clientless/tests/testnet_ironwood_boundary.rs: an env-gated walk (skips unless ~/.cache/zebra holds a testnet chain synced past the boundary) that launches a public-testnet zebrad directly through ZebradConfig, reads the activation height from the validator's getblockchaininfo.upgrades, cross-checks it against the observed activation at 4,134,000, and asserts the Ironwood pool is empty below the boundary and the Orchard pool never grows from it. zaino-fetch's BlockObject gains a value_pools accessor (and ChainBalance a balance accessor) so the walk can read the validator's verbosity-2 per-height pool balances. Co-Authored-By: Claude Fable 5 --- .../tests/testnet_ironwood_boundary.rs | 155 +++++++++++++ live-tests/e2e/tests/ironwood_activation.rs | 218 +++++++++++++++++- .../zaino-fetch/src/jsonrpsee/response.rs | 15 ++ 3 files changed, 385 insertions(+), 3 deletions(-) create mode 100644 live-tests/clientless/tests/testnet_ironwood_boundary.rs diff --git a/live-tests/clientless/tests/testnet_ironwood_boundary.rs b/live-tests/clientless/tests/testnet_ironwood_boundary.rs new file mode 100644 index 000000000..08b5a7f7f --- /dev/null +++ b/live-tests/clientless/tests/testnet_ironwood_boundary.rs @@ -0,0 +1,155 @@ +//! Observational walk of the real NU6.3 activation boundary on the public +//! testnet (glossary: "testnet" means the public test network and nothing +//! else). Historical blocks are permanent, so these invariants are checkable +//! forever, long after the epoch that produced them closed. +//! +//! The invariants, enforced one block below the boundary and at it: +//! +//! - Below the activation height the Ironwood pool holds no value. +//! - From the activation height the Orchard pool's value never increases: +//! the no-new-value rule admits only withdrawals and same-receiver change +//! (the cross-address restriction, +//! ). +//! +//! The activation height itself is read from the running validator's +//! `getblockchaininfo.upgrades` — the single source of truth for activation +//! heights — and cross-checked against the observed public-testnet +//! activation at height 4,134,000 (~2026-07-04). +//! +//! Non-hermetic and therefore env-gated: the test needs a zebrad chain +//! cache at `~/.cache/zebra` (`ZEBRAD_TESTNET_CACHE_DIR`) synced past the +//! activation height, and skips with a message when the cache is absent or +//! short. Run it manually, or from a job that maintains the cache. +//! +//! The validator is launched through `ZebradConfig` directly rather than +//! `TestManager`: the manager's launch path always writes regtest test +//! parameters into the config, while this test needs the public testnet +//! (`network_type: NetworkType::Testnet`, which makes zebrad ignore +//! `activation_heights` and `miner_address`). + +use zaino_fetch::jsonrpsee::connector::{test_node_and_return_url, JsonRpSeeConnector}; +use zaino_fetch::jsonrpsee::response::GetBlockResponse; +use zaino_testutils::ZEBRAD_TESTNET_CACHE_DIR; +use zcash_local_net::process::Process as _; +use zcash_local_net::protocol::NetworkType; +use zcash_local_net::validator::zebrad::{Zebrad, ZebradConfig}; +use zcash_local_net::validator::Validator as _; +use zebra_chain::parameters::NetworkUpgrade; + +/// The height the public testnet activated NU6.3 at, recorded from the real +/// activation. A mismatch means either a testnet reset or a wrong validator +/// pin — both worth failing loudly over. +const OBSERVED_TESTNET_NU6_3_ACTIVATION: u32 = 4_134_000; + +/// The chain value of `pool_id` as of `height`, from the validator's +/// verbosity-2 block object. +async fn pool_zats(connector: &JsonRpSeeConnector, height: u32, pool_id: &str) -> i64 { + let response = connector + .get_block(height.to_string(), Some(2)) + .await + .expect("getblock verbosity 2"); + let GetBlockResponse::Object(block) = response else { + panic!("verbosity-2 getblock must return a block object"); + }; + block + .value_pools() + .expect("verbosity-2 block object carries value pools") + .iter() + .map(|pool| pool.balance()) + .find(|pool| pool.id() == pool_id) + .unwrap_or_else(|| panic!("value pools must include {pool_id}")) + .chain_value_zat() + .zatoshis() +} + +/// multi_thread required: the test launches the validator process and polls +/// it over RPC. +#[tokio::test(flavor = "multi_thread")] +async fn value_pools_respect_the_boundary_on_testnet() { + let Some(cache_dir) = ZEBRAD_TESTNET_CACHE_DIR.clone() else { + eprintln!("skipping: no testnet cache dir configured"); + return; + }; + if !cache_dir.exists() { + eprintln!( + "skipping: no zebrad testnet chain cache at {}", + cache_dir.display() + ); + return; + } + + let config = ZebradConfig { + network_type: NetworkType::Testnet, + chain_cache: Some(cache_dir), + ..ZebradConfig::default() + }; + let mut zebrad = Zebrad::launch(config).await.expect("launch testnet zebrad"); + + let rpc_address = format!("127.0.0.1:{}", zebrad.get_port()); + let connector = JsonRpSeeConnector::new_with_basic_auth( + test_node_and_return_url( + &rpc_address, + None, + Some("xxxxxx".to_string()), + Some("xxxxxx".to_string()), + ) + .await + .expect("validator RPC reachable"), + "xxxxxx".to_string(), + "xxxxxx".to_string(), + ) + .expect("connect to the validator RPC"); + + let blockchain_info = connector + .get_blockchain_info() + .await + .expect("getblockchaininfo"); + + // The validator's reported schedule is the source of truth for the + // boundary; the recorded constant pins the real public-testnet history. + let boundary = blockchain_info + .upgrades + .values() + .find_map(|upgrade_info| { + let (upgrade, height, _status) = upgrade_info.into_parts(); + (upgrade == NetworkUpgrade::Nu6_3).then_some(height.0) + }) + .expect("the testnet validator must report an NU6.3 activation height"); + assert_eq!( + boundary, OBSERVED_TESTNET_NU6_3_ACTIVATION, + "the validator's NU6.3 height must match the observed testnet activation" + ); + + let tip = blockchain_info.blocks.0; + if tip <= boundary { + eprintln!( + "skipping: testnet cache tip {tip} has not crossed the NU6.3 boundary {boundary}" + ); + zebrad.stop(); + return; + } + + // One block below the boundary and at it, plus a block of margin on + // each side: the Ironwood pool holds no value below the activation + // height, and the Orchard pool never grows from it. + for height in (boundary - 2)..boundary { + assert_eq!( + pool_zats(&connector, height, "ironwood").await, + 0, + "the ironwood pool must hold no value at height {height}, below the boundary" + ); + } + let walk_end = boundary + 1; + let mut previous_orchard = pool_zats(&connector, boundary - 1, "orchard").await; + for height in boundary..=walk_end { + let orchard = pool_zats(&connector, height, "orchard").await; + assert!( + orchard <= previous_orchard, + "the orchard pool must never grow from the boundary; \ + height {height} holds {orchard} zats after {previous_orchard}" + ); + previous_orchard = orchard; + } + + zebrad.stop(); +} diff --git a/live-tests/e2e/tests/ironwood_activation.rs b/live-tests/e2e/tests/ironwood_activation.rs index 6e04096de..eba7c5ef1 100644 --- a/live-tests/e2e/tests/ironwood_activation.rs +++ b/live-tests/e2e/tests/ironwood_activation.rs @@ -19,6 +19,9 @@ //! | unified-address receipt lands in Ironwood | false (pool inactive) | `devtool.rs` `send_to_ironwood` | //! | shielded-receiver coinbase pays the era pool | here (wallet view); wire tier in `compact_block_wire.rs` | `devtool.rs` `receives_mining_reward`; wire tier in `compact_block_wire.rs` | //! | an Orchard note spends into an Ironwood receipt (ZIP 318 migration) | n/a (nothing to exit) | here | +//! | `shield` deposits into the era pool | here | `devtool.rs` `shield_for_validator` | +//! | receipt pool flips between the last Orchard block and the activation block | here (boundary − 1) | here (exactly at the boundary) | +//! | Orchard pool value grows only below the boundary; Ironwood holds value only from it | here (per-height value pools) | here | //! //! Era composition of the *served* chain (coinbase routing, compact-block //! action fields) is covered clientless in @@ -44,10 +47,10 @@ //! in `TEST_BINARIES_DIR`/`PATH`, alongside the usual validator binaries. use e2e::devtool::DevtoolClients; -use zaino_state::{ZcashIndexer, ZcashService}; +use zaino_state::{LightWalletIndexer, ZcashIndexer, ZcashService}; use zaino_testutils::{ - PollableTip, TestManager, TestService, ValidatorKind, NU6_3_TRANSITION_BOUNDARY, - ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS, + all_pools_i32, collect_block_range, PollableTip, TestManager, TestService, ValidatorKind, + NU6_3_TRANSITION_BOUNDARY, ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS, }; use zainodlib::error::IndexerError; use zcash_local_net::validator::zebrad::Zebrad; @@ -94,6 +97,34 @@ where (test_manager, clients) } +/// The chain value (in zatoshis) of the pool named `pool_id` as of +/// `height`, read from the served verbosity-2 block object — the same +/// per-height `valuePools` a zcashd `getblock` reports. +async fn pool_zats_at_height(subscriber: &S, height: u32, pool_id: &str) -> i64 +where + S: ZcashIndexer, + IndexerError: From, +{ + let response = subscriber + .z_get_block(height.to_string(), Some(2)) + .await + .map_err(IndexerError::from) + .expect("z_get_block verbosity 2"); + let zebra_rpc::methods::GetBlock::Object(block) = response else { + panic!("verbosity-2 getblock must return a block object"); + }; + let pools = block + .value_pools() + .as_ref() + .expect("verbosity-2 block object carries value pools"); + pools + .iter() + .find(|pool| pool.id() == pool_id) + .unwrap_or_else(|| panic!("value pools must include {pool_id}")) + .chain_value_zat() + .zatoshis() +} + /// Orchard-era receipt: with the tip still below the boundary, the faucet's /// coinbase note is an Orchard note (not Ironwood), and a unified-address /// send received before the boundary lands in the recipient's Orchard pool @@ -198,6 +229,159 @@ where "the migration send must spend an orchard note" ); + // Chain-tier consequences, read from the served per-height value pools. + // The Ironwood pool holds no value below the activation height; the + // Orchard pool grows only below it (its coinbases), holds exactly + // steady across the boundary edge (the activation block's coinbase pays + // Ironwood, and no new value may enter Orchard), and shrinks at the + // migration block by the withdrawn amount plus fee. + let boundary = NU6_3_TRANSITION_BOUNDARY; + let migration_height = boundary + 1; + let subscriber = test_manager.subscriber(); + let mut orchard_at = Vec::new(); + for height in 1..=migration_height { + let ironwood = pool_zats_at_height(subscriber, height, "ironwood").await; + let orchard = pool_zats_at_height(subscriber, height, "orchard").await; + if height < boundary { + assert_eq!( + ironwood, 0, + "the ironwood pool must hold no value at height {height}, below the boundary" + ); + } + orchard_at.push(orchard); + } + let orchard = |height: u32| orchard_at[height as usize - 1]; + for height in 2..boundary { + assert!( + orchard(height) > orchard(height - 1), + "each pre-boundary coinbase must grow the orchard pool (height {height})" + ); + } + assert_eq!( + orchard(boundary), + orchard(boundary - 1), + "the orchard pool must hold exactly steady across the boundary edge" + ); + assert!( + pool_zats_at_height(subscriber, boundary, "ironwood").await > 0, + "the activation block's coinbase must give the ironwood pool its first value" + ); + assert!( + orchard(migration_height) < orchard(boundary), + "the migration block must shrink the orchard pool" + ); + + test_manager.close().await; +} + +/// The receipt pool flips between adjacent blocks: a send confirmed in the +/// last Orchard-era block (boundary − 1) lands in Orchard, and a send built +/// one block below the boundary but confirmed in the activation block +/// itself lands in Ironwood. The second send also pins the wallet's era +/// anticipation at the edge: it is constructed while the tip is still +/// Orchard-era, targeting the first Ironwood-era height, and it spends an +/// Orchard note (the faucet holds no Ironwood note until the activation +/// block is mined) — a migration transaction in the activation block. +async fn receipts_flip_pools_exactly_at_the_boundary() +where + Service: TestService, + IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, + ::Subscriber: PollableTip + LightWalletIndexer, +{ + let (mut test_manager, mut clients) = + launch_transition_chain_and_fund_faucet::().await; + + // Tip is 2. Mine heights 3 and 4 (a second spendable Orchard note and a + // filler), so the two sends below confirm at exactly boundary − 1 and + // the boundary. + test_manager + .generate_blocks_and_wait_for_tip(NU6_3_TRANSITION_BOUNDARY - 4, test_manager.subscriber()) + .await; + clients.sync_faucet().await; + + let recipient = clients.get_recipient_address("unified").await; + + // Confirmed in block 5: the last Orchard-era block. + let last_orchard_txid = clients.send_from_faucet(&recipient, 250_000).await; + test_manager + .generate_blocks_and_wait_for_tip(1, test_manager.subscriber()) + .await; + clients.sync_faucet().await; + clients.sync_recipient().await; + let balance = clients.recipient_balance().await; + assert_eq!(e2e::Pool::Orchard.spendable_balance(&balance), 250_000); + assert_eq!(e2e::Pool::Ironwood.spendable_balance(&balance), 0); + + // Built at tip boundary − 1, confirmed in block 6: the activation block. + let first_ironwood_txid = clients.send_from_faucet(&recipient, 250_000).await; + test_manager + .generate_blocks_and_wait_for_tip(1, test_manager.subscriber()) + .await; + clients.sync_recipient().await; + let balance = clients.recipient_balance().await; + assert_eq!(e2e::Pool::Ironwood.spendable_balance(&balance), 250_000); + assert_eq!( + e2e::Pool::Orchard.spendable_balance(&balance), + 250_000, + "the pre-boundary receipt must survive the flip unchanged" + ); + + // Era composition of the two served edge blocks. + let subscriber = test_manager.subscriber(); + let boundary = u64::from(NU6_3_TRANSITION_BOUNDARY); + let blocks = collect_block_range(subscriber, boundary - 1, boundary, all_pools_i32()).await; + let [last_orchard_block, activation_block] = blocks.as_slice() else { + panic!("expected exactly the two edge blocks, got {}", blocks.len()); + }; + assert_eq!(last_orchard_block.height, boundary - 1); + assert_eq!(activation_block.height, boundary); + let last_orchard_txid = e2e::devtool::txid_from_devtool(&last_orchard_txid); + e2e::assert_pool_present(last_orchard_block, &last_orchard_txid, e2e::Pool::Orchard); + e2e::assert_pool_absent(last_orchard_block, &last_orchard_txid, e2e::Pool::Ironwood); + let first_ironwood_txid = e2e::devtool::txid_from_devtool(&first_ironwood_txid); + e2e::assert_pool_present(activation_block, &first_ironwood_txid, e2e::Pool::Ironwood); + e2e::assert_pool_absent(activation_block, &first_ironwood_txid, e2e::Pool::Orchard); + + test_manager.close().await; +} + +/// Below the boundary, `shield` deposits into the Orchard pool: the faucet +/// funds the recipient's transparent address, the recipient shields, and +/// the shielded balance (net of the ZIP-317 fee, mirroring +/// `devtool.rs::shield_for_validator`) lands in Orchard with the Ironwood +/// pool exactly empty — the era-mirror of the Ironwood-era shield cell. +async fn shield_deposits_to_orchard_before_boundary() +where + Service: TestService, + IndexerError: From<<::Subscriber as ZcashIndexer>::Error>, + ::Subscriber: PollableTip, +{ + let (mut test_manager, mut clients) = + launch_transition_chain_and_fund_faucet::().await; + + // Tip is 2; the transparent receipt confirms at 3 and the shield at 4, + // all below the boundary at 6. + let recipient_t = clients.get_recipient_address("transparent").await; + clients.send_from_faucet(&recipient_t, 250_000).await; + test_manager + .generate_blocks_and_wait_for_tip(1, test_manager.subscriber()) + .await; + clients.sync_recipient().await; + assert_eq!( + e2e::Pool::Transparent.spendable_balance(&clients.recipient_balance().await), + 250_000 + ); + + clients.shield_recipient().await; + test_manager + .generate_blocks_and_wait_for_tip(1, test_manager.subscriber()) + .await; + clients.sync_recipient().await; + + let balance = clients.recipient_balance().await; + assert_eq!(e2e::Pool::Orchard.spendable_balance(&balance), 235_000); + assert_eq!(e2e::Pool::Ironwood.spendable_balance(&balance), 0); + test_manager.close().await; } @@ -221,6 +405,20 @@ mod zebrad { async fn orchard_note_spends_to_ironwood_across_boundary() { crate::orchard_note_spends_to_ironwood_across_boundary::().await; } + + /// multi_thread required: the test manager spawns the validator and + /// indexer services. + #[tokio::test(flavor = "multi_thread")] + async fn receipts_flip_pools_exactly_at_the_boundary() { + crate::receipts_flip_pools_exactly_at_the_boundary::().await; + } + + /// multi_thread required: the test manager spawns the validator and + /// indexer services. + #[tokio::test(flavor = "multi_thread")] + async fn shield_deposits_to_orchard_before_boundary() { + crate::shield_deposits_to_orchard_before_boundary::().await; + } } mod state_service { @@ -239,5 +437,19 @@ mod zebrad { async fn orchard_note_spends_to_ironwood_across_boundary() { crate::orchard_note_spends_to_ironwood_across_boundary::().await; } + + /// multi_thread required: the test manager spawns the validator and + /// indexer services. + #[tokio::test(flavor = "multi_thread")] + async fn receipts_flip_pools_exactly_at_the_boundary() { + crate::receipts_flip_pools_exactly_at_the_boundary::().await; + } + + /// multi_thread required: the test manager spawns the validator and + /// indexer services. + #[tokio::test(flavor = "multi_thread")] + async fn shield_deposits_to_orchard_before_boundary() { + crate::shield_deposits_to_orchard_before_boundary::().await; + } } } diff --git a/packages/zaino-fetch/src/jsonrpsee/response.rs b/packages/zaino-fetch/src/jsonrpsee/response.rs index b63fe58f5..86a6fe4be 100644 --- a/packages/zaino-fetch/src/jsonrpsee/response.rs +++ b/packages/zaino-fetch/src/jsonrpsee/response.rs @@ -1245,6 +1245,21 @@ pub struct BlockObject { pub next_block_hash: Option, } +impl BlockObject { + /// Per-pool chain value balances as of this block, when the validator + /// reports them (`getblock` verbosity 2). + pub fn value_pools(&self) -> Option<&[ChainBalance]> { + self.value_pools.as_deref() + } +} + +impl ChainBalance { + /// The underlying per-pool balance entry. + pub fn balance(&self) -> &GetBlockchainInfoBalance { + &self.0 + } +} + impl TryFrom for zebra_rpc::methods::GetBlock { type Error = zebra_chain::serialization::SerializationError; From 888d0ccb432865390d9c5440168971b0e5e71906 Mon Sep 17 00:00:00 2001 From: zancas Date: Tue, 7 Jul 2026 08:05:13 -0700 Subject: [PATCH 27/58] build(makers): add an ironwood set to the test front door `makers test ironwood` runs the ironwood tests of every set, package then clientless then e2e, reusing the same engines as the other sets with the `all` set's continue-on-failure accumulation. The selection is defined once, as a nextest filter expression naming the two dedicated test binaries plus every ironwood-named test; the CONTEXT.md era-naming convention keeps new ironwood tests inside that net. The engines are invoked directly rather than through the `live` summary runner, which forwards no nextest arguments. Co-Authored-By: Claude Fable 5 --- Makefile.toml | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/Makefile.toml b/Makefile.toml index f38a60907..73f540efe 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -150,17 +150,18 @@ script.main = 'source "./tools/scripts/container-nextest-workspace.sh"' # clear = true fully replaces cargo-make's built-in `test` task (which runs # `cargo test`); without it our `script` would merge with the builtin `command`. clear = true -description = "Run a test set: `makers test [package|e2e|clientless|live|all]` (default: package). Pass --with-zcashd for the zcashd-backed tests." +description = "Run a test set: `makers test [package|e2e|clientless|live|all|ironwood]` (default: package). Pass --with-zcashd for the zcashd-backed tests." script_runner = "bash" script = ''' set -uo pipefail usage() { - echo "usage: makers test [package|e2e|clientless|live|all] [-- nextest args]" >&2 + echo "usage: makers test [package|e2e|clientless|live|all|ironwood] [-- nextest args]" >&2 echo " package (default) packages/* tests, no live validator" >&2 echo " e2e | clientless that single live partition" >&2 echo " live both live partitions + combined summary" >&2 echo " all package then live (everything)" >&2 + echo " ironwood the ironwood tests of every set, package then clientless then e2e" >&2 } # A non-flag first arg selects the set and must be a known name; a flag first @@ -170,7 +171,7 @@ usage() { set="package" if [ "$#" -gt 0 ] && [ "${1#-}" = "$1" ]; then case "$1" in - package|e2e|clientless|live|all) set="$1"; shift ;; + package|e2e|clientless|live|all|ironwood) set="$1"; shift ;; *) echo "makers test: unknown set '$1'" >&2; usage; exit 2 ;; esac fi @@ -187,6 +188,21 @@ case "$set" in makers container-test "$@" || rc=1 cargo run -q --manifest-path tools/test-runner/Cargo.toml --bin live-summary -- "$@" || rc=1 exit "$rc" ;; + ironwood) + # The one definition of the ironwood selection: the two dedicated test + # binaries, plus every test whose name carries the pool's name (the + # CONTEXT.md era-naming convention keeps new ironwood tests inside this + # net). The same expression serves every engine — a selector naming a + # binary a partition lacks simply matches nothing there. The engines are + # invoked directly rather than through the `live` summary runner, which + # forwards no nextest args. Run every set even if an earlier one fails, + # as in `all`. + ironwood_filter='binary(ironwood_activation) | binary(testnet_ironwood_boundary) | test(ironwood)' + rc=0 + makers container-test -E "$ironwood_filter" "$@" || rc=1 + makers live-clientless -E "$ironwood_filter" "$@" || rc=1 + makers live-e2e -E "$ironwood_filter" "$@" || rc=1 + exit "$rc" ;; esac ''' From 8aeb2d339dccdf9fd3f5a715aea25ea2f0478148 Mon Sep 17 00:00:00 2001 From: zancas Date: Tue, 7 Jul 2026 08:29:56 -0700 Subject: [PATCH 28/58] fix(e2e): position the boundary cells from the observed tip and pin the observed migration facts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first container run of the boundary-edge cells falsified two of my assumptions and confirmed a third. All three corrections follow from one root cause plus two observed chain facts. The root cause: TestManager's launch mines an NU-activation block after block 1, so every launch returns at tip 2, and the suite's hand-counted heights sat one block high. The flip test's sends confirmed one era late, and the migration test's pool-value walk stopped one block short of the real migration block — making its "the orchard pool does not shrink" verdict vacuous. Both tests now mine to absolute heights computed from the observed tip, immune to launch-preamble changes. With the walk actually spanning the migration block, the original predicate proves correct and now passes on both backends: the migration block debits the Orchard pool by the full spent note while the Ironwood pool absorbs the coinbase plus the migrated value. The walk reads verbosity-1 block objects rather than verbosity 2, which the fetch backend cannot deserialize (its `tx` entries arrive as maps where zaino-fetch expects txid strings); the value pools ride along at verbosity 1. The flip test's final expectation inverts: a send built one block below the boundary is itself migration-shaped — it spends an Orchard note, the faucet's only spendable kind at that tip — so its compact form correctly carries the Orchard spend's data alongside the Ironwood receipt. The walk and balance assertions now carry their observed values in their failure messages, which is what made this diagnosis one-pass. Co-Authored-By: Claude Fable 5 --- live-tests/e2e/tests/ironwood_activation.rs | 92 +++++++++++++++------ 1 file changed, 68 insertions(+), 24 deletions(-) diff --git a/live-tests/e2e/tests/ironwood_activation.rs b/live-tests/e2e/tests/ironwood_activation.rs index eba7c5ef1..fd9f86b82 100644 --- a/live-tests/e2e/tests/ironwood_activation.rs +++ b/live-tests/e2e/tests/ironwood_activation.rs @@ -57,8 +57,12 @@ use zcash_local_net::validator::zebrad::Zebrad; /// Launch an orchard-receiver-mining zebrad + Zaino on the transition /// heights, build devtool faucet/recipient wallets (their schedule derived -/// from the launched validator), mine one block (height 2: the first -/// Orchard-era coinbase note), and sync the faucet. The transition-fixture +/// from the launched validator), mine one block, and sync the faucet. The +/// launch itself already leaves the tip at 2 (`TestManager` mines an +/// NU-activation block after block 1), so the helper returns at tip 3 with +/// the faucet holding the Orchard coinbase notes of heights 2 and 3. Tests +/// that need exact boundary positioning mine to absolute heights from the +/// observed tip rather than counting from here. The transition-fixture /// analogue of `devtool.rs::launch_and_fund_faucet`. async fn launch_transition_chain_and_fund_faucet( ) -> (TestManager, DevtoolClients) @@ -98,25 +102,28 @@ where } /// The chain value (in zatoshis) of the pool named `pool_id` as of -/// `height`, read from the served verbosity-2 block object — the same -/// per-height `valuePools` a zcashd `getblock` reports. +/// `height`, read from the served verbosity-1 block object — the same +/// per-height `valuePools` a zcashd `getblock` reports. Verbosity 1, not 2: +/// the fetch backend cannot deserialize a verbosity-2 block object (its +/// `tx` entries are maps where zaino-fetch expects txid strings), and the +/// value pools ride along at verbosity 1. async fn pool_zats_at_height(subscriber: &S, height: u32, pool_id: &str) -> i64 where S: ZcashIndexer, IndexerError: From, { let response = subscriber - .z_get_block(height.to_string(), Some(2)) + .z_get_block(height.to_string(), Some(1)) .await .map_err(IndexerError::from) - .expect("z_get_block verbosity 2"); + .expect("z_get_block verbosity 1"); let zebra_rpc::methods::GetBlock::Object(block) = response else { - panic!("verbosity-2 getblock must return a block object"); + panic!("verbosity-1 getblock must return a block object"); }; let pools = block .value_pools() .as_ref() - .expect("verbosity-2 block object carries value pools"); + .expect("verbosity-1 block object carries value pools"); pools .iter() .find(|pool| pool.id() == pool_id) @@ -139,8 +146,8 @@ where let (mut test_manager, mut clients) = launch_transition_chain_and_fund_faucet::().await; - // Tip is 2: inside the Orchard era, with room to confirm the send at - // height 3 while staying below the boundary at 6. + // Tip is 3: inside the Orchard era, with room to confirm the send at + // height 4 while staying below the boundary at 6. let faucet_balance = clients.faucet_balance().await; assert!( faucet_balance.orchard_spendable > 0, @@ -191,11 +198,19 @@ where "pre-boundary coinbase should be an orchard note, got {pre_boundary_balance:?}" ); - // Tip is 2; mine to the boundary itself. Heights 3–5 add more Orchard - // coinbase notes, height 6 is the first Ironwood-era block (its coinbase - // is the faucet's first Ironwood note). + // Mine to the boundary itself, from the observed tip rather than a + // hand-count. The blocks below the boundary add more Orchard coinbase + // notes; the boundary block is the first Ironwood-era block, and its + // coinbase is the faucet's first Ironwood note. + let tip = u32::try_from(test_manager.subscriber().tip_height().await) + .expect("regtest tips fit in u32"); test_manager - .generate_blocks_and_wait_for_tip(NU6_3_TRANSITION_BOUNDARY - 2, test_manager.subscriber()) + .generate_blocks_and_wait_for_tip( + NU6_3_TRANSITION_BOUNDARY + .checked_sub(tip) + .expect("the launch preamble must leave room below the boundary"), + test_manager.subscriber(), + ) .await; clients.sync_faucet().await; let crossed_balance = clients.faucet_balance().await; @@ -250,11 +265,18 @@ where } orchard_at.push(orchard); } + for (index, orchard) in orchard_at.iter().enumerate() { + let height = index + 1; + let ironwood = pool_zats_at_height(subscriber, height as u32, "ironwood").await; + eprintln!("pool values at height {height}: orchard={orchard} ironwood={ironwood}"); + } let orchard = |height: u32| orchard_at[height as usize - 1]; for height in 2..boundary { assert!( orchard(height) > orchard(height - 1), - "each pre-boundary coinbase must grow the orchard pool (height {height})" + "each pre-boundary coinbase must grow the orchard pool (height {height}: {} after {})", + orchard(height), + orchard(height - 1) ); } assert_eq!( @@ -268,7 +290,9 @@ where ); assert!( orchard(migration_height) < orchard(boundary), - "the migration block must shrink the orchard pool" + "the migration block must shrink the orchard pool ({} at the boundary, {} at the migration block)", + orchard(boundary), + orchard(migration_height) ); test_manager.close().await; @@ -291,11 +315,18 @@ where let (mut test_manager, mut clients) = launch_transition_chain_and_fund_faucet::().await; - // Tip is 2. Mine heights 3 and 4 (a second spendable Orchard note and a - // filler), so the two sends below confirm at exactly boundary − 1 and - // the boundary. + // Position the tip at exactly boundary − 2, from the observed tip + // rather than a hand-count, so the two sends below confirm at exactly + // boundary − 1 and the boundary. + let tip = u32::try_from(test_manager.subscriber().tip_height().await) + .expect("regtest tips fit in u32"); test_manager - .generate_blocks_and_wait_for_tip(NU6_3_TRANSITION_BOUNDARY - 4, test_manager.subscriber()) + .generate_blocks_and_wait_for_tip( + NU6_3_TRANSITION_BOUNDARY + .checked_sub(2 + tip) + .expect("the launch preamble must leave room below the boundary"), + test_manager.subscriber(), + ) .await; clients.sync_faucet().await; @@ -309,8 +340,16 @@ where clients.sync_faucet().await; clients.sync_recipient().await; let balance = clients.recipient_balance().await; - assert_eq!(e2e::Pool::Orchard.spendable_balance(&balance), 250_000); - assert_eq!(e2e::Pool::Ironwood.spendable_balance(&balance), 0); + assert_eq!( + e2e::Pool::Orchard.spendable_balance(&balance), + 250_000, + "receipt confirmed at boundary - 1 must be orchard, got {balance:?}" + ); + assert_eq!( + e2e::Pool::Ironwood.spendable_balance(&balance), + 0, + "no ironwood receipt below the boundary, got {balance:?}" + ); // Built at tip boundary − 1, confirmed in block 6: the activation block. let first_ironwood_txid = clients.send_from_faucet(&recipient, 250_000).await; @@ -340,7 +379,12 @@ where e2e::assert_pool_absent(last_orchard_block, &last_orchard_txid, e2e::Pool::Ironwood); let first_ironwood_txid = e2e::devtool::txid_from_devtool(&first_ironwood_txid); e2e::assert_pool_present(activation_block, &first_ironwood_txid, e2e::Pool::Ironwood); - e2e::assert_pool_absent(activation_block, &first_ironwood_txid, e2e::Pool::Orchard); + // The second send is itself migration-shaped: built one block below the + // boundary, it spends an Orchard note (the faucet's only spendable kind + // at that tip), so its compact form carries the Orchard spend's data + // alongside the Ironwood receipt. The receipt's pool routing is what + // flips at the boundary, and that is asserted at the wallet tier above. + e2e::assert_pool_present(activation_block, &first_ironwood_txid, e2e::Pool::Orchard); test_manager.close().await; } @@ -359,7 +403,7 @@ where let (mut test_manager, mut clients) = launch_transition_chain_and_fund_faucet::().await; - // Tip is 2; the transparent receipt confirms at 3 and the shield at 4, + // Tip is 3; the transparent receipt confirms at 4 and the shield at 5, // all below the boundary at 6. let recipient_t = clients.get_recipient_address("transparent").await; clients.send_from_faucet(&recipient_t, 250_000).await; From 6402edd708e6e1c5333e6a10a4d865f17ffa5d7a Mon Sep 17 00:00:00 2001 From: zancas Date: Tue, 7 Jul 2026 10:49:16 -0700 Subject: [PATCH 29/58] test(zaino-fetch): pin the verbosity-2 getblock transaction-object shape (known red) A verbosity-2 `getblock` response carries full transaction objects in its `tx` array, where a verbosity-1 response carries txid strings. zaino-fetch's `BlockObject` types the array for the verbosity-1 shape only, so the object form fails to deserialize and the fetch backend cannot serve verbosity-2 blocks at all; the state backend serves them correctly, so the defect also breaks backend parity. This pins both shapes: the verbosity-1 test guards the behavior the fix must preserve, and the verbosity-2 test is gated `should_panic` on the failure until the fix lands (issue #1380). The verbosity-2 fixture is built by round-tripping a real test-vector transaction through zebra-rpc's own response types, so it reproduces zebrad's wire shape by construction and cannot drift from it. Co-Authored-By: Claude Fable 5 --- .../zaino-fetch/src/jsonrpsee/response.rs | 86 +++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/packages/zaino-fetch/src/jsonrpsee/response.rs b/packages/zaino-fetch/src/jsonrpsee/response.rs index b63fe58f5..1944a5b50 100644 --- a/packages/zaino-fetch/src/jsonrpsee/response.rs +++ b/packages/zaino-fetch/src/jsonrpsee/response.rs @@ -2028,3 +2028,89 @@ mod get_transaction_response { ); } } + +#[cfg(test)] +mod get_block_response { + use super::GetBlockResponse; + + /// Builds the `tx` entry of a verbosity-2 `getblock` response exactly as + /// zebrad serializes it, by round-tripping a real test-vector + /// transaction through zebra-rpc's own response types. Constructing the + /// fixture from zebra's serializer keeps it faithful to the wire across + /// zebra upgrades, with no hand-maintained JSON to drift. + fn verbosity_2_transaction_json() -> serde_json::Value { + use zebra_chain::serialization::ZcashDeserializeInto as _; + + let vector = wire_serialized_transaction_test_data::transactions::get_test_vectors() + .into_iter() + .next() + .expect("the test-vector crate provides at least one transaction"); + let transaction: zebra_chain::transaction::Transaction = vector + .tx + .as_slice() + .zcash_deserialize_into() + .expect("test-vector transactions deserialize"); + let txid = transaction.hash(); + let tx_object = zebra_rpc::client::TransactionObject::from_transaction( + std::sync::Arc::new(transaction), + Some(zebra_chain::block::Height(2)), + Some(1), + &zebra_chain::parameters::Network::Mainnet, + None, + None, + None, + txid, + ); + serde_json::to_value(zebra_rpc::methods::GetBlockTransaction::Object(Box::new( + tx_object, + ))) + .expect("zebra's response types serialize") + } + + fn block_json_with_tx(tx: serde_json::Value) -> serde_json::Value { + serde_json::json!({ + "hash": "0000000000000000000000000000000000000000000000000000000000000002", + "confirmations": 1, + "tx": [tx], + "trees": {}, + }) + } + + /// A verbosity-1 `getblock` response carries txid strings in `tx`. This + /// pins the shape the type already handles, so the verbosity-2 fix below + /// cannot regress it. + #[test] + fn parses_verbosity_1_txid_strings() { + let block_json = block_json_with_tx(serde_json::json!( + "e85e5729b34c34e8e62aa639302cb3434bb961666e13191da6dc39fcf775b320" + )); + + let parsed: GetBlockResponse = + serde_json::from_value(block_json).expect("verbosity-1 block objects deserialize"); + let GetBlockResponse::Object(block) = parsed else { + panic!("a JSON block object must parse as the object variant"); + }; + assert_eq!(block.tx.len(), 1); + } + + /// A verbosity-2 `getblock` response carries full transaction objects in + /// `tx`; the response type must deserialize that shape as well. (known + /// red: the `tx` field is typed for the verbosity-1 shape only, so the + /// object form fails and the fetch backend cannot serve verbosity-2 + /// blocks at all; issue #1380. The connector surfaces the failure as + /// `invalid type: map, expected a string` against `Box`; + /// through the untagged response enum it appears as the variant + /// mismatch below.) + #[test] + #[should_panic(expected = "data did not match any variant of untagged enum GetBlockResponse")] + fn parses_verbosity_2_transaction_objects() { + let block_json = block_json_with_tx(verbosity_2_transaction_json()); + + let parsed: GetBlockResponse = + serde_json::from_value(block_json).expect("verbosity-2 block objects deserialize"); + let GetBlockResponse::Object(block) = parsed else { + panic!("a JSON block object must parse as the object variant"); + }; + assert_eq!(block.tx.len(), 1); + } +} From 576b72df07f4d753a9c46fa31c0fd3d37e281e65 Mon Sep 17 00:00:00 2001 From: zancas Date: Tue, 7 Jul 2026 10:51:16 -0700 Subject: [PATCH 30/58] fix(zaino-fetch): deserialize verbosity-2 getblock transaction objects The `tx` field of zaino-fetch's `BlockObject` becomes a vector of zebra-rpc's `GetBlockTransaction`, the untagged enum that already models both wire shapes: hex-encoded transaction IDs at verbosity 1 and full transaction objects at verbosity 2. The fetch backend previously typed the field for the verbosity-1 shape only, so a verbosity-2 response failed to deserialize and the backend could not serve it at all, while the state backend served the same request correctly through zebra's own types (issue #1380). The change is DRY by deletion. Reusing zebra's enum removes zaino's parallel string representation of the same wire data, and the `TryFrom` conversion shrinks from a fallible txid-by-txid re-parse into a straight pass-through, taking its only error path with it. Verbosity-1 behavior is unchanged: the enum's `Hash` variant serializes to the same hex string the field produced before, and the precursor commit's verbosity-1 test pins that shape. The verbosity-2 test sheds its known-red gate and now also asserts the round trip: the parsed entry is a transaction object, and the conversion into zebra-rpc's `GetBlock` passes the object through intact. Closes #1380. Co-Authored-By: Claude Fable 5 --- .../zaino-fetch/src/jsonrpsee/response.rs | 54 +++++++++++-------- 1 file changed, 31 insertions(+), 23 deletions(-) diff --git a/packages/zaino-fetch/src/jsonrpsee/response.rs b/packages/zaino-fetch/src/jsonrpsee/response.rs index 1944a5b50..18248f5c8 100644 --- a/packages/zaino-fetch/src/jsonrpsee/response.rs +++ b/packages/zaino-fetch/src/jsonrpsee/response.rs @@ -1213,8 +1213,10 @@ pub struct BlockObject { #[serde(default, skip_serializing_if = "Option::is_none")] pub difficulty: Option, - /// List of transaction IDs in block order, hex-encoded. - pub tx: Vec, + /// The block's transactions in block order: hex-encoded transaction IDs + /// at verbosity 1, full transaction objects at verbosity 2. Both shapes + /// share zebra-rpc's untagged enum. + pub tx: Vec, /// Chain supply balance #[serde(default)] @@ -1254,15 +1256,7 @@ impl TryFrom for zebra_rpc::methods::GetBlock { Ok(zebra_rpc::methods::GetBlock::Raw(serialized_block.0)) } GetBlockResponse::Object(block) => { - let tx: Result, _> = block - .tx - .into_iter() - .map(|txid| { - txid.parse::() - .map(zebra_rpc::methods::GetBlockTransaction::Hash) - }) - .collect(); - let tx = tx?; + let tx = block.tx; Ok(zebra_rpc::methods::GetBlock::Object(Box::new( zebra_rpc::client::BlockObject::new( @@ -2042,8 +2036,8 @@ mod get_block_response { use zebra_chain::serialization::ZcashDeserializeInto as _; let vector = wire_serialized_transaction_test_data::transactions::get_test_vectors() - .into_iter() - .next() + .first() + .cloned() .expect("the test-vector crate provides at least one transaction"); let transaction: zebra_chain::transaction::Transaction = vector .tx @@ -2094,23 +2088,37 @@ mod get_block_response { } /// A verbosity-2 `getblock` response carries full transaction objects in - /// `tx`; the response type must deserialize that shape as well. (known - /// red: the `tx` field is typed for the verbosity-1 shape only, so the - /// object form fails and the fetch backend cannot serve verbosity-2 - /// blocks at all; issue #1380. The connector surfaces the failure as - /// `invalid type: map, expected a string` against `Box`; - /// through the untagged response enum it appears as the variant - /// mismatch below.) + /// `tx`; the response type must deserialize that shape as well, and the + /// conversion into zebra-rpc's `GetBlock` must preserve the objects + /// (issue #1380). #[test] - #[should_panic(expected = "data did not match any variant of untagged enum GetBlockResponse")] fn parses_verbosity_2_transaction_objects() { let block_json = block_json_with_tx(verbosity_2_transaction_json()); let parsed: GetBlockResponse = serde_json::from_value(block_json).expect("verbosity-2 block objects deserialize"); - let GetBlockResponse::Object(block) = parsed else { + let GetBlockResponse::Object(ref block) = parsed else { panic!("a JSON block object must parse as the object variant"); }; - assert_eq!(block.tx.len(), 1); + assert!( + matches!( + block.tx.as_slice(), + [zebra_rpc::methods::GetBlockTransaction::Object(_)] + ), + "the verbosity-2 entry must parse as a transaction object" + ); + + let converted: zebra_rpc::methods::GetBlock = + parsed.try_into().expect("the object form converts"); + let zebra_rpc::methods::GetBlock::Object(converted) = converted else { + panic!("the conversion must keep the object variant"); + }; + assert!( + matches!( + converted.tx().as_slice(), + [zebra_rpc::methods::GetBlockTransaction::Object(_)] + ), + "the conversion must pass the transaction object through" + ); } } From 63dc84ce2ac6a060352b071b91e179cf93077fe1 Mon Sep 17 00:00:00 2001 From: zancas Date: Tue, 7 Jul 2026 12:06:06 -0700 Subject: [PATCH 31/58] refactor(state): apply ironwood-migration polish from review of #1379 This applies the six non-blocking follow-ups tracked in #1382. Backfill errors now carry remediation advice: when the v1.2.1 to v1.3.0 migration cannot refetch a post-NU6.3 block, the error tells the operator to ensure the backing validator serves the range from NU6.3 activation through the database tip, or to wipe and re-index. The idempotent-write helper is hoisted to module scope and reused at all three migration sites that previously inlined the NO_OVERWRITE plus verify-match pattern (the spent-batch flush, the txid_location rebuild, and the v1.3.0 commitment and ironwood writes). It now takes a lazy describe closure, so the per-row success path allocates nothing; the previous version formatted two messages per height on the success path. At the txid_location site the byte-for-byte comparison subsumes the old decode-and-checksum check, since a corrupt existing row cannot match the rebuilt bytes. The three shielded-pool activation heights are resolved through a new PoolActivationHeights struct in finalised_state.rs, shared by write_blocks_to_height and the migration so the two call sites cannot drift apart. The accessors introduced by #1379 (FinalisedSource::start_validator and the two ironwood_db accessors, plus DbV1::start_validator) narrow from pub(crate) to pub(super), the minimum visibility that compiles. The v1_2_1_cache_migrates_to_current_then_validates test gains a comment justifying its multi_thread flavor: the background validator runs blocking LMDB validation inline on its task, which would starve the test's status polling on a current-thread runtime. Fixes #1382. Co-Authored-By: Claude Fable 5 --- .../src/chain_index/finalised_state.rs | 27 +++ .../finalised_state/finalised_source.rs | 4 +- .../finalised_state/finalised_source/v1.rs | 4 +- .../finalised_source/v1/write_core.rs | 20 +- .../chain_index/finalised_state/migrations.rs | 175 ++++++++---------- .../migrations/v1_2_to_v1_3.rs | 3 + 6 files changed, 115 insertions(+), 118 deletions(-) diff --git a/packages/zaino-state/src/chain_index/finalised_state.rs b/packages/zaino-state/src/chain_index/finalised_state.rs index 2873e6e49..a44aa7e63 100644 --- a/packages/zaino-state/src/chain_index/finalised_state.rs +++ b/packages/zaino-state/src/chain_index/finalised_state.rs @@ -243,6 +243,33 @@ use crate::{ use std::{sync::Arc, time::Duration}; use tokio::time::{interval, MissedTickBehavior}; +/// The activation heights of the three shielded pools whose data +/// [`build_indexed_block_from_source`] assembles, resolved once per run. +/// +/// Both the ingestion loop ([`capability::DbWrite::write_blocks_to_height`]) and the v1.2.1 → +/// v1.3.0 migration backfill need exactly this set of heights; resolving them in one place keeps +/// the two call sites from drifting apart. A `None` height means the pool's network upgrade has no +/// activation height on the given network. +struct PoolActivationHeights { + sapling: Option, + nu5: Option, + nu6_3: Option, +} + +impl PoolActivationHeights { + /// Resolves the Sapling, NU5 (Orchard), and NU6.3 (Ironwood) activation heights on + /// `zebra_network`. + fn resolve(zebra_network: &zebra_chain::parameters::Network) -> Self { + let activation_height = + |pool: super::ShieldedPool| pool.activation_upgrade().activation_height(zebra_network); + Self { + sapling: activation_height(super::ShieldedPool::Sapling), + nu5: activation_height(super::ShieldedPool::Orchard), + nu6_3: activation_height(super::ShieldedPool::Ironwood), + } + } +} + /// Fetches the block at `height_int` from `source` and builds its [`IndexedBlock`], threading /// `parent_chainwork` into the block metadata. /// diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source.rs index 70a9c7137..d2aeb1cce 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source.rs @@ -288,7 +288,7 @@ impl FinalisedSource { /// /// Called by the orchestrator only once all pending migrations have completed, so the /// validator never races a migration that populates the tables its initial scan reads. - pub(crate) fn start_validator(&self) { + pub(super) fn start_validator(&self) { match self { Self::V1(db) => db.start_validator(), Self::Ephemeral(_) => {} @@ -383,7 +383,7 @@ impl FinalisedSource { /// Provides access to the (v1.3.0) `ironwood` table, required for Migration1_2_1To1_3_0 to /// backfill ironwood rows from validator-fetched block data. - pub(crate) fn ironwood_db(&self) -> Result { + pub(super) fn ironwood_db(&self) -> Result { Ok(self.require_v1("v1 ironwood db")?.ironwood_db()) } } diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1.rs index 835579784..56256f242 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1.rs @@ -606,7 +606,7 @@ impl DbV1 { /// migrations have finished (the validator reads tables a migration populates). Takes `&self` /// (the join handle lives behind a `Mutex`) so it can be driven through the shared /// `Arc` the router holds after spawn. - pub(crate) fn start_validator(&self) { + pub(super) fn start_validator(&self) { // Clone everything the task needs so we can move it into the async block. let zaino_db = self.detached_handle(); @@ -852,7 +852,7 @@ impl DbV1 { /// Provides access to the (v1.3.0) `ironwood` table, required for Migration1_2_1To1_3_0 to /// backfill ironwood rows for post-NU6.3 blocks from validator-fetched block data. - pub(crate) fn ironwood_db(&self) -> Database { + pub(super) fn ironwood_db(&self) -> Database { self.ironwood } diff --git a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/write_core.rs b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/write_core.rs index 5c3cfc07e..8ca13c12e 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/write_core.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/finalised_source/v1/write_core.rs @@ -2,8 +2,6 @@ use super::*; -use crate::chain_index::ShieldedPool; - #[cfg(feature = "prometheus")] use crate::metric_names::*; @@ -231,20 +229,18 @@ impl DbWrite for DbV1 { height: Height, source: &S, ) -> Result<(), FinalisedStateError> { - use crate::chain_index::finalised_state::build_indexed_block_from_source; + use crate::chain_index::finalised_state::{ + build_indexed_block_from_source, PoolActivationHeights, + }; let network = self.config.network; let zebra_network = network.to_zebra_network(); - let sapling_activation_height = ShieldedPool::Sapling - .activation_upgrade() - .activation_height(&zebra_network) + let pool_activations = PoolActivationHeights::resolve(&zebra_network); + let sapling_activation_height = pool_activations + .sapling .expect("Sapling activation height must be set"); - let nu5_activation_height = ShieldedPool::Orchard - .activation_upgrade() - .activation_height(&zebra_network); - let nu6_3_activation_height = ShieldedPool::Ironwood - .activation_upgrade() - .activation_height(&zebra_network); + let nu5_activation_height = pool_activations.nu5; + let nu6_3_activation_height = pool_activations.nu6_3; // Seed `parent_chainwork` from the current tip header (the block before the first one we // write). On an empty database this is genesis with zero chainwork. Read raw rather than via diff --git a/packages/zaino-state/src/chain_index/finalised_state/migrations.rs b/packages/zaino-state/src/chain_index/finalised_state/migrations.rs index ef414f9d1..57b4cd00d 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/migrations.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/migrations.rs @@ -491,6 +491,34 @@ impl Migration for Migration1_0_0To1_1_0 { /// - No shadow database. struct Migration1_1_0To1_2_0; +/// Writes `value` under `key` with `NO_OVERWRITE`, tolerating an existing byte-identical row. +/// +/// Migrations use this to stay idempotent on crash-resume: a resumed pass may revisit rows it has +/// already committed, and each such row must match the rebuilt bytes exactly. Any difference — +/// a conflicting rebuild or a corrupt existing row — aborts the migration with the message +/// `describe` produces. `describe` runs only on that error path, so the per-row success path +/// allocates nothing. +fn put_idempotent( + txn: &mut lmdb::RwTransaction<'_>, + db: lmdb::Database, + key: &[u8], + value: &[u8], + describe: impl FnOnce() -> String, +) -> Result<(), FinalisedStateError> { + match txn.put(db, &key, &value, WriteFlags::NO_OVERWRITE) { + Ok(()) => Ok(()), + Err(lmdb::Error::KeyExist) => { + let existing = txn.get(db, &key).map_err(FinalisedStateError::LmdbError)?; + if existing == value { + Ok(()) + } else { + Err(FinalisedStateError::Custom(describe())) + } + } + Err(error) => Err(FinalisedStateError::LmdbError(error)), + } +} + /// Flushes a buffered batch of `spent` index entries in sorted key order, then commits them /// together with the Stage B progress watermark and fsyncs. /// @@ -511,26 +539,12 @@ fn flush_migration_spent_batch( let mut txn = env.begin_rw_txn()?; for (outpoint_bytes, tx_location) in buffer.iter() { let entry_bytes = StoredEntryFixed::new(outpoint_bytes, *tx_location).to_bytes()?; - match txn.put( - spent_db, - outpoint_bytes, - &entry_bytes, - WriteFlags::NO_OVERWRITE, - ) { - Ok(()) => {} - Err(lmdb::Error::KeyExist) => { - let existing = txn - .get(spent_db, outpoint_bytes) - .map_err(FinalisedStateError::LmdbError)?; - if existing != entry_bytes { - return Err(FinalisedStateError::Custom(format!( - "conflicting existing spent entry during batched migration for outpoint {}", - hex::encode(outpoint_bytes) - ))); - } - } - Err(error) => return Err(FinalisedStateError::LmdbError(error)), - } + put_idempotent(&mut txn, spent_db, outpoint_bytes, &entry_bytes, || { + format!( + "conflicting existing spent entry during batched migration for outpoint {}", + hex::encode(outpoint_bytes) + ) + })?; } let progress = StoredEntryFixed::new(progress_key, up_to_height + 1); @@ -689,42 +703,20 @@ impl Migration for Migration1_1_0To1_2_0 { let entry_bytes = StoredEntryFixed::new(txid_bytes, *tx_location).to_bytes()?; - match txn.put( + // Idempotent on resume: an existing entry must match byte-for-byte, which + // also rejects a corrupt existing row (its checksum bytes would differ). + put_idempotent( + &mut txn, txid_location_db, txid_bytes, &entry_bytes, - WriteFlags::NO_OVERWRITE, - ) { - Ok(()) => {} - - // Idempotent on resume: an existing entry must match exactly. - Err(lmdb::Error::KeyExist) => { - let existing_bytes = txn - .get(txid_location_db, txid_bytes) - .map_err(FinalisedStateError::LmdbError)?; - let existing_entry = - StoredEntryFixed::::from_bytes(existing_bytes) - .map_err(|error| { - FinalisedStateError::Custom(format!( - "corrupt existing txid_location entry: {error}" - )) - })?; - if !existing_entry.verify(txid_bytes) { - return Err(FinalisedStateError::Custom( - "existing txid_location entry checksum mismatch" - .to_string(), - )); - } - if existing_entry.inner() != tx_location { - return Err(FinalisedStateError::Custom(format!( - "conflicting txid_location entry at height {}", - height.0 - ))); - } - } - - Err(error) => return Err(FinalisedStateError::LmdbError(error)), - } + || { + format!( + "conflicting or corrupt existing txid_location entry at height {}", + height.0 + ) + }, + )?; } let progress = @@ -1069,9 +1061,8 @@ impl Migration for Migration1_2_1To1_3_0 { use crate::chain_index::finalised_state::{ build_indexed_block_from_source, - finalised_source::v1::write_core::build_block_ironwood_entry, + finalised_source::v1::write_core::build_block_ironwood_entry, PoolActivationHeights, }; - use crate::chain_index::ShieldedPool; // Temporary metadata entry recording the next height to rebuild, removed on completion. const MIGRATION_CTD_PROGRESS_KEY: &[u8] = @@ -1100,21 +1091,15 @@ impl Migration for Migration1_2_1To1_3_0 { ) .await?; - // Network-upgrade activation heights (mirrors `write_blocks_to_height`). Blocks at or above - // NU6.3 have their ironwood root/size and ironwood tx list rebuilt from the validator (the - // legacy on-disk data predates ironwood); blocks below it are rebuilt in place from the - // legacy commitment row, with no ironwood. + // Network-upgrade activation heights (shared with `write_blocks_to_height`). Blocks at or + // above NU6.3 have their ironwood root/size and ironwood tx list rebuilt from the validator + // (the legacy on-disk data predates ironwood); blocks below it are rebuilt in place from + // the legacy commitment row, with no ironwood. let network = cfg.network; - let zebra_network = network.to_zebra_network(); - let sapling_activation_height = ShieldedPool::Sapling - .activation_upgrade() - .activation_height(&zebra_network); - let nu5_activation_height = ShieldedPool::Orchard - .activation_upgrade() - .activation_height(&zebra_network); - let nu6_3_activation_height = ShieldedPool::Ironwood - .activation_upgrade() - .activation_height(&zebra_network); + let pool_activations = PoolActivationHeights::resolve(&network.to_zebra_network()); + let sapling_activation_height = pool_activations.sapling; + let nu5_activation_height = pool_activations.nu5; + let nu6_3_activation_height = pool_activations.nu6_3; // Mark migration in progress (observability only; resumption uses the progress key). { @@ -1161,30 +1146,6 @@ impl Migration for Migration1_2_1To1_3_0 { ); let started = std::time::Instant::now(); - // Idempotent write: on resume an already-written row must match byte-for-byte (the - // legacy rebuild is deterministic and the validator refetch is over immutable history). - fn put_idempotent( - txn: &mut lmdb::RwTransaction<'_>, - db: lmdb::Database, - key: &[u8], - value: &[u8], - what: &str, - ) -> Result<(), FinalisedStateError> { - match txn.put(db, &key, &value, WriteFlags::NO_OVERWRITE) { - Ok(()) => Ok(()), - Err(lmdb::Error::KeyExist) => { - let existing = txn.get(db, &key).map_err(FinalisedStateError::LmdbError)?; - if existing != value { - return Err(FinalisedStateError::Custom(format!( - "conflicting rebuilt {what}" - ))); - } - Ok(()) - } - Err(error) => Err(FinalisedStateError::LmdbError(error)), - } - } - while next_height <= db_tip { let height = Height::try_from(next_height) .map_err(|error| FinalisedStateError::Custom(error.to_string()))?; @@ -1218,7 +1179,16 @@ impl Migration for Migration1_2_1To1_3_0 { // are extracted, and neither depends on it. None, ) - .await?; + .await + .map_err(|error| { + FinalisedStateError::Custom(format!( + "v1.3.0 ironwood backfill failed at height {next_height}: {error}. \ + This backfill refetches every stored block from NU6.3 activation \ + through the database tip; ensure the backing validator serves that \ + range, or wipe the finalised-state directory and re-index from the \ + validator." + )) + })?; commitment_bytes = StoredEntryVar::new(&height_bytes, *block.commitment_tree_data()) @@ -1261,16 +1231,17 @@ impl Migration for Migration1_2_1To1_3_0 { new_ctd_db, &height_bytes, &commitment_bytes, - &format!("commitment_tree_data at height {}", height.0), + || { + format!( + "conflicting rebuilt commitment_tree_data at height {}", + height.0 + ) + }, )?; if let Some(bytes) = &ironwood_bytes { - put_idempotent( - &mut txn, - ironwood_db, - &height_bytes, - bytes, - &format!("ironwood at height {}", height.0), - )?; + put_idempotent(&mut txn, ironwood_db, &height_bytes, bytes, || { + format!("conflicting rebuilt ironwood at height {}", height.0) + })?; } let progress = StoredEntryFixed::new(MIGRATION_CTD_PROGRESS_KEY, height + 1); diff --git a/packages/zaino-state/src/chain_index/tests/finalised_state/migrations/v1_2_to_v1_3.rs b/packages/zaino-state/src/chain_index/tests/finalised_state/migrations/v1_2_to_v1_3.rs index 5d215be9c..0c7756e06 100644 --- a/packages/zaino-state/src/chain_index/tests/finalised_state/migrations/v1_2_to_v1_3.rs +++ b/packages/zaino-state/src/chain_index/tests/finalised_state/migrations/v1_2_to_v1_3.rs @@ -39,6 +39,9 @@ fn v1_2_1() -> DbVersion { /// database, then reopens it through the production `FinalisedState::spawn` path (which migrates to /// the current schema and only then starts the validator) and asserts it reaches `Ready` — i.e. the /// migration completed *before* validation, and validation then passed. +// multi_thread required: the background validator runs blocking LMDB validation +// (`validate_block_blocking`) inline on its task, which would starve this test's status polling on +// a current-thread runtime. #[tokio::test(flavor = "multi_thread")] async fn v1_2_1_cache_migrates_to_current_then_validates() { init_tracing(); From 083ced1efc5284218bde941e4b0188434edd3d3a Mon Sep 17 00:00:00 2001 From: zancas Date: Tue, 7 Jul 2026 21:52:25 -0700 Subject: [PATCH 32/58] test!: expect derived Sapling keys from z_validateaddress on all connections The merged NodeBackedIndexerService computes z_validateaddress locally on both connection types, so the zebrad Rpc path now returns the derived diversifier and diversified transmission key instead of zebrad's passthrough response, which omitted them. This failed the pinned expectation in clientless::fetch_service zebrad::validation::z_validate_address deterministically. Adopt the local-computation behavior as the single expectation: fold the Sapling assertion (keys present) into run_z_validate_suite, delete the SaplingSuite enum with its ZebradPassthroughFetchService variant and the passthrough-specific suite function, and drop the suite parameter from run_z_validate_for and its four call sites. The old "safer behavior" rationale for omitting the keys addressed delegation to a remote actor and does not apply to a local derivation; the RPC's deprecation signal is unaffected because the shared helper zaino_state::indexer::z_validate_address still warns with Z_VALIDATE_DEPRECATION on every call. See https://github.com/zingolabs/zaino/pull/1361#issuecomment-4908129940 for the failure analysis. Co-Authored-By: Claude Fable 5 --- live-tests/clientless/src/lib.rs | 75 ++++---------------- live-tests/clientless/tests/fetch_service.rs | 14 ++-- live-tests/clientless/tests/json_server.rs | 7 +- live-tests/clientless/tests/state_service.rs | 7 +- 4 files changed, 24 insertions(+), 79 deletions(-) diff --git a/live-tests/clientless/src/lib.rs b/live-tests/clientless/src/lib.rs index 9e7314757..12d93f9be 100644 --- a/live-tests/clientless/src/lib.rs +++ b/live-tests/clientless/src/lib.rs @@ -83,7 +83,17 @@ pub mod rpc { // assert_eq!(fs_sprout, expected_sprout); - // Sapling (differs by validator) + // Sapling + let expected_sapling = ValidZValidateAddress::sapling( + VALID_SAPLING_ADDRESS.to_string(), + Some(VALID_DIVERSIFIER.to_string()), + Some(VALID_DIVERSIFIED_TRANSMISSION_KEY.to_string()), + ); + assert_known_valid_eq( + rpc_call(VALID_SAPLING_ADDRESS.to_string()).await, + expected_sapling, + "Sapling", + ); // Unified (differs by validator) let expected_unified = @@ -101,70 +111,15 @@ pub mod rpc { assert_eq!(all_zeroes, ZValidateAddressResponse::invalid()); } - pub async fn run_z_validate_sapling(rpc_call: &F) - where - F: Fn(String) -> Fut, - Fut: Future, - { - let expected_sapling = ValidZValidateAddress::sapling( - VALID_SAPLING_ADDRESS.to_string(), - Some(VALID_DIVERSIFIER.to_string()), - Some(VALID_DIVERSIFIED_TRANSMISSION_KEY.to_string()), - ); - assert_known_valid_eq( - rpc_call(VALID_SAPLING_ADDRESS.to_string()).await, - expected_sapling, - "Sapling", - ); - } - - /// zebrad's JSON-RPC passthrough (via FetchService) omits `diversifier` - /// and `diversifiedtransmissionkey` from the Sapling response. This is - /// the safer behavior: address component extraction should happen - /// client-side, not by delegating to a remote actor. - /// - /// See [`DEPRECATION_NOTICE`](zaino_fetch::jsonrpsee::response::z_validate_address::DEPRECATION_NOTICE). - pub async fn run_z_validate_sapling_zebrad_passthrough_fetchservice(rpc_call: &F) - where - F: Fn(String) -> Fut, - Fut: Future, - { - let expected_sapling = ValidZValidateAddress::sapling( - VALID_SAPLING_ADDRESS.to_string(), - None::, - None::, - ); - assert_known_valid_eq( - rpc_call(VALID_SAPLING_ADDRESS.to_string()).await, - expected_sapling, - "Sapling (zebrad passthrough via FetchService — keys omitted)", - ); - } - - /// Which sapling suite to run after the shared suite. - pub enum SaplingSuite { - /// Full response — diversifier and diversifiedtransmissionkey present. - Standard, - /// zebrad's JSON-RPC passthrough (via FetchService) omits those keys. - ZebradPassthroughFetchService, - } - /// Build the `z_validate_address` rpc-call closure from `subscriber` and - /// run the shared validation suite plus the chosen sapling suite. Factors - /// the identical closure + suite-call preamble shared by the four - /// `z_validate_address` tests (fetch_service zcashd/zebrad, state_service, - /// json_server). + /// run the shared validation suite. Factors the identical closure + + /// suite-call preamble shared by the four `z_validate_address` tests + /// (fetch_service zcashd/zebrad, state_service, json_server). #[allow(deprecated)] - pub async fn run_z_validate_for(subscriber: &S, sapling: SaplingSuite) { + pub async fn run_z_validate_for(subscriber: &S) { let rpc_call = |addr: String| async move { subscriber.z_validate_address(addr).await.unwrap() }; run_z_validate_suite(&rpc_call).await; - match sapling { - SaplingSuite::Standard => run_z_validate_sapling(&rpc_call).await, - SaplingSuite::ZebradPassthroughFetchService => { - run_z_validate_sapling_zebrad_passthrough_fetchservice(&rpc_call).await - } - } } } } diff --git a/live-tests/clientless/tests/fetch_service.rs b/live-tests/clientless/tests/fetch_service.rs index f4ac94f37..ac80060df 100644 --- a/live-tests/clientless/tests/fetch_service.rs +++ b/live-tests/clientless/tests/fetch_service.rs @@ -1,6 +1,6 @@ //! These tests compare the output of `FetchService` with the output of `JsonRpcConnector`. -use clientless::rpc::z_validate_address::{run_z_validate_for, SaplingSuite}; +use clientless::rpc::z_validate_address::run_z_validate_for; use futures::StreamExt as _; use zaino_fetch::jsonrpsee::connector::JsonRpSeeConnector; use zaino_proto::proto::compact_formats::CompactBlock; @@ -416,11 +416,11 @@ async fn fetch_service_validate_address(validator: &ValidatorKi /// Launch a fetch-backend manager and run the shared `z_validate_address` /// suite against its subscriber. -async fn z_validate(validator: &ValidatorKind, suite: SaplingSuite) { +async fn z_validate(validator: &ValidatorKind) { let (mut test_manager, fetch_service_subscriber) = zaino_testutils::launch_with_fetch_subscriber::(validator, None).await; - run_z_validate_for(&fetch_service_subscriber, suite).await; + run_z_validate_for(&fetch_service_subscriber).await; test_manager.close().await; } @@ -648,7 +648,7 @@ mod zcashd { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] pub(crate) async fn z_validate_address() { - z_validate::(&ValidatorKind::Zcashd, SaplingSuite::Standard).await; + z_validate::(&ValidatorKind::Zcashd).await; } } @@ -719,11 +719,7 @@ mod zebrad { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] pub(crate) async fn z_validate_address() { - z_validate::( - &ValidatorKind::Zebrad, - SaplingSuite::ZebradPassthroughFetchService, - ) - .await; + z_validate::(&ValidatorKind::Zebrad).await; } } diff --git a/live-tests/clientless/tests/json_server.rs b/live-tests/clientless/tests/json_server.rs index 72df376c5..b8c17443c 100644 --- a/live-tests/clientless/tests/json_server.rs +++ b/live-tests/clientless/tests/json_server.rs @@ -429,11 +429,8 @@ mod zcashd { async fn z_validate_address() { let mut services = zaino_testutils::launch_zcashd_dual_fetch_services().await; - clientless::rpc::z_validate_address::run_z_validate_for( - &services.zcashd_subscriber, - clientless::rpc::z_validate_address::SaplingSuite::Standard, - ) - .await; + clientless::rpc::z_validate_address::run_z_validate_for(&services.zcashd_subscriber) + .await; services.test_manager.close().await; } diff --git a/live-tests/clientless/tests/state_service.rs b/live-tests/clientless/tests/state_service.rs index 9a4190a12..ec6257f9f 100644 --- a/live-tests/clientless/tests/state_service.rs +++ b/live-tests/clientless/tests/state_service.rs @@ -573,11 +573,8 @@ mod zebra { ) .await; - clientless::rpc::z_validate_address::run_z_validate_for( - &services.state_subscriber, - clientless::rpc::z_validate_address::SaplingSuite::Standard, - ) - .await; + clientless::rpc::z_validate_address::run_z_validate_for(&services.state_subscriber) + .await; services.test_manager.close().await; } From 5f2ac737b27853c3714974a7b1d31b0729cace62 Mon Sep 17 00:00:00 2001 From: zancas Date: Tue, 7 Jul 2026 22:22:43 -0700 Subject: [PATCH 33/58] fix(zaino-state): preserve typed error causes across the BlockchainSource boundary zaino-serve recovers zcashd-compatible RPC error codes by downcast-walking std::error::Error::source() chains (getblock_error_object_from_indexer_error looks for TransportError::ErrorStatusCode(500) to return legacy code -8; sendrawtransaction_error_object_from_indexer_error forwards RpcError codes). The connector boundary flattened every validator and read-state error to BlockchainSourceError::Unrecoverable(error.to_string()), which severed the chain, so those walks fell through to a generic -32603 internal error. Add BlockchainSourceError::UnrecoverableWithSource, which boxes the typed cause as #[source] while rendering the same top-level Display message as the old stringified form, with constructors unrecoverable(error) and unrecoverable_context(context, error) accepting both concrete and already-boxed errors (zebra's BoxError). Convert the 84 connector sites that wrapped a typed error; sites that only carry a rendered message keep the string-only Unrecoverable variant. The regression test constructs a Fetch connector against a closed port and asserts the TransportError stays reachable through the source() chain of the get_block_verbose error; it fails on the previous stringifying code. Addresses finding 1 of the PR #1361 review. Co-Authored-By: Claude Fable 5 --- .../zaino-state/src/chain_index/source.rs | 45 ++- .../chain_index/source/mockchain_source.rs | 14 +- .../chain_index/source/validator_connector.rs | 312 ++++++++++-------- 3 files changed, 232 insertions(+), 139 deletions(-) diff --git a/packages/zaino-state/src/chain_index/source.rs b/packages/zaino-state/src/chain_index/source.rs index 982b6bfe2..c06294122 100644 --- a/packages/zaino-state/src/chain_index/source.rs +++ b/packages/zaino-state/src/chain_index/source.rs @@ -379,11 +379,54 @@ pub(super) async fn wait_or_source_change( /// An error originating from a blockchain source. #[derive(Debug, thiserror::Error)] pub enum BlockchainSourceError { - /// Unrecoverable error. + /// Unrecoverable error described only by a message (no underlying + /// typed error exists, e.g. an unexpected response shape). // TODO: Add logic for handling recoverable errors if any are identified // one candidate may be ephemerable network hiccoughs #[error("critical error in backing block source: {0}")] Unrecoverable(String), + /// Unrecoverable error whose typed cause is preserved as + /// [`std::error::Error::source`]. zaino-serve recovers zcashd-compatible + /// RPC error codes by downcast-walking `source()` chains, so errors that + /// wrap a typed transport or RPC error must use this variant rather than + /// [`Self::Unrecoverable`] with a stringified cause. + #[error("critical error in backing block source: {message}")] + UnrecoverableWithSource { + /// Rendered description, including the cause's `Display` output so + /// top-level log lines match the previous stringified form. + message: String, + /// The typed cause, available to `source()`-chain walks. + #[source] + source: Box, + }, +} + +impl BlockchainSourceError { + /// Wraps a typed error, preserving it for `source()`-chain recovery. + /// Accepts both concrete error types and already-boxed errors (e.g. + /// zebra's `BoxError`). + pub(crate) fn unrecoverable( + error: impl Into>, + ) -> Self { + let source = error.into(); + Self::UnrecoverableWithSource { + message: source.to_string(), + source, + } + } + + /// Wraps a typed error with a context prefix, preserving the error for + /// `source()`-chain recovery. + pub(crate) fn unrecoverable_context( + context: impl std::fmt::Display, + error: impl Into>, + ) -> Self { + let source = error.into(); + Self::UnrecoverableWithSource { + message: format!("{context}: {source}"), + source, + } + } } /// Error type returned when invalid data is returned by the validator. diff --git a/packages/zaino-state/src/chain_index/source/mockchain_source.rs b/packages/zaino-state/src/chain_index/source/mockchain_source.rs index 74eb31d20..1bec4de8e 100644 --- a/packages/zaino-state/src/chain_index/source/mockchain_source.rs +++ b/packages/zaino-state/src/chain_index/source/mockchain_source.rs @@ -370,7 +370,7 @@ impl MockchainSource { return Ok(GetBlockHeaderResponse::Raw(HexData( header .zcash_serialize_to_vec() - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?, + .map_err(BlockchainSourceError::unrecoverable)?, ))); } @@ -592,7 +592,7 @@ impl BlockchainSource for MockchainSource { // full block, so serialize it to measure. let size = block .zcash_serialize_to_vec() - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))? + .map_err(BlockchainSourceError::unrecoverable)? .len() as i64; // `chain_supply` / `value_pools` are cumulative pool balances that the test @@ -622,8 +622,8 @@ impl BlockchainSource for MockchainSource { hash: String, verbose: bool, ) -> BlockchainSourceResult { - let hash_or_height = HashOrHeight::from_str(&hash) - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + let hash_or_height = + HashOrHeight::from_str(&hash).map_err(BlockchainSourceError::unrecoverable)?; let height_index = self.resolve_index(&hash_or_height).ok_or_else(|| { BlockchainSourceError::Unrecoverable("block height not in best chain".to_string()) })?; @@ -632,8 +632,8 @@ impl BlockchainSource for MockchainSource { } async fn get_block_deltas(&self, hash: String) -> BlockchainSourceResult { - let hash_or_height = HashOrHeight::from_str(&hash) - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + let hash_or_height = + HashOrHeight::from_str(&hash).map_err(BlockchainSourceError::unrecoverable)?; let GetBlock::Object(object) = self.get_block_verbose(hash_or_height, Some(2)).await? else { return Err(BlockchainSourceError::Unrecoverable( @@ -655,7 +655,7 @@ impl BlockchainSource for MockchainSource { continue; }; let prev_hash = zebra_chain::transaction::Hash::from_str(prevtxid) - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + .map_err(BlockchainSourceError::unrecoverable)?; if prevtx_cache.contains_key(&prev_hash) { continue; } diff --git a/packages/zaino-state/src/chain_index/source/validator_connector.rs b/packages/zaino-state/src/chain_index/source/validator_connector.rs index 603db354a..22d9a01a0 100644 --- a/packages/zaino-state/src/chain_index/source/validator_connector.rs +++ b/packages/zaino-state/src/chain_index/source/validator_connector.rs @@ -115,12 +115,12 @@ impl ValidatorConnector { common.validator_cookie_path.clone(), ) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + .map_err(BlockchainSourceError::unrecoverable)?; let info = fetcher .get_info() .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + .map_err(BlockchainSourceError::unrecoverable)?; Ok((ValidatorConnector::Fetch(fetcher), info)) } @@ -187,7 +187,7 @@ impl ValidatorConnector { .ready() .and_then(|service| service.call(ReadRequest::Tip)) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + .map_err(BlockchainSourceError::unrecoverable)?; let (syncer_height, syncer_tip_hash) = expected_read_response!(syncer_response, Tip) .ok_or_else(|| { BlockchainSourceError::Unrecoverable("no blocks in chain".to_string()) @@ -309,7 +309,7 @@ impl BlockchainSource for ValidatorConnector { { Ok(GetBlockResponse::Raw(raw_block)) => Ok(Some(Arc::new( zebra_chain::block::Block::zcash_deserialize(raw_block.as_ref()) - .map_err(|e| BlockchainSourceError::Unrecoverable(e.to_string()))?, + .map_err(BlockchainSourceError::unrecoverable)?, ))), Ok(_) => unreachable!(), Err(e) => match e { @@ -317,7 +317,7 @@ impl BlockchainSource for ValidatorConnector { // TODO/FIX: zcashd returns this transport error when a block is requested higher than current chain. is this correct? RpcRequestError::Transport(zaino_fetch::jsonrpsee::error::TransportError::ErrorStatusCode(500)) => Ok(None), RpcRequestError::ServerWorkQueueFull => Err(BlockchainSourceError::Unrecoverable("Work queue full. not yet implemented: handling of ephemeral network errors.".to_string())), - _ => Err(BlockchainSourceError::Unrecoverable(e.to_string())), + _ => Err(BlockchainSourceError::unrecoverable(e)), }, } } @@ -325,7 +325,7 @@ impl BlockchainSource for ValidatorConnector { "Read Request of Block returned Read Response of {otherwise:#?} \n\ This should be deterministically unreachable" ), - Err(e) => Err(BlockchainSourceError::Unrecoverable(e.to_string())), + Err(e) => Err(BlockchainSourceError::unrecoverable(e)), }, ValidatorConnector::Fetch(fetch) => { match fetch @@ -334,7 +334,7 @@ impl BlockchainSource for ValidatorConnector { { Ok(GetBlockResponse::Raw(raw_block)) => Ok(Some(Arc::new( zebra_chain::block::Block::zcash_deserialize(raw_block.as_ref()) - .map_err(|e| BlockchainSourceError::Unrecoverable(e.to_string()))?, + .map_err(BlockchainSourceError::unrecoverable)?, ))), Ok(_) => unreachable!(), Err(e) => match e { @@ -342,7 +342,7 @@ impl BlockchainSource for ValidatorConnector { // TODO/FIX: zcashd returns this transport error when a block is requested higher than current chain. is this correct? RpcRequestError::Transport(zaino_fetch::jsonrpsee::error::TransportError::ErrorStatusCode(500)) => Ok(None), RpcRequestError::ServerWorkQueueFull => Err(BlockchainSourceError::Unrecoverable("Work queue full. not yet implemented: handling of ephemeral network errors.".to_string())), - _ => Err(BlockchainSourceError::Unrecoverable(e.to_string())), + _ => Err(BlockchainSourceError::unrecoverable(e)), }, } } @@ -361,10 +361,10 @@ impl BlockchainSource for ValidatorConnector { ValidatorConnector::Fetch(fetch) => fetch .get_block(hash_or_height.to_string(), verbosity) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))? + .map_err(BlockchainSourceError::unrecoverable)? .try_into() .map_err(|error: zebra_chain::serialization::SerializationError| { - BlockchainSourceError::Unrecoverable(error.to_string()) + BlockchainSourceError::unrecoverable(error) }), } } @@ -376,8 +376,8 @@ impl BlockchainSource for ValidatorConnector { ) -> BlockchainSourceResult { match self { ValidatorConnector::State(state) => { - let hash_or_height = HashOrHeight::from_str(&hash) - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + let hash_or_height = + HashOrHeight::from_str(&hash).map_err(BlockchainSourceError::unrecoverable)?; let header = state .get_block_header_inner(hash_or_height, Some(verbose)) .await?; @@ -386,7 +386,7 @@ impl BlockchainSource for ValidatorConnector { ValidatorConnector::Fetch(fetch) => fetch .get_block_header(hash, verbose) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string())), + .map_err(BlockchainSourceError::unrecoverable), } } @@ -396,7 +396,7 @@ impl BlockchainSource for ValidatorConnector { ValidatorConnector::Fetch(fetch) => fetch .get_block_deltas(hash) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string())), + .map_err(BlockchainSourceError::unrecoverable), } } @@ -410,11 +410,11 @@ impl BlockchainSource for ValidatorConnector { false, ) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string())), + .map_err(BlockchainSourceError::unrecoverable), ValidatorConnector::Fetch(fetch) => Ok(fetch .get_difficulty() .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))? + .map_err(BlockchainSourceError::unrecoverable)? .0), } } @@ -425,7 +425,7 @@ impl BlockchainSource for ValidatorConnector { ValidatorConnector::Fetch(fetch) => fetch .get_blockchain_info() .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))? + .map_err(BlockchainSourceError::unrecoverable)? .try_into() .map_err(|_error| { BlockchainSourceError::Unrecoverable( @@ -442,7 +442,7 @@ impl BlockchainSource for ValidatorConnector { .json_rpc_connector() .get_info() .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))? + .map_err(BlockchainSourceError::unrecoverable)? .into()) } @@ -450,21 +450,21 @@ impl BlockchainSource for ValidatorConnector { self.json_rpc_connector() .get_peer_info() .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string())) + .map_err(BlockchainSourceError::unrecoverable) } async fn get_block_subsidy(&self, height: u32) -> BlockchainSourceResult { self.json_rpc_connector() .get_block_subsidy(height) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string())) + .map_err(BlockchainSourceError::unrecoverable) } async fn get_mining_info(&self) -> BlockchainSourceResult { self.json_rpc_connector() .get_mining_info() .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string())) + .map_err(BlockchainSourceError::unrecoverable) } async fn get_tx_out( @@ -476,7 +476,7 @@ impl BlockchainSource for ValidatorConnector { self.json_rpc_connector() .get_tx_out(txid, n, include_mempool) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string())) + .map_err(BlockchainSourceError::unrecoverable) } async fn get_spent_info( @@ -486,7 +486,7 @@ impl BlockchainSource for ValidatorConnector { self.json_rpc_connector() .get_spent_info(request) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string())) + .map_err(BlockchainSourceError::unrecoverable) } async fn get_network_sol_ps( @@ -497,7 +497,7 @@ impl BlockchainSource for ValidatorConnector { self.json_rpc_connector() .get_network_sol_ps(blocks, height) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string())) + .map_err(BlockchainSourceError::unrecoverable) } async fn send_raw_transaction( @@ -510,7 +510,7 @@ impl BlockchainSource for ValidatorConnector { .send_raw_transaction(raw_transaction_hex) .await .map(SentTransactionHash::from) - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string())) + .map_err(BlockchainSourceError::unrecoverable) } async fn get_treestate_by_id( @@ -520,7 +520,7 @@ impl BlockchainSource for ValidatorConnector { self.json_rpc_connector() .get_treestate(hash_or_height) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))? + .map_err(BlockchainSourceError::unrecoverable)? .try_into() .map_err(|_error| { BlockchainSourceError::Unrecoverable("failed to parse treestate".to_string()) @@ -559,7 +559,7 @@ impl BlockchainSource for ValidatorConnector { }) .await .map_err(|e| { - BlockchainSourceError::Unrecoverable(format!("state read failed: {e}")) + BlockchainSourceError::unrecoverable_context("state read failed", e) })?; if let zebra_state::ReadResponse::AnyChainTransaction(opt) = response { @@ -596,9 +596,10 @@ impl BlockchainSource for ValidatorConnector { .get_raw_transaction(zebra_txid.to_string(), Some(0)) .await .map_err(|e| { - BlockchainSourceError::Unrecoverable(format!( - "could not fetch transaction data: {e}" - )) + BlockchainSourceError::unrecoverable_context( + "could not fetch transaction data", + e, + ) })? { serialized_transaction } else { @@ -611,9 +612,10 @@ impl BlockchainSource for ValidatorConnector { std::io::Cursor::new(serialized_transaction.as_ref()), ) .map_err(|e| { - BlockchainSourceError::Unrecoverable(format!( - "could not deserialize transaction data: {e}" - )) + BlockchainSourceError::unrecoverable_context( + "could not deserialize transaction data", + e, + ) })?; Ok(Some((transaction.into(), GetTransactionLocation::Mempool))) } else { @@ -626,9 +628,10 @@ impl BlockchainSource for ValidatorConnector { .get_raw_transaction(txid.to_rpc_hex(), Some(1)) .await .map_err(|e| { - BlockchainSourceError::Unrecoverable(format!( - "could not fetch transaction data: {e}" - )) + BlockchainSourceError::unrecoverable_context( + "could not fetch transaction data", + e, + ) })? { transaction_object } else { @@ -641,9 +644,10 @@ impl BlockchainSource for ValidatorConnector { transaction_object.hex().as_ref(), )) .map_err(|e| { - BlockchainSourceError::Unrecoverable(format!( - "could not deserialize transaction data: {e}" - )) + BlockchainSourceError::unrecoverable_context( + "could not deserialize transaction data", + e, + ) })?; let location = match transaction_object.height() { Some(-1) => GetTransactionLocation::NonbestChain, @@ -673,7 +677,7 @@ impl BlockchainSource for ValidatorConnector { .get_raw_mempool() .await .map_err(|e| { - BlockchainSourceError::Unrecoverable(format!("could not fetch mempool data: {e}")) + BlockchainSourceError::unrecoverable_context("could not fetch mempool data", e) })? .transactions; @@ -681,9 +685,10 @@ impl BlockchainSource for ValidatorConnector { .into_iter() .map(|txid_str| { zebra_chain::transaction::Hash::from_str(&txid_str).map_err(|e| { - BlockchainSourceError::Unrecoverable(format!( - "invalid transaction id '{txid_str}': {e}" - )) + BlockchainSourceError::unrecoverable_context( + format!("invalid transaction id '{txid_str}'"), + e, + ) }) }) .collect::>()?; @@ -711,9 +716,10 @@ impl BlockchainSource for ValidatorConnector { .get_best_blockhash() .await .map_err(|e| { - BlockchainSourceError::Unrecoverable(format!( - "could not fetch best block hash from validator: {e}" - )) + BlockchainSourceError::unrecoverable_context( + "could not fetch best block hash from validator", + e, + ) })? .0, )) @@ -725,9 +731,10 @@ impl BlockchainSource for ValidatorConnector { .get_best_blockhash() .await .map_err(|e| { - BlockchainSourceError::Unrecoverable(format!( - "could not fetch best block hash from validator: {e}" - )) + BlockchainSourceError::unrecoverable_context( + "could not fetch best block hash from validator", + e, + ) })? .0, )), @@ -753,9 +760,10 @@ impl BlockchainSource for ValidatorConnector { .get_block_count() .await .map_err(|e| { - BlockchainSourceError::Unrecoverable(format!( - "could not fetch best block hash from validator: {e}" - )) + BlockchainSourceError::unrecoverable_context( + "could not fetch best block hash from validator", + e, + ) })? .into(), )) @@ -767,9 +775,10 @@ impl BlockchainSource for ValidatorConnector { .get_block_count() .await .map_err(|e| { - BlockchainSourceError::Unrecoverable(format!( - "could not fetch best block hash from validator: {e}" - )) + BlockchainSourceError::unrecoverable_context( + "could not fetch best block hash from validator", + e, + ) })? .into(), )), @@ -988,9 +997,10 @@ impl BlockchainSource for ValidatorConnector { } }) .map_err(|e| { - BlockchainSourceError::Unrecoverable(format!( - "could not get subtrees from validator: {e}" - )) + BlockchainSourceError::unrecoverable_context( + "could not get subtrees from validator", + e, + ) }) } @@ -999,9 +1009,10 @@ impl BlockchainSource for ValidatorConnector { .get_subtrees_by_index(pool.pool_string(), start_index, max_entries) .await .map_err(|e| { - BlockchainSourceError::Unrecoverable(format!( - "could not get subtrees from validator: {e}" - )) + BlockchainSourceError::unrecoverable_context( + "could not get subtrees from validator", + e, + ) })?; Ok(subtrees @@ -1022,9 +1033,10 @@ impl BlockchainSource for ValidatorConnector { }) .collect::, _>>() .map_err(|e| { - BlockchainSourceError::Unrecoverable(format!( - "could not get subtrees from validator: {e}" - )) + BlockchainSourceError::unrecoverable_context( + "could not get subtrees from validator", + e, + ) })?) } } @@ -1056,12 +1068,9 @@ impl BlockchainSource for ValidatorConnector { .await; let (sapling_tree, orchard_tree, ironwood_tree) = match ( //TODO: Better readstateservice error handling - sapling_tree_response - .map_err(|e| BlockchainSourceError::Unrecoverable(e.to_string()))?, - orchard_tree_response - .map_err(|e| BlockchainSourceError::Unrecoverable(e.to_string()))?, - ironwood_tree_response - .map_err(|e| BlockchainSourceError::Unrecoverable(e.to_string()))?, + sapling_tree_response.map_err(BlockchainSourceError::unrecoverable)?, + orchard_tree_response.map_err(BlockchainSourceError::unrecoverable)?, + ironwood_tree_response.map_err(BlockchainSourceError::unrecoverable)?, ) { ( ReadResponse::SaplingTree(saptree), @@ -1097,7 +1106,7 @@ impl BlockchainSource for ValidatorConnector { .to_string(), ) } - _ => BlockchainSourceError::Unrecoverable(e.to_string()), + _ => BlockchainSourceError::unrecoverable(e), })?; let GetTreestateResponse { sapling, @@ -1117,7 +1126,7 @@ impl BlockchainSource for ValidatorConnector { }, ) .transpose() - .map_err(|e| BlockchainSourceError::Unrecoverable(format!("io error: {e}")))?; + .map_err(|e| BlockchainSourceError::unrecoverable_context("io error", e))?; let orchard_frontier = orchard .map_or_else( || Some(Ok(CommitmentTree::empty())), @@ -1130,7 +1139,7 @@ impl BlockchainSource for ValidatorConnector { }, ) .transpose() - .map_err(|e| BlockchainSourceError::Unrecoverable(format!("io error: {e}")))?; + .map_err(|e| BlockchainSourceError::unrecoverable_context("io error", e))?; let ironwood_frontier = ironwood .map_or_else( || Some(Ok(CommitmentTree::empty())), @@ -1143,7 +1152,7 @@ impl BlockchainSource for ValidatorConnector { }, ) .transpose() - .map_err(|e| BlockchainSourceError::Unrecoverable(format!("io error: {e}")))?; + .map_err(|e| BlockchainSourceError::unrecoverable_context("io error", e))?; let sapling_root = sapling_frontier .map(|tree| { zebra_chain::sapling::tree::Root::try_from(tree.root().to_bytes()) @@ -1151,7 +1160,7 @@ impl BlockchainSource for ValidatorConnector { }) .transpose() .map_err(|e| { - BlockchainSourceError::Unrecoverable(format!("could not deser: {e}")) + BlockchainSourceError::unrecoverable_context("could not deser", e) })?; let orchard_root = orchard_frontier .map(|tree| { @@ -1160,7 +1169,7 @@ impl BlockchainSource for ValidatorConnector { }) .transpose() .map_err(|e| { - BlockchainSourceError::Unrecoverable(format!("could not deser: {e}")) + BlockchainSourceError::unrecoverable_context("could not deser", e) })?; let ironwood_root = ironwood_frontier .map(|tree| { @@ -1169,7 +1178,7 @@ impl BlockchainSource for ValidatorConnector { }) .transpose() .map_err(|e| { - BlockchainSourceError::Unrecoverable(format!("could not deser: {e}")) + BlockchainSourceError::unrecoverable_context("could not deser", e) })?; Ok((sapling_root, orchard_root, ironwood_root)) } @@ -1268,14 +1277,10 @@ impl BlockchainSource for ValidatorConnector { let response = read_state .ready() .await - .map_err(|error| { - BlockchainSourceError::Unrecoverable(error.to_string()) - })? + .map_err(BlockchainSourceError::unrecoverable)? .call(ReadRequest::BlockHeader(hash_or_height)) .await - .map_err(|error| { - BlockchainSourceError::Unrecoverable(error.to_string()) - })?; + .map_err(BlockchainSourceError::unrecoverable)?; match response { ReadResponse::BlockHeader { hash, .. } => Ok(BlockInfo::new( @@ -1296,14 +1301,10 @@ impl BlockchainSource for ValidatorConnector { let response = read_state .ready() .await - .map_err(|error| { - BlockchainSourceError::Unrecoverable(error.to_string()) - })? + .map_err(BlockchainSourceError::unrecoverable)? .call(ReadRequest::BlockHeader(hash_or_height)) .await - .map_err(|error| { - BlockchainSourceError::Unrecoverable(error.to_string()) - })?; + .map_err(BlockchainSourceError::unrecoverable)?; match response { ReadResponse::BlockHeader { hash, .. } => Ok(BlockInfo::new( @@ -1330,7 +1331,7 @@ impl BlockchainSource for ValidatorConnector { ValidatorConnector::Fetch(fetch) => fetch .get_address_deltas(params) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string())), + .map_err(BlockchainSourceError::unrecoverable), } } @@ -1343,14 +1344,14 @@ impl BlockchainSource for ValidatorConnector { let mut state = state.read_state_service.clone(); let strings_set = address_strings.valid_addresses().map_err(|error| { - BlockchainSourceError::Unrecoverable(format!("invalid address: {error}")) + BlockchainSourceError::unrecoverable_context("invalid address", error) })?; let response = state .ready() .and_then(|service| service.call(ReadRequest::AddressBalance(strings_set))) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + .map_err(BlockchainSourceError::unrecoverable)?; let (balance, received) = match response { ReadResponse::AddressBalance { balance, received } => (balance, received), @@ -1375,7 +1376,7 @@ impl BlockchainSource for ValidatorConnector { .collect(), ) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))? + .map_err(BlockchainSourceError::unrecoverable)? .into()), } } @@ -1393,7 +1394,7 @@ impl BlockchainSource for ValidatorConnector { .ready() .and_then(|service| service.call(ReadRequest::Tip)) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + .map_err(BlockchainSourceError::unrecoverable)?; let (chain_height, _chain_hash) = expected_read_response!(response, Tip) .ok_or_else(|| { @@ -1420,9 +1421,7 @@ impl BlockchainSource for ValidatorConnector { addresses: GetAddressBalanceRequest::new(addresses) .valid_addresses() .map_err(|error| { - BlockchainSourceError::Unrecoverable(format!( - "invalid address: {error}" - )) + BlockchainSourceError::unrecoverable_context("invalid address", error) })?, height_range: zebra_chain::block::Height(start) @@ -1432,7 +1431,7 @@ impl BlockchainSource for ValidatorConnector { .ready() .and_then(|service| service.call(request)) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + .map_err(BlockchainSourceError::unrecoverable)?; let hashes = expected_read_response!(response, AddressesTransactionIds); @@ -1461,14 +1460,15 @@ impl BlockchainSource for ValidatorConnector { fetch .get_address_txids(addresses, start, end) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))? + .map_err(BlockchainSourceError::unrecoverable)? .transactions .iter() .map(|txid_string| { TransactionHash::from_hex(txid_string.as_bytes()).map_err(|error| { - BlockchainSourceError::Unrecoverable(format!( - "invalid txid from getaddresstxids `{txid_string}`: {error}" - )) + BlockchainSourceError::unrecoverable_context( + format!("invalid txid from getaddresstxids `{txid_string}`"), + error, + ) }) }) .collect::, BlockchainSourceError>>() @@ -1485,7 +1485,7 @@ impl BlockchainSource for ValidatorConnector { let mut state = state.read_state_service.clone(); let valid_addresses = addresses.valid_addresses().map_err(|error| { - BlockchainSourceError::Unrecoverable(format!("invalid address: {error}")) + BlockchainSourceError::unrecoverable_context("invalid address", error) })?; let request = ReadRequest::UtxosByAddresses(valid_addresses); @@ -1493,7 +1493,7 @@ impl BlockchainSource for ValidatorConnector { .ready() .and_then(|service| service.call(request)) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + .map_err(BlockchainSourceError::unrecoverable)?; let utxos = expected_read_response!(response, AddressUtxos); let mut last_output_location = @@ -1536,7 +1536,7 @@ impl BlockchainSource for ValidatorConnector { .collect(), ) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))? + .map_err(BlockchainSourceError::unrecoverable)? .into_iter() .map(|utxos| utxos.into()) .collect()), @@ -1635,14 +1635,14 @@ impl State { GetBlockHeaderResponse::Raw(HexData( header .zcash_serialize_to_vec() - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?, + .map_err(BlockchainSourceError::unrecoverable)?, )) } else { let ReadResponse::SaplingTree(sapling_tree) = state .ready() .and_then(|service| service.call(ReadRequest::SaplingTree(hash_or_height))) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))? + .map_err(BlockchainSourceError::unrecoverable)? else { return Err(BlockchainSourceError::Unrecoverable( "Unexpected response to SaplingTree request".to_string(), @@ -1657,7 +1657,7 @@ impl State { .ready() .and_then(|service| service.call(ReadRequest::Depth(hash))) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))? + .map_err(BlockchainSourceError::unrecoverable)? else { return Err(BlockchainSourceError::Unrecoverable( "Unexpected response to Depth request".to_string(), @@ -1707,7 +1707,7 @@ impl State { .ready() .and_then(|service| service.call(request)) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + .map_err(BlockchainSourceError::unrecoverable)?; let block = expected_read_response!(response, Block); block .map(SerializedBlock::from) @@ -1787,19 +1787,19 @@ impl State { unreachable!("Unexpected responses from state service: {unexpected:?} {unexpected2:?}") } (Err(e), _) | (_, Err(e)) => { - return Err(BlockchainSourceError::Unrecoverable(e.to_string())); + return Err(BlockchainSourceError::unrecoverable(e)); } }; - let orchard_tree_response = orchard_tree_response - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + let orchard_tree_response = + orchard_tree_response.map_err(BlockchainSourceError::unrecoverable)?; let orchard_tree = expected_read_response!(orchard_tree_response, OrchardTree) .ok_or_else(|| { BlockchainSourceError::Unrecoverable("missing orchard tree".to_string()) })?; - let ironwood_tree_response = ironwood_tree_response - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + let ironwood_tree_response = + ironwood_tree_response.map_err(BlockchainSourceError::unrecoverable)?; let ironwood_tree = expected_read_response!(ironwood_tree_response, IronwoodTree) .ok_or_else(|| { BlockchainSourceError::Unrecoverable("missing ironwood tree".to_string()) @@ -1843,8 +1843,8 @@ impl State { /// spent prevout via a best-chain `ReadRequest::Transaction` (finalised + /// non-finalised), then hands off to the shared [`assemble_block_deltas`]. async fn get_block_deltas(&self, hash: String) -> BlockchainSourceResult { - let hash_or_height = HashOrHeight::from_str(&hash) - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + let hash_or_height = + HashOrHeight::from_str(&hash).map_err(BlockchainSourceError::unrecoverable)?; let GetBlock::Object(object) = self.get_block_inner(hash_or_height, Some(2)).await? else { return Err(BlockchainSourceError::Unrecoverable( "getblockdeltas: unexpected raw block".to_string(), @@ -1866,7 +1866,7 @@ impl State { continue; }; let prev_hash = zebra_chain::transaction::Hash::from_str(prevtxid) - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + .map_err(BlockchainSourceError::unrecoverable)?; if prevtx_cache.contains_key(&prev_hash) { continue; } @@ -1875,7 +1875,7 @@ impl State { .ready() .and_then(|service| service.call(ReadRequest::Transaction(prev_hash))) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + .map_err(BlockchainSourceError::unrecoverable)?; let mined_tx = expected_read_response!(response, Transaction).ok_or_else(|| { BlockchainSourceError::Unrecoverable(format!( "getblockdeltas: prevout tx {prevtxid} not in best chain" @@ -1938,7 +1938,7 @@ impl State { .ready() .and_then(|service| service.call(ReadRequest::TipPoolValues)) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + .map_err(BlockchainSourceError::unrecoverable)?; let (height, hash, balance) = match response { ReadResponse::TipPoolValues { tip_height, @@ -1954,14 +1954,14 @@ impl State { .ready() .and_then(|service| service.call(ReadRequest::UsageInfo)) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + .map_err(BlockchainSourceError::unrecoverable)?; let size_on_disk = expected_read_response!(usage_response, UsageInfo); let response = state .ready() .and_then(|service| service.call(ReadRequest::BlockHeader(hash.into()))) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + .map_err(BlockchainSourceError::unrecoverable)?; let header = match response { ReadResponse::BlockHeader { header, .. } => header, unexpected => { @@ -2020,7 +2020,7 @@ impl State { let difficulty = chain_tip_difficulty(network.clone(), self.read_state_service.clone(), false) .await - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + .map_err(BlockchainSourceError::unrecoverable)?; let verification_progress = f64::from(height.0) / f64::from(zebra_estimated_height.0); @@ -2211,7 +2211,7 @@ pub(crate) fn header_to_block_commitments( final_sapling_root: [u8; 32], ) -> BlockchainSourceResult<[u8; 32]> { let hash = match header.commitment(network, height).map_err(|error| { - BlockchainSourceError::Unrecoverable(format!("invalid block commitment: {error}")) + BlockchainSourceError::unrecoverable_context("invalid block commitment", error) })? { zebra_chain::block::Commitment::PreSaplingReserved(bytes) => bytes, zebra_chain::block::Commitment::FinalSaplingRoot(_root) => final_sapling_root, @@ -2232,10 +2232,8 @@ pub(crate) fn header_to_block_commitments( pub(crate) fn zebra_block_header_to_wire( header: GetBlockHeaderResponse, ) -> BlockchainSourceResult { - let value = serde_json::to_value(&header) - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; - serde_json::from_value(value) - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string())) + let value = serde_json::to_value(&header).map_err(BlockchainSourceError::unrecoverable)?; + serde_json::from_value(value).map_err(BlockchainSourceError::unrecoverable) } /// Returns the median of a non-empty set of block times. @@ -2286,7 +2284,7 @@ pub(crate) fn assemble_block_deltas( }; let prev_hash = zebra_chain::transaction::Hash::from_str(prevtxid) - .map_err(|error| BlockchainSourceError::Unrecoverable(error.to_string()))?; + .map_err(BlockchainSourceError::unrecoverable)?; let prev_tx = prevtx_cache.get(&prev_hash).ok_or_else(|| { BlockchainSourceError::Unrecoverable(format!( "getblockdeltas: prevout tx {prevtxid} not resolved" @@ -2307,9 +2305,10 @@ pub(crate) fn assemble_block_deltas( // Inputs are debits, so the amount leaves the address. let satoshis: Amount = (-output.value().zatoshis()).try_into().map_err(|error| { - BlockchainSourceError::Unrecoverable(format!( - "getblockdeltas: input amount out of range for {prevtxid}:{prevout}: {error}" - )) + BlockchainSourceError::unrecoverable_context( + format!("getblockdeltas: input amount out of range for {prevtxid}:{prevout}"), + error, + ) })?; inputs.push(InputDelta { @@ -2436,6 +2435,57 @@ mod fetch_pool_treestate_slot { } } +#[cfg(test)] +mod get_block_verbose { + use super::*; + + /// zaino-serve recovers zcashd-compatible RPC error codes by downcast-walking + /// [`std::error::Error::source`] chains (see + /// `getblock_error_object_from_indexer_error` and + /// `sendrawtransaction_error_object_from_indexer_error` in + /// `zaino-serve/src/rpc/jsonrpc/service.rs`). The connector boundary must + /// therefore preserve the typed cause instead of flattening it to a string, + /// or `getblock` failures surface as generic internal errors rather than the + /// legacy `-8` code lightwalletd-family clients key on. + #[tokio::test] + async fn fetch_error_keeps_transport_error_in_source_chain() { + // Port 1 refuses connections, so the request fails at the transport + // layer without contacting any validator. + let connector = zaino_fetch::jsonrpsee::connector::JsonRpSeeConnector::new_with_basic_auth( + "http://127.0.0.1:1/" + .parse() + .expect("static url literal parses"), + "user".to_string(), + "password".to_string(), + ) + .expect("connector construction is network-free"); + let source = ValidatorConnector::Fetch(connector); + + let error = source + .get_block_verbose(HashOrHeight::Height(zebra_chain::block::Height(1)), Some(1)) + .await + .expect_err("a request to a closed port must fail"); + + let mut current: Option<&(dyn std::error::Error + 'static)> = Some(&error); + let mut transport_error_reachable = false; + while let Some(source_error) = current { + if source_error + .downcast_ref::() + .is_some() + { + transport_error_reachable = true; + break; + } + current = source_error.source(); + } + assert!( + transport_error_reachable, + "the typed TransportError must stay reachable via the source() chain; \ + stringifying it strips the RPC error code the serve layer recovers" + ); + } +} + #[cfg(test)] mod ironwood_treestate_slot { /// Regression test: when the validator reported no ironwood treestate (below NU6.3 From fd322a174b065f174596b7502959b80478410ca0 Mon Sep 17 00:00:00 2001 From: zancas Date: Tue, 7 Jul 2026 22:26:42 -0700 Subject: [PATCH 34/58] fix(zaino-state): keep the typed RpcError on sendrawtransaction hex rejection validate_raw_transaction_hex returns an RpcError carrying zcashd's legacy InvalidParameter code (-8), but send_raw_transaction stringified it through ChainIndexError::internal, so zaino-serve's source()-chain walk (sendrawtransaction_error_object_from_indexer_error) could not find it and wallets submitting malformed hex received a generic -32603 internal error. Add ChainIndexError::internal_from, which preserves the typed error as source while rendering the same message, and use it at the validation site. The regression test drives ChainIndexRpcExt::send_raw_transaction over a MockchainSource with invalid hex and asserts the -8 code is recoverable from the source() chain; it fails on the stringifying code. The mock's send_raw_transaction stub would panic if reached, so the test also pins that rejection happens before any validator round-trip. Addresses finding 2 of the PR #1361 review. Co-Authored-By: Claude Fable 5 --- packages/zaino-state/src/chain_index.rs | 2 +- .../src/chain_index/tests/mockchain_tests.rs | 37 +++++++++++++++++++ packages/zaino-state/src/error.rs | 12 ++++++ 3 files changed, 50 insertions(+), 1 deletion(-) diff --git a/packages/zaino-state/src/chain_index.rs b/packages/zaino-state/src/chain_index.rs index 83dce333d..008461c3d 100644 --- a/packages/zaino-state/src/chain_index.rs +++ b/packages/zaino-state/src/chain_index.rs @@ -2865,7 +2865,7 @@ impl ChainIndexRpcExt for NodeBackedChainIndexSubscrib raw_transaction_hex: String, ) -> Result { validate_raw_transaction_hex(&raw_transaction_hex) - .map_err(|error| ChainIndexError::internal(error.to_string()))?; + .map_err(ChainIndexError::internal_from)?; self.source() .send_raw_transaction(raw_transaction_hex) .await diff --git a/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs b/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs index 95f8e489b..76a7d6107 100644 --- a/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs +++ b/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs @@ -1072,3 +1072,40 @@ async fn node_backed_indexer_service_serves_latest_block() { let latest = service.get_latest_block().await.unwrap(); assert_eq!(latest.height, expected_tip as u64); } + +/// `sendrawtransaction` rejections must carry zcashd's legacy error code: +/// zaino-serve forwards the code by downcast-walking the `source()` chain for +/// the typed `RpcError` (`sendrawtransaction_error_object_from_indexer_error`), +/// so stringifying it downgrades the legacy `-8` "invalid hex" rejection to a +/// generic `-32603` internal error. +/// +/// Invalid hex fails in local validation before the source is consulted, so +/// this also pins that the rejection happens without a validator round-trip +/// (the mock's `send_raw_transaction` would panic if reached). +#[tokio::test(flavor = "multi_thread")] +async fn send_raw_transaction_invalid_hex_keeps_legacy_error_code() { + let (_blocks, _indexer, index_reader, _mockchain) = + load_test_vectors_and_sync_chain_index(MockchainMode::Static).await; + + let error = index_reader + .send_raw_transaction("notahexstring".to_string()) + .await + .expect_err("invalid hex must be rejected"); + + let mut current: Option<&(dyn std::error::Error + 'static)> = Some(&error); + let mut rpc_error_code = None; + while let Some(source_error) = current { + if let Some(rpc_error) = + source_error.downcast_ref::() + { + rpc_error_code = Some(rpc_error.code); + break; + } + current = source_error.source(); + } + assert_eq!( + rpc_error_code, + Some(zebra_rpc::server::error::LegacyCode::InvalidParameter as i64), + "the typed RpcError (legacy code -8) must stay reachable via the source() chain" + ); +} diff --git a/packages/zaino-state/src/error.rs b/packages/zaino-state/src/error.rs index 2093ca68b..b0eb87b31 100644 --- a/packages/zaino-state/src/error.rs +++ b/packages/zaino-state/src/error.rs @@ -510,6 +510,18 @@ impl ChainIndexError { } } + /// Constructs an `InternalServerError`-kind error from a typed error, + /// preserving it as `source` so zaino-serve's RPC-error-code recovery + /// walks can downcast to it (e.g. the legacy `-8` code carried by an + /// [`RpcError`](zaino_fetch::jsonrpsee::connector::RpcError)). + pub(crate) fn internal_from(error: impl std::error::Error + Send + Sync + 'static) -> Self { + Self { + kind: ChainIndexErrorKind::InternalServerError, + message: error.to_string(), + source: Some(Box::new(error)), + } + } + pub(crate) fn backing_validator(value: impl std::error::Error + Send + Sync + 'static) -> Self { Self { kind: ChainIndexErrorKind::InternalServerError, From 39053f16816ac441888ac85382c6499d990d7beb Mon Sep 17 00:00:00 2001 From: zancas Date: Tue, 7 Jul 2026 22:31:07 -0700 Subject: [PATCH 35/58] fix(zaino-state)!: return Option from chaintip_update_subscriber instead of panicking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Before the service merge the chain-tip subscriber existed only on the State-backed subscriber type, so requesting one from an RPC-backed service was a compile error. The merged NodeBackedIndexerServiceSubscriber exposed it as an ungated pub method whose .expect() panicked at runtime whenever the service was configured with backend = "rpc" — a caller-dependent condition, not a program invariant, so the panic also violated the repository's .expect() policy. Lift chain_tip_change onto the BlockchainSource trait with a None default (only the Direct connection drives its own Zebra syncer and owns a local tip-change stream), make chaintip_update_subscriber generic over the source, and return Option. The live-test caller now unwraps explicitly on the Direct connection it launches. The regression test builds the service over a MockchainSource (a source with no tip stream, like the Rpc connection) and asserts the subscriber request yields None; before the fix the non-panicking form did not exist and the only reachable behavior was the runtime panic. Addresses finding 3 of the PR #1361 review. Co-Authored-By: Claude Fable 5 --- live-tests/clientless/tests/state_service.rs | 5 +++- .../zaino-state/src/chain_index/source.rs | 8 +++++ .../chain_index/source/validator_connector.rs | 21 +++++++------ .../src/chain_index/tests/mockchain_tests.rs | 26 ++++++++++++++++ .../src/indexer/node_backed_indexer.rs | 30 +++++++++---------- 5 files changed, 62 insertions(+), 28 deletions(-) diff --git a/live-tests/clientless/tests/state_service.rs b/live-tests/clientless/tests/state_service.rs index ec6257f9f..c1ccbd6c5 100644 --- a/live-tests/clientless/tests/state_service.rs +++ b/live-tests/clientless/tests/state_service.rs @@ -383,7 +383,10 @@ mod zebra { #[tokio::test(flavor = "multi_thread")] async fn state_service_chaintip_update_subscriber() { let services = launch_regtest(true).await; - let mut chaintip_subscriber = services.state_subscriber.chaintip_update_subscriber(); + let mut chaintip_subscriber = services + .state_subscriber + .chaintip_update_subscriber() + .expect("the Direct connection exposes a local tip-change stream"); services .test_manager .generate_blocks_and_check_each( diff --git a/packages/zaino-state/src/chain_index/source.rs b/packages/zaino-state/src/chain_index/source.rs index c06294122..5080c367e 100644 --- a/packages/zaino-state/src/chain_index/source.rs +++ b/packages/zaino-state/src/chain_index/source.rs @@ -145,6 +145,14 @@ pub trait BlockchainSource: Clone + Send + Sync + 'static { /// minimum difficulty (the `getdifficulty` RPC value). fn get_difficulty(&self) -> impl SendFut>; + /// A watch stream of the source's chain-tip changes, when the source owns + /// one locally. Only the `Direct` validator connection (which drives its + /// own Zebra syncer) does; every other source observes tips by polling, + /// and inherits this `None` default. + fn chain_tip_change(&self) -> Option { + None + } + /// Returns the `getblockchaininfo` response. fn get_blockchain_info( &self, diff --git a/packages/zaino-state/src/chain_index/source/validator_connector.rs b/packages/zaino-state/src/chain_index/source/validator_connector.rs index 22d9a01a0..5dc94d43e 100644 --- a/packages/zaino-state/src/chain_index/source/validator_connector.rs +++ b/packages/zaino-state/src/chain_index/source/validator_connector.rs @@ -224,17 +224,6 @@ impl ValidatorConnector { Ok((source, info)) } - /// Watches the Zebra syncer's chain-tip changes, when this connector owns one. - /// - /// `Some` for the `State` variant (which drives its own syncer); `None` for the - /// JSON-RPC `Fetch` variant, which has no local tip-change stream. - pub(crate) fn chain_tip_change(&self) -> Option { - match self { - ValidatorConnector::State(state) => Some(state.chain_tip_change.clone()), - ValidatorConnector::Fetch(_) => None, - } - } - /// The backing [`ReadStateService`], when this connector is `State`-backed. /// /// Test-only escape hatch: live tests recompute expected chain data directly off @@ -288,6 +277,16 @@ fn fetch_pool_treestate_slot(treestate: zebra_rpc::client::Treestate) -> Option< } impl BlockchainSource for ValidatorConnector { + /// `Some` for the `State` variant, which drives its own Zebra syncer; the + /// JSON-RPC `Fetch` variant has no local tip-change stream and keeps the + /// trait's `None` default semantics. + fn chain_tip_change(&self) -> Option { + match self { + ValidatorConnector::State(state) => Some(state.chain_tip_change.clone()), + ValidatorConnector::Fetch(_) => None, + } + } + // ********** Block methods ********** async fn get_block( diff --git a/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs b/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs index 76a7d6107..b3aa8a1b5 100644 --- a/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs +++ b/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs @@ -1073,6 +1073,32 @@ async fn node_backed_indexer_service_serves_latest_block() { assert_eq!(latest.height, expected_tip as u64); } +/// The `Rpc` connection has no local chain-tip-change stream, so requesting a +/// chain-tip subscriber over such a source must yield `None` rather than +/// panic. Before this method returned `Option`, it existed only in a +/// panicking form (`.expect("chaintip_update_subscriber requires the Direct +/// connection")`) reachable by any embedder configured with `backend = "rpc"`; +/// pre-merge the misuse was a compile error because only the State-backed +/// subscriber type had the method. +#[tokio::test(flavor = "multi_thread")] +async fn chaintip_update_subscriber_absent_without_tip_stream() { + use crate::indexer::node_backed_indexer::NodeBackedIndexerServiceSubscriber; + use zaino_common::{network::ActivationHeights, Network}; + + let (_blocks, _indexer, index_reader, _mockchain) = + load_test_vectors_and_sync_chain_index(MockchainMode::Static).await; + + let service = NodeBackedIndexerServiceSubscriber::new_for_test( + index_reader, + Network::Regtest(ActivationHeights::default()), + ); + + assert!( + service.chaintip_update_subscriber().is_none(), + "a source with no local tip-change stream must yield no subscriber, not panic" + ); +} + /// `sendrawtransaction` rejections must carry zcashd's legacy error code: /// zaino-serve forwards the code by downcast-walking the `source()` chain for /// the typed `RpcError` (`sendrawtransaction_error_object_from_indexer_error`), diff --git a/packages/zaino-state/src/indexer/node_backed_indexer.rs b/packages/zaino-state/src/indexer/node_backed_indexer.rs index bd804312a..27e666eaa 100644 --- a/packages/zaino-state/src/indexer/node_backed_indexer.rs +++ b/packages/zaino-state/src/indexer/node_backed_indexer.rs @@ -291,24 +291,22 @@ impl ChainTipSubscriber { } } -/// Methods available only on the production (`ValidatorConnector`-backed) subscriber: -/// the `Direct` connection owns a Zebra chain-tip stream and `ReadStateService` that the -/// generic source abstraction does not expose. -impl NodeBackedIndexerServiceSubscriber { - /// Gets a subscriber to any updates to the latest chain tip. - /// - /// Only meaningful for the `Direct` connection; the `Rpc` connection has no local - /// tip-change stream. - pub fn chaintip_update_subscriber(&self) -> ChainTipSubscriber { - ChainTipSubscriber { - monitor: self - .indexer - .source() - .chain_tip_change() - .expect("chaintip_update_subscriber requires the Direct connection"), - } +impl NodeBackedIndexerServiceSubscriber { + /// A subscriber to chain-tip updates, when the backing source exposes a + /// local tip-change stream. `Some` only on the `Direct` connection; the + /// `Rpc` connection (and any other stream-less source) observes tips by + /// polling the validator and yields `None`. + pub fn chaintip_update_subscriber(&self) -> Option { + Some(ChainTipSubscriber { + monitor: self.indexer.source().chain_tip_change()?, + }) } +} +/// Methods available only on the production (`ValidatorConnector`-backed) subscriber: +/// the `Direct` connection owns a `ReadStateService` that the generic source +/// abstraction does not expose. +impl NodeBackedIndexerServiceSubscriber { /// The backing Zebra [`zebra_state::ReadStateService`] (`Direct` connection only). /// /// Test-only escape hatch: live tests recompute expected chain data directly off the From dfc6d98668d0f5b3e0c2fee0ae574859175e6994 Mon Sep 17 00:00:00 2001 From: zancas Date: Tue, 7 Jul 2026 22:33:25 -0700 Subject: [PATCH 36/58] fix(zaino-state): replace the getaddressdeltas tip unwrap with a typed error The Direct arm of get_address_deltas clamped its height range with self.get_best_block_height().await?.unwrap(), panicking whenever the source reports no best height (nothing indexed yet). Production code disallows .unwrap() without exception. Extract the range clamp into clamp_deltas_range_to_tip, which turns the absent-tip case into a BlockchainSourceError and preserves the existing clamping semantics (end == 0 means "to the tip"; both bounds clamp down to the tip). The tests pin the absent-tip error, which had no non-panicking form before the fix, and the clamping behavior verbatim. Addresses finding 4 of the PR #1361 review. Co-Authored-By: Claude Fable 5 --- .../chain_index/source/validator_connector.rs | 75 ++++++++++++++++--- 1 file changed, 66 insertions(+), 9 deletions(-) diff --git a/packages/zaino-state/src/chain_index/source/validator_connector.rs b/packages/zaino-state/src/chain_index/source/validator_connector.rs index 5dc94d43e..8974271cd 100644 --- a/packages/zaino-state/src/chain_index/source/validator_connector.rs +++ b/packages/zaino-state/src/chain_index/source/validator_connector.rs @@ -1204,15 +1204,11 @@ impl BlockchainSource for ValidatorConnector { GetAddressDeltasParams::Address(a) => (vec![a.clone()], 0, 0, false), }; - let tip = self.get_best_block_height().await?.unwrap().into(); - let mut start = Height(start_raw); - let mut end = Height(end_raw); - if end == Height(0) || end > tip { - end = tip; - } - if start > tip { - start = tip; - } + let (start, end) = clamp_deltas_range_to_tip( + self.get_best_block_height().await?, + start_raw, + end_raw, + )?; let transactions: Vec> = { let tx_ids_request = @@ -2434,6 +2430,67 @@ mod fetch_pool_treestate_slot { } } +/// Clamps a `getaddressdeltas` height range to the current best tip: +/// `end == 0` means "to the tip", and both bounds clamp down to it. A source +/// that reports no best height (nothing indexed yet) is a typed error. +fn clamp_deltas_range_to_tip( + tip: Option, + start_raw: u32, + end_raw: u32, +) -> BlockchainSourceResult<(Height, Height)> { + let tip: Height = tip + .ok_or_else(|| { + BlockchainSourceError::Unrecoverable( + "getaddressdeltas: the source reports no best block height".to_string(), + ) + })? + .into(); + let mut start = Height(start_raw); + let mut end = Height(end_raw); + if end == Height(0) || end > tip { + end = tip; + } + if start > tip { + start = tip; + } + Ok((start, end)) +} + +#[cfg(test)] +mod clamp_deltas_range_to_tip { + use super::*; + + /// A source that reports no best height (nothing indexed yet, or the + /// RPC fallback found no tip) must yield a typed error, not a panic: + /// this range clamp previously lived inline in `get_address_deltas` + /// behind `get_best_block_height().await?.unwrap()`. + #[test] + fn absent_tip_is_a_typed_error() { + assert!( + clamp_deltas_range_to_tip(None, 0, 0).is_err(), + "an absent tip must surface as a BlockchainSourceError" + ); + } + + /// `end == 0` means "to the tip", and both bounds clamp down to the tip. + #[test] + fn bounds_clamp_to_tip() { + let tip = Some(zebra_chain::block::Height(100)); + + let (start, end) = + clamp_deltas_range_to_tip(tip, 5, 0).expect("a present tip clamps successfully"); + assert_eq!((start, end), (Height(5), Height(100))); + + let (start, end) = + clamp_deltas_range_to_tip(tip, 5, 400).expect("a present tip clamps successfully"); + assert_eq!((start, end), (Height(5), Height(100))); + + let (start, end) = + clamp_deltas_range_to_tip(tip, 300, 50).expect("a present tip clamps successfully"); + assert_eq!((start, end), (Height(100), Height(50))); + } +} + #[cfg(test)] mod get_block_verbose { use super::*; From e094e56660991e7c7ec4c7e213af4b8a64895037 Mon Sep 17 00:00:00 2001 From: zancas Date: Tue, 7 Jul 2026 22:38:09 -0700 Subject: [PATCH 37/58] fix(zaino-state): make the service teardown safe off the multi-thread runtime NodeBackedIndexerService's shutdown_blocking (shared by ZcashService::close and Drop) entered block_in_place + Handle::current().block_on unconditionally. block_in_place panics on a current-thread runtime and Handle::current() panics on a thread with no runtime; either way a panic inside Drop during unwind aborts the process, so merely dropping the service in the wrong context was fatal. The old StateService drop was a plain JoinHandle::abort with no runtime requirement. Guard the blocking path with Handle::try_current() and a runtime-flavor check. Contexts that cannot block fall back to NodeBackedChainIndex::shutdown_sync_best_effort, a synchronous teardown that cancels the sync worker, closes the mempool, and releases source-owned background work, skipping only the finalised DB's async shutdown step (the DB's own Drop still releases its resources). The regression tests build the service over a MockchainSource and drop it on a plain thread with no runtime and inside a current-thread runtime; both panicked before the fix. Both wrap the drop in a spawned thread because the mock harness itself requires the multi-thread flavor (its finalised-state validation uses block_in_place). Addresses finding 5 of the PR #1361 review. Co-Authored-By: Claude Fable 5 --- packages/zaino-state/src/chain_index.rs | 13 +++ .../src/chain_index/tests/mockchain_tests.rs | 55 ++++++++++++ .../src/indexer/node_backed_indexer.rs | 87 ++++++++++++++----- 3 files changed, 133 insertions(+), 22 deletions(-) diff --git a/packages/zaino-state/src/chain_index.rs b/packages/zaino-state/src/chain_index.rs index 008461c3d..47d3a9463 100644 --- a/packages/zaino-state/src/chain_index.rs +++ b/packages/zaino-state/src/chain_index.rs @@ -986,6 +986,19 @@ impl NodeBackedChainIndex { Ok(()) } + /// Synchronous best-effort teardown for contexts that cannot run async + /// work (a `Drop` on a current-thread runtime, or on a thread with no + /// runtime at all): cancels the sync worker, closes the mempool, and + /// releases source-owned background work. The finalised DB's async + /// [`shutdown`](Self::shutdown) step is skipped; the DB's own `Drop` + /// remains responsible for releasing its resources. + pub(crate) fn shutdown_sync_best_effort(&self) { + self.cancel_token.cancel(); + self.status.store(StatusType::Closing); + self.mempool.close(); + self.source.shutdown(); + } + /// Displays the status of the chain_index pub fn status(&self) -> StatusType { let finalized_status = self.finalized_db.status(); diff --git a/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs b/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs index b3aa8a1b5..6d0f0a611 100644 --- a/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs +++ b/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs @@ -1073,6 +1073,61 @@ async fn node_backed_indexer_service_serves_latest_block() { assert_eq!(latest.height, expected_tip as u64); } +/// Dropping the service must not panic on a thread with no Tokio runtime. +/// `Drop` runs the blocking teardown, which previously called +/// `Handle::current()` unconditionally; outside a runtime that panics, and a +/// panic inside `Drop` during unwind aborts the process. +/// +/// multi_thread required: the harness's finalised-state validation uses +/// `block_in_place`. The drop under test happens on a separate plain thread. +#[tokio::test(flavor = "multi_thread")] +async fn service_drop_survives_thread_without_runtime() { + use crate::indexer::node_backed_indexer::NodeBackedIndexerService; + use zaino_common::{network::ActivationHeights, Network}; + + let (_blocks, indexer, _index_reader, _mockchain) = + load_test_vectors_and_sync_chain_index(MockchainMode::Static).await; + + let service = NodeBackedIndexerService::new_for_test( + indexer, + Network::Regtest(ActivationHeights::default()), + ); + + std::thread::spawn(move || drop(service)) + .join() + .expect("dropping the service off-runtime must not panic"); +} + +/// Dropping the service must not panic on a current-thread Tokio runtime, +/// where `block_in_place` (the old unconditional teardown entry) aborts. +/// +/// multi_thread required: the harness's finalised-state validation uses +/// `block_in_place`. The drop under test happens inside a current-thread +/// runtime built on a separate thread. +#[tokio::test(flavor = "multi_thread")] +async fn service_drop_survives_current_thread_runtime() { + use crate::indexer::node_backed_indexer::NodeBackedIndexerService; + use zaino_common::{network::ActivationHeights, Network}; + + let (_blocks, indexer, _index_reader, _mockchain) = + load_test_vectors_and_sync_chain_index(MockchainMode::Static).await; + + let service = NodeBackedIndexerService::new_for_test( + indexer, + Network::Regtest(ActivationHeights::default()), + ); + + std::thread::spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("current-thread runtime builds") + .block_on(async move { drop(service) }); + }) + .join() + .expect("dropping the service on a current-thread runtime must not panic"); +} + /// The `Rpc` connection has no local chain-tip-change stream, so requesting a /// chain-tip subscriber over such a source must yield `None` rather than /// panic. Before this method returned `Option`, it existed only in a diff --git a/packages/zaino-state/src/indexer/node_backed_indexer.rs b/packages/zaino-state/src/indexer/node_backed_indexer.rs index 27e666eaa..9fac9ef83 100644 --- a/packages/zaino-state/src/indexer/node_backed_indexer.rs +++ b/packages/zaino-state/src/indexer/node_backed_indexer.rs @@ -116,12 +116,26 @@ impl NodeBackedIndexerService { /// Tears down the indexer (sync loop, finalised DB, mempool, and any source-owned /// syncer task) from a synchronous context. Shared by [`ZcashService::close`] and /// [`Drop`]. + /// + /// `block_in_place` is only legal on a multi-thread runtime; on a + /// current-thread runtime or a thread with no runtime it panics, and a + /// panic inside `Drop` during unwind aborts the process. Those contexts + /// fall back to the synchronous best-effort teardown, which skips only + /// the finalised DB's async shutdown step (the DB's own `Drop` still + /// releases its resources). fn shutdown_blocking(&self) { - tokio::task::block_in_place(|| { - tokio::runtime::Handle::current().block_on(async { - let _ = self.indexer.shutdown().await; - }); - }); + use tokio::runtime::{Handle, RuntimeFlavor}; + + match Handle::try_current() { + Ok(handle) if handle.runtime_flavor() == RuntimeFlavor::MultiThread => { + tokio::task::block_in_place(|| { + handle.block_on(async { + let _ = self.indexer.shutdown().await; + }); + }); + } + _ => self.indexer.shutdown_sync_best_effort(), + } } } @@ -240,6 +254,49 @@ impl NodeBackedIndexerServiceSubscriber { } } +/// Placeholder metadata and config for test-only service construction. +#[cfg(test)] +fn test_service_parts(network: zaino_common::Network) -> (ServiceMetadata, CommonBackendConfig) { + ( + ServiceMetadata::new( + get_build_info("test".to_string()), + network.to_zebra_network(), + "test-build".to_string(), + "test-subversion".to_string(), + ), + CommonBackendConfig::new( + "127.0.0.1:0".to_string(), + None, + None, + None, + zaino_common::ServiceConfig::default(), + zaino_common::StorageConfig::default(), + true, + network, + None, + ), + ) +} + +#[cfg(test)] +impl NodeBackedIndexerService { + /// Wraps a chain index in a service for tests, with placeholder + /// metadata/config. Lets unit tests exercise the service lifecycle over a + /// mock source (no real validator). Production builds go through + /// [`ZcashService::spawn`]. + pub(crate) fn new_for_test( + indexer: NodeBackedChainIndex, + network: zaino_common::Network, + ) -> Self { + let (data, config) = test_service_parts(network); + Self { + data, + config, + indexer, + } + } +} + #[cfg(test)] impl NodeBackedIndexerServiceSubscriber { /// Wraps a chain-index subscriber in a service subscriber for tests, with placeholder @@ -249,24 +306,10 @@ impl NodeBackedIndexerServiceSubscriber { indexer: NodeBackedChainIndexSubscriber, network: zaino_common::Network, ) -> Self { + let (data, config) = test_service_parts(network); Self { - data: ServiceMetadata::new( - get_build_info("test".to_string()), - network.to_zebra_network(), - "test-build".to_string(), - "test-subversion".to_string(), - ), - config: CommonBackendConfig::new( - "127.0.0.1:0".to_string(), - None, - None, - None, - zaino_common::ServiceConfig::default(), - zaino_common::StorageConfig::default(), - true, - network, - None, - ), + data, + config, indexer, } } From 61d11924ab2e8106896bcad2b38ced7ffb483339 Mon Sep 17 00:00:00 2001 From: zancas Date: Tue, 7 Jul 2026 22:42:30 -0700 Subject: [PATCH 38/58] fix(zaino-state): omit multi-address outputs from getblockdeltas again MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit assemble_block_deltas attributed each output's full value to addresses.first(), where the pre-merge state backend required exactly one derivable address. An output with multiple derivable addresses (e.g. bare multisig) has no single owning address: zcashd's getblockdeltas omits it, and crediting the first address fabricates balance for it — and makes the Direct and Rpc connections disagree, since the Rpc arm proxies the validator's response. Extract the output mapping into output_delta_from_verbose and restore the exactly-one-address rule there. The extraction landed first with the first-address behavior intact, so the new multi-address test failed against it and flipped green only with the rule restored; single-address attribution and nonstandard-script omission are pinned alongside. Addresses finding 6 of the PR #1361 review. Co-Authored-By: Claude Fable 5 --- .../chain_index/source/validator_connector.rs | 91 ++++++++++++++++--- 1 file changed, 78 insertions(+), 13 deletions(-) diff --git a/packages/zaino-state/src/chain_index/source/validator_connector.rs b/packages/zaino-state/src/chain_index/source/validator_connector.rs index 8974271cd..d7c6bb9c5 100644 --- a/packages/zaino-state/src/chain_index/source/validator_connector.rs +++ b/packages/zaino-state/src/chain_index/source/validator_connector.rs @@ -2242,6 +2242,29 @@ pub(crate) fn median_of_block_times(mut times: Vec) -> BlockchainSourceResu Ok(times[times.len() / 2]) } +/// Maps a verbose-transaction output to its `getblockdeltas` output delta, +/// or `None` when the output does not participate in address deltas: only +/// outputs with exactly one derivable address are attributed. Zero means no +/// address to credit (nonstandard script); more than one (e.g. bare multisig) +/// has no single owning address, and zcashd's getblockdeltas omits such +/// outputs rather than crediting the first address. +fn output_delta_from_verbose(vout: &zebra_rpc::client::Output) -> Option { + let address = + vout.script_pub_key() + .addresses() + .as_ref() + .and_then(|addresses| match addresses.as_slice() { + [address] => Some(address.clone()), + _ => None, + })?; + let satoshis: Amount = vout.value_zat().try_into().ok()?; + Some(OutputDelta { + address, + satoshis, + index: vout.n(), + }) +} + /// Assembles a `getblockdeltas` response from a verbosity-2 `getblock` object plus a /// cache of previous transactions (keyed by txid) needed to resolve each spend's address /// and value, its median time, and the running network. @@ -2318,19 +2341,7 @@ pub(crate) fn assemble_block_deltas( let outputs: Vec = txo .outputs() .iter() - .filter_map(|vout| { - let address = vout - .script_pub_key() - .addresses() - .as_ref() - .and_then(|addresses| addresses.first().cloned())?; - let satoshis: Amount = vout.value_zat().try_into().ok()?; - Some(OutputDelta { - address, - satoshis, - index: vout.n(), - }) - }) + .filter_map(output_delta_from_verbose) .collect(); deltas.push(BlockDelta { @@ -2456,6 +2467,60 @@ fn clamp_deltas_range_to_tip( Ok((start, end)) } +#[cfg(test)] +mod output_delta_from_verbose { + use super::*; + + fn output_with_addresses(addresses: Option>) -> zebra_rpc::client::Output { + zebra_rpc::client::Output::new( + 0.00000001, + 1, + 0, + zebra_rpc::client::ScriptPubKey::new( + "asm".to_string(), + zebra_chain::transparent::Script::new(&[0u8]), + addresses.as_ref().map(|a| a.len() as u32), + "pubkeyhash".to_string(), + addresses, + ), + ) + } + + /// An output with exactly one derivable address is attributed to it. + #[test] + fn single_address_output_is_attributed() { + let delta = + output_delta_from_verbose(&output_with_addresses(Some(vec!["t1address".to_string()]))) + .expect("a single-address output participates in deltas"); + assert_eq!(delta.address, "t1address"); + } + + /// An output with multiple derivable addresses (e.g. bare multisig) has no + /// single owning address: zcashd's getblockdeltas omits it, and crediting + /// the first address would fabricate balance for it. The pre-fix code took + /// `addresses.first()`. + #[test] + fn multi_address_output_is_omitted() { + assert_eq!( + output_delta_from_verbose(&output_with_addresses(Some(vec![ + "t1first".to_string(), + "t1second".to_string(), + ]))), + None, + "a multi-address output must be omitted, not attributed to the first address" + ); + } + + /// No derivable address (nonstandard script) means no delta. + #[test] + fn addressless_output_is_omitted() { + assert_eq!( + output_delta_from_verbose(&output_with_addresses(None)), + None + ); + } +} + #[cfg(test)] mod clamp_deltas_range_to_tip { use super::*; From d198d9dbcf13bd66dd6c16b671321e6b22d8f90b Mon Sep 17 00:00:00 2001 From: zancas Date: Tue, 7 Jul 2026 22:49:53 -0700 Subject: [PATCH 39/58] fix(zaino-state): proxy getchaintips to the validator while still syncing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both pre-merge backends answered getchaintips during the initial finalised-state build (no non-finalised snapshot yet — hours on mainnet) by proxying the validator's own JSON-RPC response. The merged service returned UnavailableNotSyncedEnough for that whole window instead. getbestblockhash and getblockcount are unaffected: they already reach the source through best_chaintip's syncing-window arm. Add a get_chain_tips node-passthrough to the BlockchainSource trait (both connector arms already hold the JSON-RPC client the old backends used) and extract the snapshot-or-fallback choice into chain_tips_for_snapshot, which the service method now delegates to. MockchainSource serves a single active tip at its active height, matching a validator with no side branches; the proptest mock stubs the method like its other passthroughs. The regression test drives chain_tips_for_snapshot with a StillSyncingFinalizedState snapshot over the mock and asserts the validator's tips come back; before the fix the fallback path did not exist and the only behavior was the UnavailableNotSyncedEnough error. Addresses finding 7 of the PR #1361 review. Co-Authored-By: Claude Fable 5 --- .../zaino-state/src/chain_index/source.rs | 9 +++++ .../chain_index/source/mockchain_source.rs | 28 +++++++++++++++ .../chain_index/source/validator_connector.rs | 10 ++++++ .../src/chain_index/tests/mockchain_tests.rs | 35 +++++++++++++++++++ .../chain_index/tests/proptest_blockgen.rs | 7 ++++ .../src/indexer/node_backed_indexer.rs | 26 ++++++++++---- 6 files changed, 109 insertions(+), 6 deletions(-) diff --git a/packages/zaino-state/src/chain_index/source.rs b/packages/zaino-state/src/chain_index/source.rs index 5080c367e..0a4af2e1c 100644 --- a/packages/zaino-state/src/chain_index/source.rs +++ b/packages/zaino-state/src/chain_index/source.rs @@ -169,6 +169,15 @@ pub trait BlockchainSource: Clone + Send + Sync + 'static { /// Returns the `getpeerinfo` response. fn get_peer_info(&self) -> impl SendFut>; + /// Returns the validator's `getchaintips` response. Serves as the + /// `getchaintips` fallback while the local index is still building its + /// finalised state and has no non-finalised snapshot to answer from. + fn get_chain_tips( + &self, + ) -> impl SendFut< + BlockchainSourceResult, + >; + /// Returns the `getblocksubsidy` response at the given height. fn get_block_subsidy( &self, diff --git a/packages/zaino-state/src/chain_index/source/mockchain_source.rs b/packages/zaino-state/src/chain_index/source/mockchain_source.rs index 1bec4de8e..cc41537b1 100644 --- a/packages/zaino-state/src/chain_index/source/mockchain_source.rs +++ b/packages/zaino-state/src/chain_index/source/mockchain_source.rs @@ -712,6 +712,34 @@ impl BlockchainSource for MockchainSource { unimplemented!("MockchainSource cannot serve get_peer_info until test vectors are extended") } + /// A single active tip at the mockchain's current active height, matching + /// what a validator with no side branches reports. + async fn get_chain_tips( + &self, + ) -> BlockchainSourceResult + { + if self + .force_requests_against_source_to_fail + .load(Ordering::SeqCst) + { + return Err(BlockchainSourceError::Unrecoverable( + "forced source failure".into(), + )); + } + let height = self.active_height(); + let Some(index) = self.valid_height(height) else { + return Ok(vec![]); + }; + Ok(vec![ + zaino_fetch::jsonrpsee::response::chain_tips::ChainTip::new( + height, + self.blocks[index].hash().to_string(), + 0, + zaino_fetch::jsonrpsee::response::chain_tips::ChainTipStatus::Active, + ), + ]) + } + async fn get_block_subsidy( &self, _height: u32, diff --git a/packages/zaino-state/src/chain_index/source/validator_connector.rs b/packages/zaino-state/src/chain_index/source/validator_connector.rs index d7c6bb9c5..94119dbde 100644 --- a/packages/zaino-state/src/chain_index/source/validator_connector.rs +++ b/packages/zaino-state/src/chain_index/source/validator_connector.rs @@ -452,6 +452,16 @@ impl BlockchainSource for ValidatorConnector { .map_err(BlockchainSourceError::unrecoverable) } + async fn get_chain_tips( + &self, + ) -> BlockchainSourceResult + { + self.json_rpc_connector() + .get_chain_tips() + .await + .map_err(BlockchainSourceError::unrecoverable) + } + async fn get_block_subsidy(&self, height: u32) -> BlockchainSourceResult { self.json_rpc_connector() .get_block_subsidy(height) diff --git a/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs b/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs index 6d0f0a611..d7ffa6e6f 100644 --- a/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs +++ b/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs @@ -1073,6 +1073,41 @@ async fn node_backed_indexer_service_serves_latest_block() { assert_eq!(latest.height, expected_tip as u64); } +/// During the initial finalised-state build there is no non-finalised +/// snapshot. Both pre-merge backends answered `getchaintips` in that window by +/// proxying the validator's own response; the merged service must fall back to +/// the source the same way, rather than serving UnavailableNotSyncedEnough for +/// the whole build (hours on mainnet). +#[tokio::test] +async fn get_chain_tips_falls_back_to_source_while_syncing() { + use crate::chain_index::non_finalised_state::ChainIndexSnapshot; + use crate::chain_index::tests::vectors::build_mockchain_source; + use crate::indexer::node_backed_indexer::chain_tips_for_snapshot; + + let blocks = load_test_vectors().unwrap().blocks; + let tip_height = (blocks.len() as u32) - 1; + let expected_tip_hash = blocks[tip_height as usize].zebra_block.hash().to_string(); + let mock = build_mockchain_source(blocks); + + let syncing_snapshot = ChainIndexSnapshot::StillSyncingFinalizedState { + validator_finalized_height: crate::Height(tip_height), + }; + + let tips = chain_tips_for_snapshot(&syncing_snapshot, &mock) + .await + .expect("the syncing window must proxy the source's chain tips"); + + assert_eq!( + tips, + vec![zaino_fetch::jsonrpsee::response::chain_tips::ChainTip::new( + tip_height, + expected_tip_hash, + 0, + zaino_fetch::jsonrpsee::response::chain_tips::ChainTipStatus::Active, + )] + ); +} + /// Dropping the service must not panic on a thread with no Tokio runtime. /// `Drop` runs the blocking teardown, which previously called /// `Handle::current()` unconditionally; outside a runtime that panics, and a diff --git a/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs b/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs index bdb5dc3b9..ac4255e64 100644 --- a/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs +++ b/packages/zaino-state/src/chain_index/tests/proptest_blockgen.rs @@ -1080,6 +1080,13 @@ impl BlockchainSource for ProptestMockchain { unimplemented!() } + async fn get_chain_tips( + &self, + ) -> BlockchainSourceResult + { + unimplemented!() + } + async fn get_block_subsidy( &self, _height: u32, diff --git a/packages/zaino-state/src/indexer/node_backed_indexer.rs b/packages/zaino-state/src/indexer/node_backed_indexer.rs index 9fac9ef83..913c14862 100644 --- a/packages/zaino-state/src/indexer/node_backed_indexer.rs +++ b/packages/zaino-state/src/indexer/node_backed_indexer.rs @@ -254,6 +254,25 @@ impl NodeBackedIndexerServiceSubscriber { } } +/// `getchaintips` served from the non-finalised snapshot when it exists, +/// falling back to the validator's own response during the initial +/// finalised-state build — matching both pre-merge backends, which proxied +/// the validator for that window instead of erroring. +pub(crate) async fn chain_tips_for_snapshot( + snapshot: &ChainIndexSnapshot, + source: &Source, +) -> Result { + match snapshot.get_nfs_snapshot() { + Some(non_finalized_snapshot) => Ok(chain_tips_from_nonfinalized_snapshot( + non_finalized_snapshot, + )), + None => Ok(source + .get_chain_tips() + .await + .map_err(crate::error::ChainIndexError::backing_validator)?), + } +} + /// Placeholder metadata and config for test-only service construction. #[cfg(test)] fn test_service_parts(network: zaino_common::Network) -> (ServiceMetadata, CommonBackendConfig) { @@ -619,12 +638,7 @@ impl ZcashIndexer for NodeBackedIndexerServiceSubscrib #[allow(deprecated)] async fn get_chain_tips(&self) -> Result { let snapshot = self.indexer.snapshot_nonfinalized_state().await?; - let Some(non_finalized_snapshot) = snapshot.get_nfs_snapshot() else { - return Err(NodeBackedIndexerServiceError::UnavailableNotSyncedEnough); - }; - Ok(chain_tips_from_nonfinalized_snapshot( - non_finalized_snapshot, - )) + chain_tips_for_snapshot(&snapshot, self.indexer.source()).await } /// Return information about the given Zcash address. From 0d62a5af90c75b6fcbb70b6a4b002fbc832062e2 Mon Sep 17 00:00:00 2001 From: zancas Date: Tue, 7 Jul 2026 22:52:48 -0700 Subject: [PATCH 40/58] fix(zaino-state): resolve tip-relative getblock heights, keep the -8 code z_get_block pre-parsed its identifier with HashOrHeight::from_str, which rejects the negative tip-relative heights zebra's own getblock accepts ("-1" is the tip), and flattened the rejection into an internal-error string. The old Rpc backend forwarded the raw string to the validator, so negative heights worked there; the merge regressed them, and unparsable identifiers lost zcashd's legacy InvalidParameter code. Parse with HashOrHeight::new against the best chaintip instead, matching zebra's semantics on both connection types, and attach the rejection as a typed RpcError (legacy code -8) source via ChainIndexError::internal_from so the serve layer's downcast walk can recover it. The regression tests drive z_get_block over a MockchainSource: "-1" must serve the same block as the explicit tip height, and "notablockid" must carry the -8 RpcError in its source() chain. Both failed before the fix. Addresses finding 8 of the PR #1361 review. Co-Authored-By: Claude Fable 5 --- packages/zaino-state/src/chain_index.rs | 17 +++++- .../src/chain_index/tests/mockchain_tests.rs | 55 +++++++++++++++++++ 2 files changed, 70 insertions(+), 2 deletions(-) diff --git a/packages/zaino-state/src/chain_index.rs b/packages/zaino-state/src/chain_index.rs index 47d3a9463..88b5a5c78 100644 --- a/packages/zaino-state/src/chain_index.rs +++ b/packages/zaino-state/src/chain_index.rs @@ -2762,8 +2762,21 @@ impl ChainIndexRpcExt for NodeBackedChainIndexSubscrib hash_or_height: String, verbosity: Option, ) -> Result { - let id = HashOrHeight::from_str(&hash_or_height) - .map_err(|error| ChainIndexError::internal(error.to_string()))?; + // Resolve tip-relative negative heights against the best chaintip, + // matching zebra's own `getblock` semantics (`-1` is the tip). A + // rejected identifier carries zcashd's legacy InvalidParameter code + // as a typed `RpcError` source, which the serve layer recovers by + // downcast-walking the error chain. + let snapshot = self.snapshot_nonfinalized_state().await?; + let tip = self.best_chaintip(&snapshot).await?; + let id = HashOrHeight::new(&hash_or_height, Some(tip.height.into())).map_err(|error| { + ChainIndexError::internal_from( + zaino_fetch::jsonrpsee::connector::RpcError::new_from_legacycode( + zebra_rpc::server::error::LegacyCode::InvalidParameter, + error, + ), + ) + })?; self.source() .get_block_verbose(id, verbosity) .await diff --git a/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs b/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs index d7ffa6e6f..7adc31b7b 100644 --- a/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs +++ b/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs @@ -1073,6 +1073,61 @@ async fn node_backed_indexer_service_serves_latest_block() { assert_eq!(latest.height, expected_tip as u64); } +/// zebra's `getblock` resolves negative heights against the tip (`-1` is the +/// tip block). The old Rpc backend forwarded the raw identifier string to the +/// validator, so `getblock "-1"` worked; the merged pre-parse used +/// `HashOrHeight::from_str`, which rejects negative heights. +#[tokio::test(flavor = "multi_thread")] +async fn z_get_block_resolves_negative_heights_against_the_tip() { + let (blocks, _indexer, index_reader, _mockchain) = + load_test_vectors_and_sync_chain_index(MockchainMode::Static).await; + let tip_height = (blocks.len() as u32) - 1; + wait_for_indexer_tip(&index_reader, tip_height).await; + + let by_negative_height = index_reader + .z_get_block("-1".to_string(), Some(1)) + .await + .expect("height -1 must resolve to the tip block"); + let by_tip_height = index_reader + .z_get_block(tip_height.to_string(), Some(1)) + .await + .expect("the tip height resolves"); + assert_eq!(by_negative_height, by_tip_height); +} + +/// An unparsable `getblock` identifier must carry zcashd's legacy +/// InvalidParameter code (-8) as a typed `RpcError` in the `source()` chain, +/// not be flattened into an internal-error string: the serve layer recovers +/// legacy codes by downcast-walking the chain. +#[tokio::test(flavor = "multi_thread")] +async fn z_get_block_invalid_identifier_keeps_legacy_error_code() { + let (blocks, _indexer, index_reader, _mockchain) = + load_test_vectors_and_sync_chain_index(MockchainMode::Static).await; + wait_for_indexer_tip(&index_reader, (blocks.len() as u32) - 1).await; + + let error = index_reader + .z_get_block("notablockid".to_string(), Some(1)) + .await + .expect_err("an unparsable identifier must be rejected"); + + let mut current: Option<&(dyn std::error::Error + 'static)> = Some(&error); + let mut rpc_error_code = None; + while let Some(source_error) = current { + if let Some(rpc_error) = + source_error.downcast_ref::() + { + rpc_error_code = Some(rpc_error.code); + break; + } + current = source_error.source(); + } + assert_eq!( + rpc_error_code, + Some(zebra_rpc::server::error::LegacyCode::InvalidParameter as i64), + "the typed RpcError (legacy code -8) must stay reachable via the source() chain" + ); +} + /// During the initial finalised-state build there is no non-finalised /// snapshot. Both pre-merge backends answered `getchaintips` in that window by /// proxying the validator's own response; the merged service must fall back to From 493032b5761ac85183b79adb343a53f953f24631 Mon Sep 17 00:00:00 2001 From: zancas Date: Tue, 7 Jul 2026 22:56:15 -0700 Subject: [PATCH 41/58] fix(zaino-state): never let the source syncer task outlive its chain index NodeBackedChainIndex::shutdown() released source-owned background work (the Direct connection's Zebra syncer task) only after the fallible finalised-DB shutdown, so a DB shutdown error skipped the release via the `?`; and Drop only cancelled the sync worker, never releasing the source at all. Either way the syncer task could keep running against a torn-down index. Route both paths through shutdown_sync_best_effort (introduced with the teardown-safety fix): shutdown() runs the synchronous teardown before the fallible DB step, and Drop runs it in place of the bare cancel. The cancel-before-DB ordering the shutdown doc calls out is preserved. The regression test gives MockchainSource a shutdown-recording flag, drops the index without calling shutdown(), and asserts the source was released; before the fix Drop never invoked the release. Addresses finding 9 of the PR #1361 review. Co-Authored-By: Claude Fable 5 --- packages/zaino-state/src/chain_index.rs | 19 +++++++++-------- .../chain_index/source/mockchain_source.rs | 16 ++++++++++++++ .../src/chain_index/tests/mockchain_tests.rs | 21 +++++++++++++++++++ 3 files changed, 47 insertions(+), 9 deletions(-) diff --git a/packages/zaino-state/src/chain_index.rs b/packages/zaino-state/src/chain_index.rs index 88b5a5c78..23e5dd710 100644 --- a/packages/zaino-state/src/chain_index.rs +++ b/packages/zaino-state/src/chain_index.rs @@ -976,14 +976,11 @@ impl NodeBackedChainIndex { /// the design we have. Cancelling first just removes the wasted /// failure-path round trip. pub async fn shutdown(&self) -> Result<(), FinalisedStateError> { - self.cancel_token.cancel(); - self.status.store(StatusType::Closing); - self.finalized_db.shutdown().await?; - self.mempool.close(); - // Release any source-owned background work (e.g. the Zebra chain-syncer - // task behind the `State` connector). No-op for poll-only sources. - self.source.shutdown(); - Ok(()) + // The synchronous teardown (cancellation, mempool close, source + // release) runs before the fallible DB shutdown so a DB error cannot + // skip it — the source's Zebra syncer task must not outlive the index. + self.shutdown_sync_best_effort(); + self.finalized_db.shutdown().await } /// Synchronous best-effort teardown for contexts that cannot run async @@ -1222,7 +1219,11 @@ impl Drop for NodeBackedChainIndex { /// on `cancel_token.cancelled()` wrapping the iter body), the worker /// exits at its next await checkpoint instead. fn drop(&mut self) { - self.cancel_token.cancel(); + // The full synchronous teardown, not just the cancellation: the + // source release in particular must run here, or the `State` + // connector's Zebra syncer task outlives an index that is dropped + // without an explicit `shutdown()` call. + self.shutdown_sync_best_effort(); } } diff --git a/packages/zaino-state/src/chain_index/source/mockchain_source.rs b/packages/zaino-state/src/chain_index/source/mockchain_source.rs index cc41537b1..68cf04afe 100644 --- a/packages/zaino-state/src/chain_index/source/mockchain_source.rs +++ b/packages/zaino-state/src/chain_index/source/mockchain_source.rs @@ -180,6 +180,9 @@ pub(crate) struct MockchainSource { /// neither know nor care how many `mine_blocks` events occurred /// between wakes. blocks_received_broadcaster: tokio::sync::watch::Sender<()>, + /// Records whether [`BlockchainSource::shutdown`] ran, so teardown tests + /// can assert the index releases its source. Shared across clones. + shutdown_called: Arc, } impl MockchainSource { @@ -246,9 +249,16 @@ impl MockchainSource { )), get_block_hook: Arc::new(Mutex::new(None)), blocks_received_broadcaster: tokio::sync::watch::channel(()).0, + shutdown_called: Arc::new(std::sync::atomic::AtomicBool::new(false)), } } + /// Whether [`BlockchainSource::shutdown`] has run on this source (or any + /// clone of it). + pub(crate) fn shutdown_called(&self) -> bool { + self.shutdown_called.load(Ordering::SeqCst) + } + /// When set to true, `get_best_block_height` and `get_best_block_hash` /// return `BlockchainSourceError::Unrecoverable`. pub(crate) fn set_failing(&self, fail: bool) { @@ -712,6 +722,12 @@ impl BlockchainSource for MockchainSource { unimplemented!("MockchainSource cannot serve get_peer_info until test vectors are extended") } + /// Records the release so teardown tests can assert the index shuts its + /// source down (the mock owns no real background work). + fn shutdown(&self) { + self.shutdown_called.store(true, Ordering::SeqCst); + } + /// A single active tip at the mockchain's current active height, matching /// what a validator with no side branches reports. async fn get_chain_tips( diff --git a/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs b/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs index 7adc31b7b..6c391552a 100644 --- a/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs +++ b/packages/zaino-state/src/chain_index/tests/mockchain_tests.rs @@ -1073,6 +1073,27 @@ async fn node_backed_indexer_service_serves_latest_block() { assert_eq!(latest.height, expected_tip as u64); } +/// Dropping the chain index without an explicit `shutdown()` call must still +/// release source-owned background work: `Drop` previously only cancelled the +/// sync worker, and the async `shutdown()`'s `?` on the DB teardown skipped +/// the source release when the DB shutdown errored — either way the Direct +/// connection's Zebra syncer task could outlive its index. +#[tokio::test(flavor = "multi_thread")] +async fn dropping_the_chain_index_releases_the_source() { + let (_blocks, indexer, _index_reader, mockchain) = + load_test_vectors_and_sync_chain_index(MockchainMode::Static).await; + + assert!( + !mockchain.shutdown_called(), + "the source must not be shut down while the index is live" + ); + drop(indexer); + assert!( + mockchain.shutdown_called(), + "dropping the index must release source-owned background work" + ); +} + /// zebra's `getblock` resolves negative heights against the tip (`-1` is the /// tip block). The old Rpc backend forwarded the raw identifier string to the /// validator, so `getblock "-1"` worked; the merged pre-parse used From 06fd01fa4ed1c2864c7a93bdb66c04fab4130c3e Mon Sep 17 00:00:00 2001 From: zancas Date: Tue, 7 Jul 2026 23:00:36 -0700 Subject: [PATCH 42/58] test(zaino-state): pin the getblockheader raw-to-wire serde round-trip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Direct connection's non-verbose getblockheader builds a GetBlockHeaderResponse::Raw and crosses to zaino-fetch's untagged GetBlockHeader through a serde JSON round-trip with no test at the conversion's definition: a shape regression on either side would break every non-verbose getblockheader on a Direct connection at runtime and only surface in live tests. The conversion is currently correct — the mockchain get_block_header test exercises it end to end — so this is a coverage pin rather than a fix. The test asserts Raw hex lands in the Compact variant with identical bytes; a mutation check (nulling the intermediate serde value) confirmed it fails against a broken conversion. Addresses finding 10 of the PR #1361 review. Co-Authored-By: Claude Fable 5 --- .../chain_index/source/validator_connector.rs | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/packages/zaino-state/src/chain_index/source/validator_connector.rs b/packages/zaino-state/src/chain_index/source/validator_connector.rs index 94119dbde..551fd30ce 100644 --- a/packages/zaino-state/src/chain_index/source/validator_connector.rs +++ b/packages/zaino-state/src/chain_index/source/validator_connector.rs @@ -2415,6 +2415,29 @@ pub(crate) fn assemble_block_deltas( }) } +#[cfg(test)] +mod zebra_block_header_to_wire { + use super::*; + + /// The Direct connection's non-verbose `getblockheader` builds a + /// `GetBlockHeaderResponse::Raw` and crosses to the wire type through the + /// serde round-trip under test: the raw hex must land in the untagged + /// `Compact` variant with the same bytes, or every non-verbose + /// `getblockheader` on a Direct connection errors at runtime. The + /// mockchain `get_block_header` test covers the same conversion end to + /// end; this pins it at its definition. + #[test] + fn raw_header_round_trips_to_compact_hex() { + let header_bytes = vec![0x01, 0x02, 0xab, 0xcd]; + let raw = GetBlockHeaderResponse::Raw(HexData(header_bytes.clone())); + + let wire = zebra_block_header_to_wire(raw) + .expect("the raw header shape must convert to the wire type"); + + assert_eq!(wire, GetBlockHeader::Compact(hex::encode(header_bytes))); + } +} + #[cfg(test)] mod fetch_pool_treestate_slot { /// Regression test: the validator's finalRoot must pass through to the pool slot. From 064b1775257ec07795f806aee24a6e191c3f26b6 Mon Sep 17 00:00:00 2001 From: Mark Henderson Date: Thu, 9 Jul 2026 09:14:15 +0200 Subject: [PATCH 43/58] fix: wait for the validator's first block in the connector spawn paths A freshly started validator answers JSON-RPC before it has fetched and committed its first block: zebra serves getblockchaininfo (reporting a genesis-hash placeholder on an empty state) and an empty getrawmempool while getbestblockhash returns "No blocks in state" until genesis arrives from the network, which can take minutes of peer discovery on slow networks. Starting zainod in that window fails spawn on both connector paths and the process exit-loops under any supervisor: - spawn_fetch builds the connector without checking for a servable tip, so NodeBackedChainIndex::new fails (Mempool::spawn returns Critical when get_best_block_hash errors). - spawn_state's sync-wait loop gets None from expected_read_response!(_, Tip) on an empty state and fails with "no blocks in chain". Poll get_best_blockhash in spawn_fetch until it succeeds before returning the connector, and treat the missing syncer tip in spawn_state as wait-and-retry; both log progress every 3 seconds. The validator is reachable and version-checked at this point, so waiting matches the RPC-liveness standard operators use to sequence zainod after the validator. Co-Authored-By: Claude Fable 5 --- .../chain_index/source/validator_connector.rs | 30 ++++++++++++++++--- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/packages/zaino-state/src/chain_index/source/validator_connector.rs b/packages/zaino-state/src/chain_index/source/validator_connector.rs index 551fd30ce..b0a912ed5 100644 --- a/packages/zaino-state/src/chain_index/source/validator_connector.rs +++ b/packages/zaino-state/src/chain_index/source/validator_connector.rs @@ -122,6 +122,19 @@ impl ValidatorConnector { .await .map_err(BlockchainSourceError::unrecoverable)?; + // A freshly started validator answers RPC before it has fetched and + // committed its first block: zebra serves getblockchaininfo and an + // empty getrawmempool while getbestblockhash returns "No blocks in + // state" until genesis arrives, which can take minutes of peer + // discovery. Everything downstream assumes a servable tip + // (Mempool::spawn returns Critical on get_best_block_hash and the + // ChainIndex sync loop escalates to CriticalError), so wait here + // instead of failing spawn and exit-looping the whole process. + while let Err(e) = fetcher.get_best_blockhash().await { + info!(%e, "Waiting for validator to serve its first block"); + tokio::time::sleep(std::time::Duration::from_secs(3)).await; + } + Ok((ValidatorConnector::Fetch(fetcher), info)) } @@ -188,10 +201,19 @@ impl ValidatorConnector { .and_then(|service| service.call(ReadRequest::Tip)) .await .map_err(BlockchainSourceError::unrecoverable)?; - let (syncer_height, syncer_tip_hash) = expected_read_response!(syncer_response, Tip) - .ok_or_else(|| { - BlockchainSourceError::Unrecoverable("no blocks in chain".to_string()) - })?; + // A freshly started validator answers RPC before it has committed + // its first block (getblockchaininfo reports a genesis-hash + // placeholder on an empty state), and the syncer has no tip until + // genesis arrives, which can take minutes of peer discovery. + // Everything below assumes a servable tip, so wait here instead + // of failing spawn and exit-looping the whole process. + let Some((syncer_height, syncer_tip_hash)) = + expected_read_response!(syncer_response, Tip) + else { + info!("Waiting for validator to serve its first block"); + tokio::time::sleep(std::time::Duration::from_secs(3)).await; + continue; + }; if server_height == syncer_height && server_tip_hash == syncer_tip_hash { info!( From 86f9303288979889cd6afff7760cf44473ff3e0d Mon Sep 17 00:00:00 2001 From: zancas Date: Thu, 9 Jul 2026 12:16:18 -0700 Subject: [PATCH 44/58] build: bump zcash_local_net to the tip of infrastructure PR #278 The new rev (85502cd0) picks up the twelve commits on the bump_to_NU6.3 branch since the previous pin, including deterministic listener-port allocation with an Indexer-convergence barrier, the generic Wallet abstraction, and the front proxies published as the canonical endpoints. Co-Authored-By: Claude Fable 5 --- Cargo.lock | 6 +++--- Cargo.toml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3d0124b57..08f1ca280 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6184,7 +6184,7 @@ dependencies = [ [[package]] name = "zcash_local_net" version = "0.7.0" -source = "git+https://github.com/zingolabs/infrastructure.git?rev=78511f7304ba27895967be860016ede923acec08#78511f7304ba27895967be860016ede923acec08" +source = "git+https://github.com/zingolabs/infrastructure.git?rev=85502cd0b5faf9308e590865dae51bf31fdd342e#85502cd0b5faf9308e590865dae51bf31fdd342e" dependencies = [ "hex", "reqwest 0.12.28", @@ -6735,12 +6735,12 @@ dependencies = [ [[package]] name = "zingo-consensus" version = "0.1.0" -source = "git+https://github.com/zingolabs/infrastructure.git?rev=78511f7304ba27895967be860016ede923acec08#78511f7304ba27895967be860016ede923acec08" +source = "git+https://github.com/zingolabs/infrastructure.git?rev=85502cd0b5faf9308e590865dae51bf31fdd342e#85502cd0b5faf9308e590865dae51bf31fdd342e" [[package]] name = "zingo_test_vectors" version = "0.0.1" -source = "git+https://github.com/zingolabs/infrastructure.git?rev=78511f7304ba27895967be860016ede923acec08#78511f7304ba27895967be860016ede923acec08" +source = "git+https://github.com/zingolabs/infrastructure.git?rev=85502cd0b5faf9308e590865dae51bf31fdd342e#85502cd0b5faf9308e590865dae51bf31fdd342e" [[package]] name = "zip32" diff --git a/Cargo.toml b/Cargo.toml index bacac4ad1..8578ba5ff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -143,7 +143,7 @@ zaino-testutils = { path = "live-tests/zaino-testutils" } # truth (infrastructure ADR 0003, PR #278): wallet clients derive their # heights via `WalletNetwork::from_validator`, and `ZainodConfig` carries a # payload-free network kind. Restore a release tag once one is cut upstream. -zcash_local_net = { git = "https://github.com/zingolabs/infrastructure.git", rev = "78511f7304ba27895967be860016ede923acec08" } +zcash_local_net = { git = "https://github.com/zingolabs/infrastructure.git", rev = "85502cd0b5faf9308e590865dae51bf31fdd342e" } wire_serialized_transaction_test_data = "1.0.0" config = { version = "0.15", default-features = false, features = ["toml"] } From 4285738fa7ee65150ab7d9952dbf8a9a932675c4 Mon Sep 17 00:00:00 2001 From: zancas Date: Thu, 9 Jul 2026 12:56:50 -0700 Subject: [PATCH 45/58] build: record zaino-common's rustls dependency in the lockfile The dev merge took the incoming Cargo.lock, which predates this branch keeping rustls in zaino-common (ADR-0006); cargo regenerates the entry on the first build. Co-Authored-By: Claude Fable 5 --- Cargo.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.lock b/Cargo.lock index fe728fec8..8398f82f9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5941,6 +5941,7 @@ dependencies = [ name = "zaino-common" version = "0.2.0" dependencies = [ + "rustls", "serde", "time", "tracing", From 5bc718057de458853721f02cd26e7aa52fb06ded Mon Sep 17 00:00:00 2001 From: zancas Date: Thu, 9 Jul 2026 13:03:08 -0700 Subject: [PATCH 46/58] refactor: derive the adoption plumbing from the mirror macro MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The NetworkUpgrade-variant ↔ ActivationHeights-field correspondence is recorded exactly once, in zaino-common's activation_heights_mirror! macro, but activation_heights_from_upgrades restated it as a hand-written thirteen-arm match and both it and its tests restated the all-None schedule as field-by-field literals. The macro now generates the two missing derived items — the NEVER_ACTIVATED schedule and the per-upgrade slot_mut accessor — and the adoption function collapses onto them. The generated match stays exhaustive, so a new zebra upgrade variant still fails the build until its pair is added to the mirror. Also remove the dead ChainIndexConfig::new: it had no callers and was the one constructor shape that invited supplying a network from somewhere other than validator adoption; from_backend_config is now the sole named production constructor (zaino#1076). Co-Authored-By: Claude Fable 5 --- packages/zaino-common/src/config/network.rs | 27 ++++++++++- .../chain_index/source/validator_connector.rs | 48 ++----------------- packages/zaino-state/src/config.rs | 16 ------- 3 files changed, 31 insertions(+), 60 deletions(-) diff --git a/packages/zaino-common/src/config/network.rs b/packages/zaino-common/src/config/network.rs index 5983e5580..a50cb31e0 100644 --- a/packages/zaino-common/src/config/network.rs +++ b/packages/zaino-common/src/config/network.rs @@ -98,7 +98,9 @@ impl Default for ActivationHeights { /// Records the `NetworkUpgrade`-variant ↔ `ActivationHeights`-field correspondence /// exactly once, generating everything derived from it: the two field-by-field /// `From` conversions between [`ActivationHeights`] and zebra's -/// [`ConfiguredActivationHeights`] (the structs share field names). +/// [`ConfiguredActivationHeights`] (the structs share field names), the +/// all-`None` [`ActivationHeights::NEVER_ACTIVATED`] schedule, and the +/// per-upgrade [`ActivationHeights::slot_mut`] accessor. /// /// A declarative macro rather than functions because plain `fn`s cannot abstract /// over struct fields, and the variant/field spellings (`Nu5`/`nu5`) differ only @@ -124,6 +126,29 @@ macro_rules! activation_heights_mirror { Self { $($field),* } } } + + impl ActivationHeights { + /// The all-`None` schedule: every upgrade never-activated. The + /// starting point for building heights from an external report, + /// where an upgrade absent from the report must stay unset. + pub const NEVER_ACTIVATED: Self = Self { $($field: None),* }; + + /// Mutable slot for `upgrade`'s activation height, or `None` for + /// `Genesis` (height 0 by definition; it has no slot). + pub fn slot_mut( + &mut self, + upgrade: zebra_chain::parameters::NetworkUpgrade, + ) -> Option<&mut Option> { + match upgrade { + zebra_chain::parameters::NetworkUpgrade::Genesis => None, + $( + zebra_chain::parameters::NetworkUpgrade::$variant => { + Some(&mut self.$field) + } + )* + } + } + } }; } diff --git a/packages/zaino-state/src/chain_index/source/validator_connector.rs b/packages/zaino-state/src/chain_index/source/validator_connector.rs index 919dc9c6f..1aff0432f 100644 --- a/packages/zaino-state/src/chain_index/source/validator_connector.rs +++ b/packages/zaino-state/src/chain_index/source/validator_connector.rs @@ -313,37 +313,12 @@ async fn adopt_network( fn activation_heights_from_upgrades( upgrades: &IndexMap, ) -> Result { - let mut heights = zaino_common::config::network::ActivationHeights { - before_overwinter: None, - overwinter: None, - sapling: None, - blossom: None, - heartwood: None, - canopy: None, - nu5: None, - nu6: None, - nu6_1: None, - nu6_2: None, - nu6_3: None, - nu7: None, - }; + let mut heights = zaino_common::config::network::ActivationHeights::NEVER_ACTIVATED; for upgrade_info in upgrades.values() { let (upgrade, height, _status) = upgrade_info.into_parts(); - let slot = match upgrade { - // Genesis is height 0 by definition; it has no configuration slot. - NetworkUpgrade::Genesis => continue, - NetworkUpgrade::BeforeOverwinter => &mut heights.before_overwinter, - NetworkUpgrade::Overwinter => &mut heights.overwinter, - NetworkUpgrade::Sapling => &mut heights.sapling, - NetworkUpgrade::Blossom => &mut heights.blossom, - NetworkUpgrade::Heartwood => &mut heights.heartwood, - NetworkUpgrade::Canopy => &mut heights.canopy, - NetworkUpgrade::Nu5 => &mut heights.nu5, - NetworkUpgrade::Nu6 => &mut heights.nu6, - NetworkUpgrade::Nu6_1 => &mut heights.nu6_1, - NetworkUpgrade::Nu6_2 => &mut heights.nu6_2, - NetworkUpgrade::Nu6_3 => &mut heights.nu6_3, - NetworkUpgrade::Nu7 => &mut heights.nu7, + // Genesis is height 0 by definition; it has no configuration slot. + let Some(slot) = heights.slot_mut(upgrade) else { + continue; }; if slot.replace(height.0).is_some() { return Err(format!("validator reported {upgrade:?} twice")); @@ -2792,20 +2767,7 @@ mod activation_heights_from_upgrades { /// All-`None` heights: the starting point adoption fills from the /// validator's map, and the expected value for every absent upgrade. - const NEVER_ACTIVATED: ActivationHeights = ActivationHeights { - before_overwinter: None, - overwinter: None, - sapling: None, - blossom: None, - heartwood: None, - canopy: None, - nu5: None, - nu6: None, - nu6_1: None, - nu6_2: None, - nu6_3: None, - nu7: None, - }; + const NEVER_ACTIVATED: ActivationHeights = ActivationHeights::NEVER_ACTIVATED; fn upgrades_map( json: &str, diff --git a/packages/zaino-state/src/config.rs b/packages/zaino-state/src/config.rs index 668f0dd82..3cfff0d76 100644 --- a/packages/zaino-state/src/config.rs +++ b/packages/zaino-state/src/config.rs @@ -251,22 +251,6 @@ pub struct ChainIndexConfig { } impl ChainIndexConfig { - /// Returns a new instance of [`ChainIndexConfig`]. - #[allow(dead_code)] - pub fn new( - storage: StorageConfig, - db_version: u32, - network: zebra_chain::parameters::Network, - ephemeral: bool, - ) -> Self { - ChainIndexConfig { - storage, - db_version, - network, - ephemeral, - } - } - /// Builds the chain-index config from the backend config plus the /// runtime network. The backend config carries only a network kind, so /// the adopted runtime network arrives as its own argument — there is From ef3ed37a0a95780c4a76248a5b4ac77bdb5944ff Mon Sep 17 00:00:00 2001 From: zancas Date: Thu, 9 Jul 2026 13:46:53 -0700 Subject: [PATCH 47/58] feat: verify the public-network schedule against the validator at spawn MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Public-network activation heights come only from checked-in, ecosystem-shared consensus code; regtest parameters are configured on the validator at scenario launch and every other component adopts them from it. adopt_network previously enforced the regtest half but took the compiled Mainnet/Testnet parameters on faith: a validator running a *configured* testnet (zebra supports custom testnet activation heights) would silently drift from the index — the exact bug class this branch exists to kill, on a different network kind. adopt_network now fetches getblockchaininfo on every network kind and, on Mainnet and Testnet, verifies each reported (upgrade, height) pair against the compiled parameters before trusting them, failing loud on the first disagreement. Only reported entries are checked, not report completeness: the upgrades map is branch-ID-keyed, so unconfigured upgrades are simply absent and an older validator may not know the newest ones. Also document two acceptance decisions the spec made but the code did not state: adoption happens exactly once at spawn (a validator restarted mid-session with a different schedule is out of scope), and before_overwinter is always None when adopted (BeforeOverwinter has no consensus branch ID, so it can never appear in the report; zebra's new_regtest supplies its handling). Co-Authored-By: Claude Fable 5 --- .../chain_index/source/validator_connector.rs | 148 ++++++++++++++++-- 1 file changed, 133 insertions(+), 15 deletions(-) diff --git a/packages/zaino-state/src/chain_index/source/validator_connector.rs b/packages/zaino-state/src/chain_index/source/validator_connector.rs index 1aff0432f..6ad37eb65 100644 --- a/packages/zaino-state/src/chain_index/source/validator_connector.rs +++ b/packages/zaino-state/src/chain_index/source/validator_connector.rs @@ -270,24 +270,47 @@ impl ValidatorConnector { /// /// The validator is the single source of truth for activation heights /// (zaino#1076): the config carries only a network kind, and the runtime -/// network is constructed here before anything consumes one — from zebra's -/// compiled parameters for the public networks, from the validator's -/// reported schedule for regtest. There is no fallback: a silently wrong -/// schedule is the failure mode this removes. +/// network is constructed here before anything consumes one. On regtest the +/// schedule is adopted wholesale from the validator's report. On Mainnet and +/// Testnet the compiled zebra parameters are used, but the validator's +/// reported schedule is verified against them first, so a validator running +/// a *configured* testnet (custom activation heights) fails loud here +/// instead of silently drifting from the index. There is no fallback: a +/// silently wrong schedule is the failure mode this removes. +/// +/// Adoption happens exactly once, at spawn. A validator restarted +/// mid-session with a different schedule is out of scope (the test harness +/// restarts validator and indexer together); it would invalidate the index +/// without being re-detected here. async fn adopt_network( common: &CommonBackendConfig, rpc_client: &JsonRpSeeConnector, ) -> Result { - Ok(match common.network { - zaino_common::Network::Mainnet => zebra_chain::parameters::Network::Mainnet, - zaino_common::Network::Testnet => zebra_chain::parameters::Network::new_default_testnet(), + let blockchain_info = rpc_client.get_blockchain_info().await.map_err(|error| { + BlockchainSourceError::Unrecoverable(format!( + "cannot fetch activation heights from the validator at {}: {error}", + common.validator_rpc_address + )) + })?; + + // Shared by the Mainnet/Testnet arms: the compiled network is only + // trusted after the validator's report agrees with it. + let verified = |network: zebra_chain::parameters::Network| { + verify_reported_upgrades(&network, &blockchain_info.upgrades).map_err(|reason| { + BlockchainSourceError::Unrecoverable(format!( + "the validator at {} disagrees with the compiled {network} parameters: {reason}", + common.validator_rpc_address + )) + })?; + Ok(network) + }; + + match common.network { + zaino_common::Network::Mainnet => verified(zebra_chain::parameters::Network::Mainnet), + zaino_common::Network::Testnet => { + verified(zebra_chain::parameters::Network::new_default_testnet()) + } zaino_common::Network::Regtest => { - let blockchain_info = rpc_client.get_blockchain_info().await.map_err(|error| { - BlockchainSourceError::Unrecoverable(format!( - "cannot fetch activation heights from the validator at {}: {error}", - common.validator_rpc_address - )) - })?; let heights = activation_heights_from_upgrades(&blockchain_info.upgrades).map_err(|reason| { BlockchainSourceError::Unrecoverable(format!( @@ -296,9 +319,35 @@ async fn adopt_network( )) })?; info!(?heights, "Adopted activation heights from the validator"); - heights.to_regtest_network() + Ok(heights.to_regtest_network()) } - }) + } +} + +/// Checks every `(upgrade, height)` the validator reports against `network`'s +/// compiled activation schedule, rejecting the first disagreement. +/// +/// Only reported entries are checked — the report's *completeness* is not: +/// the map is keyed by consensus branch ID, so upgrades zebra considers +/// unconfigured are simply absent, and an older validator may not know the +/// newest upgrades yet. +fn verify_reported_upgrades( + network: &zebra_chain::parameters::Network, + upgrades: &IndexMap, +) -> Result<(), String> { + for upgrade_info in upgrades.values() { + let (upgrade, height, _status) = upgrade_info.into_parts(); + let compiled = upgrade.activation_height(network); + if compiled != Some(height) { + return Err(format!( + "the validator reports {upgrade:?} activating at height {}, \ + but the compiled parameters say {:?}", + height.0, + compiled.map(|compiled_height| compiled_height.0) + )); + } + } + Ok(()) } /// Builds the regtest activation heights from the validator's reported @@ -310,6 +359,12 @@ async fn adopt_network( /// `Network` (zaino#1076). An upgrade absent from the validator's map is /// never-activated — nothing is backfilled from defaults. Mainnet and /// Testnet use zebra's compiled parameters and never take this path. +/// +/// `before_overwinter` is always `None` here: the map is keyed by consensus +/// branch ID, which `BeforeOverwinter` does not have, so it can never appear +/// in the report. Correctness relies on zebra's `new_regtest` supplying its +/// own `BeforeOverwinter` handling when `to_regtest_network` builds the +/// runtime network from these heights. fn activation_heights_from_upgrades( upgrades: &IndexMap, ) -> Result { @@ -2865,3 +2920,66 @@ mod activation_heights_from_upgrades { ); } } + +#[cfg(test)] +mod verify_reported_upgrades { + fn upgrades_map( + json: &str, + ) -> indexmap::IndexMap< + zebra_rpc::methods::ConsensusBranchIdHex, + zebra_rpc::methods::NetworkUpgradeInfo, + > { + serde_json::from_str(json).expect("upgrades fixture parses") + } + + /// A report agreeing with the compiled schedule passes. + #[test] + fn accepts_a_report_matching_the_compiled_schedule() { + let upgrades = upgrades_map( + r#"{ + "76b809bb": { "name": "Sapling", "activationheight": 419200, "status": "active" } + }"#, + ); + + super::verify_reported_upgrades(&zebra_chain::parameters::Network::Mainnet, &upgrades) + .expect("mainnet Sapling at 419200 matches the compiled parameters"); + } + + /// A reported height disagreeing with the compiled schedule fails loud — + /// the configured-testnet / wrong-network drift class. + #[test] + fn rejects_a_mismatched_height() { + let upgrades = upgrades_map( + r#"{ + "76b809bb": { "name": "Sapling", "activationheight": 419201, "status": "active" } + }"#, + ); + + let reason = + super::verify_reported_upgrades(&zebra_chain::parameters::Network::Mainnet, &upgrades) + .expect_err("a wrong Sapling height must be rejected"); + assert!( + reason.contains("419201"), + "error should name the reported height, got: {reason}" + ); + } + + /// An upgrade the compiled schedule never activates must also be + /// rejected when the validator reports a height for it. + #[test] + fn rejects_an_upgrade_the_compiled_schedule_lacks() { + let upgrades = upgrades_map( + r#"{ + "77190ad8": { "name": "NU7", "activationheight": 123, "status": "pending" } + }"#, + ); + + let reason = + super::verify_reported_upgrades(&zebra_chain::parameters::Network::Mainnet, &upgrades) + .expect_err("an unscheduled upgrade with a reported height must be rejected"); + assert!( + reason.contains("None"), + "error should show the compiled schedule has no height, got: {reason}" + ); + } +} From 0713cfb1f8c11dccdb1da53c8f443b1048158520 Mon Sep 17 00:00:00 2001 From: zancas Date: Thu, 9 Jul 2026 13:59:10 -0700 Subject: [PATCH 48/58] refactor!: name The Public Testnet exactly, everywhere it is meant The public shared-consensus test network and a hermetic regtest chain are categorically different things, and the bare word "testnet" has named both. Where the intention is the public network, the canonical vocabulary is now: "The Public Testnet" in prose, `the_pub_testnet` in identifiers, and `PubTestnet` in types. The CONTEXT.md glossary entry records the rule. The config kind renames to `Network::PubTestnet`. Its canonical config spelling becomes `"PubTestnet"`, with `#[serde(alias = "Testnet")]` keeping existing config files parsing; a zainod config test pins the legacy spelling. The Public-Testnet chain-cache static renames to ZEBRAD_THE_PUB_TESTNET_CACHE_DIR, and the boundary-walk suite renames to the_pub_testnet_ironwood_boundary (the Makefile ironwood filter follows). References to zebra's own types and config format (parameters::Network::Testnet, NetworkType::Testnet, zebrad.toml) are foreign vocabulary and keep their spelling. Co-Authored-By: Claude Fable 5 --- CONTEXT.md | 11 +++-- Makefile.toml | 2 +- live-tests/clientless/tests/state_service.rs | 10 ++-- ...s => the_pub_testnet_ironwood_boundary.rs} | 48 +++++++++--------- live-tests/e2e/tests/ironwood_activation.rs | 4 +- live-tests/zaino-testutils/src/lib.rs | 8 +-- packages/zaino-common/src/config/network.rs | 23 ++++++--- .../chain_index/source/validator_connector.rs | 16 +++--- packages/zainod/src/config.rs | 49 +++++++++++++++---- zainod-heights-from-validator-spec.md | 6 +-- 10 files changed, 109 insertions(+), 68 deletions(-) rename live-tests/clientless/tests/{testnet_ironwood_boundary.rs => the_pub_testnet_ironwood_boundary.rs} (73%) diff --git a/CONTEXT.md b/CONTEXT.md index 4b4e11ba8..6246105d7 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -32,13 +32,14 @@ _Avoid_: stale version, forgotten bump ### Chains and networks -**Testnet**: -The public Zcash test network, and only that. Testnet regimes are +**The Public Testnet**: +The public Zcash test network, and only that. Its regimes are non-hermetic — state is shared with other participants, and an epoch the public chain has left (e.g. pre-NU6.3 once NU6.3 activates there) -cannot be re-entered. -_Avoid_: "testnet" for any locally-launched chain, even one launched -under a testnet network kind +cannot be re-entered. Always this exact phrase in prose; identifiers +use `the_pub_testnet` and types use `PubTestnet`. +_Avoid_: bare "Testnet"/"testnet", and especially "testnet" for any +locally-launched chain, even one launched under a testnet network kind **Regtest net**: A hermetic, locally-launched chain whose activation heights the diff --git a/Makefile.toml b/Makefile.toml index a2a9b9bd9..83e1c47ea 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -197,7 +197,7 @@ case "$set" in # invoked directly rather than through the `live` summary runner, which # forwards no nextest args. Run every set even if an earlier one fails, # as in `all`. - ironwood_filter='binary(ironwood_activation) | binary(testnet_ironwood_boundary) | test(ironwood)' + ironwood_filter='binary(ironwood_activation) | binary(the_pub_testnet_ironwood_boundary) | test(ironwood)' rc=0 makers container-test -E "$ironwood_filter" "$@" || rc=1 makers live-clientless -E "$ironwood_filter" "$@" || rc=1 diff --git a/live-tests/clientless/tests/state_service.rs b/live-tests/clientless/tests/state_service.rs index c1ccbd6c5..59b607d40 100644 --- a/live-tests/clientless/tests/state_service.rs +++ b/live-tests/clientless/tests/state_service.rs @@ -2,7 +2,7 @@ use zaino_fetch::jsonrpsee::response::address_deltas::GetAddressDeltasParams; use zaino_state::{LightWalletIndexer, NodeBackedIndexerServiceSubscriber, ZcashIndexer}; use zaino_testutils::{StateAndFetchServices, ValidatorExt}; -use zaino_testutils::{ValidatorKind, ZEBRAD_TESTNET_CACHE_DIR}; +use zaino_testutils::{ValidatorKind, ZEBRAD_THE_PUB_TESTNET_CACHE_DIR}; use zcash_local_net::validator::zebrad::Zebrad; use zebra_chain::parameters::NetworkKind; use zebra_rpc::methods::{GetAddressBalanceRequest, GetAddressTxIdsRequest}; @@ -23,7 +23,7 @@ async fn launch_regtest(enable_zaino: bool) -> StateAndFetchServices { async fn launch_testnet_cached() -> StateAndFetchServices { zaino_testutils::launch_state_and_fetch_services::( &ValidatorKind::Zebrad, - ZEBRAD_TESTNET_CACHE_DIR.clone(), + ZEBRAD_THE_PUB_TESTNET_CACHE_DIR.clone(), false, Some(NetworkKind::Testnet), ) @@ -427,7 +427,7 @@ mod zebra { async fn testnet() { state_service_check_info::( &ValidatorKind::Zebrad, - ZEBRAD_TESTNET_CACHE_DIR.clone(), + ZEBRAD_THE_PUB_TESTNET_CACHE_DIR.clone(), NetworkKind::Testnet, ) .await; @@ -612,7 +612,7 @@ mod zebra { async fn block_object_testnet() { state_service_get_block_object( &ValidatorKind::Zebrad, - ZEBRAD_TESTNET_CACHE_DIR.clone(), + ZEBRAD_THE_PUB_TESTNET_CACHE_DIR.clone(), NetworkKind::Testnet, ) .await; @@ -628,7 +628,7 @@ mod zebra { async fn block_raw_testnet() { state_service_get_block_raw( &ValidatorKind::Zebrad, - ZEBRAD_TESTNET_CACHE_DIR.clone(), + ZEBRAD_THE_PUB_TESTNET_CACHE_DIR.clone(), NetworkKind::Testnet, ) .await; diff --git a/live-tests/clientless/tests/testnet_ironwood_boundary.rs b/live-tests/clientless/tests/the_pub_testnet_ironwood_boundary.rs similarity index 73% rename from live-tests/clientless/tests/testnet_ironwood_boundary.rs rename to live-tests/clientless/tests/the_pub_testnet_ironwood_boundary.rs index 08b5a7f7f..41833480d 100644 --- a/live-tests/clientless/tests/testnet_ironwood_boundary.rs +++ b/live-tests/clientless/tests/the_pub_testnet_ironwood_boundary.rs @@ -1,7 +1,7 @@ -//! Observational walk of the real NU6.3 activation boundary on the public -//! testnet (glossary: "testnet" means the public test network and nothing -//! else). Historical blocks are permanent, so these invariants are checkable -//! forever, long after the epoch that produced them closed. +//! Observational walk of the real NU6.3 activation boundary on The Public +//! Testnet (glossary: The Public Testnet is the public test network and +//! nothing else). Historical blocks are permanent, so these invariants are +//! checkable forever, long after the epoch that produced them closed. //! //! The invariants, enforced one block below the boundary and at it: //! @@ -13,33 +13,33 @@ //! //! The activation height itself is read from the running validator's //! `getblockchaininfo.upgrades` — the single source of truth for activation -//! heights — and cross-checked against the observed public-testnet -//! activation at height 4,134,000 (~2026-07-04). +//! heights — and cross-checked against the observed activation on The +//! Public Testnet at height 4,134,000 (~2026-07-04). //! //! Non-hermetic and therefore env-gated: the test needs a zebrad chain -//! cache at `~/.cache/zebra` (`ZEBRAD_TESTNET_CACHE_DIR`) synced past the +//! cache at `~/.cache/zebra` (`ZEBRAD_THE_PUB_TESTNET_CACHE_DIR`) synced past the //! activation height, and skips with a message when the cache is absent or //! short. Run it manually, or from a job that maintains the cache. //! //! The validator is launched through `ZebradConfig` directly rather than //! `TestManager`: the manager's launch path always writes regtest test -//! parameters into the config, while this test needs the public testnet +//! parameters into the config, while this test needs The Public Testnet //! (`network_type: NetworkType::Testnet`, which makes zebrad ignore //! `activation_heights` and `miner_address`). use zaino_fetch::jsonrpsee::connector::{test_node_and_return_url, JsonRpSeeConnector}; use zaino_fetch::jsonrpsee::response::GetBlockResponse; -use zaino_testutils::ZEBRAD_TESTNET_CACHE_DIR; +use zaino_testutils::ZEBRAD_THE_PUB_TESTNET_CACHE_DIR; use zcash_local_net::process::Process as _; use zcash_local_net::protocol::NetworkType; use zcash_local_net::validator::zebrad::{Zebrad, ZebradConfig}; use zcash_local_net::validator::Validator as _; use zebra_chain::parameters::NetworkUpgrade; -/// The height the public testnet activated NU6.3 at, recorded from the real -/// activation. A mismatch means either a testnet reset or a wrong validator -/// pin — both worth failing loudly over. -const OBSERVED_TESTNET_NU6_3_ACTIVATION: u32 = 4_134_000; +/// The height The Public Testnet activated NU6.3 at, recorded from the real +/// activation. A mismatch means either a reset of The Public Testnet or a +/// wrong validator pin — both worth failing loudly over. +const OBSERVED_NU6_3_ACTIVATION_ON_THE_PUB_TESTNET: u32 = 4_134_000; /// The chain value of `pool_id` as of `height`, from the validator's /// verbosity-2 block object. @@ -65,14 +65,14 @@ async fn pool_zats(connector: &JsonRpSeeConnector, height: u32, pool_id: &str) - /// multi_thread required: the test launches the validator process and polls /// it over RPC. #[tokio::test(flavor = "multi_thread")] -async fn value_pools_respect_the_boundary_on_testnet() { - let Some(cache_dir) = ZEBRAD_TESTNET_CACHE_DIR.clone() else { - eprintln!("skipping: no testnet cache dir configured"); +async fn value_pools_respect_the_boundary_on_the_pub_testnet() { + let Some(cache_dir) = ZEBRAD_THE_PUB_TESTNET_CACHE_DIR.clone() else { + eprintln!("skipping: no cache dir configured for The Public Testnet"); return; }; if !cache_dir.exists() { eprintln!( - "skipping: no zebrad testnet chain cache at {}", + "skipping: no zebrad chain cache of The Public Testnet at {}", cache_dir.display() ); return; @@ -83,7 +83,9 @@ async fn value_pools_respect_the_boundary_on_testnet() { chain_cache: Some(cache_dir), ..ZebradConfig::default() }; - let mut zebrad = Zebrad::launch(config).await.expect("launch testnet zebrad"); + let mut zebrad = Zebrad::launch(config) + .await + .expect("launch a zebrad on The Public Testnet"); let rpc_address = format!("127.0.0.1:{}", zebrad.get_port()); let connector = JsonRpSeeConnector::new_with_basic_auth( @@ -106,7 +108,7 @@ async fn value_pools_respect_the_boundary_on_testnet() { .expect("getblockchaininfo"); // The validator's reported schedule is the source of truth for the - // boundary; the recorded constant pins the real public-testnet history. + // boundary; the recorded constant pins the real history of The Public Testnet. let boundary = blockchain_info .upgrades .values() @@ -114,16 +116,16 @@ async fn value_pools_respect_the_boundary_on_testnet() { let (upgrade, height, _status) = upgrade_info.into_parts(); (upgrade == NetworkUpgrade::Nu6_3).then_some(height.0) }) - .expect("the testnet validator must report an NU6.3 activation height"); + .expect("the validator on The Public Testnet must report an NU6.3 activation height"); assert_eq!( - boundary, OBSERVED_TESTNET_NU6_3_ACTIVATION, - "the validator's NU6.3 height must match the observed testnet activation" + boundary, OBSERVED_NU6_3_ACTIVATION_ON_THE_PUB_TESTNET, + "the validator's NU6.3 height must match the observed activation on The Public Testnet" ); let tip = blockchain_info.blocks.0; if tip <= boundary { eprintln!( - "skipping: testnet cache tip {tip} has not crossed the NU6.3 boundary {boundary}" + "skipping: cache tip {tip} of The Public Testnet has not crossed the NU6.3 boundary {boundary}" ); zebrad.stop(); return; diff --git a/live-tests/e2e/tests/ironwood_activation.rs b/live-tests/e2e/tests/ironwood_activation.rs index 6d2fbc6b1..a5db94ea1 100644 --- a/live-tests/e2e/tests/ironwood_activation.rs +++ b/live-tests/e2e/tests/ironwood_activation.rs @@ -2,7 +2,7 @@ //! //! Every test here runs a devtool wallet on //! [`ORCHARD_THEN_IRONWOOD_ACTIVATION_HEIGHTS`] — the hermetic replay of what -//! the public testnet did once at height 4,134,000: heights 2 through 5 are +//! The Public Testnet did once at height 4,134,000: heights 2 through 5 are //! Orchard era, [`NU6_3_TRANSITION_BOUNDARY`] (6) onward is Ironwood era. //! The wallets derive their activation schedule from the running validator //! (`WalletNetwork::from_validator`, infrastructure ADR 0003), so the @@ -29,7 +29,7 @@ //! wire in `compact_block_wire.rs`; this file owns the cells that need a //! wallet on both sides of the boundary. //! -//! The public testnet cannot host the migration cell for us: its pre-NU6.3 +//! The Public Testnet cannot host the migration cell for us: its pre-NU6.3 //! epoch closed at height 4,134,000, no new value may enter Orchard from //! there (post-activation Orchard actions permit only same-receiver change //! or withdrawal — the cross-address restriction, diff --git a/live-tests/zaino-testutils/src/lib.rs b/live-tests/zaino-testutils/src/lib.rs index ce33203fc..78ecf274c 100644 --- a/live-tests/zaino-testutils/src/lib.rs +++ b/live-tests/zaino-testutils/src/lib.rs @@ -473,8 +473,8 @@ pub static ZEBRAD_CHAIN_CACHE_DIR: Lazy> = Lazy::new(|| { Some(workspace_root_path.join("live-tests/chain_cache/client_rpc_tests_large")) }); -/// Path for the Zebra chain cache in the user's home directory. -pub static ZEBRAD_TESTNET_CACHE_DIR: Lazy> = Lazy::new(|| { +/// Path for the Zebra chain cache of The Public Testnet in the user's home directory. +pub static ZEBRAD_THE_PUB_TESTNET_CACHE_DIR: Lazy> = Lazy::new(|| { let home_path = PathBuf::from(std::env::var("HOME").unwrap()); Some(home_path.join(".cache/zebra")) }); @@ -736,7 +736,7 @@ where // that adoption. let zaino_network_kind = match network_kind { NetworkKind::Mainnet => Network::Mainnet, - NetworkKind::Testnet => Network::Testnet, + NetworkKind::Testnet => Network::PubTestnet, NetworkKind::Regtest => Network::Regtest, }; @@ -1159,7 +1159,7 @@ pub async fn launch_state_and_fetch_services_mining_to( Some(NetworkKind::Testnet) => { println!("Waiting for validator to spawn.."); tokio::time::sleep(std::time::Duration::from_millis(5000)).await; - Network::Testnet + Network::PubTestnet } // The kind suffices: zainod adopts the schedule from the validator // at spawn (zaino#1076). diff --git a/packages/zaino-common/src/config/network.rs b/packages/zaino-common/src/config/network.rs index a50cb31e0..61a3dd294 100644 --- a/packages/zaino-common/src/config/network.rs +++ b/packages/zaino-common/src/config/network.rs @@ -16,8 +16,12 @@ use zebra_chain::parameters::testnet::ConfiguredActivationHeights; pub enum Network { /// Mainnet network Mainnet, - /// Testnet network - Testnet, + /// The Public Testnet: the shared-consensus test network, whose + /// activation heights come only from checked-in zebra parameters — + /// never from configuration. Accepts the legacy config spelling + /// `"Testnet"`. + #[serde(alias = "Testnet")] + PubTestnet, /// Regtest network (for local testing) Regtest, } @@ -26,13 +30,13 @@ impl fmt::Display for Network { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Network::Mainnet => write!(f, "Mainnet"), - Network::Testnet => write!(f, "Testnet"), + Network::PubTestnet => write!(f, "PubTestnet"), Network::Regtest => write!(f, "Regtest"), } } } -/// Configurable activation heights for Regtest and configured Testnets. +/// Configurable activation heights for a Regtest validator launch. /// /// We use our own type instead of the zebra type /// as the zebra type is missing a number of useful @@ -170,12 +174,15 @@ activation_heights_mirror!( impl Network { /// Determines if we should wait for the server to fully sync. Used for testing /// - /// - Mainnet/Testnet: Skip sync (false) because we don't want to sync real chains in tests + /// - Mainnet / The Public Testnet: Skip sync (false) because we don't want + /// to sync real chains in tests /// - Regtest: Enable sync (true) because regtest is local and fast to sync pub fn wait_on_server_sync(&self) -> bool { match self { - Network::Mainnet | Network::Testnet => false, // Real networks - don't try to sync the whole chain - Network::Regtest => true, // Local network - safe and fast to sync + // Real networks - don't try to sync the whole chain + Network::Mainnet | Network::PubTestnet => false, + // Local network - safe and fast to sync + Network::Regtest => true, } } } @@ -188,7 +195,7 @@ impl From for Network { if parameters.is_regtest() { Network::Regtest } else { - Network::Testnet + Network::PubTestnet } } } diff --git a/packages/zaino-state/src/chain_index/source/validator_connector.rs b/packages/zaino-state/src/chain_index/source/validator_connector.rs index 6ad37eb65..e2130c218 100644 --- a/packages/zaino-state/src/chain_index/source/validator_connector.rs +++ b/packages/zaino-state/src/chain_index/source/validator_connector.rs @@ -272,9 +272,10 @@ impl ValidatorConnector { /// (zaino#1076): the config carries only a network kind, and the runtime /// network is constructed here before anything consumes one. On regtest the /// schedule is adopted wholesale from the validator's report. On Mainnet and -/// Testnet the compiled zebra parameters are used, but the validator's -/// reported schedule is verified against them first, so a validator running -/// a *configured* testnet (custom activation heights) fails loud here +/// The Public Testnet the compiled zebra parameters are used, but the +/// validator's reported schedule is verified against them first, so a +/// validator reporting custom activation heights under the `PubTestnet` +/// kind (a regtest net in configured-testnet clothing) fails loud here /// instead of silently drifting from the index. There is no fallback: a /// silently wrong schedule is the failure mode this removes. /// @@ -293,7 +294,7 @@ async fn adopt_network( )) })?; - // Shared by the Mainnet/Testnet arms: the compiled network is only + // Shared by the Mainnet / The Public Testnet arms: the compiled network is only // trusted after the validator's report agrees with it. let verified = |network: zebra_chain::parameters::Network| { verify_reported_upgrades(&network, &blockchain_info.upgrades).map_err(|reason| { @@ -307,7 +308,7 @@ async fn adopt_network( match common.network { zaino_common::Network::Mainnet => verified(zebra_chain::parameters::Network::Mainnet), - zaino_common::Network::Testnet => { + zaino_common::Network::PubTestnet => { verified(zebra_chain::parameters::Network::new_default_testnet()) } zaino_common::Network::Regtest => { @@ -358,7 +359,8 @@ fn verify_reported_upgrades( /// runtime network here at first contact, before anything consumes a /// `Network` (zaino#1076). An upgrade absent from the validator's map is /// never-activated — nothing is backfilled from defaults. Mainnet and -/// Testnet use zebra's compiled parameters and never take this path. +/// The Public Testnet use zebra's compiled parameters and never take this +/// path. /// /// `before_overwinter` is always `None` here: the map is keyed by consensus /// branch ID, which `BeforeOverwinter` does not have, so it can never appear @@ -2946,7 +2948,7 @@ mod verify_reported_upgrades { } /// A reported height disagreeing with the compiled schedule fails loud — - /// the configured-testnet / wrong-network drift class. + /// the wrong-schedule / wrong-network drift class. #[test] fn rejects_a_mismatched_height() { let upgrades = upgrades_map( diff --git a/packages/zainod/src/config.rs b/packages/zainod/src/config.rs index 315eaf804..889359c65 100644 --- a/packages/zainod/src/config.rs +++ b/packages/zainod/src/config.rs @@ -103,7 +103,8 @@ pub struct ZainodConfig { /// When enabled, Zaino does not use a persistent on-disk finalised-state database. Finalised /// state reads are served from the configured validator/source instead. pub ephemeral_finalised_state: bool, - /// Network to connect to (Mainnet, Testnet, or Regtest). + /// Network to connect to (Mainnet, PubTestnet — The Public Testnet — or Regtest; + /// `"Testnet"` is accepted as a legacy spelling of PubTestnet). pub network: Network, /// Prometheus metrics endpoint listen address. /// @@ -266,7 +267,7 @@ impl Default for ZainodConfig { storage: StorageConfig::default(), ephemeral_finalised_state: false, zebra_db_path: default_zebra_db_path(), - network: Network::Testnet, + network: Network::PubTestnet, donation_address: None, } } @@ -602,7 +603,7 @@ key_path = "{}" let toml_content = r#" backend = "state" -network = "Testnet" +network = "PubTestnet" zebra_db_path = "/opt/zebra/data" [storage.database] @@ -621,6 +622,7 @@ listen_address = "127.0.0.1:8137" // legacy `backend = "state"` still parses via the serde alias assert_eq!(config.backend, BackendType::Direct); + assert_eq!(config.network, Network::PubTestnet); assert!(config.json_server_settings.is_none()); assert_eq!( config.validator_settings.validator_user, @@ -632,6 +634,33 @@ listen_address = "127.0.0.1:8137" ); } + /// The pre-rename config spelling of The Public Testnet still parses, + /// via the `#[serde(alias = "Testnet")]` on `Network::PubTestnet`. + #[test] + fn legacy_testnet_spelling_parses_as_the_pub_testnet() { + let _guard = EnvGuard::new(); + let temp_dir = TempDir::new().unwrap(); + + let toml_content = r#" +backend = "state" +network = "Testnet" +zebra_db_path = "/opt/zebra/data" + +[storage.database] +path = "/opt/zaino/data" + +[validator_settings] +validator_jsonrpc_listen_address = "127.0.0.1:18232" + +[grpc_settings] +listen_address = "127.0.0.1:8137" +"#; + + let config_path = create_test_config_file(&temp_dir, toml_content, "legacy_testnet.toml"); + let config = load_config(&config_path).expect("load_config failed"); + assert_eq!(config.network, Network::PubTestnet); + } + #[test] fn test_cookie_dir_logic() { let _guard = EnvGuard::new(); @@ -640,7 +669,7 @@ listen_address = "127.0.0.1:8137" // Scenario 1: auth enabled, cookie_dir empty (should use default ephemeral path) let toml_content = r#" backend = "fetch" -network = "Testnet" +network = "PubTestnet" zebra_db_path = "/zebra/db" [storage.database] @@ -670,7 +699,7 @@ listen_address = "127.0.0.1:8137" // Scenario 2: auth enabled, cookie_dir specified let toml_content2 = r#" backend = "fetch" -network = "Testnet" +network = "PubTestnet" zebra_db_path = "/zebra/db" [storage.database] @@ -697,7 +726,7 @@ listen_address = "127.0.0.1:8137" // Scenario 3: cookie_dir not specified (should be None) let toml_content3 = r#" backend = "fetch" -network = "Testnet" +network = "PubTestnet" zebra_db_path = "/zebra/db" [storage.database] @@ -808,7 +837,7 @@ listen_address = "127.0.0.1:8137" let temp_dir = TempDir::new().unwrap(); let toml_content = r#" -network = "Testnet" +network = "PubTestnet" [validator_settings] validator_jsonrpc_listen_address = "127.0.0.1:18232" @@ -899,7 +928,7 @@ listen_address = "127.0.0.1:8137" let toml_content = r#" backend = "fetch" -network = "Testnet" +network = "PubTestnet" [validator_settings] validator_jsonrpc_listen_address = "192.168.1.10:18232" @@ -930,7 +959,7 @@ listen_address = "127.0.0.1:8137" let toml_content = r#" backend = "fetch" -network = "Testnet" +network = "PubTestnet" [validator_settings] validator_jsonrpc_listen_address = "8.8.8.8:18232" @@ -1291,7 +1320,7 @@ listen_address = "127.0.0.1:8137" let toml_content = r#" backend = "fetch" -network = "Testnet" +network = "PubTestnet" ephemeral_finalised_state = true [validator_settings] diff --git a/zainod-heights-from-validator-spec.md b/zainod-heights-from-validator-spec.md index 66725a501..d78f94b0b 100644 --- a/zainod-heights-from-validator-spec.md +++ b/zainod-heights-from-validator-spec.md @@ -34,7 +34,7 @@ zebrad launch config. The wallet derives them via `WalletNetwork::from_validator`; zainod adopts them via this spec; no test-side constant needs to mirror anything. Expect the zcash_local_net pin bump to be loud: `ZainodConfig.network` narrows to a payload-free -kind (Mainnet/Testnet/Regtest), `ZcashDevtoolConfig::faucet()/recipient()` +kind (Mainnet / PubTestnet / Regtest), `ZcashDevtoolConfig::faucet()/recipient()` take a `WalletNetwork` parameter, and any test that assigned heights into a wallet config stops compiling — the fix is always to launch the validator first and derive. @@ -88,7 +88,7 @@ from it. (`state.rs:190-196`), so the ordering is achievable without restructuring. 2. **Learned heights are the only source.** `ZEBRAD_DEFAULT_ACTIVATION_HEIGHTS` ceases to exist as a truth source. Recommended shape: the *config-level* - network type degrades to a kind (Mainnet / Testnet / Regtest, no payload), + network type degrades to a kind (Mainnet / PubTestnet / Regtest, no payload), and only the *runtime* type constructed after the handshake carries heights — making pre-handshake height reads unrepresentable. A minimal variant (populate the existing payload from the handshake before first @@ -157,7 +157,7 @@ nu5 = 2, nu6 = 2, nu6_1 = 2, nu6_2 = 2, nu6_3 = 6 ## Out of scope -- Mainnet / Testnet: compiled zebra network parameters are correct there; no +- Mainnet / The Public Testnet: compiled zebra network parameters are correct there; no handshake is needed and none should be added. - The infras-side client guard lift (Provider heights) — lands on `infrastructure#278` independently. From 5ecfe7113e16867984886d003f9d901c6a891c37 Mon Sep 17 00:00:00 2001 From: zancas Date: Thu, 9 Jul 2026 14:02:15 -0700 Subject: [PATCH 49/58] style: clear the two outstanding clippy warnings ChainWork::to_u256 takes self by value (the type is Copy, and clippy's wrong_self_convention expects to_* methods on Copy types to consume self), and the address-deltas test fixture uses i.is_multiple_of(2) in place of the manual modulo test. Co-Authored-By: Claude Fable 5 --- packages/zaino-fetch/src/jsonrpsee/response/address_deltas.rs | 2 +- packages/zaino-state/src/chain_index/types/db/block.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/zaino-fetch/src/jsonrpsee/response/address_deltas.rs b/packages/zaino-fetch/src/jsonrpsee/response/address_deltas.rs index ef422bb39..f509ec857 100644 --- a/packages/zaino-fetch/src/jsonrpsee/response/address_deltas.rs +++ b/packages/zaino-fetch/src/jsonrpsee/response/address_deltas.rs @@ -278,7 +278,7 @@ mod tests { fn sample_delta_with_block_index(i: u32, bi: Option) -> AddressDelta { AddressDelta { - satoshis: if i % 2 == 0 { 1_000 } else { -500 }, + satoshis: if i.is_multiple_of(2) { 1_000 } else { -500 }, txid: format!("deadbeef{:02x}", i), index: i, height: 123_456 + i, diff --git a/packages/zaino-state/src/chain_index/types/db/block.rs b/packages/zaino-state/src/chain_index/types/db/block.rs index 1a224b5f2..cb55b4ba1 100644 --- a/packages/zaino-state/src/chain_index/types/db/block.rs +++ b/packages/zaino-state/src/chain_index/types/db/block.rs @@ -321,7 +321,7 @@ mod tests { impl ChainWork { /// Returns ChainWork as a U256. - pub(super) fn to_u256(&self) -> primitive_types::U256 { + pub(super) fn to_u256(self) -> primitive_types::U256 { primitive_types::U256::from_big_endian(&self.0) } From 46c5e93afe2d95e08ff076af7daf14ff502e8691 Mon Sep 17 00:00:00 2001 From: zancas Date: Mon, 13 Jul 2026 08:44:50 -0700 Subject: [PATCH 50/58] bump deps --- Cargo.lock | 130 ++++++++++++++++++----------------------------------- Cargo.toml | 17 ++++--- 2 files changed, 52 insertions(+), 95 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8398f82f9..e6ed72c67 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -306,26 +306,6 @@ dependencies = [ "serde", ] -[[package]] -name = "bindgen" -version = "0.69.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" -dependencies = [ - "bitflags 2.13.0", - "cexpr", - "clang-sys", - "itertools 0.12.1", - "lazy_static", - "lazycell", - "proc-macro2", - "quote", - "regex", - "rustc-hash 1.1.0", - "shlex 1.3.0", - "syn 2.0.118", -] - [[package]] name = "bindgen" version = "0.72.1" @@ -339,7 +319,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "rustc-hash 2.1.2", + "rustc-hash", "shlex 1.3.0", "syn 2.0.118", ] @@ -2247,15 +2227,6 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" -[[package]] -name = "itertools" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.13.0" @@ -2374,7 +2345,7 @@ dependencies = [ "jsonrpsee-types", "parking_lot", "rand 0.8.6", - "rustc-hash 2.1.2", + "rustc-hash", "serde", "serde_json", "thiserror 1.0.69", @@ -2481,12 +2452,6 @@ dependencies = [ "spin", ] -[[package]] -name = "lazycell" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" - [[package]] name = "left-right" version = "0.11.7" @@ -2531,14 +2496,13 @@ dependencies = [ [[package]] name = "librocksdb-sys" -version = "0.16.0+8.10.0" +version = "0.17.3+10.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce3d60bc059831dc1c83903fb45c103f75db65c5a7bf22272764d9cc683e348c" +checksum = "cef2a00ee60fe526157c9023edab23943fae1ce2ab6f4abb2a807c1746835de9" dependencies = [ - "bindgen 0.69.5", + "bindgen", "bzip2-sys", "cc", - "glob", "libc", "libz-sys", "lz4-sys", @@ -2561,7 +2525,7 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f8ce05b56f3cbc65ec7d0908adb308ed91281e022f61c8c3a0c9388b5380b17" dependencies = [ - "bindgen 0.72.1", + "bindgen", "cc", "thiserror 2.0.18", "tracing", @@ -2953,9 +2917,9 @@ checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" [[package]] name = "orchard" -version = "0.15.0-pre.1" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8e277dd4b46f5d06deae3ffb8af1a951e8622368f028c2a4d6fe59339566403" +checksum = "cca2ede6a4a77bd729eed3be9303d98a08b217edc85190dcf4dbe004086d5a65" dependencies = [ "aes", "bitvec", @@ -3473,7 +3437,7 @@ dependencies = [ "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash 2.1.2", + "rustc-hash", "rustls", "socket2", "thiserror 2.0.18", @@ -3493,7 +3457,7 @@ dependencies = [ "lru-slab", "rand 0.9.4", "ring", - "rustc-hash 2.1.2", + "rustc-hash", "rustls", "rustls-pki-types", "slab", @@ -3936,9 +3900,9 @@ dependencies = [ [[package]] name = "rocksdb" -version = "0.22.0" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bd13e55d6d7b8cd0ea569161127567cd587676c99f4472f779a0279aa60a7a7" +checksum = "ddb7af00d2b17dbd07d82c0063e25411959748ff03e8d4f96134c2ff41fce34f" dependencies = [ "libc", "librocksdb-sys", @@ -3956,12 +3920,6 @@ version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" -[[package]] -name = "rustc-hash" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" - [[package]] name = "rustc-hash" version = "2.1.2" @@ -5016,9 +4974,9 @@ dependencies = [ [[package]] name = "tower-batch-control" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e6cf52578f98b4da47335c26c4f883f7993b1a9b9d2f5420eb8dbfd5dd19a28" +checksum = "0a49cdaf2b00c1150c93a8dbbb63cae193573903a3c9a07b5d4d61251b9e32f0" dependencies = [ "futures", "futures-core", @@ -5033,9 +4991,9 @@ dependencies = [ [[package]] name = "tower-fallback" -version = "0.2.41" +version = "0.2.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4434e19ee996ee5c6aa42f11463a355138452592e5c5b5b73b6f0f19534556af" +checksum = "700cd6732c9c9e2c77501cc663708b2229abc9e8fa27d3deae15126330a1ed87" dependencies = [ "futures-core", "pin-project", @@ -6116,9 +6074,9 @@ dependencies = [ [[package]] name = "zcash_address" -version = "0.13.0-pre.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cf2918e73eff76388cda87695a6e7398f96a2e3383a0de7b77729a204ec00a0" +checksum = "5a854b28c07dba372f4410ea8ad62b4bf7d5c2bf8be32fc4b31bc0db6521a975" dependencies = [ "bech32", "bs58", @@ -6141,9 +6099,9 @@ dependencies = [ [[package]] name = "zcash_history" -version = "0.5.0-pre.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82e8634d011026cb181cb67b2a412e601dc344a2827b9c576fe3a727fdebf441" +checksum = "69d2ed9a9fa7b466a7adedab1ad5a21f8330e4943756912fc776cf7f3beae32b" dependencies = [ "blake2b_simd", "byteorder", @@ -6152,9 +6110,9 @@ dependencies = [ [[package]] name = "zcash_keys" -version = "0.15.0-pre.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a0bac3a9e5b0d954684ba1f07386d55b5ae4588b3ba2d93cad05ee09f3e0947" +checksum = "36391ebfce7df2510c6564f79e608ed93f1c0692c6464e2397b248e06dae3912" dependencies = [ "bech32", "blake2b_simd", @@ -6211,9 +6169,9 @@ dependencies = [ [[package]] name = "zcash_primitives" -version = "0.29.0-pre.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba7bfe66975658f44dba87d535f9ebb9fb4c9c0c4fecca3f5c40cbe1583dece6" +checksum = "bdbeccc05bfe63b6dee9e989c6ff5027421741562a4853c36504dd621f6a81b1" dependencies = [ "blake2b_simd", "block-buffer 0.11.0-rc.3", @@ -6242,9 +6200,9 @@ dependencies = [ [[package]] name = "zcash_proofs" -version = "0.29.0-pre.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39b90964ffe6bdc314368c7b849aaa6dcc2b495249a3bf1b9bfbdc92ba4e4f6" +checksum = "b91fb0b420fb66a765f1fa92ab7a6b631aa54c08415415ef6185256826e74fbd" dependencies = [ "bellman", "blake2b_simd", @@ -6265,9 +6223,9 @@ dependencies = [ [[package]] name = "zcash_protocol" -version = "0.10.0-pre.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97f339a9801c7f70295732a5cf822346f194202cea6425fc2fc14a5af679b004" +checksum = "2973d210e74ebd81adc50828fbbb284ab6aaa099308097c42c47dc63cf3420fc" dependencies = [ "corez", "document-features", @@ -6304,9 +6262,9 @@ dependencies = [ [[package]] name = "zcash_transparent" -version = "0.9.0-pre.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e941230f67056aad41c8fa9b926f9cc4d9d1a321f32e95c39c0b0e38e85f79b" +checksum = "72e0f8c142bed366ed5886dcfa8772ec90b5420cc127e6cdb76b6744c53a287b" dependencies = [ "bip32", "bs58", @@ -6329,9 +6287,9 @@ dependencies = [ [[package]] name = "zebra-chain" -version = "11.0.0" +version = "11.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "600a4c35ee81350384226f8178fab2b088e38d5e0a9f6cb0b1051caba30702d7" +checksum = "014adde138ea00b1cf6c873aaf56aacb5aa0d55b8e5451a3e3b1b06c461bd107" dependencies = [ "bech32", "bitflags 2.13.0", @@ -6398,9 +6356,9 @@ dependencies = [ [[package]] name = "zebra-consensus" -version = "10.0.0" +version = "11.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0aa2b2678c4ce6d18172bad73a9b72b24a4b3a65af4c2a5e9120a1470c3fc402" +checksum = "4481ced3223226db18e33b9fb2e8eab7f7cefeae1681f4e8d8d7e3c2285a5c4d" dependencies = [ "bellman", "blake2b_simd", @@ -6441,9 +6399,9 @@ dependencies = [ [[package]] name = "zebra-network" -version = "10.0.0" +version = "10.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36b1e6ca4687bc8d128553ca04bb82cc38701c4dac7107f3e56717b39d4b18da" +checksum = "9a59dc9e9ac723c4c01d985a5ecf3ddd85c0f1681574457d52173c43263905f2" dependencies = [ "bitflags 2.13.0", "byteorder", @@ -6479,9 +6437,9 @@ dependencies = [ [[package]] name = "zebra-node-services" -version = "9.0.0" +version = "9.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4446db9b576c0de14ff91c7fd7b38a59d340c1969a1604787185a52b8b7f53a6" +checksum = "006992c62b9c4742a13a306f2e7443646fa0b5b884aac63a6fba5b0be273f864" dependencies = [ "color-eyre", "jsonrpsee-types", @@ -6495,9 +6453,9 @@ dependencies = [ [[package]] name = "zebra-rpc" -version = "11.0.0" +version = "11.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb180542f1ffc0f66c20e1cd66c7741662053ccda588e3847a8ed2df8c83dae7" +checksum = "4d071b177740cdc1107f90dd50fac67e1b440a84c8509ae1529b3609fb6f801d" dependencies = [ "base64", "chrono", @@ -6556,9 +6514,9 @@ dependencies = [ [[package]] name = "zebra-script" -version = "10.0.0" +version = "10.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8648c201e3dadb1c95022eb7605d528de4bf94b6c0cebf76537805849aee76c5" +checksum = "f7c967badb1bb99573dba430948c98fbca940daf23915f2e9736e1a1d2a166f3" dependencies = [ "libzcash_script", "rand 0.8.6", @@ -6570,9 +6528,9 @@ dependencies = [ [[package]] name = "zebra-state" -version = "10.0.0" +version = "10.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b95b81181253a46885e3254cc5a20e4d82b5258116c9e43db5c8e9660c6b973" +checksum = "39295d344c1cc56ed725646cfc7bb7e16f8edae7518ea5a9e599665f979e9cf6" dependencies = [ "bincode", "chrono", diff --git a/Cargo.toml b/Cargo.toml index 8578ba5ff..0b6ece458 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,17 +40,16 @@ license = "Apache-2.0" # Librustzcash incrementalmerkletree = "0.8" -zcash_address = "0.13.0-pre.0" -zcash_keys = "0.15.0-pre.0" -zcash_protocol = "0.10.0-pre.0" -zcash_primitives = "0.29.0-pre.0" -zcash_transparent = "0.9.0-pre.0" -zcash_client_backend = "0.24.0-pre.0" +zcash_address = "0.13" +zcash_keys = "0.15" +zcash_protocol = "0.10" +zcash_primitives = "0.29" +zcash_transparent = "0.9" # Zebra -zebra-chain = "11.0" -zebra-state = "10.0" -zebra-rpc = "11.0" +zebra-chain = "11.1" +zebra-state = "10.1" +zebra-rpc = "11.1" # Runtime tokio = { version = "1.38", features = ["full"] } From 1a795db91daee239df4562f6653ec70bf33edd60 Mon Sep 17 00:00:00 2001 From: zancas Date: Mon, 13 Jul 2026 08:58:40 -0700 Subject: [PATCH 51/58] build(zainod): drop the unused zebra-chain dependency An audit of every zebra* and zcash* dependency edge (cargo machete, the compiler's unused_crate_dependencies lint, and inspection of each remaining single-reference hit) found exactly one dead edge: zainod declared zebra-chain but never referenced it. Remove it. Every other declared edge is genuinely used; zebra-chain remains the type backbone of the workspace, with some 350 references in zaino-state alone. Co-Authored-By: Claude Fable 5 --- Cargo.lock | 1 - packages/zainod/Cargo.toml | 1 - 2 files changed, 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e6ed72c67..d919c5245 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6068,7 +6068,6 @@ dependencies = [ "zaino-state", "zcash_address", "zcash_protocol", - "zebra-chain", "zebra-state", ] diff --git a/packages/zainod/Cargo.toml b/packages/zainod/Cargo.toml index 4fc9a786d..bc9d0b938 100644 --- a/packages/zainod/Cargo.toml +++ b/packages/zainod/Cargo.toml @@ -70,7 +70,6 @@ zaino-state = { workspace = true } zaino-serve = { workspace = true } # Zebra -zebra-chain = { workspace = true } zebra-state = { workspace = true } # Runtime From 1e255d083b2ab92974a98485d1ff457261ded587 Mon Sep 17 00:00:00 2001 From: zancas Date: Mon, 13 Jul 2026 08:58:40 -0700 Subject: [PATCH 52/58] build: advance ZEBRA_VERSION to the 6.0.0 final release Advance ZEBRA_VERSION in .env.testing-artifacts from 6.0.0-rc.0 to the final 6.0.0 release. Zebrad 6.0.0 depends on zebra-chain ^11.1.0, the same crate generation the workspace now pins, so the zebrad binary the live tests run against and the zebra libraries zaino links are once again from a single release. The makers check-matching-zebras guard passes. Co-Authored-By: Claude Fable 5 --- .env.testing-artifacts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.testing-artifacts b/.env.testing-artifacts index 92b2fbae2..e1e14d228 100644 --- a/.env.testing-artifacts +++ b/.env.testing-artifacts @@ -5,7 +5,7 @@ # zcashd Git tag (https://github.com/zcash/zcash/releases) ZCASH_VERSION=6.20.0 -ZEBRA_VERSION=6.0.0-rc.0 +ZEBRA_VERSION=6.0.0 # zcash-devtool git ref (https://github.com/zingolabs/zcash-devtool) — # the wallet client driven by wallet-tests via zcash_local_net, built From 483a6ca17612638b0c5bf6906921b4d3b41dfb0b Mon Sep 17 00:00:00 2001 From: zancas Date: Mon, 13 Jul 2026 09:12:38 -0700 Subject: [PATCH 53/58] build: remove unused dependency edges across the workspace A workspace-wide unused-dependency audit (the compiler's unused_crate_dependencies lint analyzed per target, cross-checked with cargo machete and a text search of every candidate's package, including feature-gated code) found twelve dead edges. Remove them: prost, bs58, nonempty, derive_more, and once_cell from zaino-state; tracing-subscriber from zainod; and tracing, corez, serde_json, tempfile, tower, and zip32 from clientless. The bs58, nonempty, and zip32 entries in [workspace.dependencies] lost their last inheritors and are removed with them. One edge that looks dead is kept deliberately: clientless does not import zainod in any test source, but its transparent_address_history_experimental feature forwards to zainod/transparent_address_history_experimental, and cargo rejects a feature forward to a non-dependency. A comment in the manifest now records that. Co-Authored-By: Claude Fable 5 --- Cargo.lock | 28 +++++----------------------- Cargo.toml | 3 --- live-tests/clientless/Cargo.toml | 9 +++------ packages/zaino-state/Cargo.toml | 5 ----- packages/zainod/Cargo.toml | 1 - 5 files changed, 8 insertions(+), 38 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d919c5245..798fd33aa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -669,14 +669,9 @@ name = "clientless" version = "0.2.0" dependencies = [ "anyhow", - "corez", "futures", "hex", - "serde_json", - "tempfile", "tokio", - "tower 0.4.13", - "tracing", "wire_serialized_transaction_test_data", "zaino-common", "zaino-fetch", @@ -688,7 +683,6 @@ dependencies = [ "zebra-chain", "zebra-rpc", "zebra-state", - "zip32", ] [[package]] @@ -2773,12 +2767,6 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "549e471b99ccaf2f89101bec68f4d244457d5a95a9c3d0672e9564124397741d" -[[package]] -name = "nonempty" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9737e026353e5cd0736f98eddae28665118eb6f6600902a7f50db585621fecb6" - [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -2936,7 +2924,7 @@ dependencies = [ "incrementalmerkletree", "lazy_static", "memuse", - "nonempty 0.11.0", + "nonempty", "pasta_curves", "rand 0.8.6", "rand_core 0.6.4", @@ -5976,11 +5964,9 @@ dependencies = [ "arc-swap", "bitflags 2.13.0", "blake2", - "bs58", "chrono", "corez", "dashmap", - "derive_more", "futures", "hex", "incrementalmerkletree", @@ -5988,11 +5974,8 @@ dependencies = [ "lmdb", "lmdb-sys", "metrics", - "nonempty 0.12.0", - "once_cell", "primitive-types 0.14.0", "proptest", - "prost", "rand 0.10.1", "reqwest 0.13.1", "sapling-crypto", @@ -6061,7 +6044,6 @@ dependencies = [ "tokio", "toml", "tracing", - "tracing-subscriber", "zaino-common", "zaino-fetch", "zaino-serve", @@ -6093,7 +6075,7 @@ checksum = "1440921903cdb86133fb9e2fe800be488015db2939a30bedb413078a1acb0306" dependencies = [ "corez", "hex", - "nonempty 0.11.0", + "nonempty", ] [[package]] @@ -6121,7 +6103,7 @@ dependencies = [ "document-features", "group", "memuse", - "nonempty 0.11.0", + "nonempty", "orchard", "rand_core 0.6.4", "sapling-crypto", @@ -6183,7 +6165,7 @@ dependencies = [ "incrementalmerkletree", "jubjub", "memuse", - "nonempty 0.11.0", + "nonempty", "orchard", "rand_core 0.6.4", "redjubjub", @@ -6271,7 +6253,7 @@ dependencies = [ "document-features", "getset", "hex", - "nonempty 0.11.0", + "nonempty", "ripemd 0.1.3", "secp256k1", "sha2 0.10.9", diff --git a/Cargo.toml b/Cargo.toml index 0b6ece458..e13a29ac6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -114,7 +114,6 @@ blake2 = "0.10" hex = "0.4" toml = "1.1" primitive-types = "0.14" -bs58 = "0.5" bitflags = "2.9" # Test @@ -146,9 +145,7 @@ zcash_local_net = { git = "https://github.com/zingolabs/infrastructure.git", rev wire_serialized_transaction_test_data = "1.0.0" config = { version = "0.15", default-features = false, features = ["toml"] } -nonempty = "0.12.0" proptest = "~1.11" -zip32 = "0.2.1" anyhow = "1.0" arc-swap = "1.7.1" derive_more = "2.0.1" diff --git a/live-tests/clientless/Cargo.toml b/live-tests/clientless/Cargo.toml index ba912e690..c8945a81a 100644 --- a/live-tests/clientless/Cargo.toml +++ b/live-tests/clientless/Cargo.toml @@ -36,7 +36,6 @@ zaino-fetch = { workspace = true } zaino-testutils = { workspace = true } # The lib's z_validate helper calls `ZcashIndexer::z_validate_address`. zaino-state = { workspace = true, features = ["test_dependencies"] } -tracing.workspace = true [dev-dependencies] anyhow = { workspace = true } @@ -46,6 +45,9 @@ anyhow = { workspace = true } zaino-state = { workspace = true, features = ["fast-test-seam"] } # Test utility +# Not imported by any test source; present so the +# transparent_address_history_experimental feature can forward to +# zainod/transparent_address_history_experimental in the spawned daemon. zainod = { workspace = true } zaino-testutils = { workspace = true } zaino-fetch = { workspace = true } @@ -55,13 +57,8 @@ wire_serialized_transaction_test_data = { workspace = true } zebra-chain = { workspace = true } zebra-state = { workspace = true } zebra-rpc = { workspace = true } -zip32 = { workspace = true } -corez = { workspace = true } -serde_json = { workspace = true } futures = { workspace = true } -tempfile = { workspace = true } -tower = { workspace = true } hex = { workspace = true } # Runtime diff --git a/packages/zaino-state/Cargo.toml b/packages/zaino-state/Cargo.toml index 4f96dbe9f..6b341c1c9 100644 --- a/packages/zaino-state/Cargo.toml +++ b/packages/zaino-state/Cargo.toml @@ -83,22 +83,17 @@ lmdb = { workspace = true } lmdb-sys = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true, features = ["preserve_order"] } -prost = { workspace = true } primitive-types = { workspace = true } blake2 = { workspace = true } sha2 = { workspace = true } corez = { workspace = true } -bs58 = { workspace = true } -nonempty = { workspace = true } arc-swap = { workspace = true } reqwest.workspace = true bitflags = { workspace = true } -derive_more = { workspace = true, features = ["from"] } [dev-dependencies] tempfile = { workspace = true } tracing-subscriber = { workspace = true } -once_cell = { workspace = true } zebra-chain = { workspace = true, features = ["proptest-impl"] } proptest.workspace = true incrementalmerkletree = "0.8.2" diff --git a/packages/zainod/Cargo.toml b/packages/zainod/Cargo.toml index bc9d0b938..c3a54016e 100644 --- a/packages/zainod/Cargo.toml +++ b/packages/zainod/Cargo.toml @@ -80,7 +80,6 @@ clap = { workspace = true, features = ["derive"] } # Tracing tracing = { workspace = true } -tracing-subscriber = { workspace = true, features = ["fmt", "env-filter", "time"] } # Network / RPC http = { workspace = true } From 46d18c623b993e4394436017be2c37a89f7c00b0 Mon Sep 17 00:00:00 2001 From: zancas Date: Mon, 13 Jul 2026 09:13:11 -0700 Subject: [PATCH 54/58] build: demote test-only dependencies to dev-dependencies The same audit that removed the dead edges found three dependencies that are real but used exclusively under #[cfg(test)]: primitive-types in zaino-state (the legacy_chainwork_reference round-trip tests), tempfile in zainod (the config tests), and tonic in zaino-testutils (the cfg(test)-gated build_client helper). Move each from [dependencies] to [dev-dependencies] so shipped builds stop linking them. The lockfile is unchanged: dependency kind does not affect resolution. Co-Authored-By: Claude Fable 5 --- live-tests/zaino-testutils/Cargo.toml | 4 +++- packages/zaino-state/Cargo.toml | 2 +- packages/zainod/Cargo.toml | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/live-tests/zaino-testutils/Cargo.toml b/live-tests/zaino-testutils/Cargo.toml index 38f2ef883..09b132e9f 100644 --- a/live-tests/zaino-testutils/Cargo.toml +++ b/live-tests/zaino-testutils/Cargo.toml @@ -65,5 +65,7 @@ futures = { workspace = true } http = { workspace = true } once_cell = { workspace = true } tokio = { workspace = true } -tonic.workspace = true tracing.workspace = true + +[dev-dependencies] +tonic.workspace = true diff --git a/packages/zaino-state/Cargo.toml b/packages/zaino-state/Cargo.toml index 6b341c1c9..0f60eab31 100644 --- a/packages/zaino-state/Cargo.toml +++ b/packages/zaino-state/Cargo.toml @@ -83,7 +83,6 @@ lmdb = { workspace = true } lmdb-sys = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true, features = ["preserve_order"] } -primitive-types = { workspace = true } blake2 = { workspace = true } sha2 = { workspace = true } corez = { workspace = true } @@ -94,6 +93,7 @@ bitflags = { workspace = true } [dev-dependencies] tempfile = { workspace = true } tracing-subscriber = { workspace = true } +primitive-types = { workspace = true } zebra-chain = { workspace = true, features = ["proptest-impl"] } proptest.workspace = true incrementalmerkletree = "0.8.2" diff --git a/packages/zainod/Cargo.toml b/packages/zainod/Cargo.toml index c3a54016e..beba27ad9 100644 --- a/packages/zainod/Cargo.toml +++ b/packages/zainod/Cargo.toml @@ -95,8 +95,8 @@ thiserror = { workspace = true } # Formats toml = { workspace = true } config = { workspace = true } -tempfile = { workspace = true } [dev-dependencies] +tempfile = { workspace = true } zcash_address = { workspace = true } zcash_protocol = { workspace = true } From 4141a5d4dceb4e1705a196348b3b774e5114bb1b Mon Sep 17 00:00:00 2001 From: zancas Date: Mon, 13 Jul 2026 09:19:18 -0700 Subject: [PATCH 55/58] feat(test-runner): add a packages row and per-suite timing to the summary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `makers test all` previously ran the packages set outside the summary runner, so the closing table covered only the two live partitions. The `all` branch now delegates to `live-summary --all`, which runs `makers container-test` first and prints its tallies as a `packages:` row above `clientless:`, `e2e:`, and `TOTAL:`, under a "test summary" header. `makers test live` is unchanged apart from the new column. The old contract is preserved: every set runs even when an earlier one fails, and the runner exits non-zero if any set did. Each row also gains a wall-clock time column, parsed from the nextest summary line's `[ 510.718s]` stamp. Durations are held as integer milliseconds in a u64 — whole seconds and a right-padded three-digit fraction parsed digit-by-digit, no floating point — and the TOTAL row sums them, which reads as true wall time because the suites run sequentially. Six new unit tests cover the parser (padding, short and absent fractions, and propagation through parse_summary), and the pre-existing `filter(..).last()` on a double-ended iterator is tightened to `rfind(..)` per clippy. Co-Authored-By: Claude Fable 5 --- Makefile.toml | 10 +- tools/test-runner/src/bin/live-summary.rs | 127 +++++++++++++++++++--- 2 files changed, 118 insertions(+), 19 deletions(-) diff --git a/Makefile.toml b/Makefile.toml index 83e1c47ea..2bcaec624 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -182,12 +182,10 @@ case "$set" in clientless) exec makers live-clientless "$@" ;; live) exec cargo run -q --manifest-path tools/test-runner/Cargo.toml --bin live-summary -- "$@" ;; all) - # Run both even if the first fails, then fail if either did, so the - # omnibus reflects the whole suite. - rc=0 - makers container-test "$@" || rc=1 - cargo run -q --manifest-path tools/test-runner/Cargo.toml --bin live-summary -- "$@" || rc=1 - exit "$rc" ;; + # The summary runner's --all mode runs the packages set and both live + # partitions (each set runs even when an earlier one fails, and the exit + # code reflects any failure), so the combined table covers everything. + exec cargo run -q --manifest-path tools/test-runner/Cargo.toml --bin live-summary -- --all "$@" ;; ironwood) # The one definition of the ironwood selection: the two dedicated test # binaries, plus every test whose name carries the pool's name (the diff --git a/tools/test-runner/src/bin/live-summary.rs b/tools/test-runner/src/bin/live-summary.rs index 47d3ff3bd..3af1d56ed 100644 --- a/tools/test-runner/src/bin/live-summary.rs +++ b/tools/test-runner/src/bin/live-summary.rs @@ -1,10 +1,12 @@ -//! `live-summary` — run both live-test partitions and print a combined summary. +//! `live-summary` — run the test partitions and print a combined summary. //! -//! Invoked from `makers test live` (and `makers test all`) as -//! `cargo run --bin live-summary -- `. Runs the `clientless` then `e2e` -//! partition (each in its own CI container via its own `makers` task), streams -//! each run's output while capturing it, parses the nextest summary line, and -//! aggregates the totals. +//! Invoked from `makers test live` as `cargo run --bin live-summary -- ` +//! and from `makers test all` with the extra `--all` flag. The live set runs +//! the `clientless` then `e2e` partition (each in its own CI container via its +//! own `makers` task); `--all` first runs the `packages` set (`makers +//! container-test`) and adds its row to the table. The runner streams each +//! run's output while capturing it, parses the nextest summary line (tallies +//! and wall-clock duration), and aggregates the totals. //! //! Unlike a cargo-make `dependencies` list (which is fail-fast), this runs BOTH //! partitions even when the first fails, so the summary reflects the whole @@ -19,6 +21,14 @@ use std::error::Error; use std::io::{BufRead, BufReader}; use std::process::{Command, Stdio}; +/// Which partitions this invocation runs and summarizes. +enum TestSet { + /// The two live partitions (`makers test live`). + Live, + /// The packages set plus the two live partitions (`makers test all`). + All, +} + /// One nextest run's tallies, zero where the summary line was absent. #[derive(Default)] struct Summary { @@ -26,6 +36,8 @@ struct Summary { passed: u64, failed: u64, skipped: u64, + /// Wall-clock milliseconds from the summary line's `[ 510.718s]` stamp. + millis: u64, } impl Summary { @@ -35,6 +47,7 @@ impl Summary { passed: self.passed + other.passed, failed: self.failed + other.failed, skipped: self.skipped + other.skipped, + millis: self.millis + other.millis, } } } @@ -104,6 +117,25 @@ fn count_before(line: &str, marker: &str) -> u64 { head[head.len() - digit_count..].parse().unwrap_or(0) } +/// The wall-clock milliseconds from the summary line's `[ 510.718s]` stamp, +/// or 0 if absent. Parsed as integers (whole seconds and up to three +/// fractional digits) — no floating point. +fn duration_millis(line: &str) -> u64 { + let Some(idx) = line.find("s]") else { + return 0; + }; + let head = &line[..idx]; + let start = head + .rfind(|c: char| !(c.is_ascii_digit() || c == '.')) + .map_or(0, |i| i + 1); + let stamp = &head[start..]; + let (whole, frac) = stamp.split_once('.').unwrap_or((stamp, "")); + let whole: u64 = whole.parse().unwrap_or(0); + // Right-pad the fraction to exactly three digits ("7" -> 700ms). + let frac: u64 = format!("{frac:0<3.3}").parse().unwrap_or(0); + whole * 1000 + frac +} + /// Parse the last nextest summary line out of a captured run. /// /// nextest prints e.g.: @@ -115,8 +147,7 @@ fn parse_summary(log: &str) -> Summary { let line = log .lines() .map(strip_ansi) - .filter(|l| l.contains("run:") && l.contains("test")) - .last() + .rfind(|l| l.contains("run:") && l.contains("test")) .unwrap_or_default(); Summary { @@ -125,18 +156,34 @@ fn parse_summary(log: &str) -> Summary { passed: count_before(&line, "passed"), failed: count_before(&line, "failed"), skipped: count_before(&line, "skipped"), + millis: duration_millis(&line), } } fn print_row(label: &str, s: &Summary) { + let secs = format!("{}.{:03}", s.millis / 1000, s.millis % 1000); println!( - " {label:<18} {:>4} run, {:>4} passed, {:>4} failed, {:>4} skipped", + " {label:<18} {:>4} run, {:>4} passed, {:>4} failed, {:>4} skipped, {secs:>10}s", s.run, s.passed, s.failed, s.skipped ); } fn main() -> Result<(), Box> { let with_zcashd = std::env::args().any(|a| a == "--with-zcashd"); + let set = if std::env::args().any(|a| a == "--all") { + TestSet::All + } else { + TestSet::Live + }; + + let packages = match set { + TestSet::All => { + println!(">>> all: running packages set"); + let (rc, log) = run_partition("container-test", with_zcashd)?; + Some((rc, parse_summary(&log))) + } + TestSet::Live => None, + }; println!(">>> live: running clientless partition"); let (cl_rc, cl_log) = run_partition("live-clientless", with_zcashd)?; @@ -147,15 +194,32 @@ fn main() -> Result<(), Box> { let cl = parse_summary(&cl_log); let e2e = parse_summary(&e2e_log); + let header = match packages { + Some(_) => "test summary", + None => "live summary", + }; + let mut total = cl.add(&e2e); + if let Some((_, p)) = &packages { + total = total.add(p); + } + println!(); - println!("====================== live summary =========================="); + println!("====================== {header} =========================="); + if let Some((_, p)) = &packages { + print_row("packages:", p); + } print_row("clientless:", &cl); print_row("e2e:", &e2e); - print_row("TOTAL:", &cl.add(&e2e)); + print_row("TOTAL:", &total); println!("=============================================================="); // A partition that errored without producing a summary line likely failed // to build; call it out so the zeros above aren't read as "all clear". + if let Some((pkg_rc, p)) = &packages { + if *pkg_rc != 0 && p.run == 0 { + println!(" warning: packages produced no nextest summary (build failure?) — see output above."); + } + } if cl_rc != 0 && cl.run == 0 { println!(" warning: clientless produced no nextest summary (build failure?) — see output above."); } @@ -163,8 +227,9 @@ fn main() -> Result<(), Box> { println!(" warning: e2e produced no nextest summary (build failure?) — see output above."); } - // Fail the front door if either partition failed. - if cl_rc != 0 || e2e_rc != 0 { + // Fail the front door if any set failed. + let pkg_rc = packages.map_or(0, |(rc, _)| rc); + if pkg_rc != 0 || cl_rc != 0 || e2e_rc != 0 { std::process::exit(1); } Ok(()) @@ -215,3 +280,39 @@ mod parse_summary { check(log, 9, 7, 1, 1); } } + +#[cfg(test)] +mod duration_millis { + use super::*; + + #[test] + fn fractional_seconds_with_padding() { + assert_eq!(duration_millis("Summary [ 73.207s] 8 tests run: 8 passed"), 73_207); + } + + #[test] + fn no_padding() { + assert_eq!(duration_millis("Summary [510.718s] 29 tests run: 23 passed"), 510_718); + } + + #[test] + fn short_fraction_is_right_padded() { + assert_eq!(duration_millis("Summary [2.7s] 1 test run: 1 passed"), 2_700); + } + + #[test] + fn whole_seconds_without_fraction() { + assert_eq!(duration_millis("Summary [2s] 1 test run: 1 passed"), 2_000); + } + + #[test] + fn missing_stamp_is_zero() { + assert_eq!(duration_millis("no summary line here"), 0); + } + + #[test] + fn survives_through_parse_summary() { + let s = parse_summary("Summary [ 1.795s] 1 test run: 0 passed, 1 failed, 114 skipped"); + assert_eq!(s.millis, 1_795); + } +} From 2f3541a4a85352bfed928453ed71b38354051373 Mon Sep 17 00:00:00 2001 From: zancas Date: Mon, 13 Jul 2026 09:37:25 -0700 Subject: [PATCH 56/58] fix(ci): refresh the trixie apt pins the point release superseded The test-environment image rebuild (first since the ZEBRA_VERSION bump changed the image tag) failed because Debian trixie point updates replaced two pinned package versions, and the archive does not keep superseded versions: protobuf-compiler 3.21.12-11 -> 3.21.12-11+deb13u1 (all three build stages) and curl 8.14.1-2+deb13u3 -> 8.14.1-2+deb13u4 (both stages). Every other pin in the Containerfile still matches the archive; verified against qa.debian.org/madison and a live trixie-slim container. Co-Authored-By: Claude Fable 5 --- live-tests/test_environment/Containerfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/live-tests/test_environment/Containerfile b/live-tests/test_environment/Containerfile index dabb4dea2..a6a19c7fd 100644 --- a/live-tests/test_environment/Containerfile +++ b/live-tests/test_environment/Containerfile @@ -46,7 +46,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ clang=1:19.0-63 \ cmake=3.31.6-2 \ pkg-config=1.8.1-4 \ - protobuf-compiler=3.21.12-11 \ + protobuf-compiler=3.21.12-11+deb13u1 \ libstdc++6=14.2.0-19 WORKDIR /zebra # Single RUN (DL3059): clone, checkout, build, and stage the binary. @@ -71,7 +71,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ autoconf=2.72-3.1 \ libtool=2.5.4-4 \ bsdmainutils=12.1.8 \ - curl=8.14.1-2+deb13u3 \ + curl=8.14.1-2+deb13u4 \ ca-certificates=20250419 \ netbase=6.5 @@ -99,7 +99,7 @@ ARG DEVTOOL_REV RUN apt-get update && apt-get install -y --no-install-recommends \ git=1:2.47.3-0+deb13u1 \ pkg-config=1.8.1-4 \ - protobuf-compiler=3.21.12-11 \ + protobuf-compiler=3.21.12-11+deb13u1 \ libsqlite3-dev=3.46.1-7+deb13u1 WORKDIR /devtool # Single RUN (DL3059): clone, checkout, build, and stage the binary. The @@ -140,12 +140,12 @@ LABEL org.zaino.test-artifacts.versions.rust="${RUST_VERSION}" # Versions pinned (DL3008) to the candidates in rust:1.95.0-trixie; bump # together with the base image (query with `apt-cache policy `). RUN apt-get update && apt-get install -y --no-install-recommends \ - curl=8.14.1-2+deb13u3 \ + curl=8.14.1-2+deb13u4 \ libclang-dev=1:19.0-63 \ build-essential=12.12 \ cmake=3.31.6-2 \ librocksdb-dev=9.10.0-1+b1 \ - protobuf-compiler=3.21.12-11 \ + protobuf-compiler=3.21.12-11+deb13u1 \ && rm -rf /var/lib/apt/lists/* # Create container_user. The GID may already belong to a base-image group From 45ed68d16a0f76776009e296d76847a10f47dd0e Mon Sep 17 00:00:00 2001 From: nachog00 Date: Mon, 13 Jul 2026 14:11:13 -0300 Subject: [PATCH 57/58] refactor(test-runner): measure wall-clock time with Instant, not output parsing Replace the duration_millis parser (which scraped nextest's human-readable summary line) with Instant::now()/elapsed() around each run_partition call. The nextest display format is not a stable contract; the runner already owns the process lifecycle so it can time it directly. This also captures build time, which the nextest stamp did not include. --- tools/test-runner/src/bin/live-summary.rs | 88 ++++++----------------- 1 file changed, 23 insertions(+), 65 deletions(-) diff --git a/tools/test-runner/src/bin/live-summary.rs b/tools/test-runner/src/bin/live-summary.rs index 3af1d56ed..2ca93c47e 100644 --- a/tools/test-runner/src/bin/live-summary.rs +++ b/tools/test-runner/src/bin/live-summary.rs @@ -5,8 +5,8 @@ //! the `clientless` then `e2e` partition (each in its own CI container via its //! own `makers` task); `--all` first runs the `packages` set (`makers //! container-test`) and adds its row to the table. The runner streams each -//! run's output while capturing it, parses the nextest summary line (tallies -//! and wall-clock duration), and aggregates the totals. +//! run's output while capturing it, parses the nextest summary line for +//! tallies, and measures wall-clock duration via `Instant`. //! //! Unlike a cargo-make `dependencies` list (which is fail-fast), this runs BOTH //! partitions even when the first fails, so the summary reflects the whole @@ -20,6 +20,7 @@ use std::error::Error; use std::io::{BufRead, BufReader}; use std::process::{Command, Stdio}; +use std::time::{Duration, Instant}; /// Which partitions this invocation runs and summarizes. enum TestSet { @@ -36,8 +37,8 @@ struct Summary { passed: u64, failed: u64, skipped: u64, - /// Wall-clock milliseconds from the summary line's `[ 510.718s]` stamp. - millis: u64, + /// Wall-clock duration measured by the runner (includes build + test time). + elapsed: Duration, } impl Summary { @@ -47,7 +48,7 @@ impl Summary { passed: self.passed + other.passed, failed: self.failed + other.failed, skipped: self.skipped + other.skipped, - millis: self.millis + other.millis, + elapsed: self.elapsed + other.elapsed, } } } @@ -117,25 +118,6 @@ fn count_before(line: &str, marker: &str) -> u64 { head[head.len() - digit_count..].parse().unwrap_or(0) } -/// The wall-clock milliseconds from the summary line's `[ 510.718s]` stamp, -/// or 0 if absent. Parsed as integers (whole seconds and up to three -/// fractional digits) — no floating point. -fn duration_millis(line: &str) -> u64 { - let Some(idx) = line.find("s]") else { - return 0; - }; - let head = &line[..idx]; - let start = head - .rfind(|c: char| !(c.is_ascii_digit() || c == '.')) - .map_or(0, |i| i + 1); - let stamp = &head[start..]; - let (whole, frac) = stamp.split_once('.').unwrap_or((stamp, "")); - let whole: u64 = whole.parse().unwrap_or(0); - // Right-pad the fraction to exactly three digits ("7" -> 700ms). - let frac: u64 = format!("{frac:0<3.3}").parse().unwrap_or(0); - whole * 1000 + frac -} - /// Parse the last nextest summary line out of a captured run. /// /// nextest prints e.g.: @@ -156,12 +138,14 @@ fn parse_summary(log: &str) -> Summary { passed: count_before(&line, "passed"), failed: count_before(&line, "failed"), skipped: count_before(&line, "skipped"), - millis: duration_millis(&line), + ..Summary::default() } } fn print_row(label: &str, s: &Summary) { - let secs = format!("{}.{:03}", s.millis / 1000, s.millis % 1000); + let total_secs = s.elapsed.as_secs(); + let millis = s.elapsed.subsec_millis(); + let secs = format!("{total_secs}.{millis:03}"); println!( " {label:<18} {:>4} run, {:>4} passed, {:>4} failed, {:>4} skipped, {secs:>10}s", s.run, s.passed, s.failed, s.skipped @@ -179,20 +163,30 @@ fn main() -> Result<(), Box> { let packages = match set { TestSet::All => { println!(">>> all: running packages set"); + let start = Instant::now(); let (rc, log) = run_partition("container-test", with_zcashd)?; - Some((rc, parse_summary(&log))) + let elapsed = start.elapsed(); + let mut s = parse_summary(&log); + s.elapsed = elapsed; + Some((rc, s)) } TestSet::Live => None, }; println!(">>> live: running clientless partition"); + let cl_start = Instant::now(); let (cl_rc, cl_log) = run_partition("live-clientless", with_zcashd)?; + let cl_elapsed = cl_start.elapsed(); println!(">>> live: running e2e partition"); + let e2e_start = Instant::now(); let (e2e_rc, e2e_log) = run_partition("live-e2e", with_zcashd)?; + let e2e_elapsed = e2e_start.elapsed(); - let cl = parse_summary(&cl_log); - let e2e = parse_summary(&e2e_log); + let mut cl = parse_summary(&cl_log); + cl.elapsed = cl_elapsed; + let mut e2e = parse_summary(&e2e_log); + e2e.elapsed = e2e_elapsed; let header = match packages { Some(_) => "test summary", @@ -280,39 +274,3 @@ mod parse_summary { check(log, 9, 7, 1, 1); } } - -#[cfg(test)] -mod duration_millis { - use super::*; - - #[test] - fn fractional_seconds_with_padding() { - assert_eq!(duration_millis("Summary [ 73.207s] 8 tests run: 8 passed"), 73_207); - } - - #[test] - fn no_padding() { - assert_eq!(duration_millis("Summary [510.718s] 29 tests run: 23 passed"), 510_718); - } - - #[test] - fn short_fraction_is_right_padded() { - assert_eq!(duration_millis("Summary [2.7s] 1 test run: 1 passed"), 2_700); - } - - #[test] - fn whole_seconds_without_fraction() { - assert_eq!(duration_millis("Summary [2s] 1 test run: 1 passed"), 2_000); - } - - #[test] - fn missing_stamp_is_zero() { - assert_eq!(duration_millis("no summary line here"), 0); - } - - #[test] - fn survives_through_parse_summary() { - let s = parse_summary("Summary [ 1.795s] 1 test run: 0 passed, 1 failed, 114 skipped"); - assert_eq!(s.millis, 1_795); - } -} From dda8497c280ac7c2488f11c1e510131f41363ed5 Mon Sep 17 00:00:00 2001 From: nachog00 Date: Mon, 13 Jul 2026 15:00:33 -0300 Subject: [PATCH 58/58] fix(ci): make publish dry-run advisory for PRs targeting rc/stable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PRs that target rc/** or stable carry unbumped dev code by design — version bumps happen on the RC branch after merge. The blocking mode was failing these PRs for an expected, non-actionable condition. Only direct pushes to rc/** or stable are blocking now; PRs targeting them are advisory. --- .github/workflows/publish-dry-run.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/publish-dry-run.yml b/.github/workflows/publish-dry-run.yml index 8add6d556..f2fb2e948 100644 --- a/.github/workflows/publish-dry-run.yml +++ b/.github/workflows/publish-dry-run.yml @@ -13,13 +13,11 @@ jobs: publish-dry-run: name: cargo publish --dry-run runs-on: ubuntu-latest - # Blocking context (pushes to rc/** or stable, and PRs targeting them): - # this job MUST pass. Advisory context (everything else): failures exit 0 - # after emitting a warning annotation, so they do not paint a red X on - # PRs (a continue-on-error'd failing job still shows as failed in the PR - # checks list even though the run succeeds — hence no job-level - # continue-on-error here). The context is computed once in the - # "Compute check mode" step below. + # Blocking context: direct pushes to rc/** or stable — the branch is in + # release-prep state and must publish cleanly. Advisory context: everything + # else, including PRs that *target* rc/** or stable. Those PRs carry + # unbumped dev code; version bumps happen on the RC branch after merge, so + # a publish failure at PR time is expected, not actionable. steps: - name: Checkout repository uses: actions/checkout@v7 @@ -30,10 +28,12 @@ jobs: - name: Compute check mode id: mode run: | - if [[ "${{ github.event_name }}" == "pull_request" ]]; then - target="${{ github.base_ref }}" - else + if [[ "${{ github.event_name }}" == "push" ]]; then target="${{ github.ref_name }}" + else + # PRs (including those targeting rc/** or stable) are always + # advisory — they carry pre-bump code by design. + target="" fi if [[ "$target" == rc/* || "$target" == stable ]]; then echo "mode=blocking" >> "$GITHUB_OUTPUT"