From 850ee975f82b62b538ae12c248f77cd3e24b2ec3 Mon Sep 17 00:00:00 2001 From: Pacu Date: Mon, 27 Apr 2026 12:49:47 -0300 Subject: [PATCH 1/5] Match lightwalletd compact block responses Align Zaino's CompactTxStreamer responses with lightwalletd for single-block lookups. GetBlock and GetBlockNullifiers now request all pool data, compact blocks use lightwalletd's protoVersion value, nullifier responses clear transparent inputs and outputs, and tree-state RPCs are delegated to the backing validator so they are not limited by the local cache view. --- packages/zaino-proto/src/proto/utils.rs | 2 + packages/zaino-state/src/backends/fetch.rs | 88 +++++-------------- packages/zaino-state/src/backends/state.rs | 68 +++----------- .../finalised_state/db/v1/compact_block.rs | 4 +- .../src/chain_index/types/db/legacy.rs | 2 +- 5 files changed, 41 insertions(+), 123 deletions(-) diff --git a/packages/zaino-proto/src/proto/utils.rs b/packages/zaino-proto/src/proto/utils.rs index 6c4de0bbc..c52a5d8df 100644 --- a/packages/zaino-proto/src/proto/utils.rs +++ b/packages/zaino-proto/src/proto/utils.rs @@ -360,6 +360,8 @@ pub fn compact_block_with_pool_types( pub fn compact_block_to_nullifiers(mut block: CompactBlock) -> CompactBlock { for ctransaction in &mut block.vtx { ctransaction.outputs = Vec::new(); + ctransaction.vin = Vec::new(); + ctransaction.vout = Vec::new(); for caction in &mut ctransaction.actions { *caction = CompactOrchardAction { nullifier: caction.nullifier.clone(), diff --git a/packages/zaino-state/src/backends/fetch.rs b/packages/zaino-state/src/backends/fetch.rs index 643d5707e..27deefdd9 100644 --- a/packages/zaino-state/src/backends/fetch.rs +++ b/packages/zaino-state/src/backends/fetch.rs @@ -2,7 +2,7 @@ use futures::StreamExt; use hex::FromHex; -use std::{io::Cursor, str::FromStr, time}; +use std::{io::Cursor, time}; use tokio::{sync::mpsc, time::timeout}; use tonic::async_trait; use tracing::{info, instrument, warn}; @@ -550,51 +550,8 @@ impl ZcashIndexer for FetchServiceSubscriber { &self, hash_or_height: String, ) -> Result { - let hash_or_height_struct: HashOrHeight = HashOrHeight::from_str(&hash_or_height)?; - let snapshot = self.indexer.snapshot_nonfinalized_state().await?; - - let block_data = match hash_or_height_struct { - HashOrHeight::Hash(hash) => self - .indexer - .get_indexed_block_by_hash(&snapshot, &hash.into()) - .await - .map_err(|_error| { - #[allow(deprecated)] - FetchServiceError::RpcError(RpcError::new_from_legacycode( - zebra_rpc::server::error::LegacyCode::InvalidParameter, - "Failed to fetch block data.", - )) - })? - .ok_or( - #[allow(deprecated)] - FetchServiceError::RpcError(RpcError::new_from_legacycode( - zebra_rpc::server::error::LegacyCode::InvalidParameter, - "Failed to fetch block data.", - )), - )?, - HashOrHeight::Height(height) => self - .indexer - .get_indexed_block_by_height(&snapshot, &height.into()) - .await - .map_err(|_error| { - #[allow(deprecated)] - FetchServiceError::RpcError(RpcError::new_from_legacycode( - zebra_rpc::server::error::LegacyCode::InvalidParameter, - "Failed to fetch block data.", - )) - })? - .ok_or( - #[allow(deprecated)] - FetchServiceError::RpcError(RpcError::new_from_legacycode( - zebra_rpc::server::error::LegacyCode::InvalidParameter, - "Failed to fetch block data.", - )), - )?, - }; - - let (sapling, orchard) = self - .indexer - .get_treestate(block_data.hash()) + self.fetcher + .get_treestate(hash_or_height) .await .map_err(|_error| { #[allow(deprecated)] @@ -602,23 +559,16 @@ impl ZcashIndexer for FetchServiceSubscriber { zebra_rpc::server::error::LegacyCode::InvalidParameter, "Failed to fetch treestate.", )) - })?; - let time: u32 = block_data.data().time().try_into().map_err(|_error| { - #[allow(deprecated)] - FetchServiceError::RpcError(RpcError::new_from_legacycode( - zebra_rpc::server::error::LegacyCode::InvalidParameter, - "Block time is out of range for u32.", - )) - })?; - - #[allow(deprecated)] - Ok(GetTreestateResponse::from_parts( - (*block_data.hash()).into(), - block_data.height().into(), - time, - sapling, - orchard, - )) + }) + .and_then(|treestate| { + treestate.try_into().map_err(|_error| { + #[allow(deprecated)] + FetchServiceError::RpcError(RpcError::new_from_legacycode( + zebra_rpc::server::error::LegacyCode::InvalidParameter, + "Failed to parse treestate.", + )) + }) + }) } /// Returns information about a range of Sapling or Orchard subtrees. @@ -888,7 +838,11 @@ impl LightWalletIndexer for FetchServiceSubscriber { match self .indexer - .get_compact_block(&snapshot, types::Height(height), PoolTypeFilter::default()) + .get_compact_block( + &snapshot, + types::Height(height), + PoolTypeFilter::includes_all(), + ) .await { Ok(Some(block)) => Ok(block), @@ -963,7 +917,11 @@ impl LightWalletIndexer for FetchServiceSubscriber { }; match self .indexer - .get_compact_block(&snapshot, types::Height(height), PoolTypeFilter::default()) + .get_compact_block( + &snapshot, + types::Height(height), + PoolTypeFilter::includes_all(), + ) .await { Ok(Some(block)) => Ok(compact_block_to_nullifiers(block)), diff --git a/packages/zaino-state/src/backends/state.rs b/packages/zaino-state/src/backends/state.rs index 0461a2f7f..a6cb3b14a 100644 --- a/packages/zaino-state/src/backends/state.rs +++ b/packages/zaino-state/src/backends/state.rs @@ -1374,65 +1374,23 @@ impl ZcashIndexer for StateServiceSubscriber { &self, hash_or_height: String, ) -> Result { - let hash_or_height_struct: HashOrHeight = HashOrHeight::from_str(&hash_or_height)?; - let snapshot = self.indexer.snapshot_nonfinalized_state().await?; - - let block_data = match hash_or_height_struct { - HashOrHeight::Hash(hash) => self - .indexer - .get_indexed_block_by_hash(&snapshot, &hash.into()) - .await - .map_err(|_error| { - StateServiceError::RpcError(RpcError::new_from_legacycode( - zebra_rpc::server::error::LegacyCode::InvalidParameter, - "Failed to fetch block data.", - )) - })? - .ok_or(StateServiceError::RpcError(RpcError::new_from_legacycode( - zebra_rpc::server::error::LegacyCode::InvalidParameter, - "Failed to fetch block data.", - )))?, - HashOrHeight::Height(height) => self - .indexer - .get_indexed_block_by_height(&snapshot, &height.into()) - .await - .map_err(|_error| { - StateServiceError::RpcError(RpcError::new_from_legacycode( - zebra_rpc::server::error::LegacyCode::InvalidParameter, - "Failed to fetch block data.", - )) - })? - .ok_or(StateServiceError::RpcError(RpcError::new_from_legacycode( - zebra_rpc::server::error::LegacyCode::InvalidParameter, - "Failed to fetch block data.", - )))?, - }; - - let (sapling, orchard) = self - .indexer - .get_treestate(block_data.hash()) + self.rpc_client + .get_treestate(hash_or_height) .await .map_err(|_error| { StateServiceError::RpcError(RpcError::new_from_legacycode( zebra_rpc::server::error::LegacyCode::InvalidParameter, "Failed to fetch treestate.", )) - })?; - let time: u32 = block_data.data().time().try_into().map_err(|_error| { - StateServiceError::RpcError(RpcError::new_from_legacycode( - zebra_rpc::server::error::LegacyCode::InvalidParameter, - "Block time is out of range for u32.", - )) - })?; - - #[allow(deprecated)] - Ok(GetTreestateResponse::from_parts( - (*block_data.hash()).into(), - block_data.height().into(), - time, - sapling, - orchard, - )) + }) + .and_then(|treestate| { + treestate.try_into().map_err(|_error| { + StateServiceError::RpcError(RpcError::new_from_legacycode( + zebra_rpc::server::error::LegacyCode::InvalidParameter, + "Failed to parse treestate.", + )) + }) + }) } async fn get_mining_info(&self) -> Result { @@ -1869,7 +1827,7 @@ impl LightWalletIndexer for StateServiceSubscriber { match self .indexer - .get_compact_block(&snapshot, block_height, PoolTypeFilter::default()) + .get_compact_block(&snapshot, block_height, PoolTypeFilter::includes_all()) .await { Ok(Some(block)) => Ok(block), @@ -1934,7 +1892,7 @@ impl LightWalletIndexer for StateServiceSubscriber { match self .indexer - .get_compact_block(&snapshot, block_height, PoolTypeFilter::default()) + .get_compact_block(&snapshot, block_height, PoolTypeFilter::includes_all()) .await { Ok(Some(block)) => Ok(compact_block_to_nullifiers(block)), diff --git a/packages/zaino-state/src/chain_index/finalised_state/db/v1/compact_block.rs b/packages/zaino-state/src/chain_index/finalised_state/db/v1/compact_block.rs index 43a8581c1..da07ab5ae 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/db/v1/compact_block.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/db/v1/compact_block.rs @@ -248,7 +248,7 @@ impl DbV1 { // ----- Construct CompactBlock ----- Ok(zaino_proto::proto::compact_formats::CompactBlock { - proto_version: 4, + proto_version: 0, height: header.context.height().0 as u64, hash: header.context.hash().0.to_vec(), prev_hash: header.context.parent_hash().0.to_vec(), @@ -1107,7 +1107,7 @@ impl DbV1 { }; let compact_block = zaino_proto::proto::compact_formats::CompactBlock { - proto_version: 4, + proto_version: 0, height: header.context.height().0 as u64, hash: header.context.hash().0.to_vec(), prev_hash: header.context.parent_hash().0.to_vec(), diff --git a/packages/zaino-state/src/chain_index/types/db/legacy.rs b/packages/zaino-state/src/chain_index/types/db/legacy.rs index f0f970b0e..865e794cf 100644 --- a/packages/zaino-state/src/chain_index/types/db/legacy.rs +++ b/packages/zaino-state/src/chain_index/types/db/legacy.rs @@ -1194,7 +1194,7 @@ impl IndexedBlock { let orchard_commitment_tree_size = self.commitment_tree_data().sizes().orchard(); zaino_proto::proto::compact_formats::CompactBlock { - proto_version: 4, + proto_version: 0, height, hash, prev_hash, From e50a469d6d96f075029ef6174e86f7f8efd8adff Mon Sep 17 00:00:00 2001 From: Pacu Date: Mon, 27 Apr 2026 12:50:04 -0300 Subject: [PATCH 2/5] Fetch compact blocks from validator on cache miss Teach the chain index to reconstruct compact blocks from the backing validator when a serviceable non-finalized height is absent from the local snapshot/finalized DB. This matches lightwalletd's validator-backed behavior for recently loaded chains and prevents GetBlock/GetBlockRange from reporting database holes while Zaino's local cache is still catching up. --- packages/zaino-state/src/chain_index.rs | 182 ++++++++++++++++++++---- 1 file changed, 157 insertions(+), 25 deletions(-) diff --git a/packages/zaino-state/src/chain_index.rs b/packages/zaino-state/src/chain_index.rs index 221d91727..39343b7aa 100644 --- a/packages/zaino-state/src/chain_index.rs +++ b/packages/zaino-state/src/chain_index.rs @@ -14,11 +14,14 @@ use crate::chain_index::non_finalised_state::ChainIndexSnapshot; use crate::chain_index::source::GetTransactionLocation; use crate::chain_index::types::db::metadata::MempoolInfo; +use crate::chain_index::types::helpers::{BlockMetadata, BlockWithMetadata, TreeRootData}; use crate::chain_index::types::BlockIndex; use crate::chain_index::types::{BestChainLocation, NonBestChainLocation}; use crate::error::{ChainIndexError, ChainIndexErrorKind, FinalisedStateError}; use crate::status::Status; -use crate::{CompactBlockStream, NamedAtomicStatus, NonFinalizedState, StatusType, SyncError}; +use crate::{ + ChainWork, CompactBlockStream, NamedAtomicStatus, NonFinalizedState, StatusType, SyncError, +}; use crate::{IndexedBlock, TransactionHash}; use std::collections::HashSet; use std::{sync::Arc, time::Duration}; @@ -801,6 +804,78 @@ pub struct NodeBackedChainIndexSubscriber( + source: &Source, + network: ZebraNetwork, + height: types::Height, + pool_types: &PoolTypeFilter, +) -> Result, ChainIndexError> { + let Some(block) = source + .get_block(HashOrHeight::Height(zebra_chain::block::Height(height.0))) + .await + .map_err(ChainIndexError::backing_validator)? + else { + return Ok(None); + }; + + let block_height = block + .coinbase_height() + .map(|height| types::Height(height.0)) + .ok_or_else(|| { + ChainIndexError::backing_validator(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "validator returned a block without a height", + )) + })?; + if block_height != height { + return Err(ChainIndexError::backing_validator(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!( + "validator returned block at height {}, expected {}", + block_height.0, height.0 + ), + ))); + } + + let tree_roots = source + .get_commitment_tree_roots(types::BlockHash::from(block.hash())) + .await + .map_err(ChainIndexError::backing_validator)?; + let (sapling_root, sapling_size, orchard_root, orchard_size) = + TreeRootData::new(tree_roots.0, tree_roots.1).extract_with_defaults(); + + let metadata = BlockMetadata::new( + sapling_root, + sapling_size.try_into().map_err(|_| { + ChainIndexError::backing_validator(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "sapling commitment tree size overflow", + )) + })?, + orchard_root, + orchard_size.try_into().map_err(|_| { + ChainIndexError::backing_validator(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "orchard commitment tree size overflow", + )) + })?, + ChainWork::from_u256(0.into()), + network, + ); + let indexed_block = + IndexedBlock::try_from(BlockWithMetadata::new(&block, metadata)).map_err(|error| { + ChainIndexError::backing_validator(std::io::Error::new( + std::io::ErrorKind::InvalidData, + error, + )) + })?; + + Ok(Some(compact_block_with_pool_types( + indexed_block.to_compact_block(), + &pool_types.to_pool_types_vector(), + ))) +} + impl NodeBackedChainIndexSubscriber { fn source(&self) -> &Source { &self.source @@ -834,6 +909,14 @@ impl NodeBackedChainIndexSubscriber { .transpose() } + async fn get_compact_block_from_node( + &self, + height: types::Height, + pool_types: &PoolTypeFilter, + ) -> Result, ChainIndexError> { + compact_block_from_source(self.source(), self.network.clone(), height, pool_types).await + } + async fn get_indexed_block_height( &self, snapshot: &NonfinalizedBlockCacheSnapshot, @@ -1230,19 +1313,19 @@ impl ChainIndex for NodeBackedChainIndexSubscriber match self - .finalized_state - .get_compact_block(height, pool_types) - .await - { - Ok(block) => block, - Err(e) => { - return Err(ChainIndexError::database_hole( - height, - Some(Box::new(e)), - )) + None => { + match self + .finalized_state + .get_compact_block(height, pool_types.clone()) + .await + { + Ok(block) => block, + Err(_) => self + .get_compact_block_from_node(height, &pool_types) + .await? + .ok_or(ChainIndexError::database_hole(height, None))?, } - }, + } })) } else { Ok(None) @@ -1357,6 +1440,9 @@ impl ChainIndex for NodeBackedChainIndexSubscriber ChainIndex for NodeBackedChainIndexSubscriber { + if channel_sender.send(Ok(compact_block)).await.is_err() { + return; + } + continue; + } + Ok(None) => { + let _ = channel_sender + .send(Err(tonic::Status::internal(format!( + "Internal error, missing nonfinalized block at height [{height_value}].", + )))) + .await; + return; + } + Err(error) => { + let _ = channel_sender + .send(Err(tonic::Status::internal(error.to_string()))) + .await; + return; + } + } }; let compact_block = compact_block_with_pool_types( indexed_block.to_compact_block(), @@ -1413,12 +1522,35 @@ impl ChainIndex for NodeBackedChainIndexSubscriber { + if channel_sender.send(Ok(compact_block)).await.is_err() { + return; + } + continue; + } + Ok(None) => { + let _ = channel_sender + .send(Err(tonic::Status::internal(format!( + "Internal error, missing nonfinalized block at height [{height_value}].", + )))) + .await; + return; + } + Err(error) => { + let _ = channel_sender + .send(Err(tonic::Status::internal(error.to_string()))) + .await; + return; + } + } }; let compact_block = compact_block_with_pool_types( indexed_block.to_compact_block(), From e09ff2e7b4abcac067518b82821d21c09705c109 Mon Sep 17 00:00:00 2001 From: Pacu Date: Mon, 27 Apr 2026 15:15:54 -0300 Subject: [PATCH 3/5] Prefer local treestate before validator fallback Restore local chain-index tree-state lookup as the primary path, and only fall back to the backing validator when the local data cannot serve the request. This keeps Zaino's local-data semantics while preserving the boot/cache-miss behavior needed for lightwalletd parity in the gRPC comparison test. --- packages/zaino-state/src/backends/fetch.rs | 58 +++++++++++++++++++++- packages/zaino-state/src/backends/state.rs | 49 +++++++++++++++++- 2 files changed, 104 insertions(+), 3 deletions(-) diff --git a/packages/zaino-state/src/backends/fetch.rs b/packages/zaino-state/src/backends/fetch.rs index 27deefdd9..11ff9aca5 100644 --- a/packages/zaino-state/src/backends/fetch.rs +++ b/packages/zaino-state/src/backends/fetch.rs @@ -2,7 +2,7 @@ use futures::StreamExt; use hex::FromHex; -use std::{io::Cursor, time}; +use std::{io::Cursor, str::FromStr, time}; use tokio::{sync::mpsc, time::timeout}; use tonic::async_trait; use tracing::{info, instrument, warn}; @@ -550,8 +550,62 @@ impl ZcashIndexer for FetchServiceSubscriber { &self, hash_or_height: String, ) -> Result { + let fallback_hash_or_height = hash_or_height.clone(); + let local_result: Result = async { + let hash_or_height_struct: HashOrHeight = HashOrHeight::from_str(&hash_or_height)?; + let snapshot = self.indexer.snapshot_nonfinalized_state().await?; + + let block_data = match hash_or_height_struct { + HashOrHeight::Hash(hash) => self + .indexer + .get_indexed_block_by_hash(&snapshot, &hash.into()) + .await? + .ok_or( + #[allow(deprecated)] + FetchServiceError::RpcError(RpcError::new_from_legacycode( + zebra_rpc::server::error::LegacyCode::InvalidParameter, + "Failed to fetch block data.", + )), + )?, + HashOrHeight::Height(height) => self + .indexer + .get_indexed_block_by_height(&snapshot, &height.into()) + .await? + .ok_or( + #[allow(deprecated)] + FetchServiceError::RpcError(RpcError::new_from_legacycode( + zebra_rpc::server::error::LegacyCode::InvalidParameter, + "Failed to fetch block data.", + )), + )?, + }; + + let (sapling, orchard) = self.indexer.get_treestate(block_data.hash()).await?; + let time: u32 = block_data.data().time().try_into().map_err(|_error| { + #[allow(deprecated)] + FetchServiceError::RpcError(RpcError::new_from_legacycode( + zebra_rpc::server::error::LegacyCode::InvalidParameter, + "Block time is out of range for u32.", + )) + })?; + + #[allow(deprecated)] + Ok(GetTreestateResponse::from_parts( + (*block_data.hash()).into(), + block_data.height().into(), + time, + sapling, + orchard, + )) + } + .await; + + if let Ok(response) = local_result { + return Ok(response); + } + self.fetcher - .get_treestate(hash_or_height) + .get_treestate(fallback_hash_or_height) .await .map_err(|_error| { #[allow(deprecated)] diff --git a/packages/zaino-state/src/backends/state.rs b/packages/zaino-state/src/backends/state.rs index a6cb3b14a..6e56ad712 100644 --- a/packages/zaino-state/src/backends/state.rs +++ b/packages/zaino-state/src/backends/state.rs @@ -1374,8 +1374,55 @@ impl ZcashIndexer for StateServiceSubscriber { &self, hash_or_height: String, ) -> Result { + let fallback_hash_or_height = hash_or_height.clone(); + let local_result: Result = async { + let hash_or_height_struct: HashOrHeight = HashOrHeight::from_str(&hash_or_height)?; + let snapshot = self.indexer.snapshot_nonfinalized_state().await?; + + let block_data = match hash_or_height_struct { + HashOrHeight::Hash(hash) => self + .indexer + .get_indexed_block_by_hash(&snapshot, &hash.into()) + .await? + .ok_or(StateServiceError::RpcError(RpcError::new_from_legacycode( + zebra_rpc::server::error::LegacyCode::InvalidParameter, + "Failed to fetch block data.", + )))?, + HashOrHeight::Height(height) => self + .indexer + .get_indexed_block_by_height(&snapshot, &height.into()) + .await? + .ok_or(StateServiceError::RpcError(RpcError::new_from_legacycode( + zebra_rpc::server::error::LegacyCode::InvalidParameter, + "Failed to fetch block data.", + )))?, + }; + + let (sapling, orchard) = self.indexer.get_treestate(block_data.hash()).await?; + let time: u32 = block_data.data().time().try_into().map_err(|_error| { + StateServiceError::RpcError(RpcError::new_from_legacycode( + zebra_rpc::server::error::LegacyCode::InvalidParameter, + "Block time is out of range for u32.", + )) + })?; + + #[allow(deprecated)] + Ok(GetTreestateResponse::from_parts( + (*block_data.hash()).into(), + block_data.height().into(), + time, + sapling, + orchard, + )) + } + .await; + + if let Ok(response) = local_result { + return Ok(response); + } + self.rpc_client - .get_treestate(hash_or_height) + .get_treestate(fallback_hash_or_height) .await .map_err(|_error| { StateServiceError::RpcError(RpcError::new_from_legacycode( From 0333a69249210744f0fd931898990b28c662cc41 Mon Sep 17 00:00:00 2001 From: Francisco Gindre Date: Wed, 27 May 2026 14:49:41 -0300 Subject: [PATCH 4/5] PR Suggestion --- packages/zaino-state/src/chain_index.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/zaino-state/src/chain_index.rs b/packages/zaino-state/src/chain_index.rs index c828d3274..1c540ac00 100644 --- a/packages/zaino-state/src/chain_index.rs +++ b/packages/zaino-state/src/chain_index.rs @@ -1053,6 +1053,7 @@ async fn compact_block_from_source( "orchard commitment tree size overflow", )) })?, + // TODO: Define an empty value https://github.com/zingolabs/zaino/issues/1158 ChainWork::from_u256(0.into()), network, ); From 8e391217f802b12dc9e9bb1d1c181e46cdd65475 Mon Sep 17 00:00:00 2001 From: Francisco Gindre Date: Wed, 27 May 2026 14:51:09 -0300 Subject: [PATCH 5/5] Fix compiler error --- packages/zaino-state/src/chain_index.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/zaino-state/src/chain_index.rs b/packages/zaino-state/src/chain_index.rs index 1c540ac00..183d79486 100644 --- a/packages/zaino-state/src/chain_index.rs +++ b/packages/zaino-state/src/chain_index.rs @@ -20,7 +20,7 @@ use crate::chain_index::types::{BestChainLocation, NonBestChainLocation}; use crate::error::{ChainIndexError, ChainIndexErrorKind, FinalisedStateError}; use crate::status::Status; use crate::{ - ChainWork, CompactBlockStream, NamedAtomicStatus, NonFinalizedState, StatusType, SyncError, + ChainWork, CompactBlockStream, NamedAtomicStatus, NonFinalizedState, StatusType, SyncError, TxOutCompact, }; use crate::{IndexedBlock, Outpoint, TransactionHash}; use std::collections::HashSet;