diff --git a/CHANGELOG.md b/CHANGELOG.md index ed4c0a3f453..2a454951408 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,14 @@ and this project adheres to [Semantic Versioning](https://semver.org). ### Performance +- Skip the transparent address index (balances, address→utxo, address→tx) on a + pruned, checkpoint-syncing node. The index is RPC-only state, not consensus, so + the minimal fast-validator configuration no longer does the per-block + address-balance reads or the index writes (like pruned mode already skips + raw-transaction storage). Address-lookup RPCs (`getaddressbalance`, + `getaddressutxos`, `getaddresstxids`) return an explicit "index disabled" + error in this mode instead of wrong (empty) results. Archive nodes, and pruned + nodes with checkpoint sync disabled (full semantic verification), are unchanged. - Compute the v5 ZIP-244 txid and authorizing-data digest natively. Both previously routed through `Transaction::to_librustzcash`, which re-serializes and reparses the whole transaction — decompressing every Jubjub and Pallas diff --git a/zebra-state/src/config.rs b/zebra-state/src/config.rs index e65133df9d4..657c5314501 100644 --- a/zebra-state/src/config.rs +++ b/zebra-state/src/config.rs @@ -258,6 +258,22 @@ impl Config { } } + /// Whether to omit archive/indexer-only data that is not required for consensus. + /// + /// These indexes are RPC-only state, not consensus. They are skipped only for + /// the minimal fast-validator configuration: a node that is both + /// [`StorageMode::Pruned`] **and** checkpoint-syncing + /// ([`checkpoint_sync`](Config::checkpoint_sync)). This drops historical + /// transparent address-index writes and transparent finalized-spender lookups, + /// just as pruned mode drops raw-transaction storage. + /// + /// An archive node keeps these indexes; so does a node with checkpoint sync + /// disabled (full semantic verification), even when pruned. RPCs backed by + /// skipped indexes return an error rather than wrong (empty) results. + pub fn skip_archive_indexes(&self) -> bool { + matches!(self.storage_mode, StorageMode::Pruned(_)) && self.checkpoint_sync + } + /// Validates the configured [`StorageMode`]. /// /// This must be called before opening the database, so that a misconfigured @@ -433,6 +449,39 @@ impl Default for Config { mod tests { use super::*; + #[test] + fn skip_archive_indexes_only_when_pruned_and_checkpoint_syncing() { + let pruned_checkpoint = Config { + storage_mode: StorageMode::Pruned(PruningConfig::default()), + checkpoint_sync: true, + ..Default::default() + }; + assert!( + pruned_checkpoint.skip_archive_indexes(), + "pruned + checkpoint-sync skips archive-only indexes" + ); + + let archive = Config { + storage_mode: StorageMode::Archive, + checkpoint_sync: true, + ..Default::default() + }; + assert!( + !archive.skip_archive_indexes(), + "archive mode keeps archive-only indexes" + ); + + let pruned_legacy = Config { + storage_mode: StorageMode::Pruned(PruningConfig::default()), + checkpoint_sync: false, + ..Default::default() + }; + assert!( + !pruned_legacy.skip_archive_indexes(), + "pruned with checkpoint sync disabled keeps archive-only indexes" + ); + } + #[test] fn storage_mode_deserializes_from_documented_toml() { let archive: Config = toml::from_str(r#"storage_mode = "archive""#) diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs index 9edb075cb87..fa2ebfe5f8e 100644 --- a/zebra-state/src/service.rs +++ b/zebra-state/src/service.rs @@ -88,6 +88,13 @@ use self::queued_blocks::{QueuedCheckpointVerified, QueuedSemanticallyVerified, pub use self::traits::{ReadState, State}; +/// Returned for transparent archive/indexer lookups when the required indexes were +/// not built. These indexes are RPC-only, not consensus. +const ARCHIVE_INDEXES_DISABLED: &str = + "the transparent archive indexes are disabled in pruned checkpoint-sync mode; \ + getaddressbalance, getaddressutxos, getaddresstxids, and finalized transparent \ + spender lookups require an archive node or full semantic verification"; + /// A read-write service for Zebra's cached blockchain state. /// /// This service modifies and provides access to: @@ -1658,9 +1665,19 @@ impl Service for ReadStateService { } #[cfg(feature = "indexer")] - ReadRequest::SpendingTransactionId(spend) => Ok(ReadResponse::TransactionId( - read::spending_transaction_hash(state.latest_best_chain(), &state.db, spend), - )), + ReadRequest::SpendingTransactionId(spend) => { + let spending_transaction_hash = + read::spending_transaction_hash(state.latest_best_chain(), &state.db, spend); + + if spending_transaction_hash.is_none() + && matches!(spend, crate::request::Spend::OutPoint(_)) + && state.db.config().skip_archive_indexes() + { + Err(ARCHIVE_INDEXES_DISABLED.into()) + } else { + Ok(ReadResponse::TransactionId(spending_transaction_hash)) + } + } ReadRequest::UnspentBestChainUtxo(outpoint) => Ok(ReadResponse::UnspentBestChainUtxo( read::unspent_utxo(state.latest_best_chain(), &state.db, outpoint), @@ -1807,31 +1824,47 @@ impl Service for ReadStateService { // For the get_address_balance RPC. ReadRequest::AddressBalance(addresses) => { - let (balance, received) = - read::transparent_balance(state.latest_best_chain(), &state.db, addresses)?; - Ok(ReadResponse::AddressBalance { balance, received }) + if state.db.config().skip_archive_indexes() { + Err(ARCHIVE_INDEXES_DISABLED.into()) + } else { + let (balance, received) = + read::transparent_balance(state.latest_best_chain(), &state.db, addresses)?; + Ok(ReadResponse::AddressBalance { balance, received }) + } } // For the get_address_tx_ids RPC. ReadRequest::TransactionIdsByAddresses { addresses, height_range, - } => read::transparent_tx_ids( - state.latest_best_chain(), - &state.db, - addresses, - height_range, - ) - .map(ReadResponse::AddressesTransactionIds), + } => { + if state.db.config().skip_archive_indexes() { + Err(ARCHIVE_INDEXES_DISABLED.into()) + } else { + read::transparent_tx_ids( + state.latest_best_chain(), + &state.db, + addresses, + height_range, + ) + .map(ReadResponse::AddressesTransactionIds) + } + } // For the get_address_utxos RPC. - ReadRequest::UtxosByAddresses(addresses) => read::address_utxos( - &state.network, - state.latest_best_chain(), - &state.db, - addresses, - ) - .map(ReadResponse::AddressUtxos), + ReadRequest::UtxosByAddresses(addresses) => { + if state.db.config().skip_archive_indexes() { + Err(ARCHIVE_INDEXES_DISABLED.into()) + } else { + read::address_utxos( + &state.network, + state.latest_best_chain(), + &state.db, + addresses, + ) + .map(ReadResponse::AddressUtxos) + } + } ReadRequest::CheckBestChainTipNullifiersAndAnchors(unmined_tx) => { let latest_non_finalized_best_chain = state.latest_best_chain(); diff --git a/zebra-state/src/service/finalized_state/disk_format/upgrade/block_info_and_address_received.rs b/zebra-state/src/service/finalized_state/disk_format/upgrade/block_info_and_address_received.rs index ad059d7d844..52aaf7a516b 100644 --- a/zebra-state/src/service/finalized_state/disk_format/upgrade/block_info_and_address_received.rs +++ b/zebra-state/src/service/finalized_state/disk_format/upgrade/block_info_and_address_received.rs @@ -273,6 +273,14 @@ impl DiskFormatUpgrade for Upgrade { return Err(CancelFormatChange); } + // A pruned, checkpoint-syncing node does not build transparent archive-only + // indexes ([`Config::skip_archive_indexes`]), so the received-balance check below + // does not apply — the balances are intentionally absent. The BlockInfo check + // above still runs, since block info is written regardless of the address index. + if db.config().skip_archive_indexes() { + return Ok(Ok(())); + } + // Check that all recipient addresses of transparent transfers in the range have a non-zero received balance. // Collect the set of addresses that received transparent funds in the last query range (last 1000 blocks). diff --git a/zebra-state/src/service/finalized_state/zebra_db/block.rs b/zebra-state/src/service/finalized_state/zebra_db/block.rs index 44eeb7aa3a8..8ee67f96973 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block.rs @@ -886,21 +886,6 @@ impl ZebraDb { .map(|(_outpoint, out_loc, utxo)| (out_loc, utxo)) .collect(); - // Get the transparent addresses with changed balances/UTXOs - let changed_addresses: HashSet = spent_utxos_by_out_loc - .values() - .chain( - finalized - .new_outputs - .values() - .map(|ordered_utxo| &ordered_utxo.utxo), - ) - .filter_map(|utxo| utxo.output.address(network)) - .unique() - .collect(); - - // Get the current address balances, before the transactions in this block - // Like the spent-UTXO reads above, the per-address balance lookups are // cache-served but serial. Fan them across the rayon pool once a block // touches enough addresses to amortize the fork-join cost. @@ -934,14 +919,39 @@ impl ZebraDb { // reading all of the pending merge operands (potentially hundreds), and applying pending merge operands to the // fully-merged value such that it's much faster to read entries that have been updated with insertions than it // is to read entries that have been updated with merge operations. - let address_balances: AddressBalanceLocationUpdates = if self.finished_format_upgrades() { - AddressBalanceLocationUpdates::Insert(read_addr_locs(changed_addresses, |addr| { - self.address_balance_location(addr) - })) + // + // When archive-only indexes are skipped (pruned + checkpoint-sync fast-validator), + // none of the per-address balance reads happen and `address_balances` is left + // empty; the gated transparent index passes below then write no address entries. + let address_balances: AddressBalanceLocationUpdates = if self + .config() + .skip_archive_indexes() + { + AddressBalanceLocationUpdates::Insert(HashMap::new()) } else { - AddressBalanceLocationUpdates::Merge(read_addr_locs(changed_addresses, |addr| { - Some(self.address_balance_location(addr)?.into_new_change()) - })) + // Transparent addresses with changed balances/UTXOs in this block. + let changed_addresses: HashSet = spent_utxos_by_out_loc + .values() + .chain( + finalized + .new_outputs + .values() + .map(|ordered_utxo| &ordered_utxo.utxo), + ) + .filter_map(|utxo| utxo.output.address(network)) + .unique() + .collect(); + + // Get the current address balances, before the transactions in this block. + if self.finished_format_upgrades() { + AddressBalanceLocationUpdates::Insert(read_addr_locs(changed_addresses, |addr| { + self.address_balance_location(addr) + })) + } else { + AddressBalanceLocationUpdates::Merge(read_addr_locs(changed_addresses, |addr| { + Some(self.address_balance_location(addr)?.into_new_change()) + })) + } }; let mut batch = DiskWriteBatch::new(); diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/prune.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/prune.rs index 9bf0a03f4a0..9f6af1af4d0 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/prune.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/prune.rs @@ -56,6 +56,74 @@ fn new_state_with_blocks(config: &Config, network: &Network) -> FinalizedState { state } +/// Returns the number of entries in the column family `cf_name`. +fn cf_len(state: &FinalizedState, cf_name: &str) -> usize { + use crate::service::finalized_state::disk_format::RawBytes; + let cf = state + .db + .db + .cf_handle(cf_name) + .expect("column family exists"); + state + .db + .db + .zs_forward_range_iter::<_, RawBytes, RawBytes, _>(&cf, ..) + .count() +} + +/// A pruned + checkpoint-syncing node commits the UTXO set but skips transparent +/// archive-only index column families; an archive node populates all of them. +#[test] +fn pruned_checkpoint_commit_skips_the_transparent_address_index() { + let _init_guard = zebra_test::init(); + let network = Mainnet; + + const BALANCE_CF: &str = "balance_by_transparent_addr"; + const UTXO_LOC_CF: &str = "utxo_loc_by_transparent_addr_loc"; + const TX_LOC_CF: &str = "tx_loc_by_transparent_addr_loc"; + const SPENT_TX_LOC_CF: &str = "tx_loc_by_spent_out_loc"; + const UTXO_SET_CF: &str = "utxo_by_out_loc"; + + // Archive mode keeps the address index: the early coinbase outputs populate all + // three address column families. + let archive = new_state_with_blocks(&Config::ephemeral(), &network); + assert!( + !archive.db.config().skip_archive_indexes(), + "archive mode keeps transparent archive-only indexes" + ); + assert!( + cf_len(&archive, BALANCE_CF) > 0 + && cf_len(&archive, UTXO_LOC_CF) > 0 + && cf_len(&archive, TX_LOC_CF) > 0, + "archive commit populates the address index: balance={}, utxo_loc={}, tx_loc={}", + cf_len(&archive, BALANCE_CF), + cf_len(&archive, UTXO_LOC_CF), + cf_len(&archive, TX_LOC_CF), + ); + + // Pruned + checkpoint-sync skips archive-only indexes entirely, but still writes + // the consensus-critical UTXO set. + let pruned = new_state_with_blocks(&pruned_config(), &network); + assert!( + pruned.db.config().skip_archive_indexes(), + "pruned + checkpoint-sync skips transparent archive-only indexes" + ); + assert_eq!( + ( + cf_len(&pruned, BALANCE_CF), + cf_len(&pruned, UTXO_LOC_CF), + cf_len(&pruned, TX_LOC_CF), + cf_len(&pruned, SPENT_TX_LOC_CF), + ), + (0, 0, 0, 0), + "pruned + checkpoint-sync writes no transparent archive-only index entries" + ); + assert!( + cf_len(&pruned, UTXO_SET_CF) > 0, + "the UTXO set is still written when the address index is skipped" + ); +} + /// Opens a fresh finalized state with a checkpoint retention start and commits /// blocks `0..=TEST_BLOCKS` for `network`. fn new_state_with_checkpoint_retention( diff --git a/zebra-state/src/service/finalized_state/zebra_db/rollback.rs b/zebra-state/src/service/finalized_state/zebra_db/rollback.rs index b77ac70862d..6c71b3c4551 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/rollback.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/rollback.rs @@ -681,6 +681,12 @@ fn reverse_transparent_block( // (see `prepare_transparent_transaction_batch`). Undoing the operations in the exact reverse // order retraces those same in-range intermediate balances, so the checked balance arithmetic // below cannot spuriously overflow or underflow. + // A pruned, checkpoint-syncing node never built the transparent archive-only indexes + // ([`Config::skip_archive_indexes`]), so there is nothing to un-credit / un-debit + // or remove from the finalized transparent spender index. The UTXO-set reversal + // (`utxo_by_out_loc`) below still runs. + let skip_index = db.config().skip_archive_indexes(); + for (tx_index, transaction) in block.transactions.iter().enumerate().rev() { let tx_location = TransactionLocation::from_usize(height, tx_index); @@ -689,21 +695,23 @@ fn reverse_transparent_block( let created_output_location = OutputLocation::from_usize(height, tx_index, output_index); - if let Some(address) = output.address(network) { - let address_location = - cached_address_balance(db, address_balances, &address)?.address_location(); - - batch.zs_delete( - db.db.cf_handle("tx_loc_by_transparent_addr_loc").unwrap(), - AddressTransaction::new(address_location, tx_location), - ); - batch.zs_delete( - db.db.cf_handle("utxo_loc_by_transparent_addr_loc").unwrap(), - AddressUnspentOutput::new(address_location, created_output_location), - ); - - sub_address_balance(db, address_balances, &address, output.value())?; - sub_address_received(db, address_balances, &address, output.value()); + if !skip_index { + if let Some(address) = output.address(network) { + let address_location = + cached_address_balance(db, address_balances, &address)?.address_location(); + + batch.zs_delete( + db.db.cf_handle("tx_loc_by_transparent_addr_loc").unwrap(), + AddressTransaction::new(address_location, tx_location), + ); + batch.zs_delete( + db.db.cf_handle("utxo_loc_by_transparent_addr_loc").unwrap(), + AddressUnspentOutput::new(address_location, created_output_location), + ); + + sub_address_balance(db, address_balances, &address, output.value())?; + sub_address_received(db, address_balances, &address, output.value()); + } } batch.zs_delete( @@ -716,21 +724,23 @@ fn reverse_transparent_block( for spent_outpoint in transaction.inputs().iter().filter_map(Input::outpoint) { let (spent_output_location, spent_utxo) = finalized_output(db, &spent_outpoint)?; - if let Some(address) = spent_utxo.output.address(network) { - let address_location = - cached_address_balance(db, address_balances, &address)?.address_location(); - - batch.zs_delete( - db.db.cf_handle("tx_loc_by_transparent_addr_loc").unwrap(), - AddressTransaction::new(address_location, tx_location), - ); - batch.zs_insert( - db.db.cf_handle("utxo_loc_by_transparent_addr_loc").unwrap(), - AddressUnspentOutput::new(address_location, spent_output_location), - (), - ); - - add_address_balance(db, address_balances, &address, spent_utxo.output.value())?; + if !skip_index { + if let Some(address) = spent_utxo.output.address(network) { + let address_location = + cached_address_balance(db, address_balances, &address)?.address_location(); + + batch.zs_delete( + db.db.cf_handle("tx_loc_by_transparent_addr_loc").unwrap(), + AddressTransaction::new(address_location, tx_location), + ); + batch.zs_insert( + db.db.cf_handle("utxo_loc_by_transparent_addr_loc").unwrap(), + AddressUnspentOutput::new(address_location, spent_output_location), + (), + ); + + add_address_balance(db, address_balances, &address, spent_utxo.output.value())?; + } } batch.zs_insert( diff --git a/zebra-state/src/service/finalized_state/zebra_db/transparent.rs b/zebra-state/src/service/finalized_state/zebra_db/transparent.rs index a5aec765587..d63791e622f 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/transparent.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/transparent.rs @@ -434,18 +434,27 @@ impl DiskWriteBatch { let db = &zebra_db.db; let FinalizedBlock { block, height, .. } = finalized; + // A pruned, checkpoint-syncing node skips transparent archive-only indexes. + // The UTXO-set passes still run (they write/delete `utxo_by_out_loc`), but + // the address-balance update, address index writes, and finalized + // transparent spender index writes are all elided. The UTXO/value-pool/ + // nullifier state is unchanged. + let skip_index = zebra_db.config().skip_archive_indexes(); + // Update the in-memory `address_balances` transaction-by-transaction, debiting inputs // before crediting outputs within each transaction. This ordering keeps every // intermediate per-address balance within the consensus range, even when the block // contains a same-address transparent self-spend chain whose batch credit-first // intermediate balance would otherwise exceed MAX_MONEY. - Self::prepare_transparent_address_balance_updates( - network, - *height, - &block.transactions, - spent_utxos_by_outpoint, - &mut address_balances, - ); + if !skip_index { + Self::prepare_transparent_address_balance_updates( + network, + *height, + &block.transactions, + spent_utxos_by_outpoint, + &mut address_balances, + ); + } // Write the new and spent transparent output index entries. These passes no longer // touch `address_balances`; they only read each entry's `address_location()`. @@ -454,31 +463,37 @@ impl DiskWriteBatch { network, new_outputs_by_out_loc, &address_balances, + skip_index, ); self.prepare_spent_transparent_outputs_batch( db, network, spent_utxos_by_out_loc, &address_balances, + skip_index, ); // Index the transparent addresses that spent in each transaction - for (tx_index, transaction) in block.transactions.iter().enumerate() { - let spending_tx_location = TransactionLocation::from_usize(*height, tx_index); - - self.prepare_spending_transparent_tx_ids_batch( - zebra_db, - network, - spending_tx_location, - transaction, - spent_utxos_by_outpoint, - #[cfg(feature = "indexer")] - out_loc_by_outpoint, - &address_balances, - ); + if !skip_index { + for (tx_index, transaction) in block.transactions.iter().enumerate() { + let spending_tx_location = TransactionLocation::from_usize(*height, tx_index); + + self.prepare_spending_transparent_tx_ids_batch( + zebra_db, + network, + spending_tx_location, + transaction, + spent_utxos_by_outpoint, + #[cfg(feature = "indexer")] + out_loc_by_outpoint, + &address_balances, + ); + } } - self.prepare_transparent_balances_batch(db, address_balances); + if !skip_index { + self.prepare_transparent_balances_batch(db, address_balances); + } } /// Update `address_balances` in memory for the transparent transfers in `transactions`, @@ -591,6 +606,7 @@ impl DiskWriteBatch { network: &Network, new_outputs_by_out_loc: &BTreeMap, address_balances: &AddressBalanceLocationUpdates, + skip_index: bool, ) { let utxo_by_out_loc = db.cf_handle("utxo_by_out_loc").unwrap(); let utxo_loc_by_transparent_addr_loc = @@ -601,7 +617,9 @@ impl DiskWriteBatch { // Index all new transparent outputs for (new_output_location, utxo) in new_outputs_by_out_loc { let unspent_output = &utxo.output; - let receiving_address = unspent_output.address(network); + let receiving_address = (!skip_index) + .then(|| unspent_output.address(network)) + .flatten(); if let Some(receiving_address) = receiving_address { let receiving_address_location = match address_balances { @@ -663,6 +681,7 @@ impl DiskWriteBatch { network: &Network, spent_utxos_by_out_loc: &BTreeMap, address_balances: &AddressBalanceLocationUpdates, + skip_index: bool, ) { let utxo_by_out_loc = db.cf_handle("utxo_by_out_loc").unwrap(); let utxo_loc_by_transparent_addr_loc = @@ -673,7 +692,9 @@ impl DiskWriteBatch { // Coinbase inputs represent new coins, so there are no UTXOs to mark as spent. for (spent_output_location, utxo) in spent_utxos_by_out_loc { let spent_output = &utxo.output; - let sending_address = spent_output.address(network); + let sending_address = (!skip_index) + .then(|| spent_output.address(network)) + .flatten(); // Fetch the link from the address to the AddressLocation, from memory. if let Some(sending_address) = sending_address { diff --git a/zebra-state/src/service/tests.rs b/zebra-state/src/service/tests.rs index 06cc4e36237..9a3c372ea6c 100644 --- a/zebra-state/src/service/tests.rs +++ b/zebra-state/src/service/tests.rs @@ -644,6 +644,37 @@ async fn header_only_service_requests_preserve_body_boundary() -> std::result::R Ok(()) } +#[cfg(feature = "indexer")] +#[tokio::test(flavor = "multi_thread")] +async fn pruned_checkpoint_transparent_spender_lookup_returns_disabled_index_error( +) -> std::result::Result<(), BoxError> { + let _init_guard = zebra_test::init(); + let network = Network::Mainnet; + let config = Config { + storage_mode: crate::config::StorageMode::Pruned(crate::PruningConfig::default()), + checkpoint_sync: true, + ..Config::ephemeral() + }; + let (_, read_state, _, _) = StateService::new(config, &network, Height::MAX, 0).await; + + let outpoint = transparent::OutPoint::from_usize(transaction::Hash([0; 32]), 0); + let error = read_state + .oneshot(ReadRequest::SpendingTransactionId( + crate::request::Spend::OutPoint(outpoint), + )) + .await + .expect_err("transparent spender lookup should require archive indexes"); + + assert!( + error + .to_string() + .contains("transparent archive indexes are disabled"), + "unexpected error: {error}", + ); + + Ok(()) +} + /// A node still in the finalized (checkpoint) write phase must be able to commit /// a Zakura header range. ///