From da581cdd9a8681b9678229bac22b1a6966a8ed23 Mon Sep 17 00:00:00 2001 From: zancas Date: Wed, 1 Jul 2026 21:13:33 -0700 Subject: [PATCH 01/19] Rename `makers test` set `package` -> `packages` The front-door test set is named for the `packages/` directory it runs, so the plural is the faithful name. Hard-cut: `makers test package` now errors with `unknown set` rather than aliasing. Updates every reference site: Makefile.toml (default, case matcher, dispatch, descriptions, comment block), help.sh, docs/testing.md, README.md, and the live-tests/CONTEXT.md glossary term (now records the retired singular under _Avoid_). ADR-0004 gets a one-line supersession footer; its historical text is preserved. Co-Authored-By: Claude Opus 4.8 (1M context) --- Makefile.toml | 32 +++++++++---------- README.md | 2 +- ...ame-integration-partition-to-clientless.md | 3 ++ docs/testing.md | 6 ++-- live-tests/CONTEXT.md | 12 ++++--- tools/scripts/help.sh | 8 ++--- 6 files changed, 34 insertions(+), 29 deletions(-) diff --git a/Makefile.toml b/Makefile.toml index f38a60907..3595ca15a 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -93,7 +93,7 @@ script.main = 'source "./tools/scripts/push-image.sh"' [tasks.container-test] clear = true -description = "Engine: run nextest in the local CI container, forwarding args (zcashd_support OFF by default; --with-zcashd adds --features zcashd_support). Devs use `makers test [package|e2e|clientless|live|all]`; live-clientless / live-e2e and the `test` front door call this." +description = "Engine: run nextest in the local CI container, forwarding args (zcashd_support OFF by default; --with-zcashd adds --features zcashd_support). Devs use `makers test [packages|e2e|clientless|live|all]`; live-clientless / live-e2e and the `test` front door call this." # This task runs tests inside the container built from live-tests/test_environment. # The entrypoint.sh script in the container sets up test binaries (zcashd, zebrad, zcash-cli) # by creating symlinks from /home/container_user/artifacts to the expected live-tests/test_binaries/bins location. @@ -132,16 +132,16 @@ script.main = 'source "./tools/scripts/container-nextest-workspace.sh"' # =================================================================== # Developer-facing test front door: a single `makers test [SET]` task. # -# makers test packaged (the default) -# makers test package packages/* production-crate tests (no validator) +# makers test packages (the default) +# makers test packages packages/* production-crate tests (no validator) # makers test e2e the e2e live partition # makers test clientless the clientless live partition # makers test live both live partitions + combined summary -# makers test all package then live (everything) +# makers test all packages then live (everything) # -# SET names mirror the crate/dir names (`e2e`, `clientless`); `package`, -# `live`, and `all` are the groupings. `live` = all non-package tests; `all` -# = `package` + `live` (see live-tests/CONTEXT.md). Each set delegates to an +# SET names mirror the crate/dir names (`e2e`, `clientless`); `packages`, +# `live`, and `all` are the groupings. `live` = all non-packages tests; `all` +# = `packages` + `live` (see live-tests/CONTEXT.md). Each set delegates to an # existing engine rather than re-implementing a run. `--with-zcashd` (handled # by the engines) adds the zcashd-backed tests to any set. # =================================================================== @@ -150,33 +150,33 @@ script.main = 'source "./tools/scripts/container-nextest-workspace.sh"' # clear = true fully replaces cargo-make's built-in `test` task (which runs # `cargo test`); without it our `script` would merge with the builtin `command`. clear = true -description = "Run a test set: `makers test [package|e2e|clientless|live|all]` (default: package). Pass --with-zcashd for the zcashd-backed tests." +description = "Run a test set: `makers test [packages|e2e|clientless|live|all]` (default: packages). Pass --with-zcashd for the zcashd-backed tests." script_runner = "bash" script = ''' set -uo pipefail usage() { - echo "usage: makers test [package|e2e|clientless|live|all] [-- nextest args]" >&2 - echo " package (default) packages/* tests, no live validator" >&2 + echo "usage: makers test [packages|e2e|clientless|live|all] [-- nextest args]" >&2 + echo " packages (default) packages/* tests, no live validator" >&2 echo " e2e | clientless that single live partition" >&2 echo " live both live partitions + combined summary" >&2 - echo " all package then live (everything)" >&2 + echo " all packages then live (everything)" >&2 } # A non-flag first arg selects the set and must be a known name; a flag first -# arg (or none) keeps the default `package` and is forwarded to nextest. So -# `makers test --with-zcashd` runs package with the flag, while a mistyped set +# arg (or none) keeps the default `packages` and is forwarded to nextest. So +# `makers test --with-zcashd` runs packages with the flag, while a mistyped set # name errors instead of silently running the default. -set="package" +set="packages" if [ "$#" -gt 0 ] && [ "${1#-}" = "$1" ]; then case "$1" in - package|e2e|clientless|live|all) set="$1"; shift ;; + packages|e2e|clientless|live|all) set="$1"; shift ;; *) echo "makers test: unknown set '$1'" >&2; usage; exit 2 ;; esac fi case "$set" in - package) exec makers container-test "$@" ;; + packages) exec makers container-test "$@" ;; e2e) exec makers live-e2e "$@" ;; clientless) exec makers live-clientless "$@" ;; live) exec cargo run -q --manifest-path tools/test-runner/Cargo.toml --bin live-summary -- "$@" ;; diff --git a/README.md b/README.md index a72a9a495..0b8ca92f4 100644 --- a/README.md +++ b/README.md @@ -106,7 +106,7 @@ The test suites run inside a **podman** container via `makers` (cargo-make): ```sh makers test # packages/* tests that need no live validator (default) makers test live # both live partitions (clientless + e2e) + combined summary -makers test all # everything: package then live +makers test all # everything: packages then live ``` zcashd-backed tests are **off by default**; add `--with-zcashd` to include them diff --git a/docs/adr/0004-rename-integration-partition-to-clientless.md b/docs/adr/0004-rename-integration-partition-to-clientless.md index c98de87d4..cb80fbfa7 100644 --- a/docs/adr/0004-rename-integration-partition-to-clientless.md +++ b/docs/adr/0004-rename-integration-partition-to-clientless.md @@ -50,3 +50,6 @@ unchanged. "clientless" is no longer avoided. - ADR-0003's naming bullet and its three-front-door task surface are superseded; its two-crate split and the `live` umbrella decision still stand. + +> **Later note:** the `package` set was renamed `packages` (plural, matching the +> `packages/` dir); `makers test package` now errors. See `live-tests/CONTEXT.md`. diff --git a/docs/testing.md b/docs/testing.md index f5c6d8f10..9bfd24575 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -26,14 +26,14 @@ The `makers` tasks below build and run the test suites inside a **podman** container, so you don't need the validator binaries on your host `$PATH`. The container image is built or pulled automatically on first run. -The single front door is `makers test [SET]`, where `SET` defaults to `package`: +The single front door is `makers test [SET]`, where `SET` defaults to `packages`: -- `makers test` (or `makers test package`) — the **package** set: the +- `makers test` (or `makers test packages`) — the **packages** set: the `packages/*` production-crate tests that need no live validator. - `makers test e2e` / `makers test clientless` — one live partition. - `makers test live` — both live partitions (`clientless` then `e2e`) against a live validator, with a combined pass/fail summary. -- `makers test all` — the whole suite: package then live. +- `makers test all` — the whole suite: packages then live. (`container-test`, `live-clientless`, and `live-e2e` are the internal engines the `test` front door delegates to; invoke them directly only when you need to diff --git a/live-tests/CONTEXT.md b/live-tests/CONTEXT.md index d00e0f064..1bef4076e 100644 --- a/live-tests/CONTEXT.md +++ b/live-tests/CONTEXT.md @@ -40,15 +40,17 @@ owns shared fixtures and the feature-forwarding surface (`zcashd_support`, `transparent_address_history_experimental`). _Avoid_: test helpers, testlib. -**package (test set)**: +**packages (test set)**: The tests of the `packages/*` production crates — the workspace `default-members`, exercised by bare `cargo nextest run` and by `makers test` -(the default with no argument, equivalently `makers test package`). Named for +(the default with no argument, equivalently `makers test packages`). Named for where the crates live (`packages/`); the functional reason they run apart from the live partitions is that they need **no** live validator. They are *not* network-free, though: e.g. zaino-serve's gRPC `spawn` regression test binds a loopback socket and stands up a tonic server. The absent ingredient is a validator, not the network. -_Avoid_: offline (overclaims "no network" — these tests bind loopback sockets); -unit test (the set includes crate-level integration tests); container test -(names the run mechanism, which the live suite shares). +_Avoid_: package (singular — the set is plural, matching the `packages/` dir; +`makers test package` now errors with `unknown set`); offline (overclaims "no +network" — these tests bind loopback sockets); unit test (the set includes +crate-level integration tests); container test (names the run mechanism, which +the live suite shares). diff --git a/tools/scripts/help.sh b/tools/scripts/help.sh index 524c80b6c..7f962451c 100755 --- a/tools/scripts/help.sh +++ b/tools/scripts/help.sh @@ -12,7 +12,7 @@ echo "" echo "Common usage:" echo " makers test # packages/* tests, no live validator (default)" echo " makers test live # both live partitions + combined summary" -echo " makers test all # everything: package then live" +echo " makers test all # everything: packages then live" echo "" echo "If you modify '.env.testing-artifacts', the test command will \ automatically:" @@ -21,15 +21,15 @@ echo " - Build a new local container image if needed" echo "" echo "Available commands:" echo "" -echo " test [SET] Front door. SET = package (default) | e2e | \ +echo " test [SET] Front door. SET = packages (default) | e2e | \ clientless | live | all" -echo " package packages/* tests, no live \ +echo " packages packages/* tests, no live \ validator" echo " e2e the e2e live partition" echo " clientless the clientless live partition" echo " live both live partitions + \ combined summary" -echo " all package then live (everything)" +echo " all packages then live (everything)" echo " container-test Engine: run nextest in the container \ (used by the front door; invoke directly to forward engine flags)" echo " live-clientless Engine: run the clientless live-test \ From d53e9f0a207fba1c72adc9efb157ba6505c736be Mon Sep 17 00:00:00 2001 From: zancas Date: Mon, 29 Jun 2026 14:33:54 -0700 Subject: [PATCH 02/19] Add outp_to_spend_index feature and read-free spend extractor Introduce the default-off `outp_to_spend_index` experimental feature gating a self-contained module for the parallel-buildable finalised spend-index POC. Adds the pure, statically read-free extractor extract_spends(&[IndexedBlock]) -> Vec<(Outpoint, TransactionHash)>, mapping each consumed transparent outpoint to its spending txid (coinbase null-prevout inputs skipped), plus a unit test for the coinbase-skip rule. Co-Authored-By: Claude Opus 4.8 --- packages/zaino-state/Cargo.toml | 10 ++- .../src/chain_index/finalised_state.rs | 2 + .../finalised_state/outp_to_spend_index.rs | 90 +++++++++++++++++++ 3 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs diff --git a/packages/zaino-state/Cargo.toml b/packages/zaino-state/Cargo.toml index 4f96dbe9f..e5b3a4320 100644 --- a/packages/zaino-state/Cargo.toml +++ b/packages/zaino-state/Cargo.toml @@ -32,13 +32,21 @@ test_dependencies = [] # **Experimental and alpha features** # Exposes the **complete** set of experimental / alpha features currently implemented in Zaino. -experimental_features = ["transparent_address_history_experimental"] +experimental_features = [ + "transparent_address_history_experimental", + "outp_to_spend_index", +] # Activates transparent address history capability in zaino # # NOTE: currently this is only implemented in the finalised state. transparent_address_history_experimental = [] +# Standalone, parallel-buildable finalised spend index +# (transparent_outpoint -> spending_txid). Proof of concept; off by default. +# See docs/adr/0002-finalised-spend-index-parallel-build.md. +outp_to_spend_index = [] + [dependencies] zaino-common = { workspace = true } zaino-fetch = { workspace = true } diff --git a/packages/zaino-state/src/chain_index/finalised_state.rs b/packages/zaino-state/src/chain_index/finalised_state.rs index 575ba1e5b..473302b4f 100644 --- a/packages/zaino-state/src/chain_index/finalised_state.rs +++ b/packages/zaino-state/src/chain_index/finalised_state.rs @@ -218,6 +218,8 @@ pub(crate) mod capability; pub(crate) mod entry; pub(crate) mod finalised_source; pub(crate) mod migrations; +#[cfg(feature = "outp_to_spend_index")] +mod outp_to_spend_index; pub(crate) mod reader; pub(crate) mod router; diff --git a/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs b/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs new file mode 100644 index 000000000..05f717767 --- /dev/null +++ b/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs @@ -0,0 +1,90 @@ +//! Standalone, parallel-buildable finalised spend index: maps each transparent +//! outpoint to the txid of the transaction that consumed it. +//! +//! Proof of concept, gated behind the `outp_to_spend_index` feature. See +//! `docs/adr/0002-finalised-spend-index-parallel-build.md`. +//! +//! This slice holds the **read-free extractor** — the pure first stage of the +//! build. The independent sync loop and the sorted-merge collator land in later +//! slices. + +use crate::chain_index::types::TransactionHash; +use crate::{IndexedBlock, Outpoint, TransparentCompactTx}; + +/// One transparent spend: the consumed outpoint paired with the txid of the +/// transaction that consumed it. +pub(super) type SpendRecord = (Outpoint, TransactionHash); + +/// Extracts every transparent spend recorded in `blocks`. +/// +/// Pure and statically read-free: it is handed only block data — no database +/// and no validator handle — so a previous-output lookup is unrepresentable, +/// not merely discouraged. Each transparent input yields +/// `(prevout_outpoint, spending_txid)` directly, where the spending txid is the +/// containing transaction's own id; nothing outside the block stream is +/// consulted. +/// +/// TODO (collator slice): replace with +/// `extract_spends_into(blocks, &mut Vec)` so the per-worker loop +/// reuses one buffer across batches — `clear()` retains capacity, dropping +/// steady-state allocation to ~zero, and the collator sorts that buffer in +/// place. Left as the simple collecting form for now: the extractor is dwarfed +/// by zebra block I/O and the sort, so buffer reuse only pays once wired into +/// the loop. +pub(super) fn extract_spends(blocks: &[IndexedBlock]) -> Vec { + blocks + .iter() + .flat_map(|block| block.transactions().iter()) + .flat_map(|tx| spends_in_transaction(*tx.txid(), tx.transparent())) + .collect() +} + +/// The spends contributed by one transaction: each outpoint it consumes paired +/// with `spending_txid`. The non-coinbase filtering lives in +/// [`TransparentCompactTx::spent_outpoints`]; this only attaches the spender. +fn spends_in_transaction( + spending_txid: TransactionHash, + transparent: &TransparentCompactTx, +) -> impl Iterator + '_ { + transparent + .spent_outpoints() + .map(move |outpoint| (outpoint, spending_txid)) +} + +#[cfg(test)] +mod spends_in_transaction { + use super::*; + use crate::TxInCompact; + + /// Arbitrary fill byte for the spending transaction's txid, kept distinct + /// from the prevout txids below so the two can't be confused. + const SPENDER_TXID_BYTE: u8 = 9; + + fn txid(byte: u8) -> TransactionHash { + TransactionHash::from([byte; 32]) + } + + #[test] + fn skips_coinbase_input_keeps_real_spends() { + let spender = txid(SPENDER_TXID_BYTE); + let transparent = TransparentCompactTx::new( + vec![ + TxInCompact::null_prevout(), // coinbase input → contributes nothing + TxInCompact::new([1u8; 32], 0), // spends output 0 of txid 0x01.. + TxInCompact::new([2u8; 32], 7), // spends output 7 of txid 0x02.. + ], + vec![], + ); + + let records: Vec = + super::spends_in_transaction(spender, &transparent).collect(); + + assert_eq!( + records, + vec![ + (Outpoint::new([1u8; 32], 0), spender), + (Outpoint::new([2u8; 32], 7), spender), + ] + ); + } +} From 0efa0aa94511af2dd2b68214d8f4e9596038a830 Mon Sep 17 00:00:00 2001 From: zancas Date: Mon, 29 Jun 2026 16:58:45 -0700 Subject: [PATCH 03/19] Add spend-index collator core (collate) Add `collate`: encode each spend record as (outpoint key, bare 32-byte txid value), sort by key bytes (LMDB memcmp order, ready for MDB_APPEND), and reject duplicate keys (the disjoint-key invariant). Adds the EncodedSpend type and collate tests. The extractor carries a TODO for the buffer-reuse variant, to land with the per-worker loop. No integrity commitment (not MVP); the MDB_APPEND write folds into the loop slice, which opens the index's LMDB environment. Co-Authored-By: Claude Opus 4.8 --- .../finalised_state/outp_to_spend_index.rs | 86 ++++++++++++++++--- 1 file changed, 75 insertions(+), 11 deletions(-) diff --git a/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs b/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs index 05f717767..a4a18cf57 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs @@ -4,17 +4,25 @@ //! Proof of concept, gated behind the `outp_to_spend_index` feature. See //! `docs/adr/0002-finalised-spend-index-parallel-build.md`. //! -//! This slice holds the **read-free extractor** — the pure first stage of the -//! build. The independent sync loop and the sorted-merge collator land in later -//! slices. +//! Pure, read-free build stages live here: **extract** spends from a block +//! batch and **collate** them into LMDB key order. The `MDB_APPEND` write and +//! the independent sync loop that drives these land in later slices. (Table- +//! level integrity over the entries is deferred; it is not MVP.) use crate::chain_index::types::TransactionHash; -use crate::{IndexedBlock, Outpoint, TransparentCompactTx}; +use crate::error::FinalisedStateError; +use crate::{IndexedBlock, Outpoint, TransparentCompactTx, ZainoVersionedSerde as _}; /// One transparent spend: the consumed outpoint paired with the txid of the /// transaction that consumed it. pub(super) type SpendRecord = (Outpoint, TransactionHash); +/// A spend entry encoded for storage: the LMDB key (the encoded outpoint) and +/// value (the bare 32-byte spending txid), ready for an `MDB_APPEND` load. +pub(super) type EncodedSpend = (Vec, [u8; 32]); + +// ── Extract ────────────────────────────────────────────────────────────────── + /// Extracts every transparent spend recorded in `blocks`. /// /// Pure and statically read-free: it is handed only block data — no database @@ -24,13 +32,12 @@ pub(super) type SpendRecord = (Outpoint, TransactionHash); /// containing transaction's own id; nothing outside the block stream is /// consulted. /// -/// TODO (collator slice): replace with -/// `extract_spends_into(blocks, &mut Vec)` so the per-worker loop -/// reuses one buffer across batches — `clear()` retains capacity, dropping -/// steady-state allocation to ~zero, and the collator sorts that buffer in -/// place. Left as the simple collecting form for now: the extractor is dwarfed -/// by zebra block I/O and the sort, so buffer reuse only pays once wired into -/// the loop. +/// TODO (loop slice): add a buffer-filling variant that writes into a +/// caller-owned `&mut Vec`, once the per-worker loop exists to +/// reuse one allocation across batches (`Vec::clear` keeps capacity ⇒ ~zero +/// steady-state alloc, and the collator can sort that buffer in place). +/// Deferred until a real reusing caller anchors it: the extractor is dwarfed by +/// zebra block I/O and the sort, so buffer reuse only pays then. pub(super) fn extract_spends(blocks: &[IndexedBlock]) -> Vec { blocks .iter() @@ -51,6 +58,35 @@ fn spends_in_transaction( .map(move |outpoint| (outpoint, spending_txid)) } +// ── Collate ────────────────────────────────────────────────────────────────── + +/// Encodes and sorts `records` into LMDB key order — byte-wise on the encoded +/// outpoint key, matching LMDB's default comparator — so the result can be +/// bulk-loaded with `MDB_APPEND`. +/// +/// Spend keys are globally disjoint — each outpoint is spent at most once on a +/// chain — so this is a pure sort needing no cross-record reconciliation; a +/// duplicate key means corrupt input and is rejected. +pub(super) fn collate(records: &[SpendRecord]) -> Result, FinalisedStateError> { + let mut encoded = records + .iter() + .map(|(outpoint, spending_txid)| { + Ok((outpoint.to_bytes()?, <[u8; 32]>::from(*spending_txid))) + }) + .collect::, FinalisedStateError>>()?; + + encoded.sort_by(|a, b| a.0.cmp(&b.0)); + + if let Some(duplicate) = encoded.windows(2).find(|pair| pair[0].0 == pair[1].0) { + return Err(FinalisedStateError::Custom(format!( + "duplicate spend-index key during collation: {:?}", + duplicate[0].0 + ))); + } + + Ok(encoded) +} + #[cfg(test)] mod spends_in_transaction { use super::*; @@ -88,3 +124,31 @@ mod spends_in_transaction { ); } } + +#[cfg(test)] +mod collate { + use super::*; + + fn record(outpoint_byte: u8, index: u32) -> SpendRecord { + ( + Outpoint::new([outpoint_byte; 32], index), + TransactionHash::from([0xaau8; 32]), + ) + } + + #[test] + fn sorts_into_ascending_key_order() { + let records = [record(3, 0), record(1, 0), record(2, 0)]; + let encoded = super::collate(&records).expect("disjoint keys collate cleanly"); + assert!( + encoded.windows(2).all(|pair| pair[0].0 < pair[1].0), + "collated keys must be strictly ascending for MDB_APPEND", + ); + } + + #[test] + fn rejects_duplicate_keys() { + let records = [record(1, 0), record(1, 0)]; + assert!(super::collate(&records).is_err()); + } +} From 4ebc5320a9a54e58e6c8cef57679a0d61b5e10b3 Mon Sep 17 00:00:00 2001 From: zancas Date: Mon, 29 Jun 2026 21:34:27 -0700 Subject: [PATCH 04/19] Test collate's little-endian index tie-break ordering MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pin that collate sorts by encoded key bytes (LMDB memcmp order), not numerically: for two spends sharing a prev_txid, the LE-encoded output index makes 256 (00 01 00 00) sort before 1 (01 00 00 00) — exactly what MDB_APPEND expects. Regression-guards the sort<->storage agreement. Co-Authored-By: Claude Opus 4.8 --- .../finalised_state/outp_to_spend_index.rs | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs b/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs index a4a18cf57..7188c09a4 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs @@ -151,4 +151,28 @@ mod collate { let records = [record(1, 0), record(1, 0)]; assert!(super::collate(&records).is_err()); } + + #[test] + fn little_endian_index_tie_break_matches_byte_order() { + // Same creating txid, two output indices. The index is LE-encoded, so + // 256 (bytes `00 01 00 00`) sorts before 1 (`01 00 00 00`) under memcmp + // — exactly the order LMDB's default comparator (and thus MDB_APPEND) + // requires. collate must reproduce that, not a numeric ordering. + let encoded = + super::collate(&[record(7, 1), record(7, 256)]).expect("disjoint keys collate cleanly"); + + assert!( + encoded[0].0 < encoded[1].0, + "keys must be strictly ascending for MDB_APPEND", + ); + // The trailing LE u32 of each key shows index 256 precedes index 1. + assert_eq!( + &encoded[0].0[encoded[0].0.len() - 4..], + &256u32.to_le_bytes()[..] + ); + assert_eq!( + &encoded[1].0[encoded[1].0.len() - 4..], + &1u32.to_le_bytes()[..] + ); + } } From e6dbc0e74833d311ec3bddefe4371757ea147c75 Mon Sep 17 00:00:00 2001 From: zancas Date: Mon, 29 Jun 2026 22:05:48 -0700 Subject: [PATCH 05/19] Add SpendIndexDb: standalone LMDB store for the spend index Open the index's own LMDB environment (no WRITE_MAP, mirroring the finalised state) holding a single outpoint -> spending_txid database. bulk_load writes collated entries via MDB_APPEND in one transaction (one-shot, globally-sorted input; a debug_assert documents the contract and LMDB self-checks it at runtime). spending_txid is a point read (None = unspent in finalised state). Round-trip test over a temp environment. Co-Authored-By: Claude Opus 4.8 --- .../finalised_state/outp_to_spend_index.rs | 143 ++++++++++++++++-- 1 file changed, 128 insertions(+), 15 deletions(-) diff --git a/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs b/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs index 7188c09a4..a5b8a3657 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs @@ -4,10 +4,15 @@ //! Proof of concept, gated behind the `outp_to_spend_index` feature. See //! `docs/adr/0002-finalised-spend-index-parallel-build.md`. //! -//! Pure, read-free build stages live here: **extract** spends from a block -//! batch and **collate** them into LMDB key order. The `MDB_APPEND` write and -//! the independent sync loop that drives these land in later slices. (Table- -//! level integrity over the entries is deferred; it is not MVP.) +//! The build stages live here: **extract** spends from a block batch, +//! **collate** them into LMDB key order, and bulk-load them into the index's +//! own LMDB store via `MDB_APPEND` ([`SpendIndexDb`]). The independent sync +//! loop that drives these (a genesis re-stream from zebra) lands in a later +//! slice. (Table-level integrity over the entries is deferred; it is not MVP.) + +use std::path::Path; + +use lmdb::{Database, DatabaseFlags, Environment, EnvironmentFlags, Transaction as _, WriteFlags}; use crate::chain_index::types::TransactionHash; use crate::error::FinalisedStateError; @@ -70,9 +75,7 @@ fn spends_in_transaction( pub(super) fn collate(records: &[SpendRecord]) -> Result, FinalisedStateError> { let mut encoded = records .iter() - .map(|(outpoint, spending_txid)| { - Ok((outpoint.to_bytes()?, <[u8; 32]>::from(*spending_txid))) - }) + .map(|(outpoint, spending_txid)| Ok((outpoint.to_bytes()?, <[u8; 32]>::from(*spending_txid)))) .collect::, FinalisedStateError>>()?; encoded.sort_by(|a, b| a.0.cmp(&b.0)); @@ -87,6 +90,92 @@ pub(super) fn collate(records: &[SpendRecord]) -> Result, Fina Ok(encoded) } +// ── Store ──────────────────────────────────────────────────────────────────── + +/// LMDB database name for the spend index (versioned, mirroring the +/// finalised-state naming convention). +const SPEND_DB_NAME: &str = "outp_to_spend_index_1_0_0"; + +/// The spend index's **own** LMDB store: a single database keyed by the encoded +/// outpoint, valued by the bare 32-byte spending txid. Standalone — its own +/// `Environment`, independent of the finalised-state monolith's database. +pub(super) struct SpendIndexDb { + env: Environment, + spend: Database, +} + +impl SpendIndexDb { + /// Opens (creating if absent) the spend index at `path` with the given LMDB + /// `map_size`. Mirrors the finalised-state environment: **no `WRITE_MAP`** + /// (a crash discards uncommitted data rather than corrupting), with + /// reader-friendly flags. `path` is a directory, created if missing. + pub(super) fn open(path: &Path, map_size: usize) -> Result { + std::fs::create_dir_all(path).map_err(|error| { + FinalisedStateError::Custom(format!( + "spend index: create dir {}: {error}", + path.display() + )) + })?; + let env = Environment::new() + .set_max_dbs(1) + .set_map_size(map_size) + .set_flags(EnvironmentFlags::NO_TLS | EnvironmentFlags::NO_READAHEAD) + .open(path)?; + let spend = env.create_db(Some(SPEND_DB_NAME), DatabaseFlags::empty())?; + Ok(Self { env, spend }) + } + + /// Bulk-loads collated (sorted, disjoint-key) entries with `MDB_APPEND` — a + /// sequential B-tree fill — in one transaction, then forces durability. + /// `entries` must be in ascending key order (see [`collate`]); a key out of + /// order or already present makes LMDB reject the append. + pub(super) fn bulk_load(&self, entries: &[EncodedSpend]) -> Result<(), FinalisedStateError> { + // One-shot / global-order contract: `entries` must be strictly ascending + // by key — `collate`'s output for *all* spends at once. `MDB_APPEND` + // requires every key to exceed the current maximum, so there is no + // re-sort across calls: a second `bulk_load` would need every key to + // exceed the first call's maximum. LMDB self-checks this at runtime (it + // rejects a non-increasing key with `KeyExist`); this assert fails earlier + // and with a clearer message in debug builds if a caller hands over an + // unsorted, duplicate, or per-batch (not globally sorted) input. + debug_assert!( + entries.windows(2).all(|pair| pair[0].0 < pair[1].0), + "bulk_load: entries must be strictly ascending, globally-sorted keys \ + (MDB_APPEND is one-shot — feed all spends through a single collate)", + ); + let mut txn = self.env.begin_rw_txn()?; + for (key, value) in entries { + txn.put(self.spend, key, value, WriteFlags::APPEND)?; + } + txn.commit()?; + self.env.sync(true)?; + Ok(()) + } + + /// The txid that consumed `outpoint`, or `None` if it is unspent in + /// finalised state. A single point lookup; the caller unions this with the + /// non-finalised window at serve time (the deferred seam union). + pub(super) fn spending_txid( + &self, + outpoint: &Outpoint, + ) -> Result, FinalisedStateError> { + let key = outpoint.to_bytes()?; + let ro = self.env.begin_ro_txn()?; + match ro.get(self.spend, &key) { + Ok(bytes) => { + let array: [u8; 32] = bytes.try_into().map_err(|_| { + FinalisedStateError::Custom( + "spend index: stored value is not a 32-byte txid".to_string(), + ) + })?; + Ok(Some(TransactionHash::from(array))) + } + Err(lmdb::Error::NotFound) => Ok(None), + Err(error) => Err(error.into()), + } + } +} + #[cfg(test)] mod spends_in_transaction { use super::*; @@ -166,13 +255,37 @@ mod collate { "keys must be strictly ascending for MDB_APPEND", ); // The trailing LE u32 of each key shows index 256 precedes index 1. - assert_eq!( - &encoded[0].0[encoded[0].0.len() - 4..], - &256u32.to_le_bytes()[..] - ); - assert_eq!( - &encoded[1].0[encoded[1].0.len() - 4..], - &1u32.to_le_bytes()[..] - ); + assert_eq!(&encoded[0].0[encoded[0].0.len() - 4..], &256u32.to_le_bytes()[..]); + assert_eq!(&encoded[1].0[encoded[1].0.len() - 4..], &1u32.to_le_bytes()[..]); + } +} + +#[cfg(test)] +mod spend_index_db { + use super::*; + + #[test] + fn bulk_load_then_point_read_round_trips() { + let dir = + std::env::temp_dir().join(format!("outp_to_spend_index_roundtrip_{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + + let db = SpendIndexDb::open(&dir, 1 << 20).expect("open spend index"); + + let spent_a = Outpoint::new([1u8; 32], 0); + let spent_b = Outpoint::new([2u8; 32], 7); + let spender = TransactionHash::from([9u8; 32]); + + let collated = + super::collate(&[(spent_a, spender), (spent_b, spender)]).expect("collate"); + db.bulk_load(&collated).expect("bulk load"); + + assert_eq!(db.spending_txid(&spent_a).expect("read a"), Some(spender)); + assert_eq!(db.spending_txid(&spent_b).expect("read b"), Some(spender)); + // An outpoint with no recorded spend reads back as unspent. + let unspent = Outpoint::new([3u8; 32], 0); + assert_eq!(db.spending_txid(&unspent).expect("read unspent"), None); + + std::fs::remove_dir_all(&dir).ok(); } } From f7694bcba1539844bdf5a8c604b4d8b4faa0029e Mon Sep 17 00:00:00 2001 From: zancas Date: Mon, 29 Jun 2026 22:41:14 -0700 Subject: [PATCH 06/19] Add StateService-only spend-index sync loop StateSource: a compile-time StateService-only BlockchainSource wrapping the zebra State connector; its forwarders are generated by a declarative macro that delegates to the tested ValidatorConnector::State dispatch (one construction site). SpendIndexSource marker admits StateSource + MockchainSource but not the zcashd/JSON-RPC Fetch path, so 'never FetchService' is a compile-time fact. SpendIndexSync is a move-only, single-run handle whose run(self) streams [start_height, finalised_tip] from the source, extracts spends, collates once, and MDB_APPEND-loads the index. start_height is configurable (genesis for a full index, or e.g. a network-upgrade activation). from_state is the production constructor and rejects the Fetch variant. Co-Authored-By: Claude Opus 4.8 --- .../finalised_state/outp_to_spend_index.rs | 117 +++++++++++++++++- .../chain_index/source/validator_connector.rs | 57 +++++++++ 2 files changed, 173 insertions(+), 1 deletion(-) diff --git a/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs b/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs index a5b8a3657..9dd2e7b6c 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs @@ -14,9 +14,15 @@ use std::path::Path; use lmdb::{Database, DatabaseFlags, Environment, EnvironmentFlags, Transaction as _, WriteFlags}; +use crate::chain_index::finalised_state::build_indexed_block_from_source; +use crate::chain_index::source::validator_connector::{StateSource, ValidatorConnector}; +use crate::chain_index::source::BlockchainSource; use crate::chain_index::types::TransactionHash; +use crate::chain_index::NON_FINALIZED_DEPTH; use crate::error::FinalisedStateError; -use crate::{IndexedBlock, Outpoint, TransparentCompactTx, ZainoVersionedSerde as _}; +use crate::{ChainWork, IndexedBlock, Outpoint, TransparentCompactTx, ZainoVersionedSerde as _}; +use zaino_common::Network; +use zebra_chain::parameters::NetworkUpgrade; /// One transparent spend: the consumed outpoint paired with the txid of the /// transaction that consumed it. @@ -176,6 +182,115 @@ impl SpendIndexDb { } } +// ── Sync loop ──────────────────────────────────────────────────────────────── + +/// The sources the spend-index POC may build from: zebra's StateService +/// ([`StateSource`]) and the test mockchain — **never** the JSON-RPC/zcashd +/// `FetchService`. Binding [`SpendIndexSync`] to `S: SpendIndexSource` makes +/// "never FetchService" a compile-time fact: the `Fetch` path does not +/// implement this trait. +pub(super) trait SpendIndexSource: BlockchainSource {} + +impl SpendIndexSource for StateSource {} + +#[cfg(test)] +impl SpendIndexSource for crate::chain_index::source::mockchain_source::MockchainSource {} + +/// The independent, single-run builder for the spend index. +/// +/// Move-only (`!Clone`) with a private constructor and a `self`-consuming +/// [`run`](Self::run): the type system makes a second concurrent build +/// unrepresentable — "only one loop at a time" without a runtime guard. POC +/// build model: re-stream `start_height` → finalised tip from `source`, extract +/// every spend, collate once, and bulk-load the index in a single `MDB_APPEND` +/// pass. +pub(super) struct SpendIndexSync { + source: S, + db: SpendIndexDb, + network: Network, + /// First height to index, inclusive. Genesis (`0`) for a full index, or a + /// higher height — e.g. a network-upgrade activation — to cover only spends + /// occurring from that epoch onward. + start_height: u32, +} + +impl SpendIndexSync { + /// Private mint: production enters via [`SpendIndexSync::from_state`] + /// (StateService only); tests construct directly with a `MockchainSource`. + fn new(source: S, db: SpendIndexDb, network: Network, start_height: u32) -> Self { + Self { + source, + db, + network, + start_height, + } + } + + /// Builds the spend index from its start height to the finalised tip, consuming the + /// handle. One-shot (POC): no resume, no batching — a single global + /// `collate` + `MDB_APPEND`. + pub(super) async fn run(self) -> Result<(), FinalisedStateError> { + let Some(best) = self.source.get_best_block_height().await.map_err(|error| { + FinalisedStateError::Custom(format!("spend index: fetch best height: {error:?}")) + })? + else { + return Ok(()); // empty chain — nothing finalised to index + }; + + // Finalised tip: below the reorg-possible window, clamped at genesis. + let finalised_tip = best.0.saturating_sub(NON_FINALIZED_DEPTH); + + let zebra_network = self.network.to_zebra_network(); + let sapling_activation = NetworkUpgrade::Sapling + .activation_height(&zebra_network) + .expect("Sapling activation height is set for every network"); + let nu5_activation = NetworkUpgrade::Nu5.activation_height(&zebra_network); + + // Stream finalised blocks, extracting spends and dropping each block; + // only the (smaller) spend records are retained for the one-shot sort. + // Chainwork is irrelevant to spend extraction, so a zero parent is fine. + let mut spends: Vec = Vec::new(); + for height in self.start_height..=finalised_tip { + let block = build_indexed_block_from_source( + &self.source, + self.network, + sapling_activation, + nu5_activation, + height, + ChainWork::from_u256(0.into()), + ) + .await?; + spends.extend(extract_spends(std::slice::from_ref(&block))); + } + + let collated = collate(&spends)?; + tokio::task::block_in_place(|| self.db.bulk_load(&collated))?; + Ok(()) + } +} + +impl SpendIndexSync { + /// Production constructor — **StateService only**. The `Fetch` + /// (JSON-RPC/zcashd) backend is rejected here, the one place the concrete + /// `ValidatorConnector` variant is visible; the `S: SpendIndexSource` bound + /// already keeps a `Fetch` source from compiling elsewhere. + pub(super) fn from_state( + connector: ValidatorConnector, + db: SpendIndexDb, + network: Network, + start_height: u32, + ) -> Result { + match connector { + ValidatorConnector::State(state) => { + Ok(Self::new(StateSource(state), db, network, start_height)) + } + ValidatorConnector::Fetch(_) => Err(FinalisedStateError::Custom( + "spend-index POC requires the StateService backend, not FetchService".to_string(), + )), + } + } +} + #[cfg(test)] mod spends_in_transaction { use super::*; diff --git a/packages/zaino-state/src/chain_index/source/validator_connector.rs b/packages/zaino-state/src/chain_index/source/validator_connector.rs index 8560853e1..55fecabdb 100644 --- a/packages/zaino-state/src/chain_index/source/validator_connector.rs +++ b/packages/zaino-state/src/chain_index/source/validator_connector.rs @@ -1077,3 +1077,60 @@ impl BlockchainSource for ValidatorConnector { } } } + +/// A compile-time **StateService-only** view of a validator source. +/// +/// Wraps the zebra [`State`] connector — never the JSON-RPC `Fetch` backend — so +/// a source can be *statically* known to be backed by zebra's `ReadStateService`. +/// The finalised spend-index POC builds only from this (or a test mockchain), +/// never `FetchService`; binding the loop to a `SpendIndexSource` that excludes +/// the `Fetch` path makes that a compile-time guarantee. +/// +/// It reuses the tested [`ValidatorConnector::State`] dispatch by wrapping a +/// cheap `State` clone per call (that dispatch already clones the service per +/// call). Gated with the `outp_to_spend_index` feature. +#[cfg(feature = "outp_to_spend_index")] +#[derive(Clone, Debug)] +pub(crate) struct StateSource(pub(crate) State); + +/// Generates [`StateSource`]'s `BlockchainSource` impl: one forwarder per listed +/// method, each delegating to the tested `ValidatorConnector::State` dispatch. +/// +/// A declarative macro rather than a `fn` because the forwarders span N distinct +/// `async` signatures — a pattern a function cannot capture (per the repo's +/// "macros only where `fn` cannot express it" rule). The macro emits the whole +/// `#[async_trait] impl` so `async_trait` transforms concrete `async fn`s after +/// expansion, and the `ValidatorConnector::State(self.0.clone())` construction +/// appears exactly once — here. +#[cfg(feature = "outp_to_spend_index")] +macro_rules! impl_state_source_forwarders { + ( $( fn $method:ident ( $( $arg:ident : $arg_ty:ty ),* ) -> $ret:ty; )+ ) => { + #[async_trait] + impl BlockchainSource for StateSource { + $( + async fn $method(&self, $( $arg : $arg_ty ),*) -> $ret { + ValidatorConnector::State(self.0.clone()) + .$method($( $arg ),*) + .await + } + )+ + } + }; +} + +#[cfg(feature = "outp_to_spend_index")] +impl_state_source_forwarders! { + fn get_block(id: HashOrHeight) -> BlockchainSourceResult>>; + fn get_transaction(txid: TransactionHash) -> BlockchainSourceResult, GetTransactionLocation)>>; + fn get_mempool_txids() -> BlockchainSourceResult>>; + fn get_best_block_hash() -> BlockchainSourceResult>; + fn get_best_block_height() -> BlockchainSourceResult>; + fn get_treestate(id: BlockHash) -> BlockchainSourceResult<(Option>, Option>)>; + fn get_subtree_roots(pool: ShieldedPool, start_index: u16, max_entries: Option) -> BlockchainSourceResult>; + fn get_commitment_tree_roots(id: BlockHash) -> BlockchainSourceResult<(Option<(zebra_chain::sapling::tree::Root, u64)>, Option<(zebra_chain::orchard::tree::Root, u64)>)>; + fn get_address_deltas(params: GetAddressDeltasParams) -> BlockchainSourceResult; + fn get_address_balance(address_strings: GetAddressBalanceRequest) -> BlockchainSourceResult; + fn get_address_txids(request: GetAddressTxIdsRequest) -> BlockchainSourceResult>; + fn get_address_utxos(address_strings: GetAddressBalanceRequest) -> BlockchainSourceResult>; + fn nonfinalized_listener() -> Result)>>, Box>; +} From 01fba9c32f3988b025237e5ac5706f554cf7e6dc Mon Sep 17 00:00:00 2001 From: zancas Date: Mon, 29 Jun 2026 22:46:56 -0700 Subject: [PATCH 07/19] fmt --- .../finalised_state/outp_to_spend_index.rs | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs b/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs index 9dd2e7b6c..a9c8819e8 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs @@ -81,7 +81,9 @@ fn spends_in_transaction( pub(super) fn collate(records: &[SpendRecord]) -> Result, FinalisedStateError> { let mut encoded = records .iter() - .map(|(outpoint, spending_txid)| Ok((outpoint.to_bytes()?, <[u8; 32]>::from(*spending_txid)))) + .map(|(outpoint, spending_txid)| { + Ok((outpoint.to_bytes()?, <[u8; 32]>::from(*spending_txid))) + }) .collect::, FinalisedStateError>>()?; encoded.sort_by(|a, b| a.0.cmp(&b.0)); @@ -370,8 +372,14 @@ mod collate { "keys must be strictly ascending for MDB_APPEND", ); // The trailing LE u32 of each key shows index 256 precedes index 1. - assert_eq!(&encoded[0].0[encoded[0].0.len() - 4..], &256u32.to_le_bytes()[..]); - assert_eq!(&encoded[1].0[encoded[1].0.len() - 4..], &1u32.to_le_bytes()[..]); + assert_eq!( + &encoded[0].0[encoded[0].0.len() - 4..], + &256u32.to_le_bytes()[..] + ); + assert_eq!( + &encoded[1].0[encoded[1].0.len() - 4..], + &1u32.to_le_bytes()[..] + ); } } @@ -381,8 +389,10 @@ mod spend_index_db { #[test] fn bulk_load_then_point_read_round_trips() { - let dir = - std::env::temp_dir().join(format!("outp_to_spend_index_roundtrip_{}", std::process::id())); + let dir = std::env::temp_dir().join(format!( + "outp_to_spend_index_roundtrip_{}", + std::process::id() + )); let _ = std::fs::remove_dir_all(&dir); let db = SpendIndexDb::open(&dir, 1 << 20).expect("open spend index"); @@ -391,8 +401,7 @@ mod spend_index_db { let spent_b = Outpoint::new([2u8; 32], 7); let spender = TransactionHash::from([9u8; 32]); - let collated = - super::collate(&[(spent_a, spender), (spent_b, spender)]).expect("collate"); + let collated = super::collate(&[(spent_a, spender), (spent_b, spender)]).expect("collate"); db.bulk_load(&collated).expect("bulk load"); assert_eq!(db.spending_txid(&spent_a).expect("read a"), Some(spender)); From 12816ed3d9979d17087edb579e5138c6051eeb4e Mon Sep 17 00:00:00 2001 From: zancas Date: Mon, 29 Jun 2026 22:57:48 -0700 Subject: [PATCH 08/19] Test the spend-index sync loop against a mockchain source Drives SpendIndexSync::run over the 201-block regtest fixture via a MockchainSource and asserts run indexes exactly the finalised-range spends: every finalised spend mapped to its spender, every non-finalised spend absent, and an unspent outpoint reading back as None. The mockchain caps get_best_block_height at its max loaded height (200), so at the cfg(test) depth of 100 the finalised tip is 100; regtest coinbase maturity puts every fixture spend at height >=101, so this exercises the loop end to end in the exclusion direction (fetch -> finalised boundary -> empty collate -> LMDB -> read-back). Finalised-spend mapping with real records is covered by the spend_index_db round-trip; an end-to-end presence check needs a longer synthetic chain. multi_thread because run uses block_in_place for the LMDB write. Co-Authored-By: Claude Opus 4.8 --- .../finalised_state/outp_to_spend_index.rs | 96 +++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs b/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs index a9c8819e8..6d7d40ace 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs @@ -413,3 +413,99 @@ mod spend_index_db { std::fs::remove_dir_all(&dir).ok(); } } + +#[cfg(test)] +mod spend_index_sync { + use super::*; + use crate::chain_index::source::mockchain_source::MockchainSource; + use crate::chain_index::tests::vectors::{ + build_active_mockchain_source, indexed_block_chain, load_test_vectors, TestVectorBlockData, + }; + + fn fixture_blocks() -> Vec { + load_test_vectors().expect("load test vectors").blocks + } + + /// Builds the spend index from `mock` into a fresh temp dir, returning a + /// reader handle to what was persisted plus the dir to clean up. `run` + /// consumes its writer handle, so the db is reopened for reading. + /// + /// `multi_thread` is required of the callers: `run` uses + /// `tokio::task::block_in_place` for the LMDB write, which panics on a + /// current-thread runtime. The network only drives the block builder's + /// commitment-root activation checks (the mockchain supplies roots + /// regardless) and is irrelevant to transparent-spend extraction. + async fn build_index(mock: MockchainSource, tag: &str) -> (SpendIndexDb, std::path::PathBuf) { + let dir = + std::env::temp_dir().join(format!("outp_to_spend_index_{tag}_{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let db = SpendIndexDb::open(&dir, 1 << 24).expect("open spend index"); + SpendIndexSync::new(mock, db, Network::Mainnet, 0) + .run() + .await + .expect("spend-index build runs"); + ( + SpendIndexDb::open(&dir, 1 << 24).expect("reopen spend index"), + dir, + ) + } + + /// `run` must index *exactly* the transparent spends in the finalised range + /// `0..=finalised_tip`: every finalised spend mapped to its spender, every + /// non-finalised spend absent. + /// + /// Coverage note: the 201-block fixture caps `get_best_block_height` at 200 + /// (the mockchain reveals loaded blocks, it cannot mint new ones), so at the + /// `#[cfg(test)]` depth of 100 the finalised tip is pinned at 100. Regtest + /// coinbase maturity (100 blocks) puts every fixture spend at height ≥101, so + /// the finalised range holds no spends and this test exercises the exclusion + /// direction end to end (fetch → finalised boundary → empty collate → LMDB → + /// read-back). Finalised-spend *mapping* with real records is covered by the + /// `spend_index_db` round-trip test; an end-to-end presence check needs a + /// longer synthetic chain (tracked in zingolabs/zaino#1334). + #[tokio::test(flavor = "multi_thread")] + async fn run_indexes_exactly_the_finalised_spends() { + let blocks = fixture_blocks(); + let chain_top = blocks.last().expect("non-empty chain").height; + // No mining: the mock's tip is its max loaded height, so this matches + // `run`'s own `best - NON_FINALIZED_DEPTH`. + let finalised_tip = chain_top - NON_FINALIZED_DEPTH; + + let mock = build_active_mockchain_source(chain_top, blocks.clone()); + let (db, dir) = build_index(mock, "run").await; + + let mut saw_non_finalised_spend = false; + for block in indexed_block_chain(&blocks) { + let finalised = block.context.index.height.0 <= finalised_tip; + for (outpoint, spending_txid) in extract_spends(std::slice::from_ref(&block)) { + let indexed = db.spending_txid(&outpoint).expect("read spend"); + if finalised { + assert_eq!( + indexed, + Some(spending_txid), + "finalised spend {outpoint:?} must be indexed to its spender", + ); + } else { + saw_non_finalised_spend = true; + assert_eq!( + indexed, None, + "non-finalised spend {outpoint:?} must not be indexed", + ); + } + } + } + assert!( + saw_non_finalised_spend, + "fixture should contain spends above the seam to exercise exclusion", + ); + + // A never-spent outpoint reads back as unspent. + assert_eq!( + db.spending_txid(&Outpoint::new([0xff; 32], u32::MAX)) + .expect("read unspent"), + None, + ); + + std::fs::remove_dir_all(&dir).ok(); + } +} From 1d8672923166fd45ae96cbe984855630cc400f3a Mon Sep 17 00:00:00 2001 From: zancas Date: Tue, 30 Jun 2026 11:43:13 -0700 Subject: [PATCH 09/19] Wire the finalised spend-index build into the ChainIndex owner Adds a feature-gated BlockchainSource::finalised_spend_index_source bridge (default None; ValidatorConnector yields a StateSource for its State variant, None for Fetch) so the generic NodeBackedChainIndex can spawn the build without naming the concrete source. new_with_sync_timings spawns the one-shot build via outp_to_spend_index::spawn_build, which opens the index's own LMDB env (a sibling of the chain-index DB), builds from the Sapling activation height to the finalised tip, and logs its outcome. Replaces the redundant from_state constructor: the State/Fetch guard now lives in the trait override, and the never-Fetch guarantee still holds at the loop via SpendIndexSource. Co-Authored-By: Claude Opus 4.8 --- packages/zaino-state/src/chain_index.rs | 6 ++ .../src/chain_index/finalised_state.rs | 2 +- .../finalised_state/outp_to_spend_index.rs | 74 +++++++++++++------ .../zaino-state/src/chain_index/source.rs | 12 +++ .../chain_index/source/validator_connector.rs | 9 +++ 5 files changed, 79 insertions(+), 24 deletions(-) diff --git a/packages/zaino-state/src/chain_index.rs b/packages/zaino-state/src/chain_index.rs index 87e0d7f68..77f75272d 100644 --- a/packages/zaino-state/src/chain_index.rs +++ b/packages/zaino-state/src/chain_index.rs @@ -807,6 +807,12 @@ impl NodeBackedChainIndex { }; chain_index.sync_loop_handle = Some(chain_index.start_sync_loop()); + #[cfg(feature = "outp_to_spend_index")] + if let Some(spend_index_source) = chain_index.source.finalised_spend_index_source() { + let _build = + finalised_state::outp_to_spend_index::spawn_build(spend_index_source, &config); + } + Ok(chain_index) } diff --git a/packages/zaino-state/src/chain_index/finalised_state.rs b/packages/zaino-state/src/chain_index/finalised_state.rs index 473302b4f..28be49747 100644 --- a/packages/zaino-state/src/chain_index/finalised_state.rs +++ b/packages/zaino-state/src/chain_index/finalised_state.rs @@ -219,7 +219,7 @@ pub(crate) mod entry; pub(crate) mod finalised_source; pub(crate) mod migrations; #[cfg(feature = "outp_to_spend_index")] -mod outp_to_spend_index; +pub(crate) mod outp_to_spend_index; pub(crate) mod reader; pub(crate) mod router; diff --git a/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs b/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs index 6d7d40ace..97568a6e2 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs @@ -6,19 +6,23 @@ //! //! The build stages live here: **extract** spends from a block batch, //! **collate** them into LMDB key order, and bulk-load them into the index's -//! own LMDB store via `MDB_APPEND` ([`SpendIndexDb`]). The independent sync -//! loop that drives these (a genesis re-stream from zebra) lands in a later -//! slice. (Table-level integrity over the entries is deferred; it is not MVP.) +//! own LMDB store via `MDB_APPEND` ([`SpendIndexDb`]). The independent, +//! move-only sync loop ([`SpendIndexSync`]) drives these in one shot over +//! `[start_height, finalised_tip]` streamed from a [`SpendIndexSource`], and +//! [`spawn_build`] wires it onto the owning `ChainIndex` (StateService only, +//! from the Sapling activation height). (Table-level integrity over the entries +//! is deferred; it is not MVP.) use std::path::Path; use lmdb::{Database, DatabaseFlags, Environment, EnvironmentFlags, Transaction as _, WriteFlags}; use crate::chain_index::finalised_state::build_indexed_block_from_source; -use crate::chain_index::source::validator_connector::{StateSource, ValidatorConnector}; +use crate::chain_index::source::validator_connector::StateSource; use crate::chain_index::source::BlockchainSource; use crate::chain_index::types::TransactionHash; use crate::chain_index::NON_FINALIZED_DEPTH; +use crate::config::ChainIndexConfig; use crate::error::FinalisedStateError; use crate::{ChainWork, IndexedBlock, Outpoint, TransparentCompactTx, ZainoVersionedSerde as _}; use zaino_common::Network; @@ -271,28 +275,52 @@ impl SpendIndexSync { } } -impl SpendIndexSync { - /// Production constructor — **StateService only**. The `Fetch` - /// (JSON-RPC/zcashd) backend is rejected here, the one place the concrete - /// `ValidatorConnector` variant is visible; the `S: SpendIndexSource` bound - /// already keeps a `Fetch` source from compiling elsewhere. - pub(super) fn from_state( - connector: ValidatorConnector, - db: SpendIndexDb, - network: Network, - start_height: u32, - ) -> Result { - match connector { - ValidatorConnector::State(state) => { - Ok(Self::new(StateSource(state), db, network, start_height)) - } - ValidatorConnector::Fetch(_) => Err(FinalisedStateError::Custom( - "spend-index POC requires the StateService backend, not FetchService".to_string(), - )), - } +/// Map size for the spend index's own LMDB env. LMDB grows the file lazily (no +/// `WRITE_MAP` here), so this is an upper bound, not a preallocation. +const SPEND_INDEX_MAP_SIZE: usize = 32 * 1024 * 1024 * 1024; + +/// The spend index's on-disk location: a sibling of the configured chain-index +/// database directory. +fn spend_index_dir(cfg: &ChainIndexConfig) -> std::path::PathBuf { + let main = &cfg.storage.database.path; + match main.parent() { + Some(parent) => parent.join("outp_to_spend_index"), + None => main.join("outp_to_spend_index"), } } +/// Spawns the one-shot finalised spend-index build as a background task. +/// +/// The build streams `[Sapling activation, finalised tip]` from `source` — the +/// zebra StateService, supplied by +/// [`BlockchainSource::finalised_spend_index_source`] — into the index's own +/// LMDB env. The task logs its outcome. The caller currently detaches the +/// handle (POC); production should track it for shutdown/cancellation. +pub(crate) fn spawn_build( + source: StateSource, + cfg: &ChainIndexConfig, +) -> tokio::task::JoinHandle> { + let network = cfg.network.clone(); + let dir = spend_index_dir(cfg); + tokio::spawn(async move { + let start_height = NetworkUpgrade::Sapling + .activation_height(&network.to_zebra_network()) + .ok_or_else(|| { + FinalisedStateError::Custom("network has no Sapling activation height".to_string()) + })? + .0; + let db = SpendIndexDb::open(&dir, SPEND_INDEX_MAP_SIZE)?; + let result = SpendIndexSync::new(source, db, network, start_height) + .run() + .await; + match &result { + Ok(()) => tracing::info!("finalised spend-index build complete"), + Err(error) => tracing::error!("finalised spend-index build failed: {error}"), + } + result + }) +} + #[cfg(test)] mod spends_in_transaction { use super::*; diff --git a/packages/zaino-state/src/chain_index/source.rs b/packages/zaino-state/src/chain_index/source.rs index 73e3e13de..0aebf04dc 100644 --- a/packages/zaino-state/src/chain_index/source.rs +++ b/packages/zaino-state/src/chain_index/source.rs @@ -229,6 +229,18 @@ pub trait BlockchainSource: Clone + Send + Sync + 'static { fn subscribe_to_blocks_received(&self) -> Option> { None } + + /// The source from which a finalised spend index may be built, if this + /// backend supports one. Returns `None` for backends that must not build it + /// (the JSON-RPC `Fetch` path) and by default for those that don't; the + /// zebra StateService yields a [`StateSource`](validator_connector::StateSource). + // `StateSource` is intentionally `pub(crate)`; this is internal wiring on a + // `pub` trait, so exposing the more-private return type is expected. + #[cfg(feature = "outp_to_spend_index")] + #[allow(private_interfaces)] + fn finalised_spend_index_source(&self) -> Option { + None + } } /// Sleep up to `duration`, but return early if `change_rx` resolves first. diff --git a/packages/zaino-state/src/chain_index/source/validator_connector.rs b/packages/zaino-state/src/chain_index/source/validator_connector.rs index 55fecabdb..bf8333bac 100644 --- a/packages/zaino-state/src/chain_index/source/validator_connector.rs +++ b/packages/zaino-state/src/chain_index/source/validator_connector.rs @@ -1076,6 +1076,15 @@ impl BlockchainSource for ValidatorConnector { ValidatorConnector::Fetch(_fetch) => Ok(None), } } + + #[cfg(feature = "outp_to_spend_index")] + #[allow(private_interfaces)] + fn finalised_spend_index_source(&self) -> Option { + match self { + ValidatorConnector::State(state) => Some(StateSource(state.clone())), + ValidatorConnector::Fetch(_) => None, + } + } } /// A compile-time **StateService-only** view of a validator source. From 51c0d68c009c07f0ac16282981fbafb88c75dd1b Mon Sep 17 00:00:00 2001 From: zancas Date: Tue, 30 Jun 2026 11:51:32 -0700 Subject: [PATCH 10/19] Reconcile the spend index with dev's async_trait and ChainWork changes Dev dropped async_trait (#1314, native AFIT) and reworked ChainWork into a NonZeroU128 primitive passed as Option at call sites (#1313). Update the spend-index code to match: impl_state_source_forwarders! no longer emits #[async_trait] (its async fn forwarders satisfy BlockchainSource's native -> impl SendFut<_> methods), and SpendIndexSync::run passes None for build_indexed_block_from_source's now-Option parent_chainwork (chainwork is irrelevant to spend extraction), dropping the unused ChainWork import. Co-Authored-By: Claude Opus 4.8 --- .../chain_index/finalised_state/outp_to_spend_index.rs | 6 +++--- .../src/chain_index/source/validator_connector.rs | 9 ++++----- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs b/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs index 97568a6e2..71c883fb4 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs @@ -24,7 +24,7 @@ use crate::chain_index::types::TransactionHash; use crate::chain_index::NON_FINALIZED_DEPTH; use crate::config::ChainIndexConfig; use crate::error::FinalisedStateError; -use crate::{ChainWork, IndexedBlock, Outpoint, TransparentCompactTx, ZainoVersionedSerde as _}; +use crate::{IndexedBlock, Outpoint, TransparentCompactTx, ZainoVersionedSerde as _}; use zaino_common::Network; use zebra_chain::parameters::NetworkUpgrade; @@ -254,7 +254,7 @@ impl SpendIndexSync { // Stream finalised blocks, extracting spends and dropping each block; // only the (smaller) spend records are retained for the one-shot sort. - // Chainwork is irrelevant to spend extraction, so a zero parent is fine. + // Chainwork is irrelevant to spend extraction, so a `None` parent is fine. let mut spends: Vec = Vec::new(); for height in self.start_height..=finalised_tip { let block = build_indexed_block_from_source( @@ -263,7 +263,7 @@ impl SpendIndexSync { sapling_activation, nu5_activation, height, - ChainWork::from_u256(0.into()), + None, ) .await?; spends.extend(extract_spends(std::slice::from_ref(&block))); diff --git a/packages/zaino-state/src/chain_index/source/validator_connector.rs b/packages/zaino-state/src/chain_index/source/validator_connector.rs index bf8333bac..c2967899d 100644 --- a/packages/zaino-state/src/chain_index/source/validator_connector.rs +++ b/packages/zaino-state/src/chain_index/source/validator_connector.rs @@ -1107,14 +1107,13 @@ pub(crate) struct StateSource(pub(crate) State); /// /// A declarative macro rather than a `fn` because the forwarders span N distinct /// `async` signatures — a pattern a function cannot capture (per the repo's -/// "macros only where `fn` cannot express it" rule). The macro emits the whole -/// `#[async_trait] impl` so `async_trait` transforms concrete `async fn`s after -/// expansion, and the `ValidatorConnector::State(self.0.clone())` construction -/// appears exactly once — here. +/// "macros only where `fn` cannot express it" rule). Each generated `async fn` +/// satisfies the trait's native `-> impl SendFut<_>` method, and the +/// `ValidatorConnector::State(self.0.clone())` construction appears exactly +/// once — here. #[cfg(feature = "outp_to_spend_index")] macro_rules! impl_state_source_forwarders { ( $( fn $method:ident ( $( $arg:ident : $arg_ty:ty ),* ) -> $ret:ty; )+ ) => { - #[async_trait] impl BlockchainSource for StateSource { $( async fn $method(&self, $( $arg : $arg_ty ),*) -> $ret { From 9efed25c9d9e2825fb0e9fd8fa33d6311f1d7feb Mon Sep 17 00:00:00 2001 From: zancas Date: Tue, 30 Jun 2026 11:56:14 -0700 Subject: [PATCH 11/19] Track and abort the spend-index build with the ChainIndex MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Promotes the one-shot spend-index build from a detached task to a tracked one: NodeBackedChainIndex holds its JoinHandle (feature-gated) and aborts it in both shutdown() and Drop, so the build never outlives the index. Aborting is safe: run() accumulates spends in memory and writes the index in a single MDB_APPEND transaction at the very end, so an abort either leaves the in-memory work undone (nothing written) or lands after the atomic commit — never a partial index. Co-Authored-By: Claude Opus 4.8 --- packages/zaino-state/src/chain_index.rs | 22 ++++++++++++++++--- .../finalised_state/outp_to_spend_index.rs | 4 ++-- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/packages/zaino-state/src/chain_index.rs b/packages/zaino-state/src/chain_index.rs index 77f75272d..0730159be 100644 --- a/packages/zaino-state/src/chain_index.rs +++ b/packages/zaino-state/src/chain_index.rs @@ -703,6 +703,10 @@ pub struct NodeBackedChainIndex { non_finalized_state: Arc>>, finalized_db: std::sync::Arc>, sync_loop_handle: Option>>, + /// Handle to the one-shot finalised spend-index build, if one was spawned. + /// Aborted on shutdown/drop so the build never outlives the index. + #[cfg(feature = "outp_to_spend_index")] + spend_index_build: Option>>, status: NamedAtomicStatus, network: ZebraNetwork, source: Source, @@ -799,6 +803,8 @@ impl NodeBackedChainIndex { non_finalized_state: Arc::new(ArcSwapOption::empty()), finalized_db, sync_loop_handle: None, + #[cfg(feature = "outp_to_spend_index")] + spend_index_build: None, status: NamedAtomicStatus::new("ChainIndex", StatusType::Spawning), network: config.network.to_zebra_network(), source, @@ -808,9 +814,11 @@ impl NodeBackedChainIndex { chain_index.sync_loop_handle = Some(chain_index.start_sync_loop()); #[cfg(feature = "outp_to_spend_index")] - if let Some(spend_index_source) = chain_index.source.finalised_spend_index_source() { - let _build = - finalised_state::outp_to_spend_index::spawn_build(spend_index_source, &config); + { + chain_index.spend_index_build = chain_index + .source + .finalised_spend_index_source() + .map(|source| finalised_state::outp_to_spend_index::spawn_build(source, &config)); } Ok(chain_index) @@ -843,6 +851,10 @@ impl NodeBackedChainIndex { /// failure-path round trip. pub async fn shutdown(&self) -> Result<(), FinalisedStateError> { self.cancel_token.cancel(); + #[cfg(feature = "outp_to_spend_index")] + if let Some(handle) = &self.spend_index_build { + handle.abort(); + } self.status.store(StatusType::Closing); self.finalized_db.shutdown().await?; self.mempool.close(); @@ -1073,6 +1085,10 @@ impl Drop for NodeBackedChainIndex { /// exits at its next await checkpoint instead. fn drop(&mut self) { self.cancel_token.cancel(); + #[cfg(feature = "outp_to_spend_index")] + if let Some(handle) = &self.spend_index_build { + handle.abort(); + } } } diff --git a/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs b/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs index 71c883fb4..5c530cd54 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs @@ -294,8 +294,8 @@ fn spend_index_dir(cfg: &ChainIndexConfig) -> std::path::PathBuf { /// The build streams `[Sapling activation, finalised tip]` from `source` — the /// zebra StateService, supplied by /// [`BlockchainSource::finalised_spend_index_source`] — into the index's own -/// LMDB env. The task logs its outcome. The caller currently detaches the -/// handle (POC); production should track it for shutdown/cancellation. +/// LMDB env. The task logs its outcome; the owning `ChainIndex` holds the +/// returned handle and aborts it on shutdown/drop. pub(crate) fn spawn_build( source: StateSource, cfg: &ChainIndexConfig, From a20096f1d67f464e1c865ddbabbe9612a47cc987 Mon Sep 17 00:00:00 2001 From: zancas Date: Tue, 30 Jun 2026 12:17:32 -0700 Subject: [PATCH 12/19] Add an end-to-end spend-index presence test on a synthetic chain The 201-block fixture can't exercise a non-empty index from run (coinbase maturity puts every fixture spend above the depth-100 seam). This synthesises a chain with zebra's block generator (allow_all_transparent_coinbase_spends lets transparent spends land below the seam), deterministically searches via a seeded TestRunner for a chain whose finalised range holds a spend, runs the index, and asserts every finalised spend is mapped to its spender. Addresses the presence half (Part 1) of zingolabs/zaino#1334. Co-Authored-By: Claude Opus 4.8 --- .../finalised_state/outp_to_spend_index.rs | 148 ++++++++++++++++++ 1 file changed, 148 insertions(+) diff --git a/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs b/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs index 5c530cd54..2b413a755 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs @@ -537,3 +537,151 @@ mod spend_index_sync { std::fs::remove_dir_all(&dir).ok(); } } + +/// End-to-end **presence** test: `run` must build a *non-empty* index when the +/// finalised range contains transparent spends. The 201-block fixture can't +/// exercise this (coinbase maturity puts its spends above the depth-100 seam), +/// so this synthesises a chain with zebra's block generator, which — with +/// `allow_all_transparent_coinbase_spends` — produces transparent spends that +/// can land below the seam. Tracks zingolabs/zaino#1334. +#[cfg(test)] +mod spend_index_presence { + use super::*; + use crate::chain_index::source::mockchain_source::MockchainSource; + use proptest::prelude::{Arbitrary, Strategy}; + use proptest::strategy::ValueTree; + use proptest::test_runner::TestRunner; + use std::sync::Arc; + use zebra_chain::block::arbitrary::{ + allow_all_transparent_coinbase_spends, LedgerStateOverride, + }; + use zebra_chain::block::{Block, Height}; + use zebra_chain::parameters::{Network as ZebraNetwork, GENESIS_PREVIOUS_BLOCK_HASH}; + use zebra_chain::transparent::Input; + use zebra_chain::LedgerState; + + // Long enough that the depth-100 seam leaves room for finalised spends. + const CHAIN_LEN: usize = 200; + // Generate-until-found budget — a chain with a finalised transparent spend + // is overwhelmingly likely within a couple of tries. + const MAX_GEN_ATTEMPTS: usize = 40; + + fn build_mock(blocks: Vec>) -> MockchainSource { + let hashes = blocks + .iter() + .map(|block| crate::BlockHash::from(block.hash())) + .collect(); + // Synthetic blocks carry no commitment roots/treestates; with a + // non-regtest network the block builder falls back to default roots, and + // spends are root-independent anyway. + let roots = vec![(None, None); blocks.len()]; + let treestates = vec![(Vec::new(), Vec::new()); blocks.len()]; + MockchainSource::new(blocks, roots, treestates, hashes) + } + + /// True if a non-coinbase transparent input (a spend) occurs at or below + /// `finalised_tip`. Block index equals height for a genesis-rooted chain. + fn has_finalised_spend(blocks: &[Arc], finalised_tip: u32) -> bool { + blocks.iter().take(finalised_tip as usize + 1).any(|block| { + block.transactions.iter().any(|tx| { + tx.inputs() + .iter() + .any(|i| matches!(i, Input::PrevOut { .. })) + }) + }) + } + + #[tokio::test(flavor = "multi_thread")] + async fn run_builds_a_nonempty_index_from_a_synthetic_chain() { + let strategy = LedgerState::arbitrary_with(LedgerStateOverride { + height_override: Some(Height(0)), + previous_block_hash_override: Some(GENESIS_PREVIOUS_BLOCK_HASH), + network_upgrade_override: None, + transaction_version_override: None, + transaction_has_valid_network_upgrade: true, + always_has_coinbase: true, + network_override: Some(ZebraNetwork::Mainnet), + }) + .prop_flat_map(|ledger| { + Block::partial_chain_strategy( + ledger, + CHAIN_LEN, + allow_all_transparent_coinbase_spends, + true, + ) + }); + let mut runner = TestRunner::deterministic(); + + // Deterministically search for a generated chain with a finalised spend, + // so the presence direction is genuinely exercised. + let mut found = None; + for _ in 0..MAX_GEN_ATTEMPTS { + let blocks = strategy + .new_tree(&mut runner) + .expect("generate chain") + .current() + .0; + let finalised_tip = (blocks.len() as u32 - 1).saturating_sub(NON_FINALIZED_DEPTH); + if has_finalised_spend(&blocks, finalised_tip) { + found = Some((blocks, finalised_tip)); + break; + } + } + let (blocks, finalised_tip) = + found.expect("a synthetic chain with a finalised transparent spend within budget"); + + let mock = build_mock(blocks); + + // Reference: the spends in the finalised range, built the way `run` + // builds blocks (so this checks fetch + boundary + collate + LMDB + // round-trip; the extractor is unit-tested above). + let zebra_network = Network::Mainnet.to_zebra_network(); + let sapling = NetworkUpgrade::Sapling + .activation_height(&zebra_network) + .expect("Sapling activation height"); + let nu5 = NetworkUpgrade::Nu5.activation_height(&zebra_network); + let mut finalised_blocks = Vec::new(); + for height in 0..=finalised_tip { + finalised_blocks.push( + build_indexed_block_from_source( + &mock, + Network::Mainnet, + sapling, + nu5, + height, + None, + ) + .await + .expect("build finalised block"), + ); + } + let expected = extract_spends(&finalised_blocks); + assert!( + !expected.is_empty(), + "the chosen chain must contain finalised transparent spends", + ); + + // Build the index from genesis and read it back. + let dir = std::env::temp_dir().join(format!( + "outp_to_spend_index_synthetic_{}", + std::process::id() + )); + let _ = std::fs::remove_dir_all(&dir); + let db = SpendIndexDb::open(&dir, 1 << 24).expect("open spend index"); + SpendIndexSync::new(mock, db, Network::Mainnet, 0) + .run() + .await + .expect("spend-index build runs"); + let db = SpendIndexDb::open(&dir, 1 << 24).expect("reopen spend index"); + + for (outpoint, spending_txid) in &expected { + assert_eq!( + db.spending_txid(outpoint).expect("read finalised spend"), + Some(*spending_txid), + "finalised spend {outpoint:?} must be indexed to its spender", + ); + } + + std::fs::remove_dir_all(&dir).ok(); + } +} From 4e617687246319d711af660dbbb6e3109d6037aa Mon Sep 17 00:00:00 2001 From: zancas Date: Wed, 1 Jul 2026 23:06:57 -0700 Subject: [PATCH 13/19] Record ADR 0006 for the finalised spend index; add glossary terms Land the decision record #1328 promised before implementation (the 0002 slot the issue reserved was taken by the live-tests ADR). Beyond the issue's content, the record captures what has crystallised since: - PR #1167's get_outpoint_spenders already performs the serve-time union with the non-finalised state (FullChain scope), so the remaining serving step is swapping its finalised leg from the monolith's spent table (outpoint -> TxLocation -> txid) to this index; get_outpoint_spenders(Finalised) doubles as an in-tree parity oracle alongside zebra's SpendingTransactionId. - The start height is a first-class index floor: absence means "unspent within the built range". Serving deployments must run a genesis floor, enforced by config validation; the shipped config defaults to genesis with the Sapling-activation floor present only as a commented-out option. - The one-shot pre-materialized MDB_APPEND pass is the deliberate speed-of-light baseline for the sync benchmark (~30 GB peak at genesis floor); a batched k-way-merge build is a measurement-gated contingency, not a foregone production requirement. - Acceptance bar: one mainnet run producing the benchmark numbers, a parity diff against zebra, and golden vectors captured before zebra drops its indexer feature; a synthetic-chain parity test runs first to prove the oracle plumbing. The chain-index glossary gains "index floor" (distinct from the finalized floor) and "spending transaction" (the term #1328 defined and promised to this file). Co-Authored-By: Claude Fable 5 --- ...06-finalised-spend-index-parallel-build.md | 204 ++++++++++++++++++ .../zaino-state/src/chain_index/CONTEXT.md | 14 ++ 2 files changed, 218 insertions(+) create mode 100644 docs/adr/0006-finalised-spend-index-parallel-build.md diff --git a/docs/adr/0006-finalised-spend-index-parallel-build.md b/docs/adr/0006-finalised-spend-index-parallel-build.md new file mode 100644 index 000000000..e3dc35a99 --- /dev/null +++ b/docs/adr/0006-finalised-spend-index-parallel-build.md @@ -0,0 +1,204 @@ +# A standalone, parallel-buildable finalised spend index + +## Status + +accepted (pilot; refines the umbrella schema discussion in issue #1326, +scoped by issue #1328, implemented in PR #1330) + +Note: issues #1328/#1326 reserved the path +`docs/adr/0002-finalised-spend-index-parallel-build.md` for this record; +that number was taken by the live-tests ADR before this file landed, so +the record lives here as 0006. + +## Context + +Serving "which transaction spent this transparent outpoint?" requires an +index mapping `transparent_outpoint → spending_txid`. zebra already +answers this via `ReadRequest::SpendingTransactionId`, but only behind +its non-default `indexer` feature, and that service is slated for +removal once zaino provides the index — so zaino becomes the sole +provider and the index becomes required, not merely offered. zebra's +answer is therefore a temporary correctness oracle: zaino's index is +diffed against it, and golden vectors must be captured before the +feature is removed. + +On the serving side, zaino already answers the query: +`ChainIndex::get_outpoint_spenders` (PR #1167) takes a batch of +outpoints and a `ChainScope` — `FullChain` scans the non-finalised best +chain first, then falls back to the finalised state; `Finalised` is the +reorg-stable subset. Its finalised leg reads the monolith's `spent` +table (`outpoint → TxLocation`) and resolves each deduped `TxLocation` +to a txid through the transaction-location tables. That method is the +consumer this index ultimately backs. + +Independently, the finalised-state build path is a monolith: one +`write_block_with_options` writes all tables per block in a single +transaction, so no index can be built, rebuilt, or optimised in +isolation. The spend index is the simplest useful pilot for breaking +that coupling: its keys are globally disjoint (an outpoint is spent at +most once chain-wide), so per-batch results combine by pure sorted merge +with zero cross-batch reconciliation — unlike value- or script-bearing +indexes, which need a shared live-UTXO primary. + +Terminology (see `packages/zaino-state/src/chain_index/CONTEXT.md`): an +outpoint's first field already names its **creating** transaction; the +transaction that lists the outpoint as an input's prevout is its +**spending** transaction. This index maps to the spending txid only. + +## Decision + +Build the spend index in zaino as a standalone pilot, gated behind the +default-off Cargo feature `outp_to_spend_index` (a member of +`experimental_features`), decomposed into three roles so that +read-freedom is scoped to exactly one of them: + +- **Block source** (impure): a `BlockchainSource` producing + `IndexedBlock`s. Bound at compile time to zebra's StateService or the + test mockchain via the `SpendIndexSource` trait — the JSON-RPC/zcashd + `FetchService` does not implement it, so "never FetchService" is a + compile-time fact. +- **Extractor** (statically read-free): a free function handed only + `&[IndexedBlock]` — no `&self`, no database, no validator handle in + scope — so a previous-output lookup is unrepresentable, not merely + discouraged. Each transparent input yields + `(prevout_outpoint, containing_txid)`. Null-prevout inputs (only the + coinbase input) are skipped; spends *of* coinbase outputs are ordinary + spends and are indexed. +- **Collator/store** (impure, single-writer): encode, byte-sort, reject + duplicate keys as corrupt input, and bulk-load with `MDB_APPEND` into + the index's **own** LMDB environment (database + `outp_to_spend_index_1_0_0`, no `WRITE_MAP`, sited as a sibling + directory of the chain-index database), a sequential B-tree fill. + +Schema: key = the 36-byte encoded outpoint (`txid[32] ‖ LE(u32) index`) +because that is what the query hands us; value = the bare 32-byte +spending txid; absence ⇒ unspent **within the index's built range** +`[index floor, finalised tip]` — below its floor the index asserts +nothing. The value is +deliberately **not** a `TxLocation`: that would reintroduce a dependency +on the `txids`/`txid_location` tables and break the index's isolation. + +Build model (pilot): a clean sync that re-streams blocks from the +source over `[start_height, finalised_tip]`, where +`finalised_tip = best_height − non-finalized depth` — feeding the +extractor only finalised blocks makes the build reorg-immune by +construction. No resume watermark, no backfill from on-disk block +tables, no migration; a crash reruns from the start height. + +The start height — the **index floor** — is a first-class, configurable +property of the index that survives into production, not pilot +scaffolding: genesis buys full coverage; a later floor (e.g. a +network-upgrade activation) buys a cheaper build at the cost of +range-scoped answers. Serving discipline: a deployment that serves +spend queries must run a genesis floor, enforced by config validation, +so a floor-truncated answer never reaches a client; non-genesis floors +are for deployments that build without serving (pilots, tests, +experiments). The shipped configuration defaults to genesis and carries +the Sapling-activation floor only as a commented-out option in the +config file. Because absence is only meaningful relative to the built +range, a serving layer must know the index's floor, so the floor must +ultimately be persisted with the index (deferred; the pilot hardcodes +it). The pilot enters through one `spawn_build` call with the floor +pinned to Sapling activation. + +Single-loop enforcement is pushed as far into the type system as it +goes: the sync handle is move-only (`!Clone`) with a private constructor +and a `self`-consuming `run`, so duplicating the handle or running two +loops concurrently is rejected by the move checker. The residual — the +owner minting two handles — is closed by the owning `ChainIndex` making +exactly one `spawn_build` call, deliberately not by a runtime guard +(`OnceLock`/atomic). The owner holds the returned `JoinHandle` and +aborts the build on shutdown/drop. + +## Considered options + +- **Keep serving from the monolith's `spent` (`outpoint → TxLocation`) + table alone** — what `get_outpoint_spenders`' finalised leg does + today. Rejected as the end state: that table is written only by the + monolithic per-block write path, so it cannot be built, rebuilt, or + optimised in isolation, and its `TxLocation` value forces a second + resolution hop through the transaction-location tables — exactly the + coupling the bare-txid value avoids. It remains the serving path + until this index replaces the finalised leg. +- **Unconditional genesis start (no index floor).** Rejected: it would + make "absence ⇒ unspent" hold outright, but forces every deployment + to carry the full deep history whether or not it serves it. The + configurable floor keeps the build/storage cost proportional to what + a deployment actually answers for; the cost — absence is range-scoped + — is carried explicitly by the serving layer. +- **Serving with a non-genesis floor** — either falling back to the + monolith's `spent` table below the floor, or letting + `get_outpoint_spenders`' "`None` = unspent or unknown" contract + absorb the weaker answer. Both rejected: the fallback keeps the + monolith coupling alive indefinitely, and contract absorption + silently serves worse answers than the monolith's `spent` table does + today. Config validation (serving ⇒ genesis floor) prevents both. +- **Proxy zebra's `indexer` service instead of building.** Rejected: + ties a served capability to a non-default zebra feature that is + planned for removal, and forfeits both serving scale and the pilot + value for the parallel-sync architecture. +- **Value = `TxLocation` instead of bare txid.** Rejected: couples the + new index to the monolith's transaction-location tables, defeating + the isolation this pilot exists to prove. +- **Runtime single-instance guard.** Rejected in favour of the + move-only consuming handle plus a single constructor call site; + a `OnceLock`/atomic guard would paper over ownership the type system + can express. +- **Backfill from zaino's on-disk block tables.** Dropped: re-streaming + from the validator keeps the build independent of every other table + and is the model production resume will refine, not replace. +- **Immutable sorted segment (SSTable-like flat file) as the store.** + Deferred to the v2 discussion in #1326: a better long-term fit for an + append-only finalised index than LMDB's update-in-place B-tree, but + the pilot reuses the engine already in the tree. + +## Consequences + +- ≈162M entries at mainnet height ≈3.4M ⇒ roughly 13–15 GB on disk, + growing linearly with chain history; the pilot's LMDB map size is a + 32 GB lazy upper bound, not a preallocation. +- The build holds all extracted spends in memory for one global sort + and a single `MDB_APPEND` transaction; `bulk_load` is one-shot (a + second call would need every key to exceed the first call's maximum). + This is deliberate, not a pilot shortcut: the pre-materialized, + globally sorted single append pass is the speed-of-light baseline + for the sync benchmark, and at genesis floor on mainnet it peaks + around 30 GB — within a reasonably resourced build machine. A + batched variant (per-worker sorted runs, k-way merge feeding the + appender — licensed by the disjoint-key property) is a contingency + adopted only if measurement shows it beating the one-shot pass or + memory genuinely binding, not a foregone production requirement. +- The serve-time union with the non-finalised state already exists as + `get_outpoint_spenders`' `FullChain` scope (PR #1167); the pilot index + is built but not yet served. The deferred serving step is swapping + that method's finalised leg from the monolith's `spent` table to this + index — dropping the `TxLocation → txid` resolution hop — with no + change to the method's contract (it already returns bare txids). + Queried directly, the pilot index alone returns `None` for an + outpoint spent within the non-finalized depth, and for anything + spent below its floor (pilot: Sapling activation). +- With a floor above genesis, `None` from the index means "unspent or + spent below the floor". Config validation keeps that answer away from + clients — serving requires a genesis floor — so the swap needs no + fallback to the monolith's `spent` table and the isolation goal + stands. The floor must still be persisted in the index's environment + so builders, restarts, and servers agree on the built range; it is + the value the serving-side validation checks. +- `get_outpoint_spenders(Finalised)` over the monolith's `spent` table + is a second, in-tree parity oracle for this index, alongside zebra's + `SpendingTransactionId` — same chain, same process, no non-default + zebra feature required. +- Table-level integrity (the order-independent XOR-of-BLAKE2b + commitment over entries sketched in #1328) is deferred; the pilot + index has no checksum and relies on being deterministically + rebuildable. +- The pilot's acceptance bar is a mainnet run of the one-shot build on + a well-resourced machine, producing three artifacts together: the + sync-time benchmark numbers (the pilot's reason to exist), a parity + diff against zebra's `SpendingTransactionId`, and golden vectors — + which must in any case be captured while zebra still ships the + `indexer` feature. A synthetic-chain parity test runs first, to prove + the oracle plumbing before spending a mainnet stream on it. +- Generalising the extractor/collator decomposition to value- and + script-bearing indexes (txout-set accumulator, address history) — + which need a shared live-UTXO primary — remains with #1326. diff --git a/packages/zaino-state/src/chain_index/CONTEXT.md b/packages/zaino-state/src/chain_index/CONTEXT.md index 37634b70a..17544b2e3 100644 --- a/packages/zaino-state/src/chain_index/CONTEXT.md +++ b/packages/zaino-state/src/chain_index/CONTEXT.md @@ -32,3 +32,17 @@ blocks. ## Eviction Removal of a block from the NFS once the finalized floor rises past its height. A block is evicted when it passes below the seam. + +## Index floor +The lowest height a given finalized index covers, chosen per index at +build time. The index asserts nothing about chain activity below its +floor: an absent key means "no such entry within the built range", not +"none on the whole chain". Distinct from the finalized floor, which is +chain-wide and delimits NFS eviction. + +## Spending transaction +For a transparent outpoint, the transaction that consumes it — the one +listing the outpoint as an input's prevout. Distinct from the outpoint's +creating transaction, which the outpoint itself already names in its +first field. A spend index maps outpoints to spending transactions, +never to creating ones. From 5abe7bdf7ffadd7ae114c9dee05c63bbcada227d Mon Sep 17 00:00:00 2001 From: zancas Date: Wed, 1 Jul 2026 23:07:15 -0700 Subject: [PATCH 14/19] Fix spend-index rot invisible to default builds; point docs at ADR 0006 cargo check --features outp_to_spend_index fails at the previous HEAD: the gated module still referenced NON_FINALIZED_DEPTH after this branch renamed the constant to OPERATIONAL_NFS_DEPTH. A default-off feature is compiled by no default build, so nothing caught the rename missing it. Fix the five references, and a clone-on-Copy in spawn_build. Also repoint the module header and feature comment from the phantom docs/adr/0002 path to the landed ADR 0006, and rewrite the SpendIndexDb::spending_txid doc to name the real serving plan: the serve-time union already exists as get_outpoint_spenders (FullChain scope, #1167); wiring this store in as that method's finalised leg is the deferred step. Co-Authored-By: Claude Fable 5 --- packages/zaino-state/Cargo.toml | 2 +- .../finalised_state/outp_to_spend_index.rs | 22 +++++++++++-------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/packages/zaino-state/Cargo.toml b/packages/zaino-state/Cargo.toml index e5b3a4320..c88f13838 100644 --- a/packages/zaino-state/Cargo.toml +++ b/packages/zaino-state/Cargo.toml @@ -44,7 +44,7 @@ transparent_address_history_experimental = [] # Standalone, parallel-buildable finalised spend index # (transparent_outpoint -> spending_txid). Proof of concept; off by default. -# See docs/adr/0002-finalised-spend-index-parallel-build.md. +# See docs/adr/0006-finalised-spend-index-parallel-build.md. outp_to_spend_index = [] [dependencies] diff --git a/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs b/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs index 2b413a755..7782e88fc 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs @@ -2,7 +2,7 @@ //! outpoint to the txid of the transaction that consumed it. //! //! Proof of concept, gated behind the `outp_to_spend_index` feature. See -//! `docs/adr/0002-finalised-spend-index-parallel-build.md`. +//! `docs/adr/0006-finalised-spend-index-parallel-build.md`. //! //! The build stages live here: **extract** spends from a block batch, //! **collate** them into LMDB key order, and bulk-load them into the index's @@ -21,7 +21,7 @@ use crate::chain_index::finalised_state::build_indexed_block_from_source; use crate::chain_index::source::validator_connector::StateSource; use crate::chain_index::source::BlockchainSource; use crate::chain_index::types::TransactionHash; -use crate::chain_index::NON_FINALIZED_DEPTH; +use crate::chain_index::OPERATIONAL_NFS_DEPTH; use crate::config::ChainIndexConfig; use crate::error::FinalisedStateError; use crate::{IndexedBlock, Outpoint, TransparentCompactTx, ZainoVersionedSerde as _}; @@ -165,8 +165,12 @@ impl SpendIndexDb { } /// The txid that consumed `outpoint`, or `None` if it is unspent in - /// finalised state. A single point lookup; the caller unions this with the - /// non-finalised window at serve time (the deferred seam union). + /// finalised state. A single point lookup. The serve-time union with the + /// non-finalised window already exists as + /// `ChainIndex::get_outpoint_spenders` (`ChainScope::FullChain`, #1167); + /// wiring this store in as that method's finalised leg — replacing the + /// monolith's `spent` table + `TxLocation → txid` resolution — is the + /// deferred serving step. pub(super) fn spending_txid( &self, outpoint: &Outpoint, @@ -244,7 +248,7 @@ impl SpendIndexSync { }; // Finalised tip: below the reorg-possible window, clamped at genesis. - let finalised_tip = best.0.saturating_sub(NON_FINALIZED_DEPTH); + let finalised_tip = best.0.saturating_sub(OPERATIONAL_NFS_DEPTH); let zebra_network = self.network.to_zebra_network(); let sapling_activation = NetworkUpgrade::Sapling @@ -300,7 +304,7 @@ pub(crate) fn spawn_build( source: StateSource, cfg: &ChainIndexConfig, ) -> tokio::task::JoinHandle> { - let network = cfg.network.clone(); + let network = cfg.network; let dir = spend_index_dir(cfg); tokio::spawn(async move { let start_height = NetworkUpgrade::Sapling @@ -496,8 +500,8 @@ mod spend_index_sync { let blocks = fixture_blocks(); let chain_top = blocks.last().expect("non-empty chain").height; // No mining: the mock's tip is its max loaded height, so this matches - // `run`'s own `best - NON_FINALIZED_DEPTH`. - let finalised_tip = chain_top - NON_FINALIZED_DEPTH; + // `run`'s own `best - OPERATIONAL_NFS_DEPTH`. + let finalised_tip = chain_top - OPERATIONAL_NFS_DEPTH; let mock = build_active_mockchain_source(chain_top, blocks.clone()); let (db, dir) = build_index(mock, "run").await; @@ -621,7 +625,7 @@ mod spend_index_presence { .expect("generate chain") .current() .0; - let finalised_tip = (blocks.len() as u32 - 1).saturating_sub(NON_FINALIZED_DEPTH); + let finalised_tip = (blocks.len() as u32 - 1).saturating_sub(OPERATIONAL_NFS_DEPTH); if has_finalised_spend(&blocks, finalised_tip) { found = Some((blocks, finalised_tip)); break; From 5fc70a663a736288732f68b7154a701225585b74 Mon Sep 17 00:00:00 2001 From: zancas Date: Wed, 1 Jul 2026 23:07:32 -0700 Subject: [PATCH 15/19] Compile experimental_features in makers lint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The same rot pattern struck twice: this branch's constant rename missed the default-off spend-index module, and dev's async-trait -> native-AFIT migration broke transparent_address_history_experimental (E0446: pub(crate) AddrEventBytes leaking through the pub TransparentHistExt trait). Both were invisible because no default build compiles gated code, and the existing clippy --all-features lint is red with dev-inherited warnings, so it guards nothing in practice. Add a check-experimental task (cargo check --tests --features experimental_features) to the makers lint front door, and fix what its first run caught: TransparentHistExt narrows to pub(crate) — it has no users outside zaino-state and no re-export, so minimum visibility applies. Co-Authored-By: Claude Fable 5 --- .../src/chain_index/finalised_state/capability.rs | 2 +- tools/makefiles/lints.toml | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/zaino-state/src/chain_index/finalised_state/capability.rs b/packages/zaino-state/src/chain_index/finalised_state/capability.rs index 0193c5d69..39becec45 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/capability.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/capability.rs @@ -964,7 +964,7 @@ pub trait IndexedBlockExt: Send + Sync { /// Range semantics: /// - Methods that accept `start_height` and `end_height` interpret the range as inclusive: /// `[start_height, end_height]` -pub trait TransparentHistExt: Send + Sync { +pub(crate) trait TransparentHistExt: Send + Sync { /// Fetch all address history records for a given transparent address. /// /// Returns: diff --git a/tools/makefiles/lints.toml b/tools/makefiles/lints.toml index cac50b8d3..9676a2104 100644 --- a/tools/makefiles/lints.toml +++ b/tools/makefiles/lints.toml @@ -11,6 +11,11 @@ env = { RUSTDOCFLAGS = "-D warnings" } command = "cargo" args = ["doc", "--no-deps", "--all-features"] +[tasks.check-experimental] +description = "Compile (incl. test targets) with the experimental_features umbrella on. Gated modules are invisible to default builds, so without this they rot silently (e.g. a crate-wide rename missing a default-off module)." +command = "cargo" +args = ["check", "--tests", "--features", "experimental_features"] + [tasks.lint-boundary-conversions] description = "Forbid From/TryFrom impls across the persistence and wire boundaries. Both must use named inherent methods (from_business/into_business for persistence; to_wire/try_from_wire for wire)." # See CLAUDE.md §'Persistence-boundary conversions' and §'Wire-boundary @@ -103,6 +108,7 @@ dependencies = [ "fmt", "clippy", "doc", + "check-experimental", "lint-boundary-conversions", "lint-workbench", "lint-toolchain-pin", From e8c12c11adfff3185e4da12a7fec2c86f1d882c3 Mon Sep 17 00:00:00 2001 From: zancas Date: Thu, 2 Jul 2026 19:38:08 -0700 Subject: [PATCH 16/19] Parallelize the spend-index build; instrument it for benchmarking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three moves toward the pilot's acceptance bar (ADR 0006: a mainnet benchmark of an optimally fast index sync), informed by emersonian's PR #1241, which measured the monolith's serial two-await-per-block fetch collapsing to ~1 blk/s in the sandblast band with both zaino and zebra CPU-idle: - Instrument the build. run() returns SpendIndexBuildStats — worker count, blocks, spends, and wall-clock per stage (stream/extract, collate, bulk-load). Collate and load are identical across build variants, so comparing runs isolates the streaming stage. spawn_build logs the stats and gains loud, ignorable-when-unset benchmark env knobs: ZAINO_SPEND_INDEX_START_HEIGHT / _END_HEIGHT bound the built range; ZAINO_SPEND_INDEX_WORKERS sets the fan-out. - Parallelize the streaming stage. Workers pull fixed-size 1000-block chunks from a shared atomic queue — chunk-pulling self-balances the block-weight skew (sandblast blocks dwarf 2016-era ones) — and extract into worker-local buffers; one global sort and one MDB_APPEND pass follow, so workers never touch the store (the single-writer discipline whose violation PR #1275 diagnosed as LMDB SIGSEGV). There is one code path, not two: workers = 1 IS the serial baseline, so serial-vs-parallel comparisons vary only the fan-out. The move-only single-build guarantee is untouched. - Make the fetch roots-free. Workers extract directly from the zebra block (extract_spends_from_zebra_block): one get_block per height, no get_commitment_tree_roots await, no compact conversion of the discarded shielded data. The compact-form extractor remains as the test oracle, so the existing sync-loop and presence tests now cross-check the two extraction paths over the same chains. SpendIndexSync drops its network field (it only fed activation heights the extractor no longer needs). Co-Authored-By: Claude Fable 5 --- ...06-finalised-spend-index-parallel-build.md | 29 +- packages/zaino-state/src/chain_index.rs | 6 +- .../finalised_state/outp_to_spend_index.rs | 307 +++++++++++++++--- 3 files changed, 291 insertions(+), 51 deletions(-) diff --git a/docs/adr/0006-finalised-spend-index-parallel-build.md b/docs/adr/0006-finalised-spend-index-parallel-build.md index e3dc35a99..505a07d27 100644 --- a/docs/adr/0006-finalised-spend-index-parallel-build.md +++ b/docs/adr/0006-finalised-spend-index-parallel-build.md @@ -52,18 +52,26 @@ default-off Cargo feature `outp_to_spend_index` (a member of `experimental_features`), decomposed into three roles so that read-freedom is scoped to exactly one of them: -- **Block source** (impure): a `BlockchainSource` producing - `IndexedBlock`s. Bound at compile time to zebra's StateService or the +- **Block source** (impure): a `BlockchainSource` producing zebra + blocks. Bound at compile time to zebra's StateService or the test mockchain via the `SpendIndexSource` trait — the JSON-RPC/zcashd `FetchService` does not implement it, so "never FetchService" is a - compile-time fact. + compile-time fact. The fetch is **roots-free**: one `get_block` per + height, skipping the monolith ingestion's second sequential await + (`get_commitment_tree_roots`) and its compact conversion of shielded + data — the spend index needs neither, and PR #1241 measured that + two-await-per-block pattern collapsing to ~1 blk/s in the sandblast + band. - **Extractor** (statically read-free): a free function handed only - `&[IndexedBlock]` — no `&self`, no database, no validator handle in + fetched block data — no `&self`, no database, no validator handle in scope — so a previous-output lookup is unrepresentable, not merely discouraged. Each transparent input yields `(prevout_outpoint, containing_txid)`. Null-prevout inputs (only the coinbase input) are skipped; spends *of* coinbase outputs are ordinary - spends and are indexed. + spends and are indexed. It exists in two forms — over raw zebra blocks + (the build path) and over zaino's compact form (the original, kept as + the test oracle) — and the sync-loop tests assert the two agree over + the same chains. - **Collator/store** (impure, single-writer): encode, byte-sort, reject duplicate keys as corrupt input, and bulk-load with `MDB_APPEND` into the index's **own** LMDB environment (database @@ -83,7 +91,16 @@ source over `[start_height, finalised_tip]`, where `finalised_tip = best_height − non-finalized depth` — feeding the extractor only finalised blocks makes the build reorg-immune by construction. No resume watermark, no backfill from on-disk block -tables, no migration; a crash reruns from the start height. +tables, no migration; a crash reruns from the start height. The +streaming stage fans out across worker tasks pulling fixed-size height +chunks from a shared queue (chunk-pulling self-balances the block-weight +skew across the chain); collation stays one global sort feeding one +append pass, and workers never touch the store — the single-writer +discipline whose violation PR #1275 diagnosed as LMDB corruption. +There is one code path, not two: `workers = 1` *is* the serial +baseline, so serial-vs-parallel benchmarks vary only the fan-out, and +the build reports per-stage timings (stream/extract, collate, load) +plus its worker count to make every run a self-describing measurement. The start height — the **index floor** — is a first-class, configurable property of the index that survives into production, not pilot diff --git a/packages/zaino-state/src/chain_index.rs b/packages/zaino-state/src/chain_index.rs index 0730159be..c67fcc746 100644 --- a/packages/zaino-state/src/chain_index.rs +++ b/packages/zaino-state/src/chain_index.rs @@ -706,7 +706,11 @@ pub struct NodeBackedChainIndex { /// Handle to the one-shot finalised spend-index build, if one was spawned. /// Aborted on shutdown/drop so the build never outlives the index. #[cfg(feature = "outp_to_spend_index")] - spend_index_build: Option>>, + spend_index_build: Option< + tokio::task::JoinHandle< + Result, + >, + >, status: NamedAtomicStatus, network: ZebraNetwork, source: Source, diff --git a/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs b/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs index 7782e88fc..aab028ed3 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs @@ -8,10 +8,14 @@ //! **collate** them into LMDB key order, and bulk-load them into the index's //! own LMDB store via `MDB_APPEND` ([`SpendIndexDb`]). The independent, //! move-only sync loop ([`SpendIndexSync`]) drives these in one shot over -//! `[start_height, finalised_tip]` streamed from a [`SpendIndexSource`], and -//! [`spawn_build`] wires it onto the owning `ChainIndex` (StateService only, -//! from the Sapling activation height). (Table-level integrity over the entries -//! is deferred; it is not MVP.) +//! `[start_height, finalised_tip]` streamed from a [`SpendIndexSource`]: +//! block sourcing/extraction fans out across worker tasks pulling fixed-size +//! height chunks from a shared queue (`workers = 1` is the serial baseline — +//! the same path minus concurrency, so serial and parallel builds compare +//! stage-for-stage), and collation stays one global sort feeding one +//! `MDB_APPEND` pass. [`spawn_build`] wires it onto the owning `ChainIndex` +//! (StateService only, from the Sapling activation height). (Table-level +//! integrity over the entries is deferred; it is not MVP.) use std::path::Path; @@ -206,79 +210,213 @@ impl SpendIndexSource for StateSource {} #[cfg(test)] impl SpendIndexSource for crate::chain_index::source::mockchain_source::MockchainSource {} +/// Stage timings and volumes from one one-shot build: the benchmark's +/// measurement surface. `stream_and_extract` covers the sequential +/// fetch → decode → extract loop — the stage a parallel build transforms — +/// while `collate` and `bulk_load` are identical across build variants, so +/// comparing two builds' stats isolates the streaming stage. +#[derive(Debug, Default, Clone)] +pub(crate) struct SpendIndexBuildStats { + /// Worker tasks the streaming stage fanned out across (`1` = the serial + /// baseline), recorded so every measurement is self-describing. + pub(crate) workers: u32, + /// Blocks fetched and scanned. + pub(crate) blocks: u32, + /// Spend records extracted (= entries loaded). + pub(crate) spends: usize, + /// Wall-clock of the fetch → decode → extract loop. + pub(crate) stream_and_extract: std::time::Duration, + /// Wall-clock of the global encode + sort + duplicate check. + pub(crate) collate: std::time::Duration, + /// Wall-clock of the `MDB_APPEND` load and durability sync. + pub(crate) bulk_load: std::time::Duration, +} + /// The independent, single-run builder for the spend index. /// /// Move-only (`!Clone`) with a private constructor and a `self`-consuming /// [`run`](Self::run): the type system makes a second concurrent build -/// unrepresentable — "only one loop at a time" without a runtime guard. POC -/// build model: re-stream `start_height` → finalised tip from `source`, extract -/// every spend, collate once, and bulk-load the index in a single `MDB_APPEND` -/// pass. +/// unrepresentable — "only one loop at a time" without a runtime guard. (The +/// guarantee constrains *builds*; the fan-out of worker tasks inside one +/// `run` is not a second build.) POC build model: re-stream `start_height` → +/// finalised tip from `source` across `workers` chunk-pulling tasks, extract +/// every spend, collate once, and bulk-load the index in a single +/// `MDB_APPEND` pass. pub(super) struct SpendIndexSync { source: S, db: SpendIndexDb, - network: Network, /// First height to index, inclusive. Genesis (`0`) for a full index, or a /// higher height — e.g. a network-upgrade activation — to cover only spends /// occurring from that epoch onward. start_height: u32, + /// Benchmark knob: when set, the build stops at + /// `min(finalised_tip, cap)` instead of the finalised tip, so a baseline + /// run can cover a fixed block range. `None` (production) builds to the + /// finalised tip. + end_height_cap: Option, + /// Worker tasks for the streaming stage. Defaults to the machine's + /// available parallelism; `1` selects the serial baseline (the identical + /// path minus concurrency), which is what parallel runs are measured + /// against. + workers: usize, } impl SpendIndexSync { /// Private mint: production enters via [`SpendIndexSync::from_state`] /// (StateService only); tests construct directly with a `MockchainSource`. - fn new(source: S, db: SpendIndexDb, network: Network, start_height: u32) -> Self { + fn new(source: S, db: SpendIndexDb, start_height: u32) -> Self { Self { source, db, - network, start_height, + end_height_cap: None, + workers: std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get), } } - /// Builds the spend index from its start height to the finalised tip, consuming the - /// handle. One-shot (POC): no resume, no batching — a single global - /// `collate` + `MDB_APPEND`. - pub(super) async fn run(self) -> Result<(), FinalisedStateError> { + /// Builds the spend index from its start height to the finalised tip, + /// consuming the handle and returning the per-stage timings. One-shot + /// (POC): no resume, no batching — a single global `collate` + + /// `MDB_APPEND`. + pub(super) async fn run(self) -> Result { let Some(best) = self.source.get_best_block_height().await.map_err(|error| { FinalisedStateError::Custom(format!("spend index: fetch best height: {error:?}")) })? else { - return Ok(()); // empty chain — nothing finalised to index + return Ok(SpendIndexBuildStats::default()); // empty chain — nothing finalised to index }; // Finalised tip: below the reorg-possible window, clamped at genesis. + // A benchmark cap lowers it further so a run covers a fixed range. let finalised_tip = best.0.saturating_sub(OPERATIONAL_NFS_DEPTH); + let finalised_tip = self + .end_height_cap + .map_or(finalised_tip, |cap| finalised_tip.min(cap)); + + // Stream finalised blocks across the workers, extracting spends and + // dropping each block; only the (smaller) spend records are retained + // for the one-shot sort. Fixed-size chunks pulled from a shared queue + // self-balance the block-weight skew across the chain (late blocks + // are far heavier than early ones), so equal-width per-worker ranges + // are not needed. + let workers = self.workers.max(1); + let chunks = + std::sync::Arc::new(chunk_ranges(self.start_height, finalised_tip, CHUNK_BLOCKS)); + let next_chunk = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + + let mut stats = SpendIndexBuildStats { + workers: workers as u32, + blocks: chunks.iter().map(|(lo, hi)| hi - lo + 1).sum(), + ..Default::default() + }; - let zebra_network = self.network.to_zebra_network(); - let sapling_activation = NetworkUpgrade::Sapling - .activation_height(&zebra_network) - .expect("Sapling activation height is set for every network"); - let nu5_activation = NetworkUpgrade::Nu5.activation_height(&zebra_network); - - // Stream finalised blocks, extracting spends and dropping each block; - // only the (smaller) spend records are retained for the one-shot sort. - // Chainwork is irrelevant to spend extraction, so a `None` parent is fine. + let stream_started = std::time::Instant::now(); + let mut handles = Vec::with_capacity(workers); + for _ in 0..workers { + handles.push(tokio::spawn(extract_chunks( + self.source.clone(), + std::sync::Arc::clone(&chunks), + std::sync::Arc::clone(&next_chunk), + ))); + } let mut spends: Vec = Vec::new(); - for height in self.start_height..=finalised_tip { - let block = build_indexed_block_from_source( - &self.source, - self.network, - sapling_activation, - nu5_activation, - height, - None, - ) - .await?; - spends.extend(extract_spends(std::slice::from_ref(&block))); + for handle in handles { + let worker_records = handle.await.map_err(|error| { + FinalisedStateError::Custom(format!("spend index: worker task died: {error}")) + })??; + spends.extend(worker_records); } + stats.stream_and_extract = stream_started.elapsed(); + stats.spends = spends.len(); + let collate_started = std::time::Instant::now(); let collated = collate(&spends)?; + stats.collate = collate_started.elapsed(); + + let load_started = std::time::Instant::now(); tokio::task::block_in_place(|| self.db.bulk_load(&collated))?; - Ok(()) + stats.bulk_load = load_started.elapsed(); + + Ok(stats) + } +} + +/// Blocks per work-queue chunk. Small relative to any real build range so +/// chunk-pulling self-balances the block-weight skew, large enough that queue +/// bookkeeping (one relaxed `fetch_add` per chunk) is noise. +const CHUNK_BLOCKS: u32 = 1_000; + +/// Splits inclusive `[start, end]` into consecutive inclusive ranges of at +/// most `size` blocks. Empty when `start > end`. +fn chunk_ranges(start: u32, end: u32, size: u32) -> Vec<(u32, u32)> { + let mut ranges = Vec::new(); + let mut lo = start; + while lo <= end { + let hi = lo.saturating_add(size - 1).min(end); + ranges.push((lo, hi)); + match hi.checked_add(1) { + Some(next) => lo = next, + None => break, // end == u32::MAX: the final range is pushed + } + } + ranges +} + +/// One streaming worker: pulls chunk indices from the shared queue until it +/// drains, fetching each height and extracting its spends into a +/// worker-local buffer (returned for the caller to concatenate). +/// +/// Roots-free: the per-block cost is one `get_block` plus a +/// transparent-input walk — no commitment-tree-root fetch and no compact +/// conversion of the (discarded) shielded data. The monolith's per-block +/// ingestion awaits `get_block` *and* `get_commitment_tree_roots` in +/// sequence, the fetch-latency pattern PR #1241 measured collapsing to +/// ~1 blk/s in the sandblast band; the spend index never needs the second +/// call. +async fn extract_chunks( + source: S, + chunks: std::sync::Arc>, + next_chunk: std::sync::Arc, +) -> Result, FinalisedStateError> { + let mut records = Vec::new(); + loop { + let index = next_chunk.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let Some(&(lo, hi)) = chunks.get(index) else { + return Ok(records); + }; + for height in lo..=hi { + let block = source + .get_block(zebra_state::HashOrHeight::Height( + zebra_chain::block::Height(height), + )) + .await? + .ok_or_else(|| { + FinalisedStateError::Custom(format!( + "spend index: no block at finalised height {height}" + )) + })?; + records.extend(extract_spends_from_zebra_block(&block)); + } } } +/// Extracts every transparent spend in one zebra block: each non-coinbase +/// input yields `(prevout_outpoint, containing_txid)`. Statically read-free — +/// handed only block data — like [`extract_spends`], its compact-form twin; +/// the sync-loop tests assert both paths agree over the same chains. +fn extract_spends_from_zebra_block( + block: &zebra_chain::block::Block, +) -> impl Iterator + '_ { + block.transactions.iter().flat_map(|tx| { + let spending_txid = TransactionHash::from(tx.hash().0); + tx.inputs().iter().filter_map(move |input| { + input + .outpoint() + .map(|prevout| (Outpoint::new(prevout.hash.0, prevout.index), spending_txid)) + }) + }) +} + /// Map size for the spend index's own LMDB env. LMDB grows the file lazily (no /// `WRITE_MAP` here), so this is an upper bound, not a preallocation. const SPEND_INDEX_MAP_SIZE: usize = 32 * 1024 * 1024 * 1024; @@ -303,7 +441,7 @@ fn spend_index_dir(cfg: &ChainIndexConfig) -> std::path::PathBuf { pub(crate) fn spawn_build( source: StateSource, cfg: &ChainIndexConfig, -) -> tokio::task::JoinHandle> { +) -> tokio::task::JoinHandle> { let network = cfg.network; let dir = spend_index_dir(cfg); tokio::spawn(async move { @@ -314,17 +452,63 @@ pub(crate) fn spawn_build( })? .0; let db = SpendIndexDb::open(&dir, SPEND_INDEX_MAP_SIZE)?; - let result = SpendIndexSync::new(source, db, network, start_height) - .run() - .await; + let mut sync = SpendIndexSync::new(source, db, start_height); + if let Some(start) = parsed_env::(BENCH_START_HEIGHT_ENV) { + tracing::warn!( + start, + "spend-index build start overridden via {BENCH_START_HEIGHT_ENV} (benchmark knob)", + ); + sync.start_height = start; + } + if let Some(cap) = parsed_env::(BENCH_END_HEIGHT_ENV) { + tracing::warn!( + cap, + "spend-index build height-capped via {BENCH_END_HEIGHT_ENV} (benchmark knob)", + ); + sync.end_height_cap = Some(cap); + } + if let Some(workers) = parsed_env::(WORKERS_ENV) { + tracing::warn!( + workers, + "spend-index build worker count set via {WORKERS_ENV} (1 = serial baseline)", + ); + sync.workers = workers.max(1); + } + let result = sync.run().await; match &result { - Ok(()) => tracing::info!("finalised spend-index build complete"), + Ok(stats) => tracing::info!(?stats, "finalised spend-index build complete"), Err(error) => tracing::error!("finalised spend-index build failed: {error}"), } result }) } +/// Benchmark env knobs. Each is ignored when unset and announced with a +/// `warn!` when active, so a bounded or tuned build can't pass silently for +/// a production one. Start/end bound the built range (e.g. a fixed 100k-block +/// window); the worker knob selects the streaming fan-out, with `1` the +/// serial baseline that parallel runs are measured against. +const BENCH_START_HEIGHT_ENV: &str = "ZAINO_SPEND_INDEX_START_HEIGHT"; +const BENCH_END_HEIGHT_ENV: &str = "ZAINO_SPEND_INDEX_END_HEIGHT"; +const WORKERS_ENV: &str = "ZAINO_SPEND_INDEX_WORKERS"; + +/// The parsed value of env knob `name`, or `None` when unset or unparseable +/// (the latter is logged, not fatal — a benchmark knob must not take the +/// production build down). +fn parsed_env(name: &str) -> Option +where + T::Err: std::fmt::Display, +{ + let raw = std::env::var(name).ok()?; + match raw.parse() { + Ok(value) => Some(value), + Err(error) => { + tracing::warn!(%raw, "ignoring unparseable {name}: {error}"); + None + } + } +} + #[cfg(test)] mod spends_in_transaction { use super::*; @@ -415,6 +599,41 @@ mod collate { } } +#[cfg(test)] +mod chunk_ranges { + #[test] + fn covers_the_range_exactly_once_with_ragged_tail() { + assert_eq!( + super::chunk_ranges(10, 34, 10), + vec![(10, 19), (20, 29), (30, 34)], + ); + } + + #[test] + fn exact_multiple_has_no_ragged_tail() { + assert_eq!(super::chunk_ranges(0, 19, 10), vec![(0, 9), (10, 19)]); + } + + #[test] + fn single_block_range_is_one_chunk() { + assert_eq!(super::chunk_ranges(7, 7, 10), vec![(7, 7)]); + } + + #[test] + fn empty_when_start_exceeds_end() { + assert!(super::chunk_ranges(8, 7, 10).is_empty()); + } + + #[test] + fn end_at_u32_max_terminates() { + let ranges = super::chunk_ranges(u32::MAX - 5, u32::MAX, 4); + assert_eq!( + ranges, + vec![(u32::MAX - 5, u32::MAX - 2), (u32::MAX - 1, u32::MAX)], + ); + } +} + #[cfg(test)] mod spend_index_db { use super::*; @@ -472,7 +691,7 @@ mod spend_index_sync { std::env::temp_dir().join(format!("outp_to_spend_index_{tag}_{}", std::process::id())); let _ = std::fs::remove_dir_all(&dir); let db = SpendIndexDb::open(&dir, 1 << 24).expect("open spend index"); - SpendIndexSync::new(mock, db, Network::Mainnet, 0) + SpendIndexSync::new(mock, db, 0) .run() .await .expect("spend-index build runs"); @@ -672,7 +891,7 @@ mod spend_index_presence { )); let _ = std::fs::remove_dir_all(&dir); let db = SpendIndexDb::open(&dir, 1 << 24).expect("open spend index"); - SpendIndexSync::new(mock, db, Network::Mainnet, 0) + SpendIndexSync::new(mock, db, 0) .run() .await .expect("spend-index build runs"); From 8645d5b1d8b2964fbed6ba538f53f48a7e9420ff Mon Sep 17 00:00:00 2001 From: zancas Date: Thu, 2 Jul 2026 19:40:18 -0700 Subject: [PATCH 17/19] Add zebra-state dev-dep: the SpendingTransactionId parity oracle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wire zaino-state's dev-dependencies to zebra-state with the indexer and proptest-impl features: indexer exposes zebra's SpendingTransactionId read request — the temporary correctness oracle ADR 0006 designates for spend-index parity, to be captured as golden vectors before zebra removes the feature — and proptest-impl exposes populated_state for building oracle-backed test states. Dev-only; the production feature set is unchanged. Deliberately temporary, like the oracle itself: once parity is demonstrated and the golden vectors are captured, this dependency is reverted along with zebra's indexer feature. Co-Authored-By: Claude Fable 5 --- Cargo.lock | 3 +++ packages/zaino-state/Cargo.toml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index cd489ac56..cb575bbaf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6644,6 +6644,8 @@ dependencies = [ "lazy_static", "metrics", "mset", + "proptest", + "proptest-derive", "rayon", "regex", "rlimit", @@ -6658,6 +6660,7 @@ dependencies = [ "tracing", "zebra-chain", "zebra-node-services", + "zebra-test", ] [[package]] diff --git a/packages/zaino-state/Cargo.toml b/packages/zaino-state/Cargo.toml index c88f13838..493e14870 100644 --- a/packages/zaino-state/Cargo.toml +++ b/packages/zaino-state/Cargo.toml @@ -108,6 +108,9 @@ tempfile = { workspace = true } tracing-subscriber = { workspace = true } once_cell = { workspace = true } zebra-chain = { workspace = true, features = ["proptest-impl"] } +# `indexer` exposes zebra's SpendingTransactionId oracle; `proptest-impl` +# exposes `populated_state`. Dev-only: off in production builds. +zebra-state = { workspace = true, features = ["indexer", "proptest-impl"] } proptest.workspace = true incrementalmerkletree = "0.8.2" rand = "0.10.1" From 1d60faa4b5be6128dcc2b90c8d389d31ed5494f8 Mon Sep 17 00:00:00 2001 From: zancas Date: Thu, 2 Jul 2026 20:32:11 -0700 Subject: [PATCH 18/19] Test spend-index parity against zebra's SpendingTransactionId oracle The synthetic-chain leg of ADR 0006's acceptance bar, and the first consumer of the zebra-state indexer dev-dep: build the index through the real run() path (chunk queue, roots-free extraction, MDB_APPEND) from a deterministic synthetic chain, commit the same blocks into a real zebra ReadStateService via populated_state (checkpoint-verified, so the oracle's finalized spending_tx_loc index covers the whole chain), then compare every prevout spent anywhere on the chain: - at or below the seam, the index and the oracle must agree byte-for-byte on the spending txid; - above the seam, the one designed divergence is asserted rather than skipped: the finalised-only index is silent while the whole-chain oracle answers; - a never-spent outpoint is None from both implementations. The presence test's deterministic chain generation moves to a shared synthetic_chain test module (seeded TestRunner, so both tests see the same chain on every run); its assertions are unchanged. Co-Authored-By: Claude Fable 5 --- .../finalised_state/outp_to_spend_index.rs | 157 ++++++++++++++++-- 1 file changed, 140 insertions(+), 17 deletions(-) diff --git a/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs b/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs index aab028ed3..ac889d114 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs @@ -761,15 +761,15 @@ mod spend_index_sync { } } -/// End-to-end **presence** test: `run` must build a *non-empty* index when the -/// finalised range contains transparent spends. The 201-block fixture can't -/// exercise this (coinbase maturity puts its spends above the depth-100 seam), -/// so this synthesises a chain with zebra's block generator, which — with -/// `allow_all_transparent_coinbase_spends` — produces transparent spends that -/// can land below the seam. Tracks zingolabs/zaino#1334. +/// Deterministic synthetic-chain generation shared by the presence and +/// zebra-parity tests. The 201-block fixture can't put a transparent spend +/// below the depth-100 test seam (coinbase maturity forbids it), so these +/// chains come from zebra's block generator, whose +/// `allow_all_transparent_coinbase_spends` predicate lets spends land in the +/// finalised range. #[cfg(test)] -mod spend_index_presence { - use super::*; +mod synthetic_chain { + use super::OPERATIONAL_NFS_DEPTH; use crate::chain_index::source::mockchain_source::MockchainSource; use proptest::prelude::{Arbitrary, Strategy}; use proptest::strategy::ValueTree; @@ -789,7 +789,7 @@ mod spend_index_presence { // is overwhelmingly likely within a couple of tries. const MAX_GEN_ATTEMPTS: usize = 40; - fn build_mock(blocks: Vec>) -> MockchainSource { + pub(super) fn build_mock(blocks: Vec>) -> MockchainSource { let hashes = blocks .iter() .map(|block| crate::BlockHash::from(block.hash())) @@ -814,8 +814,10 @@ mod spend_index_presence { }) } - #[tokio::test(flavor = "multi_thread")] - async fn run_builds_a_nonempty_index_from_a_synthetic_chain() { + /// A deterministically generated genesis-rooted mainnet chain holding at + /// least one transparent spend at or below the seam, plus its finalised + /// tip. The seeded `TestRunner` yields the same chain on every run. + pub(super) fn with_finalised_spend() -> (Vec>, u32) { let strategy = LedgerState::arbitrary_with(LedgerStateOverride { height_override: Some(Height(0)), previous_block_hash_override: Some(GENESIS_PREVIOUS_BLOCK_HASH), @@ -837,7 +839,6 @@ mod spend_index_presence { // Deterministically search for a generated chain with a finalised spend, // so the presence direction is genuinely exercised. - let mut found = None; for _ in 0..MAX_GEN_ATTEMPTS { let blocks = strategy .new_tree(&mut runner) @@ -846,14 +847,25 @@ mod spend_index_presence { .0; let finalised_tip = (blocks.len() as u32 - 1).saturating_sub(OPERATIONAL_NFS_DEPTH); if has_finalised_spend(&blocks, finalised_tip) { - found = Some((blocks, finalised_tip)); - break; + return (blocks, finalised_tip); } } - let (blocks, finalised_tip) = - found.expect("a synthetic chain with a finalised transparent spend within budget"); + panic!("no synthetic chain with a finalised transparent spend within budget"); + } +} - let mock = build_mock(blocks); +/// End-to-end **presence** test: `run` must build a *non-empty* index when the +/// finalised range contains transparent spends (chain from +/// [`synthetic_chain`]). Tracks zingolabs/zaino#1334. +#[cfg(test)] +mod spend_index_presence { + use super::*; + + #[tokio::test(flavor = "multi_thread")] + async fn run_builds_a_nonempty_index_from_a_synthetic_chain() { + let (blocks, finalised_tip) = synthetic_chain::with_finalised_spend(); + + let mock = synthetic_chain::build_mock(blocks); // Reference: the spends in the finalised range, built the way `run` // builds blocks (so this checks fetch + boundary + collate + LMDB @@ -908,3 +920,114 @@ mod spend_index_presence { std::fs::remove_dir_all(&dir).ok(); } } + +/// Cross-implementation **parity** test: the built index must agree +/// byte-for-byte with zebra's `SpendingTransactionId` read request — the +/// temporary correctness oracle ADR 0006 designates — served here by a real +/// `ReadStateService` (dev-features `indexer` + `proptest-impl`) over the +/// same synthetic chain the index is built from. +#[cfg(test)] +mod spend_index_zebra_parity { + use super::*; + use tower::ServiceExt as _; + use zebra_chain::parameters::Network as ZebraNetwork; + use zebra_state::{ReadRequest, ReadResponse, ReadStateService, Spend}; + + /// The oracle's answer for one prevout: the txid of its spending + /// transaction, if zebra knows one. + async fn oracle_spender( + read_state: &ReadStateService, + prevout: zebra_chain::transparent::OutPoint, + ) -> Option { + match read_state + .clone() + .oneshot(ReadRequest::SpendingTransactionId(Spend::OutPoint(prevout))) + .await + .expect("oracle answers spending-txid reads") + { + ReadResponse::TransactionId(answer) => answer, + other => panic!("unexpected oracle response: {other:?}"), + } + } + + /// For every prevout spent anywhere in the chain: at or below the seam + /// the index and the oracle must agree byte-for-byte; above it the + /// finalised-only index is silent by design while the whole-chain oracle + /// still answers; a never-spent outpoint is `None` in both. + /// + /// `multi_thread`: `run` uses `block_in_place` for the LMDB write. + #[tokio::test(flavor = "multi_thread")] + async fn index_agrees_with_zebra_over_a_synthetic_chain() { + let (blocks, finalised_tip) = synthetic_chain::with_finalised_spend(); + + // zaino: build the index from genesis over the mockchain. + let dir = std::env::temp_dir().join(format!( + "outp_to_spend_index_zebra_parity_{}", + std::process::id() + )); + let _ = std::fs::remove_dir_all(&dir); + let db = SpendIndexDb::open(&dir, 1 << 24).expect("open spend index"); + SpendIndexSync::new(synthetic_chain::build_mock(blocks.clone()), db, 0) + .run() + .await + .expect("spend-index build runs"); + let db = SpendIndexDb::open(&dir, 1 << 24).expect("reopen spend index"); + + // zebra: checkpoint-committing the same blocks lands them all in the + // finalized state, so the oracle covers the whole chain. + let (_state, read_state, _tip, _tip_change) = + zebra_state::populated_state(blocks.clone(), &ZebraNetwork::Mainnet).await; + + let mut finalised_spends = 0usize; + for (height, block) in blocks.iter().enumerate() { + for tx in &block.transactions { + for input in tx.inputs() { + let Some(prevout) = input.outpoint() else { + continue; // coinbase input: not a spend + }; + let indexed = db + .spending_txid(&Outpoint::new(prevout.hash.0, prevout.index)) + .expect("read spend index"); + let oracle = oracle_spender(&read_state, prevout).await; + assert!( + oracle.is_some(), + "oracle must know the spend of {prevout:?}", + ); + if height as u32 <= finalised_tip { + assert_eq!( + indexed.map(<[u8; 32]>::from), + oracle.map(|hash| hash.0), + "finalised spend of {prevout:?}: index and oracle must agree", + ); + finalised_spends += 1; + } else { + // The one expected divergence: finalised-only index + // vs whole-chain oracle. + assert_eq!( + indexed, None, + "above-seam spend of {prevout:?} must not be indexed", + ); + } + } + } + } + assert!( + finalised_spends > 0, + "chain must exercise the byte-for-byte agreement direction", + ); + + // A never-spent outpoint: `None` from both implementations. + let absent = zebra_chain::transparent::OutPoint { + hash: zebra_chain::transaction::Hash([0xEE; 32]), + index: u32::MAX, + }; + assert_eq!( + db.spending_txid(&Outpoint::new(absent.hash.0, absent.index)) + .expect("read absent outpoint"), + None, + ); + assert_eq!(oracle_spender(&read_state, absent).await, None); + + std::fs::remove_dir_all(&dir).ok(); + } +} From 1ce4e6774ff6078f17554856949e884ef01ef82b Mon Sep 17 00:00:00 2001 From: zancas Date: Fri, 3 Jul 2026 00:11:51 -0700 Subject: [PATCH 19/19] Gate the spend-index module's test-only imports behind cfg(test) The roots-free refactor left build_indexed_block_from_source and zaino_common::Network used only by the test mods (the presence test's compact-form expected-value path), so the feature-on lib build warned on both. Surfaced by the parity test run's test-profile build. Co-Authored-By: Claude Fable 5 --- .../src/chain_index/finalised_state/outp_to_spend_index.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs b/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs index ac889d114..c9da99941 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs @@ -21,6 +21,9 @@ use std::path::Path; use lmdb::{Database, DatabaseFlags, Environment, EnvironmentFlags, Transaction as _, WriteFlags}; +// Test-only: the presence test builds its expected values through the +// compact-form path the production build no longer uses. +#[cfg(test)] use crate::chain_index::finalised_state::build_indexed_block_from_source; use crate::chain_index::source::validator_connector::StateSource; use crate::chain_index::source::BlockchainSource; @@ -29,6 +32,7 @@ use crate::chain_index::OPERATIONAL_NFS_DEPTH; use crate::config::ChainIndexConfig; use crate::error::FinalisedStateError; use crate::{IndexedBlock, Outpoint, TransparentCompactTx, ZainoVersionedSerde as _}; +#[cfg(test)] use zaino_common::Network; use zebra_chain::parameters::NetworkUpgrade;