diff --git a/Cargo.lock b/Cargo.lock index cd489ac56..cb575bbaf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6644,6 +6644,8 @@ dependencies = [ "lazy_static", "metrics", "mset", + "proptest", + "proptest-derive", "rayon", "regex", "rlimit", @@ -6658,6 +6660,7 @@ dependencies = [ "tracing", "zebra-chain", "zebra-node-services", + "zebra-test", ] [[package]] diff --git a/Makefile.toml b/Makefile.toml index f38a60907..3595ca15a 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -93,7 +93,7 @@ script.main = 'source "./tools/scripts/push-image.sh"' [tasks.container-test] clear = true -description = "Engine: run nextest in the local CI container, forwarding args (zcashd_support OFF by default; --with-zcashd adds --features zcashd_support). Devs use `makers test [package|e2e|clientless|live|all]`; live-clientless / live-e2e and the `test` front door call this." +description = "Engine: run nextest in the local CI container, forwarding args (zcashd_support OFF by default; --with-zcashd adds --features zcashd_support). Devs use `makers test [packages|e2e|clientless|live|all]`; live-clientless / live-e2e and the `test` front door call this." # This task runs tests inside the container built from live-tests/test_environment. # The entrypoint.sh script in the container sets up test binaries (zcashd, zebrad, zcash-cli) # by creating symlinks from /home/container_user/artifacts to the expected live-tests/test_binaries/bins location. @@ -132,16 +132,16 @@ script.main = 'source "./tools/scripts/container-nextest-workspace.sh"' # =================================================================== # Developer-facing test front door: a single `makers test [SET]` task. # -# makers test packaged (the default) -# makers test package packages/* production-crate tests (no validator) +# makers test packages (the default) +# makers test packages packages/* production-crate tests (no validator) # makers test e2e the e2e live partition # makers test clientless the clientless live partition # makers test live both live partitions + combined summary -# makers test all package then live (everything) +# makers test all packages then live (everything) # -# SET names mirror the crate/dir names (`e2e`, `clientless`); `package`, -# `live`, and `all` are the groupings. `live` = all non-package tests; `all` -# = `package` + `live` (see live-tests/CONTEXT.md). Each set delegates to an +# SET names mirror the crate/dir names (`e2e`, `clientless`); `packages`, +# `live`, and `all` are the groupings. `live` = all non-packages tests; `all` +# = `packages` + `live` (see live-tests/CONTEXT.md). Each set delegates to an # existing engine rather than re-implementing a run. `--with-zcashd` (handled # by the engines) adds the zcashd-backed tests to any set. # =================================================================== @@ -150,33 +150,33 @@ script.main = 'source "./tools/scripts/container-nextest-workspace.sh"' # clear = true fully replaces cargo-make's built-in `test` task (which runs # `cargo test`); without it our `script` would merge with the builtin `command`. clear = true -description = "Run a test set: `makers test [package|e2e|clientless|live|all]` (default: package). Pass --with-zcashd for the zcashd-backed tests." +description = "Run a test set: `makers test [packages|e2e|clientless|live|all]` (default: packages). Pass --with-zcashd for the zcashd-backed tests." script_runner = "bash" script = ''' set -uo pipefail usage() { - echo "usage: makers test [package|e2e|clientless|live|all] [-- nextest args]" >&2 - echo " package (default) packages/* tests, no live validator" >&2 + echo "usage: makers test [packages|e2e|clientless|live|all] [-- nextest args]" >&2 + echo " packages (default) packages/* tests, no live validator" >&2 echo " e2e | clientless that single live partition" >&2 echo " live both live partitions + combined summary" >&2 - echo " all package then live (everything)" >&2 + echo " all packages then live (everything)" >&2 } # A non-flag first arg selects the set and must be a known name; a flag first -# arg (or none) keeps the default `package` and is forwarded to nextest. So -# `makers test --with-zcashd` runs package with the flag, while a mistyped set +# arg (or none) keeps the default `packages` and is forwarded to nextest. So +# `makers test --with-zcashd` runs packages with the flag, while a mistyped set # name errors instead of silently running the default. -set="package" +set="packages" if [ "$#" -gt 0 ] && [ "${1#-}" = "$1" ]; then case "$1" in - package|e2e|clientless|live|all) set="$1"; shift ;; + packages|e2e|clientless|live|all) set="$1"; shift ;; *) echo "makers test: unknown set '$1'" >&2; usage; exit 2 ;; esac fi case "$set" in - package) exec makers container-test "$@" ;; + packages) exec makers container-test "$@" ;; e2e) exec makers live-e2e "$@" ;; clientless) exec makers live-clientless "$@" ;; live) exec cargo run -q --manifest-path tools/test-runner/Cargo.toml --bin live-summary -- "$@" ;; diff --git a/README.md b/README.md index a72a9a495..0b8ca92f4 100644 --- a/README.md +++ b/README.md @@ -106,7 +106,7 @@ The test suites run inside a **podman** container via `makers` (cargo-make): ```sh makers test # packages/* tests that need no live validator (default) makers test live # both live partitions (clientless + e2e) + combined summary -makers test all # everything: package then live +makers test all # everything: packages then live ``` zcashd-backed tests are **off by default**; add `--with-zcashd` to include them diff --git a/docs/adr/0004-rename-integration-partition-to-clientless.md b/docs/adr/0004-rename-integration-partition-to-clientless.md index c98de87d4..cb80fbfa7 100644 --- a/docs/adr/0004-rename-integration-partition-to-clientless.md +++ b/docs/adr/0004-rename-integration-partition-to-clientless.md @@ -50,3 +50,6 @@ unchanged. "clientless" is no longer avoided. - ADR-0003's naming bullet and its three-front-door task surface are superseded; its two-crate split and the `live` umbrella decision still stand. + +> **Later note:** the `package` set was renamed `packages` (plural, matching the +> `packages/` dir); `makers test package` now errors. See `live-tests/CONTEXT.md`. diff --git a/docs/adr/0006-finalised-spend-index-parallel-build.md b/docs/adr/0006-finalised-spend-index-parallel-build.md new file mode 100644 index 000000000..505a07d27 --- /dev/null +++ b/docs/adr/0006-finalised-spend-index-parallel-build.md @@ -0,0 +1,221 @@ +# A standalone, parallel-buildable finalised spend index + +## Status + +accepted (pilot; refines the umbrella schema discussion in issue #1326, +scoped by issue #1328, implemented in PR #1330) + +Note: issues #1328/#1326 reserved the path +`docs/adr/0002-finalised-spend-index-parallel-build.md` for this record; +that number was taken by the live-tests ADR before this file landed, so +the record lives here as 0006. + +## Context + +Serving "which transaction spent this transparent outpoint?" requires an +index mapping `transparent_outpoint → spending_txid`. zebra already +answers this via `ReadRequest::SpendingTransactionId`, but only behind +its non-default `indexer` feature, and that service is slated for +removal once zaino provides the index — so zaino becomes the sole +provider and the index becomes required, not merely offered. zebra's +answer is therefore a temporary correctness oracle: zaino's index is +diffed against it, and golden vectors must be captured before the +feature is removed. + +On the serving side, zaino already answers the query: +`ChainIndex::get_outpoint_spenders` (PR #1167) takes a batch of +outpoints and a `ChainScope` — `FullChain` scans the non-finalised best +chain first, then falls back to the finalised state; `Finalised` is the +reorg-stable subset. Its finalised leg reads the monolith's `spent` +table (`outpoint → TxLocation`) and resolves each deduped `TxLocation` +to a txid through the transaction-location tables. That method is the +consumer this index ultimately backs. + +Independently, the finalised-state build path is a monolith: one +`write_block_with_options` writes all tables per block in a single +transaction, so no index can be built, rebuilt, or optimised in +isolation. The spend index is the simplest useful pilot for breaking +that coupling: its keys are globally disjoint (an outpoint is spent at +most once chain-wide), so per-batch results combine by pure sorted merge +with zero cross-batch reconciliation — unlike value- or script-bearing +indexes, which need a shared live-UTXO primary. + +Terminology (see `packages/zaino-state/src/chain_index/CONTEXT.md`): an +outpoint's first field already names its **creating** transaction; the +transaction that lists the outpoint as an input's prevout is its +**spending** transaction. This index maps to the spending txid only. + +## Decision + +Build the spend index in zaino as a standalone pilot, gated behind the +default-off Cargo feature `outp_to_spend_index` (a member of +`experimental_features`), decomposed into three roles so that +read-freedom is scoped to exactly one of them: + +- **Block source** (impure): a `BlockchainSource` producing zebra + blocks. Bound at compile time to zebra's StateService or the + test mockchain via the `SpendIndexSource` trait — the JSON-RPC/zcashd + `FetchService` does not implement it, so "never FetchService" is a + compile-time fact. The fetch is **roots-free**: one `get_block` per + height, skipping the monolith ingestion's second sequential await + (`get_commitment_tree_roots`) and its compact conversion of shielded + data — the spend index needs neither, and PR #1241 measured that + two-await-per-block pattern collapsing to ~1 blk/s in the sandblast + band. +- **Extractor** (statically read-free): a free function handed only + fetched block data — no `&self`, no database, no validator handle in + scope — so a previous-output lookup is unrepresentable, not merely + discouraged. Each transparent input yields + `(prevout_outpoint, containing_txid)`. Null-prevout inputs (only the + coinbase input) are skipped; spends *of* coinbase outputs are ordinary + spends and are indexed. It exists in two forms — over raw zebra blocks + (the build path) and over zaino's compact form (the original, kept as + the test oracle) — and the sync-loop tests assert the two agree over + the same chains. +- **Collator/store** (impure, single-writer): encode, byte-sort, reject + duplicate keys as corrupt input, and bulk-load with `MDB_APPEND` into + the index's **own** LMDB environment (database + `outp_to_spend_index_1_0_0`, no `WRITE_MAP`, sited as a sibling + directory of the chain-index database), a sequential B-tree fill. + +Schema: key = the 36-byte encoded outpoint (`txid[32] ‖ LE(u32) index`) +because that is what the query hands us; value = the bare 32-byte +spending txid; absence ⇒ unspent **within the index's built range** +`[index floor, finalised tip]` — below its floor the index asserts +nothing. The value is +deliberately **not** a `TxLocation`: that would reintroduce a dependency +on the `txids`/`txid_location` tables and break the index's isolation. + +Build model (pilot): a clean sync that re-streams blocks from the +source over `[start_height, finalised_tip]`, where +`finalised_tip = best_height − non-finalized depth` — feeding the +extractor only finalised blocks makes the build reorg-immune by +construction. No resume watermark, no backfill from on-disk block +tables, no migration; a crash reruns from the start height. The +streaming stage fans out across worker tasks pulling fixed-size height +chunks from a shared queue (chunk-pulling self-balances the block-weight +skew across the chain); collation stays one global sort feeding one +append pass, and workers never touch the store — the single-writer +discipline whose violation PR #1275 diagnosed as LMDB corruption. +There is one code path, not two: `workers = 1` *is* the serial +baseline, so serial-vs-parallel benchmarks vary only the fan-out, and +the build reports per-stage timings (stream/extract, collate, load) +plus its worker count to make every run a self-describing measurement. + +The start height — the **index floor** — is a first-class, configurable +property of the index that survives into production, not pilot +scaffolding: genesis buys full coverage; a later floor (e.g. a +network-upgrade activation) buys a cheaper build at the cost of +range-scoped answers. Serving discipline: a deployment that serves +spend queries must run a genesis floor, enforced by config validation, +so a floor-truncated answer never reaches a client; non-genesis floors +are for deployments that build without serving (pilots, tests, +experiments). The shipped configuration defaults to genesis and carries +the Sapling-activation floor only as a commented-out option in the +config file. Because absence is only meaningful relative to the built +range, a serving layer must know the index's floor, so the floor must +ultimately be persisted with the index (deferred; the pilot hardcodes +it). The pilot enters through one `spawn_build` call with the floor +pinned to Sapling activation. + +Single-loop enforcement is pushed as far into the type system as it +goes: the sync handle is move-only (`!Clone`) with a private constructor +and a `self`-consuming `run`, so duplicating the handle or running two +loops concurrently is rejected by the move checker. The residual — the +owner minting two handles — is closed by the owning `ChainIndex` making +exactly one `spawn_build` call, deliberately not by a runtime guard +(`OnceLock`/atomic). The owner holds the returned `JoinHandle` and +aborts the build on shutdown/drop. + +## Considered options + +- **Keep serving from the monolith's `spent` (`outpoint → TxLocation`) + table alone** — what `get_outpoint_spenders`' finalised leg does + today. Rejected as the end state: that table is written only by the + monolithic per-block write path, so it cannot be built, rebuilt, or + optimised in isolation, and its `TxLocation` value forces a second + resolution hop through the transaction-location tables — exactly the + coupling the bare-txid value avoids. It remains the serving path + until this index replaces the finalised leg. +- **Unconditional genesis start (no index floor).** Rejected: it would + make "absence ⇒ unspent" hold outright, but forces every deployment + to carry the full deep history whether or not it serves it. The + configurable floor keeps the build/storage cost proportional to what + a deployment actually answers for; the cost — absence is range-scoped + — is carried explicitly by the serving layer. +- **Serving with a non-genesis floor** — either falling back to the + monolith's `spent` table below the floor, or letting + `get_outpoint_spenders`' "`None` = unspent or unknown" contract + absorb the weaker answer. Both rejected: the fallback keeps the + monolith coupling alive indefinitely, and contract absorption + silently serves worse answers than the monolith's `spent` table does + today. Config validation (serving ⇒ genesis floor) prevents both. +- **Proxy zebra's `indexer` service instead of building.** Rejected: + ties a served capability to a non-default zebra feature that is + planned for removal, and forfeits both serving scale and the pilot + value for the parallel-sync architecture. +- **Value = `TxLocation` instead of bare txid.** Rejected: couples the + new index to the monolith's transaction-location tables, defeating + the isolation this pilot exists to prove. +- **Runtime single-instance guard.** Rejected in favour of the + move-only consuming handle plus a single constructor call site; + a `OnceLock`/atomic guard would paper over ownership the type system + can express. +- **Backfill from zaino's on-disk block tables.** Dropped: re-streaming + from the validator keeps the build independent of every other table + and is the model production resume will refine, not replace. +- **Immutable sorted segment (SSTable-like flat file) as the store.** + Deferred to the v2 discussion in #1326: a better long-term fit for an + append-only finalised index than LMDB's update-in-place B-tree, but + the pilot reuses the engine already in the tree. + +## Consequences + +- ≈162M entries at mainnet height ≈3.4M ⇒ roughly 13–15 GB on disk, + growing linearly with chain history; the pilot's LMDB map size is a + 32 GB lazy upper bound, not a preallocation. +- The build holds all extracted spends in memory for one global sort + and a single `MDB_APPEND` transaction; `bulk_load` is one-shot (a + second call would need every key to exceed the first call's maximum). + This is deliberate, not a pilot shortcut: the pre-materialized, + globally sorted single append pass is the speed-of-light baseline + for the sync benchmark, and at genesis floor on mainnet it peaks + around 30 GB — within a reasonably resourced build machine. A + batched variant (per-worker sorted runs, k-way merge feeding the + appender — licensed by the disjoint-key property) is a contingency + adopted only if measurement shows it beating the one-shot pass or + memory genuinely binding, not a foregone production requirement. +- The serve-time union with the non-finalised state already exists as + `get_outpoint_spenders`' `FullChain` scope (PR #1167); the pilot index + is built but not yet served. The deferred serving step is swapping + that method's finalised leg from the monolith's `spent` table to this + index — dropping the `TxLocation → txid` resolution hop — with no + change to the method's contract (it already returns bare txids). + Queried directly, the pilot index alone returns `None` for an + outpoint spent within the non-finalized depth, and for anything + spent below its floor (pilot: Sapling activation). +- With a floor above genesis, `None` from the index means "unspent or + spent below the floor". Config validation keeps that answer away from + clients — serving requires a genesis floor — so the swap needs no + fallback to the monolith's `spent` table and the isolation goal + stands. The floor must still be persisted in the index's environment + so builders, restarts, and servers agree on the built range; it is + the value the serving-side validation checks. +- `get_outpoint_spenders(Finalised)` over the monolith's `spent` table + is a second, in-tree parity oracle for this index, alongside zebra's + `SpendingTransactionId` — same chain, same process, no non-default + zebra feature required. +- Table-level integrity (the order-independent XOR-of-BLAKE2b + commitment over entries sketched in #1328) is deferred; the pilot + index has no checksum and relies on being deterministically + rebuildable. +- The pilot's acceptance bar is a mainnet run of the one-shot build on + a well-resourced machine, producing three artifacts together: the + sync-time benchmark numbers (the pilot's reason to exist), a parity + diff against zebra's `SpendingTransactionId`, and golden vectors — + which must in any case be captured while zebra still ships the + `indexer` feature. A synthetic-chain parity test runs first, to prove + the oracle plumbing before spending a mainnet stream on it. +- Generalising the extractor/collator decomposition to value- and + script-bearing indexes (txout-set accumulator, address history) — + which need a shared live-UTXO primary — remains with #1326. diff --git a/docs/testing.md b/docs/testing.md index f5c6d8f10..9bfd24575 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -26,14 +26,14 @@ The `makers` tasks below build and run the test suites inside a **podman** container, so you don't need the validator binaries on your host `$PATH`. The container image is built or pulled automatically on first run. -The single front door is `makers test [SET]`, where `SET` defaults to `package`: +The single front door is `makers test [SET]`, where `SET` defaults to `packages`: -- `makers test` (or `makers test package`) — the **package** set: the +- `makers test` (or `makers test packages`) — the **packages** set: the `packages/*` production-crate tests that need no live validator. - `makers test e2e` / `makers test clientless` — one live partition. - `makers test live` — both live partitions (`clientless` then `e2e`) against a live validator, with a combined pass/fail summary. -- `makers test all` — the whole suite: package then live. +- `makers test all` — the whole suite: packages then live. (`container-test`, `live-clientless`, and `live-e2e` are the internal engines the `test` front door delegates to; invoke them directly only when you need to diff --git a/live-tests/CONTEXT.md b/live-tests/CONTEXT.md index d00e0f064..1bef4076e 100644 --- a/live-tests/CONTEXT.md +++ b/live-tests/CONTEXT.md @@ -40,15 +40,17 @@ owns shared fixtures and the feature-forwarding surface (`zcashd_support`, `transparent_address_history_experimental`). _Avoid_: test helpers, testlib. -**package (test set)**: +**packages (test set)**: The tests of the `packages/*` production crates — the workspace `default-members`, exercised by bare `cargo nextest run` and by `makers test` -(the default with no argument, equivalently `makers test package`). Named for +(the default with no argument, equivalently `makers test packages`). Named for where the crates live (`packages/`); the functional reason they run apart from the live partitions is that they need **no** live validator. They are *not* network-free, though: e.g. zaino-serve's gRPC `spawn` regression test binds a loopback socket and stands up a tonic server. The absent ingredient is a validator, not the network. -_Avoid_: offline (overclaims "no network" — these tests bind loopback sockets); -unit test (the set includes crate-level integration tests); container test -(names the run mechanism, which the live suite shares). +_Avoid_: package (singular — the set is plural, matching the `packages/` dir; +`makers test package` now errors with `unknown set`); offline (overclaims "no +network" — these tests bind loopback sockets); unit test (the set includes +crate-level integration tests); container test (names the run mechanism, which +the live suite shares). diff --git a/packages/zaino-state/Cargo.toml b/packages/zaino-state/Cargo.toml index 4f96dbe9f..493e14870 100644 --- a/packages/zaino-state/Cargo.toml +++ b/packages/zaino-state/Cargo.toml @@ -32,13 +32,21 @@ test_dependencies = [] # **Experimental and alpha features** # Exposes the **complete** set of experimental / alpha features currently implemented in Zaino. -experimental_features = ["transparent_address_history_experimental"] +experimental_features = [ + "transparent_address_history_experimental", + "outp_to_spend_index", +] # Activates transparent address history capability in zaino # # NOTE: currently this is only implemented in the finalised state. transparent_address_history_experimental = [] +# Standalone, parallel-buildable finalised spend index +# (transparent_outpoint -> spending_txid). Proof of concept; off by default. +# See docs/adr/0006-finalised-spend-index-parallel-build.md. +outp_to_spend_index = [] + [dependencies] zaino-common = { workspace = true } zaino-fetch = { workspace = true } @@ -100,6 +108,9 @@ tempfile = { workspace = true } tracing-subscriber = { workspace = true } once_cell = { workspace = true } zebra-chain = { workspace = true, features = ["proptest-impl"] } +# `indexer` exposes zebra's SpendingTransactionId oracle; `proptest-impl` +# exposes `populated_state`. Dev-only: off in production builds. +zebra-state = { workspace = true, features = ["indexer", "proptest-impl"] } proptest.workspace = true incrementalmerkletree = "0.8.2" rand = "0.10.1" diff --git a/packages/zaino-state/src/chain_index.rs b/packages/zaino-state/src/chain_index.rs index 87e0d7f68..c67fcc746 100644 --- a/packages/zaino-state/src/chain_index.rs +++ b/packages/zaino-state/src/chain_index.rs @@ -703,6 +703,14 @@ pub struct NodeBackedChainIndex { non_finalized_state: Arc>>, finalized_db: std::sync::Arc>, sync_loop_handle: Option>>, + /// Handle to the one-shot finalised spend-index build, if one was spawned. + /// Aborted on shutdown/drop so the build never outlives the index. + #[cfg(feature = "outp_to_spend_index")] + spend_index_build: Option< + tokio::task::JoinHandle< + Result, + >, + >, status: NamedAtomicStatus, network: ZebraNetwork, source: Source, @@ -799,6 +807,8 @@ impl NodeBackedChainIndex { non_finalized_state: Arc::new(ArcSwapOption::empty()), finalized_db, sync_loop_handle: None, + #[cfg(feature = "outp_to_spend_index")] + spend_index_build: None, status: NamedAtomicStatus::new("ChainIndex", StatusType::Spawning), network: config.network.to_zebra_network(), source, @@ -807,6 +817,14 @@ impl NodeBackedChainIndex { }; chain_index.sync_loop_handle = Some(chain_index.start_sync_loop()); + #[cfg(feature = "outp_to_spend_index")] + { + chain_index.spend_index_build = chain_index + .source + .finalised_spend_index_source() + .map(|source| finalised_state::outp_to_spend_index::spawn_build(source, &config)); + } + Ok(chain_index) } @@ -837,6 +855,10 @@ impl NodeBackedChainIndex { /// failure-path round trip. pub async fn shutdown(&self) -> Result<(), FinalisedStateError> { self.cancel_token.cancel(); + #[cfg(feature = "outp_to_spend_index")] + if let Some(handle) = &self.spend_index_build { + handle.abort(); + } self.status.store(StatusType::Closing); self.finalized_db.shutdown().await?; self.mempool.close(); @@ -1067,6 +1089,10 @@ impl Drop for NodeBackedChainIndex { /// exits at its next await checkpoint instead. fn drop(&mut self) { self.cancel_token.cancel(); + #[cfg(feature = "outp_to_spend_index")] + if let Some(handle) = &self.spend_index_build { + handle.abort(); + } } } diff --git a/packages/zaino-state/src/chain_index/CONTEXT.md b/packages/zaino-state/src/chain_index/CONTEXT.md index 37634b70a..17544b2e3 100644 --- a/packages/zaino-state/src/chain_index/CONTEXT.md +++ b/packages/zaino-state/src/chain_index/CONTEXT.md @@ -32,3 +32,17 @@ blocks. ## Eviction Removal of a block from the NFS once the finalized floor rises past its height. A block is evicted when it passes below the seam. + +## Index floor +The lowest height a given finalized index covers, chosen per index at +build time. The index asserts nothing about chain activity below its +floor: an absent key means "no such entry within the built range", not +"none on the whole chain". Distinct from the finalized floor, which is +chain-wide and delimits NFS eviction. + +## Spending transaction +For a transparent outpoint, the transaction that consumes it — the one +listing the outpoint as an input's prevout. Distinct from the outpoint's +creating transaction, which the outpoint itself already names in its +first field. A spend index maps outpoints to spending transactions, +never to creating ones. diff --git a/packages/zaino-state/src/chain_index/finalised_state.rs b/packages/zaino-state/src/chain_index/finalised_state.rs index 575ba1e5b..28be49747 100644 --- a/packages/zaino-state/src/chain_index/finalised_state.rs +++ b/packages/zaino-state/src/chain_index/finalised_state.rs @@ -218,6 +218,8 @@ pub(crate) mod capability; pub(crate) mod entry; pub(crate) mod finalised_source; pub(crate) mod migrations; +#[cfg(feature = "outp_to_spend_index")] +pub(crate) mod outp_to_spend_index; pub(crate) mod reader; pub(crate) mod router; diff --git a/packages/zaino-state/src/chain_index/finalised_state/capability.rs b/packages/zaino-state/src/chain_index/finalised_state/capability.rs index 0193c5d69..39becec45 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/capability.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/capability.rs @@ -964,7 +964,7 @@ pub trait IndexedBlockExt: Send + Sync { /// Range semantics: /// - Methods that accept `start_height` and `end_height` interpret the range as inclusive: /// `[start_height, end_height]` -pub trait TransparentHistExt: Send + Sync { +pub(crate) trait TransparentHistExt: Send + Sync { /// Fetch all address history records for a given transparent address. /// /// Returns: diff --git a/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs b/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs new file mode 100644 index 000000000..c9da99941 --- /dev/null +++ b/packages/zaino-state/src/chain_index/finalised_state/outp_to_spend_index.rs @@ -0,0 +1,1037 @@ +//! Standalone, parallel-buildable finalised spend index: maps each transparent +//! outpoint to the txid of the transaction that consumed it. +//! +//! Proof of concept, gated behind the `outp_to_spend_index` feature. See +//! `docs/adr/0006-finalised-spend-index-parallel-build.md`. +//! +//! The build stages live here: **extract** spends from a block batch, +//! **collate** them into LMDB key order, and bulk-load them into the index's +//! own LMDB store via `MDB_APPEND` ([`SpendIndexDb`]). The independent, +//! move-only sync loop ([`SpendIndexSync`]) drives these in one shot over +//! `[start_height, finalised_tip]` streamed from a [`SpendIndexSource`]: +//! block sourcing/extraction fans out across worker tasks pulling fixed-size +//! height chunks from a shared queue (`workers = 1` is the serial baseline — +//! the same path minus concurrency, so serial and parallel builds compare +//! stage-for-stage), and collation stays one global sort feeding one +//! `MDB_APPEND` pass. [`spawn_build`] wires it onto the owning `ChainIndex` +//! (StateService only, from the Sapling activation height). (Table-level +//! integrity over the entries is deferred; it is not MVP.) + +use std::path::Path; + +use lmdb::{Database, DatabaseFlags, Environment, EnvironmentFlags, Transaction as _, WriteFlags}; + +// Test-only: the presence test builds its expected values through the +// compact-form path the production build no longer uses. +#[cfg(test)] +use crate::chain_index::finalised_state::build_indexed_block_from_source; +use crate::chain_index::source::validator_connector::StateSource; +use crate::chain_index::source::BlockchainSource; +use crate::chain_index::types::TransactionHash; +use crate::chain_index::OPERATIONAL_NFS_DEPTH; +use crate::config::ChainIndexConfig; +use crate::error::FinalisedStateError; +use crate::{IndexedBlock, Outpoint, TransparentCompactTx, ZainoVersionedSerde as _}; +#[cfg(test)] +use zaino_common::Network; +use zebra_chain::parameters::NetworkUpgrade; + +/// One transparent spend: the consumed outpoint paired with the txid of the +/// transaction that consumed it. +pub(super) type SpendRecord = (Outpoint, TransactionHash); + +/// A spend entry encoded for storage: the LMDB key (the encoded outpoint) and +/// value (the bare 32-byte spending txid), ready for an `MDB_APPEND` load. +pub(super) type EncodedSpend = (Vec, [u8; 32]); + +// ── Extract ────────────────────────────────────────────────────────────────── + +/// Extracts every transparent spend recorded in `blocks`. +/// +/// Pure and statically read-free: it is handed only block data — no database +/// and no validator handle — so a previous-output lookup is unrepresentable, +/// not merely discouraged. Each transparent input yields +/// `(prevout_outpoint, spending_txid)` directly, where the spending txid is the +/// containing transaction's own id; nothing outside the block stream is +/// consulted. +/// +/// TODO (loop slice): add a buffer-filling variant that writes into a +/// caller-owned `&mut Vec`, once the per-worker loop exists to +/// reuse one allocation across batches (`Vec::clear` keeps capacity ⇒ ~zero +/// steady-state alloc, and the collator can sort that buffer in place). +/// Deferred until a real reusing caller anchors it: the extractor is dwarfed by +/// zebra block I/O and the sort, so buffer reuse only pays then. +pub(super) fn extract_spends(blocks: &[IndexedBlock]) -> Vec { + blocks + .iter() + .flat_map(|block| block.transactions().iter()) + .flat_map(|tx| spends_in_transaction(*tx.txid(), tx.transparent())) + .collect() +} + +/// The spends contributed by one transaction: each outpoint it consumes paired +/// with `spending_txid`. The non-coinbase filtering lives in +/// [`TransparentCompactTx::spent_outpoints`]; this only attaches the spender. +fn spends_in_transaction( + spending_txid: TransactionHash, + transparent: &TransparentCompactTx, +) -> impl Iterator + '_ { + transparent + .spent_outpoints() + .map(move |outpoint| (outpoint, spending_txid)) +} + +// ── Collate ────────────────────────────────────────────────────────────────── + +/// Encodes and sorts `records` into LMDB key order — byte-wise on the encoded +/// outpoint key, matching LMDB's default comparator — so the result can be +/// bulk-loaded with `MDB_APPEND`. +/// +/// Spend keys are globally disjoint — each outpoint is spent at most once on a +/// chain — so this is a pure sort needing no cross-record reconciliation; a +/// duplicate key means corrupt input and is rejected. +pub(super) fn collate(records: &[SpendRecord]) -> Result, FinalisedStateError> { + let mut encoded = records + .iter() + .map(|(outpoint, spending_txid)| { + Ok((outpoint.to_bytes()?, <[u8; 32]>::from(*spending_txid))) + }) + .collect::, FinalisedStateError>>()?; + + encoded.sort_by(|a, b| a.0.cmp(&b.0)); + + if let Some(duplicate) = encoded.windows(2).find(|pair| pair[0].0 == pair[1].0) { + return Err(FinalisedStateError::Custom(format!( + "duplicate spend-index key during collation: {:?}", + duplicate[0].0 + ))); + } + + Ok(encoded) +} + +// ── Store ──────────────────────────────────────────────────────────────────── + +/// LMDB database name for the spend index (versioned, mirroring the +/// finalised-state naming convention). +const SPEND_DB_NAME: &str = "outp_to_spend_index_1_0_0"; + +/// The spend index's **own** LMDB store: a single database keyed by the encoded +/// outpoint, valued by the bare 32-byte spending txid. Standalone — its own +/// `Environment`, independent of the finalised-state monolith's database. +pub(super) struct SpendIndexDb { + env: Environment, + spend: Database, +} + +impl SpendIndexDb { + /// Opens (creating if absent) the spend index at `path` with the given LMDB + /// `map_size`. Mirrors the finalised-state environment: **no `WRITE_MAP`** + /// (a crash discards uncommitted data rather than corrupting), with + /// reader-friendly flags. `path` is a directory, created if missing. + pub(super) fn open(path: &Path, map_size: usize) -> Result { + std::fs::create_dir_all(path).map_err(|error| { + FinalisedStateError::Custom(format!( + "spend index: create dir {}: {error}", + path.display() + )) + })?; + let env = Environment::new() + .set_max_dbs(1) + .set_map_size(map_size) + .set_flags(EnvironmentFlags::NO_TLS | EnvironmentFlags::NO_READAHEAD) + .open(path)?; + let spend = env.create_db(Some(SPEND_DB_NAME), DatabaseFlags::empty())?; + Ok(Self { env, spend }) + } + + /// Bulk-loads collated (sorted, disjoint-key) entries with `MDB_APPEND` — a + /// sequential B-tree fill — in one transaction, then forces durability. + /// `entries` must be in ascending key order (see [`collate`]); a key out of + /// order or already present makes LMDB reject the append. + pub(super) fn bulk_load(&self, entries: &[EncodedSpend]) -> Result<(), FinalisedStateError> { + // One-shot / global-order contract: `entries` must be strictly ascending + // by key — `collate`'s output for *all* spends at once. `MDB_APPEND` + // requires every key to exceed the current maximum, so there is no + // re-sort across calls: a second `bulk_load` would need every key to + // exceed the first call's maximum. LMDB self-checks this at runtime (it + // rejects a non-increasing key with `KeyExist`); this assert fails earlier + // and with a clearer message in debug builds if a caller hands over an + // unsorted, duplicate, or per-batch (not globally sorted) input. + debug_assert!( + entries.windows(2).all(|pair| pair[0].0 < pair[1].0), + "bulk_load: entries must be strictly ascending, globally-sorted keys \ + (MDB_APPEND is one-shot — feed all spends through a single collate)", + ); + let mut txn = self.env.begin_rw_txn()?; + for (key, value) in entries { + txn.put(self.spend, key, value, WriteFlags::APPEND)?; + } + txn.commit()?; + self.env.sync(true)?; + Ok(()) + } + + /// The txid that consumed `outpoint`, or `None` if it is unspent in + /// finalised state. A single point lookup. The serve-time union with the + /// non-finalised window already exists as + /// `ChainIndex::get_outpoint_spenders` (`ChainScope::FullChain`, #1167); + /// wiring this store in as that method's finalised leg — replacing the + /// monolith's `spent` table + `TxLocation → txid` resolution — is the + /// deferred serving step. + pub(super) fn spending_txid( + &self, + outpoint: &Outpoint, + ) -> Result, FinalisedStateError> { + let key = outpoint.to_bytes()?; + let ro = self.env.begin_ro_txn()?; + match ro.get(self.spend, &key) { + Ok(bytes) => { + let array: [u8; 32] = bytes.try_into().map_err(|_| { + FinalisedStateError::Custom( + "spend index: stored value is not a 32-byte txid".to_string(), + ) + })?; + Ok(Some(TransactionHash::from(array))) + } + Err(lmdb::Error::NotFound) => Ok(None), + Err(error) => Err(error.into()), + } + } +} + +// ── Sync loop ──────────────────────────────────────────────────────────────── + +/// The sources the spend-index POC may build from: zebra's StateService +/// ([`StateSource`]) and the test mockchain — **never** the JSON-RPC/zcashd +/// `FetchService`. Binding [`SpendIndexSync`] to `S: SpendIndexSource` makes +/// "never FetchService" a compile-time fact: the `Fetch` path does not +/// implement this trait. +pub(super) trait SpendIndexSource: BlockchainSource {} + +impl SpendIndexSource for StateSource {} + +#[cfg(test)] +impl SpendIndexSource for crate::chain_index::source::mockchain_source::MockchainSource {} + +/// Stage timings and volumes from one one-shot build: the benchmark's +/// measurement surface. `stream_and_extract` covers the sequential +/// fetch → decode → extract loop — the stage a parallel build transforms — +/// while `collate` and `bulk_load` are identical across build variants, so +/// comparing two builds' stats isolates the streaming stage. +#[derive(Debug, Default, Clone)] +pub(crate) struct SpendIndexBuildStats { + /// Worker tasks the streaming stage fanned out across (`1` = the serial + /// baseline), recorded so every measurement is self-describing. + pub(crate) workers: u32, + /// Blocks fetched and scanned. + pub(crate) blocks: u32, + /// Spend records extracted (= entries loaded). + pub(crate) spends: usize, + /// Wall-clock of the fetch → decode → extract loop. + pub(crate) stream_and_extract: std::time::Duration, + /// Wall-clock of the global encode + sort + duplicate check. + pub(crate) collate: std::time::Duration, + /// Wall-clock of the `MDB_APPEND` load and durability sync. + pub(crate) bulk_load: std::time::Duration, +} + +/// The independent, single-run builder for the spend index. +/// +/// Move-only (`!Clone`) with a private constructor and a `self`-consuming +/// [`run`](Self::run): the type system makes a second concurrent build +/// unrepresentable — "only one loop at a time" without a runtime guard. (The +/// guarantee constrains *builds*; the fan-out of worker tasks inside one +/// `run` is not a second build.) POC build model: re-stream `start_height` → +/// finalised tip from `source` across `workers` chunk-pulling tasks, extract +/// every spend, collate once, and bulk-load the index in a single +/// `MDB_APPEND` pass. +pub(super) struct SpendIndexSync { + source: S, + db: SpendIndexDb, + /// First height to index, inclusive. Genesis (`0`) for a full index, or a + /// higher height — e.g. a network-upgrade activation — to cover only spends + /// occurring from that epoch onward. + start_height: u32, + /// Benchmark knob: when set, the build stops at + /// `min(finalised_tip, cap)` instead of the finalised tip, so a baseline + /// run can cover a fixed block range. `None` (production) builds to the + /// finalised tip. + end_height_cap: Option, + /// Worker tasks for the streaming stage. Defaults to the machine's + /// available parallelism; `1` selects the serial baseline (the identical + /// path minus concurrency), which is what parallel runs are measured + /// against. + workers: usize, +} + +impl SpendIndexSync { + /// Private mint: production enters via [`SpendIndexSync::from_state`] + /// (StateService only); tests construct directly with a `MockchainSource`. + fn new(source: S, db: SpendIndexDb, start_height: u32) -> Self { + Self { + source, + db, + start_height, + end_height_cap: None, + workers: std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get), + } + } + + /// Builds the spend index from its start height to the finalised tip, + /// consuming the handle and returning the per-stage timings. One-shot + /// (POC): no resume, no batching — a single global `collate` + + /// `MDB_APPEND`. + pub(super) async fn run(self) -> Result { + let Some(best) = self.source.get_best_block_height().await.map_err(|error| { + FinalisedStateError::Custom(format!("spend index: fetch best height: {error:?}")) + })? + else { + return Ok(SpendIndexBuildStats::default()); // empty chain — nothing finalised to index + }; + + // Finalised tip: below the reorg-possible window, clamped at genesis. + // A benchmark cap lowers it further so a run covers a fixed range. + let finalised_tip = best.0.saturating_sub(OPERATIONAL_NFS_DEPTH); + let finalised_tip = self + .end_height_cap + .map_or(finalised_tip, |cap| finalised_tip.min(cap)); + + // Stream finalised blocks across the workers, extracting spends and + // dropping each block; only the (smaller) spend records are retained + // for the one-shot sort. Fixed-size chunks pulled from a shared queue + // self-balance the block-weight skew across the chain (late blocks + // are far heavier than early ones), so equal-width per-worker ranges + // are not needed. + let workers = self.workers.max(1); + let chunks = + std::sync::Arc::new(chunk_ranges(self.start_height, finalised_tip, CHUNK_BLOCKS)); + let next_chunk = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + + let mut stats = SpendIndexBuildStats { + workers: workers as u32, + blocks: chunks.iter().map(|(lo, hi)| hi - lo + 1).sum(), + ..Default::default() + }; + + let stream_started = std::time::Instant::now(); + let mut handles = Vec::with_capacity(workers); + for _ in 0..workers { + handles.push(tokio::spawn(extract_chunks( + self.source.clone(), + std::sync::Arc::clone(&chunks), + std::sync::Arc::clone(&next_chunk), + ))); + } + let mut spends: Vec = Vec::new(); + for handle in handles { + let worker_records = handle.await.map_err(|error| { + FinalisedStateError::Custom(format!("spend index: worker task died: {error}")) + })??; + spends.extend(worker_records); + } + stats.stream_and_extract = stream_started.elapsed(); + stats.spends = spends.len(); + + let collate_started = std::time::Instant::now(); + let collated = collate(&spends)?; + stats.collate = collate_started.elapsed(); + + let load_started = std::time::Instant::now(); + tokio::task::block_in_place(|| self.db.bulk_load(&collated))?; + stats.bulk_load = load_started.elapsed(); + + Ok(stats) + } +} + +/// Blocks per work-queue chunk. Small relative to any real build range so +/// chunk-pulling self-balances the block-weight skew, large enough that queue +/// bookkeeping (one relaxed `fetch_add` per chunk) is noise. +const CHUNK_BLOCKS: u32 = 1_000; + +/// Splits inclusive `[start, end]` into consecutive inclusive ranges of at +/// most `size` blocks. Empty when `start > end`. +fn chunk_ranges(start: u32, end: u32, size: u32) -> Vec<(u32, u32)> { + let mut ranges = Vec::new(); + let mut lo = start; + while lo <= end { + let hi = lo.saturating_add(size - 1).min(end); + ranges.push((lo, hi)); + match hi.checked_add(1) { + Some(next) => lo = next, + None => break, // end == u32::MAX: the final range is pushed + } + } + ranges +} + +/// One streaming worker: pulls chunk indices from the shared queue until it +/// drains, fetching each height and extracting its spends into a +/// worker-local buffer (returned for the caller to concatenate). +/// +/// Roots-free: the per-block cost is one `get_block` plus a +/// transparent-input walk — no commitment-tree-root fetch and no compact +/// conversion of the (discarded) shielded data. The monolith's per-block +/// ingestion awaits `get_block` *and* `get_commitment_tree_roots` in +/// sequence, the fetch-latency pattern PR #1241 measured collapsing to +/// ~1 blk/s in the sandblast band; the spend index never needs the second +/// call. +async fn extract_chunks( + source: S, + chunks: std::sync::Arc>, + next_chunk: std::sync::Arc, +) -> Result, FinalisedStateError> { + let mut records = Vec::new(); + loop { + let index = next_chunk.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let Some(&(lo, hi)) = chunks.get(index) else { + return Ok(records); + }; + for height in lo..=hi { + let block = source + .get_block(zebra_state::HashOrHeight::Height( + zebra_chain::block::Height(height), + )) + .await? + .ok_or_else(|| { + FinalisedStateError::Custom(format!( + "spend index: no block at finalised height {height}" + )) + })?; + records.extend(extract_spends_from_zebra_block(&block)); + } + } +} + +/// Extracts every transparent spend in one zebra block: each non-coinbase +/// input yields `(prevout_outpoint, containing_txid)`. Statically read-free — +/// handed only block data — like [`extract_spends`], its compact-form twin; +/// the sync-loop tests assert both paths agree over the same chains. +fn extract_spends_from_zebra_block( + block: &zebra_chain::block::Block, +) -> impl Iterator + '_ { + block.transactions.iter().flat_map(|tx| { + let spending_txid = TransactionHash::from(tx.hash().0); + tx.inputs().iter().filter_map(move |input| { + input + .outpoint() + .map(|prevout| (Outpoint::new(prevout.hash.0, prevout.index), spending_txid)) + }) + }) +} + +/// Map size for the spend index's own LMDB env. LMDB grows the file lazily (no +/// `WRITE_MAP` here), so this is an upper bound, not a preallocation. +const SPEND_INDEX_MAP_SIZE: usize = 32 * 1024 * 1024 * 1024; + +/// The spend index's on-disk location: a sibling of the configured chain-index +/// database directory. +fn spend_index_dir(cfg: &ChainIndexConfig) -> std::path::PathBuf { + let main = &cfg.storage.database.path; + match main.parent() { + Some(parent) => parent.join("outp_to_spend_index"), + None => main.join("outp_to_spend_index"), + } +} + +/// Spawns the one-shot finalised spend-index build as a background task. +/// +/// The build streams `[Sapling activation, finalised tip]` from `source` — the +/// zebra StateService, supplied by +/// [`BlockchainSource::finalised_spend_index_source`] — into the index's own +/// LMDB env. The task logs its outcome; the owning `ChainIndex` holds the +/// returned handle and aborts it on shutdown/drop. +pub(crate) fn spawn_build( + source: StateSource, + cfg: &ChainIndexConfig, +) -> tokio::task::JoinHandle> { + let network = cfg.network; + let dir = spend_index_dir(cfg); + tokio::spawn(async move { + let start_height = NetworkUpgrade::Sapling + .activation_height(&network.to_zebra_network()) + .ok_or_else(|| { + FinalisedStateError::Custom("network has no Sapling activation height".to_string()) + })? + .0; + let db = SpendIndexDb::open(&dir, SPEND_INDEX_MAP_SIZE)?; + let mut sync = SpendIndexSync::new(source, db, start_height); + if let Some(start) = parsed_env::(BENCH_START_HEIGHT_ENV) { + tracing::warn!( + start, + "spend-index build start overridden via {BENCH_START_HEIGHT_ENV} (benchmark knob)", + ); + sync.start_height = start; + } + if let Some(cap) = parsed_env::(BENCH_END_HEIGHT_ENV) { + tracing::warn!( + cap, + "spend-index build height-capped via {BENCH_END_HEIGHT_ENV} (benchmark knob)", + ); + sync.end_height_cap = Some(cap); + } + if let Some(workers) = parsed_env::(WORKERS_ENV) { + tracing::warn!( + workers, + "spend-index build worker count set via {WORKERS_ENV} (1 = serial baseline)", + ); + sync.workers = workers.max(1); + } + let result = sync.run().await; + match &result { + Ok(stats) => tracing::info!(?stats, "finalised spend-index build complete"), + Err(error) => tracing::error!("finalised spend-index build failed: {error}"), + } + result + }) +} + +/// Benchmark env knobs. Each is ignored when unset and announced with a +/// `warn!` when active, so a bounded or tuned build can't pass silently for +/// a production one. Start/end bound the built range (e.g. a fixed 100k-block +/// window); the worker knob selects the streaming fan-out, with `1` the +/// serial baseline that parallel runs are measured against. +const BENCH_START_HEIGHT_ENV: &str = "ZAINO_SPEND_INDEX_START_HEIGHT"; +const BENCH_END_HEIGHT_ENV: &str = "ZAINO_SPEND_INDEX_END_HEIGHT"; +const WORKERS_ENV: &str = "ZAINO_SPEND_INDEX_WORKERS"; + +/// The parsed value of env knob `name`, or `None` when unset or unparseable +/// (the latter is logged, not fatal — a benchmark knob must not take the +/// production build down). +fn parsed_env(name: &str) -> Option +where + T::Err: std::fmt::Display, +{ + let raw = std::env::var(name).ok()?; + match raw.parse() { + Ok(value) => Some(value), + Err(error) => { + tracing::warn!(%raw, "ignoring unparseable {name}: {error}"); + None + } + } +} + +#[cfg(test)] +mod spends_in_transaction { + use super::*; + use crate::TxInCompact; + + /// Arbitrary fill byte for the spending transaction's txid, kept distinct + /// from the prevout txids below so the two can't be confused. + const SPENDER_TXID_BYTE: u8 = 9; + + fn txid(byte: u8) -> TransactionHash { + TransactionHash::from([byte; 32]) + } + + #[test] + fn skips_coinbase_input_keeps_real_spends() { + let spender = txid(SPENDER_TXID_BYTE); + let transparent = TransparentCompactTx::new( + vec![ + TxInCompact::null_prevout(), // coinbase input → contributes nothing + TxInCompact::new([1u8; 32], 0), // spends output 0 of txid 0x01.. + TxInCompact::new([2u8; 32], 7), // spends output 7 of txid 0x02.. + ], + vec![], + ); + + let records: Vec = + super::spends_in_transaction(spender, &transparent).collect(); + + assert_eq!( + records, + vec![ + (Outpoint::new([1u8; 32], 0), spender), + (Outpoint::new([2u8; 32], 7), spender), + ] + ); + } +} + +#[cfg(test)] +mod collate { + use super::*; + + fn record(outpoint_byte: u8, index: u32) -> SpendRecord { + ( + Outpoint::new([outpoint_byte; 32], index), + TransactionHash::from([0xaau8; 32]), + ) + } + + #[test] + fn sorts_into_ascending_key_order() { + let records = [record(3, 0), record(1, 0), record(2, 0)]; + let encoded = super::collate(&records).expect("disjoint keys collate cleanly"); + assert!( + encoded.windows(2).all(|pair| pair[0].0 < pair[1].0), + "collated keys must be strictly ascending for MDB_APPEND", + ); + } + + #[test] + fn rejects_duplicate_keys() { + let records = [record(1, 0), record(1, 0)]; + assert!(super::collate(&records).is_err()); + } + + #[test] + fn little_endian_index_tie_break_matches_byte_order() { + // Same creating txid, two output indices. The index is LE-encoded, so + // 256 (bytes `00 01 00 00`) sorts before 1 (`01 00 00 00`) under memcmp + // — exactly the order LMDB's default comparator (and thus MDB_APPEND) + // requires. collate must reproduce that, not a numeric ordering. + let encoded = + super::collate(&[record(7, 1), record(7, 256)]).expect("disjoint keys collate cleanly"); + + assert!( + encoded[0].0 < encoded[1].0, + "keys must be strictly ascending for MDB_APPEND", + ); + // The trailing LE u32 of each key shows index 256 precedes index 1. + assert_eq!( + &encoded[0].0[encoded[0].0.len() - 4..], + &256u32.to_le_bytes()[..] + ); + assert_eq!( + &encoded[1].0[encoded[1].0.len() - 4..], + &1u32.to_le_bytes()[..] + ); + } +} + +#[cfg(test)] +mod chunk_ranges { + #[test] + fn covers_the_range_exactly_once_with_ragged_tail() { + assert_eq!( + super::chunk_ranges(10, 34, 10), + vec![(10, 19), (20, 29), (30, 34)], + ); + } + + #[test] + fn exact_multiple_has_no_ragged_tail() { + assert_eq!(super::chunk_ranges(0, 19, 10), vec![(0, 9), (10, 19)]); + } + + #[test] + fn single_block_range_is_one_chunk() { + assert_eq!(super::chunk_ranges(7, 7, 10), vec![(7, 7)]); + } + + #[test] + fn empty_when_start_exceeds_end() { + assert!(super::chunk_ranges(8, 7, 10).is_empty()); + } + + #[test] + fn end_at_u32_max_terminates() { + let ranges = super::chunk_ranges(u32::MAX - 5, u32::MAX, 4); + assert_eq!( + ranges, + vec![(u32::MAX - 5, u32::MAX - 2), (u32::MAX - 1, u32::MAX)], + ); + } +} + +#[cfg(test)] +mod spend_index_db { + use super::*; + + #[test] + fn bulk_load_then_point_read_round_trips() { + let dir = std::env::temp_dir().join(format!( + "outp_to_spend_index_roundtrip_{}", + std::process::id() + )); + let _ = std::fs::remove_dir_all(&dir); + + let db = SpendIndexDb::open(&dir, 1 << 20).expect("open spend index"); + + let spent_a = Outpoint::new([1u8; 32], 0); + let spent_b = Outpoint::new([2u8; 32], 7); + let spender = TransactionHash::from([9u8; 32]); + + let collated = super::collate(&[(spent_a, spender), (spent_b, spender)]).expect("collate"); + db.bulk_load(&collated).expect("bulk load"); + + assert_eq!(db.spending_txid(&spent_a).expect("read a"), Some(spender)); + assert_eq!(db.spending_txid(&spent_b).expect("read b"), Some(spender)); + // An outpoint with no recorded spend reads back as unspent. + let unspent = Outpoint::new([3u8; 32], 0); + assert_eq!(db.spending_txid(&unspent).expect("read unspent"), None); + + std::fs::remove_dir_all(&dir).ok(); + } +} + +#[cfg(test)] +mod spend_index_sync { + use super::*; + use crate::chain_index::source::mockchain_source::MockchainSource; + use crate::chain_index::tests::vectors::{ + build_active_mockchain_source, indexed_block_chain, load_test_vectors, TestVectorBlockData, + }; + + fn fixture_blocks() -> Vec { + load_test_vectors().expect("load test vectors").blocks + } + + /// Builds the spend index from `mock` into a fresh temp dir, returning a + /// reader handle to what was persisted plus the dir to clean up. `run` + /// consumes its writer handle, so the db is reopened for reading. + /// + /// `multi_thread` is required of the callers: `run` uses + /// `tokio::task::block_in_place` for the LMDB write, which panics on a + /// current-thread runtime. The network only drives the block builder's + /// commitment-root activation checks (the mockchain supplies roots + /// regardless) and is irrelevant to transparent-spend extraction. + async fn build_index(mock: MockchainSource, tag: &str) -> (SpendIndexDb, std::path::PathBuf) { + let dir = + std::env::temp_dir().join(format!("outp_to_spend_index_{tag}_{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let db = SpendIndexDb::open(&dir, 1 << 24).expect("open spend index"); + SpendIndexSync::new(mock, db, 0) + .run() + .await + .expect("spend-index build runs"); + ( + SpendIndexDb::open(&dir, 1 << 24).expect("reopen spend index"), + dir, + ) + } + + /// `run` must index *exactly* the transparent spends in the finalised range + /// `0..=finalised_tip`: every finalised spend mapped to its spender, every + /// non-finalised spend absent. + /// + /// Coverage note: the 201-block fixture caps `get_best_block_height` at 200 + /// (the mockchain reveals loaded blocks, it cannot mint new ones), so at the + /// `#[cfg(test)]` depth of 100 the finalised tip is pinned at 100. Regtest + /// coinbase maturity (100 blocks) puts every fixture spend at height ≥101, so + /// the finalised range holds no spends and this test exercises the exclusion + /// direction end to end (fetch → finalised boundary → empty collate → LMDB → + /// read-back). Finalised-spend *mapping* with real records is covered by the + /// `spend_index_db` round-trip test; an end-to-end presence check needs a + /// longer synthetic chain (tracked in zingolabs/zaino#1334). + #[tokio::test(flavor = "multi_thread")] + async fn run_indexes_exactly_the_finalised_spends() { + let blocks = fixture_blocks(); + let chain_top = blocks.last().expect("non-empty chain").height; + // No mining: the mock's tip is its max loaded height, so this matches + // `run`'s own `best - OPERATIONAL_NFS_DEPTH`. + let finalised_tip = chain_top - OPERATIONAL_NFS_DEPTH; + + let mock = build_active_mockchain_source(chain_top, blocks.clone()); + let (db, dir) = build_index(mock, "run").await; + + let mut saw_non_finalised_spend = false; + for block in indexed_block_chain(&blocks) { + let finalised = block.context.index.height.0 <= finalised_tip; + for (outpoint, spending_txid) in extract_spends(std::slice::from_ref(&block)) { + let indexed = db.spending_txid(&outpoint).expect("read spend"); + if finalised { + assert_eq!( + indexed, + Some(spending_txid), + "finalised spend {outpoint:?} must be indexed to its spender", + ); + } else { + saw_non_finalised_spend = true; + assert_eq!( + indexed, None, + "non-finalised spend {outpoint:?} must not be indexed", + ); + } + } + } + assert!( + saw_non_finalised_spend, + "fixture should contain spends above the seam to exercise exclusion", + ); + + // A never-spent outpoint reads back as unspent. + assert_eq!( + db.spending_txid(&Outpoint::new([0xff; 32], u32::MAX)) + .expect("read unspent"), + None, + ); + + std::fs::remove_dir_all(&dir).ok(); + } +} + +/// Deterministic synthetic-chain generation shared by the presence and +/// zebra-parity tests. The 201-block fixture can't put a transparent spend +/// below the depth-100 test seam (coinbase maturity forbids it), so these +/// chains come from zebra's block generator, whose +/// `allow_all_transparent_coinbase_spends` predicate lets spends land in the +/// finalised range. +#[cfg(test)] +mod synthetic_chain { + use super::OPERATIONAL_NFS_DEPTH; + use crate::chain_index::source::mockchain_source::MockchainSource; + use proptest::prelude::{Arbitrary, Strategy}; + use proptest::strategy::ValueTree; + use proptest::test_runner::TestRunner; + use std::sync::Arc; + use zebra_chain::block::arbitrary::{ + allow_all_transparent_coinbase_spends, LedgerStateOverride, + }; + use zebra_chain::block::{Block, Height}; + use zebra_chain::parameters::{Network as ZebraNetwork, GENESIS_PREVIOUS_BLOCK_HASH}; + use zebra_chain::transparent::Input; + use zebra_chain::LedgerState; + + // Long enough that the depth-100 seam leaves room for finalised spends. + const CHAIN_LEN: usize = 200; + // Generate-until-found budget — a chain with a finalised transparent spend + // is overwhelmingly likely within a couple of tries. + const MAX_GEN_ATTEMPTS: usize = 40; + + pub(super) fn build_mock(blocks: Vec>) -> MockchainSource { + let hashes = blocks + .iter() + .map(|block| crate::BlockHash::from(block.hash())) + .collect(); + // Synthetic blocks carry no commitment roots/treestates; with a + // non-regtest network the block builder falls back to default roots, and + // spends are root-independent anyway. + let roots = vec![(None, None); blocks.len()]; + let treestates = vec![(Vec::new(), Vec::new()); blocks.len()]; + MockchainSource::new(blocks, roots, treestates, hashes) + } + + /// True if a non-coinbase transparent input (a spend) occurs at or below + /// `finalised_tip`. Block index equals height for a genesis-rooted chain. + fn has_finalised_spend(blocks: &[Arc], finalised_tip: u32) -> bool { + blocks.iter().take(finalised_tip as usize + 1).any(|block| { + block.transactions.iter().any(|tx| { + tx.inputs() + .iter() + .any(|i| matches!(i, Input::PrevOut { .. })) + }) + }) + } + + /// A deterministically generated genesis-rooted mainnet chain holding at + /// least one transparent spend at or below the seam, plus its finalised + /// tip. The seeded `TestRunner` yields the same chain on every run. + pub(super) fn with_finalised_spend() -> (Vec>, u32) { + let strategy = LedgerState::arbitrary_with(LedgerStateOverride { + height_override: Some(Height(0)), + previous_block_hash_override: Some(GENESIS_PREVIOUS_BLOCK_HASH), + network_upgrade_override: None, + transaction_version_override: None, + transaction_has_valid_network_upgrade: true, + always_has_coinbase: true, + network_override: Some(ZebraNetwork::Mainnet), + }) + .prop_flat_map(|ledger| { + Block::partial_chain_strategy( + ledger, + CHAIN_LEN, + allow_all_transparent_coinbase_spends, + true, + ) + }); + let mut runner = TestRunner::deterministic(); + + // Deterministically search for a generated chain with a finalised spend, + // so the presence direction is genuinely exercised. + for _ in 0..MAX_GEN_ATTEMPTS { + let blocks = strategy + .new_tree(&mut runner) + .expect("generate chain") + .current() + .0; + let finalised_tip = (blocks.len() as u32 - 1).saturating_sub(OPERATIONAL_NFS_DEPTH); + if has_finalised_spend(&blocks, finalised_tip) { + return (blocks, finalised_tip); + } + } + panic!("no synthetic chain with a finalised transparent spend within budget"); + } +} + +/// End-to-end **presence** test: `run` must build a *non-empty* index when the +/// finalised range contains transparent spends (chain from +/// [`synthetic_chain`]). Tracks zingolabs/zaino#1334. +#[cfg(test)] +mod spend_index_presence { + use super::*; + + #[tokio::test(flavor = "multi_thread")] + async fn run_builds_a_nonempty_index_from_a_synthetic_chain() { + let (blocks, finalised_tip) = synthetic_chain::with_finalised_spend(); + + let mock = synthetic_chain::build_mock(blocks); + + // Reference: the spends in the finalised range, built the way `run` + // builds blocks (so this checks fetch + boundary + collate + LMDB + // round-trip; the extractor is unit-tested above). + let zebra_network = Network::Mainnet.to_zebra_network(); + let sapling = NetworkUpgrade::Sapling + .activation_height(&zebra_network) + .expect("Sapling activation height"); + let nu5 = NetworkUpgrade::Nu5.activation_height(&zebra_network); + let mut finalised_blocks = Vec::new(); + for height in 0..=finalised_tip { + finalised_blocks.push( + build_indexed_block_from_source( + &mock, + Network::Mainnet, + sapling, + nu5, + height, + None, + ) + .await + .expect("build finalised block"), + ); + } + let expected = extract_spends(&finalised_blocks); + assert!( + !expected.is_empty(), + "the chosen chain must contain finalised transparent spends", + ); + + // Build the index from genesis and read it back. + let dir = std::env::temp_dir().join(format!( + "outp_to_spend_index_synthetic_{}", + std::process::id() + )); + let _ = std::fs::remove_dir_all(&dir); + let db = SpendIndexDb::open(&dir, 1 << 24).expect("open spend index"); + SpendIndexSync::new(mock, db, 0) + .run() + .await + .expect("spend-index build runs"); + let db = SpendIndexDb::open(&dir, 1 << 24).expect("reopen spend index"); + + for (outpoint, spending_txid) in &expected { + assert_eq!( + db.spending_txid(outpoint).expect("read finalised spend"), + Some(*spending_txid), + "finalised spend {outpoint:?} must be indexed to its spender", + ); + } + + std::fs::remove_dir_all(&dir).ok(); + } +} + +/// Cross-implementation **parity** test: the built index must agree +/// byte-for-byte with zebra's `SpendingTransactionId` read request — the +/// temporary correctness oracle ADR 0006 designates — served here by a real +/// `ReadStateService` (dev-features `indexer` + `proptest-impl`) over the +/// same synthetic chain the index is built from. +#[cfg(test)] +mod spend_index_zebra_parity { + use super::*; + use tower::ServiceExt as _; + use zebra_chain::parameters::Network as ZebraNetwork; + use zebra_state::{ReadRequest, ReadResponse, ReadStateService, Spend}; + + /// The oracle's answer for one prevout: the txid of its spending + /// transaction, if zebra knows one. + async fn oracle_spender( + read_state: &ReadStateService, + prevout: zebra_chain::transparent::OutPoint, + ) -> Option { + match read_state + .clone() + .oneshot(ReadRequest::SpendingTransactionId(Spend::OutPoint(prevout))) + .await + .expect("oracle answers spending-txid reads") + { + ReadResponse::TransactionId(answer) => answer, + other => panic!("unexpected oracle response: {other:?}"), + } + } + + /// For every prevout spent anywhere in the chain: at or below the seam + /// the index and the oracle must agree byte-for-byte; above it the + /// finalised-only index is silent by design while the whole-chain oracle + /// still answers; a never-spent outpoint is `None` in both. + /// + /// `multi_thread`: `run` uses `block_in_place` for the LMDB write. + #[tokio::test(flavor = "multi_thread")] + async fn index_agrees_with_zebra_over_a_synthetic_chain() { + let (blocks, finalised_tip) = synthetic_chain::with_finalised_spend(); + + // zaino: build the index from genesis over the mockchain. + let dir = std::env::temp_dir().join(format!( + "outp_to_spend_index_zebra_parity_{}", + std::process::id() + )); + let _ = std::fs::remove_dir_all(&dir); + let db = SpendIndexDb::open(&dir, 1 << 24).expect("open spend index"); + SpendIndexSync::new(synthetic_chain::build_mock(blocks.clone()), db, 0) + .run() + .await + .expect("spend-index build runs"); + let db = SpendIndexDb::open(&dir, 1 << 24).expect("reopen spend index"); + + // zebra: checkpoint-committing the same blocks lands them all in the + // finalized state, so the oracle covers the whole chain. + let (_state, read_state, _tip, _tip_change) = + zebra_state::populated_state(blocks.clone(), &ZebraNetwork::Mainnet).await; + + let mut finalised_spends = 0usize; + for (height, block) in blocks.iter().enumerate() { + for tx in &block.transactions { + for input in tx.inputs() { + let Some(prevout) = input.outpoint() else { + continue; // coinbase input: not a spend + }; + let indexed = db + .spending_txid(&Outpoint::new(prevout.hash.0, prevout.index)) + .expect("read spend index"); + let oracle = oracle_spender(&read_state, prevout).await; + assert!( + oracle.is_some(), + "oracle must know the spend of {prevout:?}", + ); + if height as u32 <= finalised_tip { + assert_eq!( + indexed.map(<[u8; 32]>::from), + oracle.map(|hash| hash.0), + "finalised spend of {prevout:?}: index and oracle must agree", + ); + finalised_spends += 1; + } else { + // The one expected divergence: finalised-only index + // vs whole-chain oracle. + assert_eq!( + indexed, None, + "above-seam spend of {prevout:?} must not be indexed", + ); + } + } + } + } + assert!( + finalised_spends > 0, + "chain must exercise the byte-for-byte agreement direction", + ); + + // A never-spent outpoint: `None` from both implementations. + let absent = zebra_chain::transparent::OutPoint { + hash: zebra_chain::transaction::Hash([0xEE; 32]), + index: u32::MAX, + }; + assert_eq!( + db.spending_txid(&Outpoint::new(absent.hash.0, absent.index)) + .expect("read absent outpoint"), + None, + ); + assert_eq!(oracle_spender(&read_state, absent).await, None); + + std::fs::remove_dir_all(&dir).ok(); + } +} diff --git a/packages/zaino-state/src/chain_index/source.rs b/packages/zaino-state/src/chain_index/source.rs index 73e3e13de..0aebf04dc 100644 --- a/packages/zaino-state/src/chain_index/source.rs +++ b/packages/zaino-state/src/chain_index/source.rs @@ -229,6 +229,18 @@ pub trait BlockchainSource: Clone + Send + Sync + 'static { fn subscribe_to_blocks_received(&self) -> Option> { None } + + /// The source from which a finalised spend index may be built, if this + /// backend supports one. Returns `None` for backends that must not build it + /// (the JSON-RPC `Fetch` path) and by default for those that don't; the + /// zebra StateService yields a [`StateSource`](validator_connector::StateSource). + // `StateSource` is intentionally `pub(crate)`; this is internal wiring on a + // `pub` trait, so exposing the more-private return type is expected. + #[cfg(feature = "outp_to_spend_index")] + #[allow(private_interfaces)] + fn finalised_spend_index_source(&self) -> Option { + None + } } /// Sleep up to `duration`, but return early if `change_rx` resolves first. diff --git a/packages/zaino-state/src/chain_index/source/validator_connector.rs b/packages/zaino-state/src/chain_index/source/validator_connector.rs index 8560853e1..c2967899d 100644 --- a/packages/zaino-state/src/chain_index/source/validator_connector.rs +++ b/packages/zaino-state/src/chain_index/source/validator_connector.rs @@ -1076,4 +1076,69 @@ impl BlockchainSource for ValidatorConnector { ValidatorConnector::Fetch(_fetch) => Ok(None), } } + + #[cfg(feature = "outp_to_spend_index")] + #[allow(private_interfaces)] + fn finalised_spend_index_source(&self) -> Option { + match self { + ValidatorConnector::State(state) => Some(StateSource(state.clone())), + ValidatorConnector::Fetch(_) => None, + } + } +} + +/// A compile-time **StateService-only** view of a validator source. +/// +/// Wraps the zebra [`State`] connector — never the JSON-RPC `Fetch` backend — so +/// a source can be *statically* known to be backed by zebra's `ReadStateService`. +/// The finalised spend-index POC builds only from this (or a test mockchain), +/// never `FetchService`; binding the loop to a `SpendIndexSource` that excludes +/// the `Fetch` path makes that a compile-time guarantee. +/// +/// It reuses the tested [`ValidatorConnector::State`] dispatch by wrapping a +/// cheap `State` clone per call (that dispatch already clones the service per +/// call). Gated with the `outp_to_spend_index` feature. +#[cfg(feature = "outp_to_spend_index")] +#[derive(Clone, Debug)] +pub(crate) struct StateSource(pub(crate) State); + +/// Generates [`StateSource`]'s `BlockchainSource` impl: one forwarder per listed +/// method, each delegating to the tested `ValidatorConnector::State` dispatch. +/// +/// A declarative macro rather than a `fn` because the forwarders span N distinct +/// `async` signatures — a pattern a function cannot capture (per the repo's +/// "macros only where `fn` cannot express it" rule). Each generated `async fn` +/// satisfies the trait's native `-> impl SendFut<_>` method, and the +/// `ValidatorConnector::State(self.0.clone())` construction appears exactly +/// once — here. +#[cfg(feature = "outp_to_spend_index")] +macro_rules! impl_state_source_forwarders { + ( $( fn $method:ident ( $( $arg:ident : $arg_ty:ty ),* ) -> $ret:ty; )+ ) => { + impl BlockchainSource for StateSource { + $( + async fn $method(&self, $( $arg : $arg_ty ),*) -> $ret { + ValidatorConnector::State(self.0.clone()) + .$method($( $arg ),*) + .await + } + )+ + } + }; +} + +#[cfg(feature = "outp_to_spend_index")] +impl_state_source_forwarders! { + fn get_block(id: HashOrHeight) -> BlockchainSourceResult>>; + fn get_transaction(txid: TransactionHash) -> BlockchainSourceResult, GetTransactionLocation)>>; + fn get_mempool_txids() -> BlockchainSourceResult>>; + fn get_best_block_hash() -> BlockchainSourceResult>; + fn get_best_block_height() -> BlockchainSourceResult>; + fn get_treestate(id: BlockHash) -> BlockchainSourceResult<(Option>, Option>)>; + fn get_subtree_roots(pool: ShieldedPool, start_index: u16, max_entries: Option) -> BlockchainSourceResult>; + fn get_commitment_tree_roots(id: BlockHash) -> BlockchainSourceResult<(Option<(zebra_chain::sapling::tree::Root, u64)>, Option<(zebra_chain::orchard::tree::Root, u64)>)>; + fn get_address_deltas(params: GetAddressDeltasParams) -> BlockchainSourceResult; + fn get_address_balance(address_strings: GetAddressBalanceRequest) -> BlockchainSourceResult; + fn get_address_txids(request: GetAddressTxIdsRequest) -> BlockchainSourceResult>; + fn get_address_utxos(address_strings: GetAddressBalanceRequest) -> BlockchainSourceResult>; + fn nonfinalized_listener() -> Result)>>, Box>; } diff --git a/tools/makefiles/lints.toml b/tools/makefiles/lints.toml index cac50b8d3..9676a2104 100644 --- a/tools/makefiles/lints.toml +++ b/tools/makefiles/lints.toml @@ -11,6 +11,11 @@ env = { RUSTDOCFLAGS = "-D warnings" } command = "cargo" args = ["doc", "--no-deps", "--all-features"] +[tasks.check-experimental] +description = "Compile (incl. test targets) with the experimental_features umbrella on. Gated modules are invisible to default builds, so without this they rot silently (e.g. a crate-wide rename missing a default-off module)." +command = "cargo" +args = ["check", "--tests", "--features", "experimental_features"] + [tasks.lint-boundary-conversions] description = "Forbid From/TryFrom impls across the persistence and wire boundaries. Both must use named inherent methods (from_business/into_business for persistence; to_wire/try_from_wire for wire)." # See CLAUDE.md §'Persistence-boundary conversions' and §'Wire-boundary @@ -103,6 +108,7 @@ dependencies = [ "fmt", "clippy", "doc", + "check-experimental", "lint-boundary-conversions", "lint-workbench", "lint-toolchain-pin", diff --git a/tools/scripts/help.sh b/tools/scripts/help.sh index 524c80b6c..7f962451c 100755 --- a/tools/scripts/help.sh +++ b/tools/scripts/help.sh @@ -12,7 +12,7 @@ echo "" echo "Common usage:" echo " makers test # packages/* tests, no live validator (default)" echo " makers test live # both live partitions + combined summary" -echo " makers test all # everything: package then live" +echo " makers test all # everything: packages then live" echo "" echo "If you modify '.env.testing-artifacts', the test command will \ automatically:" @@ -21,15 +21,15 @@ echo " - Build a new local container image if needed" echo "" echo "Available commands:" echo "" -echo " test [SET] Front door. SET = package (default) | e2e | \ +echo " test [SET] Front door. SET = packages (default) | e2e | \ clientless | live | all" -echo " package packages/* tests, no live \ +echo " packages packages/* tests, no live \ validator" echo " e2e the e2e live partition" echo " clientless the clientless live partition" echo " live both live partitions + \ combined summary" -echo " all package then live (everything)" +echo " all packages then live (everything)" echo " container-test Engine: run nextest in the container \ (used by the front door; invoke directly to forward engine flags)" echo " live-clientless Engine: run the clientless live-test \